Skip to content
This repository
Browse code

Added option to DBMaker to disable file-system locking entirely.

  • Loading branch information...
commit 68fbaf54f97527eb1969a57a48acbc649f1907c6 1 parent 25369ed
Hamish Morgan hamishmorgan authored
4 src/main/java/net/kotek/jdbm/DBCache.java
@@ -40,11 +40,11 @@
40 40 */
41 41 public DBCache(String filename, boolean readonly, boolean transactionDisabled,
42 42 Cipher cipherIn, Cipher cipherOut, boolean useRandomAccessFile,
43   - boolean deleteFilesAfterClose){
  43 + boolean deleteFilesAfterClose,boolean lockingDisabled){
44 44
45 45 super(filename, readonly, transactionDisabled,
46 46 cipherIn, cipherOut, useRandomAccessFile,
47   - deleteFilesAfterClose);
  47 + deleteFilesAfterClose,lockingDisabled);
48 48
49 49 }
50 50
4 src/main/java/net/kotek/jdbm/DBCacheMRU.java
@@ -69,10 +69,10 @@
69 69 */
70 70 public DBCacheMRU(String filename, boolean readonly, boolean transactionDisabled,
71 71 Cipher cipherIn, Cipher cipherOut, boolean useRandomAccessFile,
72   - boolean deleteFilesAfterClose, int cacheMaxRecords) {
  72 + boolean deleteFilesAfterClose, int cacheMaxRecords, boolean lockingDisabled) {
73 73 super(filename, readonly, transactionDisabled,
74 74 cipherIn, cipherOut, useRandomAccessFile,
75   - deleteFilesAfterClose);
  75 + deleteFilesAfterClose,lockingDisabled);
76 76
77 77 _hash = new LongHashMap<CacheEntry>(cacheMaxRecords);
78 78 _max = cacheMaxRecords;
4 src/main/java/net/kotek/jdbm/DBCacheRef.java
@@ -79,11 +79,11 @@
79 79 public DBCacheRef(String filename, boolean readonly, boolean transactionDisabled,
80 80 Cipher cipherIn, Cipher cipherOut, boolean useRandomAccessFile,
81 81 boolean deleteFilesAfterClose,
82   - byte cacheType, boolean cacheAutoClearOnLowMem) {
  82 + byte cacheType, boolean cacheAutoClearOnLowMem, boolean lockingDisabled) {
83 83
84 84 super(filename, readonly, transactionDisabled,
85 85 cipherIn, cipherOut, useRandomAccessFile,
86   - deleteFilesAfterClose);
  86 + deleteFilesAfterClose, lockingDisabled);
87 87
88 88
89 89 this._cacheType = cacheType;
26 src/main/java/net/kotek/jdbm/DBMaker.java
@@ -36,6 +36,7 @@
36 36 private String location = null;
37 37
38 38 private boolean disableTransactions = false;
  39 + private boolean lockingDisabled = false;
39 40 private boolean readonly = false;
40 41 private String password = null;
41 42 private boolean useAES256Bit = true;
@@ -231,7 +232,24 @@ public DBMaker disableTransactions() {
231 232 this.disableTransactions = true;
232 233 return this;
233 234 }
234   -
  235 +
  236 + /**
  237 + * Disable file system based locking (for file systems that do not support it).
  238 + *
  239 + * Locking is not supported by many remote or distributed file systems; such
  240 + * as Lustre and NFS. Attempts to perform locks will result in an
  241 + * IOException with the message "Function not implemented".
  242 + *
  243 + * Disabling locking will avoid this issue, though of course it comes with
  244 + * all the issues of uncontrolled file access.
  245 + *
  246 + * @return this builder
  247 + */
  248 + public DBMaker disableLocking(){
  249 + this.lockingDisabled = true;
  250 + return this;
  251 + }
  252 +
235 253 /**
236 254 * By default JDBM uses mapped memory buffers to read from files.
237 255 * But this may behave strangely on some platforms.
@@ -314,11 +332,11 @@ public DB make() {
314 332
315 333
316 334 if (cacheType == DBCacheRef.MRU){
317   - db = new DBCacheMRU(location, readonly, disableTransactions, cipherIn, cipherOut,useRandomAccessFile,deleteFilesAfterCloseFlag, mruCacheSize);
  335 + db = new DBCacheMRU(location, readonly, disableTransactions, cipherIn, cipherOut,useRandomAccessFile,deleteFilesAfterCloseFlag, mruCacheSize,lockingDisabled);
318 336 }else if( cacheType == DBCacheRef.SOFT || cacheType == DBCacheRef.HARD || cacheType == DBCacheRef.WEAK) {
319   - db = new DBCacheRef(location, readonly, disableTransactions, cipherIn, cipherOut,useRandomAccessFile,deleteFilesAfterCloseFlag, cacheType,autoClearRefCacheOnLowMem);
  337 + db = new DBCacheRef(location, readonly, disableTransactions, cipherIn, cipherOut,useRandomAccessFile,deleteFilesAfterCloseFlag, cacheType,autoClearRefCacheOnLowMem,lockingDisabled);
320 338 } else if (cacheType == DBCacheRef.NONE) {
321   - db = new DBStore(location, readonly, disableTransactions, cipherIn, cipherOut,useRandomAccessFile,deleteFilesAfterCloseFlag);
  339 + db = new DBStore(location, readonly, disableTransactions, cipherIn, cipherOut,useRandomAccessFile,deleteFilesAfterCloseFlag,lockingDisabled);
322 340 } else {
323 341 throw new IllegalArgumentException("Unknown cache type: " + cacheType);
324 342 }
13 src/main/java/net/kotek/jdbm/DBStore.java
@@ -93,7 +93,7 @@
93 93 */
94 94 private Cipher cipherIn;
95 95 private boolean useRandomAccessFile;
96   -
  96 + private boolean lockingDisabled;
97 97
98 98 void checkCanWrite() {
99 99 if (readonly)
@@ -129,8 +129,8 @@ void checkCanWrite() {
129 129
130 130 private final String _filename;
131 131
132   - public DBStore(String filename, boolean readonly, boolean transactionDisabled) throws IOException {
133   - this(filename, readonly, transactionDisabled, null, null, false,false);
  132 + public DBStore(String filename, boolean readonly, boolean transactionDisabled, boolean lockingDisabled) throws IOException {
  133 + this(filename, readonly, transactionDisabled, null, null, false,false,false);
134 134 }
135 135
136 136
@@ -142,7 +142,7 @@ public DBStore(String filename, boolean readonly, boolean transactionDisabled) t
142 142 */
143 143 public DBStore(String filename, boolean readonly, boolean transactionDisabled,
144 144 Cipher cipherIn, Cipher cipherOut, boolean useRandomAccessFile,
145   - boolean deleteFilesAfterClose){
  145 + boolean deleteFilesAfterClose, boolean lockingDisabled){
146 146 _filename = filename;
147 147 this.readonly = readonly;
148 148 this.transactionsDisabled = transactionDisabled;
@@ -150,13 +150,14 @@ public DBStore(String filename, boolean readonly, boolean transactionDisabled,
150 150 this.cipherOut = cipherOut;
151 151 this.useRandomAccessFile = useRandomAccessFile;
152 152 this.deleteFilesAfterClose = deleteFilesAfterClose;
  153 + this.lockingDisabled = lockingDisabled;
153 154 reopen();
154 155 }
155 156
156 157
157 158 private void reopen() {
158 159 try{
159   - _file = new PageFile(_filename, readonly, transactionsDisabled, cipherIn, cipherOut,useRandomAccessFile);
  160 + _file = new PageFile(_filename, readonly, transactionsDisabled, cipherIn, cipherOut,useRandomAccessFile,lockingDisabled);
160 161 _pageman = new PageManager(_file);
161 162 _physMgr = new PhysicalRowIdManager(_file, _pageman);
162 163
@@ -669,7 +670,7 @@ public synchronized void defrag(boolean sortCollections) {
669 670 commit();
670 671 final String filename2 = _filename + "_defrag" + System.currentTimeMillis();
671 672 final String filename1 = _filename;
672   - DBStore db2 = new DBStore(filename2, false, true, cipherIn, cipherOut, false,false);
  673 + DBStore db2 = new DBStore(filename2, false, true, cipherIn, cipherOut, false,false,false);
673 674
674 675 //recreate logical file with original page layout
675 676 {
8 src/main/java/net/kotek/jdbm/PageFile.java
@@ -81,7 +81,7 @@
81 81 * @throws IOException whenever the creation of the underlying
82 82 * RandomAccessFile throws it.
83 83 */
84   - PageFile(String fileName, boolean readonly, boolean transactionsDisabled, Cipher cipherIn, Cipher cipherOut, boolean useRandomAccessFile) throws IOException {
  84 + PageFile(String fileName, boolean readonly, boolean transactionsDisabled, Cipher cipherIn, Cipher cipherOut, boolean useRandomAccessFile, boolean lockingDisabled) throws IOException {
85 85 this.cipherIn = cipherIn;
86 86 this.cipherOut = cipherOut;
87 87 this.transactionsDisabled = transactionsDisabled;
@@ -92,9 +92,9 @@
92 92 // }else if (fileName.contains("!/"))
93 93 // this.storage = new StorageZip(fileName);
94 94 else if(useRandomAccessFile)
95   - this.storage = new StorageDisk(fileName,readonly);
  95 + this.storage = new StorageDisk(fileName,readonly,lockingDisabled);
96 96 else
97   - this.storage = new StorageDiskMapped(fileName,readonly,transactionsDisabled);
  97 + this.storage = new StorageDiskMapped(fileName,readonly,transactionsDisabled,lockingDisabled);
98 98
99 99 if (this.storage.isReadonly() && !readonly)
100 100 throw new IllegalArgumentException("This type of storage is readonly, you should call readonly() on DBMaker");
@@ -106,7 +106,7 @@ else if(useRandomAccessFile)
106 106 }
107 107
108 108 public PageFile(String filename) throws IOException {
109   - this(filename, false, false, null, null,false);
  109 + this(filename, false, false, null, null,false,false);
110 110 }
111 111
112 112
6 src/main/java/net/kotek/jdbm/StorageDisk.java
@@ -21,14 +21,16 @@
21 21
22 22 private long lastPageNumber = Long.MIN_VALUE;
23 23 private boolean readonly;
  24 + private boolean lockingDisabled;
24 25
25   - public StorageDisk(String fileName,boolean readonly) throws IOException {
  26 + public StorageDisk(String fileName,boolean readonly, boolean lockingDisabled) throws IOException {
26 27 this.fileName = fileName;
27 28 this.readonly = readonly;
  29 + this.lockingDisabled = lockingDisabled;
28 30 //make sure first file can be opened
29 31 //lock it
30 32 try {
31   - if(!readonly)
  33 + if(!readonly && !lockingDisabled)
32 34 getRaf(0).getChannel().tryLock();
33 35 } catch (IOException e) {
34 36 throw new IOException("Could not lock DB file: " + fileName, e);
7 src/main/java/net/kotek/jdbm/StorageDiskMapped.java
@@ -37,16 +37,19 @@
37 37 private String fileName;
38 38 private boolean transactionsDisabled;
39 39 private boolean readonly;
  40 + private boolean lockingDisabled;
40 41
41 42
42   - public StorageDiskMapped(String fileName, boolean readonly, boolean transactionsDisabled) throws IOException {
  43 + public StorageDiskMapped(String fileName, boolean readonly, boolean transactionsDisabled, boolean lockingDisabled) throws IOException {
43 44 this.fileName = fileName;
44 45 this.transactionsDisabled = transactionsDisabled;
45 46 this.readonly = readonly;
  47 + this.lockingDisabled = lockingDisabled;
46 48 //make sure first file can be opened
47 49 //lock it
48 50 try {
49   - getChannel(0).lock();
  51 + if(!lockingDisabled)
  52 + getChannel(0).lock();
50 53 } catch (IOException e) {
51 54 throw new IOException("Could not lock DB file: " + fileName, e);
52 55 } catch (OverlappingFileLockException e) {
6 src/test/java/net/kotek/jdbm/BTreeKeyCompressionTest.java
@@ -13,7 +13,7 @@
13 13 public void testExpand() throws IOException {
14 14 long init = Long.MAX_VALUE - size * 2;
15 15 String file = newTestFile();
16   - DB db = new DBStore(file, false, false);
  16 + DB db = new DBStore(file, false, false,false);
17 17 SortedMap<Long, String> map = db.createTreeMap("aa");
18 18 for (long i = init; i < init + size; i++) {
19 19 map.put(i, "");
@@ -104,7 +104,7 @@ public void testCornersLimitsInt() throws IOException {
104 104 public void testStrings() throws IOException {
105 105 long init = Long.MAX_VALUE - size * 2;
106 106 String file = newTestFile();
107   - DB db = new DBStore(file, false, false);
  107 + DB db = new DBStore(file, false, false,false);
108 108 SortedMap<String, String> map = db.createTreeMap("aa");
109 109 for (long i = init; i < init + size / 10; i++) {
110 110 map.put("aaaaa" + i, "");
@@ -112,7 +112,7 @@ public void testStrings() throws IOException {
112 112 db.commit();
113 113 db.defrag(true);
114 114 db.close();
115   - db = new DBStore(file, false, false);
  115 + db = new DBStore(file, false, false,false);
116 116 map = db.getTreeMap("aa");
117 117 for (long i = init; i < init + size / 10; i++) {
118 118 assertTrue(map.containsKey("aaaaa" + i));
16 src/test/java/net/kotek/jdbm/DefragTest.java
@@ -8,18 +8,18 @@
8 8
9 9 public void testDefrag1() throws IOException {
10 10 String file = newTestFile();
11   - DBStore m = new DBStore(file, false, false);
  11 + DBStore m = new DBStore(file, false, false,false);
12 12 long loc = m.insert("123");
13 13 m.defrag(true);
14 14 m.close();
15   - m = new DBStore(file, false, false);
  15 + m = new DBStore(file, false, false,false);
16 16 assertEquals(m.fetch(loc), "123");
17 17 }
18 18
19 19
20 20 public void testDefrag2() throws IOException {
21 21 String file = newTestFile();
22   - DBStore m = new DBStore(file, false, false);
  22 + DBStore m = new DBStore(file, false, false,false);
23 23 TreeMap<Long, String> map = new TreeMap<Long, String>();
24 24 for (int i = 0; i < 10000; i++) {
25 25 long loc = m.insert("" + i);
@@ -28,7 +28,7 @@ public void testDefrag2() throws IOException {
28 28
29 29 m.defrag(true);
30 30 m.close();
31   - m = new DBStore(file, false, false);
  31 + m = new DBStore(file, false, false,false);
32 32 for (Long l : map.keySet()) {
33 33 String val = map.get(l);
34 34 assertEquals(val, m.fetch(l));
@@ -38,7 +38,7 @@ public void testDefrag2() throws IOException {
38 38
39 39 public void testDefragBtree() throws IOException {
40 40 String file = newTestFile();
41   - DBStore m = new DBStore(file, false, false);
  41 + DBStore m = new DBStore(file, false, false,false);
42 42 Map t = m.createTreeMap("aa");
43 43 TreeMap t2 = new TreeMap();
44 44 for (int i = 0; i < 10000; i++) {
@@ -48,14 +48,14 @@ public void testDefragBtree() throws IOException {
48 48
49 49 m.defrag(true);
50 50 m.close();
51   - m = new DBStore(file, false, false);
  51 + m = new DBStore(file, false, false,false);
52 52 t = m.getTreeMap("aa");
53 53 assertEquals(t, t2);
54 54 }
55 55
56 56 public void testDefragLinkedList() throws Exception {
57 57 String file = newTestFile();
58   - DBStore r = new DBStore(file, false, false);
  58 + DBStore r = new DBStore(file, false, false,false);
59 59 List l = r.createLinkedList("test");
60 60 Map<Long, Double> junk = new LinkedHashMap<Long, Double>();
61 61
@@ -72,7 +72,7 @@ public void testDefragLinkedList() throws Exception {
72 72 r.defrag(true);
73 73
74 74 r.close();
75   - r = new DBStore(file, false, false);
  75 + r = new DBStore(file, false, false,false);
76 76 assertEquals(oldRecCount, r.countRecords());
77 77
78 78 //compare that list was unchanged
2  src/test/java/net/kotek/jdbm/StorageZipTest.java
@@ -15,7 +15,7 @@ public void test_archive_creation() throws IOException {
15 15 tmp.deleteOnExit();
16 16
17 17 //first create archie and put it in zip file
18   - DBStore r = new DBStore(newTestFile(), false, true);
  18 + DBStore r = new DBStore(newTestFile(), false, true,false);
19 19 Set<Long> h = r.createHashSet("hash");
20 20 for (Long l = 0L; l < 1e3; l++) {
21 21 h.add(l);
2  src/test/java/net/kotek/jdbm/TestLargeData.java
@@ -7,7 +7,7 @@
7 7
8 8 public void testLargeData() throws IOException {
9 9
10   - DBAbstract db = new DBStore(newTestFile(), false, false);
  10 + DBAbstract db = new DBStore(newTestFile(), false, false,false);
11 11
12 12 byte[] data = UtilTT.makeRecord(1000000, (byte) 12);
13 13 final long id = db.insert(data);
4 src/test/java/net/kotek/jdbm/TestStress.java
@@ -79,7 +79,7 @@ private int getRandomAllocatedRoot() {
79 79 public void testBasics() throws Exception {
80 80
81 81 String file = newTestFile();
82   - DBStore db = new DBStore(file, false, false);
  82 + DBStore db = new DBStore(file, false, false,false);
83 83
84 84 // as this code is meant to test data structure calculcations
85 85 // and stuff like that, we may want to disable transactions
@@ -105,7 +105,7 @@ public void testBasics() throws Exception {
105 105 System.out.print(" (reopened at round "
106 106 + i / RPPROMILLE + ")");
107 107 db.close();
108   - db = new DBStore(file, false, false);
  108 + db = new DBStore(file, false, false,false);
109 109 // db.disableTransactions();
110 110 }
111 111

0 comments on commit 68fbaf5

Please sign in to comment.
Something went wrong with that request. Please try again.