Skip to content
This repository
Browse code

Implement two stage write combining.

  • Loading branch information...
commit d5035a52b87b60a457f376c2bcc41da2be4c18d4 1 parent e02af6b
Tim Caswell authored August 23, 2010
326  lib/nstore.js
@@ -18,7 +18,9 @@ var nStore = module.exports = Class.extend({
18 18
     this.filename = filename;
19 19
     this.fd = null;
20 20
     this.index = Hash.new();
21  
-    this.writeQueue = Queue.new();
  21
+    this.toWriteCallbacks = [];
  22
+    this.toWrite = Hash.new();
  23
+    this.isWriting = Hash.new();
22 24
     this.stale = 0;
23 25
     this.dbLength = 0;
24 26
     this.busy = false;
@@ -26,15 +28,17 @@ var nStore = module.exports = Class.extend({
26 28
     this.loadDatabase(callback);
27 29
   },
28 30
 
29  
-  // getter property that returns the number of documents in the database
30 31
   get length() {
31 32
     return this.index.length;
32 33
   },
33 34
 
34  
-  getKey: function getKey() {
35  
-    var key = (0x100000000 * ((Date.now() & 0x3ffff)+Math.random())).toString(32);
36  
-    if (this.index.hasOwnProperty(key)) {
37  
-      return this.getKey();
  35
+  genKey: function genKey() {
  36
+    // Lower 0x20 from Date.now and 0x100000000 from Math.random
  37
+    // Last character is from timestamp, the rest is random using full
  38
+    // precision of Math.random (32 bit integer)
  39
+    var key = (0x2000000000 * Math.random() + (Date.now() & 0x1f)).toString(32);
  40
+    if (this.index.hasOwnProperty(key) || this.toWrite.hasOwnProperty(key) || this.isWriting.hasOwnProperty(key)) {
  41
+      return this.genKey();
38 42
     }
39 43
     return key;
40 44
   },
@@ -114,91 +118,48 @@ var nStore = module.exports = Class.extend({
114 118
     });
115 119
   },
116 120
 
117  
-  compactDatabase: function (clear, callback) {
118  
-    if ((!clear && this.stale === 0) || this.busy) return;
119  
-    var tmpFile = Path.join(Path.dirname(this.filename), this.getKey() + ".tmpdb"),
120  
-        tmpDb;
121  
-
122  
-    this.busy = true;
123  
-    var self = this;
124  
-    Step(
125  
-      function makeNewDb() {
126  
-        tmpDb = nStore.new(tmpFile, this);
127  
-      },
128  
-      function copyData(err) {
129  
-        if (err) throw err;
130  
-        if (clear) return true;
131  
-        var group = this.group();
132  
-        var copy = Step.fn(
133  
-          function (key) {
134  
-            self.get(key, this);
135  
-          },
136  
-          function (err, doc, key) {
137  
-            if (err) throw err;
138  
-            if (self.filterFn && self.filterFn(doc, key)) {
139  
-              return true;
140  
-            }
141  
-            tmpDb.save(key, doc, this);
142  
-          }
143  
-        );
144  
-
145  
-        self.index.forEach(function (info, key) {
146  
-          copy(key, group());
147  
-        });
148  
-      },
149  
-      function closeOld(err) {
150  
-        if (err) throw err;
151  
-        fs.close(self.fd, this);
152  
-      },
153  
-      function moveNew(err) {
154  
-        if (err) throw err;
155  
-        fs.rename(tmpFile, self.filename, this);
156  
-      },
157  
-      function transitionState(err) {
158  
-        if (err) throw err;
159  
-        self.dbLength = tmpDb.dbLength;
160  
-        self.index = tmpDb.index;
161  
-        self.fd = tmpDb.fd;
162  
-        self.stale = tmpDb.stale;
163  
-        return true;
164  
-      },
165  
-      function cleanup(err) {
166  
-        self.busy = false;
167  
-        process.nextTick(function () {
168  
-          self.checkQueue();
169  
-        });
170  
-        if (err) throw err;
171  
-        return true;
172  
-      },
173  
-      function prologue(err) {
174  
-        if (callback) {
175  
-          callback(err);
176  
-        }
177  
-      }
178  
-    );
179  
-
180  
-  },
181  
-
182 121
   save: function save(key, doc, callback) {
183 122
     if (!key) {
184  
-      key = this.getKey();
  123
+      key = this.genKey();
185 124
     }
186  
-    this.writeQueue.push({
187  
-      key: key.toString(),
188  
-      doc: doc,
189  
-      callback: callback
  125
+    this.toWrite[key] = doc;
  126
+    this.toWriteCallbacks.push(function (err) {
  127
+      if (err) callback(err);
  128
+      else callback(err, key);
190 129
     });
191 130
     this.checkQueue();
192 131
   },
193 132
 
194 133
   // Load a single record from the disk
195 134
   get: function getByKey(key, callback) {
  135
+    function missing() {
  136
+      var error = new Error("Document does not exist for " + key);
  137
+      error.errno = process.ENOENT;
  138
+      callback(error);
  139
+    }
  140
+    // Check the cache of just written values
  141
+    if (this.toWrite.hasOwnProperty(key)) {
  142
+      var value = this.toWrite[key];
  143
+      if (value === undefined) return missing();
  144
+      process.nextTick(function () {
  145
+        callback(null, value, key);
  146
+      });
  147
+      return;
  148
+    }
  149
+    // Check the cache of in-progress  values
  150
+    if (this.isWriting.hasOwnProperty(key)) {
  151
+      var value = this.isWriting[key];
  152
+      if (value === undefined) return missing();
  153
+      process.nextTick(function () {
  154
+        callback(null, value, key);
  155
+      });
  156
+      return;
  157
+    }
  158
+    // Read from disk otherwise
196 159
     try {
197 160
       var info = this.index[key];
198 161
       if (!info) {
199  
-        var error = new Error("Document does not exist for " + key);
200  
-        error.errno = process.ENOENT;
201  
-        callback(error);
  162
+        missing();
202 163
         return;
203 164
       }
204 165
 
@@ -216,6 +177,142 @@ var nStore = module.exports = Class.extend({
216 177
     }
217 178
   },
218 179
 
  180
+  // Checks the save queue to see if there is a record to write to disk
  181
+  checkQueue: function checkQueue() {
  182
+    // Only run when not locked
  183
+    if (this.busy) return;
  184
+    // Skip if there is nothing to write
  185
+    if (this.toWriteCallbacks.length === 0) return;
  186
+    // Lock the table
  187
+    this.busy = true;
  188
+
  189
+    // Grab items off the queue
  190
+    this.isWriting = this.toWrite;
  191
+
  192
+    this.toWrite = Hash.new();
  193
+
  194
+    var callbacks = this.toWriteCallbacks.splice(0, this.toWriteCallbacks.length);
  195
+    function callback(err) {
  196
+      for (var i = 0, l = callbacks.length; i < l; i++) {
  197
+        callbacks[i](err);
  198
+      }
  199
+      callbacks.length = 0;
  200
+      self.busy = false;
  201
+      self.checkQueue();
  202
+    }
  203
+
  204
+    var updates = Hash.new(),
  205
+        offset = this.dbLength,
  206
+        self = this,
  207
+        output;
  208
+
  209
+    // Use Step to handle errors
  210
+    Step(
  211
+      function () {
  212
+        // Serialize the data to be written
  213
+        output = self.isWriting.map(function (value, key) {
  214
+          var doc = value === undefined ? "" : JSON.stringify(value),
  215
+              docLength = Buffer.byteLength(doc),
  216
+              keyLength = Buffer.byteLength(key);
  217
+
  218
+          // New data for the disk index
  219
+          updates[key] = {
  220
+            offset: offset + keyLength + 1,
  221
+            length: docLength
  222
+          };
  223
+
  224
+          offset += keyLength + docLength + 2;
  225
+
  226
+          return key + "\t" + doc + "\n";
  227
+        }).join("");
  228
+        output = new Buffer(output);
  229
+        File.write(self.fd, output, self.dbLength, this);
  230
+      },
  231
+      function (err) {
  232
+        if (err) return callback(err);
  233
+        updates.forEach(function (value, key) {
  234
+          if (self.index.hasOwnProperty(key)) {
  235
+            self.stale++;
  236
+            if (value.length === 0) {
  237
+              delete self.index[key];
  238
+            }
  239
+          }
  240
+          if (value !== undefined) {
  241
+            self.index[key] = value;
  242
+          }
  243
+        });
  244
+        self.dbLength += output.length;
  245
+        callback();
  246
+      },
  247
+      callback
  248
+    );
  249
+  },
  250
+
  251
+  // compactDatabase: function (clear, callback) {
  252
+  //   if ((!clear && this.stale === 0) || this.busy) return;
  253
+  //   var tmpFile = Path.join(Path.dirname(this.filename), this.genKey() + ".tmpdb"),
  254
+  //       tmpDb;
  255
+  //
  256
+  //   this.busy = true;
  257
+  //   var self = this;
  258
+  //   Step(
  259
+  //     function makeNewDb() {
  260
+  //       tmpDb = nStore.new(tmpFile, this);
  261
+  //     },
  262
+  //     function copyData(err) {
  263
+  //       if (err) throw err;
  264
+  //       if (clear) return true;
  265
+  //       var group = this.group();
  266
+  //       var copy = Step.fn(
  267
+  //         function (key) {
  268
+  //           self.get(key, this);
  269
+  //         },
  270
+  //         function (err, doc, key) {
  271
+  //           if (err) throw err;
  272
+  //           if (self.filterFn && self.filterFn(doc, key)) {
  273
+  //             return true;
  274
+  //           }
  275
+  //           tmpDb.save(key, doc, this);
  276
+  //         }
  277
+  //       );
  278
+  //
  279
+  //       self.index.forEach(function (info, key) {
  280
+  //         copy(key, group());
  281
+  //       });
  282
+  //     },
  283
+  //     function closeOld(err) {
  284
+  //       if (err) throw err;
  285
+  //       fs.close(self.fd, this);
  286
+  //     },
  287
+  //     function moveNew(err) {
  288
+  //       if (err) throw err;
  289
+  //       fs.rename(tmpFile, self.filename, this);
  290
+  //     },
  291
+  //     function transitionState(err) {
  292
+  //       if (err) throw err;
  293
+  //       self.dbLength = tmpDb.dbLength;
  294
+  //       self.index = tmpDb.index;
  295
+  //       self.fd = tmpDb.fd;
  296
+  //       self.stale = tmpDb.stale;
  297
+  //       return true;
  298
+  //     },
  299
+  //     function cleanup(err) {
  300
+  //       self.busy = false;
  301
+  //       process.nextTick(function () {
  302
+  //         self.checkQueue();
  303
+  //       });
  304
+  //       if (err) throw err;
  305
+  //       return true;
  306
+  //     },
  307
+  //     function prologue(err) {
  308
+  //       if (callback) {
  309
+  //         callback(err);
  310
+  //       }
  311
+  //     }
  312
+  //   );
  313
+  //
  314
+  // },
  315
+
219 316
   remove: function removeByKey(key, callback) {
220 317
     try {
221 318
       var info = this.index[key];
@@ -225,80 +322,11 @@ var nStore = module.exports = Class.extend({
225 322
         callback(error);
226 323
         return;
227 324
       }
228  
-      this.save(key, null, callback);
  325
+      this.save(key, undefined, callback);
229 326
     } catch(err) {
230 327
       callback(err);
231 328
     }
232 329
   },
233 330
 
234  
-  clear: function clearAll(callback) {
235  
-    if (this.busy) {
236  
-      var self = this;
237  
-      process.nextTick(function () {
238  
-        self.clear(callback);
239  
-      });
240  
-      return;
241  
-    }
242  
-    this.compactDatabase(true, callback);
243  
-  },
244  
-
245  
-  // Checks the save queue to see if there is a record to write to disk
246  
-  checkQueue: function checkQueue() {
247  
-    if (this.busy) return;
248  
-    var next = this.writeQueue.shift();
249  
-    if (next === undefined) {
250  
-      // Compact when the db is over half stale
251  
-      if (this.stale > (this.length - this.stale)) {
252  
-        this.compactDatabase();
253  
-      }
254  
-      return;
255  
-    }
256  
-    this.busy = true;
257  
-    var self = this;
258  
-    try {
259  
-      var line = new Buffer(next.key + "\t" + JSON.stringify(next.doc) + "\n");
260  
-      var keyLength = Buffer.byteLength(next.key);
261  
-    } catch(err) {
262  
-      console.log(err.stack);
263  
-      if (next.callback) {
264  
-        next.callback(err);
265  
-      }
266  
-      return;
267  
-    }
268  
-    Step(
269  
-      function writeDocument() {
270  
-        File.write(self.fd, line, self.dbLength, this);
271  
-      },
272  
-      function updateIndex(err) {
273  
-        if (err) throw err;
274  
-        // Count stale records
275  
-        if (self.index.hasOwnProperty(next.key)) { self.stale++; }
276  
-        if (next.doc) {
277  
-          // Update index
278  
-          self.index[next.key] = {
279  
-            position: self.dbLength + keyLength + 1,
280  
-            length: line.length - keyLength - 2
281  
-          };
282  
-        } else {
283  
-          delete self.index[next.key];
284  
-        }
285  
-        // Update the pointer to the end of the database
286  
-        self.dbLength += line.length;
287  
-        return true;
288  
-      },
289  
-      function done(err) {
290  
-        self.busy = false;
291  
-        if (err) throw err;
292  
-        self.checkQueue();
293  
-        return true;
294  
-      },
295  
-      function (err) {
296  
-        if (next.callback) {
297  
-          next.callback(err, next.key);
298  
-        }
299  
-      }
300  
-    );
301  
-  },
302  
-
303 331
 });
304 332
 
2  lib/nstore/cache.js
@@ -14,7 +14,7 @@ module.exports = function CachePlugin(maxSize) {
14 14
   var self = {
15 15
     save: function save(key, doc, callback) {
16 16
       // Go ahead and generate the auto key so we know what to cache
17  
-      if (!key) key = this.getKey();
  17
+      if (!key) key = this.genKey();
18 18
       push(key, doc);
19 19
       // Call super
20 20
       self.__proto__.save.call(this, key, doc, callback);
2  test/helper.js
@@ -35,4 +35,4 @@ function clean() {
35 35
 clean();
36 36
 
37 37
 // Clean on exit too
38  
-process.addListener('exit', clean);
  38
+// process.addListener('exit', clean);
12  test/stressTest.js
... ...
@@ -1,10 +1,11 @@
1 1
 require('./helper');
2 2
 
3  
-console.dir(process.memoryUsage());
  3
+// console.dir(process.memoryUsage());
4 4
 
5 5
 var counter = 0;
6 6
 const NUM = 1000000;
7 7
 var obj = {F:"B"};
  8
+expect("saveall");
8 9
 var documents = nStore.new('fixtures/new.db', function () {
9 10
   for (var i = 0; i < NUM; i++) {
10 11
     documents.save((i % 1000) + 1, obj, function () {
@@ -12,14 +13,13 @@ var documents = nStore.new('fixtures/new.db', function () {
12 13
     // documents.save(null, obj, function () {
13 14
       counter++;
14 15
       if (counter === NUM) {
15  
-        documents.compactDatabase();
  16
+        fulfill("saveall");
16 17
       }
17  
-    
18 18
     });
19 19
   }
20 20
 });
21 21
 
22 22
 
23  
-process.on('exit', function () {
24  
-  console.dir(process.memoryUsage());
25  
-});
  23
+// process.on('exit', function () {
  24
+//   console.dir(process.memoryUsage());
  25
+// });

0 notes on commit d5035a5

Please sign in to comment.
Something went wrong with that request. Please try again.