Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Implement two stage write combining.

  • Loading branch information...
commit d5035a52b87b60a457f376c2bcc41da2be4c18d4 1 parent e02af6b
@creationix authored
View
326 lib/nstore.js
@@ -18,7 +18,9 @@ var nStore = module.exports = Class.extend({
this.filename = filename;
this.fd = null;
this.index = Hash.new();
- this.writeQueue = Queue.new();
+ this.toWriteCallbacks = [];
+ this.toWrite = Hash.new();
+ this.isWriting = Hash.new();
this.stale = 0;
this.dbLength = 0;
this.busy = false;
@@ -26,15 +28,17 @@ var nStore = module.exports = Class.extend({
this.loadDatabase(callback);
},
- // getter property that returns the number of documents in the database
get length() {
return this.index.length;
},
- getKey: function getKey() {
- var key = (0x100000000 * ((Date.now() & 0x3ffff)+Math.random())).toString(32);
- if (this.index.hasOwnProperty(key)) {
- return this.getKey();
+ genKey: function genKey() {
+ // Lower 0x20 from Date.now and 0x100000000 from Math.random
+ // Last character is from timestamp, the rest is random using full
+ // precision of Math.random (32 bit integer)
+ var key = (0x2000000000 * Math.random() + (Date.now() & 0x1f)).toString(32);
+ if (this.index.hasOwnProperty(key) || this.toWrite.hasOwnProperty(key) || this.isWriting.hasOwnProperty(key)) {
+ return this.genKey();
}
return key;
},
@@ -114,91 +118,48 @@ var nStore = module.exports = Class.extend({
});
},
- compactDatabase: function (clear, callback) {
- if ((!clear && this.stale === 0) || this.busy) return;
- var tmpFile = Path.join(Path.dirname(this.filename), this.getKey() + ".tmpdb"),
- tmpDb;
-
- this.busy = true;
- var self = this;
- Step(
- function makeNewDb() {
- tmpDb = nStore.new(tmpFile, this);
- },
- function copyData(err) {
- if (err) throw err;
- if (clear) return true;
- var group = this.group();
- var copy = Step.fn(
- function (key) {
- self.get(key, this);
- },
- function (err, doc, key) {
- if (err) throw err;
- if (self.filterFn && self.filterFn(doc, key)) {
- return true;
- }
- tmpDb.save(key, doc, this);
- }
- );
-
- self.index.forEach(function (info, key) {
- copy(key, group());
- });
- },
- function closeOld(err) {
- if (err) throw err;
- fs.close(self.fd, this);
- },
- function moveNew(err) {
- if (err) throw err;
- fs.rename(tmpFile, self.filename, this);
- },
- function transitionState(err) {
- if (err) throw err;
- self.dbLength = tmpDb.dbLength;
- self.index = tmpDb.index;
- self.fd = tmpDb.fd;
- self.stale = tmpDb.stale;
- return true;
- },
- function cleanup(err) {
- self.busy = false;
- process.nextTick(function () {
- self.checkQueue();
- });
- if (err) throw err;
- return true;
- },
- function prologue(err) {
- if (callback) {
- callback(err);
- }
- }
- );
-
- },
-
save: function save(key, doc, callback) {
if (!key) {
- key = this.getKey();
+ key = this.genKey();
}
- this.writeQueue.push({
- key: key.toString(),
- doc: doc,
- callback: callback
+ this.toWrite[key] = doc;
+ this.toWriteCallbacks.push(function (err) {
+ if (err) callback(err);
+ else callback(err, key);
});
this.checkQueue();
},
// Load a single record from the disk
get: function getByKey(key, callback) {
+ function missing() {
+ var error = new Error("Document does not exist for " + key);
+ error.errno = process.ENOENT;
+ callback(error);
+ }
+ // Check the cache of just written values
+ if (this.toWrite.hasOwnProperty(key)) {
+ var value = this.toWrite[key];
+ if (value === undefined) return missing();
+ process.nextTick(function () {
+ callback(null, value, key);
+ });
+ return;
+ }
+ // Check the cache of in-progress values
+ if (this.isWriting.hasOwnProperty(key)) {
+ var value = this.isWriting[key];
+ if (value === undefined) return missing();
+ process.nextTick(function () {
+ callback(null, value, key);
+ });
+ return;
+ }
+ // Read from disk otherwise
try {
var info = this.index[key];
if (!info) {
- var error = new Error("Document does not exist for " + key);
- error.errno = process.ENOENT;
- callback(error);
+ missing();
return;
}
@@ -216,6 +177,142 @@ var nStore = module.exports = Class.extend({
}
},
+ // Checks the save queue to see if there is a record to write to disk
+ checkQueue: function checkQueue() {
+ // Only run when not locked
+ if (this.busy) return;
+ // Skip if there is nothing to write
+ if (this.toWriteCallbacks.length === 0) return;
+ // Lock the table
+ this.busy = true;
+
+ // Grab items off the queue
+ this.isWriting = this.toWrite;
+
+ this.toWrite = Hash.new();
+
+ var callbacks = this.toWriteCallbacks.splice(0, this.toWriteCallbacks.length);
+ function callback(err) {
+ for (var i = 0, l = callbacks.length; i < l; i++) {
+ callbacks[i](err);
+ }
+ callbacks.length = 0;
+ self.busy = false;
+ self.checkQueue();
+ }
+
+ var updates = Hash.new(),
+ offset = this.dbLength,
+ self = this,
+ output;
+
+ // Use Step to handle errors
+ Step(
+ function () {
+ // Serialize the data to be written
+ output = self.isWriting.map(function (value, key) {
+ var doc = value === undefined ? "" : JSON.stringify(value),
+ docLength = Buffer.byteLength(doc),
+ keyLength = Buffer.byteLength(key);
+
+ // New data for the disk index
+ updates[key] = {
+ offset: offset + keyLength + 1,
+ length: docLength
+ };
+
+ offset += keyLength + docLength + 2;
+
+ return key + "\t" + doc + "\n";
+ }).join("");
+ output = new Buffer(output);
+ File.write(self.fd, output, self.dbLength, this);
+ },
+ function (err) {
+ if (err) return callback(err);
+ updates.forEach(function (value, key) {
+ if (self.index.hasOwnProperty(key)) {
+ self.stale++;
+ if (value.length === 0) {
+ delete self.index[key];
+ }
+ }
+ if (value !== undefined) {
+ self.index[key] = value;
+ }
+ });
+ self.dbLength += output.length;
+ callback();
+ },
+ callback
+ );
+ },
+
+ // compactDatabase: function (clear, callback) {
+ // if ((!clear && this.stale === 0) || this.busy) return;
+ // var tmpFile = Path.join(Path.dirname(this.filename), this.genKey() + ".tmpdb"),
+ // tmpDb;
+ //
+ // this.busy = true;
+ // var self = this;
+ // Step(
+ // function makeNewDb() {
+ // tmpDb = nStore.new(tmpFile, this);
+ // },
+ // function copyData(err) {
+ // if (err) throw err;
+ // if (clear) return true;
+ // var group = this.group();
+ // var copy = Step.fn(
+ // function (key) {
+ // self.get(key, this);
+ // },
+ // function (err, doc, key) {
+ // if (err) throw err;
+ // if (self.filterFn && self.filterFn(doc, key)) {
+ // return true;
+ // }
+ // tmpDb.save(key, doc, this);
+ // }
+ // );
+ //
+ // self.index.forEach(function (info, key) {
+ // copy(key, group());
+ // });
+ // },
+ // function closeOld(err) {
+ // if (err) throw err;
+ // fs.close(self.fd, this);
+ // },
+ // function moveNew(err) {
+ // if (err) throw err;
+ // fs.rename(tmpFile, self.filename, this);
+ // },
+ // function transitionState(err) {
+ // if (err) throw err;
+ // self.dbLength = tmpDb.dbLength;
+ // self.index = tmpDb.index;
+ // self.fd = tmpDb.fd;
+ // self.stale = tmpDb.stale;
+ // return true;
+ // },
+ // function cleanup(err) {
+ // self.busy = false;
+ // process.nextTick(function () {
+ // self.checkQueue();
+ // });
+ // if (err) throw err;
+ // return true;
+ // },
+ // function prologue(err) {
+ // if (callback) {
+ // callback(err);
+ // }
+ // }
+ // );
+ //
+ // },
+
remove: function removeByKey(key, callback) {
try {
var info = this.index[key];
@@ -225,80 +322,11 @@ var nStore = module.exports = Class.extend({
callback(error);
return;
}
- this.save(key, null, callback);
+ this.save(key, undefined, callback);
} catch(err) {
callback(err);
}
},
- clear: function clearAll(callback) {
- if (this.busy) {
- var self = this;
- process.nextTick(function () {
- self.clear(callback);
- });
- return;
- }
- this.compactDatabase(true, callback);
- },
-
- // Checks the save queue to see if there is a record to write to disk
- checkQueue: function checkQueue() {
- if (this.busy) return;
- var next = this.writeQueue.shift();
- if (next === undefined) {
- // Compact when the db is over half stale
- if (this.stale > (this.length - this.stale)) {
- this.compactDatabase();
- }
- return;
- }
- this.busy = true;
- var self = this;
- try {
- var line = new Buffer(next.key + "\t" + JSON.stringify(next.doc) + "\n");
- var keyLength = Buffer.byteLength(next.key);
- } catch(err) {
- console.log(err.stack);
- if (next.callback) {
- next.callback(err);
- }
- return;
- }
- Step(
- function writeDocument() {
- File.write(self.fd, line, self.dbLength, this);
- },
- function updateIndex(err) {
- if (err) throw err;
- // Count stale records
- if (self.index.hasOwnProperty(next.key)) { self.stale++; }
- if (next.doc) {
- // Update index
- self.index[next.key] = {
- position: self.dbLength + keyLength + 1,
- length: line.length - keyLength - 2
- };
- } else {
- delete self.index[next.key];
- }
- // Update the pointer to the end of the database
- self.dbLength += line.length;
- return true;
- },
- function done(err) {
- self.busy = false;
- if (err) throw err;
- self.checkQueue();
- return true;
- },
- function (err) {
- if (next.callback) {
- next.callback(err, next.key);
- }
- }
- );
- },
-
});
View
2  lib/nstore/cache.js
@@ -14,7 +14,7 @@ module.exports = function CachePlugin(maxSize) {
var self = {
save: function save(key, doc, callback) {
// Go ahead and generate the auto key so we know what to cache
- if (!key) key = this.getKey();
+ if (!key) key = this.genKey();
push(key, doc);
// Call super
self.__proto__.save.call(this, key, doc, callback);
View
2  test/helper.js
@@ -35,4 +35,4 @@ function clean() {
clean();
// Clean on exit too
-process.addListener('exit', clean);
+// process.addListener('exit', clean);
View
12 test/stressTest.js
@@ -1,10 +1,11 @@
require('./helper');
-console.dir(process.memoryUsage());
+// console.dir(process.memoryUsage());
var counter = 0;
const NUM = 1000000;
var obj = {F:"B"};
+expect("saveall");
var documents = nStore.new('fixtures/new.db', function () {
for (var i = 0; i < NUM; i++) {
documents.save((i % 1000) + 1, obj, function () {
@@ -12,14 +13,13 @@ var documents = nStore.new('fixtures/new.db', function () {
// documents.save(null, obj, function () {
counter++;
if (counter === NUM) {
- documents.compactDatabase();
+ fulfill("saveall");
}
-
});
}
});
-process.on('exit', function () {
- console.dir(process.memoryUsage());
-});
+// process.on('exit', function () {
+// console.dir(process.memoryUsage());
+// });
Please sign in to comment.
Something went wrong with that request. Please try again.