Added pluggable storage (louischatriot#419): - added new, optional "s…

…torage" options to Datastore and Persistence, to allow overriding the default "storage.js" module

Necessary changes to make that possible:
- turn all static method of "Persistence" class into normal methods
- adjust static calls to Persistence.ensureDirectoryExists() into normal class method invocations (applies to benchmarks and tests only)
- reimplemented (benchmarks) commonUtilities prepareDb() to remove Persistence dependency. The module already relies on a Node environment and so can also use "mkdirp" module directly. The new implementation behaves exactly as before (ie. ignores mkdirp() errors)

All tests and benchmarks still passing.

benchmarks/commonUtilities.js

@@ -4,6 +4,7 @@
 
 var customUtils = require('../lib/customUtils')
   , fs = require('fs')
+  , mkdirp = require('mkdirp')
   , path = require('path')
   , Datastore = require('../lib/datastore')
   , Persistence = require('../lib/persistence')
@@ -51,7 +52,7 @@ module.exports.getConfiguration = function (benchDb) {
  * Ensure the workspace exists and the db datafile is empty
  */
 module.exports.prepareDb = function (filename, cb) {
-  Persistence.ensureDirectoryExists(path.dirname(filename), function () {
+  mkdirp(path.dirname(filename), function () {
     fs.exists(filename, function (exists) {
       if (exists) {
         fs.unlink(filename, cb);

lib/datastore.js

@@ -22,6 +22,7 @@ var customUtils = require('./customUtils')
  * @param {Function} options.afterSerialization/options.beforeDeserialization Optional, serialization hooks
  * @param {Number} options.corruptAlertThreshold Optional, threshold after which an alert is thrown if too much data is corrupt
  * @param {Function} options.compareStrings Optional, string comparison function that overrides default for sorting
+ * @param {Object} options.storage Optional, custom storage engine for the database files. Must implement all methods exported by the standard "storage.js" module (included)
  *
  * Event Emitter - Events
  * * compaction.done - Fired whenever a compaction operation was finished
@@ -57,6 +58,7 @@ function Datastore (options) {
                                       , afterSerialization: options.afterSerialization
                                       , beforeDeserialization: options.beforeDeserialization
                                       , corruptAlertThreshold: options.corruptAlertThreshold
+                                      , storage: options.storage
                                       });
 
   // This new executor is ready if we don't use persistence

lib/persistence.js

@@ -5,7 +5,7 @@
  * * Persistence.persistNewState(newDocs, callback) where newDocs is an array of documents and callback has signature err
  */
 
-var storage = require('./storage')
+var defaultStorage = require('./storage')
   , path = require('path')
   , model = require('./model')
   , async = require('async')
@@ -19,6 +19,7 @@ var storage = require('./storage')
  * @param {Datastore} options.db
  * @param {Boolean} options.nodeWebkitAppName Optional, specify the name of your NW app if you want options.filename to be relative to the directory where
  *                                            Node Webkit stores application data such as cookies and local storage (the best place to store data in my opinion)
+ * @param {Object} options.storage Optional, custom storage engine for the database files. Must implement all methods exported by the standard "storage.js" module (included)
  */
 function Persistence (options) {
   var i, j, randomString;
@@ -27,6 +28,7 @@ function Persistence (options) {
   this.inMemoryOnly = this.db.inMemoryOnly;
   this.filename = this.db.filename;
   this.corruptAlertThreshold = options.corruptAlertThreshold !== undefined ? options.corruptAlertThreshold : 0.1;
+  this.storage = options.storage || defaultStorage;
 
   if (!this.inMemoryOnly && this.filename && this.filename.charAt(this.filename.length - 1) === '~') {
     throw new Error("The datafile name can't end with a ~, which is reserved for crash safe backup files");
@@ -59,7 +61,7 @@ function Persistence (options) {
     console.log("require('nw.gui').App.dataPath");
     console.log("See https://github.com/rogerwang/node-webkit/issues/500");
     console.log("==================================================================");
-    this.filename = Persistence.getNWAppFilename(options.nodeWebkitAppName, this.filename);
+    this.filename = this.getNWAppFilename(options.nodeWebkitAppName, this.filename);
   }
 };
 
@@ -68,11 +70,11 @@ function Persistence (options) {
  * Check if a directory exists and create it on the fly if it is not the case
  * cb is optional, signature: err
  */
-Persistence.ensureDirectoryExists = function (dir, cb) {
+Persistence.prototype.ensureDirectoryExists = function (dir, cb) {
   var callback = cb || function () {}
     ;
 
-  storage.mkdirp(dir, function (err) { return callback(err); });
+  this.storage.mkdirp(dir, function (err) { return callback(err); });
 };
 
 
@@ -82,7 +84,7 @@ Persistence.ensureDirectoryExists = function (dir, cb) {
  * Return the path the datafile if the given filename is relative to the directory where Node Webkit stores
  * data for this application. Probably the best place to store data
  */
-Persistence.getNWAppFilename = function (appName, relativeFilename) {
+Persistence.prototype.getNWAppFilename = function (appName, relativeFilename) {
   var home;
 
   switch (process.platform) {
@@ -134,7 +136,7 @@ Persistence.prototype.persistCachedDatabase = function (cb) {
     }
   });
 
-  storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
+  this.storage.crashSafeWriteFile(this.filename, toPersist, function (err) {
     if (err) { return callback(err); }
     self.db.emit('compaction.done');
     return callback(null);
@@ -197,7 +199,7 @@ Persistence.prototype.persistNewState = function (newDocs, cb) {
 
   if (toPersist.length === 0) { return callback(null); }
 
-  storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
+  this.storage.appendFile(self.filename, toPersist, 'utf8', function (err) {
     return callback(err);
   });
 };
@@ -272,9 +274,9 @@ Persistence.prototype.loadDatabase = function (cb) {
 
   async.waterfall([
     function (cb) {
-      Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
-        storage.ensureDatafileIntegrity(self.filename, function (err) {
-          storage.readFile(self.filename, 'utf8', function (err, rawData) {
+      self.ensureDirectoryExists(path.dirname(self.filename), function (err) {
+        self.storage.ensureDatafileIntegrity(self.filename, function (err) {
+          self.storage.readFile(self.filename, 'utf8', function (err, rawData) {
             if (err) { return cb(err); }
 
             try {

test/cursor.test.js

@@ -22,7 +22,7 @@ describe('Cursor', function () {
 
     async.waterfall([
       function (cb) {
-        Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
+        d.persistence.ensureDirectoryExists(path.dirname(testDb), function () {
           fs.exists(testDb, function (exists) {
             if (exists) {
               fs.unlink(testDb, cb);

test/db.test.js

@@ -22,7 +22,7 @@ describe('Database', function () {
 
     async.waterfall([
       function (cb) {
-        Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
+        d.persistence.ensureDirectoryExists(path.dirname(testDb), function () {
           fs.exists(testDb, function (exists) {
             if (exists) {
               fs.unlink(testDb, cb);

test/executor.test.js

@@ -137,7 +137,7 @@ describe('Executor', function () {
 
       async.waterfall([
         function (cb) {
-          Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
+          d.persistence.ensureDirectoryExists(path.dirname(testDb), function () {
             fs.exists(testDb, function (exists) {
               if (exists) {
                 fs.unlink(testDb, cb);

test/persistence.test.js

@@ -24,7 +24,7 @@ describe('Persistence', function () {
 
     async.waterfall([
       function (cb) {
-        Persistence.ensureDirectoryExists(path.dirname(testDb), function () {
+        d.persistence.ensureDirectoryExists(path.dirname(testDb), function () {
           fs.exists(testDb, function (exists) {
             if (exists) {
               fs.unlink(testDb, cb);