Refactor callbacks and errors

lib/indexing/calibrater.js

@@ -1,8 +1,8 @@
 
 exports.calibrate = function(indexes, callback) {
-  documentFrequencies(indexes, function(msg) {
-    countDocuments(indexes, function(msg) {
-      callback(msg);
+  documentFrequencies(indexes, function(err, msg) {
+    countDocuments(indexes, function(err, msg) {
+      callback(err, msg);
     });
   });
 };
@@ -11,11 +11,11 @@ exports.getTotalDocs = function(indexes, callback) {
   indexes.get('search-index.totalDocs', function(err, value){
     if (err) {
       searchIndexLogger.info('Index is empty or not calibrated');
-      callback(0);
+      callback(null, 0);
     }
     else {
       searchIndexLogger.info(value + ' documents searchable');
-      callback(value);
+      callback(null, value);
     }
   });
 }
@@ -30,7 +30,7 @@ exports.incrementallyCalibrate = function(indexesMultiply, tf, callback) {
       }
     }
     searchIndexLogger.info('counting documents');
-    countDocuments(indexesMultiply, function(msg) {
+    countDocuments(indexesMultiply, function(err, msg) {
       searchIndexLogger.info('sorting tf sets');
       //resort all keys
       for (var k in tf['reducedTF']) {
@@ -49,7 +49,7 @@ exports.incrementallyCalibrate = function(indexesMultiply, tf, callback) {
             }
           }
           searchIndexLogger.info('counting documents');
-          countDocuments(indexesMultiply, function(msg) {
+          countDocuments(indexesMultiply, function(err, msg) {
             searchIndexLogger.info('sorting tf sets');
             //resort all keys
             for (var k in tf['reducedTFSortOnTF']) {
@@ -65,7 +65,7 @@ exports.incrementallyCalibrate = function(indexesMultiply, tf, callback) {
             }
             searchIndexLogger.info('reinserting tf sets');
             indexesMultiply.put(tf['reducedTFSortOnTF'], function(err){
-              callback('[success] incremental calibration complete')
+              callback(null, '[success] incremental calibration complete')
             });
           });
         });
@@ -89,7 +89,7 @@ countDocuments = function(indexes, callback) {
     })
     .on('end', function() {
       indexes.put('search-index.totalDocs', tally, function(){
-        callback('calibrated ' + tally + ' docs');
+        callback(null, 'calibrated ' + tally + ' docs');
       });
     });  
 };
@@ -123,7 +123,7 @@ documentFrequencies = function(indexes, callback) {
       }
     })
     .on('close', function() {
-      callback('done');
+      callback(null, 'done');
     });  
 }
 

lib/indexing/deleter.js

@@ -26,7 +26,7 @@ function decrementRIArray(docID, indexesMultiply, indexes, keys, callback) {
       decrementKeys.push(batchOperation);
     }
     indexes.batch(decrementKeys, function (err) {
-      callback('[success] index recalibrated');
+      callback(null, '[success] index recalibrated');
     })
   })
 }
@@ -66,7 +66,7 @@ function decrementTFArray(docID, indexesMultiply, indexes, keys, callback) {
       if (err) {
         searchIndexLogger.error('problem decrementing keys: ' + err);
       }
-      else callback('index recalibrated');
+      else callback(null, 'index recalibrated');
     })
   })
 }
@@ -87,11 +87,11 @@ function deleteArtifacts(docID, indexes, indexesMultiply, callback) {
     var deleteIDsFromTheseRIArrayKeys = [];
     for (var i = 0; i < deleteIDsFromTheseArrayKeys.length; i++)
       deleteIDsFromTheseRIArrayKeys[i] = 'RI~' + deleteIDsFromTheseArrayKeys[i];
-    decrementTFArray(docID, indexesMultiply, indexes, deleteIDsFromTheseTFArrayKeys, function(err){
-      decrementRIArray(docID, indexesMultiply, indexes, deleteIDsFromTheseRIArrayKeys, function(err){
+    decrementTFArray(docID, indexesMultiply, indexes, deleteIDsFromTheseTFArrayKeys, function(err, msg){
+      decrementRIArray(docID, indexesMultiply, indexes, deleteIDsFromTheseRIArrayKeys, function(err, msg){
         indexesMultiply.del(deleteKeys, function (err, data) {
           // database no longer has 'foo', 'boom' or 'whoa' ('wha' was never there)
-          callback(false);
+          callback(null, false);
         })
       });
     });
@@ -100,21 +100,22 @@ function deleteArtifacts(docID, indexes, indexesMultiply, callback) {
 
 exports.doesDocExist = function(docID, indexes, callback) {
   var testKey = 'DOCUMENT~' + docID + '~';
-  indexPeek.indexValue(testKey, indexes, function(msg){
-    if (msg == '[warning] key not found') return callback(false);
-    return callback(true);
+  indexPeek.indexValue(testKey, indexes, function(err, msg){
+    if (msg == '[warning] key not found') return callback(null, false);
+    return callback(null, true);
   });
 }
 
 exports.deleteDoc = function(docID, indexes, indexesMultiply, callback) {
-  this.doesDocExist(docID, indexes, function(docExists){
+  this.doesDocExist(docID, indexes, function(err, docExists){
     if (docExists) {
-      deleteArtifacts(docID, indexes, indexesMultiply, function(msg) {
-        return callback(true);
+      deleteArtifacts(docID, indexes, indexesMultiply, function(err, msg) {
+        return callback(null, true);
       });
     }
     else {
-      return callback({'notFound': docID + ' cannot be deleted because it is not in index'});
+      return callback(new Error('NotFound'));
+      //{'notFound': docID + ' cannot be deleted because it is not in index'});
     }
   });
 }

lib/indexing/indexPeek.js

@@ -11,19 +11,21 @@ exports.indexRange = function (start, stop, index, callback) {
       callback(err);
     })
     .on('end', function() {
-      callback(dump);
+      callback(null, dump);
     })
     .on('close', function() {
-      callback(dump);
+      callback(null, dump);
     });
 }
 
 exports.indexValue = function (key, index, callback) {
   index.get(key, function (err, value) {
     if (err) {
       if (err.notFound)
-        return callback(true, false)
+        return callback(null, false);
+      else
+        return callback(err);
     }
-    return callback(false, value);
+    return callback(null, value);
   });
 }

lib/indexing/indexer.js

@@ -150,12 +150,15 @@ function indexDoc(reverseIndex, docID, doc, facets, callback) {
 
   //put key-values into database
   reverseIndex.batch(fieldBatch, function (err) {
+    if (err) {
+      searchIndexLogger.error('Ooops!', err);
+      return callback(err);
+    }
+
     var msg = {};
     msg['status'] = 'Indexed doc ' + docID;
     msg['tfValues'] = tfValues;
-    callback(msg);
-    if (err) return searchIndexLogger.error('Ooops!', err);
-    return;
+    return callback(null, msg);
   });
 }
 
@@ -173,15 +176,15 @@ exports.addDocToIndex = function(indexes, indexesMultiply, batch, batchName, fil
         //verify document format
         if (Object.prototype.toString.call(batch[i]) != '[object Object]') {
           searchIndexLogger.error('malformed document: check document formatting at batch index ' + i);
-          return callback(true);
+          return callback(new Error('Malformed document'));
         }
         var docID = '';
         if (batch[i]['id'])
           docID = batch[i]['id'];
         else
           docID = Date.now() + '-' + Math.random()*10000000000000000 + '-' + hash(batch[i]);
-        deleter.deleteDoc(docID, indexes, indexesMultiply, function(msgg) {
-          indexDoc(indexes, docID, batch[i], filters, function(msg) {
+        deleter.deleteDoc(docID, indexes, indexesMultiply, function(err, msgg) {
+          indexDoc(indexes, docID, batch[i], filters, function(err, msg) {
             searchIndexLogger.info(msg.status);
             tf[docID] = msg.tfValues;
             indexDocs(++i);
@@ -190,10 +193,10 @@ exports.addDocToIndex = function(indexes, indexesMultiply, batch, batchName, fil
       }
       else {
         searchIndexLogger.info('starting calibration');
-        calibrater.incrementallyCalibrate(indexesMultiply, reduceTF(tf), function(msg) {
+        calibrater.incrementallyCalibrate(indexesMultiply, reduceTF(tf), function(err, msg) {
           searchIndexLogger.info(msg);
           searchIndexLogger.info('indexed batch: ' + batchName);
-          callback(false);
+          callback(null);
         });
       }
     };

lib/indexing/replicator.js

@@ -1,23 +1,21 @@
 var level = require('levelup'),
-JSONStream = require('JSONStream'),
-zlib = require('zlib'),
-inflater = zlib.createGunzip(),
-gzip = zlib.createGzip();
+    JSONStream = require('JSONStream'),
+    zlib = require('zlib'),
+    inflater = zlib.createGunzip(),
+    gzip = zlib.createGzip();
 
 
-exports.replicateFromSnapShot = function(readStream, indexes, callback) {
-  var err = false;
-  readStream.pipe(inflater)
-    .pipe(JSONStream.parse())
-    .pipe(indexes.createWriteStream())
-    .on('close', function() {
-      callback(err);
-    });
+exports.replicateFromSnapShot = function (readStream, indexes, callback) {
+    readStream.pipe(inflater)
+        .pipe(JSONStream.parse())
+        .pipe(indexes.createWriteStream())
+        .on('close', callback);
 }
 
 
-exports.createSnapShot = function(indexes, callback) {
-  callback(indexes.createReadStream()
-    .pipe(JSONStream.stringify('','\n',''))
-    .pipe(gzip));
+exports.createSnapShot = function (indexes, callback) {
+    indexes.createReadStream()
+        .on('finish', callback)
+        .pipe(JSONStream.stringify('', '\n', ''))
+        .pipe(gzip);
 }

lib/matchers/matcher.js

@@ -4,7 +4,7 @@ var threshold = 3; //only give matches for strings longer or equal to than this
 exports.matcher = function(reverseIndex, beginsWith, callback) {
   //sggestion string is too short
   var suggestions = [];
-  if (beginsWith.length < threshold) return callback(false, suggestions);
+  if (beginsWith.length < threshold) return callback(null, suggestions);
   reverseIndex.createReadStream({
     start: 'TF~*~' + beginsWith,
     end: 'TF~*~' + beginsWith + '~~~'})
@@ -24,6 +24,6 @@ exports.matcher = function(reverseIndex, beginsWith, callback) {
       for (var i = 0; i < sortedSuggestions.length; i++) {
         simpleSortedSuggestions.push(sortedSuggestions[i][0]);
       }
-      callback(false, simpleSortedSuggestions);
+      callback(null, simpleSortedSuggestions);
     });
 }

lib/search-index.js

@@ -50,7 +50,7 @@ var SearchIndex = module.exports = function (options) {
     });
   }
 
-  calibrater.getTotalDocs(indexes, function(totalDocs) {
+  calibrater.getTotalDocs(indexes, function(err, totalDocs) {
     searcher.setTotalDocs(totalDocs);
   });
 
@@ -117,7 +117,7 @@ SearchIndex.match = function(beginsWith, callback) {
 SearchIndex.tellMeAboutMySearchIndex = function(callback) {
   if (!indexes) SearchIndex();
 
-  calibrater.getTotalDocs(indexes, function(totalDocs) {
+  calibrater.getTotalDocs(indexes, function(err, totalDocs) {
     var metadata = {};
     metadata['totalDocs'] = totalDocs;
     callback(metadata);
@@ -145,8 +145,8 @@ SearchIndex.snapShot = function(callback) {
 //******************************************
 SearchIndex.indexRange = function(options, callback) {
   if (!indexes) SearchIndex();
-  indexPeek.indexRange(options.start, options.stop, indexes, function(err) {
-    callback(err);
+  indexPeek.indexRange(options.start, options.stop, indexes, function(err, dump) {
+    callback(err, dump);
   });
 };
 SearchIndex.indexValue = function(options, callback) {
@@ -160,7 +160,7 @@ SearchIndex.indexValue = function(options, callback) {
 SearchIndex.calibrate = function(callback) {
   if (!indexes) SearchIndex();
 
-  calibrater.calibrate(indexes, function(msg) {
-    callback(msg);
+  calibrater.calibrate(indexes, function(err, msg) {
+    callback(err, msg);
   });
 };

lib/search/docGetter.js

@@ -38,7 +38,7 @@ exports.getDocArtifacts = function (reverseIndex, reverseIndexMultiply, docID, c
       for (k in data) {
         if (data[k]) res[k] = data[k];
       }
-      callback(false, res);
+      callback(null, res);
     })
   })
 }

test/spec/0indexing-spec.js

@@ -1,5 +1,5 @@
 var fs = require('fs');
-var si = require('../../')({logLevel:'info'});
+var si = require('../../')({logSilent: true});
 
 
 describe('indexing and search', function () {
@@ -10,16 +10,18 @@ describe('indexing and search', function () {
   it('should index one file of test data', function () {
     runs(function() {
       this.err = undefined;
+      this.done = false;
       var that = this;
       si.add({'batchName': 'reuters-000.json', 'filters': ['places']}, data, function(err) {
         that.err = err;
+        that.done = true;
       });
     });
     waitsFor(function() {
-      return this.err != undefined;
+      return this.done != false;
     }, 'err not to be empty (search err returned)', 30000)
     runs(function () {
-      expect(this.err).toEqual(false);
+      expect(this.err).toEqual(null);
     });
   });
 
@@ -36,9 +38,9 @@ describe('indexing and search', function () {
     });
     waitsFor(function() {
       return this.err != undefined;
-    }, 'err to be true', 30000)
+    }, 'err to be Malformed document', 30000)
     runs(function () {
-      expect(this.err).toEqual(true);
+      expect(this.err.message).toEqual('Malformed document');
     });
   });
 

test/spec/1.5get-doc-spec.js

@@ -1,5 +1,5 @@
 var fs = require('fs');
-var si = require('../../');
+var si = require('../../')({logSilent: false});
 var should = require('should');
 
 describe('get-ting', function () {

test/spec/1searching-spec.js

@@ -1,5 +1,5 @@
 var fs = require('fs');
-var si = require('../../');
+var si = require('../../')({logLevel:false});
 
 
 describe('indexing and search', function () {