save, load and renew cursor

README.md

@@ -22,4 +22,4 @@ The command is long running, and will watch for file changes until the process d
 > You can omit the `$ACCESS_TOKEN` and enter it interactively.
 
 ## Reset
-Every time you run this command, a JSON file is stored in `~/.anyfetch-file-watcher/$ACCESS_TOKEN/$STRIPPED_DIRECTORY.json` with cursor details. If you want to resend everything, remove this file.
+Every time you run this command, a JSON file is stored in `~/.anyfetch-file-watcher/$STRIPPED_DIRECTORY.json` with cursor details. If you want to resend everything, remove this file.

lib/helpers/save-path.js

@@ -0,0 +1,23 @@
+"use strict";
+var fs = require("fs");
+
+var getSavePath = function(dir) {
+  var homeDir = process.env.HOMEPATH || process.env.HOME;
+  if(!getSavePath.hasMkdir) {
+    try {
+      fs.mkdirSync(homeDir + "/.anyfetch-file-watcher/");
+    }
+    catch(err) {}
+    getSavePath.hasMkdir = true;
+  }
+  return homeDir + "/.anyfetch-file-watcher/" + dir.trim().replace(/(\/|\\)/g, '');
+};
+getSavePath.hasMkdir = false;
+
+
+var saveCursor = function(dir, newCursor, cb) {
+  fs.writeFile(getSavePath(dir), JSON.stringify(newCursor), cb);
+};
+
+module.exports = saveCursor;
+module.exports.getSavePath = getSavePath;

lib/helpers/upload.js

@@ -1,11 +1,12 @@
 "use strict";
 
-var Anyfetch = require('anyfetch');
+var AnyfetchClient = require('anyfetch');
 var fs = require('fs');
 var path = require('path');
+var async = require('async');
 
-module.exports = function uploadFile(filePath, accessToken, baseIdentifier, cb) {
-  var anyfetch = new Anyfetch();
+var uploadFile = function(dir, filePath, accessToken, baseIdentifier, cb) {
+  var anyfetch = new AnyfetchClient();
   anyfetch.setAccessToken(accessToken);
 
   // Send a document to anyFetch
@@ -16,10 +17,22 @@ module.exports = function uploadFile(filePath, accessToken, baseIdentifier, cb)
   var fileConfig = function() {
     // Wrap this in a function to avoid creating the stream before reading it.
     return {
-      file: fs.createReadStream(filePath),
-      filename: path.baseName(filePath),
+      file: fs.createReadStream(dir + filePath),
+      filename: path.basename(dir + filePath),
     };
   };
 
-  anyfetch.sendDocumentAndFile(document, fileConfig, cb);
+  anyfetch.sendDocumentAndFile(document, fileConfig, function(err) {
+    cb(err);
+  });
 };
+
+var queue = async.queue(function worker(task, cb) {
+  uploadFile(task.dir, task.filePath, task.accessToken, task.baseIdentifier, cb);
+}, 4);
+
+module.exports = function pushToQueue(task) {
+  queue.push(task);
+};
+
+module.exports.uploadFile = uploadFile;

lib/index.js

@@ -0,0 +1,45 @@
+"use strict";
+var fs = require('fs');
+var async = require('async');
+var os = require('os');
+
+var retrieveFiles = require('./helpers/list-files').retrieveFiles;
+var pushToQueue = require('./helpers/upload');
+var saveCursor = require('./helpers/save-path');
+var getSavePath = saveCursor.getSavePath;
+
+
+module.exports = function update(dir, accessToken, cb) {
+  async.waterfall([
+    function getOldCursor(cb) {
+      fs.readFile(getSavePath(dir), function(err, cursor) {
+        if(err && err.code !== 'ENOENT') {
+          cb(err);
+        }
+        else {
+          cb(null, cursor);
+        }
+      });
+    },
+    function updateCursor(oldCursor, cb) {
+      retrieveFiles(dir, oldCursor, cb);
+    },
+    function saveNewCursor(filesToUpload, newCursor, cb) {
+      saveCursor(dir, newCursor, function(err) {
+        cb(err, filesToUpload);
+      });
+    },
+    function uploadFiles(files, cb) {
+      files.forEach(function(key) {
+        var task = {
+          'dir': dir,
+          'filePath': key,
+          'accessToken': accessToken,
+          'baseIdentifier': os.hostname()
+        };
+        pushToQueue(task);
+      });
+      cb();
+    }
+  ], cb);
+};

package.json

@@ -1,10 +1,10 @@
 {
   "name": "anyfetch-file-watcher",
   "description": "Watch for file changes, and send them to anyFetch.",
-  "version": "0.3.15",
+  "version": "0.0.1",
   "author": "Matthieu Bacconnier <matthieu@anyfetch.com>",
   "scripts": {
-    "test": "NODE_ENV=test mocha --recursive  -R spec test/",
+    "test": "NODE_ENV=test mocha --recursive  -R spec test/ -t 5000",
     "lint": "jshint lib/ test/"
   },
   "main": "./lib/",

test/helpers/list-files.js

@@ -39,7 +39,10 @@ describe("Retrieve file", function () {
       }
 
       // Should contain new files and updated files
-      fileToUpload.should.eql(['/txt3.txt', '/test/txt1.doc', '/test/txt2.txt']);
+      fileToUpload.should.include('/txt3.txt');
+      fileToUpload.should.include('/test/txt1.doc');
+      fileToUpload.should.include('/test/txt2.txt');
+      fileToUpload.should.have.lengthOf(3);
       newCursor.should.eql({
         '/txt1.txt': fs.statSync(__dirname + '/../sample-directory/txt1.txt').mtime.getTime(),
         '/txt2.txt': fs.statSync(__dirname + '/../sample-directory/txt2.txt').mtime.getTime(),

test/helpers/upload.js

@@ -0,0 +1,40 @@
+'use strict';
+
+require('should');
+
+var uploadFile = require('../../lib/helpers/upload.js').uploadFile;
+var Anyfetch = require('anyfetch');
+
+describe('uploadFile', function() {
+
+  process.env.ANYFETCH_API_URL = 'http://localhost:1338';
+  var countFile = 0;
+  var mockServerHandler = function(url){
+    if (url.indexOf("/file") !== -1) {
+      countFile += 1;
+    }
+  };
+  var apiServer;
+
+  before(function() {
+    // Create a fake HTTP server
+    apiServer = Anyfetch.debug.createTestApiServer(mockServerHandler);
+    apiServer.listen(1338);
+  });
+
+  after(function(){
+    apiServer.close();
+  });
+
+  it('should upload the file', function(done) {
+
+    uploadFile(__dirname + "/..", "/sample-directory/txt1.txt", "randomAccessToken", "randomBaseIdentifier", function(err) {
+      if(err) {
+        throw err;
+      }
+      countFile.should.be.eql(1);
+      done();
+    });
+
+  });
+});

test/index.js

@@ -0,0 +1,49 @@
+'use strict';
+
+require('should');
+
+var path = require('path');
+var Anyfetch = require('anyfetch');
+var fs = require('fs');
+
+var update = require('../lib/index.js');
+var getSavePath = require('../lib/helpers/save-path.js').getSavePath;
+
+
+describe('update() function', function() {
+  process.env.ANYFETCH_API_URL = 'http://localhost:1338';
+  var countFile = 0;
+  var mockServerHandler = function(url){
+    if (url.indexOf("/file") !== -1) {
+      countFile += 1;
+    }
+  };
+
+  var apiServer;
+  before(function() {
+    // Create a fake HTTP server
+    apiServer = Anyfetch.debug.createTestApiServer(mockServerHandler);
+    apiServer.listen(1338);
+  });
+
+  after(function(){
+    apiServer.close();
+
+    // Clean cursor
+
+    fs.unlinkSync(getSavePath(path.resolve(__dirname + "/../test/sample-directory")));
+  });
+
+  it('should update account', function(done) {
+    update(path.resolve(__dirname + "/../test/sample-directory"), "randomAccessToken", function(err) {
+      if(err) {
+        throw err;
+      }
+      setTimeout(function() {
+        countFile.should.eql(5);
+        done();
+      }, 200);
+    });
+
+  });
+});