diff --git a/README.md b/README.md index dab64fb..162d6e9 100644 --- a/README.md +++ b/README.md @@ -22,4 +22,4 @@ The command is long running, and will watch for file changes until the process d > You can omit the `$ACCESS_TOKEN` and enter it interactively. ## Reset -Every time you run this command, a JSON file is stored in `~/.anyfetch-file-watcher/$ACCESS_TOKEN/$STRIPPED_DIRECTORY.json` with cursor details. If you want to resend everything, remove this file. +Every time you run this command, a JSON file is stored in `~/.anyfetch-file-watcher/$STRIPPED_DIRECTORY.json` with cursor details. If you want to resend everything, remove this file. diff --git a/lib/helpers/save-path.js b/lib/helpers/save-path.js new file mode 100644 index 0000000..84a2c17 --- /dev/null +++ b/lib/helpers/save-path.js @@ -0,0 +1,23 @@ +"use strict"; +var fs = require("fs"); + +var getSavePath = function(dir) { + var homeDir = process.env.HOMEPATH || process.env.HOME; + if(!getSavePath.hasMkdir) { + try { + fs.mkdirSync(homeDir + "/.anyfetch-file-watcher/"); + } + catch(err) {} + getSavePath.hasMkdir = true; + } + return homeDir + "/.anyfetch-file-watcher/" + dir.trim().replace(/(\/|\\)/g, ''); +}; +getSavePath.hasMkdir = false; + + +var saveCursor = function(dir, newCursor, cb) { + fs.writeFile(getSavePath(dir), JSON.stringify(newCursor), cb); +}; + +module.exports = saveCursor; +module.exports.getSavePath = getSavePath; diff --git a/lib/helpers/upload.js b/lib/helpers/upload.js index 69b33ac..2224c71 100644 --- a/lib/helpers/upload.js +++ b/lib/helpers/upload.js @@ -1,11 +1,12 @@ "use strict"; -var Anyfetch = require('anyfetch'); +var AnyfetchClient = require('anyfetch'); var fs = require('fs'); var path = require('path'); +var async = require('async'); -module.exports = function uploadFile(filePath, accessToken, baseIdentifier, cb) { - var anyfetch = new Anyfetch(); +var uploadFile = function(dir, filePath, accessToken, baseIdentifier, cb) { + var anyfetch = new AnyfetchClient(); anyfetch.setAccessToken(accessToken); // Send a document to anyFetch @@ -16,10 +17,22 @@ module.exports = function uploadFile(filePath, accessToken, baseIdentifier, cb) var fileConfig = function() { // Wrap this in a function to avoid creating the stream before reading it. return { - file: fs.createReadStream(filePath), - filename: path.baseName(filePath), + file: fs.createReadStream(dir + filePath), + filename: path.basename(dir + filePath), }; }; - anyfetch.sendDocumentAndFile(document, fileConfig, cb); + anyfetch.sendDocumentAndFile(document, fileConfig, function(err) { + cb(err); + }); }; + +var queue = async.queue(function worker(task, cb) { + uploadFile(task.dir, task.filePath, task.accessToken, task.baseIdentifier, cb); +}, 4); + +module.exports = function pushToQueue(task) { + queue.push(task); +}; + +module.exports.uploadFile = uploadFile; diff --git a/lib/index.js b/lib/index.js new file mode 100644 index 0000000..238ac4f --- /dev/null +++ b/lib/index.js @@ -0,0 +1,45 @@ +"use strict"; +var fs = require('fs'); +var async = require('async'); +var os = require('os'); + +var retrieveFiles = require('./helpers/list-files').retrieveFiles; +var pushToQueue = require('./helpers/upload'); +var saveCursor = require('./helpers/save-path'); +var getSavePath = saveCursor.getSavePath; + + +module.exports = function update(dir, accessToken, cb) { + async.waterfall([ + function getOldCursor(cb) { + fs.readFile(getSavePath(dir), function(err, cursor) { + if(err && err.code !== 'ENOENT') { + cb(err); + } + else { + cb(null, cursor); + } + }); + }, + function updateCursor(oldCursor, cb) { + retrieveFiles(dir, oldCursor, cb); + }, + function saveNewCursor(filesToUpload, newCursor, cb) { + saveCursor(dir, newCursor, function(err) { + cb(err, filesToUpload); + }); + }, + function uploadFiles(files, cb) { + files.forEach(function(key) { + var task = { + 'dir': dir, + 'filePath': key, + 'accessToken': accessToken, + 'baseIdentifier': os.hostname() + }; + pushToQueue(task); + }); + cb(); + } + ], cb); +}; diff --git a/package.json b/package.json index 76917b5..9f501b5 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "anyfetch-file-watcher", "description": "Watch for file changes, and send them to anyFetch.", - "version": "0.3.15", + "version": "0.0.1", "author": "Matthieu Bacconnier ", "scripts": { - "test": "NODE_ENV=test mocha --recursive -R spec test/", + "test": "NODE_ENV=test mocha --recursive -R spec test/ -t 5000", "lint": "jshint lib/ test/" }, "main": "./lib/", diff --git a/test/helpers/list-files.js b/test/helpers/list-files.js index f2c4d00..bf7f495 100644 --- a/test/helpers/list-files.js +++ b/test/helpers/list-files.js @@ -39,7 +39,10 @@ describe("Retrieve file", function () { } // Should contain new files and updated files - fileToUpload.should.eql(['/txt3.txt', '/test/txt1.doc', '/test/txt2.txt']); + fileToUpload.should.include('/txt3.txt'); + fileToUpload.should.include('/test/txt1.doc'); + fileToUpload.should.include('/test/txt2.txt'); + fileToUpload.should.have.lengthOf(3); newCursor.should.eql({ '/txt1.txt': fs.statSync(__dirname + '/../sample-directory/txt1.txt').mtime.getTime(), '/txt2.txt': fs.statSync(__dirname + '/../sample-directory/txt2.txt').mtime.getTime(), diff --git a/test/helpers/upload.js b/test/helpers/upload.js new file mode 100644 index 0000000..f66e28b --- /dev/null +++ b/test/helpers/upload.js @@ -0,0 +1,40 @@ +'use strict'; + +require('should'); + +var uploadFile = require('../../lib/helpers/upload.js').uploadFile; +var Anyfetch = require('anyfetch'); + +describe('uploadFile', function() { + + process.env.ANYFETCH_API_URL = 'http://localhost:1338'; + var countFile = 0; + var mockServerHandler = function(url){ + if (url.indexOf("/file") !== -1) { + countFile += 1; + } + }; + var apiServer; + + before(function() { + // Create a fake HTTP server + apiServer = Anyfetch.debug.createTestApiServer(mockServerHandler); + apiServer.listen(1338); + }); + + after(function(){ + apiServer.close(); + }); + + it('should upload the file', function(done) { + + uploadFile(__dirname + "/..", "/sample-directory/txt1.txt", "randomAccessToken", "randomBaseIdentifier", function(err) { + if(err) { + throw err; + } + countFile.should.be.eql(1); + done(); + }); + + }); +}); diff --git a/test/index.js b/test/index.js new file mode 100644 index 0000000..8946ef8 --- /dev/null +++ b/test/index.js @@ -0,0 +1,49 @@ +'use strict'; + +require('should'); + +var path = require('path'); +var Anyfetch = require('anyfetch'); +var fs = require('fs'); + +var update = require('../lib/index.js'); +var getSavePath = require('../lib/helpers/save-path.js').getSavePath; + + +describe('update() function', function() { + process.env.ANYFETCH_API_URL = 'http://localhost:1338'; + var countFile = 0; + var mockServerHandler = function(url){ + if (url.indexOf("/file") !== -1) { + countFile += 1; + } + }; + + var apiServer; + before(function() { + // Create a fake HTTP server + apiServer = Anyfetch.debug.createTestApiServer(mockServerHandler); + apiServer.listen(1338); + }); + + after(function(){ + apiServer.close(); + + // Clean cursor + + fs.unlinkSync(getSavePath(path.resolve(__dirname + "/../test/sample-directory"))); + }); + + it('should update account', function(done) { + update(path.resolve(__dirname + "/../test/sample-directory"), "randomAccessToken", function(err) { + if(err) { + throw err; + } + setTimeout(function() { + countFile.should.eql(5); + done(); + }, 200); + }); + + }); +});