Merge pull request #6 from Supernomad/chunked-v3

Chunked v3

.travis.yml

@@ -1,4 +1,8 @@
 language: node_js
 node_js:
   - "0.12"
-before_install: npm install -g grunt-cli
+  - "0.11"
+  - "0.10"
+before_install: npm install -g grunt-cli
+install: npm install
+script: grunt ci

Gruntfile.js

@@ -15,7 +15,7 @@ module.exports = function(grunt) {
         },
         watch: {
             files: ['test/**/*tests.js'],
-            tasks: ['default'],
+            tasks: ['test'],
         },
         mocha_istanbul: {
             coverage: {
@@ -39,6 +39,9 @@ module.exports = function(grunt) {
         }
     });
 
-    grunt.registerTask('build', ['mkdir:testDir', 'mocha_istanbul:coverage', 'coveralls:coverage', 'codacy:coverage']);
-    grunt.registerTask('default', ['mkdir:testDir', 'mocha_istanbul:coverage']);
+    grunt.registerTask('test', ['mkdir:testDir', 'mocha_istanbul:coverage']);
+    grunt.registerTask('coverage', ['codacy:coverage', 'coveralls:coverage']);
+
+    grunt.registerTask('ci', ['test', 'coverage']);
+    grunt.registerTask('default', ['test']);
 };

apis/chunked-api.js

@@ -5,30 +5,20 @@ var express = require('express'),
 var multer = require('multer'),
 	bodyParser = require('body-parser'),
 	typeHelper = require('../libs/helpers/typeHelper'),
+	apiCache = require('../libs/caching/localCache'),
 	io = require('../libs/io'),
-	apiCache = null,
-	uploadRoutes = null,
-	downloadRoutes = null,
 	tempChunkPath = __dirname + '/tmp/chunks';
 
-function configure(cache, options) {
-	if(!typeHelper.isObject(cache)) {
-		apiCache = require('../libs/caching/localCache');
-	}
-	else {
-		apiCache = cache;
-	}
-
-	if(!typeHelper.isObject(options)) {
-		options = {};
+function configure(options) {
+	if(typeHelper.isObject(options) && typeHelper.isString(options.tempChunkPath)) {
+		// Set the temporary upload chunk storage path
+		tempChunkPath = options.tempChunkPath;
 	}
 
-	uploadRoutes = require('../routes/chunked-upload-routes')(apiCache, io, options);
-	downloadRoutes = require('../routes/chunked-download-routes')(apiCache, io, options);
-
 	// Request parsing
 	router.use(bodyParser.json());
 	router.use(bodyParser.urlencoded({extended: false}));
+	// Handle multi-part file uploads
 	router.use(multer({
 		dest: tempChunkPath,
 	  	rename: function (fieldname, filename) {
@@ -37,17 +27,21 @@ function configure(cache, options) {
 	}));
 
 	// Upload routes
+	var uploadRoutes = require('../routes/chunked-upload-routes')(apiCache, io, options);
 	router.get(uploadRoutes.get.uri, uploadRoutes.get.handler);
 	router.post(uploadRoutes.post.uri, uploadRoutes.post.handler);
 	router.put(uploadRoutes.put.uri, uploadRoutes.put.handler);
 	router.delete(uploadRoutes.delete.uri, uploadRoutes.delete.handler);
 	router.use(uploadRoutes.error.handler);
 
 	// Download routes
+	var downloadRoutes = require('../routes/chunked-download-routes')(apiCache, io, options);
 	router.get(downloadRoutes.get.uri, downloadRoutes.get.handler);
 	router.post(downloadRoutes.post.uri, downloadRoutes.post.handler);
 	router.delete(downloadRoutes.delete.uri, downloadRoutes.delete.handler);
 	router.use(downloadRoutes.error.handler);
+
+	// Return the configured router to be 'use'd via another express router or app
 	return router;
 }
 

libs/managers/downloadManager.js

@@ -0,0 +1,102 @@
+/* global Buffer */
+var async = require('async'),
+	apiModels = require('./../models/apiModels'),
+	errorModels = require('./../models/errorModels'),
+	typeHelper = require('./../helpers/typeHelper'),
+	dataCache = null,
+	io = null;
+
+function createDownload(downloadRequest, chunkSize, ttl, done) {
+	async.waterfall([
+		function(callback) {
+			io.GetFileStats(downloadRequest.path, function(error, stats) {
+				if(typeHelper.doesExist(error)) {
+					callback(error);
+				} else {
+					var download = new apiModels.Download(downloadRequest, stats.size, chunkSize);
+					callback(null, download);
+				}
+			});
+		},
+		function(download, callback) {
+			dataCache.create(download.id, download, ttl, function(error, success) {
+				if(!success) {
+					callback(errorModels.ServerError());
+				} else {
+					callback(error, download);
+				}
+			});
+		}
+	], function(error, download) {
+		done(error, download);
+	});
+}
+
+function restoreDownload(downloadId, done) {
+	dataCache.restore(downloadId, function(error, download) {
+		if(typeHelper.doesExist(error)) {
+			done(error);
+		} else if (!typeHelper.doesExist(download.value)) {
+			done(errorModels.DownloadMissing());
+		} else {
+			done(null, download.value);
+		}
+	});
+}
+
+function updateDownload(downloadId, index, chunkSize, ttl, done) {
+	async.waterfall([
+		function(callback) {
+			restoreDownload(downloadId, callback);
+		},
+		function(download, callback) {
+			download.chunks[index] = true;
+			dataCache.update(download.id, download, ttl, function(error, success) {
+				callback(error, download, success);
+			});
+		},
+		function(download, success, callback) {
+			if(success) {
+				var buff = new Buffer(chunkSize);
+				io.ReadFileChunk(download.path, buff, 0, buff.length, index * chunkSize, function(error, read, buffer) {
+					callback(error, buffer);
+				});
+			} else {
+				callback(errorModels.ServerError());
+			}
+		}
+	], function(error, result) {
+		done(error, result);
+	});
+}
+
+function deleteDownload(downloadId, done) {
+	async.waterfall([
+		function(callback) {
+			restoreDownload(downloadId, callback);
+		},
+		function(download, callback) {
+			dataCache.delete(download.id, function(error) {
+				callback(error);
+			});
+		}
+	], function(error) {
+		done(error);
+	});
+}
+
+var manager = {
+	createDownload: createDownload,
+	restoreDownload: restoreDownload,
+	updateDownload:updateDownload,
+	deleteDownload: deleteDownload
+};
+
+function configure(cache, storage) {
+	dataCache = cache;
+	io = storage;
+
+	return manager;
+}
+
+module.exports = configure;

libs/managers/uploadManager.js

@@ -0,0 +1,137 @@
+/* global Buffer */
+var async = require('async'),
+	apiModels = require('./../models/apiModels'),
+	errorModels = require('./../models/errorModels'),
+	typeHelper = require('./../helpers/typeHelper'),
+	dataCache = null,
+	io = null;
+
+function createUpload(uploadRequest, ttl, done) {
+	var upload = new apiModels.Upload(uploadRequest);
+
+	async.waterfall([
+		function createUploadInCache(callback) {
+			dataCache.create(upload.id, upload, ttl, function (error, success) {
+				if(!success) {
+					callback(errorModels.ServerError());
+				} else {
+					callback(error, upload);
+				}
+			});
+		},
+		function createTemporaryFile(upload, callback) {
+			var buff = new Buffer(upload.fileSize);
+			buff.fill(0);
+
+			io.CreateFile(upload.tempPath, buff, 0, buff.length, function(error) {
+				callback(error, upload);
+			});
+		}
+	], function uploadCreated(error, upload) {
+		done(error, upload);
+	});
+}
+
+function restoreUpload(uploadId, done) {
+	dataCache.restore(uploadId, function name(error, upload) {
+		if(typeHelper.doesExist(error)) {
+			done(error);
+		} else if(!typeHelper.doesExist(upload.value)) {
+			done(errorModels.UploadMissing());
+		} else {
+			done(null, upload.value);
+		}
+	});
+}
+
+function updateUpload(uploadId, index, file, ttl, done) {
+	async.waterfall([
+		function(callback) {
+			restoreUpload(uploadId, callback);
+		},
+		function(upload, callback) {
+			upload.chunks[index] = true;
+			dataCache.update(upload.Id, upload, ttl, function(error, success) {
+				if(!success) {
+					callback(errorModels.ServerError());
+				} else {
+					callback(error, upload);
+				}
+			});
+		},
+		function(upload, callback) {
+			io.ReadFile(file.path, function(error, data) {
+				callback(error, upload, data);
+			});
+		},
+		function(upload, data, callback) {
+			io.WriteFileChunk(upload.TempPath, data, 0, data.length, index * upload.ChunkSize, function(error) {
+				callback(error, upload);
+			});
+		},
+		function(upload, callback) {
+			async.every(upload.chunks, function(item, isTrue) {
+				isTrue(item === true);
+			}, function(result) {
+				callback(null, upload, result);
+			});
+		},
+		function(upload, complete, callback) {
+			if(complete) {
+				async.series([
+					function(call) {
+						io.RenameFile(upload.TempPath, upload.FinalPath, function(error) {
+							call(error);
+						});
+					},
+					function(call) {
+						dataCache.delete(upload.id, function(error) {
+							call(error);
+						});
+					}
+				], function(error) {
+					callback(error, upload, complete);
+				});
+			} else {
+				callback(null, upload, complete);
+			}
+		}
+	], function(error, upload, complete) {
+		done(error, upload, complete);
+	});
+}
+
+function deleteUpload(uploadId, done) {
+	async.waterfall([
+		function(callback) {
+			restoreUpload(uploadId, callback);
+		},
+		function(upload, callback) {
+			dataCache.delete(upload.id, function (error) {
+				callback(error, upload);
+			});
+		},
+		function(upload, callback) {
+			io.DeleteFile(upload.tempPath, function(deleteError) {
+				callback(deleteError);
+			});
+		}
+	], function(error) {
+		done(error);
+	});
+}
+
+var manager = {
+	createUpload: createUpload,
+	restoreUpload: restoreUpload,
+	updateUpload: updateUpload,
+	deleteUpload: deleteUpload
+};
+
+function configure(cache, storage) {
+	dataCache = cache;
+	io = storage;
+
+	return manager;
+}
+module.exports = configure;