Skip to content

Commit

Permalink
fixes #77 : Generic refactoring
Browse files Browse the repository at this point in the history 
- Moved local & s3 storage factories to separate modules
- Moved settings object to separate module
- Cleanup of (un)used module references
- Small bug fixes
  • Loading branch information
@remie
remie committed Sep 15, 2015
1 parent b56898d commit 976a326
Show file tree
Hide file tree
Showing 6 changed files with 385 additions and 369 deletions.
6 changes: 2 additions & 4 deletions app.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@ nconf.set('basedir', __dirname);

// ------------------------------------------------------------------------------------------ App Dependencies

var fs = require("fs");
var path = require("path");
var _ = require("lodash");
var scheduler = require('./lib/scheduler.js');
var youtransfer = require('./lib/youtransfer.js');
Expand Down Expand Up @@ -105,14 +103,14 @@ app.use(function(req, res, next) {
// ------------------------------------------------------------------------------------------ App Routing

// Initialize routes
require('./lib/routes.js')(app, nconf);
require('./lib/routes')(app, nconf);

// ------------------------------------------------------------------------------------------ App Scheduling

youtransfer.settings.get(function(err, settings) {
if(!err) {
scheduler.add('cleanup', settings.cleanupSchedule, youtransfer.cleanup);
youtransfer.on('settings.push', function(err, data) {
youtransfer.settings.on('settings.push', function(err, data) {
scheduler.reschedule('cleanup', data.cleanupSchedule, youtransfer.cleanup);
});
}
Expand Down
132 changes: 132 additions & 0 deletions lib/localstorage.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
'use strict';

// ------------------------------------------------------------------------------------------ Dependencies

require('date-utils');
var fs = require("fs");
var del = require("del");
var path = require('path');
var mime = require('mime');
var _ = require("lodash");
var archiver = require('archiver');

// ------------------------------------------------------------------------------------------ Module definition

module.exports = function(options) {
return new LocalFileStorage(options);
}

function LocalFileStorage(options) {
options = options || {};
this.options = options;

if(_.isString(this.options)) {
this.localstoragepath = this.options;
} else if(_.isObject(this.options)) {
this.localstoragepath = options.localstoragepath || __dirname;
} else {
throw "Invalid options provided";
}
}

LocalFileStorage.prototype.upload = function(file, context, next) {
var basedir = path.dirname(context.path);
fs.mkdir(basedir, function() {
fs.readFile(file.path, function (err, data) {
if(err) {
next(err, context);
} else {
fs.writeFile(context.path, data, function(err) {
if(err) {
next(err, context);
} else {
fs.writeFile(context.jsonPath, JSON.stringify(context), 'utf-8', function(err) {
next(err, context);
});
}
});
}
});
});
};

LocalFileStorage.prototype.bundle = function(bundle, next) {
var basedir = path.dirname(bundle.path);
fs.mkdir(basedir, function() {
fs.writeFile(bundle.path, JSON.stringify(bundle), 'utf-8', function(err) {
next(err);
});
});
};

LocalFileStorage.prototype.archive = function(token, res, next) {
try {
var basedir = this.localstoragepath;
if(token) {
var bundle = require(path.join(basedir, token + '.json'));
if(bundle) {

res.setHeader('Content-disposition', 'attachment; filename="bundle.zip"');
res.setHeader('Content-type', 'application/octet-stream');

var archive = archiver('zip');
_.each(bundle.files, function(file, key) {
archive.file(path.join(basedir, file.id + '.binary'), { name: file.name });
});

archive.pipe(res);
archive.finalize();
} else {
throw 'Invalid bundle identifier';
}
} else {
throw 'Bundle identifier unknown';
}
} catch(err) {
next(err);
}
};

LocalFileStorage.prototype.download = function(token, res, next) {
try {
var basedir = this.localstoragepath;
if(token) {
token = token.trim();
var context = require(path.join(basedir, token + '.json'));
if(context) {
var file = path.join(basedir, token + '.binary');
var mimetype = mime.lookup(file) || context.type;

res.setHeader('Content-disposition', 'attachment; filename="' + context.name + '"');
res.setHeader('Content-length', context.size);
res.setHeader('Content-type', mimetype);

var filestream = fs.createReadStream(file);
filestream.pipe(res);
} else {
throw "token not found exception";
}
} else {
throw "invalid token exception";
}
} catch (err) {
next(err);
}
};

LocalFileStorage.prototype.purge = function() {
var basedir = this.localstoragepath;
fs.readdir(basedir, function(err, files) {
_.each(files, function(file, key) {
if(file.match(/.json$/)) {
var context = require(path.join(basedir, file));
if(context.expires) {
var expires = new Date(context.expires);
if(Date.compare(expires, new Date()) < 0) {
del([context.path, context.jsonPath]);
}
}
}
});
});
};
158 changes: 158 additions & 0 deletions lib/s3storage.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
'use strict';

// ------------------------------------------------------------------------------------------ Dependencies

require('date-utils');
var fs = require("fs");
var _ = require("lodash");
var zlib = require('zlib');
var aws = require('aws-sdk');
var archiver = require('archiver');

// ------------------------------------------------------------------------------------------ Module definition

module.exports = function(options) {
return new S3Storage(options);
}

function S3Storage(options) {
options = options || {};
this.options = options;

if(_.isObject(this.options)) {
this.s3obj = new aws.S3({
accessKeyId: this.options.S3AccessKeyId,
secretAccessKey: this.options.S3SecretAccessKey,
region: this.options.S3Region,
sslEnabled: this.options.S3SSLEnabled,
params: {
Bucket: this.options.S3Bucket
}
});
} else {
throw "Invalid options provided";
}
}


S3Storage.prototype.upload = function(file, context, next) {
try {
var s3obj = this.s3obj;
var body = fs.createReadStream(file.path).pipe(zlib.createGzip());
console.log('s3 upload');
s3obj.upload({
Key: file.id,
Body: body,
Metadata: {
json: JSON.stringify(context)
},
}, function(err, data) {
next(err, context);
});
} catch(err) {
console.log(err);
next(err, context);
}
};

S3Storage.prototype.bundle = function(bundle, next) {
try {
var s3obj = this.s3obj;
s3obj.upload({
Key: bundle.id,
Body: JSON.stringify(bundle),
Metadata: {
json: JSON.stringify(bundle)
}
}, function(err, data) {
next(err);
});
} catch(err) {
next(err);
}
};

S3Storage.prototype.archive = function(token, res, next) {
try {
if(token) {
var s3obj = this.s3obj;
s3obj.getObject({
Key: token
}, function(err, data) {
if(err) {
next(err);
} else {
var bundle = JSON.parse(data.Body);
if(bundle) {
res.setHeader('Content-disposition', 'attachment; filename="bundle.zip"');
res.setHeader('Content-type', 'application/octet-stream');
var archive = archiver('zip');

var completed = _.after(bundle.files.length, function() {
archive.pipe(res);
archive.finalize();
});

_.each(bundle.files, function(file, key) {
var gunzip = zlib.createGunzip();
archive.append(s3obj.getObject({
Key: file.id
}).createReadStream().pipe(gunzip), { name: file.name});
completed();
});
} else {
throw 'invalid bundle identifier';
}
}
});
} else {
throw 'invalid token exception'
}
} catch(err) {
next(err);
}
},

S3Storage.prototype.download = function(token, res, next) {
try {
var s3obj = this.s3obj;
s3obj.getObject({
Key: token
}, function(err, data) {
if(err) {
throw err;
} else {
var context = JSON.parse(data.Metadata.json || {});
res.setHeader('Content-disposition', 'attachment; filename="' + context.name + '"');
res.setHeader('Content-type', 'application/octet-stream');
res.setHeader('Content-length', context.size);
zlib.gunzip(data.Body, function(err, unzipped) {
res.send(unzipped);
});
}
});
} catch(err) {
next(err);
}
},

S3Storage.prototype.purge = function() {
try {
var s3obj = this.s3obj;
s3obj.listObjects(function(err, data) {
_.each(data.Contents, function(item, key) {
s3obj.headObject({
Key: item.Key
}, function(err, data) {
var context = JSON.parse(data.Metadata.json);
var expires = new Date(context.expires);
if(Date.compare(expires, new Date()) < 0) {
s3obj.deleteObject({
Key: item.Key
});
};
});
});
});
} catch(err) { }
};
62 changes: 62 additions & 0 deletions lib/settings.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
'use strict';

// ------------------------------------------------------------------------------------------ Dependencies

var fs = require("fs");
var path = require('path');
var _ = require("lodash");
var nconf = require('nconf');
var EventEmitter = require('events').EventEmitter;

// ------------------------------------------------------------------------------------------ Module definition

module.exports = (function() {

var self = new EventEmitter();

self.get = function(next) {
fs.readFile('./settings.json', 'utf8', function(err, content) {
if(err) {
next(err, {});
} else {
try {
var config = nconf.get();
var settings = JSON.parse(content);
var output = _.assign(config, settings);

if(!path.isAbsolute(output.localstoragepath)) {
output.localstoragepath = path.resolve(nconf.get('basedir'), output.localstoragepath);
}

next(null, output);
} catch(err) {
next(err, {});
}
}
});
}

self.push = function(settings, next) {
var current = {};
settings = _.pick(settings, _.identity);

fs.readFile('./settings.json', 'utf8', function(err, content) {
if(err) {
next(err);
} else {
try {
var current = JSON.parse(content);
var output = _.assign(current, settings);
fs.writeFile('./settings.json', JSON.stringify(output), function(err) {
self.emit('settings.push', err, output);
next(err);
});
} catch(err) {
next(err);
}
};
});
}

return self;
})();
Loading

0 comments on commit 976a326

Please sign in to comment.