diff --git a/README.md b/README.md index 4a8680c..2b12fc2 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,8 @@ client.upload('/tmp/awesome.jpg', destination, function(err) { * **opts:** additional thumbnailing options. * **notify:** webhook to notify when thumbnailing is complete. * **prefix:** prefix for thumbnails created (defaults to original filename). + * **bucket:** bucket to download image from (defaults to server's default bucket). + * **region:** aws-region to download image from (defaults to server's default region). Thumbnail Descriptions ---------------------- @@ -168,6 +170,11 @@ of S3 resources. A great example of this would be converting a set of images int } ``` +* **Creating Video Thumbnails on Heroku** + +* install the [ffmpeg](https://github.com/shunjikonishi/heroku-buildpack-ffmpeg) custom buildpack. +* use a custom strategy that utilizes `ffmpeg` for thumbnail generation, rather than `convert`. + The custom strategy can be used for a variety of purposes, _experiment with it :tm:_ Production Notes diff --git a/bin/thumbd.js b/bin/thumbd.js index ceab8bc..5c64501 100755 --- a/bin/thumbd.js +++ b/bin/thumbd.js @@ -3,7 +3,6 @@ var thumbd = require('../lib'), _ = require('underscore'), fs = require('fs'), - knox = require('knox'), argv = require('optimist').argv, mode = argv._.shift(), config = require('../lib/config').Config, @@ -24,7 +23,8 @@ var thumbd = require('../lib'), aws_region: 'awsRegion', descriptions: 'descriptions', remote_image: 'remoteImage', - sqs_queue: 'sqsQueue' + sqs_queue: 'sqsQueue', + bucket: 's3Bucket' }; /** @@ -55,22 +55,8 @@ switch (mode) { var opts = buildOpts(serverOpts); config.extend(opts); - var knoxOpts = { - key: config.get('awsKey'), - secret: config.get('awsSecret'), - bucket: config.get('s3Bucket'), - region: config.get('awsRegion') - } - - // Knox wants 'us-standard' instead of 'us-east-1' - if (config.get('awsRegion') == 'us-east-1') { - knoxOpts.region = 'us-standard'; - } - - var s3 = knox.createClient(knoxOpts); - - var grabber = new thumbd.Grabber(s3); - var saver = new thumbd.Saver(s3); + var grabber = new thumbd.Grabber(); + var saver = new thumbd.Saver(); var thumbnailer = new thumbd.Thumbnailer(); (new thumbd.Worker({ @@ -82,15 +68,20 @@ switch (mode) { case 'thumbnail': - var opts = buildOpts(thumbnailOpts); + var opts = buildOpts(thumbnailOpts), + client = new thumbd.Client(), + extraOpts = {}; + + // allow region/bucket to vary on a job by job basis. + if (argv.bucket) extraOpts.bucket = argv.bucket; + if (argv.region) extraOpts.region = argv.region; + config.extend(opts); - // create a client for submitting - // thumbnailing jobs. - var client = new thumbd.Client(); client.thumbnail( opts.remoteImage, JSON.parse(fs.readFileSync(opts.descriptions).toString()), + extraOpts, function(err, res) { if (err) { console.log(err); @@ -103,8 +94,8 @@ switch (mode) { default: console.log( "Usage: thumbd \n\n", - "where is one of:\n", - "\tthumbd server --aws_key= --aws_secret= --tmp_dir= --sqs_queue= --bucket= --s3_acl= --s3_storage_class=\n", - "\tthumbd thumbnail --remote_image= --descriptions= --aws_key= --aws_secret= --sqs_queue=\n" + "where is one of:\n\n", + "\tthumbd server --aws_key= --aws_secret= --tmp_dir= --sqs_queue= --bucket= --s3_acl= --s3_storage_class=\n\n", + "\tthumbd thumbnail --remote_image= --descriptions= --aws_key= --aws_secret= --sqs_queue= --bucket= --region=\n" ); } diff --git a/lib/grabber.js b/lib/grabber.js index 4cd09a7..ea5c7a2 100644 --- a/lib/grabber.js +++ b/lib/grabber.js @@ -1,5 +1,5 @@ -var knox = require('knox'), - tmp = require('tmp'), +var tmp = require('tmp'), + utils = require('./utils'), fs = require('fs'), http = require('http'), https = require('https'), @@ -10,19 +10,7 @@ var knox = require('knox'), * * @param object s3 The S3 client */ -function Grabber(s3) { - if (s3) { - this.s3 = s3; - return; - } - - this.s3 = knox.createClient({ - key: config.get('awsKey'), - secret: config.get('awsSecret'), - bucket: config.get('s3Bucket'), - region: config.get('awsRegion') - }); -} +function Grabber() {} /** * Download an image from S3 or over http(s) @@ -30,7 +18,7 @@ function Grabber(s3) { * @param string remoteImagePath The image url / s3 path * @param function} callback The callback function */ -Grabber.prototype.download = function(remoteImagePath, callback) { +Grabber.prototype.download = function(bucket, region, remoteImagePath, callback) { var _this = this, extension = remoteImagePath.split('.').pop(); @@ -49,7 +37,7 @@ Grabber.prototype.download = function(remoteImagePath, callback) { if (remoteImagePath.match(/https?:\/\//)) { // we are thumbnailing a remote image. _this.getFileHTTP(remoteImagePath, localImagePath, stream, callback); } else { // we are thumbnailing an Object in our thumbnail S3 bucket. - _this.getFileS3(remoteImagePath, localImagePath, stream, callback); + _this.getFileS3(bucket, region, remoteImagePath, localImagePath, stream, callback); } }); @@ -101,9 +89,9 @@ Grabber.prototype.getFileHTTP = function(remoteImagePath, localImagePath, stream * @param WriteStream stream The stream object for the local file * @param function callback The callback function */ -Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream, callback) { +Grabber.prototype.getFileS3 = function(bucket, region, remoteImagePath, localImagePath, stream, callback) { var _this = this, - req = this.s3.getFile(remoteImagePath, function(err, res) { + req = utils.s3(bucket, region).getFile(remoteImagePath, function(err, res) { // no response should count as an error. if (!res) res = {statusCode: 503} @@ -137,7 +125,6 @@ Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream, stream.end(); callback(err); }); - }; exports.Grabber = Grabber; diff --git a/lib/saver.js b/lib/saver.js index 6f79a5c..a88bf5a 100644 --- a/lib/saver.js +++ b/lib/saver.js @@ -1,4 +1,4 @@ -var knox = require('knox'), +var utils = require('./utils'), url = require('url'), config = require('./config').Config; @@ -7,19 +7,7 @@ var knox = require('knox'), * * @param object s3 The S3 client */ -function Saver(s3) { - if (s3) { - this.s3 = s3; - return; - } - - this.s3 = knox.createClient({ - key: config.get('awsKey'), - secret: config.get('awsSecret'), - bucket: config.get('s3Bucket'), - region: config.get('awsRegion') - }); -} +function Saver() {} /** * Save the (local or remote file) to disk @@ -28,7 +16,7 @@ function Saver(s3) { * @param string destination The local file path * @param function callback The callback function. Optional */ -Saver.prototype.save = function(source, destination, callback) { +Saver.prototype.save = function(bucket, region, source, destination, callback) { if (typeof callback === 'undefined') { callback = function(){}; } @@ -42,7 +30,7 @@ Saver.prototype.save = function(source, destination, callback) { destination = this.destinationFromURL(destination); } - this.s3.putFile(source, destination, headers, function(err, res) { + utils.s3(bucket, region).putFile(source, destination, headers, function(err, res) { if (err) return callback(err); res.on('error', function(err) { diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000..0b12b4d --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,19 @@ +var knox = require('knox'), + config = require('./config').Config; + +/** +* Create an S3 client, for a specific bucket. +* +* @param string bucket S3 bucket to connect to. +*/ +exports.s3 = function(bucket, region) { + // Knox wants 'us-standard' instead of 'us-east-1'. + if (region == 'us-east-1') region = 'us-standard'; + + return knox.createClient({ + key: config.get('awsKey'), + secret: config.get('awsSecret'), + bucket: bucket, + region: region + }); +}; diff --git a/lib/worker.js b/lib/worker.js index 1cca3f0..4980209 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -111,7 +111,7 @@ Worker.prototype._runJob = function(handle, job, callback) { async.waterfall([ function(done) { async.mapLimit(job.resources, 5, function(resource, done) { - _this._downloadFromS3(resource, done); + _this._downloadFromS3(job.bucket, job.region, resource, done); }, done); }, function(localPaths, done) { @@ -142,8 +142,12 @@ Worker.prototype._runJob = function(handle, job, callback) { * @param string remoteImagePath The s3 path to the image * @param function callback The callback function */ -Worker.prototype._downloadFromS3 = function(remoteImagePath, callback) { - this.grabber.download(remoteImagePath, function(err, localPath) { +Worker.prototype._downloadFromS3 = function(bucket, region, remoteImagePath, callback) { + // allow a default bucket to be overridden. + var bucket = bucket || config.get('s3Bucket'), + region = region || config.get('awsRegion'); + + this.grabber.download(bucket, region, remoteImagePath, function(err, localPath) { // Leave the job in the queue if an error occurs. if (err) { callback(err); @@ -164,7 +168,9 @@ Worker.prototype._downloadFromS3 = function(remoteImagePath, callback) { Worker.prototype._createThumbnails = function(localPaths, job, callback) { var _this = this, - work = []; + work = [], + bucket = job.bucket || config.get('s3Bucket'), + region = job.region || config.get('awsRegion'); // Create thumbnailing work for each thumbnail description. job.descriptions.forEach(function(description) { @@ -179,7 +185,7 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) { console.log(err); done(); } else { - _this._saveThumbnailToS3(convertedImagePath, remoteImagePath, function(err) { + _this._saveThumbnailToS3(bucket, region, convertedImagePath, remoteImagePath, function(err) { if (err) console.log(err); done(null, remoteImagePath); }); @@ -201,8 +207,8 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) { * @param string remoteImagePath The S3 path for the image * @param function callback The callback function */ -Worker.prototype._saveThumbnailToS3 = function(convertedImagePath, remoteImagePath, callback) { - this.saver.save(convertedImagePath, remoteImagePath, function(err) { +Worker.prototype._saveThumbnailToS3 = function(bucket, region, convertedImagePath, remoteImagePath, callback) { + this.saver.save(bucket, region, convertedImagePath, remoteImagePath, function(err) { fs.unlink(convertedImagePath, function() { callback(err); });