allow bucket to be modified on a job by job basis

README.md

@@ -108,6 +108,8 @@ client.upload('/tmp/awesome.jpg', destination, function(err) {
 * **opts:** additional thumbnailing options.
 	* **notify:** webhook to notify when thumbnailing is complete.
 	* **prefix:** prefix for thumbnails created (defaults to original filename).
+	* **bucket:** bucket to download image from (defaults to server's default bucket).
+	* **region:** aws-region to download image from (defaults to server's default region).
 
 Thumbnail Descriptions
 ----------------------
@@ -168,6 +170,11 @@ of S3 resources. A great example of this would be converting a set of images int
 }
 ```
 
+* **Creating Video Thumbnails on Heroku**
+
+* install the [ffmpeg](https://github.com/shunjikonishi/heroku-buildpack-ffmpeg) custom buildpack.
+* use a custom strategy that utilizes `ffmpeg` for thumbnail generation, rather than `convert`.
+
 The custom strategy can be used for a variety of purposes, _experiment with it :tm:_
 
 Production Notes

bin/thumbd.js

@@ -3,7 +3,6 @@
 var thumbd = require('../lib'),
 	_ = require('underscore'),
 	fs = require('fs'),
-	knox = require('knox'),
 	argv = require('optimist').argv,
 	mode = argv._.shift(),
 	config = require('../lib/config').Config,
@@ -24,7 +23,8 @@ var thumbd = require('../lib'),
 		aws_region: 'awsRegion',
 		descriptions: 'descriptions',
 		remote_image: 'remoteImage',
-		sqs_queue: 'sqsQueue'
+		sqs_queue: 'sqsQueue',
+		bucket: 's3Bucket'
 	};
 
 /**
@@ -55,22 +55,8 @@ switch (mode) {
 		var opts = buildOpts(serverOpts);
 		config.extend(opts);
 
-		var knoxOpts = {
-			key: config.get('awsKey'),
-			secret: config.get('awsSecret'),
-			bucket: config.get('s3Bucket'),
-			region: config.get('awsRegion')
-		}
-
-		// Knox wants 'us-standard' instead of 'us-east-1'
-		if (config.get('awsRegion') == 'us-east-1') {
-			knoxOpts.region = 'us-standard';
-		}
-
-		var s3 = knox.createClient(knoxOpts);
-
-		var grabber = new thumbd.Grabber(s3);
-		var saver = new thumbd.Saver(s3);
+		var grabber = new thumbd.Grabber();
+		var saver = new thumbd.Saver();
 		var thumbnailer = new thumbd.Thumbnailer();
 
 		(new thumbd.Worker({
@@ -82,15 +68,20 @@ switch (mode) {
 
 	case 'thumbnail':
 
-		var opts = buildOpts(thumbnailOpts);
+		var opts = buildOpts(thumbnailOpts),
+			client = new thumbd.Client(),
+			extraOpts = {};
+
+		// allow region/bucket to vary on a job by job basis.
+		if (argv.bucket) extraOpts.bucket = argv.bucket;
+		if (argv.region) extraOpts.region = argv.region;
+
 		config.extend(opts);
 
-		// create a client for submitting
-		// thumbnailing jobs.
-		var client = new thumbd.Client();
 		client.thumbnail(
 			opts.remoteImage,
 			JSON.parse(fs.readFileSync(opts.descriptions).toString()),
+			extraOpts,
 			function(err, res) {
 				if (err) {
 					console.log(err);
@@ -103,8 +94,8 @@ switch (mode) {
 	default:
 		console.log(
 			"Usage: thumbd <command>\n\n",
-			"where <command> is one of:\n",
-			"\tthumbd server --aws_key=<key> --aws_secret=<secret> --tmp_dir=</tmp> --sqs_queue=<sqs queue name> --bucket=<s3 thumbnail bucket> --s3_acl=<private or public-read> --s3_storage_class=<STANDARD or REDUCED_REDUNDANCY>\n",
-			"\tthumbd thumbnail --remote_image=<path to image s3 or http> --descriptions=<path to thumbnail description JSON file> --aws_key=<key> --aws_secret=<secret> --sqs_queue=<sqs queue name>\n"
+			"where <command> is one of:\n\n",
+			"\tthumbd server --aws_key=<key> --aws_secret=<secret> --tmp_dir=</tmp> --sqs_queue=<sqs queue name> --bucket=<default s3 bucket> --s3_acl=<private or public-read> --s3_storage_class=<STANDARD or REDUCED_REDUNDANCY>\n\n",
+			"\tthumbd thumbnail --remote_image=<path to image s3 or http> --descriptions=<path to thumbnail description JSON file> --aws_key=<key> --aws_secret=<secret> --sqs_queue=<sqs queue name> --bucket=<s3 bucket> --region=<s3 region>\n"
 		);
 }

lib/grabber.js

@@ -1,5 +1,5 @@
-var knox = require('knox'),
-	tmp = require('tmp'),
+var tmp = require('tmp'),
+	utils = require('./utils'),
 	fs = require('fs'),
 	http = require('http'),
 	https = require('https'),
@@ -10,27 +10,15 @@ var knox = require('knox'),
  *
  * @param object s3 The S3 client
  */
-function Grabber(s3) {
-	if (s3) {
-		this.s3 = s3;
-		return;
-	}
-
-	this.s3 = knox.createClient({
-		key: config.get('awsKey'),
-		secret: config.get('awsSecret'),
-		bucket: config.get('s3Bucket'),
-		region: config.get('awsRegion')
-	});
-}
+function Grabber() {}
 
 /**
  * Download an image from S3 or over http(s)
  *
  * @param string remoteImagePath The image url / s3 path
  * @param function} callback The callback function
  */
-Grabber.prototype.download = function(remoteImagePath, callback) {
+Grabber.prototype.download = function(bucket, region, remoteImagePath, callback) {
 	var _this = this,
 		extension = remoteImagePath.split('.').pop();
 
@@ -49,7 +37,7 @@ Grabber.prototype.download = function(remoteImagePath, callback) {
 		if (remoteImagePath.match(/https?:\/\//)) { // we are thumbnailing a remote image.
 			_this.getFileHTTP(remoteImagePath, localImagePath, stream, callback);
 		} else { // we are thumbnailing an Object in our thumbnail S3 bucket.
-			_this.getFileS3(remoteImagePath, localImagePath, stream, callback);
+			_this.getFileS3(bucket, region, remoteImagePath, localImagePath, stream, callback);
 		}
 
 	});
@@ -101,9 +89,9 @@ Grabber.prototype.getFileHTTP = function(remoteImagePath, localImagePath, stream
  * @param WriteStream stream The stream object for the local file
  * @param function callback The callback function
  */
-Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream, callback) {
+Grabber.prototype.getFileS3 = function(bucket, region, remoteImagePath, localImagePath, stream, callback) {
 	var _this = this,
-		req = this.s3.getFile(remoteImagePath, function(err, res) {
+		req = utils.s3(bucket, region).getFile(remoteImagePath, function(err, res) {
 
 			// no response should count as an error.
 			if (!res) res = {statusCode: 503}
@@ -137,7 +125,6 @@ Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream,
 			stream.end();
 			callback(err);
 		});
-
 };
 
 exports.Grabber = Grabber;

lib/saver.js

@@ -1,4 +1,4 @@
-var knox = require('knox'),
+var utils = require('./utils'),
 	url = require('url'),
 	config = require('./config').Config;
 
@@ -7,19 +7,7 @@ var knox = require('knox'),
  *
  * @param object s3 The S3 client
  */
-function Saver(s3) {
-	if (s3) {
-		this.s3 = s3;
-		return;
-	}
-
-	this.s3 = knox.createClient({
-		key: config.get('awsKey'),
-		secret: config.get('awsSecret'),
-		bucket: config.get('s3Bucket'),
-		region: config.get('awsRegion')
-	});
-}
+function Saver() {}
 
 /**
  * Save the (local or remote file) to disk
@@ -28,7 +16,7 @@ function Saver(s3) {
  * @param string destination The local file path
  * @param function callback The callback function. Optional
  */
-Saver.prototype.save = function(source, destination, callback) {
+Saver.prototype.save = function(bucket, region, source, destination, callback) {
 	if (typeof callback === 'undefined') {
 		callback = function(){};
 	}
@@ -42,7 +30,7 @@ Saver.prototype.save = function(source, destination, callback) {
 		destination = this.destinationFromURL(destination);
 	}
 
-	this.s3.putFile(source, destination, headers, function(err, res) {
+	utils.s3(bucket, region).putFile(source, destination, headers, function(err, res) {
 		if (err) return callback(err);
 
 		res.on('error', function(err) {

lib/utils.js

@@ -0,0 +1,19 @@
+var knox = require('knox'),
+  config = require('./config').Config;
+
+/**
+* Create an S3 client, for a specific bucket.
+*
+* @param string bucket S3 bucket to connect to.
+*/
+exports.s3 = function(bucket, region) {
+  // Knox wants 'us-standard' instead of 'us-east-1'.
+	if (region == 'us-east-1') region = 'us-standard';
+
+  return knox.createClient({
+    key: config.get('awsKey'),
+    secret: config.get('awsSecret'),
+    bucket: bucket,
+    region: region
+  });
+};

lib/worker.js

@@ -111,7 +111,7 @@ Worker.prototype._runJob = function(handle, job, callback) {
 	async.waterfall([
 		function(done) {
 			async.mapLimit(job.resources, 5, function(resource, done) {
-				_this._downloadFromS3(resource, done);
+				_this._downloadFromS3(job.bucket, job.region, resource, done);
 			}, done);
 		},
 		function(localPaths, done) {
@@ -142,8 +142,12 @@ Worker.prototype._runJob = function(handle, job, callback) {
  * @param string remoteImagePath The s3 path to the image
  * @param function callback The callback function
  */
-Worker.prototype._downloadFromS3 = function(remoteImagePath, callback) {
-	this.grabber.download(remoteImagePath, function(err, localPath) {
+Worker.prototype._downloadFromS3 = function(bucket, region, remoteImagePath, callback) {
+	// allow a default bucket to be overridden.
+	var bucket = bucket || config.get('s3Bucket'),
+		region = region || config.get('awsRegion');
+
+	this.grabber.download(bucket, region, remoteImagePath, function(err, localPath) {
 		// Leave the job in the queue if an error occurs.
 		if (err) {
 			callback(err);
@@ -164,7 +168,9 @@ Worker.prototype._downloadFromS3 = function(remoteImagePath, callback) {
 Worker.prototype._createThumbnails = function(localPaths, job, callback) {
 
 	var _this = this,
-		work = [];
+		work = [],
+		bucket = job.bucket || config.get('s3Bucket'),
+		region = job.region || config.get('awsRegion');
 
 	// Create thumbnailing work for each thumbnail description.
 	job.descriptions.forEach(function(description) {
@@ -179,7 +185,7 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) {
 					console.log(err);
 					done();
 				} else {
-					_this._saveThumbnailToS3(convertedImagePath, remoteImagePath, function(err) {
+					_this._saveThumbnailToS3(bucket, region, convertedImagePath, remoteImagePath, function(err) {
 						if (err) console.log(err);
 						done(null, remoteImagePath);
 					});
@@ -201,8 +207,8 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) {
  * @param string remoteImagePath The S3 path for the image
  * @param function callback The callback function
  */
-Worker.prototype._saveThumbnailToS3 = function(convertedImagePath, remoteImagePath, callback) {
-	this.saver.save(convertedImagePath, remoteImagePath, function(err) {
+Worker.prototype._saveThumbnailToS3 = function(bucket, region, convertedImagePath, remoteImagePath, callback) {
+	this.saver.save(bucket, region, convertedImagePath, remoteImagePath, function(err) {
 		fs.unlink(convertedImagePath, function() {
 			callback(err);
 		});