Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

allow bucket to be modified on a job by job basis #56

Merged
merged 3 commits into from

1 participant

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Jul 23, 2014
  1. added utils file.

    authored
Commits on Jul 26, 2014
This page is out of date. Refresh to see the latest.
View
7 README.md
@@ -108,6 +108,8 @@ client.upload('/tmp/awesome.jpg', destination, function(err) {
* **opts:** additional thumbnailing options.
* **notify:** webhook to notify when thumbnailing is complete.
* **prefix:** prefix for thumbnails created (defaults to original filename).
+ * **bucket:** bucket to download image from (defaults to server's default bucket).
+ * **region:** aws-region to download image from (defaults to server's default region).
Thumbnail Descriptions
----------------------
@@ -168,6 +170,11 @@ of S3 resources. A great example of this would be converting a set of images int
}
```
+* **Creating Video Thumbnails on Heroku**
+
+* install the [ffmpeg](https://github.com/shunjikonishi/heroku-buildpack-ffmpeg) custom buildpack.
+* use a custom strategy that utilizes `ffmpeg` for thumbnail generation, rather than `convert`.
+
The custom strategy can be used for a variety of purposes, _experiment with it :tm:_
Production Notes
View
41 bin/thumbd.js
@@ -3,7 +3,6 @@
var thumbd = require('../lib'),
_ = require('underscore'),
fs = require('fs'),
- knox = require('knox'),
argv = require('optimist').argv,
mode = argv._.shift(),
config = require('../lib/config').Config,
@@ -24,7 +23,8 @@ var thumbd = require('../lib'),
aws_region: 'awsRegion',
descriptions: 'descriptions',
remote_image: 'remoteImage',
- sqs_queue: 'sqsQueue'
+ sqs_queue: 'sqsQueue',
+ bucket: 's3Bucket'
};
/**
@@ -55,22 +55,8 @@ switch (mode) {
var opts = buildOpts(serverOpts);
config.extend(opts);
- var knoxOpts = {
- key: config.get('awsKey'),
- secret: config.get('awsSecret'),
- bucket: config.get('s3Bucket'),
- region: config.get('awsRegion')
- }
-
- // Knox wants 'us-standard' instead of 'us-east-1'
- if (config.get('awsRegion') == 'us-east-1') {
- knoxOpts.region = 'us-standard';
- }
-
- var s3 = knox.createClient(knoxOpts);
-
- var grabber = new thumbd.Grabber(s3);
- var saver = new thumbd.Saver(s3);
+ var grabber = new thumbd.Grabber();
+ var saver = new thumbd.Saver();
var thumbnailer = new thumbd.Thumbnailer();
(new thumbd.Worker({
@@ -82,15 +68,20 @@ switch (mode) {
case 'thumbnail':
- var opts = buildOpts(thumbnailOpts);
+ var opts = buildOpts(thumbnailOpts),
+ client = new thumbd.Client(),
+ extraOpts = {};
+
+ // allow region/bucket to vary on a job by job basis.
+ if (argv.bucket) extraOpts.bucket = argv.bucket;
+ if (argv.region) extraOpts.region = argv.region;
+
config.extend(opts);
- // create a client for submitting
- // thumbnailing jobs.
- var client = new thumbd.Client();
client.thumbnail(
opts.remoteImage,
JSON.parse(fs.readFileSync(opts.descriptions).toString()),
+ extraOpts,
function(err, res) {
if (err) {
console.log(err);
@@ -103,8 +94,8 @@ switch (mode) {
default:
console.log(
"Usage: thumbd <command>\n\n",
- "where <command> is one of:\n",
- "\tthumbd server --aws_key=<key> --aws_secret=<secret> --tmp_dir=</tmp> --sqs_queue=<sqs queue name> --bucket=<s3 thumbnail bucket> --s3_acl=<private or public-read> --s3_storage_class=<STANDARD or REDUCED_REDUNDANCY>\n",
- "\tthumbd thumbnail --remote_image=<path to image s3 or http> --descriptions=<path to thumbnail description JSON file> --aws_key=<key> --aws_secret=<secret> --sqs_queue=<sqs queue name>\n"
+ "where <command> is one of:\n\n",
+ "\tthumbd server --aws_key=<key> --aws_secret=<secret> --tmp_dir=</tmp> --sqs_queue=<sqs queue name> --bucket=<default s3 bucket> --s3_acl=<private or public-read> --s3_storage_class=<STANDARD or REDUCED_REDUNDANCY>\n\n",
+ "\tthumbd thumbnail --remote_image=<path to image s3 or http> --descriptions=<path to thumbnail description JSON file> --aws_key=<key> --aws_secret=<secret> --sqs_queue=<sqs queue name> --bucket=<s3 bucket> --region=<s3 region>\n"
);
}
View
27 lib/grabber.js
@@ -1,5 +1,5 @@
-var knox = require('knox'),
- tmp = require('tmp'),
+var tmp = require('tmp'),
+ utils = require('./utils'),
fs = require('fs'),
http = require('http'),
https = require('https'),
@@ -10,19 +10,7 @@ var knox = require('knox'),
*
* @param object s3 The S3 client
*/
-function Grabber(s3) {
- if (s3) {
- this.s3 = s3;
- return;
- }
-
- this.s3 = knox.createClient({
- key: config.get('awsKey'),
- secret: config.get('awsSecret'),
- bucket: config.get('s3Bucket'),
- region: config.get('awsRegion')
- });
-}
+function Grabber() {}
/**
* Download an image from S3 or over http(s)
@@ -30,7 +18,7 @@ function Grabber(s3) {
* @param string remoteImagePath The image url / s3 path
* @param function} callback The callback function
*/
-Grabber.prototype.download = function(remoteImagePath, callback) {
+Grabber.prototype.download = function(bucket, region, remoteImagePath, callback) {
var _this = this,
extension = remoteImagePath.split('.').pop();
@@ -49,7 +37,7 @@ Grabber.prototype.download = function(remoteImagePath, callback) {
if (remoteImagePath.match(/https?:\/\//)) { // we are thumbnailing a remote image.
_this.getFileHTTP(remoteImagePath, localImagePath, stream, callback);
} else { // we are thumbnailing an Object in our thumbnail S3 bucket.
- _this.getFileS3(remoteImagePath, localImagePath, stream, callback);
+ _this.getFileS3(bucket, region, remoteImagePath, localImagePath, stream, callback);
}
});
@@ -101,9 +89,9 @@ Grabber.prototype.getFileHTTP = function(remoteImagePath, localImagePath, stream
* @param WriteStream stream The stream object for the local file
* @param function callback The callback function
*/
-Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream, callback) {
+Grabber.prototype.getFileS3 = function(bucket, region, remoteImagePath, localImagePath, stream, callback) {
var _this = this,
- req = this.s3.getFile(remoteImagePath, function(err, res) {
+ req = utils.s3(bucket, region).getFile(remoteImagePath, function(err, res) {
// no response should count as an error.
if (!res) res = {statusCode: 503}
@@ -137,7 +125,6 @@ Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream,
stream.end();
callback(err);
});
-
};
exports.Grabber = Grabber;
View
20 lib/saver.js
@@ -1,4 +1,4 @@
-var knox = require('knox'),
+var utils = require('./utils'),
url = require('url'),
config = require('./config').Config;
@@ -7,19 +7,7 @@ var knox = require('knox'),
*
* @param object s3 The S3 client
*/
-function Saver(s3) {
- if (s3) {
- this.s3 = s3;
- return;
- }
-
- this.s3 = knox.createClient({
- key: config.get('awsKey'),
- secret: config.get('awsSecret'),
- bucket: config.get('s3Bucket'),
- region: config.get('awsRegion')
- });
-}
+function Saver() {}
/**
* Save the (local or remote file) to disk
@@ -28,7 +16,7 @@ function Saver(s3) {
* @param string destination The local file path
* @param function callback The callback function. Optional
*/
-Saver.prototype.save = function(source, destination, callback) {
+Saver.prototype.save = function(bucket, region, source, destination, callback) {
if (typeof callback === 'undefined') {
callback = function(){};
}
@@ -42,7 +30,7 @@ Saver.prototype.save = function(source, destination, callback) {
destination = this.destinationFromURL(destination);
}
- this.s3.putFile(source, destination, headers, function(err, res) {
+ utils.s3(bucket, region).putFile(source, destination, headers, function(err, res) {
if (err) return callback(err);
res.on('error', function(err) {
View
19 lib/utils.js
@@ -0,0 +1,19 @@
+var knox = require('knox'),
+ config = require('./config').Config;
+
+/**
+* Create an S3 client, for a specific bucket.
+*
+* @param string bucket S3 bucket to connect to.
+*/
+exports.s3 = function(bucket, region) {
+ // Knox wants 'us-standard' instead of 'us-east-1'.
+ if (region == 'us-east-1') region = 'us-standard';
+
+ return knox.createClient({
+ key: config.get('awsKey'),
+ secret: config.get('awsSecret'),
+ bucket: bucket,
+ region: region
+ });
+};
View
20 lib/worker.js
@@ -111,7 +111,7 @@ Worker.prototype._runJob = function(handle, job, callback) {
async.waterfall([
function(done) {
async.mapLimit(job.resources, 5, function(resource, done) {
- _this._downloadFromS3(resource, done);
+ _this._downloadFromS3(job.bucket, job.region, resource, done);
}, done);
},
function(localPaths, done) {
@@ -142,8 +142,12 @@ Worker.prototype._runJob = function(handle, job, callback) {
* @param string remoteImagePath The s3 path to the image
* @param function callback The callback function
*/
-Worker.prototype._downloadFromS3 = function(remoteImagePath, callback) {
- this.grabber.download(remoteImagePath, function(err, localPath) {
+Worker.prototype._downloadFromS3 = function(bucket, region, remoteImagePath, callback) {
+ // allow a default bucket to be overridden.
+ var bucket = bucket || config.get('s3Bucket'),
+ region = region || config.get('awsRegion');
+
+ this.grabber.download(bucket, region, remoteImagePath, function(err, localPath) {
// Leave the job in the queue if an error occurs.
if (err) {
callback(err);
@@ -164,7 +168,9 @@ Worker.prototype._downloadFromS3 = function(remoteImagePath, callback) {
Worker.prototype._createThumbnails = function(localPaths, job, callback) {
var _this = this,
- work = [];
+ work = [],
+ bucket = job.bucket || config.get('s3Bucket'),
+ region = job.region || config.get('awsRegion');
// Create thumbnailing work for each thumbnail description.
job.descriptions.forEach(function(description) {
@@ -179,7 +185,7 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) {
console.log(err);
done();
} else {
- _this._saveThumbnailToS3(convertedImagePath, remoteImagePath, function(err) {
+ _this._saveThumbnailToS3(bucket, region, convertedImagePath, remoteImagePath, function(err) {
if (err) console.log(err);
done(null, remoteImagePath);
});
@@ -201,8 +207,8 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) {
* @param string remoteImagePath The S3 path for the image
* @param function callback The callback function
*/
-Worker.prototype._saveThumbnailToS3 = function(convertedImagePath, remoteImagePath, callback) {
- this.saver.save(convertedImagePath, remoteImagePath, function(err) {
+Worker.prototype._saveThumbnailToS3 = function(bucket, region, convertedImagePath, remoteImagePath, callback) {
+ this.saver.save(bucket, region, convertedImagePath, remoteImagePath, function(err) {
fs.unlink(convertedImagePath, function() {
callback(err);
});
Something went wrong with that request. Please try again.