Add the amazon-s3-sync-down.js script

README.md

@@ -52,4 +52,37 @@ Usage:
           --object OBJECT \
           --filename FILENAME
 
+### amazon-s3-sync-down.js && amazon-s3-sync-up.js ###
+
+Both of these files takes a bucket name to sync to and tries to sync the current directory (and it's descendents) to
+the bucket. The down script downloads any files not locally present. The up script uploads files not in the
+bucket. Neither script tries to resolve conflicts in filesize or file contents and will instead just want you about
+them.
+
+    $ ./bin/amazon-s3-sync-down.js -b BUCKET_NAME
+    $ ./bin/amazon-s3-sync-up.js -b BUCKET_NAME
+
+# Author #
+
+Written by [Andrew Chilton](http://chilts.org/) - [Blog](http://chilts.org/blog/) - [Twitter](https://twitter.com/andychilton).
+
+# License #
+
+The MIT License : http://opensource.org/licenses/MIT
+
+Copyright (c) 2011-2012 AppsAttic Ltd. http://appsattic.com/
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 (Ends)

bin/amazon-s3-sync-up.js

@@ -0,0 +1,173 @@
+#!/usr/bin/env node
+// --------------------------------------------------------------------------------------------------------------------
+
+var fs = require('fs');
+var crypto = require('crypto');
+
+var fmt = require('fmt');
+var async = require('async');
+var tmp = require('tmp');
+var mkdirp = require('mkdirp');
+var walk = require('walkdir');
+
+var awssum    = require('awssum');
+var amazon    = awssum.load('amazon/amazon');
+var S3        = awssum.load('amazon/s3').S3;
+var common    = require('./lib/amazon-s3-common.js');
+
+var accessKeyId     = process.env.ACCESS_KEY_ID;
+var secretAccessKey = process.env.SECRET_ACCESS_KEY;
+var awsAccountId    = process.env.AWS_ACCOUNT_ID;
+
+// --------------------------------------------------------------------------------------------------------------------
+
+var argv = require('optimist')
+    .usage('Usage: $0 --bucket name [--concurrency n]')
+
+    .demand('b')
+    .alias('b', 'bucket')
+    .describe('b', 'bucket to sync with')
+
+    .alias('c', 'concurrency')
+    .default('c', 3)
+    .describe('c', 'concurrency level for uploads/downloads')
+
+    .describe('debug', 'to turn debugging on')
+    .argv;
+
+// --------------------------------------------------------------------------------------------------------------------
+
+var s3 = new S3({
+    accessKeyId     : accessKeyId,
+    secretAccessKey : secretAccessKey,
+    awsAccountId    : awsAccountId,
+    region          : amazon.US_EAST_1
+});
+
+fmt.sep();
+fmt.title('amazon-s3-sync-up.js');
+fmt.field('Bucket', argv.bucket);
+fmt.field('Concurrency', argv.concurrency);
+fmt.line();
+
+// set up some queues which we can put things onto
+var s3Objects = {};
+var checkFileIsInS3Queue = async.queue(checkFileIsInS3, argv.concurrency);
+var uploadItemQueue      = async.queue(uploadItem,      argv.concurrency);
+var checkMd5IsSameQueue  = async.queue(checkMd5IsSame,  argv.concurrency);
+
+// firstly, list all objects in the bucket
+common.listObjectsAll(s3, argv.bucket, function(err, objects) {
+    if (err) {
+        fmt.field('Error', err);
+        return;
+    }
+
+    // save these objects into the s3Objects global
+    objects.forEach(function (object, i) {
+        s3Objects[object.Key] = object;
+    });
+
+    // console.log(s3Objects);
+
+    // get all the files in this directory down
+    var emitter = walk('./');
+    emitter.on('file', function(filename, stat) {
+        var relativeFile = filename.substr(process.cwd().length + 1);
+
+        // ignore backup files
+        if ( relativeFile.match(/~$/) ) {
+            return;
+        }
+
+        // ignore .git files
+        if ( relativeFile.match(/^\.git\//) ) {
+            return;
+        }
+
+        // push to the next queue
+        checkFileIsInS3Queue.push({
+            'filename' : relativeFile,
+            'size'     : stat.size,
+        });
+    });
+    emitter.on('end', function() {
+        // console.log('Entire directory has been walked.');
+    });
+});
+
+// --------------------------------------------------------------------------------------------------------------------
+
+function checkFileIsInS3(item, callback) {
+    // firstly, see if the item is in S3
+    if ( !s3Objects[item.filename] ) {
+        // not in S3, so add to the queue to upload
+        uploadItemQueue.push(item);
+        callback();
+        return;
+    }
+
+    // item _is_ in S3, check the lengths are the same
+    if ( s3Objects[item.filename].Size !== item.size ) {
+        fmt.field('SizeMismatch', item.filename + ' (file=' + item.size + ', object=' + s3Objects[item.filename].Size + ')');
+        callback();
+        return;
+    }
+
+    // filename and S3 object are the same size, check the MD5 of the file
+    checkMd5IsSameQueue.push(item);
+    callback();
+}
+
+function checkMd5IsSame(item, callback) {
+    // get the MD5 of this file (we know it exists)
+    fs.readFile(item.filename, function(err, data) {
+        // get the MD5 of this file
+        var md5 = crypto.createHash('md5');
+        md5.update(data);
+        var md5hex = md5.digest('hex');
+
+        // check if the calculated MD5 is the same as the ETag in the S3 item
+        if ( md5hex !== s3Objects[item.filename].ETag ) {
+            // different, just tell the user they are different
+            fmt.field('MD5Mismatch', item.filename + ' (file=' + md5hex + ', object=' + s3Objects[item.filename].ETag + ')');
+        }
+        callback();
+    });
+}
+
+function uploadItem(item, callback) {
+    // create a read stream
+    var bodyStream = fs.createReadStream( item.filename );
+
+    var options = {
+        BucketName    : argv.bucket,
+        ObjectName    : item.filename,
+        ContentLength : item.size,
+        Body          : bodyStream
+    };
+
+    s3.PutObject(options, function(err, data) {
+        if (err) {
+            fmt.field('UploadFailed', item.filename);
+
+            // put this item back on the queue if retries is less than the cut-off
+            if ( item.retries > 2 ) {
+                fmt.field('UploadCancelled', item.filename);
+            }
+            else {
+                // try again
+                item.retries = item.retries ? item.retries+1 : 1;
+                uploadItemQueue.push(item);
+            }
+
+            callback();
+            return;
+        }
+
+        fmt.field('Uploaded', item.filename);
+        callback();
+    });
+}
+
+// --------------------------------------------------------------------------------------------------------------------

bin/lib/amazon-s3-common.js

@@ -0,0 +1,77 @@
+// --------------------------------------------------------------------------------------------------------------------
+//
+// amazon-s3-common.js - common functions for Amazon S3
+//
+// Copyright (c) 2012 AppsAttic Ltd - http://www.appsattic.com/
+//
+// Written by - Andrew Chilton
+// * Email    - andychilton@gmail.com
+// * Site     - http://chilts.org/
+// * Blog     - http://chilts.org/blog/
+// * Twitter  - https://twitter.com/andychilton
+//
+// License: http://opensource.org/licenses/MIT
+//
+// --------------------------------------------------------------------------------------------------------------------
+
+function listObjectsAll(s3, bucketname, callback) {
+    var items = [];
+
+    function doRequest(marker) {
+        var options = {
+            BucketName : bucketname,
+        };
+        if ( marker ) {
+            options.Marker = marker;
+        }
+
+        s3.ListObjects(options, function(err, data) {
+            if (err) {
+                callback(err, null);
+                return;
+            }
+
+            // loop through all the items and add them on to our saved list
+            data.Body.ListBucketResult.Contents.forEach(function(v, i) {
+                // remove the leading '/'
+                // v.Key = v.Key.substr(1);
+
+                // remove the extra "" around the MD5
+                v.ETag = v.ETag.substr(1, 32);
+
+                // convert the size to a number
+                v.Size = parseInt(v.Size, 10);
+
+                // now push onto the array
+                items.push(v);
+            });
+
+            // if the result is truncated, fetch some more
+            if (data.Body.ListBucketResult.IsTruncated === 'true') {
+                doRequest(data.Body.ListBucketResult.Contents[data.Body.ListBucketResult.Contents.length-1].Key);
+            }
+            else {
+                // all finished
+                callback(null, items);
+            }
+        });
+
+    }
+
+    // start the recursion off
+    doRequest();
+}
+
+// --------------------------------------------------------------------------------------------------------------------
+
+exports.listObjectsAllMock = function(a, b, c) {
+    var objects = [
+        { Key : 'ToDo', Size : 142, ETag : '27b1173716f04e46b9b3ee771547c558' },
+        { Key : 'README.md', Size : 1402, ETag : '145c4bb641693c5f8db8933eb4d912ba' },
+        { Key : 'package.json', Size : 67, ETag : 'doesnt matter, size is different' },
+    ];
+    c(null, objects)
+};
+exports.listObjectsAll = listObjectsAll;
+
+// --------------------------------------------------------------------------------------------------------------------

package.json

@@ -18,7 +18,8 @@
     "async": "~0.1.18",
     "fmt": "~0.1.0",
     "tmp": "0.0.12",
-    "mkdirp": "~0.3.2"
+    "mkdirp": "~0.3.2",
+    "walkdir": "~0.0.2"
   },
   "engines": {
     "node": ">= 0.6.0"