support uploading directories

lib/storage/bucket.js

@@ -23,6 +23,7 @@
 var async = require('async');
 var extend = require('extend');
 var fs = require('fs');
+var globby = require('globby');
 var mime = require('mime-types');
 var path = require('path');
 
@@ -44,6 +45,12 @@ var File = require('./file.js');
  */
 var util = require('../common/util.js');
 
+/**
+ * @const {number}
+ * @private
+ */
+var MAX_PARALLEL_UPLOADS = 5;
+
 /**
  * @const {string}
  * @private
@@ -652,40 +659,44 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
 };
 
 /**
- * Upload a file to the bucket. This is a convenience method that wraps the
- * functionality provided by a File object, {module:storage/file}.
+ * Upload files to your bucket using glob patterns.
  *
- * @param {string} localPath - The fully qualified path to the file you wish to
- *     upload to your bucket.
+ * If the input matches more than a single file, your callback will receive an
+ * array of {module:storage/file} objects.
+ *
+ * @param {string|string[]} pattern - A glob pattern, or array of patterns,
+ *     matching the files you would like uploaded. See
+ *     [sindresorhus/globby](http://goo.gl/42g2v7) for an overview.
  * @param {object=} options - Configuration options.
- * @param {string|module:storage/file} options.destination - The place to save
- *     your file. If given a string, the file will be uploaded to the bucket
- *     using the string as a filename. When given a File object, your local file
- *     will be uploaded to the File object's bucket and under the File object's
- *     name. Lastly, when this argument is omitted, the file is uploaded to your
- *     bucket using the name of the local file.
- * @param {object=} options.metadata - Metadata to set for your file.
- * @param {boolean=} options.resumable - Force a resumable upload. (default:
+ * @param {string} options.basePath - A parent directory to use as the tip of
+ *     the resulting hierarchy in your bucket. File names are determined using
+ *     this value, which defaults to the given `directoryPath`. See the example
+ *     below for more.
+ * @param {boolean} options.force - Suppress errors until all files have been
+ *     processed. (default: false)
+ * @param {object} options.globOptions - Glob options as defined by
+ *     [`node-glob`](http://goo.gl/14UhaI).
+ * @param {boolean} options.resumable - Force a resumable upload. (default:
  *     true for files larger than 5MB). Read more about resumable uploads
  *     [here](http://goo.gl/1JWqCF). NOTE: This behavior is only possible with
  *     this method, and not {module:storage/file#createWriteStream}. When
  *     working with streams, the file format and size is unknown until it's
  *     completely consumed. Because of this, it's best for you to be explicit
  *     for what makes sense given your input.
- * @param {function} callback - The callback function.
  * @param {string|boolean} options.validation - Possible values: `"md5"`,
  *     `"crc32c"`, or `false`. By default, data integrity is validated with an
  *     MD5 checksum for maximum reliability. CRC32c will provide better
  *     performance with less reliability. You may also choose to skip validation
  *     completely, however this is **not recommended**.
+ * @param {function} callback - The callback function.
  *
  * @example
  * //-
  * // The easiest way to upload a file.
  * //-
  * bucket.upload('/local/path/image.png', function(err, file, apiResponse) {
  *   // Your bucket now contains:
- *   // - "image.png" (with the contents of `/local/path/image.png')
+ *   //   - "image.png" (with the contents of `/local/path/image.png')
  *
  *   // `file` is an instance of a File object that refers to your new file.
  * });
@@ -707,14 +718,14 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
  *
  * bucket.upload('local-image.png', options, function(err, file) {
  *   // Your bucket now contains:
- *   // - "new-image.png" (with the contents of `local-image.png')
+ *   //   - "new-image.png" (with the contents of `local-image.png')
  *
  *   // `file` is an instance of a File object that refers to your new file.
  * });
  *
  * //-
- * // You may also re-use a File object, {module:storage/file}, that references
- * // the file you wish to create or overwrite.
+ * // You may also re-use a File object ({module:storage/file}) that references
+ * // the file you wish to write to.
  * //-
  * var options = {
  *   destination: bucket.file('existing-file.png'),
@@ -723,68 +734,255 @@ Bucket.prototype.setMetadata = function(metadata, callback) {
  *
  * bucket.upload('local-img.png', options, function(err, newFile) {
  *   // Your bucket now contains:
- *   // - "existing-file.png" (with the contents of `local-img.png')
+ *   //   - "existing-file.png" (with the contents of `local-img.png')
  *
  *   // Note:
- *   // The `newFile` parameter is equal to `file`.
+ *   // The `newFile` parameter is equal to `options.destination`.
+ * });
+ *
+ * //-
+ * // For the power users, glob patterns are also supported, using
+ * // <a href="https://github.com/sindresorhus/globby">sindresorhus/globby</a>
+ * // under the hood.
+ * //
+ * // NOTE: All of the options globby accepts can be provided with
+ * // `options.globOptions`. For a full list of options, see
+ * // <a href="http://goo.gl/14UhaI">node-glob's options object</a>.
+ * //-
+ * bucket.upload('/Users/stephen/Desktop/*.{jpg|png}', function(errors, files) {
+ *   // `errors` will always be an array.
+ *   // `files` is an array of all successfully uploaded files that the glob
+ *   // pattern matched.
+ * });
+ *
+ * //-
+ * // If you're uploading many files, you may wish to suppress errors until all
+ * // of the files have been processed.
+ * //-
+ * var options = {
+ *   force: true
+ * };
+ *
+ * bucket.upload('/Users/stephen/Photos/*', options, function(errors, files) {
+ *   // `errors` will always be an array.
+ *   // `files` is an array of all successfully uploaded files that the glob
+ *   // pattern matched.
  * });
  */
-Bucket.prototype.upload = function(localPath, options, callback) {
+Bucket.prototype.upload = function(pattern, options, callback) {
+  var self = this;
+
   if (util.is(options, 'function')) {
     callback = options;
     options = {};
   }
 
-  var newFile;
-  if (options.destination instanceof File) {
-    newFile = options.destination;
-  } else if (util.is(options.destination, 'string')) {
-    // Use the string as the name of the file.
-    newFile = this.file(options.destination);
-  } else {
-    // Resort to using the name of the incoming file.
-    newFile = this.file(path.basename(localPath));
+  options = options || {};
+
+  var errors = [];
+  var files = [];
+
+  var globOptions = extend({}, options.globOptions, { nodir: true });
+
+  globby(pattern, globOptions, function(err, filePaths) {
+    if (err) {
+      callback(err);
+      return;
+    }
+
+    var uploadFileFns = filePaths.map(function(filePath) {
+      return function(next) {
+        var fileName;
+
+        if (options.basePath) {
+          fileName = path.relative(options.basePath, filePath);
+        } else {
+          fileName = path.basename(filePath);
+        }
+
+        var opts = extend({}, options, { destination: fileName });
+
+        self.uploadFile(filePath, opts, function(err, file) {
+          if (err) {
+            errors.push(err);
+          } else {
+            files.push(file);
+          }
+
+          next(options.force ? null : err);
+        });
+      };
+    });
+
+    async.parallelLimit(uploadFileFns, MAX_PARALLEL_UPLOADS, function() {
+      callback(errors, files);
+    });
+  });
+};
+
+/**
+ * Upload the contents of a directory to the root of your bucket. The structure
+ * of the given directory will be maintained.
+ *
+ * @param {string} directoryPath - Path to the directory you wish to upload.
+ * @param {object=} options - Configuration object.
+ * @param {string} options.basePath - A parent directory to use as the tip of
+ *     the resulting hierarchy in your bucket. File names are determined using
+ *     this value, which defaults to the given `directoryPath`. See the example
+ *     below for more.
+ * @param {boolean} options.resumable - Force a resumable upload. (default:
+ *     true for files larger than 5MB). Read more about resumable uploads
+ *     [here](http://goo.gl/1JWqCF). NOTE: This behavior is only possible with
+ *     this method, and not {module:storage/file#createWriteStream}. When
+ *     working with streams, the file format and size is unknown until it's
+ *     completely consumed. Because of this, it's best for you to be explicit
+ *     for what makes sense given your input.
+ * @param {string|boolean} options.validation - Possible values: `"md5"`,
+ *     `"crc32c"`, or `false`. By default, data integrity is validated with an
+ *     MD5 checksum for maximum reliability. CRC32c will provide better
+ *     performance with less reliability. You may also choose to skip validation
+ *     completely, however this is **not recommended**.
+ * @param {function} callback - The callback function.
+ *
+ * @example
+ * var zooPhotosPath = '/Users/stephen/Photos/zoo';
+ *
+ * bucket.uploadDirectory(zooPhotosPath, function(err, files) {
+ *   // Your bucket now contains:
+ *   //   - "monkeys/monkey-1.jpg"
+ *   //   - "zebras/zebra-1.jpg"
+ *   //   - "sleeping-panda.jpg"
+ * });
+ *
+ * //-
+ * // You can also specify a `basePath` if you need more control.
+ * //-
+ * var options = {
+ *   basePath: '/Users/stephen/Photos';
+ * };
+ *
+ * bucket.uploadDirectory(zooPhotosPath, options, function(err, files) {
+ *   // Your bucket now contains:
+ *   //   - "zoo/monkeys/monkey-1.jpg"
+ *   //   - "zoo/zebras/zebra-1.jpg"
+ *   //   - "zoo/sleeping-panda.jpg"
+ * });
+ */
+Bucket.prototype.uploadDirectory = function(directoryPath, options, callback) {
+  if (util.is(options, 'function')) {
+    callback = options;
+    options = {};
   }
 
-  var metadata = options.metadata || {};
-  var contentType = mime.contentType(path.basename(localPath));
+  options = options || {};
 
-  if (contentType && !metadata.contentType) {
-    metadata.contentType = contentType;
+  if (!options.basePath) {
+    options.basePath = directoryPath;
   }
 
-  var resumable;
-  if (util.is(options.resumable, 'boolean')) {
-    resumable = options.resumable;
-    upload();
-  } else {
-    // Determine if the upload should be resumable if it's over the threshold.
-    fs.stat(localPath, function(err, fd) {
-      if (err) {
-        callback(err);
-        return;
-      }
+  this.upload(path.join(directoryPath, '**/*'), options, callback);
+};
 
-      resumable = fd.size > RESUMABLE_THRESHOLD;
+/**
+ * Upload a single file to your bucket.
+ *
+ * NOTE: It's often easier to use {module:storage/bucket#upload}, which can also
+ * accept a glob pattern.
+ *
+ * @param {string} filePath - Fully qualified path to a file.
+ * @param {object=} options - Configuration object.
+ * @param {string|module:storage/file} options.destination - The place to save
+ *     your file. If given a string, the file will be uploaded to the bucket
+ *     using the string as a filename. When given a File object, your local file
+ *     will be uploaded to the File object's bucket and under the File object's
+ *     name. Lastly, when this argument is omitted, the file is uploaded to your
+ *     bucket using the name of the local file.
+ * @param {object} options.metadata - Metadata to set for your file.
+ * @param {boolean} options.resumable - Force a resumable upload. (default:
+ *     true for files larger than 5MB). Read more about resumable uploads
+ *     [here](http://goo.gl/1JWqCF). NOTE: This behavior is only possible with
+ *     this method, and not {module:storage/file#createWriteStream}. When
+ *     working with streams, the file format and size is unknown until it's
+ *     completely consumed. Because of this, it's best for you to be explicit
+ *     for what makes sense given your input.
+ * @param {number} options.size - Byte size of the file. This is used to
+ *     determine if a resumable or simple upload technique should be used. If
+ *     not provided, the file will be `stat`ed for its size.
+ * @param {string|boolean} options.validation - Possible values: `"md5"`,
+ *     `"crc32c"`, or `false`. By default, data integrity is validated with an
+ *     MD5 checksum for maximum reliability. CRC32c will provide better
+ *     performance with less reliability. You may also choose to skip validation
+ *     completely, however this is **not recommended**.
+ * @param {function} callback - The callback function.
+ */
+Bucket.prototype.uploadFile = function(filePath, options, callback) {
+  var self = this;
 
-      upload();
-    });
+  if (util.is(options, 'function')) {
+    callback = options;
+    options = {};
   }
 
-  function upload() {
-    fs.createReadStream(localPath)
-      .pipe(newFile.createWriteStream({
-        validation: options.validation,
-        resumable: resumable,
-        metadata: metadata
-      }))
-      .on('error', function(err) {
-        callback(err);
-      })
-      .on('complete', function() {
-        callback(null, newFile);
+  options = options || {};
+
+  if (!util.is(options.resumable, 'boolean')) {
+    // User didn't specify a preference of resumable or simple upload. Check the
+    // file's size to determine which to use.
+    if (!util.is(options.size, 'number')) {
+      fs.stat(filePath, function(err, stats) {
+        if (err) {
+          callback(err);
+          return;
+        }
+
+        options.size = stats.size;
+        self.uploadFile(filePath, options, callback);
       });
+      return;
+    }
+
+    options.resumable = options.size > RESUMABLE_THRESHOLD;
+  }
+
+  if (util.is(options.destination, 'string')) {
+    options.destination = this.file(options.destination);
+  }
+
+  if (!options.destination) {
+    options.destination = this.file(path.basename(filePath));
+  }
+
+  this.uploadFile_(filePath, options, callback);
+};
+
+/**
+ * All of the public methods {module:storage#upload},
+ * {module:storage#uploadDirectory}, and {module:storage#uploadFile} call this
+ * wrapper around {module:storage/file#createWriteStream}.
+ *
+ * Additionally, this method will try to set a contentType and charset.
+ *
+ * @private
+ */
+Bucket.prototype.uploadFile_ = function(filePath, options, callback) {
+  var file = options.destination;
+  var metadata = options.metadata || {};
+  var contentType = mime.contentType(path.basename(filePath));
+
+  if (contentType && !metadata.contentType) {
+    metadata.contentType = contentType;
   }
+
+  fs.createReadStream(filePath)
+    .pipe(file.createWriteStream({
+      validation: options.validation,
+      resumable: options.resumable,
+      metadata: metadata
+    }))
+    .on('error', callback)
+    .on('complete', function() {
+      callback(null, file);
+    });
 };
 
 /**