Skip to content
Browse files

Merge pull request from @Zugwalt - fixes #6

  • Loading branch information...
1 parent a525bd7 commit c7ebb42e42251d3a28de28971ee80b827327c2b7 @nathanoehlman committed Jul 18, 2013
Showing with 60 additions and 12 deletions.
  1. +33 −12 lib/multipartupload.js
  2. +27 −0 test/multipart.js
View
45 lib/multipartupload.js
@@ -22,6 +22,11 @@ function MultiPartUpload(opts, callback) {
if (opts.stream && opts.file) {
throw new Error('You cannot provide both a stream and a file to upload');
}
+
+ if (opts.noDisk && opts.partSize && opts.partSize > 10485760) {
+ throw new Error('Keep in-memory part sizes 10MB or less');
+ }
+
callback = callback || function(err, results) {};
this.objectName = opts.objectName;
@@ -30,6 +35,7 @@ function MultiPartUpload(opts, callback) {
this.partSize = opts.partSize || 5242880; // 5MB default
this.uploadId = null;
this.uploads = new Batch();
+ this.noDisk = opts.noDisk;
this.maxUploadSize = opts.maxUploadSize || 1/0; // infinity default
this.currentUploadSize = 0;
this.aborted = false;
@@ -39,7 +45,7 @@ function MultiPartUpload(opts, callback) {
}
// initialise the tmp directory based on opts (fallback to os.tmpDir())
- this.tmpDir = opts.tmpDir || os.tmpDir();
+ this.tmpDir = !this.noDisk && (opts.tmpDir || os.tmpDir());
if (opts.stream) {
this._putStream(opts.stream, callback);
@@ -127,12 +133,13 @@ MultiPartUpload.prototype._handleStream = function(stream, callback) {
function newPart() {
var partId = parts.length + 1,
partFileName = path.resolve(path.join(mpu.tmpDir, 'mpu-' + this.objectName + '-' + random_seed() + '-' + (mpu.uploadId || Date.now()) + '-' + partId)),
- partFile = fs.createWriteStream(partFileName),
+ partFile = !mpu.noDisk && fs.createWriteStream(partFileName),
part = {
id: partId,
stream: partFile,
fileName: partFileName,
- length: 0
+ length: 0,
+ data: ''
};
parts.push(part);
@@ -143,7 +150,7 @@ MultiPartUpload.prototype._handleStream = function(stream, callback) {
if (!part) return;
// Ensure the stream is closed
- if (part.stream.writable) {
+ if (part.stream && part.stream.writable) {
part.stream.end();
}
mpu.uploads.push(mpu._uploadPart.bind(mpu, part));
@@ -178,7 +185,11 @@ MultiPartUpload.prototype._handleStream = function(stream, callback) {
current = newPart();
}
- current.stream.write(buffer);
+ if (current.stream) {
+ current.stream.write(buffer);
+ } else {
+ current.data += buffer;
+ }
current.length += buffer.length;
// Check if we have a part
@@ -223,12 +234,12 @@ MultiPartUpload.prototype._uploadPart = function(part, callback) {
var url = this.objectName + '?partNumber=' + part.id + '&uploadId=' + this.uploadId,
headers = { 'Content-Length': part.length },
req = this.client.request('PUT', url, headers),
- partStream = fs.createReadStream(part.fileName),
+ partStream = !this.noDisk && fs.createReadStream(part.fileName),
mpu = this;
// Wait for the upload to complete
req.on('response', function(res) {
- if (res.statusCode != 200) return callback({part: part.id, message: 'Upload failed'});
+ if (res.statusCode != 200) return callback({part: part.id, message: 'Upload failed with status code '+res.statusCode});
// Grab the etag and return it
var etag = res.headers.etag,
@@ -237,9 +248,14 @@ MultiPartUpload.prototype._uploadPart = function(part, callback) {
mpu.emit('uploaded', result);
// Remove the temporary file
- fs.unlink(part.fileName, function(err) {
- return callback(err, result);
- });
+ if (!mpu.noDisk) {
+ fs.unlink(part.fileName, function(err) {
+ return callback(err, result);
+ });
+ } else {
+ return callback(null, result);
+ }
+
});
// Handle errors
@@ -249,7 +265,12 @@ MultiPartUpload.prototype._uploadPart = function(part, callback) {
return callback(result);
});
- partStream.pipe(req);
+ if (!this.noDisk) {
+ partStream.pipe(req);
+ } else {
+ req.write(part.data);
+ req.end();
+ }
mpu.emit('uploading', part.id);
};
@@ -264,7 +285,7 @@ MultiPartUpload.prototype._completeUploads = function(callback) {
if (err) return callback(err);
- var size = 0, parts, body;
+ var size = 0, parts;
parts = _.map(results, function(value) {
size += value.size;
return util.format('<Part><PartNumber>%d</PartNumber><ETag>%s</ETag></Part>', value.part, value.etag);
View
27 test/multipart.js
@@ -167,4 +167,31 @@ describe('Knox multipart form uploads', function() {
stream.start();
});
+
+ it('should be able to upload a file using the noDisk option', function(done) {
+
+ var testLength = 7242880,
+ chunkSize = 2048,
+ stream = new mockstream.MockDataStream({chunkSize: chunkSize, streamLength: testLength}),
+ opts = {
+ client: client, objectName: Date.now() + '.txt', stream: stream, noDisk: true
+ },
+ mpu = null;
+
+ // Upload the file
+ mpu = new MultiPartUpload(opts, function(err, body) {
+ if (err) return done(err);
+ assert.equal(body['Key'], opts.objectName);
+
+ // Clean up after ourselves
+ client.deleteFile(opts.objectName, function(err, res) {
+ if (err) return done('Could not delete file [' + err + ']');
+ return done();
+ });
+
+ });
+
+ stream.start();
+
+ });
});

0 comments on commit c7ebb42

Please sign in to comment.
Something went wrong with that request. Please try again.