diff --git a/lib/form_data.js b/lib/form_data.js index 3a1bb82..e65f131 100644 --- a/lib/form_data.js +++ b/lib/form_data.js @@ -31,6 +31,8 @@ function FormData(options) { this._overheadLength = 0; this._valueLength = 0; this._valuesToMeasure = []; + this._chunked = false; + this._calculatedLength = null; CombinedStream.call(this); @@ -79,7 +81,7 @@ FormData.prototype.append = function(field, value, options) { }; FormData.prototype._trackLength = function(header, value, options) { - var valueLength = 0; + var valueLength = 0, hasLength = true; // used w/ getLengthSync(), when length is known. // e.g. for streaming directly from a remote server, @@ -91,6 +93,8 @@ FormData.prototype._trackLength = function(header, value, options) { valueLength = value.length; } else if (typeof value === 'string') { valueLength = Buffer.byteLength(value); + } else { + hasLength = false; } this._valueLength += valueLength; @@ -100,15 +104,24 @@ FormData.prototype._trackLength = function(header, value, options) { Buffer.byteLength(header) + FormData.LINE_BREAK.length; - // empty or either doesn't have path or not an http response - if (!value || ( !value.path && !(value.readable && value.hasOwnProperty('httpVersion')) )) { + if (!value) { + return; + } + + // no need to bother with the length, we already know it + if (hasLength) { return; } - // no need to bother with the length - if (!options.knownLength) { - this._valuesToMeasure.push(value); + // empty or either doesn't have path or not an http response + if (( !value.path && !(value.readable && value.hasOwnProperty('httpVersion')) )) { + this._chunked = true; + this._calculatedLength = NaN; + return; } + + // measure it later + this._valuesToMeasure.push(value); }; FormData.prototype._lengthRetriever = function(value, callback) { @@ -162,7 +175,8 @@ FormData.prototype._lengthRetriever = function(value, callback) { // something else } else { - callback('Unknown stream'); + // As NaN, it will become chunked + callback(null, NaN); } }; @@ -296,6 +310,10 @@ FormData.prototype.getHeaders = function(userHeaders) { 'content-type': 'multipart/form-data; boundary=' + this.getBoundary() }; + if (this._chunked) { + formHeaders['transfer-encoding'] = 'chunked'; + } + for (header in userHeaders) { if (userHeaders.hasOwnProperty(header)) { formHeaders[header.toLowerCase()] = userHeaders[header]; @@ -353,14 +371,31 @@ FormData.prototype.getLengthSync = function() { FormData.prototype.hasKnownLength = function() { var hasKnownLength = true; - if (this._valuesToMeasure.length) { + if (this._valuesToMeasure.length || this._chunked) { hasKnownLength = false; } return hasKnownLength; }; +// Public API to check if the stream will be chunked +FormData.prototype.isChunked = function() { + return this._chunked; +}; + FormData.prototype.getLength = function(cb) { + + // Have we already calculated the length? (or know that it's chunked) + if (this._calculatedLength !== null) { + if (this._chunked) { + // An error (or a NaN value) will be compatible with current libraries' behavior, + // i.e. request.js: https://github.com/request/request/blob/8162961dfdb73dc35a5a4bfeefb858c2ed2ccbb7/request.js#L572 + return cb(new Error('This form-data does not have a length and should be sent with `Transfer-Encoding: chunked`'), NaN); + } else { + return cb(null, this._calculatedLength); + } + } + var knownLength = this._overheadLength + this._valueLength; if (this._streams.length) { @@ -372,7 +407,7 @@ FormData.prototype.getLength = function(cb) { return; } - asynckit.parallel(this._valuesToMeasure, this._lengthRetriever, function(err, values) { + asynckit.parallel(this._valuesToMeasure, this._lengthRetriever, (function(err, values) { if (err) { cb(err); return; @@ -382,8 +417,15 @@ FormData.prototype.getLength = function(cb) { knownLength += length; }); + this._chunked = isNaN(knownLength); + this._calculatedLength = knownLength; + + if (this._chunked) { + return cb(new Error('This form-data does not have a length and should be sent with `Transfer-Encoding: chunked`'), knownLength); + } + cb(null, knownLength); - }); + }).bind(this)); }; FormData.prototype.submit = function(params, cb) { @@ -426,15 +468,18 @@ FormData.prototype.submit = function(params, cb) { // get content length and fire away this.getLength(function(err, length) { - if (err) { + if (isNaN(length)) { + request.setHeader('Transfer-Encoding', 'chunked'); + } else if (err) { this._error(err); return; + } else { + // add content length + request.setHeader('Content-Length', length); } - // add content length - request.setHeader('Content-Length', length); - this.pipe(request); + if (cb) { request.on('error', cb); request.on('response', cb.bind(this, null)); diff --git a/test/common.js b/test/common.js index 7da855c..17a8d2a 100644 --- a/test/common.js +++ b/test/common.js @@ -66,7 +66,7 @@ common.actions.populateFields = function(form, fields) if ((typeof field.value == 'function')) { field.value = field.value(); } - form.append(name, field.value); + form.append(name, field.value, field.options); } }; diff --git a/test/integration/test-custom-headers-object.js b/test/integration/test-custom-headers-object.js index dd40187..b547c50 100644 --- a/test/integration/test-custom-headers-object.js +++ b/test/integration/test-custom-headers-object.js @@ -9,7 +9,12 @@ var http = require('http'); var FormData = require(common.dir.lib + '/form_data'); -var testHeader = { 'X-Test-Fake': 123 }; +var testHeader = { + 'X-Test-Fake': 123, + + // Cover "skip nullish header" code branch + 'X-Empty-Header': undefined +}; var expectedLength; @@ -49,6 +54,13 @@ server.listen(common.port, function() { form.append('my_buffer', buffer, options); + // Cover the "userHeaders.hasOwnProperty(header)" code branch + var headers = form.getHeaders({ + 'X-Custom-Header': 'value' + }); + + assert.strictEqual(headers['x-custom-header'], 'value'); + // (available to req handler) expectedLength = form._lastBoundary().length + form._overheadLength + options.knownLength; diff --git a/test/integration/test-form-get-length.js b/test/integration/test-form-get-length.js index 6b03069..ca23612 100644 --- a/test/integration/test-form-get-length.js +++ b/test/integration/test-form-get-length.js @@ -3,6 +3,7 @@ var assert = common.assert; var FormData = require(common.dir.lib + '/form_data'); var fake = require('fake').create(); var fs = require('fs'); +var Stream = require('stream'); (function testEmptyForm() { var form = new FormData(); @@ -89,3 +90,39 @@ var fs = require('fs'); fake.expectAnytime(callback, [null, expectedLength]); form.getLength(callback); })(); + +(function testPassthroughStreamData() { + + var fields = [ + { + name: 'my_field', + value: 'Test 123' + }, + { + name: 'my_image', + value: fs.createReadStream(common.dir.fixture + '/unicycle.jpg').pipe(new Stream.PassThrough()) + }, + { + name: 'my_buffer', + value: new Buffer('123') + }, + { + name: 'my_txt', + value: fs.createReadStream(common.dir.fixture + '/veggies.txt').pipe(new Stream.PassThrough()) + } + ]; + + var form = new FormData(); + + fields.forEach(function(field) { + form.append(field.name, field.value); + }); + + var callback = fake.callback('testPassthroughStreamData-getLength'); + fake.expectAnytime(callback, []); + form.getLength(function (err, length) { + assert.ok(err, 'getLength should send an error'); + assert.ok(isNaN(length), 'length should be NaN'); + callback(); + }); +})(); diff --git a/test/integration/test-submit-chunked-request.js b/test/integration/test-submit-chunked-request.js new file mode 100644 index 0000000..213a1be --- /dev/null +++ b/test/integration/test-submit-chunked-request.js @@ -0,0 +1,69 @@ +var common = require('../common'); +var assert = common.assert; +var mime = require('mime-types'); +var request = require('request'); +var FormData = require(common.dir.lib + '/form_data'); + +var remoteFile = 'http://localhost:' + common.staticPort + '/unicycle.jpg'; + +// wrap non simple values into function +// just to deal with ReadStream "autostart" +var FIELDS = { + 'remote_file': { + type: mime.lookup(common.dir.fixture + '/unicycle.jpg'), + value: function() { + return request(remoteFile) + .on('response', function(response) { + // Remove content-length header from response, force it to be chunked + delete response.headers['content-length']; + }); + } + } +}; + +// count total +var fieldsPassed = Object.keys(FIELDS).length; + +// prepare form-receiving http server +var server = common.testFields(FIELDS, function(fields){ + fieldsPassed = fields; +}); + +server.listen(common.port, function() { + + var form = new FormData(); + + common.actions.populateFields(form, FIELDS); + + // custom params object passed to submit + form.submit({ + port: common.port, + path: '/' + }, function(err, res) { + + if (err) { + throw err; + } + + // Only now we know that it's a chunked stream, as form.submit() called getLength() + + assert.strictEqual(form.getHeaders()['transfer-encoding'], 'chunked'); + assert.strictEqual(form.isChunked(), true); + + // Verify status code + assert.strictEqual(res.statusCode, 200); + + // Try to get length again - that should take an already cached value and cover the _calculatedLength code branch + form.getLength(function (ex, length) { + assert.ok(isNaN(length)); + }); + + res.resume(); + server.close(); + }); + +}); + +process.on('exit', function() { + assert.strictEqual(fieldsPassed, 0); +}); diff --git a/test/integration/test-submit-chunked.js b/test/integration/test-submit-chunked.js new file mode 100644 index 0000000..94a1346 --- /dev/null +++ b/test/integration/test-submit-chunked.js @@ -0,0 +1,84 @@ +var common = require('../common'); +var assert = common.assert; +var mime = require('mime-types'); +var fs = require('fs'); +var Stream = require('stream'); +var FormData = require(common.dir.lib + '/form_data'); + +// wrap non simple values into function +// just to deal with ReadStream "autostart" +var FIELDS = { + 'my_field': { + value: 'my_value' + }, + 'my_empty_field': { + value: '' // Will cover the (!value) code branch + }, + 'my_buffer': { + type: FormData.DEFAULT_CONTENT_TYPE, + value: common.defaultTypeValue + }, + 'my_file': { + type: mime.lookup(common.dir.fixture + '/unicycle.jpg'), + value: function() { return fs.createReadStream(common.dir.fixture + '/unicycle.jpg').pipe(new Stream.PassThrough()); }, + options: { + filename: 'unicycle.jpg', + } + } +}; + +var TEST_FIELDS = { + 'my_field': { + value: 'my_value' + }, + 'my_empty_field': { + value: '' + }, + 'my_buffer': { + type: FormData.DEFAULT_CONTENT_TYPE, + value: common.defaultTypeValue() + }, + 'my_file': { + type: mime.lookup(common.dir.fixture + '/unicycle.jpg'), + value: fs.createReadStream(common.dir.fixture + '/unicycle.jpg') + } +}; + +// count total +var fieldsPassed = Object.keys(FIELDS).length; + +// prepare form-receiving http server +var server = common.testFields(TEST_FIELDS, function(fields){ + fieldsPassed = fields; +}); + +server.listen(common.port, function() { + + var form = new FormData(); + + common.actions.populateFields(form, FIELDS); + + assert.strictEqual(form.getHeaders()['transfer-encoding'], 'chunked'); + assert.strictEqual(form.isChunked(), true); + + // custom params object passed to submit + form.submit({ + port: common.port, + path: '/' + }, function(err, res) { + + if (err) { + throw err; + } + + assert.strictEqual(res.statusCode, 200); + + res.resume(); + server.close(); + }); + +}); + +process.on('exit', function() { + assert.strictEqual(fieldsPassed, 0); +});