From 4d939813d640c50eecd34ead17b080fcae4dc499 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Thu, 28 Jul 2016 14:49:54 -0700 Subject: [PATCH 01/13] Add blob & arrayBuffer body constructors - Return a Node `Buffer` from the Response.blob() constructor. The `Buffer` signature is pretty much the same as `Blob`, so for now this might be a good compromise (until there is better `Blob` support on Node). - Return an `ArrayBuffer` from `Response.arrayBuffer()`. This is cheap, as Node's `Buffer` is implemented in terms of `ArrayBuffer` already. - Clean up `Body` state a bit: - Move _decode related vars to the closure, so that they are GC'ed as soon as possible. This is especially interesting for the _raw response chunks. --- lib/body.js | 101 +++++++++++++++++++++++++++++++-------------------- test/test.js | 34 ++++++++++++++++- 2 files changed, 94 insertions(+), 41 deletions(-) diff --git a/lib/body.js b/lib/body.js index 8e69b5d6e..0c8ea66a4 100644 --- a/lib/body.js +++ b/lib/body.js @@ -1,3 +1,4 @@ +'use strict'; /** * body.js @@ -24,12 +25,10 @@ function Body(body, opts) { opts = opts || {}; this.body = body; + this._decodedBody = null; this.bodyUsed = false; this.size = opts.size || 0; this.timeout = opts.timeout || 0; - this._raw = []; - this._abort = false; - } /** @@ -51,13 +50,44 @@ Body.prototype.json = function() { * @return Promise */ Body.prototype.text = function() { + if (typeof this.body === 'string' && !this.bodyUsed) { + this.bodyUsed = true; + return Promise.resolve(this.body); + } else { + return this._decode() + .then(body => this._convert(body)); + } +}; - return this._decode(); +/** + * Decode response as a blob. We are using a Node Buffer, which is close + * enough (for now). + * + * @return Promise + */ +Body.prototype.blob = function() { + if (Buffer.isBuffer(this._decodedBody) && !this.bodyUsed) { + this.bodyUsed = true; + return Promise.resolve(this._decodeBody); + } else { + return this._decode(); + } +}; +/** + * Return response body as an ArrayBuffer. + * + * @return Promise + */ +Body.prototype.arrayBuffer = function() { + return this._decode() + // Convert to ArrayBuffer + .then(body => body.buffer.slice(body.byteOffset, + body.byteOffset + body.byteLength)); }; /** - * Decode buffers into utf-8 string + * Accumulate the body & return a Buffer. * * @return Promise */ @@ -68,31 +98,29 @@ Body.prototype._decode = function() { if (this.bodyUsed) { return Body.Promise.reject(new Error('body used already for: ' + this.url)); } - this.bodyUsed = true; - this._bytes = 0; - this._abort = false; - this._raw = []; + + let bytes = 0; + let abort = false; + let accum = []; return new Body.Promise(function(resolve, reject) { var resTimeout; if (typeof self.body === 'string') { - self._bytes = self.body.length; - self._raw = [new Buffer(self.body)]; - return resolve(self._convert()); + self._decodedBody = new Buffer(self.body); + return resolve(self._decodedBody); } if (self.body instanceof Buffer) { - self._bytes = self.body.length; - self._raw = [self.body]; - return resolve(self._convert()); + self._decodedBody = self.body; + return resolve(self._decodedBody); } // allow timeout on slow response body if (self.timeout) { resTimeout = setTimeout(function() { - self._abort = true; + abort = true; reject(new FetchError('response timeout at ' + self.url + ' over limit: ' + self.timeout, 'body-timeout')); }, self.timeout); } @@ -103,27 +131,28 @@ Body.prototype._decode = function() { }); self.body.on('data', function(chunk) { - if (self._abort || chunk === null) { + if (abort || chunk === null) { return; } - if (self.size && self._bytes + chunk.length > self.size) { - self._abort = true; + if (self.size && bytes + chunk.length > self.size) { + abort = true; reject(new FetchError('content size at ' + self.url + ' over limit: ' + self.size, 'max-size')); return; } - self._bytes += chunk.length; - self._raw.push(chunk); + bytes += chunk.length; + accum.push(chunk); }); self.body.on('end', function() { - if (self._abort) { + if (abort) { return; } clearTimeout(resTimeout); - resolve(self._convert()); + self._decodedBody = Buffer.concat(accum); + resolve(self._decodedBody); }); }); @@ -136,7 +165,7 @@ Body.prototype._decode = function() { * @param String encoding Target encoding * @return String */ -Body.prototype._convert = function(encoding) { +Body.prototype._convert = function(body, encoding) { encoding = encoding || 'utf-8'; @@ -149,14 +178,8 @@ Body.prototype._convert = function(encoding) { } // no charset in content type, peek at response body for at most 1024 bytes - if (!res && this._raw.length > 0) { - for (var i = 0; i < this._raw.length; i++) { - str += this._raw[i].toString() - if (str.length > 1024) { - break; - } - } - str = str.substr(0, 1024); + if (!res && body.length > 0) { + str = body.slice(0, 1024).toString(); } // html5 @@ -188,14 +211,12 @@ Body.prototype._convert = function(encoding) { charset = 'gb18030'; } } - - // turn raw buffers into utf-8 string - return convert( - Buffer.concat(this._raw) - , encoding - , charset - ).toString(); - + if (encoding !== charset) { + // turn raw buffers into utf-8 string + return convert(body, encoding, charset).toString(); + } else { + return body.toString(charset); + } }; /** diff --git a/test/test.js b/test/test.js index 22510233a..5c4929db9 100644 --- a/test/test.js +++ b/test/test.js @@ -120,6 +120,36 @@ describe('node-fetch', function() { }); }); + it('should accept plain text response (blob)', function() { + url = base + '/plain'; + return fetch(url).then(function(res) { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return res.blob().then(function(result) { + expect(res.bodyUsed).to.be.true; + expect(Buffer.isBuffer(result)).to.equal(true); + expect(result.toString()).to.equal('text'); + }); + }); + }); + + it('should accept plain text response (arrayBuffer)', function() { + url = base + '/plain'; + return fetch(url).then(function(res) { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return res.arrayBuffer().then(function(result) { + expect(res.bodyUsed).to.be.true; + expect(Buffer.isBuffer(result)).to.equal(false); + expect(new Buffer(result).toString()).to.equal('text'); + const uarr = new Uint8Array(result); + expect(uarr[0]).to.equal(116); // 't' + expect(uarr[1]).to.equal(101); // 'e' + expect(uarr[2]).to.equal(120); // 'x' + expect(uarr[3]).to.equal(116); // 't' + expect(uarr[4]).to.equal(undefined); + }); + }); + }); + it('should accept html response (like plain text)', function() { url = base + '/html'; return fetch(url).then(function(res) { @@ -1347,10 +1377,12 @@ describe('node-fetch', function() { }); }); - it('should support text() and json() method in Body constructor', function() { + it('should support text(), json(), blob() and arrayBuffer() methods in Body constructor', function() { var body = new Body('a=1'); expect(body).to.have.property('text'); expect(body).to.have.property('json'); + expect(body).to.have.property('blob'); + expect(body).to.have.property('arrayBuffer'); });
 it('should create custom FetchError', function() { From 4689b6e13afc6a996eee196fb222da3dce81511d Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Fri, 29 Jul 2016 12:25:37 -0700 Subject: [PATCH 02/13] Use ReadableStream in Response & request bodies The fetch spec prescribes the use of a ReadableStream (from the streams spec) as response & request bodies. With ReadableStream exposed in recent Chrome, client-side code is starting to take advantage of streaming content transformations. To more faithfully implement the fetch spec & support isomorphic stream processing code, this patch adds support for ReadableStream, using the whatwg reference implementation: - `Response.body` is now a ReadableStream. - The `Body` constructor dynamically converts Node streams to `ReadableStream`, but still accepts FormData. - Requests accept ReadableStream, Node stream, or FormData bodies. --- index.js | 20 +++++++---- lib/body.js | 86 +++++++++++++++++++++++++--------------------- lib/web_streams.js | 78 +++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- test/test.js | 78 ++++++++++++++++++++--------------------- 5 files changed, 179 insertions(+), 86 deletions(-) create mode 100644 lib/web_streams.js diff --git a/index.js b/index.js index 10fb8ff23..16c27ab7c 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,7 @@ var Response = require('./lib/response'); var Headers = require('./lib/headers'); var Request = require('./lib/request'); var FetchError = require('./lib/fetch-error'); +var webStreams = require('./lib/web_streams'); // commonjs module.exports = Fetch; @@ -163,7 +164,7 @@ function Fetch(url, opts) { } // handle compression - var body = res.pipe(new stream.PassThrough()); + var body = res; var headers = new Headers(res.headers); if (options.compress && headers.has('content-encoding')) { @@ -179,6 +180,9 @@ function Fetch(url, opts) { } } + // Convert to ReadableStream + body = webStreams.readable.nodeToWeb(body); + // normalize location header for manual redirect mode if (options.redirect === 'manual' && headers.has('location')) { headers.set('location', resolve_url(options.url, headers.get('location'))); @@ -200,12 +204,16 @@ function Fetch(url, opts) { // accept string or readable stream as body if (typeof options.body === 'string') { req.write(options.body); - req.end(); - } else if (typeof options.body === 'object' && options.body.pipe) { - options.body.pipe(req); - } else { - req.end(); + } else if (typeof options.body === 'object') { + if (options.body.pipe) { + // Node stream + return options.body.pipe(req); + } else if (options.body.getReader) { + const nodeBody = webStreams.readable.webToNode(options.body); + return nodeBody.pipe(req); + } } + req.end(); }); }; diff --git a/lib/body.js b/lib/body.js index 0c8ea66a4..899d89baa 100644 --- a/lib/body.js +++ b/lib/body.js @@ -7,9 +7,10 @@ */ var convert = require('encoding').convert; -var bodyStream = require('is-stream'); +var isNodeStream = require('is-stream'); var PassThrough = require('stream').PassThrough; var FetchError = require('./fetch-error'); +var webStreams = require('./web_streams'); module.exports = Body; @@ -25,6 +26,10 @@ function Body(body, opts) { opts = opts || {}; this.body = body; + if (isNodeStream(body) && !body.getBoundary) { + // Node ReadableStream && not FormData. Convert to ReadableStream. + this.body = webStreams.readable.nodeToWeb(body); + } this._decodedBody = null; this.bodyUsed = false; this.size = opts.size || 0; @@ -100,9 +105,8 @@ Body.prototype._decode = function() { } this.bodyUsed = true; - let bytes = 0; - let abort = false; let accum = []; + let accumBytes = 0; return new Body.Promise(function(resolve, reject) { var resTimeout; @@ -117,42 +121,49 @@ Body.prototype._decode = function() { return resolve(self._decodedBody); } + + var reader = self.body.getReader(); + // allow timeout on slow response body if (self.timeout) { resTimeout = setTimeout(function() { - abort = true; + reader.cancel(); reject(new FetchError('response timeout at ' + self.url + ' over limit: ' + self.timeout, 'body-timeout')); }, self.timeout); } - // handle stream error, such as incorrect content-encoding - self.body.on('error', function(err) { - reject(new FetchError('invalid response body at: ' + self.url + ' reason: ' + err.message, 'system', err)); - }); - - self.body.on('data', function(chunk) { - if (abort || chunk === null) { - return; - } - - if (self.size && bytes + chunk.length > self.size) { - abort = true; - reject(new FetchError('content size at ' + self.url + ' over limit: ' + self.size, 'max-size')); - return; - } - - bytes += chunk.length; - accum.push(chunk); - }); - - self.body.on('end', function() { - if (abort) { - return; + function pump() { + return reader.read() + .then(res => { + if (res.done) { + clearTimeout(resTimeout); + // Make sure all elements are indeed buffers + for (var i = 0; i < accum.length; i++) { + let chunk = accum[i]; + if (!Buffer.isBuffer(chunk)) { + accum[i] = new Buffer('' + chunk); + } + } + self._decodedBody = Buffer.concat(accum); + return self._decodedBody; + } + const chunk = res.value; + accum.push(chunk); + accumBytes += chunk.length; + if (self.size && accumBytes > self.size) { + reader.cancel(); + throw new FetchError('content size at ' + self.url + ' over limit: ' + self.size, 'max-size'); + } + return pump(); + }); + } + return pump().then(resolve, err => { + if (err instanceof FetchError) { + reject(err); + } else { + reject(new FetchError('invalid response body at: ' + self.url + + ' reason: ' + err.message, 'system', err)); } - - clearTimeout(resTimeout); - self._decodedBody = Buffer.concat(accum); - resolve(self._decodedBody); }); }); @@ -236,15 +247,10 @@ Body.prototype._clone = function(instance) { // check that body is a stream and not form-data object // note: we can't clone the form-data object without having it as a dependency - if (bodyStream(body) && typeof body.getBoundary !== 'function') { - // tee instance body - p1 = new PassThrough(); - p2 = new PassThrough(); - body.pipe(p1); - body.pipe(p2); - // set instance body to teed body and return the other teed body - instance.body = p1; - body = p2; + if (body instanceof ReadableStream && typeof body.getBoundary !== 'function') { + let streams = instance.body.tee(); + instance.body = streams[0]; + body = streams[1]; } return body; diff --git a/lib/web_streams.js b/lib/web_streams.js new file mode 100644 index 000000000..000a23c5c --- /dev/null +++ b/lib/web_streams.js @@ -0,0 +1,78 @@ +'use strict'; + +const Readable = require('stream').Readable; +if (!global.ReadableStream) { + global.ReadableStream = require('web-streams-polyfill').ReadableStream; +} + +/** + * Web / node stream conversion functions + */ + +function readableNodeToWeb(nodeStream) { + return new ReadableStream({ + start(controller) { + nodeStream.pause(); + nodeStream.on('data', chunk => { + controller.enqueue(chunk); + nodeStream.pause(); + }); + nodeStream.on('end', () => controller.close()); + nodeStream.on('error', (e) => controller.error(e)); + }, + pull(controller) { + nodeStream.resume(); + }, + cancel(reason) { + nodeStream.pause(); + } + }); +} + +class NodeReadable extends Readable { + constructor(webStream, options) { + super(options); + this._webStream = webStream; + this._reader = webStream.getReader(); + this._reading = false; + } + + _read(size) { + if (this._reading) { + return; + } + this._reading = true; + const doRead = () => { + this._reader.read() + .then(res => { + if (res.done) { + this.push(null); + return; + } + if (this.push(res.value)) { + return doRead(size); + } else { + this._reading = false; + } + }); + } + doRead(); + } +} + +function readableWebToNode(webStream) { + return new NodeReadable(webStream); +} + +module.exports = { + readable: { + nodeToWeb: readableNodeToWeb, + webToNode: readableWebToNode, + }, +}; + +// Simple round-trip test. +// let nodeReader = require('fs').createReadStream('/tmp/test.txt'); +// let webReader = readableNodeToWeb(nodeReader); +// let roundTrippedReader = readableWebToNode(webReader); +// roundTrippedReader.pipe(process.stdout); diff --git a/package.json b/package.json index a312abd1c..9d1e5441c 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ }, "dependencies": { "encoding": "^0.1.11", - "is-stream": "^1.0.1" + "is-stream": "^1.0.1", + "web-streams-polyfill": "git+https://github.com/gwicke/web-streams-polyfill#spec_performance_improvements" } } diff --git a/test/test.js b/test/test.js index 5c4929db9..18f58c3ed 100644 --- a/test/test.js +++ b/test/test.js @@ -1,3 +1,4 @@ +'use strict'; // test tools var chai = require('chai'); @@ -98,7 +99,7 @@ describe('node-fetch', function() { return fetch(url).then(function(res) { expect(res).to.be.an.instanceof(Response); expect(res.headers).to.be.an.instanceof(Headers); - expect(res.body).to.be.an.instanceof(stream.Transform); + expect(res.body).to.be.an.instanceof(ReadableStream); expect(res.bodyUsed).to.be.false; expect(res.url).to.equal(url); @@ -800,7 +801,7 @@ describe('node-fetch', function() { expect(res.status).to.equal(200); expect(res.statusText).to.equal('OK'); expect(res.headers.get('content-type')).to.equal('text/plain'); - expect(res.body).to.be.an.instanceof(stream.Transform); + expect(res.body).to.be.an.instanceof(ReadableStream); }); }); @@ -813,7 +814,7 @@ describe('node-fetch', function() { expect(res.status).to.equal(200); expect(res.statusText).to.equal('OK'); expect(res.headers.get('allow')).to.equal('GET, HEAD, OPTIONS'); - expect(res.body).to.be.an.instanceof(stream.Transform); + expect(res.body).to.be.an.instanceof(ReadableStream); }); }); @@ -954,16 +955,19 @@ describe('node-fetch', function() { it('should allow piping response body as stream', function(done) { url = base + '/hello'; fetch(url).then(function(res) { - expect(res.body).to.be.an.instanceof(stream.Transform); - res.body.on('data', function(chunk) { - if (chunk === null) { - return; - } - expect(chunk.toString()).to.equal('world'); - }); - res.body.on('end', function() { - done(); - }); + expect(res.body).to.be.an.instanceof(ReadableStream); + const reader = res.body.getReader(); + return reader.read() + .then(res => { + expect(res.value.toString()).to.equal('world'); + return reader.read().then(res => { + if (res.done) { + done(); + } else { + done(new Error("Expected stream to be done")); + } + }); + }); }); }); @@ -972,31 +976,28 @@ describe('node-fetch', function() { return fetch(url).then(function(res) { var counter = 0; var r1 = res.clone(); - expect(res.body).to.be.an.instanceof(stream.Transform); - expect(r1.body).to.be.an.instanceof(stream.Transform); - res.body.on('data', function(chunk) { - if (chunk === null) { - return; - } - expect(chunk.toString()).to.equal('world'); - }); - res.body.on('end', function() { - counter++; - if (counter == 2) { - done(); - } - }); - r1.body.on('data', function(chunk) { - if (chunk === null) { - return; - } - expect(chunk.toString()).to.equal('world'); - }); - r1.body.on('end', function() { - counter++; - if (counter == 2) { - done(); + expect(res.body).to.be.an.instanceof(ReadableStream); + expect(r1.body).to.be.an.instanceof(ReadableStream); + function drain(stream) { + const reader = stream.getReader(); + let chunks = []; + function pump() { + return reader.read() + .then(res => { + if (res.done) { return chunks; } + chunks.push(res.value); + return pump(); + }); } + return pump(); + } + return drain(res.body).then(chunks => { + expect(chunks[0].toString()).to.equal('world'); + return drain(r1.body); + }) + .then(chunks => { + expect(chunks[0].toString()).to.equal('world'); + done(); }); }); }); @@ -1072,7 +1073,7 @@ describe('node-fetch', function() { result.push([key, val]); }); - expected = [ + const expected = [ ["a", "1"] , ["b", "2"] , ["b", "3"] @@ -1259,7 +1260,6 @@ describe('node-fetch', function() { it('should support clone() method in Response constructor', function() { var body = resumer().queue('a=1').end(); - body = body.pipe(new stream.PassThrough()); var res = new Response(body, { headers: { a: '1' From a563ea4dfb4d10938a6213f27166a26ddecef165 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Fri, 29 Jul 2016 12:53:30 -0700 Subject: [PATCH 03/13] Test node 4 & 6, and drop 0.10 / 0.12 --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1faa00a3c..2279580b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: node_js node_js: - - "0.10" - - "0.12" - - "node" + - "4" + - "6" before_install: npm install -g npm -script: npm run coverage \ No newline at end of file +script: npm run coverage From 7d8c8db9ebcbaa09155ed6457954f391944e088b Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Fri, 29 Jul 2016 17:54:09 -0700 Subject: [PATCH 04/13] Use node-web-streams package --- index.js | 40 ++++++++++++++---------- lib/body.js | 24 ++++++++++---- lib/web_streams.js | 78 ---------------------------------------------- package.json | 2 +- test/test.js | 1 + 5 files changed, 43 insertions(+), 102 deletions(-) delete mode 100644 lib/web_streams.js diff --git a/index.js b/index.js index 16c27ab7c..28d4ac0cc 100644 --- a/index.js +++ b/index.js @@ -17,7 +17,7 @@ var Response = require('./lib/response'); var Headers = require('./lib/headers'); var Request = require('./lib/request'); var FetchError = require('./lib/fetch-error'); -var webStreams = require('./lib/web_streams'); +var webStreams = require('node-web-streams'); // commonjs module.exports = Fetch; @@ -92,8 +92,9 @@ function Fetch(url, opts) { // bring node-fetch closer to browser behavior by setting content-length automatically if (!headers.has('content-length') && /post|put|patch|delete/i.test(options.method)) { - if (typeof options.body === 'string') { - headers.set('content-length', Buffer.byteLength(options.body)); + if (typeof options._rawBody === 'string' || Buffer.isBuffer(options._rawBody)) { + options._rawBody = new Buffer(options._rawBody); + headers.set('content-length', options._rawBody.length); // detect form data input from form-data module, this hack avoid the need to add content-length header manually } else if (options.body && typeof options.body.getLengthSync === 'function' && options.body._lengthRetrievers.length == 0) { headers.set('content-length', options.body.getLengthSync().toString()); @@ -180,8 +181,8 @@ function Fetch(url, opts) { } } - // Convert to ReadableStream - body = webStreams.readable.nodeToWeb(body); + // Convert to ReadableStream + body = webStreams.toWebReadableStream(body); // normalize location header for manual redirect mode if (options.redirect === 'manual' && headers.has('location')) { @@ -201,18 +202,23 @@ function Fetch(url, opts) { resolve(output); }); - // accept string or readable stream as body - if (typeof options.body === 'string') { - req.write(options.body); - } else if (typeof options.body === 'object') { - if (options.body.pipe) { - // Node stream - return options.body.pipe(req); - } else if (options.body.getReader) { - const nodeBody = webStreams.readable.webToNode(options.body); - return nodeBody.pipe(req); - } - } + // Request body handling + if (options.body !== undefined) { + if (typeof options._rawBody === 'string' || Buffer.isBuffer(options._rawBody)) { + // Fast path for simple strings / buffers, avoid chunked + // encoding. + return req.end(options._rawBody); + } else if (options.body.pipe) { + // Node stream (likely FormData). + return options.body.pipe(req); + } else if (options.body.getReader) { + // ReadableStream + const nodeBody = webStreams.toNodeReadable(options.body); + return nodeBody.pipe(req); + } else { + throw new TypeError('Unexpected Request body type: ' + (typeof options.body)); + } + } req.end(); }); diff --git a/lib/body.js b/lib/body.js index 899d89baa..2b5b9ee4e 100644 --- a/lib/body.js +++ b/lib/body.js @@ -10,7 +10,7 @@ var convert = require('encoding').convert; var isNodeStream = require('is-stream'); var PassThrough = require('stream').PassThrough; var FetchError = require('./fetch-error'); -var webStreams = require('./web_streams'); +var webStreams = require('node-web-streams'); module.exports = Body; @@ -25,10 +25,22 @@ function Body(body, opts) { opts = opts || {}; + this._rawBody = body; this.body = body; - if (isNodeStream(body) && !body.getBoundary) { - // Node ReadableStream && not FormData. Convert to ReadableStream. - this.body = webStreams.readable.nodeToWeb(body); + if (body) { + if (body instanceof webStreams.ReadableStream) { + this.body = body; + } else if (typeof body === 'string' + || Buffer.isBuffer(body) + || isNodeStream(body) && body.readable && !body.getBoundary) { + // Node Readable && not FormData. Convert to ReadableStream. + this.body = webStreams.toWebReadableStream(body); + } else if (body && body.getBoundary) { + // FormData instance. Make an exception, for now. + this.body = body; + } else { + throw new TypeError("Unsupported Response body type: " + (typeof body)); + } } this._decodedBody = null; this.bodyUsed = false; @@ -190,7 +202,7 @@ Body.prototype._convert = function(body, encoding) { // no charset in content type, peek at response body for at most 1024 bytes if (!res && body.length > 0) { - str = body.slice(0, 1024).toString(); + str = body.slice(0, 1024).toString(); } // html5 @@ -247,7 +259,7 @@ Body.prototype._clone = function(instance) { // check that body is a stream and not form-data object // note: we can't clone the form-data object without having it as a dependency - if (body instanceof ReadableStream && typeof body.getBoundary !== 'function') { + if (body instanceof webStreams.ReadableStream && typeof body.getBoundary !== 'function') { let streams = instance.body.tee(); instance.body = streams[0]; body = streams[1]; diff --git a/lib/web_streams.js b/lib/web_streams.js deleted file mode 100644 index 000a23c5c..000000000 --- a/lib/web_streams.js +++ /dev/null @@ -1,78 +0,0 @@ -'use strict'; - -const Readable = require('stream').Readable; -if (!global.ReadableStream) { - global.ReadableStream = require('web-streams-polyfill').ReadableStream; -} - -/** - * Web / node stream conversion functions - */ - -function readableNodeToWeb(nodeStream) { - return new ReadableStream({ - start(controller) { - nodeStream.pause(); - nodeStream.on('data', chunk => { - controller.enqueue(chunk); - nodeStream.pause(); - }); - nodeStream.on('end', () => controller.close()); - nodeStream.on('error', (e) => controller.error(e)); - }, - pull(controller) { - nodeStream.resume(); - }, - cancel(reason) { - nodeStream.pause(); - } - }); -} - -class NodeReadable extends Readable { - constructor(webStream, options) { - super(options); - this._webStream = webStream; - this._reader = webStream.getReader(); - this._reading = false; - } - - _read(size) { - if (this._reading) { - return; - } - this._reading = true; - const doRead = () => { - this._reader.read() - .then(res => { - if (res.done) { - this.push(null); - return; - } - if (this.push(res.value)) { - return doRead(size); - } else { - this._reading = false; - } - }); - } - doRead(); - } -} - -function readableWebToNode(webStream) { - return new NodeReadable(webStream); -} - -module.exports = { - readable: { - nodeToWeb: readableNodeToWeb, - webToNode: readableWebToNode, - }, -}; - -// Simple round-trip test. -// let nodeReader = require('fs').createReadStream('/tmp/test.txt'); -// let webReader = readableNodeToWeb(nodeReader); -// let roundTrippedReader = readableWebToNode(webReader); -// roundTrippedReader.pipe(process.stdout); diff --git a/package.json b/package.json index 9d1e5441c..9152f0d5e 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,6 @@ "dependencies": { "encoding": "^0.1.11", "is-stream": "^1.0.1", - "web-streams-polyfill": "git+https://github.com/gwicke/web-streams-polyfill#spec_performance_improvements" + "node-web-streams": "^0.2.1" } } diff --git a/test/test.js b/test/test.js index 18f58c3ed..1ff51e96a 100644 --- a/test/test.js +++ b/test/test.js @@ -25,6 +25,7 @@ var Body = require('../lib/body.js'); var FetchError = require('../lib/fetch-error.js'); // test with native promise on node 0.11, and bluebird for node 0.10 fetch.Promise = fetch.Promise || bluebird; +var ReadableStream = require('node-web-streams').ReadableStream; var url, opts, local, base; From 20707634caaedcd038bd480796d687c527b53492 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Sat, 30 Jul 2016 10:52:14 -0700 Subject: [PATCH 05/13] Small bug fix in blob() --- lib/body.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/body.js b/lib/body.js index 2b5b9ee4e..ebda946bd 100644 --- a/lib/body.js +++ b/lib/body.js @@ -85,7 +85,7 @@ Body.prototype.text = function() { Body.prototype.blob = function() { if (Buffer.isBuffer(this._decodedBody) && !this.bodyUsed) { this.bodyUsed = true; - return Promise.resolve(this._decodeBody); + return Promise.resolve(this._decodedBody); } else { return this._decode(); } From 54ca3fdd8121635a495a6ee3e64b11bed100d434 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Sat, 30 Jul 2016 12:54:34 -0700 Subject: [PATCH 06/13] Slightly smarter text() handling --- lib/body.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/body.js b/lib/body.js index ebda946bd..37d93fce2 100644 --- a/lib/body.js +++ b/lib/body.js @@ -67,12 +67,18 @@ Body.prototype.json = function() { * @return Promise */ Body.prototype.text = function() { - if (typeof this.body === 'string' && !this.bodyUsed) { + if (this.bodyUsed) { + return this._decode(); + } + if (typeof this._rawBody === 'string') { + this.bodyUsed = true; + return Promise.resolve(this._rawBody); + } else if (this._decodedBody) { this.bodyUsed = true; - return Promise.resolve(this.body); + return Promise.resolve(this._decodedBody.toString()); } else { return this._decode() - .then(body => this._convert(body)); + .then(body => body.toString()); } }; From 00ae892bafdf601c854bf4ebbc2bc41bbe224a87 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Mon, 1 Aug 2016 10:08:37 -0700 Subject: [PATCH 07/13] Convert body, request & response to ES6 syntax Apart from being cleaner, this will let us leverage ES6 getters & setters for lazy ReadableStream construction & type conversion. --- lib/body.js | 518 ++++++++++++++++++++++++------------------------ lib/request.js | 111 +++++------ lib/response.js | 69 +++---- 3 files changed, 338 insertions(+), 360 deletions(-) diff --git a/lib/body.js b/lib/body.js index 37d93fce2..a8e05e2fd 100644 --- a/lib/body.js +++ b/lib/body.js @@ -1,18 +1,10 @@ 'use strict'; -/** - * body.js - * - * Body interface provides common methods for Request and Response - */ - -var convert = require('encoding').convert; -var isNodeStream = require('is-stream'); -var PassThrough = require('stream').PassThrough; -var FetchError = require('./fetch-error'); -var webStreams = require('node-web-streams'); - -module.exports = Body; +const encoding = require('encoding'); +const isNodeStream = require('is-stream'); +const PassThrough = require('stream').PassThrough; +const FetchError = require('./fetch-error'); +const webStreams = require('node-web-streams'); /** * Body class @@ -21,258 +13,256 @@ module.exports = Body; * @param Object opts Response options * @return Void */ -function Body(body, opts) { - - opts = opts || {}; - - this._rawBody = body; - this.body = body; - if (body) { - if (body instanceof webStreams.ReadableStream) { - this.body = body; - } else if (typeof body === 'string' - || Buffer.isBuffer(body) - || isNodeStream(body) && body.readable && !body.getBoundary) { - // Node Readable && not FormData. Convert to ReadableStream. - this.body = webStreams.toWebReadableStream(body); - } else if (body && body.getBoundary) { - // FormData instance. Make an exception, for now. - this.body = body; - } else { - throw new TypeError("Unsupported Response body type: " + (typeof body)); - } - } - this._decodedBody = null; - this.bodyUsed = false; - this.size = opts.size || 0; - this.timeout = opts.timeout || 0; -} - -/** - * Decode response as json - * - * @return Promise - */ -Body.prototype.json = function() { - - return this._decode().then(function(text) { - return JSON.parse(text); - }); - -}; - -/** - * Decode response as text - * - * @return Promise - */ -Body.prototype.text = function() { - if (this.bodyUsed) { - return this._decode(); - } - if (typeof this._rawBody === 'string') { - this.bodyUsed = true; - return Promise.resolve(this._rawBody); - } else if (this._decodedBody) { - this.bodyUsed = true; - return Promise.resolve(this._decodedBody.toString()); - } else { - return this._decode() - .then(body => body.toString()); - } -}; - -/** - * Decode response as a blob. We are using a Node Buffer, which is close - * enough (for now). - * - * @return Promise - */ -Body.prototype.blob = function() { - if (Buffer.isBuffer(this._decodedBody) && !this.bodyUsed) { - this.bodyUsed = true; - return Promise.resolve(this._decodedBody); - } else { - return this._decode(); - } -}; - -/** - * Return response body as an ArrayBuffer. - * - * @return Promise - */ -Body.prototype.arrayBuffer = function() { - return this._decode() - // Convert to ArrayBuffer - .then(body => body.buffer.slice(body.byteOffset, - body.byteOffset + body.byteLength)); -}; - -/** - * Accumulate the body & return a Buffer. - * - * @return Promise - */ -Body.prototype._decode = function() { - - var self = this; - - if (this.bodyUsed) { - return Body.Promise.reject(new Error('body used already for: ' + this.url)); - } - this.bodyUsed = true; - - let accum = []; - let accumBytes = 0; - - return new Body.Promise(function(resolve, reject) { - var resTimeout; - - if (typeof self.body === 'string') { - self._decodedBody = new Buffer(self.body); - return resolve(self._decodedBody); - } - - if (self.body instanceof Buffer) { - self._decodedBody = self.body; - return resolve(self._decodedBody); - } - - - var reader = self.body.getReader(); - - // allow timeout on slow response body - if (self.timeout) { - resTimeout = setTimeout(function() { - reader.cancel(); - reject(new FetchError('response timeout at ' + self.url + ' over limit: ' + self.timeout, 'body-timeout')); - }, self.timeout); - } - - function pump() { - return reader.read() - .then(res => { - if (res.done) { - clearTimeout(resTimeout); - // Make sure all elements are indeed buffers - for (var i = 0; i < accum.length; i++) { - let chunk = accum[i]; - if (!Buffer.isBuffer(chunk)) { - accum[i] = new Buffer('' + chunk); - } - } - self._decodedBody = Buffer.concat(accum); - return self._decodedBody; - } - const chunk = res.value; - accum.push(chunk); - accumBytes += chunk.length; - if (self.size && accumBytes > self.size) { - reader.cancel(); - throw new FetchError('content size at ' + self.url + ' over limit: ' + self.size, 'max-size'); - } - return pump(); - }); - } - return pump().then(resolve, err => { - if (err instanceof FetchError) { - reject(err); - } else { - reject(new FetchError('invalid response body at: ' + self.url - + ' reason: ' + err.message, 'system', err)); - } - }); - }); - -}; - -/** - * Detect buffer encoding and convert to target encoding - * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding - * - * @param String encoding Target encoding - * @return String - */ -Body.prototype._convert = function(body, encoding) { - - encoding = encoding || 'utf-8'; - - var charset = 'utf-8'; - var res, str; - - // header - if (this.headers.has('content-type')) { - res = /charset=([^;]*)/i.exec(this.headers.get('content-type')); - } - - // no charset in content type, peek at response body for at most 1024 bytes - if (!res && body.length > 0) { - str = body.slice(0, 1024).toString(); - } - - // html5 - if (!res && str) { - res = / JSON.parse(text)); + + } + + /** + * Decode response as text + * + * @return Promise + */ + text() { + if (this.bodyUsed) { + return this._decode(); + } + if (typeof this._rawBody === 'string') { + this.bodyUsed = true; + return Promise.resolve(this._rawBody); + } else if (this._decodedBody) { + this.bodyUsed = true; + return Promise.resolve(this._decodedBody.toString()); + } else { + return this._decode() + .then(body => body.toString()); + } + } + + /** + * Decode response as a blob. We are using a Node Buffer, which is close + * enough (for now). + * + * @return Promise + */ + blob() { + if (Buffer.isBuffer(this._decodedBody) && !this.bodyUsed) { + this.bodyUsed = true; + return Promise.resolve(this._decodedBody); + } else { + return this._decode(); + } + } + + /** + * Return response body as an ArrayBuffer. + * + * @return Promise + */ + arrayBuffer() { + return this._decode() + // Convert to ArrayBuffer + .then(body => body.buffer.slice(body.byteOffset, + body.byteOffset + body.byteLength)); + } + + /** + * Accumulate the body & return a Buffer. + * + * @return Promise + */ + _decode() { + + const self = this; + + if (this.bodyUsed) { + return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); + } + this.bodyUsed = true; + + let accum = []; + let accumBytes = 0; + + return new Body.Promise((resolve, reject) => { + let resTimeout; + + if (typeof self.body === 'string') { + self._decodedBody = new Buffer(self.body); + return resolve(self._decodedBody); + } + + if (self.body instanceof Buffer) { + self._decodedBody = self.body; + return resolve(self._decodedBody); + } + + + const reader = self.body.getReader(); + + // allow timeout on slow response body + if (self.timeout) { + resTimeout = setTimeout(() => { + reader.cancel(); + reject(new FetchError(`response timeout at ${self.url} over limit: ${self.timeout}`, 'body-timeout')); + }, self.timeout); + } + + function pump() { + return reader.read() + .then(res => { + if (res.done) { + clearTimeout(resTimeout); + // Make sure all elements are indeed buffers + for (let i = 0; i < accum.length; i++) { + let chunk = accum[i]; + if (!Buffer.isBuffer(chunk)) { + accum[i] = new Buffer(`${chunk}`); + } + } + self._decodedBody = Buffer.concat(accum); + return self._decodedBody; + } + const chunk = res.value; + accum.push(chunk); + accumBytes += chunk.length; + if (self.size && accumBytes > self.size) { + reader.cancel(); + throw new FetchError(`content size at ${self.url} over limit: ${self.size}`, 'max-size'); + } + return pump(); + }); + } + return pump().then(resolve, err => { + if (err instanceof FetchError) { + reject(err); + } else { + reject(new FetchError(`invalid response body at: ${self.url} reason: ${err.message}`, 'system', err)); + } + }); + }); + + } + + /** + * Detect buffer encoding and convert to target encoding + * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding + * + * @param String encoding Target encoding + * @return String + */ + _convert(body, encoding) { + encoding = encoding || 'utf-8'; + let charset = 'utf-8'; + let res, str; + + // header + if (this.headers.has('content-type')) { + res = /charset=([^;]*)/i.exec(this.headers.get('content-type')); + } + + // no charset in content type, peek at response body for at most 1024 bytes + if (!res && body.length > 0) { + str = body.slice(0, 1024).toString(); + } + + // html5 + if (!res && str) { + res = /= 200 && this.status < 300; - - Body.call(this, body, opts); - +class Response extends Body { + constructor(body, opts) { + opts = opts || {}; + super(body, opts); + this.url = opts.url; + this.status = opts.status || 200; + this.statusText = opts.statusText || http.STATUS_CODES[this.status]; + this.headers = new Headers(opts.headers); + this.ok = this.status >= 200 && this.status < 300; + } + + /** + * Clone this response + * + * @return Response + */ + clone() { + return new Response(Body._clone(this), { + url: this.url + , status: this.status + , statusText: this.statusText + , headers: this.headers + , ok: this.ok + }); + } } -Response.prototype = Object.create(Body.prototype); - -/** - * Clone this response - * - * @return Response - */ -Response.prototype.clone = function() { - return new Response(this._clone(this), { - url: this.url - , status: this.status - , statusText: this.statusText - , headers: this.headers - , ok: this.ok - }); -}; +module.exports = Response; From 9e3a3e1f545600098904e82c19a65a2c26c85b86 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Mon, 1 Aug 2016 10:15:17 -0700 Subject: [PATCH 08/13] Re-enable full charset detection --- lib/body.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/body.js b/lib/body.js index a8e05e2fd..e56e69465 100644 --- a/lib/body.js +++ b/lib/body.js @@ -1,6 +1,6 @@ 'use strict'; -const encoding = require('encoding'); +const convertEncoding = require('encoding').convert; const isNodeStream = require('is-stream'); const PassThrough = require('stream').PassThrough; const FetchError = require('./fetch-error'); @@ -62,13 +62,13 @@ class Body { } if (typeof this._rawBody === 'string') { this.bodyUsed = true; - return Promise.resolve(this._rawBody); - } else if (this._decodedBody) { + return Promise.resolve(this._convert(this._rawBody)); + } else if (this._decodedBody) { this.bodyUsed = true; - return Promise.resolve(this._decodedBody.toString()); + return Promise.resolve(this._convert(this._decodedBody.toString())); } else { return this._decode() - .then(body => body.toString()); + .then(body => this._convert(body)); } } @@ -229,7 +229,7 @@ class Body { } if (encoding !== charset) { // turn raw buffers into utf-8 string - return encoding.convert(body, encoding, charset).toString(); + return convertEncoding(body, encoding, charset).toString(); } else { return body.toString(charset); } From 3756688d0adc7df773e0d094586fecbd826fb563 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Mon, 1 Aug 2016 14:12:25 -0700 Subject: [PATCH 09/13] Remove charset detection & use ES6 getters for body - Remove charset detection, as this differs from the spec & browser implementations. Resolves #1. - Use ES6 getters for Request.body / Response.body access. This lets us avoid constructing a ReadableStream in the large number of cases where the full body is consumed with `.text()`, `.json()` or `.blob()`. - Add a fast path avoiding stream construction for request bodies in index.js. --- index.js | 111 +++++++++++++++++----------------- lib/body.js | 150 +++++++++++++--------------------------------- test/test.js | 164 +++++++++++++++++++++++++-------------------------- 3 files changed, 181 insertions(+), 244 deletions(-) diff --git a/index.js b/index.js index 28d4ac0cc..5ba22bc5e 100644 --- a/index.js +++ b/index.js @@ -49,27 +49,27 @@ function Fetch(url, opts) { // wrap http.request into fetch return new Fetch.Promise(function(resolve, reject) { // build request object - var options = new Request(url, opts); + const request = new Request(url, opts); - if (!options.protocol || !options.hostname) { + if (!request.protocol || !request.hostname) { throw new Error('only absolute urls are supported'); } - if (options.protocol !== 'http:' && options.protocol !== 'https:') { + if (request.protocol !== 'http:' && request.protocol !== 'https:') { throw new Error('only http(s) protocols are supported'); } var send; - if (options.protocol === 'https:') { + if (request.protocol === 'https:') { send = https.request; } else { send = http.request; } // normalize headers - var headers = new Headers(options.headers); + var headers = new Headers(request.headers); - if (options.compress) { + if (request.compress) { headers.set('accept-encoding', 'gzip,deflate'); } @@ -77,7 +77,7 @@ function Fetch(url, opts) { headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); } - if (!headers.has('connection') && !options.agent) { + if (!headers.has('connection') && !request.agent) { headers.set('connection', 'close'); } @@ -86,81 +86,86 @@ function Fetch(url, opts) { } // detect form data input from form-data module, this hack avoid the need to pass multipart header manually - if (!headers.has('content-type') && options.body && typeof options.body.getBoundary === 'function') { - headers.set('content-type', 'multipart/form-data; boundary=' + options.body.getBoundary()); + if (!headers.has('content-type') && request._rawBody + && typeof request._rawBody.getBoundary === 'function') { + headers.set('content-type', 'multipart/form-data; boundary=' + request.body.getBoundary()); } // bring node-fetch closer to browser behavior by setting content-length automatically - if (!headers.has('content-length') && /post|put|patch|delete/i.test(options.method)) { - if (typeof options._rawBody === 'string' || Buffer.isBuffer(options._rawBody)) { - options._rawBody = new Buffer(options._rawBody); - headers.set('content-length', options._rawBody.length); + if (!headers.has('content-length') && /post|put|patch|delete/i.test(request.method)) { + if (typeof request._rawBody === 'string') { + request._rawBody = new Buffer(request._rawBody); + } + if (Buffer.isBuffer(request._rawBody)) { + headers.set('content-length', request._rawBody.length); // detect form data input from form-data module, this hack avoid the need to add content-length header manually - } else if (options.body && typeof options.body.getLengthSync === 'function' && options.body._lengthRetrievers.length == 0) { - headers.set('content-length', options.body.getLengthSync().toString()); + } else if (request._rawBody + && typeof request._rawBody.getLengthSync === 'function' + && request._rawBody._lengthRetrievers.length == 0) { + headers.set('content-length', request._rawBody.getLengthSync().toString()); // this is only necessary for older nodejs releases (before iojs merge) - } else if (options.body === undefined || options.body === null) { + } else if (request._rawBody === undefined || request._rawBody === null) { headers.set('content-length', '0'); } } - options.headers = headers.raw(); + request.headers = headers.raw(); // http.request only support string as host header, this hack make custom host header possible - if (options.headers.host) { - options.headers.host = options.headers.host[0]; + if (request.headers.host) { + request.headers.host = request.headers.host[0]; } // send request - var req = send(options); + var req = send(request); var reqTimeout; - if (options.timeout) { + if (request.timeout) { req.once('socket', function(socket) { reqTimeout = setTimeout(function() { req.abort(); - reject(new FetchError('network timeout at: ' + options.url, 'request-timeout')); - }, options.timeout); + reject(new FetchError('network timeout at: ' + request.url, 'request-timeout')); + }, request.timeout); }); } req.on('error', function(err) { clearTimeout(reqTimeout); - reject(new FetchError('request to ' + options.url + ' failed, reason: ' + err.message, 'system', err)); + reject(new FetchError('request to ' + request.url + ' failed, reason: ' + err.message, 'system', err)); }); req.on('response', function(res) { clearTimeout(reqTimeout); // handle redirect - if (self.isRedirect(res.statusCode) && options.redirect !== 'manual') { - if (options.redirect === 'error') { - reject(new FetchError('redirect mode is set to error: ' + options.url, 'no-redirect')); + if (self.isRedirect(res.statusCode) && request.redirect !== 'manual') { + if (request.redirect === 'error') { + reject(new FetchError('redirect mode is set to error: ' + request.url, 'no-redirect')); return; } - if (options.counter >= options.follow) { - reject(new FetchError('maximum redirect reached at: ' + options.url, 'max-redirect')); + if (request.counter >= request.follow) { + reject(new FetchError('maximum redirect reached at: ' + request.url, 'max-redirect')); return; } if (!res.headers.location) { - reject(new FetchError('redirect location header missing at: ' + options.url, 'invalid-redirect')); + reject(new FetchError('redirect location header missing at: ' + request.url, 'invalid-redirect')); return; } // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect if (res.statusCode === 303 - || ((res.statusCode === 301 || res.statusCode === 302) && options.method === 'POST')) + || ((res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST')) { - options.method = 'GET'; - delete options.body; - delete options.headers['content-length']; + request.method = 'GET'; + request.body = undefined; + delete request.headers['content-length']; } - options.counter++; + request.counter++; - resolve(Fetch(resolve_url(options.url, res.headers.location), options)); + resolve(Fetch(resolve_url(request.url, res.headers.location), request)); return; } @@ -168,7 +173,7 @@ function Fetch(url, opts) { var body = res; var headers = new Headers(res.headers); - if (options.compress && headers.has('content-encoding')) { + if (request.compress && headers.has('content-encoding')) { var name = headers.get('content-encoding'); // no need to pipe no content and not modified response body @@ -185,38 +190,36 @@ function Fetch(url, opts) { body = webStreams.toWebReadableStream(body); // normalize location header for manual redirect mode - if (options.redirect === 'manual' && headers.has('location')) { - headers.set('location', resolve_url(options.url, headers.get('location'))); + if (request.redirect === 'manual' && headers.has('location')) { + headers.set('location', resolve_url(request.url, headers.get('location'))); } // response object var output = new Response(body, { - url: options.url + url: request.url , status: res.statusCode , statusText: res.statusMessage , headers: headers - , size: options.size - , timeout: options.timeout + , size: request.size + , timeout: request.timeout }); resolve(output); }); // Request body handling - if (options.body !== undefined) { - if (typeof options._rawBody === 'string' || Buffer.isBuffer(options._rawBody)) { - // Fast path for simple strings / buffers, avoid chunked - // encoding. - return req.end(options._rawBody); - } else if (options.body.pipe) { + if (request._rawBody !== undefined && request._rawBody !== null) { + if (Buffer.isBuffer(request._rawBody)) { + // Fast path for simple buffers. Avoid stream wrapper & + // chunked encoding. + return req.end(request._rawBody); + } else if (request._rawBody.pipe) { // Node stream (likely FormData). - return options.body.pipe(req); - } else if (options.body.getReader) { - // ReadableStream - const nodeBody = webStreams.toNodeReadable(options.body); - return nodeBody.pipe(req); + return request._rawBody.pipe(req); } else { - throw new TypeError('Unexpected Request body type: ' + (typeof options.body)); + // Standard ReadableStream + const nodeBody = webStreams.toNodeReadable(request.body); + return nodeBody.pipe(req); } } req.end(); diff --git a/lib/body.js b/lib/body.js index e56e69465..c35be9af5 100644 --- a/lib/body.js +++ b/lib/body.js @@ -17,38 +17,43 @@ class Body { constructor(body, opts) { opts = opts || {}; this._rawBody = body; - this.body = body; - if (body) { - if (body instanceof webStreams.ReadableStream) { - this.body = body; - } else if (typeof body === 'string' - || Buffer.isBuffer(body) - || isNodeStream(body) && body.readable && !body.getBoundary) { - // Node Readable && not FormData. Convert to ReadableStream. - this.body = webStreams.toWebReadableStream(body); - } else if (body && body.getBoundary) { - // FormData instance. Make an exception, for now. - this.body = body; - } else { - throw new TypeError(`Unsupported Response body type: ${typeof body}`); - } - } - this._decodedBody = null; this.bodyUsed = false; this.size = opts.size || 0; this.timeout = opts.timeout || 0; this.url = opts.url; } + get body() { + const rawBody = this._rawBody; + if (!rawBody) { + return null; + } else if (rawBody instanceof webStreams.ReadableStream) { + return rawBody; + } else if (typeof rawBody === 'string' + || Buffer.isBuffer(rawBody) + || isNodeStream(rawBody) && rawBody.readable && !rawBody.getBoundary) { + // Convert to ReadableStream. + this._rawBody = webStreams.toWebReadableStream(rawBody); + return this._rawBody; + } else if (rawBody && rawBody.getBoundary) { + // FormData instance. Make an exception, for now. + return rawBody; + } else { + throw new TypeError(`Unsupported Response body type: ${typeof body}`); + } + } + + set body(val) { + this._rawBody = val; + } + /** * Decode response as json * * @return Promise */ json() { - - return this._decode().then(text => JSON.parse(text)); - + return this._consumeBody().then(text => JSON.parse(text)); } /** @@ -57,18 +62,11 @@ class Body { * @return Promise */ text() { - if (this.bodyUsed) { - return this._decode(); - } - if (typeof this._rawBody === 'string') { - this.bodyUsed = true; - return Promise.resolve(this._convert(this._rawBody)); - } else if (this._decodedBody) { + if (typeof this._rawBody === 'string' && !this.bodyUsed) { this.bodyUsed = true; - return Promise.resolve(this._convert(this._decodedBody.toString())); - } else { - return this._decode() - .then(body => this._convert(body)); + return Promise.resolve(this._rawBody); + } else { + return this._consumeBody().then(body => body.toString()); } } @@ -79,11 +77,11 @@ class Body { * @return Promise */ blob() { - if (Buffer.isBuffer(this._decodedBody) && !this.bodyUsed) { + if (Buffer.isBuffer(this._rawBody) && !this.bodyUsed) { this.bodyUsed = true; - return Promise.resolve(this._decodedBody); + return Promise.resolve(this._rawBody); } else { - return this._decode(); + return this._consumeBody(); } } @@ -93,7 +91,7 @@ class Body { * @return Promise */ arrayBuffer() { - return this._decode() + return this._consumeBody() // Convert to ArrayBuffer .then(body => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength)); @@ -104,7 +102,7 @@ class Body { * * @return Promise */ - _decode() { + _consumeBody() { const self = this; @@ -112,24 +110,18 @@ class Body { return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); } this.bodyUsed = true; + if (Buffer.isBuffer(this._rawBody)) { + return Promise.resolve(this._rawBody); + } else if (typeof this._rawBody === 'string') { + return Promise.resolve(new Buffer(this._rawBody)); + } + // Get ready to actually consume the body let accum = []; let accumBytes = 0; - return new Body.Promise((resolve, reject) => { let resTimeout; - if (typeof self.body === 'string') { - self._decodedBody = new Buffer(self.body); - return resolve(self._decodedBody); - } - - if (self.body instanceof Buffer) { - self._decodedBody = self.body; - return resolve(self._decodedBody); - } - - const reader = self.body.getReader(); // allow timeout on slow response body @@ -152,8 +144,8 @@ class Body { accum[i] = new Buffer(`${chunk}`); } } - self._decodedBody = Buffer.concat(accum); - return self._decodedBody; + self._rawBody = Buffer.concat(accum); + return self._rawBody; } const chunk = res.value; accum.push(chunk); @@ -176,64 +168,6 @@ class Body { } - /** - * Detect buffer encoding and convert to target encoding - * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding - * - * @param String encoding Target encoding - * @return String - */ - _convert(body, encoding) { - encoding = encoding || 'utf-8'; - let charset = 'utf-8'; - let res, str; - - // header - if (this.headers.has('content-type')) { - res = /charset=([^;]*)/i.exec(this.headers.get('content-type')); - } - - // no charset in content type, peek at response body for at most 1024 bytes - if (!res && body.length > 0) { - str = body.slice(0, 1024).toString(); - } - - // html5 - if (!res && str) { - res = /日本語'); - }); - }); - }); - - it('should support encoding decode, content-type detect', function() { - url = base + '/encoding/shift-jis'; - return fetch(url).then(function(res) { - expect(res.status).to.equal(200); - return res.text().then(function(result) { - expect(result).to.equal('
日本語
'); - }); - }); - }); - - it('should support encoding decode, html5 detect', function() { - url = base + '/encoding/gbk'; - return fetch(url).then(function(res) { - expect(res.status).to.equal(200); - return res.text().then(function(result) { - expect(result).to.equal('
中文
'); - }); - }); - }); - - it('should support encoding decode, html4 detect', function() { - url = base + '/encoding/gb2312'; - return fetch(url).then(function(res) { - expect(res.status).to.equal(200); - return res.text().then(function(result) { - expect(result).to.equal('
中文
'); - }); - }); - }); +// it('should support encoding decode, xml dtd detect', function() { +// url = base + '/encoding/euc-jp'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// return res.text().then(function(result) { +// expect(result).to.equal('日本語'); +// }); +// }); +// }); +// +// it('should support encoding decode, content-type detect', function() { +// url = base + '/encoding/shift-jis'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// return res.text().then(function(result) { +// expect(result).to.equal('
日本語
'); +// }); +// }); +// }); +// +// it('should support encoding decode, html5 detect', function() { +// url = base + '/encoding/gbk'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// return res.text().then(function(result) { +// expect(result).to.equal('
中文
'); +// }); +// }); +// }); +// +// it('should support encoding decode, html4 detect', function() { +// url = base + '/encoding/gb2312'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// return res.text().then(function(result) { +// expect(result).to.equal('
中文
'); +// }); +// }); +// }); it('should default to utf8 encoding', function() { url = base + '/encoding/utf8'; @@ -909,49 +909,49 @@ describe('node-fetch', function() { }); }); - it('should support uncommon content-type order, charset in front', function() { - url = base + '/encoding/order1'; - return fetch(url).then(function(res) { - expect(res.status).to.equal(200); - return res.text().then(function(result) { - expect(result).to.equal('中文'); - }); - }); - }); - - it('should support uncommon content-type order, end with qs', function() { - url = base + '/encoding/order2'; - return fetch(url).then(function(res) { - expect(res.status).to.equal(200); - return res.text().then(function(result) { - expect(result).to.equal('中文'); - }); - }); - }); - - it('should support chunked encoding, html4 detect', function() { - url = base + '/encoding/chunked'; - return fetch(url).then(function(res) { - expect(res.status).to.equal(200); - // because node v0.12 doesn't have str.repeat - var padding = new Array(10 + 1).join('a'); - return res.text().then(function(result) { - expect(result).to.equal(padding + '
日本語
'); - }); - }); - }); - - it('should only do encoding detection up to 1024 bytes', function() { - url = base + '/encoding/invalid'; - return fetch(url).then(function(res) { - expect(res.status).to.equal(200); - // because node v0.12 doesn't have str.repeat - var padding = new Array(1200 + 1).join('a'); - return res.text().then(function(result) { - expect(result).to.not.equal(padding + '中文'); - }); - }); - }); +// it('should support uncommon content-type order, charset in front', function() { +// url = base + '/encoding/order1'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// return res.text().then(function(result) { +// expect(result).to.equal('中文'); +// }); +// }); +// }); +// +// it('should support uncommon content-type order, end with qs', function() { +// url = base + '/encoding/order2'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// return res.text().then(function(result) { +// expect(result).to.equal('中文'); +// }); +// }); +// }); +// +// it('should support chunked encoding, html4 detect', function() { +// url = base + '/encoding/chunked'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// // because node v0.12 doesn't have str.repeat +// var padding = new Array(10 + 1).join('a'); +// return res.text().then(function(result) { +// expect(result).to.equal(padding + '
日本語
'); +// }); +// }); +// }); +// +// it('should only do encoding detection up to 1024 bytes', function() { +// url = base + '/encoding/invalid'; +// return fetch(url).then(function(res) { +// expect(res.status).to.equal(200); +// // because node v0.12 doesn't have str.repeat +// var padding = new Array(1200 + 1).join('a'); +// return res.text().then(function(result) { +// expect(result).to.not.equal(padding + '中文'); +// }); +// }); +// }); it('should allow piping response body as stream', function(done) { url = base + '/hello'; From dfa09f12a69c21e22d410d58ea0ee601023fd503 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Mon, 1 Aug 2016 17:57:04 -0700 Subject: [PATCH 10/13] Migrate index.js to ES6 & make it a plain function - Convert to basic ES6. - Convert `fetch` into a regular function. This is in line with the spec & browser implementations, where calling `fetch` as a constructor is explicitly not supported. (Example: Chrome returns "TypeError: fetch is not a constructor".) Additionally, constructors masquerading as functions is no longer supported in ES6. --- index.js | 433 +++++++++++++++++++++++++++---------------------------- 1 file changed, 216 insertions(+), 217 deletions(-) diff --git a/index.js b/index.js index 5ba22bc5e..67e52ef4f 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,4 @@ +'use strict'; /** * index.js @@ -5,24 +6,20 @@ * a request API compatible with window.fetch */ -var parse_url = require('url').parse; -var resolve_url = require('url').resolve; -var http = require('http'); -var https = require('https'); -var zlib = require('zlib'); -var stream = require('stream'); +const parse_url = require('url').parse; +const resolve_url = require('url').resolve; +const http = require('http'); +const https = require('https'); +const zlib = require('zlib'); +const stream = require('stream'); -var Body = require('./lib/body'); -var Response = require('./lib/response'); -var Headers = require('./lib/headers'); -var Request = require('./lib/request'); -var FetchError = require('./lib/fetch-error'); -var webStreams = require('node-web-streams'); +const Body = require('./lib/body'); +const Response = require('./lib/response'); +const Headers = require('./lib/headers'); +const Request = require('./lib/request'); +const FetchError = require('./lib/fetch-error'); +const webStreams = require('node-web-streams'); -// commonjs -module.exports = Fetch; -// es6 default export compatibility -module.exports.default = module.exports; /** * Fetch class @@ -31,201 +28,196 @@ module.exports.default = module.exports; * @param Object opts Fetch options * @return Promise */ -function Fetch(url, opts) { - - // allow call as function - if (!(this instanceof Fetch)) - return new Fetch(url, opts); - - // allow custom promise - if (!Fetch.Promise) { - throw new Error('native promise missing, set Fetch.Promise to your favorite alternative'); - } - - Body.Promise = Fetch.Promise; - - var self = this; - - // wrap http.request into fetch - return new Fetch.Promise(function(resolve, reject) { - // build request object - const request = new Request(url, opts); - - if (!request.protocol || !request.hostname) { - throw new Error('only absolute urls are supported'); - } - - if (request.protocol !== 'http:' && request.protocol !== 'https:') { - throw new Error('only http(s) protocols are supported'); - } - - var send; - if (request.protocol === 'https:') { - send = https.request; - } else { - send = http.request; - } - - // normalize headers - var headers = new Headers(request.headers); - - if (request.compress) { - headers.set('accept-encoding', 'gzip,deflate'); - } - - if (!headers.has('user-agent')) { - headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); - } - - if (!headers.has('connection') && !request.agent) { - headers.set('connection', 'close'); - } - - if (!headers.has('accept')) { - headers.set('accept', '*/*'); - } - - // detect form data input from form-data module, this hack avoid the need to pass multipart header manually - if (!headers.has('content-type') && request._rawBody - && typeof request._rawBody.getBoundary === 'function') { - headers.set('content-type', 'multipart/form-data; boundary=' + request.body.getBoundary()); - } - - // bring node-fetch closer to browser behavior by setting content-length automatically - if (!headers.has('content-length') && /post|put|patch|delete/i.test(request.method)) { - if (typeof request._rawBody === 'string') { - request._rawBody = new Buffer(request._rawBody); - } - if (Buffer.isBuffer(request._rawBody)) { - headers.set('content-length', request._rawBody.length); - // detect form data input from form-data module, this hack avoid the need to add content-length header manually - } else if (request._rawBody - && typeof request._rawBody.getLengthSync === 'function' - && request._rawBody._lengthRetrievers.length == 0) { - headers.set('content-length', request._rawBody.getLengthSync().toString()); - // this is only necessary for older nodejs releases (before iojs merge) - } else if (request._rawBody === undefined || request._rawBody === null) { - headers.set('content-length', '0'); - } - } - - request.headers = headers.raw(); - - // http.request only support string as host header, this hack make custom host header possible - if (request.headers.host) { - request.headers.host = request.headers.host[0]; - } - - // send request - var req = send(request); - var reqTimeout; - - if (request.timeout) { - req.once('socket', function(socket) { - reqTimeout = setTimeout(function() { - req.abort(); - reject(new FetchError('network timeout at: ' + request.url, 'request-timeout')); - }, request.timeout); - }); - } - - req.on('error', function(err) { - clearTimeout(reqTimeout); - reject(new FetchError('request to ' + request.url + ' failed, reason: ' + err.message, 'system', err)); - }); - - req.on('response', function(res) { - clearTimeout(reqTimeout); - - // handle redirect - if (self.isRedirect(res.statusCode) && request.redirect !== 'manual') { - if (request.redirect === 'error') { - reject(new FetchError('redirect mode is set to error: ' + request.url, 'no-redirect')); - return; - } - - if (request.counter >= request.follow) { - reject(new FetchError('maximum redirect reached at: ' + request.url, 'max-redirect')); - return; - } - - if (!res.headers.location) { - reject(new FetchError('redirect location header missing at: ' + request.url, 'invalid-redirect')); - return; - } - - // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect - if (res.statusCode === 303 - || ((res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST')) - { - request.method = 'GET'; - request.body = undefined; - delete request.headers['content-length']; - } - - request.counter++; - - resolve(Fetch(resolve_url(request.url, res.headers.location), request)); - return; - } - - // handle compression - var body = res; - var headers = new Headers(res.headers); - - if (request.compress && headers.has('content-encoding')) { - var name = headers.get('content-encoding'); - - // no need to pipe no content and not modified response body - if (res.statusCode !== 204 && res.statusCode !== 304) { - if (name == 'gzip' || name == 'x-gzip') { - body = body.pipe(zlib.createGunzip()); - } else if (name == 'deflate' || name == 'x-deflate') { - body = body.pipe(zlib.createInflate()); - } - } - } - - // Convert to ReadableStream - body = webStreams.toWebReadableStream(body); - - // normalize location header for manual redirect mode - if (request.redirect === 'manual' && headers.has('location')) { - headers.set('location', resolve_url(request.url, headers.get('location'))); - } - - // response object - var output = new Response(body, { - url: request.url - , status: res.statusCode - , statusText: res.statusMessage - , headers: headers - , size: request.size - , timeout: request.timeout - }); - - resolve(output); - }); - - // Request body handling - if (request._rawBody !== undefined && request._rawBody !== null) { - if (Buffer.isBuffer(request._rawBody)) { - // Fast path for simple buffers. Avoid stream wrapper & - // chunked encoding. - return req.end(request._rawBody); - } else if (request._rawBody.pipe) { - // Node stream (likely FormData). - return request._rawBody.pipe(req); - } else { - // Standard ReadableStream - const nodeBody = webStreams.toNodeReadable(request.body); - return nodeBody.pipe(req); - } - } - req.end(); - }); +function fetch(url, opts) { + + // allow custom promise + if (!fetch.Promise) { + throw new Error('native promise missing, set Fetch.Promise to your favorite alternative'); + } + + Body.Promise = fetch.Promise; + + + // wrap http.request into fetch + return new fetch.Promise((resolve, reject) => { + // build request object + const request = new Request(url, opts); + + if (!request.protocol || !request.hostname) { + throw new Error('only absolute urls are supported'); + } + + if (request.protocol !== 'http:' && request.protocol !== 'https:') { + throw new Error('only http(s) protocols are supported'); + } + + let send; + if (request.protocol === 'https:') { + send = https.request; + } else { + send = http.request; + } + + // normalize headers + const headers = new Headers(request.headers); + + if (request.compress) { + headers.set('accept-encoding', 'gzip,deflate'); + } + + if (!headers.has('user-agent')) { + headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); + } + + if (!headers.has('connection') && !request.agent) { + headers.set('connection', 'close'); + } + + if (!headers.has('accept')) { + headers.set('accept', '*/*'); + } + + // detect form data input from form-data module, this hack avoid the need to pass multipart header manually + if (!headers.has('content-type') && request._rawBody + && typeof request._rawBody.getBoundary === 'function') { + headers.set('content-type', `multipart/form-data; boundary=${request.body.getBoundary()}`); + } + + // bring node-fetch closer to browser behavior by setting content-length automatically + if (!headers.has('content-length') && /post|put|patch|delete/i.test(request.method)) { + if (typeof request._rawBody === 'string') { + request._rawBody = new Buffer(request._rawBody); + } + if (Buffer.isBuffer(request._rawBody)) { + headers.set('content-length', request._rawBody.length); + // detect form data input from form-data module, this hack avoid the need to add content-length header manually + } else if (request._rawBody + && typeof request._rawBody.getLengthSync === 'function' + && request._rawBody._lengthRetrievers.length === 0) { + headers.set('content-length', request._rawBody.getLengthSync().toString()); + // this is only necessary for older nodejs releases (before iojs merge) + } else if (request._rawBody === undefined || request._rawBody === null) { + headers.set('content-length', '0'); + } + } + + request.headers = headers.raw(); + + // http.request only support string as host header, this hack make custom host header possible + if (request.headers.host) { + request.headers.host = request.headers.host[0]; + } + + // send request + const req = send(request); + let reqTimeout; + + if (request.timeout) { + req.once('socket', socket => { + reqTimeout = setTimeout(() => { + req.abort(); + reject(new FetchError(`network timeout at: ${request.url}`, 'request-timeout')); + }, request.timeout); + }); + } + + req.on('error', err => { + clearTimeout(reqTimeout); + reject(new FetchError(`request to ${request.url} failed, reason: ${err.message}`, 'system', err)); + }); + + req.on('response', res => { + clearTimeout(reqTimeout); + + // handle redirect + if (fetch.isRedirect(res.statusCode) && request.redirect !== 'manual') { + if (request.redirect === 'error') { + reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); + return; + } + + if (request.counter >= request.follow) { + reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); + return; + } + + if (!res.headers.location) { + reject(new FetchError(`redirect location header missing at: ${request.url}`, 'invalid-redirect')); + return; + } + + // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect + if (res.statusCode === 303 + || ((res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST')) + { + request.method = 'GET'; + request.body = undefined; + delete request.headers['content-length']; + } + + request.counter++; + + resolve(fetch(resolve_url(request.url, res.headers.location), request)); + return; + } + + // handle compression + let body = res; + const headers = new Headers(res.headers); + + if (request.compress && headers.has('content-encoding')) { + const name = headers.get('content-encoding'); + + // no need to pipe no content and not modified response body + if (res.statusCode !== 204 && res.statusCode !== 304) { + if (name === 'gzip' || name === 'x-gzip') { + body = body.pipe(zlib.createGunzip()); + } else if (name === 'deflate' || name === 'x-deflate') { + body = body.pipe(zlib.createInflate()); + } + } + } + + // Convert to ReadableStream + body = webStreams.toWebReadableStream(body); + + // normalize location header for manual redirect mode + if (request.redirect === 'manual' && headers.has('location')) { + headers.set('location', resolve_url(request.url, headers.get('location'))); + } + + // response object + const output = new Response(body, { + url: request.url, + status: res.statusCode, + statusText: res.statusMessage, + headers: headers, + size: request.size, + timeout: request.timeout + }); + + resolve(output); + }); + + // Request body handling + if (request._rawBody !== undefined && request._rawBody !== null) { + if (Buffer.isBuffer(request._rawBody)) { + // Fast path for simple buffers. Avoid stream wrapper & + // chunked encoding. + return req.end(request._rawBody); + } else if (request._rawBody.pipe) { + // Node stream (likely FormData). + return request._rawBody.pipe(req); + } else { + // Standard ReadableStream + const nodeBody = webStreams.toNodeReadable(request.body); + return nodeBody.pipe(req); + } + } + req.end(); + }); -}; +} /** * Redirect code matching @@ -233,12 +225,19 @@ function Fetch(url, opts) { * @param Number code Status code * @return Boolean */ -Fetch.prototype.isRedirect = function(code) { - return code === 301 || code === 302 || code === 303 || code === 307 || code === 308; -} +fetch.isRedirect = function isRedirect(code) { + return code === 301 || code === 302 || code === 303 || code === 307 || code === 308; +}; + // expose Promise -Fetch.Promise = global.Promise; -Fetch.Response = Response; -Fetch.Headers = Headers; -Fetch.Request = Request; +fetch.Promise = global.Promise; +fetch.Response = Response; +fetch.Headers = Headers; +fetch.Request = Request; + + +// commonjs +module.exports = fetch; +// es6 default export compatibility +module.exports.default = module.exports; From 54552da9b0e29b90850c08edf31ebcbbd83a424a Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Mon, 1 Aug 2016 19:24:15 -0700 Subject: [PATCH 11/13] Add a basic .jshintrc --- .jshintrc | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .jshintrc diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 000000000..daaeb3daa --- /dev/null +++ b/.jshintrc @@ -0,0 +1,31 @@ +{ + "predef": [ + "setImmediate", + "Map", + "Set" + ], + + "bitwise": true, + "laxbreak": true, + "curly": true, + "eqeqeq": true, + "immed": true, + "latedef": "nofunc", + "newcap": true, + "noarg": true, + "noempty": true, + "nonew": true, + "regexp": false, + "undef": true, + "strict": true, + "trailing": true, + + "smarttabs": true, + "multistr": true, + + "node": true, + + "nomen": false, + "loopfunc": true, + "esnext": true +} From f11ad2144dee73c981466ee3e2a54ba6c7fada54 Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Mon, 1 Aug 2016 20:29:36 -0700 Subject: [PATCH 12/13] De-tabify body.js --- lib/body.js | 64 ++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/lib/body.js b/lib/body.js index c35be9af5..2ed1a76f4 100644 --- a/lib/body.js +++ b/lib/body.js @@ -15,37 +15,37 @@ const webStreams = require('node-web-streams'); */ class Body { constructor(body, opts) { - opts = opts || {}; + opts = opts || {}; this._rawBody = body; this.bodyUsed = false; this.size = opts.size || 0; this.timeout = opts.timeout || 0; - this.url = opts.url; + this.url = opts.url; } - get body() { - const rawBody = this._rawBody; - if (!rawBody) { - return null; - } else if (rawBody instanceof webStreams.ReadableStream) { - return rawBody; - } else if (typeof rawBody === 'string' - || Buffer.isBuffer(rawBody) - || isNodeStream(rawBody) && rawBody.readable && !rawBody.getBoundary) { - // Convert to ReadableStream. - this._rawBody = webStreams.toWebReadableStream(rawBody); - return this._rawBody; - } else if (rawBody && rawBody.getBoundary) { - // FormData instance. Make an exception, for now. - return rawBody; - } else { - throw new TypeError(`Unsupported Response body type: ${typeof body}`); - } - } - - set body(val) { - this._rawBody = val; - } + get body() { + const rawBody = this._rawBody; + if (!rawBody) { + return null; + } else if (rawBody instanceof webStreams.ReadableStream) { + return rawBody; + } else if (typeof rawBody === 'string' + || Buffer.isBuffer(rawBody) + || isNodeStream(rawBody) && rawBody.readable && !rawBody.getBoundary) { + // Convert to ReadableStream. + this._rawBody = webStreams.toWebReadableStream(rawBody); + return this._rawBody; + } else if (rawBody && rawBody.getBoundary) { + // FormData instance. Make an exception, for now. + return rawBody; + } else { + throw new TypeError(`Unsupported Response body type: ${typeof body}`); + } + } + + set body(val) { + this._rawBody = val; + } /** * Decode response as json @@ -65,7 +65,7 @@ class Body { if (typeof this._rawBody === 'string' && !this.bodyUsed) { this.bodyUsed = true; return Promise.resolve(this._rawBody); - } else { + } else { return this._consumeBody().then(body => body.toString()); } } @@ -110,13 +110,13 @@ class Body { return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); } this.bodyUsed = true; - if (Buffer.isBuffer(this._rawBody)) { - return Promise.resolve(this._rawBody); - } else if (typeof this._rawBody === 'string') { - return Promise.resolve(new Buffer(this._rawBody)); - } + if (Buffer.isBuffer(this._rawBody)) { + return Promise.resolve(this._rawBody); + } else if (typeof this._rawBody === 'string') { + return Promise.resolve(new Buffer(this._rawBody)); + } - // Get ready to actually consume the body + // Get ready to actually consume the body let accum = []; let accumBytes = 0; return new Body.Promise((resolve, reject) => { From d459135d82a09b8e5d8ccfba4136ba92c850d57d Mon Sep 17 00:00:00 2001 From: Gabriel Wicke Date: Tue, 2 Aug 2016 21:27:50 -0700 Subject: [PATCH 13/13] Update README to reflect changes --- README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0176c4240..a36edb6da 100644 --- a/README.md +++ b/README.md @@ -11,20 +11,28 @@ A light-weight module that brings `window.fetch` to Node.js # Motivation -I really like the notion of Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch): it bridges the API gap between client-side and server-side http requests, so developers have less to worry about. +I really like the notion of Matt Andrews' +[isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch): it +bridges the API gap between client-side and server-side http requests, so +developers have less to worry about. -Instead of implementing `XMLHttpRequest` in node to run browser-specific [fetch polyfill](https://github.com/github/fetch), why not go from node's `http` to `fetch` API directly? Node has native stream support, browserify build targets (browsers) don't, so underneath they are going to be vastly different anyway. +Instead of implementing `XMLHttpRequest` in node to run browser-specific +[fetch polyfill](https://github.com/github/fetch), why not go from node's +`http` to `fetch` API directly? Node has native stream support, browserify +build targets (browsers) don't, so underneath they are going to be vastly +different anyway. -Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. +Hence `node-fetch`, minimal code for a `window.fetch` compatible API on +Node.js runtime. # Features - Stay consistent with `window.fetch` API. - Make conscious trade-off when following [whatwg fetch spec](https://fetch.spec.whatwg.org/) and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known difference. -- Use native promise, but allow substituting it with [insert your favorite promise library]. -- Use native stream for body, on both request and response. -- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to utf-8 automatically. +- Use native promise, but allow substituting it with [insert your favorite +- promise library]. +- Use WhatWG `ReadableStream` for streaming bodies, on both request and response. - Useful extensions such as timeout, redirect limit, response size limit, explicit reject errors.