Merge pull request #192 from hapijs/gunzip

Add a gunzipping option
hapijs · Sep 1, 2017 · 12a42b6 · 12a42b6
2 parents 2e845a3 + 1d39aff
commit 12a42b6
Show file tree

Hide file tree

Showing 3 changed files with 282 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -141,11 +141,20 @@ Returns an instance of the node.js [ClientRequest](http://nodejs.org/api/http.ht
         - `true`, 'smart' - only try `JSON.parse` if the response indicates a JSON content-type.
         - `strict` - as 'smart', except returns an error for non-JSON content-type.
         - `force` - try `JSON.parse` regardless of the content-type header.
+    - `gunzip` - A value indicating the behavior to adopt when the payload is gzipped. Defaults to `undefined` meaning no gunzipping.
+        - `true` - only try to gunzip if the response indicates a gzip content-encoding.
+        - `false` - explicitly disable gunzipping.
+        - `force` - try to gunzip regardless of the content-encoding header.
     - `maxBytes` - The maximum allowed response payload size. Defaults to unlimited.
 - `callback` - The callback function using the signature `function (err, payload)` where:
     - `err` - Any error that may have occurred while reading the response.
     - `payload` - The payload in the form of a Buffer or (optionally) parsed JavaScript object (JSON).
 
+#### Notes about gunzip
+
+When using gunzip, HTTP headers `Content-Encoding`, `Content-Length`, `Content-Range` and `ETag` won't reflect the reality as the payload has been uncompressed.
+
+Node v4 does not detect premature ending of gzipped content, if the payload is partial, you will not get an error on this specific version of node.js.
 
 ### `get(uri, [options], callback)`
 

diff --git a/lib/index.js b/lib/index.js
@@ -7,6 +7,7 @@ const Url = require('url');
 const Http = require('http');
 const Https = require('https');
 const Stream = require('stream');
+const Zlib = require('zlib');
 const Hoek = require('hoek');
 const Boom = require('boom');
 const Payload = require('./payload');
@@ -102,6 +103,9 @@ internals.Client.prototype.request = function (method, url, options, callback, _
     Hoek.assert(options.redirected === undefined || options.redirected === null || typeof options.redirected === 'function',
         'options.redirected must be a function');
 
+    Hoek.assert(options.gunzip === undefined || typeof options.gunzip === 'boolean' || options.gunzip === 'force',
+        'options.gunzip must be a boolean or "force"');
+
     options.beforeRedirect = options.beforeRedirect || ((redirectMethod, statusCode, location, resHeaders, redirectOptions, next) => next());
 
     if (options.baseUrl) {
@@ -127,6 +131,10 @@ internals.Client.prototype.request = function (method, url, options, callback, _
         }
     }
 
+    if (options.gunzip && internals.findHeader('accept-encoding', uri.headers) === undefined) {
+        uri.headers['accept-encoding'] = 'gzip';
+    }
+
     const payloadSupported = (uri.method !== 'GET' && uri.method !== 'HEAD' && options.payload !== null && options.payload !== undefined);
     if (payloadSupported &&
         (typeof options.payload === 'string' || Buffer.isBuffer(options.payload)) &&
@@ -425,6 +433,21 @@ internals.Client.prototype.read = function (res, options, callback) {
 
     reader.once('finish', onReaderFinish);
 
+    if (options.gunzip) {
+        const contentEncoding = options.gunzip === 'force' ?
+            'gzip' :
+            (res.headers && internals.findHeader('content-encoding', res.headers)) || '';
+
+        if (/^(x-)?gzip(\s*,\s*identity)?$/.test(contentEncoding)) {
+            const gunzip = Zlib.createGunzip();
+
+            gunzip.once('error', onReaderError);
+
+            res.pipe(gunzip).pipe(reader);
+            return;
+        }
+    }
+
     res.pipe(reader);
 };
 
@@ -555,6 +578,7 @@ internals.tryParseBuffer = function (buffer) {
     }
     catch (err) {
         result.err = err;
+        result.json = buffer;
     }
     return result;
 };

diff --git a/test/index.js b/test/index.js
@@ -8,6 +8,7 @@ const Path = require('path');
 const Fs = require('fs');
 const Events = require('events');
 const Stream = require('stream');
+const Zlib = require('zlib');
 const Hoek = require('hoek');
 const Lab = require('lab');
 const Reload = require('require-reload');
@@ -17,8 +18,10 @@ const Wreck = require('../');
 // Declare internals
 
 const internals = {
+    isv4: /^v4/.test(process.version),
     reload: Reload(require),
     payload: new Array(1640).join('0123456789'), // make sure we have a payload larger than 16384 bytes for chunking coverage
+    gzippedPayload: Zlib.gzipSync(new Array(1640).join('0123456789')),
     socket: __dirname + '/server.sock',
     emitSymbol: Symbol.for('wreck')
 };
@@ -1850,6 +1853,16 @@ describe('read()', () => {
         });
     });
 
+    it('handles responses with no headers (with gunzip)', (done) => {
+
+        const res = Wreck.toReadableStream(internals.gzippedPayload);
+        Wreck.read(res, { json: true, gunzip: true }, (err) => {
+
+            expect(err).to.equal(null);
+            done();
+        });
+    });
+
     it('skips destroy when not available', (done) => {
 
         const server = Http.createServer((req, res) => {
@@ -2371,6 +2384,242 @@ describe('json', () => {
     });
 });
 
+describe('gunzip', () => {
+
+    describe('true', () => {
+
+        it('automatically handles gzip', (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.equal('gzip');
+                res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' });
+                res.end(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    json: true,
+                    gunzip: true
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.not.exist();
+                    expect(res.statusCode).to.equal(200);
+                    expect(payload).to.not.equal(null);
+                    expect(payload.foo).to.exist();
+                    server.close();
+                    done();
+                });
+            });
+        });
+
+        it('automatically handles gzip (with identity)', (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.equal('gzip');
+                res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip, identity' });
+                res.end(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    json: true,
+                    gunzip: true
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.not.exist();
+                    expect(res.statusCode).to.equal(200);
+                    expect(payload).to.not.equal(null);
+                    expect(payload.foo).to.exist();
+                    server.close();
+                    done();
+                });
+            });
+        });
+
+        it('automatically handles gzip (without json)', (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.equal('gzip');
+                res.writeHead(200, { 'Content-Encoding': 'gzip' });
+                res.end(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    gunzip: true
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.not.exist();
+                    expect(res.statusCode).to.equal(200);
+                    expect(payload.toString()).to.equal('{"foo":"bar"}');
+                    server.close();
+                    done();
+                });
+            });
+        });
+
+        it('automatically handles gzip (ignores when not gzipped)', (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.equal('gzip');
+                res.writeHead(200, { 'Content-Type': 'application/json' });
+                res.end(JSON.stringify({ foo: 'bar' }));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    json: true,
+                    gunzip: true
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.not.exist();
+                    expect(res.statusCode).to.equal(200);
+                    expect(payload).to.not.equal(null);
+                    expect(payload.foo).to.exist();
+                    server.close();
+                    done();
+                });
+            });
+        });
+
+        it('handles gzip errors', { skip: internals.isv4 }, (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.equal('gzip');
+                res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' });
+                res.end(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })).slice(0, 10));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    json: true,
+                    gunzip: true
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.be.an.error(internals.isv4 ? 'Unexpected end of input' : 'unexpected end of file');
+                    expect(res.statusCode).to.equal(200);
+                    server.close();
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('false/undefined', () => {
+
+        it('fails parsing gzipped content', (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.not.exist();
+                res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' });
+                res.end(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    json: true
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.be.an.error(internals.isv4 ? 'Unexpected token \u001f' : 'Unexpected token \u001f in JSON at position 0');
+                    expect(res.statusCode).to.equal(200);
+                    expect(payload).to.equal(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })));
+                    server.close();
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('force', () => {
+
+        it('forcefully handles gzip', (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.equal('gzip');
+                res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' });
+                res.end(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    json: true,
+                    gunzip: 'force'
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.not.exist();
+                    expect(res.statusCode).to.equal(200);
+                    expect(payload).to.not.equal(null);
+                    expect(payload.foo).to.exist();
+                    server.close();
+                    done();
+                });
+            });
+        });
+
+        it('handles gzip errors', { skip: internals.isv4 }, (done) => {
+
+            const server = Http.createServer((req, res) => {
+
+                expect(req.headers['accept-encoding']).to.equal('gzip');
+                res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' });
+                res.end(Zlib.gzipSync(JSON.stringify({ foo: 'bar' })).slice(0, 10));
+            });
+
+            server.listen(0, () => {
+
+                const port = server.address().port;
+                const options = {
+                    json: true,
+                    gunzip: 'force'
+                };
+
+                Wreck.get('http://localhost:' + port, options, (err, res, payload) => {
+
+                    expect(err).to.be.an.error(internals.isv4 ? 'Unexpected end of input' : 'unexpected end of file');
+                    expect(res.statusCode).to.equal(200);
+                    server.close();
+                    done();
+                });
+            });
+        });
+    });
+});
+
 describe('toReadableStream()', () => {
 
     it('handle empty payload', (done) => {