From aec0333a9198cb67aaab5144f6e98cc04db0a40a Mon Sep 17 00:00:00 2001 From: Marcelo Gornstein Date: Tue, 2 Sep 2014 13:05:14 -0300 Subject: [PATCH 1/2] This will add the capability for handling gzipped content to the http connector. On one hand, the response is handled with buffers instead of assuming it is a string, and then depending on the presence of the "content-encoding" header, will try to uncompress the response or return it as it is. Either way, it will try to handle the final result as an utf8 encoded string --- src/lib/connectors/http.js | 24 +++++++++++++---- test/unit/specs/http_connector.js | 44 +++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/src/lib/connectors/http.js b/src/lib/connectors/http.js index 3bb0d98ea..46badfd70 100644 --- a/src/lib/connectors/http.js +++ b/src/lib/connectors/http.js @@ -16,6 +16,7 @@ var _ = require('../utils'); var qs = require('querystring'); var ForeverAgent = require('./_custom_agent'); var ConnectionAbstract = require('../connection'); +var zlib = require('zlib'); /** * Connector used to talk to an elasticsearch node via HTTP @@ -123,8 +124,9 @@ HttpConnector.prototype.request = function (params, cb) { var request; var response; var status = 0; - var headers; + var headers = {}; var log = this.log; + var buffers = []; var reqParams = this.makeReqParams(params); @@ -144,7 +146,20 @@ HttpConnector.prototype.request = function (params, cb) { if (err) { cb(err); } else { - cb(err, response, status, headers); + response = Buffer.concat(buffers); + if (headers['content-encoding'] && headers['content-encoding'].match(/gzip/i)) { + zlib.unzip(response, function(gzErr, uncompressedResponse) { + if(gzErr) { + err = gzErr; + response = response.toString('binary'); + } else { + response = uncompressedResponse.toString('utf8'); + } + cb(err, response, status, headers); + }); + } else { + cb(err, response.toString('utf8'), status, headers); + } } }, this); @@ -152,11 +167,10 @@ HttpConnector.prototype.request = function (params, cb) { incoming = _incoming; status = incoming.statusCode; headers = incoming.headers; - incoming.setEncoding('utf8'); response = ''; - + buffers = []; incoming.on('data', function (d) { - response += d; + buffers.push(new Buffer(d)); }); incoming.on('error', cleanUp); diff --git a/test/unit/specs/http_connector.js b/test/unit/specs/http_connector.js index 63d7f8839..5c629c6b5 100644 --- a/test/unit/specs/http_connector.js +++ b/test/unit/specs/http_connector.js @@ -17,6 +17,8 @@ describe('Http Connector', function () { var expectSubObject = require('../../utils/expect_sub_object'); var MockRequest = require('../../mocks/request'); var MockIncommingMessage = require('../../mocks/incomming_message'); + var zlib = require('zlib'); + var estr = require('event-stream'); nock.disableNetConnect(); @@ -302,6 +304,48 @@ describe('Http Connector', function () { }); }); + it('collects the whole request body (compressed)', function (done) { + var server = nock('http://esjs.com:9200'); + var con = new HttpConnection(new Host('http://esjs.com:9200')); + var body = '{ "USER": "doc" }'; + zlib.deflate(body, function(err, compressedBody) { + server + .get('/users/1') + .reply(200, compressedBody, {'Content-Encoding': 'gzip'}); + + con.request({ + method: 'GET', + path: '/users/1' + }, function (err, resp, status) { + expect(err).to.be(undefined); + expect(resp).to.eql(body); + expect(status).to.eql(200); + server.done(); + done(); + }); + }); + }); + + it('Can handle uncompress errors', function (done) { + var server = nock('http://esjs.com:9200'); + var con = new HttpConnection(new Host('http://esjs.com:9200')); + var body = 'blah'; + server + .get('/users/1') + .reply(200, body, {'Content-Encoding': 'gzip'}); + + con.request({ + method: 'GET', + path: '/users/1' + }, function (err, resp, status) { + expect(err.errno).to.be(-3); + expect(resp).to.eql(body); + expect(status).to.eql(200); + server.done(); + done(); + }); + }); + it('Ignores serialization errors', function (done) { var server = nock('http://esjs.com:9200'); var con = new HttpConnection(new Host('http://esjs.com:9200')); From 83b6400000b7a9ed7895af7bec353df1f9bac64d Mon Sep 17 00:00:00 2001 From: Marcelo Gornstein Date: Thu, 4 Sep 2014 18:14:42 -0300 Subject: [PATCH 2/2] adding check and test for deflate algo --- src/lib/connectors/http.js | 3 ++- test/unit/specs/http_connector.js | 36 ++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/lib/connectors/http.js b/src/lib/connectors/http.js index 46badfd70..8014f3bcc 100644 --- a/src/lib/connectors/http.js +++ b/src/lib/connectors/http.js @@ -147,7 +147,8 @@ HttpConnector.prototype.request = function (params, cb) { cb(err); } else { response = Buffer.concat(buffers); - if (headers['content-encoding'] && headers['content-encoding'].match(/gzip/i)) { + var zipHdr = headers['content-encoding']; + if (zipHdr && (zipHdr.match(/gzip/i) || zipHdr.match(/deflate/i))) { zlib.unzip(response, function(gzErr, uncompressedResponse) { if(gzErr) { err = gzErr; diff --git a/test/unit/specs/http_connector.js b/test/unit/specs/http_connector.js index 5c629c6b5..84ac67cc8 100644 --- a/test/unit/specs/http_connector.js +++ b/test/unit/specs/http_connector.js @@ -304,11 +304,15 @@ describe('Http Connector', function () { }); }); - it('collects the whole request body (compressed)', function (done) { + it('collects the whole request body (gzip compressed)', function (done) { var server = nock('http://esjs.com:9200'); var con = new HttpConnection(new Host('http://esjs.com:9200')); - var body = '{ "USER": "doc" }'; - zlib.deflate(body, function(err, compressedBody) { + var elements = []; + for(var i = 0; i < 500; i++) { + elements.push({ "USER": "doc" }); + } + var body = JSON.stringify(elements); + zlib.gzip(body, function(err, compressedBody) { server .get('/users/1') .reply(200, compressedBody, {'Content-Encoding': 'gzip'}); @@ -326,6 +330,32 @@ describe('Http Connector', function () { }); }); + it('collects the whole request body (deflate compressed)', function (done) { + var server = nock('http://esjs.com:9200'); + var con = new HttpConnection(new Host('http://esjs.com:9200')); + var elements = []; + for(var i = 0; i < 500; i++) { + elements.push({ "USER": "doc" }); + } + var body = JSON.stringify(elements); + zlib.deflate(body, function(err, compressedBody) { + server + .get('/users/1') + .reply(200, compressedBody, {'Content-Encoding': 'deflate'}); + + con.request({ + method: 'GET', + path: '/users/1' + }, function (err, resp, status) { + expect(err).to.be(undefined); + expect(resp).to.eql(body); + expect(status).to.eql(200); + server.done(); + done(); + }); + }); + }); + it('Can handle uncompress errors', function (done) { var server = nock('http://esjs.com:9200'); var con = new HttpConnection(new Host('http://esjs.com:9200'));