Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Support `options.gzip = true` to handle gzip response. fixed #26

* If the options.gzip = true, urllib will auto add
  options.headers["Accept-Encoding"] = 'gzip'
  and auto ungzip the gzip response content.
* User also can disable gzip with options.gzip = false
  or ignore this argument.
  • Loading branch information...
commit 5b6fa22f311f83acdee6e9ea8510d25528f64dea 1 parent b0028a6
@fengmk2 fengmk2 authored
View
3  .gitignore
@@ -1,5 +1,4 @@
.project
.settings
node_modules/
-coverage.html
-lib-cov
+coverage/
View
3  .npmignore
@@ -4,7 +4,6 @@ logo.png
examples/
test/
Makefile
-lib-cov/
-coverage.html
.jshintrc
.jshintignore
+coverage/
View
2  .travis.yml
@@ -3,4 +3,4 @@ node_js:
- '0.8'
- '0.10'
- '0.11'
-script: make test-coveralls
+script: make test-all
View
27 Makefile
@@ -10,27 +10,36 @@ jshint: install
@./node_modules/.bin/jshint .
test: install
- @NODE_ENV=test ./node_modules/mocha/bin/mocha \
+ @NODE_ENV=test ./node_modules/.bin/mocha \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
$(MOCHA_OPTS) \
$(TESTS)
test-cov:
- @$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=travis-cov
+ @NODE_ENV=test node --harmony \
+ node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha \
+ -- -u exports \
+ --reporter $(REPORTER) \
+ --timeout $(TIMEOUT) \
+ $(MOCHA_OPTS) \
+ $(TESTS)
+ @-$(MAKE) check-coverage
-test-cov-html:
- @rm -f coverage.html
- @$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=html-cov > coverage.html
- @ls -lh coverage.html
+check-coverage:
+ @./node_modules/.bin/istanbul check-coverage \
+ --statements 96 \
+ --branches 93 \
+ --functions 94 \
+ --lines 96
-test-coveralls: jshint test
- @echo TRAVIS_JOB_ID $(TRAVIS_JOB_ID)
- @-$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=mocha-lcov-reporter | ./node_modules/coveralls/bin/coveralls.js
+cov:
+ @./node_modules/.bin/cov coverage
test-all: jshint test test-cov
autod: install
@./node_modules/.bin/autod -w
+ @$(MAKE) install
.PHONY: test
View
5 README.md
@@ -1,7 +1,6 @@
# urllib
[![Build Status](https://secure.travis-ci.org/fengmk2/urllib.png?branch=master)](http://travis-ci.org/fengmk2/urllib)
-[![Coverage Status](https://coveralls.io/repos/fengmk2/urllib/badge.png)](https://coveralls.io/r/fengmk2/urllib)
[![NPM](https://nodei.co/npm/urllib.png?downloads=true&stars=true)](https://nodei.co/npm/urllib/)
@@ -10,6 +9,8 @@
Help in opening URLs (mostly HTTP) in a complex world — basic
and digest authentication, redirections, cookies, timeout and more.
+[Code coverage: 94%](http://qtestbucket.qiniudn.com/cov/html/urllib/0.5.8/index.html)
+
## Install
```bash
@@ -69,6 +70,7 @@ urllib.request('http://cnodejs.org/', { wd: 'nodejs' }, function (err, data, res
- ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
- ***maxRedirects*** Number - The maximum number of redirects to follow, defaults to 10.
- ***beforeRequest*** Function - Before request hook, you can change every thing here.
+ - ***gzip*** Boolean - Accept gzip response content and auto decode it, default is `false`.
- ***callback(err, data, res)*** Function - Optional callback.
- **err** Error - Would be `null` if no error accured.
- **data** Buffer | Object - The data responsed. Would be a Buffer if `dataType` is set to `text` or an JSON parsed into Object if it's set to `json`.
@@ -209,6 +211,7 @@ urllib.request('http://127.0.0.1:1984/socket.end', function (err, data, res) {
* [√] Auto redirect handle
* [√] https & self-signed certificate
* [√] Connection timeout & Response timeout
+* [√] Support `Accept-Encoding=gzip` by `options.gzip = true`
## Authors
View
63 lib/urllib.js
@@ -1,8 +1,11 @@
/**!
* urllib - lib/urllib.js
*
- * Copyright(c) 2011 - 2014 fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ * Copyright(c) 2011 - 2014 fengmk2 and other contributors.
* MIT Licensed
+ *
+ * Authors:
+ * fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
"use strict";
@@ -17,6 +20,7 @@ var urlutil = require('url');
var qs = require('querystring');
var path = require('path');
var fs = require('fs');
+var zlib = require('zlib');
var debug = require('debug')('urllib');
var pkg = JSON.parse(fs.readFileSync(path.join(__dirname, '../', 'package.json')));
@@ -81,6 +85,7 @@ var REQUEST_ID = 0;
* - {Boolean} [followRedirect]: Follow HTTP 3xx responses as redirects. defaults to false.
* - {Number} [maxRedirects]: The maximum number of redirects to follow, defaults to 10.
* - {Function(options)} [beforeRequest]: Before request hook, you can change every thing here.
+ * - {Boolean} [gzip]: Accept gzip response content and auto decode it, default is `false`.
* @param {Function} callback, callback(error, data, res)
* @return {HttpRequest} req object.
* @api public
@@ -248,6 +253,30 @@ exports.request = function (url, args, callback) {
options.headers['User-Agent'] = USER_AGENT;
}
+ var enableGzip = args.gzip === true;
+ if (enableGzip) {
+ var acceptEncoding = options.headers['Accept-Encoding'] || options.headers['accept-encoding'];
+ if (acceptEncoding) {
+ enableGzip = false; // user want to handle response content decode themself
+ } else {
+ options.headers['Accept-Encoding'] = 'gzip';
+ }
+ }
+
+ var decodeContent = function (res, body, cb) {
+ if (body.length === 0 || !enableGzip) {
+ return cb(null, body);
+ }
+
+ var encoding = res.headers['content-encoding'];
+ if (!encoding || encoding.toLowerCase() !== 'gzip') {
+ return cb(null, body);
+ }
+
+ debug('gunzip %d length body', body.length);
+ zlib.gunzip(body, cb);
+ };
+
var writeStream = args.writeStream;
var reqId = ++REQUEST_ID;
@@ -322,26 +351,30 @@ exports.request = function (url, args, callback) {
});
res.on('end', function () {
- var data = Buffer.concat(chunks, size);
+ var body = Buffer.concat(chunks, size);
debug('Request#%d %s: `res end` event emit, total size %d, _dumped: %s', reqId, options.path, size, res._dumped);
if (__err && connnected) {
// req.abort() after `res data` event emit.
- return done(__err, data, res);
+ return done(__err, body, res);
}
var result = handleRedirect(res);
- var err = result.error;
+ if (result.error) {
+ return done(result.error, body, res);
+ }
if (result.redirect) {
return;
}
- if (args.dataType === 'json') {
- if (size === 0) {
- data = null;
- } else {
+ decodeContent(res, body, function (err, data) {
+ if (err) {
+ return done(err, body, res);
+ }
+
+ if (args.dataType === 'json') {
try {
- data = JSON.parse(data);
+ data = size > 0 ? JSON.parse(data) : null;
} catch (e) {
if (e.name === 'SyntaxError') {
e.name = 'JSONResponseFormatError';
@@ -349,14 +382,14 @@ exports.request = function (url, args, callback) {
err = e;
}
}
- }
- if (res.aborted) {
- err = new Error('Remote socket was terminated before `response.end()` was called');
- err.name = 'RemoteSocketClosedError';
- }
+ if (res.aborted) {
+ err = new Error('Remote socket was terminated before `response.end()` was called');
+ err.name = 'RemoteSocketClosedError';
+ }
- done(err, data, res);
+ done(err, data, res);
+ });
});
});
View
16 package.json
@@ -3,7 +3,7 @@
"version": "0.5.8",
"description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
"keywords": [ "urllib", "http", "urlopen", "curl", "wget", "request", "https" ],
- "author": "fengmk2 <fengmk2@gmail.com> (http://github.com/fengmk2)",
+ "author": "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",
"homepage": "http://github.com/fengmk2/urllib",
"main": "index.js",
"repository": {
@@ -14,12 +14,6 @@
"test": "make test-all"
},
"config": {
- "blanket": {
- "pattern": "urllib/lib",
- "data-cover-flags": {
- "debug": false
- }
- },
"travis-cov": { "threshold": 95 }
},
"dependencies": {
@@ -28,15 +22,13 @@
"devDependencies": {
"agentkeepalive": "0.2.2",
"autod": "*",
- "blanket": "*",
- "coveralls": "*",
+ "cov": "*",
"formstream": "0.0.8",
+ "istanbul": "*",
"jshint": "*",
"mocha": ">=0.14.1",
- "mocha-lcov-reporter": "*",
"pedding": "0.0.3",
- "should": "3.1.3",
- "travis-cov": "*"
+ "should": "3.1.3"
},
"engines": { "node": ">= 0.8.0" },
"license": "MIT"
View
14 test/fixtures/server.js
@@ -1,8 +1,11 @@
/**!
* urllib - test/fixtures/server.js
*
- * Copyright(c) 2011 - 2014 fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ * Copyright(c) 2011 - 2014 fengmk2 and other contributors.
* MIT Licensed
+ *
+ * Authors:
+ * fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
"use strict";
@@ -15,6 +18,7 @@ var should = require('should');
var http = require('http');
var querystring = require('querystring');
var fs = require('fs');
+var zlib = require('zlib');
var server = http.createServer(function (req, res) {
req.headers['user-agent'].should.match(/^node\-urllib\/\d+\.\d+\.\d+$/);
@@ -121,6 +125,13 @@ var server = http.createServer(function (req, res) {
res.end(JSON.stringify(JSON.parse(Buffer.concat(chunks))));
}
return;
+ } else if (req.url.indexOf('/no-gzip') === 0) {
+ fs.createReadStream(__filename).pipe(res);
+ return;
+ } else if (req.url.indexOf('/gzip') === 0) {
+ res.setHeader('Content-Encoding', 'gzip');
+ fs.createReadStream(__filename).pipe(zlib.createGzip()).pipe(res);
+ return;
}
var url = req.url.split('?');
@@ -136,7 +147,6 @@ var server = http.createServer(function (req, res) {
'Content-Type': 'text/html',
});
res.end(ret.replace('##{charset}##', get.charset ? get.charset : ''));
-
});
});
View
87 test/urllib.test.js
@@ -1,11 +1,14 @@
/**!
* urllib - test/urllib.test.js
*
- * Copyright(c) 2011 - 2014 fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ * Copyright(c) 2011 - 2014 fengmk2 and other contributors.
* MIT Licensed
+ *
+ * Authors:
+ * fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
-"use strict";
+'use strict';
/**
* Module dependencies.
@@ -776,6 +779,86 @@ describe('urllib.test.js', function () {
});
});
+ describe('gzip content', function () {
+ it('should auto accept and decode gzip response content', function (done) {
+ urllib.request('http://r.cnpmjs.org/npm', {dataType: 'json', gzip: true, timeout: 10000},
+ function (err, data, res) {
+ should.not.exist(err);
+ data.name.should.equal('npm');
+ res.should.have.header('content-encoding', 'gzip');
+ res.should.have.header('content-type', 'application/json');
+ done();
+ });
+ });
+
+ it('should redirect and gzip', function (done) {
+ urllib.request('http://dist.cnpmjs.org/v0.10.1/SHASUMS.txt', {followRedirect: true, gzip: true},
+ function (err, data, res) {
+ should.not.exist(err);
+ data.toString().should.include('e213170fe5ec7721b31149fba1a7a691c50b5379');
+ res.should.have.header('content-encoding', 'gzip');
+ res.should.have.header('content-type', 'text/plain');
+ done();
+ });
+ });
+
+ it('should not ungzip binary content', function (done) {
+ urllib.request('http://dist.u.qiniudn.com/v0.10.0/node.exp', {gzip: true}, function (err, data, res) {
+ should.not.exist(err);
+ should.not.exist(res.headers['content-encoding']);
+ res.should.have.header('content-type', 'application/octet-stream');
+ // console.log(res.headers);
+ done();
+ });
+ });
+
+ it('should not return gzip response content', function (done) {
+ done = pedding(3, done);
+ urllib.request('http://r.cnpmjs.org', {dataType: 'json'},
+ function (err, data, res) {
+ should.not.exist(err);
+ data.db_name.should.equal('registry');
+ should.not.exist(res.headers['content-encoding']);
+ res.should.have.header('content-type', 'application/json');
+ done();
+ });
+
+ urllib.request('http://r.cnpmjs.org', {dataType: 'json', gzip: false},
+ function (err, data, res) {
+ should.not.exist(err);
+ data.db_name.should.equal('registry');
+ should.not.exist(res.headers['content-encoding']);
+ res.should.have.header('content-type', 'application/json');
+ done();
+ });
+
+ urllib.request('http://r.cnpmjs.org', {dataType: 'json', gzip: true},
+ function (err, data, res) {
+ should.not.exist(err);
+ data.db_name.should.equal('registry');
+ res.should.have.header('content-encoding', 'gzip');
+ res.should.have.header('content-type', 'application/json');
+ done();
+ });
+ });
+
+ it('should not ungzip content when server not accept gzip', function (done) {
+ urllib.request(host + '/no-gzip', {gzip: true}, function (err, data, res) {
+ should.not.exist(err);
+ should.not.exist(res.headers['content-encoding']);
+ done();
+ });
+ });
+
+ it('should gzip content when server accept gzip', function (done) {
+ urllib.request(host + '/gzip', {gzip: true}, function (err, data, res) {
+ should.not.exist(err);
+ res.should.have.header('content-encoding', 'gzip');
+ done();
+ });
+ });
+ });
+
describe('204 status response', function () {
it('should not convert json data when status 204', function (done) {
urllib.request(host + '/204', {dataType: 'json'}, function (err, data, res) {
Please sign in to comment.
Something went wrong with that request. Please try again.