Permalink
Browse files

Support `options.gzip = true` to handle gzip response. fixed #26

* If the options.gzip = true, urllib will auto add
  options.headers["Accept-Encoding"] = 'gzip'
  and auto ungzip the gzip response content.
* User also can disable gzip with options.gzip = false
  or ignore this argument.
  • Loading branch information...
fengmk2 committed Mar 10, 2014
1 parent b0028a6 commit 5b6fa22f311f83acdee6e9ea8510d25528f64dea
Showing with 174 additions and 46 deletions.
  1. +1 −2 .gitignore
  2. +1 −2 .npmignore
  3. +1 −1 .travis.yml
  4. +18 −9 Makefile
  5. +4 −1 README.md
  6. +48 −15 lib/urllib.js
  7. +4 −12 package.json
  8. +12 −2 test/fixtures/server.js
  9. +85 −2 test/urllib.test.js
View
@@ -1,5 +1,4 @@
.project
.settings
node_modules/
-coverage.html
-lib-cov
+coverage/
View
@@ -4,7 +4,6 @@ logo.png
examples/
test/
Makefile
-lib-cov/
-coverage.html
.jshintrc
.jshintignore
+coverage/
View
@@ -3,4 +3,4 @@ node_js:
- '0.8'
- '0.10'
- '0.11'
-script: make test-coveralls
+script: make test-all
View
@@ -10,27 +10,36 @@ jshint: install
@./node_modules/.bin/jshint .
test: install
- @NODE_ENV=test ./node_modules/mocha/bin/mocha \
+ @NODE_ENV=test ./node_modules/.bin/mocha \
--reporter $(REPORTER) \
--timeout $(TIMEOUT) \
$(MOCHA_OPTS) \
$(TESTS)
test-cov:
- @$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=travis-cov
+ @NODE_ENV=test node --harmony \
+ node_modules/.bin/istanbul cover ./node_modules/.bin/_mocha \
+ -- -u exports \
+ --reporter $(REPORTER) \
+ --timeout $(TIMEOUT) \
+ $(MOCHA_OPTS) \
+ $(TESTS)
+ @-$(MAKE) check-coverage
-test-cov-html:
- @rm -f coverage.html
- @$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=html-cov > coverage.html
- @ls -lh coverage.html
+check-coverage:
+ @./node_modules/.bin/istanbul check-coverage \
+ --statements 96 \
+ --branches 93 \
+ --functions 94 \
+ --lines 96
-test-coveralls: jshint test
- @echo TRAVIS_JOB_ID $(TRAVIS_JOB_ID)
- @-$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=mocha-lcov-reporter | ./node_modules/coveralls/bin/coveralls.js
+cov:
+ @./node_modules/.bin/cov coverage
test-all: jshint test test-cov
autod: install
@./node_modules/.bin/autod -w
+ @$(MAKE) install
.PHONY: test
View
@@ -1,7 +1,6 @@
# urllib
[![Build Status](https://secure.travis-ci.org/fengmk2/urllib.png?branch=master)](http://travis-ci.org/fengmk2/urllib)
-[![Coverage Status](https://coveralls.io/repos/fengmk2/urllib/badge.png)](https://coveralls.io/r/fengmk2/urllib)
[![NPM](https://nodei.co/npm/urllib.png?downloads=true&stars=true)](https://nodei.co/npm/urllib/)
@@ -10,6 +9,8 @@
Help in opening URLs (mostly HTTP) in a complex world — basic
and digest authentication, redirections, cookies, timeout and more.
+[Code coverage: 94%](http://qtestbucket.qiniudn.com/cov/html/urllib/0.5.8/index.html)
+
## Install
```bash
@@ -69,6 +70,7 @@ urllib.request('http://cnodejs.org/', { wd: 'nodejs' }, function (err, data, res
- ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
- ***maxRedirects*** Number - The maximum number of redirects to follow, defaults to 10.
- ***beforeRequest*** Function - Before request hook, you can change every thing here.
+ - ***gzip*** Boolean - Accept gzip response content and auto decode it, default is `false`.
- ***callback(err, data, res)*** Function - Optional callback.
- **err** Error - Would be `null` if no error accured.
- **data** Buffer | Object - The data responsed. Would be a Buffer if `dataType` is set to `text` or an JSON parsed into Object if it's set to `json`.
@@ -209,6 +211,7 @@ urllib.request('http://127.0.0.1:1984/socket.end', function (err, data, res) {
* [√] Auto redirect handle
* [√] https & self-signed certificate
* [√] Connection timeout & Response timeout
+* [√] Support `Accept-Encoding=gzip` by `options.gzip = true`
## Authors
View
@@ -1,8 +1,11 @@
/**!
* urllib - lib/urllib.js
*
- * Copyright(c) 2011 - 2014 fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ * Copyright(c) 2011 - 2014 fengmk2 and other contributors.
* MIT Licensed
+ *
+ * Authors:
+ * fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
"use strict";
@@ -17,6 +20,7 @@ var urlutil = require('url');
var qs = require('querystring');
var path = require('path');
var fs = require('fs');
+var zlib = require('zlib');
var debug = require('debug')('urllib');
var pkg = JSON.parse(fs.readFileSync(path.join(__dirname, '../', 'package.json')));
@@ -81,6 +85,7 @@ var REQUEST_ID = 0;
* - {Boolean} [followRedirect]: Follow HTTP 3xx responses as redirects. defaults to false.
* - {Number} [maxRedirects]: The maximum number of redirects to follow, defaults to 10.
* - {Function(options)} [beforeRequest]: Before request hook, you can change every thing here.
+ * - {Boolean} [gzip]: Accept gzip response content and auto decode it, default is `false`.
* @param {Function} callback, callback(error, data, res)
* @return {HttpRequest} req object.
* @api public
@@ -248,6 +253,30 @@ exports.request = function (url, args, callback) {
options.headers['User-Agent'] = USER_AGENT;
}
+ var enableGzip = args.gzip === true;
+ if (enableGzip) {
+ var acceptEncoding = options.headers['Accept-Encoding'] || options.headers['accept-encoding'];
+ if (acceptEncoding) {
+ enableGzip = false; // user want to handle response content decode themself
+ } else {
+ options.headers['Accept-Encoding'] = 'gzip';
+ }
+ }
+
+ var decodeContent = function (res, body, cb) {
+ if (body.length === 0 || !enableGzip) {
+ return cb(null, body);
+ }
+
+ var encoding = res.headers['content-encoding'];
+ if (!encoding || encoding.toLowerCase() !== 'gzip') {
+ return cb(null, body);
+ }
+
+ debug('gunzip %d length body', body.length);
+ zlib.gunzip(body, cb);
+ };
+
var writeStream = args.writeStream;
var reqId = ++REQUEST_ID;
@@ -322,41 +351,45 @@ exports.request = function (url, args, callback) {
});
res.on('end', function () {
- var data = Buffer.concat(chunks, size);
+ var body = Buffer.concat(chunks, size);
debug('Request#%d %s: `res end` event emit, total size %d, _dumped: %s', reqId, options.path, size, res._dumped);
if (__err && connnected) {
// req.abort() after `res data` event emit.
- return done(__err, data, res);
+ return done(__err, body, res);
}
var result = handleRedirect(res);
- var err = result.error;
+ if (result.error) {
+ return done(result.error, body, res);
+ }
if (result.redirect) {
return;
}
- if (args.dataType === 'json') {
- if (size === 0) {
- data = null;
- } else {
+ decodeContent(res, body, function (err, data) {
+ if (err) {
+ return done(err, body, res);
+ }
+
+ if (args.dataType === 'json') {
try {
- data = JSON.parse(data);
+ data = size > 0 ? JSON.parse(data) : null;
} catch (e) {
if (e.name === 'SyntaxError') {
e.name = 'JSONResponseFormatError';
}
err = e;
}
}
- }
- if (res.aborted) {
- err = new Error('Remote socket was terminated before `response.end()` was called');
- err.name = 'RemoteSocketClosedError';
- }
+ if (res.aborted) {
+ err = new Error('Remote socket was terminated before `response.end()` was called');
+ err.name = 'RemoteSocketClosedError';
+ }
- done(err, data, res);
+ done(err, data, res);
+ });
});
});
View
@@ -3,7 +3,7 @@
"version": "0.5.8",
"description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
"keywords": [ "urllib", "http", "urlopen", "curl", "wget", "request", "https" ],
- "author": "fengmk2 <fengmk2@gmail.com> (http://github.com/fengmk2)",
+ "author": "fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)",
"homepage": "http://github.com/fengmk2/urllib",
"main": "index.js",
"repository": {
@@ -14,12 +14,6 @@
"test": "make test-all"
},
"config": {
- "blanket": {
- "pattern": "urllib/lib",
- "data-cover-flags": {
- "debug": false
- }
- },
"travis-cov": { "threshold": 95 }
},
"dependencies": {
@@ -28,15 +22,13 @@
"devDependencies": {
"agentkeepalive": "0.2.2",
"autod": "*",
- "blanket": "*",
- "coveralls": "*",
+ "cov": "*",
"formstream": "0.0.8",
+ "istanbul": "*",
"jshint": "*",
"mocha": ">=0.14.1",
- "mocha-lcov-reporter": "*",
"pedding": "0.0.3",
- "should": "3.1.3",
- "travis-cov": "*"
+ "should": "3.1.3"
},
"engines": { "node": ">= 0.8.0" },
"license": "MIT"
View
@@ -1,8 +1,11 @@
/**!
* urllib - test/fixtures/server.js
*
- * Copyright(c) 2011 - 2014 fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
+ * Copyright(c) 2011 - 2014 fengmk2 and other contributors.
* MIT Licensed
+ *
+ * Authors:
+ * fengmk2 <fengmk2@gmail.com> (http://fengmk2.github.com)
*/
"use strict";
@@ -15,6 +18,7 @@ var should = require('should');
var http = require('http');
var querystring = require('querystring');
var fs = require('fs');
+var zlib = require('zlib');
var server = http.createServer(function (req, res) {
req.headers['user-agent'].should.match(/^node\-urllib\/\d+\.\d+\.\d+$/);
@@ -121,6 +125,13 @@ var server = http.createServer(function (req, res) {
res.end(JSON.stringify(JSON.parse(Buffer.concat(chunks))));
}
return;
+ } else if (req.url.indexOf('/no-gzip') === 0) {
+ fs.createReadStream(__filename).pipe(res);
+ return;
+ } else if (req.url.indexOf('/gzip') === 0) {
+ res.setHeader('Content-Encoding', 'gzip');
+ fs.createReadStream(__filename).pipe(zlib.createGzip()).pipe(res);
+ return;
}
var url = req.url.split('?');
@@ -136,7 +147,6 @@ var server = http.createServer(function (req, res) {
'Content-Type': 'text/html',
});
res.end(ret.replace('##{charset}##', get.charset ? get.charset : ''));
-
});
});
Oops, something went wrong.

0 comments on commit 5b6fa22

Please sign in to comment.