Merge 8dfe782 into f938fb6

lib/urllib.js

@@ -25,6 +25,8 @@ var fs = require('fs');
 var zlib = require('zlib');
 var ua = require('default-user-agent');
 var digestAuthHeader = require('digest-header');
+var typer = require('media-typer');
+
 var EventEmitter = require('events').EventEmitter;
 var _Promise;
 
@@ -498,9 +500,21 @@ exports.requestWithCallback = function (url, args, callback) {
         if (err) {
           return done(err, body, res);
         }
-
         // if body not decode, dont touch it
-        if (!encoding && args.dataType) {
+        if (!encoding && ~['json', 'text'].indexOf(args.dataType)) {
+          // try to decode charset
+          try {
+            var type = typer.parse(res.headers['content-type']);
+            var charset = type.parameters.charset || 'utf-8';
+            debug('auto decode response body to charset: %s', charset);
+            data = Buffer.isEncoding(charset)
+              ? data.toString(charset)
+              : require('iconv-lite').decode(data, charset);
+          } catch (_) {
+            // if error, dont touch it
+            return done(null, data, res);
+          }
+
           if (args.dataType === 'json') {
             if (size === 0) {
               data = null;

package.json

@@ -32,7 +32,9 @@
     "debug": "~1.0.4",
     "default-user-agent": "~0.0.1",
     "digest-header": "~0.0.1",
-    "native-or-bluebird": "~1.0.0"
+    "native-or-bluebird": "~1.0.0",
+    "media-typer": "~0.3.0",
+    "iconv-lite": "~0.4.4"
   },
   "devDependencies": {
     "agentkeepalive": "~0.2.0",

test/fixtures/server.js

@@ -19,6 +19,7 @@ var http = require('http');
 var querystring = require('querystring');
 var fs = require('fs');
 var zlib = require('zlib');
+var iconv = require('iconv-lite');
 
 var server = http.createServer(function (req, res) {
   // req.headers['user-agent'].should.match(/^node\-urllib\/\d+\.\d+\.\d+ node\//);
@@ -135,6 +136,17 @@ var server = http.createServer(function (req, res) {
     } else if (req.url === '/ua') {
       res.end(JSON.stringify(req.headers));
       return;
+    } else if (req.url === '/gbk/json') {
+      res.setHeader('Content-Type', 'application/json;charset=gbk');
+      var content = iconv.encode(JSON.stringify({hello: '你好'}), 'gbk');
+      return res.end(content);
+    } else if (req.url === '/gbk/text') {
+      res.setHeader('Content-Type', 'text/plain;charset=gbk');
+      var content = iconv.encode('你好', 'gbk');
+      return res.end(content);
+    } else if (req.url === '/errorcharset') {
+      res.setHeader('Content-Type', 'text/plain;charset=notfound');
+      return res.end('你好');
     }
 
     var url = req.url.split('?');

test/urllib.test.js

@@ -997,4 +997,38 @@ describe('urllib.test.js', function () {
       });
     });
   });
+
+  describe.only('charset support', function () {
+    it('should auto decode when dataType = json', function (done) {
+      urllib.request(host + '/gbk/json', {
+        dataType: 'json'
+      }, function (err, data, res) {
+        res.should.have.header('content-type', 'application/json;charset=gbk');
+        data.should.eql({
+          hello: '你好'
+        });
+        done(err);
+      });
+    });
+
+    it('should auto decode when dataType = text', function (done) {
+      urllib.request(host + '/gbk/text', {
+        dataType: 'text'
+      }, function (err, data, res) {
+        res.should.have.header('content-type', 'text/plain;charset=gbk');
+        data.should.equal('你好');
+        done(err);
+      });
+    });
+
+    it('should ignore wrong charset', function (done) {
+      urllib.request(host + '/errorcharset', {
+        dataType: 'text'
+      }, function (err, data, res) {
+        res.should.have.header('content-type', 'text/plain;charset=notfound');
+        data.should.eql(new Buffer('你好'));
+        done(err);
+      });
+    });
+  });
 });