feat: Support urllib@3 (#5000)

closes #4847

config/config.default.js

@@ -290,6 +290,7 @@ module.exports = appInfo => {
    * @property {Number} httpsAgent.freeSocketTimeout - httpss agent socket keepalive max free time, default is 4000 ms.
    * @property {Number} httpsAgent.maxSockets - https agent max socket number of one host, default is `Number.MAX_SAFE_INTEGER` @ses https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER
    * @property {Number} httpsAgent.maxFreeSockets - https agent max free socket number of one host, default is 256.
+   * @property {Boolean} useHttpClientNext - use urllib@3 HttpClient
    */
   config.httpclient = {
     enableDNSCache: false,
@@ -311,6 +312,7 @@ module.exports = appInfo => {
       maxSockets: Number.MAX_SAFE_INTEGER,
       maxFreeSockets: 256,
     },
+    useHttpClientNext: false,
   };
 
   /**

lib/core/httpclient_next.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const { HttpClient } = require('urllib-next');
+const ms = require('humanize-ms');
+
+class HttpClientNext extends HttpClient {
+  constructor(app) {
+    normalizeConfig(app);
+    const config = app.config.httpclient;
+    super({
+      app,
+      defaultArgs: config.request,
+    });
+    this.app = app;
+  }
+
+  async request(url, options) {
+    options = options || {};
+    if (options.ctx && options.ctx.tracer) {
+      options.tracer = options.ctx.tracer;
+    } else {
+      options.tracer = options.tracer || this.app.tracer;
+    }
+    return await super.request(url, options);
+  }
+
+  async curl(...args) {
+    return await this.request(...args);
+  }
+}
+
+function normalizeConfig(app) {
+  const config = app.config.httpclient;
+  if (typeof config.request.timeout === 'string') {
+    config.request.timeout = ms(config.request.timeout);
+  }
+}
+
+module.exports = HttpClientNext;

lib/egg.js

@@ -15,6 +15,7 @@ const ContextHttpClient = require('./core/context_httpclient');
 const Messenger = require('./core/messenger');
 const DNSCacheHttpClient = require('./core/dnscache_httpclient');
 const HttpClient = require('./core/httpclient');
+const HttpClientNext = require('./core/httpclient_next');
 const createLoggers = require('./core/logger');
 const Singleton = require('./core/singleton');
 const utils = require('./core/utils');
@@ -291,7 +292,9 @@ class EggApplication extends EggCore {
    */
   get httpclient() {
     if (!this[HTTPCLIENT]) {
-      if (this.config.httpclient.enableDNSCache) {
+      if (this.config.httpclient.useHttpClientNext) {
+        this[HTTPCLIENT] = new HttpClientNext(this);
+      } else if (this.config.httpclient.enableDNSCache) {
         this[HTTPCLIENT] = new DNSCacheHttpClient(this);
       } else {
         this[HTTPCLIENT] = new this.HttpClient(this);

package.json

@@ -53,6 +53,7 @@
     "semver": "^7.3.2",
     "sendmessage": "^1.1.0",
     "urllib": "^2.33.0",
+    "urllib-next": "^3.1.1",
     "utility": "^1.15.0",
     "ylru": "^1.2.1"
   },

test/fixtures/apps/httpclient-next-with-tracer/config/config.default.js

@@ -0,0 +1,6 @@
+module.exports = {
+  httpclient: {
+    useHttpClientNext: true,
+    timeout: '5s',
+  },
+};

test/fixtures/apps/httpclient-next-with-tracer/config/plugin.js

@@ -0,0 +1,8 @@
+'use strict';
+
+module.exports = {
+  tracer: {
+    enable: true,
+    package: 'egg-tracer',
+  },
+}

test/fixtures/apps/httpclient-next-with-tracer/package.json

@@ -0,0 +1,3 @@
+{
+  "name": "httpclient-next-with-tracer"
+}

test/lib/core/httpclient.test.js

@@ -4,10 +4,12 @@ const assert = require('assert');
 const mm = require('egg-mock');
 const urllib = require('urllib');
 const Httpclient = require('../../../lib/core/httpclient');
+const HttpclientNext = require('../../../lib/core/httpclient_next');
 const utils = require('../../utils');
 
 describe('test/lib/core/httpclient.test.js', () => {
   let client;
+  let clientNext;
   let url;
 
   before(() => {
@@ -26,6 +28,19 @@ describe('test/lib/core/httpclient.test.js', () => {
       info.args.headers['mock-traceid'] = 'mock-traceid';
       info.args.headers['mock-rpcid'] = 'mock-rpcid';
     });
+
+    clientNext = new HttpclientNext({
+      config: {
+        httpclient: {
+          request: {},
+        },
+      },
+    });
+    clientNext.on('request', info => {
+      info.args.headers = info.args.headers || {};
+      info.args.headers['mock-traceid'] = 'mock-traceid';
+      info.args.headers['mock-rpcid'] = 'mock-rpcid';
+    });
   });
   before(async () => {
     url = await utils.startLocalServer();
@@ -113,6 +128,55 @@ describe('test/lib/core/httpclient.test.js', () => {
     });
   });
 
+  describe('HttpClientNext', () => {
+    it('should request ok with log', async () => {
+      const args = {
+        dataType: 'text',
+      };
+      let info;
+      clientNext.once('response', meta => {
+        info = meta;
+      });
+      const { status } = await clientNext.request(url, args);
+      assert(status === 200);
+      assert(info.req.options.headers['mock-traceid'] === 'mock-traceid');
+      assert(info.req.options.headers['mock-rpcid'] === 'mock-rpcid');
+      assert(info.req.args.headers['mock-traceid'] === 'mock-traceid');
+      assert(info.req.args.headers['mock-rpcid'] === 'mock-rpcid');
+    });
+
+    it('should curl ok with log', async () => {
+      const args = {
+        dataType: 'text',
+      };
+      let info;
+      clientNext.once('response', meta => {
+        info = meta;
+      });
+      const { status } = await clientNext.curl(url, args);
+      assert(status === 200);
+      assert(info.req.options.headers['mock-traceid'] === 'mock-traceid');
+      assert(info.req.options.headers['mock-rpcid'] === 'mock-rpcid');
+      assert(info.req.args.headers['mock-traceid'] === 'mock-traceid');
+      assert(info.req.args.headers['mock-rpcid'] === 'mock-rpcid');
+    });
+
+    it('should request with error', async () => {
+      await assert.rejects(async () => {
+        const response = await clientNext.request(url + '/error', {
+          dataType: 'json',
+        });
+        console.log(response);
+      }, err => {
+        assert.equal(err.name, 'JSONResponseFormatError');
+        assert.match(err.message, /this is an error/);
+        assert(err.res);
+        assert.equal(err.res.status, 500);
+        return true;
+      });
+    });
+  });
+
   describe('httpclient.httpAgent.timeout < 30000', () => {
     let app;
     before(() => {
@@ -320,6 +384,136 @@ describe('test/lib/core/httpclient.test.js', () => {
     });
   });
 
+  describe('httpclient next with tracer', () => {
+    let app;
+    before(() => {
+      app = utils.app('apps/httpclient-next-with-tracer');
+      return app.ready();
+    });
+
+    after(() => app.close());
+
+    it('should app request auto set tracer', async () => {
+      const httpclient = app.httpclient;
+
+      let reqTracer;
+      let resTracer;
+
+      httpclient.on('request', function(options) {
+        reqTracer = options.args.tracer;
+      });
+
+      httpclient.on('response', function(options) {
+        resTracer = options.req.args.tracer;
+      });
+
+      const opaque = { now: Date.now() };
+      let res = await httpclient.request(url, {
+        method: 'GET',
+        dataType: 'text',
+        opaque,
+      });
+      assert(res.opaque === opaque);
+
+      assert(res.status === 200);
+      assert(reqTracer);
+      assert(resTracer);
+      assert(reqTracer === resTracer);
+
+      assert(reqTracer.traceId);
+      assert(reqTracer.traceId === resTracer.traceId);
+
+      reqTracer = null;
+      resTracer = null;
+
+      res = await httpclient.request(url);
+
+      assert(res.status === 200);
+      assert(reqTracer === resTracer);
+
+      assert(reqTracer.traceId);
+      assert(reqTracer.traceId === resTracer.traceId);
+    });
+
+    it('should agent request auto set tracer', async () => {
+      const httpclient = app.agent.httpclient;
+
+      let reqTracer;
+      let resTracer;
+
+      httpclient.on('request', function(options) {
+        reqTracer = options.args.tracer;
+      });
+
+      httpclient.on('response', function(options) {
+        resTracer = options.req.args.tracer;
+      });
+
+      const res = await httpclient.request(url, {
+        method: 'GET',
+      });
+
+      assert(res.status === 200);
+      assert(reqTracer === resTracer);
+
+      assert(reqTracer.traceId);
+      assert(reqTracer.traceId === resTracer.traceId);
+    });
+
+    it('should app request with ctx and tracer', async () => {
+      const httpclient = app.httpclient;
+
+      let reqTracer;
+      let resTracer;
+
+      httpclient.on('request', function(options) {
+        reqTracer = options.args.tracer;
+      });
+
+      httpclient.on('response', function(options) {
+        resTracer = options.req.args.tracer;
+      });
+
+      let res = await httpclient.request(url, {
+        method: 'GET',
+      });
+
+      assert(res.status === 200);
+
+      assert(reqTracer.traceId);
+      assert(reqTracer.traceId === resTracer.traceId);
+
+      reqTracer = null;
+      resTracer = null;
+      res = await httpclient.request(url, {
+        method: 'GET',
+        ctx: {},
+        tracer: {
+          id: '1234',
+        },
+      });
+
+      assert(res.status === 200);
+      assert(reqTracer.id === resTracer.id);
+      assert(reqTracer.id === '1234');
+
+      reqTracer = null;
+      resTracer = null;
+      res = await httpclient.request(url, {
+        method: 'GET',
+        ctx: {
+          tracer: {
+            id: '5678',
+          },
+        },
+      });
+
+      assert(res.status === 200);
+      assert(reqTracer.id === resTracer.id);
+      assert(reqTracer.id === '5678');
+    });
+  });
+
   describe.skip('before app ready multi httpclient request tracer', () => {
     let app;
     before(() => {

test/lib/core/logger.test.js

@@ -174,6 +174,11 @@ describe('test/lib/core/logger.test.js', () => {
   });
 
   it('agent and app error should output to common-error.log', done => {
+    // unstable on Windows, skip it
+    // https://github.com/eggjs/egg/runs/7977866117?check_suite_focus=true
+    if (process.platform === 'win32') {
+      return done();
+    }
     const baseDir = utils.getFilepath('apps/logger');
     mm.env('default');
     mm(process.env, 'EGG_LOG', 'none');