Merge pull request #238 from midwayjs/feat/record_http_response

feat: record http client response data

packages/hook/src/patch/HttpClient.ts

@@ -4,9 +4,15 @@ import { Patcher } from 'pandora-metrics';
 import * as semver from 'semver';
 import { HttpClientShimmer } from './shimmers/http-client/Shimmer';
 
+export type bufferTransformer = (buffer) => object | string;
+
 export class HttpClientPatcher extends Patcher {
 
-  constructor(options = {}) {
+  constructor(options: {
+    forceHttps?: boolean,
+    recordResponse?: boolean,
+    bufferTransformer?: bufferTransformer
+  }) {
     super(Object.assign({
       shimmerClass: HttpClientShimmer,
       remoteTracing: true

packages/hook/src/patch/shimmers/http-client/Shimmer.ts

@@ -1,14 +1,20 @@
-const assert = require('assert');
-const debug = require('debug')('PandoraHook:HttpClient:Shimmer');
+import * as assert from 'assert';
 import { DEFAULT_HOST, DEFAULT_PORT, HEADER_SPAN_ID, HEADER_TRACE_ID } from '../../../utils/Constants';
 import { nodeVersion } from '../../../utils/Utils';
 import { ClientRequest } from 'http';
 
+const debug = require('debug')('PandoraHook:HttpClient:Shimmer');
+
 // TODO: 接受参数，处理或记录请求详情
 
+export type bufferTransformer = (buffer) => object | string;
+
 export class HttpClientShimmer {
 
-  options = {};
+  options: {
+    recordResponse?: boolean
+    bufferTransformer?: bufferTransformer
+  } = {};
   shimmer = null;
   traceManager = null;
 
@@ -134,6 +140,11 @@ export class HttpClientShimmer {
   }
 
   protected _requestError(res, span) {
+
+    // clear cache when request error
+    delete res.__responseSize;
+    delete res.__chunks;
+
     span.setTag('http.error_code', {
       type: 'string',
       value: res.code
@@ -159,7 +170,10 @@ export class HttpClientShimmer {
   protected _responseEnd(res, span) {
     const socket = res.socket;
     const remoteIp = socket ? (socket.remoteAddress ? `${socket.remoteAddress}:${socket.remotePort}` : '') : '';
-    const responseSize = (res.headers && res.headers['content-length']) || res.responseSize;
+    const responseSize = (res.headers && res.headers['content-length']) || res.__responseSize;
+
+    delete res.__responseSize;
+    delete res.__chunks;
 
     span.setTag('http.status_code', {
       type: 'number',
@@ -179,19 +193,39 @@ export class HttpClientShimmer {
 
   protected _finish(res, span) {}
 
+  bufferTransformer(buffer): string {
+    try {
+      return buffer.toString('utf8');
+    } catch (error) {
+      debug('transform response data error. ', error);
+      return '';
+    }
+  }
+
   handleResponse(tracer, span, res) {
     const traceManager = this.traceManager;
     const shimmer = this.shimmer;
     const self = this;
+    const recordResponse = this.options.recordResponse;
+    const bufferTransformer = this.options.bufferTransformer || self.bufferTransformer;
+
+    res.__responseSize = 0;
+    res.__chunks = [];
 
-    res.responseSize = 0;
     shimmer.wrap(res, 'emit', function wrapResponseEmit(emit) {
       const bindResponseEmit = traceManager.bind(emit);
 
       return function wrappedResponseEmit(this: ClientRequest, event) {
         if (event === 'end') {
           if (span) {
 
+            if (recordResponse) {
+              const response = bufferTransformer(res.__chunks);
+              span.log({
+                response
+              });
+            }
+
             span.error(false);
 
             self._responseEnd(res, span);
@@ -201,8 +235,12 @@ export class HttpClientShimmer {
             self._finish(res, span);
           }
         } else if (event === 'data') {
-          const chunk = arguments[0];
-          res.responseSize += chunk.length;
+          const chunk = arguments[1] || [];
+          res.__responseSize += chunk.length;
+
+          if (recordResponse) {
+            res.__chunks.push(chunk);
+          }
         }
 
         return bindResponseEmit.apply(this, arguments);

packages/hook/test/fixtures/http-client-record-response/index.ts

@@ -0,0 +1,83 @@
+import { RunUtil } from '../../RunUtil';
+import * as assert from 'assert';
+// 放在前面，把 http.ClientRequest 先复写
+const nock = require('nock');
+import { HttpServerPatcher, HttpClientPatcher } from '../../../src/';
+const httpServerPatcher = new HttpServerPatcher();
+const httpClientPatcher = new HttpClientPatcher({
+  // nock 复写了 https.request 方法，没有像原始一样调用 http.request，所以需要强制复写
+  forceHttps: true,
+  recordResponse: true
+});
+
+RunUtil.run(function(done) {
+  httpServerPatcher.run();
+  httpClientPatcher.run();
+
+  const http = require('http');
+  const https = require('https');
+
+  process.on(<any>'PANDORA_PROCESS_MESSAGE_TRACE', (report: any) => {
+    const spans = report.spans;
+    assert(spans.length === 2);
+    const logs = spans[1].logs;
+    const fields = logs[0].fields;
+    assert(fields[0].key === 'response');
+    assert(fields[0].value === 'Response from TaoBao.');
+
+    done();
+  });
+
+  nock('https://www.taobao.com')
+    .get('/')
+    .reply(200, 'Response from TaoBao.');
+
+  function request(agent, options) {
+
+    return new Promise((resolve, reject) => {
+      const req = agent.request(options, (res) => {
+        let data = '';
+
+        res.on('data', (d) => {
+          data += d;
+        });
+
+        res.on('end', () => {
+          resolve([res, data]);
+        });
+      });
+
+      req.on('error', (e) => {
+        reject(e);
+      });
+
+      req.end();
+    });
+  }
+
+  const server = http.createServer((req, res) => {
+    request(https, {
+      hostname: 'www.taobao.com',
+      path: '/',
+      method: 'GET'
+    }).then((response) => {
+      res.end('ok');
+    });
+  });
+
+  server.listen(0, () => {
+    const port = server.address().port;
+
+    setTimeout(function() {
+      request(http, {
+        hostname: 'localhost',
+        port: port,
+        path: '/',
+        method: 'GET'
+      }).catch((err) => {
+        console.log('err: ', err);
+      });
+    }, 500);
+  });
+});
+

packages/hook/test/fixtures/http-client-record-response/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "http-client-test",
+  "version": "1.0.0",
+  "main": "index.js",
+  "dependencies": {
+    "urllib": "^2.25.1"
+  }
+}

packages/hook/test/index.test.ts

@@ -96,6 +96,10 @@ describe('unit test', () => {
     it('should urllib work ok', done => {
       fork('urllib', done);
     });
+
+    it('should record response data', done => {
+      fork('http-client-record-response', done);
+    });
   });
 
   describe('mysql', () => {