From b5e1a29dcbc95dc5cb7a99e1f1b5144b08e9448a Mon Sep 17 00:00:00 2001 From: MarioDu Date: Wed, 25 Apr 2018 17:34:43 +0800 Subject: [PATCH] feat: record http client response data --- packages/hook/src/patch/HttpClient.ts | 8 +- .../src/patch/shimmers/http-client/Shimmer.ts | 52 ++++++++++-- .../http-client-record-response/index.ts | 83 +++++++++++++++++++ .../http-client-record-response/package.json | 8 ++ packages/hook/test/index.test.ts | 4 + 5 files changed, 147 insertions(+), 8 deletions(-) create mode 100644 packages/hook/test/fixtures/http-client-record-response/index.ts create mode 100644 packages/hook/test/fixtures/http-client-record-response/package.json diff --git a/packages/hook/src/patch/HttpClient.ts b/packages/hook/src/patch/HttpClient.ts index e785ee67..dd777c06 100644 --- a/packages/hook/src/patch/HttpClient.ts +++ b/packages/hook/src/patch/HttpClient.ts @@ -4,9 +4,15 @@ import { Patcher } from 'pandora-metrics'; import * as semver from 'semver'; import { HttpClientShimmer } from './shimmers/http-client/Shimmer'; +export type bufferTransformer = (buffer) => object | string; + export class HttpClientPatcher extends Patcher { - constructor(options = {}) { + constructor(options: { + forceHttps?: boolean, + recordResponse?: boolean, + bufferTransformer?: bufferTransformer + }) { super(Object.assign({ shimmerClass: HttpClientShimmer, remoteTracing: true diff --git a/packages/hook/src/patch/shimmers/http-client/Shimmer.ts b/packages/hook/src/patch/shimmers/http-client/Shimmer.ts index 1f119e1a..1807d137 100644 --- a/packages/hook/src/patch/shimmers/http-client/Shimmer.ts +++ b/packages/hook/src/patch/shimmers/http-client/Shimmer.ts @@ -1,14 +1,20 @@ -const assert = require('assert'); -const debug = require('debug')('PandoraHook:HttpClient:Shimmer'); +import * as assert from 'assert'; import { DEFAULT_HOST, DEFAULT_PORT, HEADER_SPAN_ID, HEADER_TRACE_ID } from '../../../utils/Constants'; import { nodeVersion } from '../../../utils/Utils'; import { ClientRequest } from 'http'; +const debug = require('debug')('PandoraHook:HttpClient:Shimmer'); + // TODO: 接受参数，处理或记录请求详情 +export type bufferTransformer = (buffer) => object | string; + export class HttpClientShimmer { - options = {}; + options: { + recordResponse?: boolean + bufferTransformer?: bufferTransformer + } = {}; shimmer = null; traceManager = null; @@ -134,6 +140,11 @@ export class HttpClientShimmer { } protected _requestError(res, span) { + + // clear cache when request error + delete res.__responseSize; + delete res.__chunks; + span.setTag('http.error_code', { type: 'string', value: res.code @@ -159,7 +170,10 @@ export class HttpClientShimmer { protected _responseEnd(res, span) { const socket = res.socket; const remoteIp = socket ? (socket.remoteAddress ? `${socket.remoteAddress}:${socket.remotePort}` : '') : ''; - const responseSize = (res.headers && res.headers['content-length']) || res.responseSize; + const responseSize = (res.headers && res.headers['content-length']) || res.__responseSize; + + delete res.__responseSize; + delete res.__chunks; span.setTag('http.status_code', { type: 'number', @@ -179,12 +193,25 @@ export class HttpClientShimmer { protected _finish(res, span) {} + bufferTransformer(buffer): string { + try { + return buffer.toString('utf8'); + } catch (error) { + debug('transform response data error. ', error); + return ''; + } + } + handleResponse(tracer, span, res) { const traceManager = this.traceManager; const shimmer = this.shimmer; const self = this; + const recordResponse = this.options.recordResponse; + const bufferTransformer = this.options.bufferTransformer || self.bufferTransformer; + + res.__responseSize = 0; + res.__chunks = []; - res.responseSize = 0; shimmer.wrap(res, 'emit', function wrapResponseEmit(emit) { const bindResponseEmit = traceManager.bind(emit); @@ -192,6 +219,13 @@ export class HttpClientShimmer { if (event === 'end') { if (span) { + if (recordResponse) { + const response = bufferTransformer(res.__chunks); + span.log({ + response + }); + } + span.error(false); self._responseEnd(res, span); @@ -201,8 +235,12 @@ export class HttpClientShimmer { self._finish(res, span); } } else if (event === 'data') { - const chunk = arguments[0]; - res.responseSize += chunk.length; + const chunk = arguments[1] || []; + res.__responseSize += chunk.length; + + if (recordResponse) { + res.__chunks.push(chunk); + } } return bindResponseEmit.apply(this, arguments); diff --git a/packages/hook/test/fixtures/http-client-record-response/index.ts b/packages/hook/test/fixtures/http-client-record-response/index.ts new file mode 100644 index 00000000..e8d6fa74 --- /dev/null +++ b/packages/hook/test/fixtures/http-client-record-response/index.ts @@ -0,0 +1,83 @@ +import { RunUtil } from '../../RunUtil'; +import * as assert from 'assert'; +// 放在前面，把 http.ClientRequest 先复写 +const nock = require('nock'); +import { HttpServerPatcher, HttpClientPatcher } from '../../../src/'; +const httpServerPatcher = new HttpServerPatcher(); +const httpClientPatcher = new HttpClientPatcher({ + // nock 复写了 https.request 方法，没有像原始一样调用 http.request，所以需要强制复写 + forceHttps: true, + recordResponse: true +}); + +RunUtil.run(function(done) { + httpServerPatcher.run(); + httpClientPatcher.run(); + + const http = require('http'); + const https = require('https'); + + process.on('PANDORA_PROCESS_MESSAGE_TRACE', (report: any) => { + const spans = report.spans; + assert(spans.length === 2); + const logs = spans[1].logs; + const fields = logs[0].fields; + assert(fields[0].key === 'response'); + assert(fields[0].value === 'Response from TaoBao.'); + + done(); + }); + + nock('https://www.taobao.com') + .get('/') + .reply(200, 'Response from TaoBao.'); + + function request(agent, options) { + + return new Promise((resolve, reject) => { + const req = agent.request(options, (res) => { + let data = ''; + + res.on('data', (d) => { + data += d; + }); + + res.on('end', () => { + resolve([res, data]); + }); + }); + + req.on('error', (e) => { + reject(e); + }); + + req.end(); + }); + } + + const server = http.createServer((req, res) => { + request(https, { + hostname: 'www.taobao.com', + path: '/', + method: 'GET' + }).then((response) => { + res.end('ok'); + }); + }); + + server.listen(0, () => { + const port = server.address().port; + + setTimeout(function() { + request(http, { + hostname: 'localhost', + port: port, + path: '/', + method: 'GET' + }).catch((err) => { + console.log('err: ', err); + }); + }, 500); + }); +}); + diff --git a/packages/hook/test/fixtures/http-client-record-response/package.json b/packages/hook/test/fixtures/http-client-record-response/package.json new file mode 100644 index 00000000..0c5c1100 --- /dev/null +++ b/packages/hook/test/fixtures/http-client-record-response/package.json @@ -0,0 +1,8 @@ +{ + "name": "http-client-test", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "urllib": "^2.25.1" + } +} diff --git a/packages/hook/test/index.test.ts b/packages/hook/test/index.test.ts index 71248705..b43a3bd6 100644 --- a/packages/hook/test/index.test.ts +++ b/packages/hook/test/index.test.ts @@ -92,6 +92,10 @@ describe('unit test', () => { it('should urllib work ok', done => { fork('urllib', done); }); + + it('should record response data', done => { + fork('http-client-record-response', done); + }); }); describe('mysql', () => {