From f0436fc0ef7f2e6ef8760c0969ea712cd619c75e Mon Sep 17 00:00:00 2001 From: Chris Cordi <3335965+ccordi@users.noreply.github.com> Date: Mon, 13 Sep 2021 22:02:20 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20add=20http/s=20proxy=20s?= =?UTF-8?q?upport=20for=20RPC=20endpoints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.schema.json | 4 ++++ docs/cli.md | 5 +++++ docs/configuration.md | 1 + package.json | 2 ++ src/cliflags.ts | 5 +++++ src/config.ts | 3 +++ src/eth/http.ts | 33 +++++++++++++++++++++++++-------- test/config.test.ts | 1 + yarn.lock | 38 +++++++++++++++++++++++++++++++++++++- 9 files changed, 83 insertions(+), 9 deletions(-) diff --git a/config.schema.json b/config.schema.json index 2af9922..6092806 100644 --- a/config.schema.json +++ b/config.schema.json @@ -349,6 +349,10 @@ "description": "Maximum number of sockets HEC will use (per host)", "type": "number" }, + "proxyUrl": { + "description": "Optional proxy URL to route HTTP requests through. Note: this disables internal httpClient stats", + "type": "string" + }, "requestKeepAlive": { "description": "Keep sockets to JSON RPC open", "type": "boolean" diff --git a/docs/cli.md b/docs/cli.md index d445899..d5e8d5c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -61,6 +61,10 @@ OPTIONS Disable to allow ethereum client to connect to HTTPS without rejecting invalid (self-signed) certificates + --eth-rpc-proxy=eth-rpc-proxy + Optional proxy URL required to reach the target ethereum node. Supported + format is HTTP/HTTPS. Note: this disables internal httpClient stats + --eth-rpc-url=eth-rpc-url URL to reach the target ethereum node. Supported format is currently only HTTP(s) for JSON RPC @@ -147,6 +151,7 @@ OPTIONS | `SPLUNK_METRICS_HEC_TOKEN` | `string` | HEC token to use for sending metrics. You can alternatively configure different indexes to correctly route your data | | `SPLUNK_INTERNAL_HEC_TOKEN` | `string` | HEC token to use for sending internal metrics. You can alternatively configure different indexes to correctly route your data | | `ETH_RPC_URL` | `string` | URL to reach the target ethereum node. Supported format is currently only HTTP(s) for JSON RPC | +| `ETH_RPC_PROXY` | `string` | Optional proxy URL required to reach the target ethereum node. Supported format is HTTP/HTTPS. Note: this disables internal httpClient stats | | `ETH_REJECT_INVALID_CERTS` | `boolean` | Disable to allow ethereum client to connect to HTTPS without rejecting invalid (self-signed) certificates | | `ABI_DIR` | `string` | Directory containing ABI definitions (JSON files). This directory will be searched recursively | | `START_AT_BLOCK` | `string` | First block to start ingesting from. Possible values are "genesis", "latest", an absolute block number or a negative number describing how many blocks before the latest one to start at | diff --git a/docs/configuration.md b/docs/configuration.md index 4f2423b..a0002e5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -109,6 +109,7 @@ Settings for ethlogger connecting to the ethereum node via JSON RPC over HTTP | `validateCertificate` | `boolean` | If set to false, the HTTP client will ignore certificate errors (eg. when using self-signed certs) | | `requestKeepAlive` | `boolean` | Keep sockets to JSON RPC open | | `maxSockets` | `number` | Maximum number of sockets HEC will use (per host) | +| `proxyUrl` | `string` | Optional proxy URL to route HTTP requests through. Note: this disables internal httpClient stats | ### EthereumClient diff --git a/package.json b/package.json index 8f99339..fb7730d 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,8 @@ "bl": "^4.0.0", "debug": "^4.1.1", "fs-extra": "^8.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", "js-yaml": "^3.13.1", "lodash": "^4.17.15", "node-fetch": "^2.6.0", diff --git a/src/cliflags.ts b/src/cliflags.ts index 5b2eb49..65c4e4c 100644 --- a/src/cliflags.ts +++ b/src/cliflags.ts @@ -117,6 +117,11 @@ export const CLI_FLAGS = { env: 'ETH_RPC_URL', description: 'URL to reach the target ethereum node. Supported format is currently only HTTP(s) for JSON RPC', }), + 'eth-rpc-proxy': flags.string({ + env: 'ETH_RPC_PROXY', + description: + 'Optional proxy URL required to reach the target ethereum node. Supported format is HTTP/HTTPS. Note: this disables internal httpClient stats', + }), 'eth-reject-invalid-certs': flags.boolean({ allowNo: true, env: 'ETH_REJECT_INVALID_CERTS', diff --git a/src/config.ts b/src/config.ts index e53c8fe..84763ce 100644 --- a/src/config.ts +++ b/src/config.ts @@ -317,6 +317,8 @@ export interface HttpTransportConfigSchema { requestKeepAlive?: boolean; /** Maximum number of sockets HEC will use (per host) */ maxSockets?: number; + /** Optional proxy URL to route HTTP requests through. Note: this disables internal httpClient stats */ + proxyUrl?: string; } export interface HttpTransportConfig extends HttpTransportConfigSchema { @@ -741,6 +743,7 @@ export async function loadEthloggerConfig(flags: CliFlags, dryRun: boolean = fal maxSockets: defaults.eth?.http?.maxSockets!, requestKeepAlive: defaults.eth?.http?.requestKeepAlive!, timeout: parseDuration(defaults.eth?.http?.timeout!), + proxyUrl: flags['eth-rpc-proxy'] ?? defaults.eth?.http?.proxyUrl, validateCertificate: flags['eth-reject-invalid-certs'] ?? parseBooleanEnvVar(CLI_FLAGS['eth-reject-invalid-certs'].env) ?? diff --git a/src/eth/http.ts b/src/eth/http.ts index 71036c4..580ac76 100644 --- a/src/eth/http.ts +++ b/src/eth/http.ts @@ -6,6 +6,9 @@ import { isHttps } from '../utils/httputils'; import { AggregateMetric, httpClientStats } from '../utils/stats'; import { checkError, JsonRpcRequest, JsonRpcResponse, validateJsonRpcResponse } from './jsonrpc'; import { EthereumTransport } from './transport'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { HttpProxyAgent } from 'http-proxy-agent'; +import { parse } from 'url'; const { debug, trace } = createModuleDebug('eth:http'); @@ -29,7 +32,7 @@ const initialCounters = { export class HttpTransport implements EthereumTransport { private config: HttpTransportConfig & typeof CONFIG_DEFAULTS; - private httpAgent: HttpAgent | HttpsAgent; + private httpAgent: HttpAgent | HttpsAgent | HttpProxyAgent | HttpsProxyAgent; private counters = { ...initialCounters }; private aggregates = { requestDuration: new AggregateMetric(), batchSize: new AggregateMetric() }; @@ -39,12 +42,24 @@ export class HttpTransport implements EthereumTransport { keepAlive: true, maxSockets: 256, }; - this.httpAgent = isHttps(url) - ? new HttpsAgent({ - ...baseAgentOptions, - rejectUnauthorized: this.config.validateCertificate, - }) - : new HttpAgent(baseAgentOptions); + + if (this.config.proxyUrl) { + const proxyUrlOptions = parse(this.config.proxyUrl); + const baseProxyAgentOptions = { ...baseAgentOptions, ...proxyUrlOptions }; + this.httpAgent = isHttps(url) + ? new HttpsProxyAgent({ + ...baseProxyAgentOptions, + rejectUnauthorized: this.config.validateCertificate, + }) + : new HttpProxyAgent(baseProxyAgentOptions); + } else { + this.httpAgent = isHttps(url) + ? new HttpsAgent({ + ...baseAgentOptions, + rejectUnauthorized: this.config.validateCertificate, + }) + : new HttpAgent(baseAgentOptions); + } } public get source() { @@ -140,7 +155,9 @@ export class HttpTransport implements EthereumTransport { public get stats() { return { ...this.counters, - httpClient: httpClientStats(this.httpAgent.getCurrentStatus()), + httpClient: this.httpAgent.hasOwnProperty('getCurrentStatus') + ? httpClientStats((this.httpAgent as any).getCurrentStatus()) + : {}, }; } diff --git a/test/config.test.ts b/test/config.test.ts index 4efb70f..44e7ddc 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -42,6 +42,7 @@ test('defaults', async () => { }, "http": Object { "maxSockets": 256, + "proxyUrl": undefined, "requestKeepAlive": true, "timeout": 60000, "validateCertificate": true, diff --git a/yarn.lock b/yarn.lock index dc110f8..f386f6f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -681,6 +681,11 @@ lodash "^4.17.4" read-pkg-up "^7.0.0" +"@tootallnate/once@1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" + integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== + "@types/babel__core@^7.1.0": version "7.1.3" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.3.tgz#e441ea7df63cd080dfcd02ab199e6d16a735fc30" @@ -976,6 +981,13 @@ agent-base@4, agent-base@^4.3.0: dependencies: es6-promisify "^5.0.0" +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + agent-base@~4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9" @@ -2043,6 +2055,13 @@ debug@3.1.0: dependencies: ms "2.0.0" +debug@4: + version "4.3.2" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" + integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== + dependencies: + ms "2.1.2" + debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -3217,6 +3236,15 @@ http-proxy-agent@^2.1.0: agent-base "4" debug "3.1.0" +http-proxy-agent@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" + integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== + dependencies: + "@tootallnate/once" "1" + agent-base "6" + debug "4" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -3242,6 +3270,14 @@ https-proxy-agent@^3.0.0: agent-base "^4.3.0" debug "^3.1.0" +https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" @@ -4992,7 +5028,7 @@ ms@2.0.0: resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= -ms@^2.0.0, ms@^2.1.1: +ms@2.1.2, ms@^2.0.0, ms@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==