Skip to content

Commit

Permalink
feat: 🎸 add http/s proxy support for RPC endpoints
Browse files Browse the repository at this point in the history
  • Loading branch information
@ccordi
ccordi committed Sep 14, 2021
1 parent b566ce7 commit f0436fc
Show file tree
Hide file tree
Showing 9 changed files with 83 additions and 9 deletions.
4 changes: 4 additions & 0 deletions config.schema.json
View file
Expand Up @@ -349,6 +349,10 @@
"description": "Maximum number of sockets HEC will use (per host)",
"type": "number"
},
"proxyUrl": {
"description": "Optional proxy URL to route HTTP requests through. Note: this disables internal httpClient stats",
"type": "string"
},
"requestKeepAlive": {
"description": "Keep sockets to JSON RPC open",
"type": "boolean"
Expand Down
5 changes: 5 additions & 0 deletions docs/cli.md
View file
Expand Up @@ -61,6 +61,10 @@ OPTIONS
Disable to allow ethereum client to connect to HTTPS without rejecting
invalid (self-signed) certificates
--eth-rpc-proxy=eth-rpc-proxy
Optional proxy URL required to reach the target ethereum node. Supported
format is HTTP/HTTPS. Note: this disables internal httpClient stats
--eth-rpc-url=eth-rpc-url
URL to reach the target ethereum node. Supported format is currently only
HTTP(s) for JSON RPC
Expand Down Expand Up @@ -147,6 +151,7 @@ OPTIONS
| `SPLUNK_METRICS_HEC_TOKEN` | `string` | HEC token to use for sending metrics. You can alternatively configure different indexes to correctly route your data |
| `SPLUNK_INTERNAL_HEC_TOKEN` | `string` | HEC token to use for sending internal metrics. You can alternatively configure different indexes to correctly route your data |
| `ETH_RPC_URL` | `string` | URL to reach the target ethereum node. Supported format is currently only HTTP(s) for JSON RPC |
| `ETH_RPC_PROXY` | `string` | Optional proxy URL required to reach the target ethereum node. Supported format is HTTP/HTTPS. Note: this disables internal httpClient stats |
| `ETH_REJECT_INVALID_CERTS` | `boolean` | Disable to allow ethereum client to connect to HTTPS without rejecting invalid (self-signed) certificates |
| `ABI_DIR` | `string` | Directory containing ABI definitions (JSON files). This directory will be searched recursively |
| `START_AT_BLOCK` | `string` | First block to start ingesting from. Possible values are "genesis", "latest", an absolute block number or a negative number describing how many blocks before the latest one to start at |
Expand Down
1 change: 1 addition & 0 deletions docs/configuration.md
View file
Expand Up @@ -109,6 +109,7 @@ Settings for ethlogger connecting to the ethereum node via JSON RPC over HTTP
| `validateCertificate` | `boolean` | If set to false, the HTTP client will ignore certificate errors (eg. when using self-signed certs) |
| `requestKeepAlive` | `boolean` | Keep sockets to JSON RPC open |
| `maxSockets` | `number` | Maximum number of sockets HEC will use (per host) |
| `proxyUrl` | `string` | Optional proxy URL to route HTTP requests through. Note: this disables internal httpClient stats |

### EthereumClient

Expand Down
2 changes: 2 additions & 0 deletions package.json
View file
Expand Up @@ -22,6 +22,8 @@
"bl": "^4.0.0",
"debug": "^4.1.1",
"fs-extra": "^8.1.0",
"http-proxy-agent": "^4.0.1",
"https-proxy-agent": "^5.0.0",
"js-yaml": "^3.13.1",
"lodash": "^4.17.15",
"node-fetch": "^2.6.0",
Expand Down
5 changes: 5 additions & 0 deletions src/cliflags.ts
View file
Expand Up @@ -117,6 +117,11 @@ export const CLI_FLAGS = {
env: 'ETH_RPC_URL',
description: 'URL to reach the target ethereum node. Supported format is currently only HTTP(s) for JSON RPC',
}),
'eth-rpc-proxy': flags.string({
env: 'ETH_RPC_PROXY',
description:
'Optional proxy URL required to reach the target ethereum node. Supported format is HTTP/HTTPS. Note: this disables internal httpClient stats',
}),
'eth-reject-invalid-certs': flags.boolean({
allowNo: true,
env: 'ETH_REJECT_INVALID_CERTS',
Expand Down
3 changes: 3 additions & 0 deletions src/config.ts
View file
Expand Up @@ -317,6 +317,8 @@ export interface HttpTransportConfigSchema {
requestKeepAlive?: boolean;
/** Maximum number of sockets HEC will use (per host) */
maxSockets?: number;
/** Optional proxy URL to route HTTP requests through. Note: this disables internal httpClient stats */
proxyUrl?: string;
}

export interface HttpTransportConfig extends HttpTransportConfigSchema {
Expand Down Expand Up @@ -741,6 +743,7 @@ export async function loadEthloggerConfig(flags: CliFlags, dryRun: boolean = fal
maxSockets: defaults.eth?.http?.maxSockets!,
requestKeepAlive: defaults.eth?.http?.requestKeepAlive!,
timeout: parseDuration(defaults.eth?.http?.timeout!),
proxyUrl: flags['eth-rpc-proxy'] ?? defaults.eth?.http?.proxyUrl,
validateCertificate:
flags['eth-reject-invalid-certs'] ??
parseBooleanEnvVar(CLI_FLAGS['eth-reject-invalid-certs'].env) ??
Expand Down
33 changes: 25 additions & 8 deletions src/eth/http.ts
View file
Expand Up @@ -6,6 +6,9 @@ import { isHttps } from '../utils/httputils';
import { AggregateMetric, httpClientStats } from '../utils/stats';
import { checkError, JsonRpcRequest, JsonRpcResponse, validateJsonRpcResponse } from './jsonrpc';
import { EthereumTransport } from './transport';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { HttpProxyAgent } from 'http-proxy-agent';
import { parse } from 'url';

const { debug, trace } = createModuleDebug('eth:http');

Expand All @@ -29,7 +32,7 @@ const initialCounters = {

export class HttpTransport implements EthereumTransport {
private config: HttpTransportConfig & typeof CONFIG_DEFAULTS;
private httpAgent: HttpAgent | HttpsAgent;
private httpAgent: HttpAgent | HttpsAgent | HttpProxyAgent | HttpsProxyAgent;
private counters = { ...initialCounters };
private aggregates = { requestDuration: new AggregateMetric(), batchSize: new AggregateMetric() };

Expand All @@ -39,12 +42,24 @@ export class HttpTransport implements EthereumTransport {
keepAlive: true,
maxSockets: 256,
};
this.httpAgent = isHttps(url)
? new HttpsAgent({
...baseAgentOptions,
rejectUnauthorized: this.config.validateCertificate,
})
: new HttpAgent(baseAgentOptions);

if (this.config.proxyUrl) {
const proxyUrlOptions = parse(this.config.proxyUrl);
const baseProxyAgentOptions = { ...baseAgentOptions, ...proxyUrlOptions };
this.httpAgent = isHttps(url)
? new HttpsProxyAgent({
...baseProxyAgentOptions,
rejectUnauthorized: this.config.validateCertificate,
})
: new HttpProxyAgent(baseProxyAgentOptions);
} else {
this.httpAgent = isHttps(url)
? new HttpsAgent({
...baseAgentOptions,
rejectUnauthorized: this.config.validateCertificate,
})
: new HttpAgent(baseAgentOptions);
}
}

public get source() {
Expand Down Expand Up @@ -140,7 +155,9 @@ export class HttpTransport implements EthereumTransport {
public get stats() {
return {
...this.counters,
httpClient: httpClientStats(this.httpAgent.getCurrentStatus()),
httpClient: this.httpAgent.hasOwnProperty('getCurrentStatus')
? httpClientStats((this.httpAgent as any).getCurrentStatus())
: {},
};
}

Expand Down
1 change: 1 addition & 0 deletions test/config.test.ts
View file
Expand Up @@ -42,6 +42,7 @@ test('defaults', async () => {
},
"http": Object {
"maxSockets": 256,
"proxyUrl": undefined,
"requestKeepAlive": true,
"timeout": 60000,
"validateCertificate": true,
Expand Down
38 changes: 37 additions & 1 deletion yarn.lock
View file
Expand Up @@ -681,6 +681,11 @@
lodash "^4.17.4"
read-pkg-up "^7.0.0"

"@tootallnate/once@1":
version "1.1.2"
resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==

"@types/babel__core@^7.1.0":
version "7.1.3"
resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.3.tgz#e441ea7df63cd080dfcd02ab199e6d16a735fc30"
Expand Down Expand Up @@ -976,6 +981,13 @@ agent-base@4, agent-base@^4.3.0:
dependencies:
es6-promisify "^5.0.0"

agent-base@6:
version "6.0.2"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==
dependencies:
debug "4"

agent-base@~4.2.1:
version "4.2.1"
resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9"
Expand Down Expand Up @@ -2043,6 +2055,13 @@ debug@3.1.0:
dependencies:
ms "2.0.0"

debug@4:
version "4.3.2"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"
integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==
dependencies:
ms "2.1.2"

debug@^2.2.0, debug@^2.3.3:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
Expand Down Expand Up @@ -3217,6 +3236,15 @@ http-proxy-agent@^2.1.0:
agent-base "4"
debug "3.1.0"

http-proxy-agent@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a"
integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==
dependencies:
"@tootallnate/once" "1"
agent-base "6"
debug "4"

http-signature@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
Expand All @@ -3242,6 +3270,14 @@ https-proxy-agent@^3.0.0:
agent-base "^4.3.0"
debug "^3.1.0"

https-proxy-agent@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2"
integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==
dependencies:
agent-base "6"
debug "4"

human-signals@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3"
Expand Down Expand Up @@ -4992,7 +5028,7 @@ ms@2.0.0:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=

ms@^2.0.0, ms@^2.1.1:
ms@2.1.2, ms@^2.0.0, ms@^2.1.1:
version "2.1.2"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
Expand Down

0 comments on commit f0436fc

Please sign in to comment.