From 2dbbec0eecd94ecf256b6d638d9f027ea5ba9a10 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 24 Dec 2023 20:28:44 +0100 Subject: [PATCH 1/3] feat(proxy): support no_proxy --- README.md | 4 ++- lib/proxy.d.ts | 18 ++++++++++- src/proxy.ts | 86 +++++++++++++++++++++++++++++++++++--------------- 3 files changed, 81 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 7d35553..6dcf6d2 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,9 @@ Node.js has no built-in support for HTTP Proxies for fetch (see [nodejs/undici#1 This package bundles a compact and simple proxy-supported solution for both Node.js versions without native fetch using [HTTP Agent](https://github.com/TooTallNate/proxy-agents/tree/main/packages/proxy-agent) and versions with native fetch using [Undici Proxy Agent](https://undici.nodejs.org/#/docs/api/ProxyAgent). -By default, `https_proxy`, `http_proxy`, `HTTPS_PROXY`, and `HTTP_PROXY` environment variables will be checked and used (in order) for the proxy and if not any of them are set, the proxy will be disabled. +By default, `https_proxy`, `http_proxy`, `HTTPS_PROXY`, and `HTTP_PROXY` environment variables will be checked and used (in order) for the proxy and if not any of them are set, the proxy will be disabled. You can override it using `url` option passed to `createFetch` and `createProxy` utils. + +By default, `no_proxy` and `NO_PROXY` environment variables will be checked and used for the (comma seperated) list of hosts to ignore proxy for. You can override it using `noProxy` option passed to `createFetch` and `createProxy` utils. The enties staring with a dot will be used to check domain and also any subdomain. > [!NOTE] > Using export conditions, this utility adds proxy support for Node.js and for other runtimes, it will simply return native fetch. diff --git a/lib/proxy.d.ts b/lib/proxy.d.ts index f98fcee..8bdb103 100644 --- a/lib/proxy.d.ts +++ b/lib/proxy.d.ts @@ -2,7 +2,23 @@ import type * as http from "node:http"; import type * as https from "node:https"; import type * as undici from "undici"; -export type ProxyOptions = { url?: string }; +export type ProxyOptions = { + /** + * HTTP(s) Proxy URL + * + * Default is read from `https_proxy`, `http_proxy`, `HTTPS_PROXY` or `HTTP_PROXY` environment variables + * */ + url?: string; + + /** + * List of hosts to skip proxy for (comma seperated or array of strings) + * + * Default is read from `no_proxy` or `NO_PROXY` environment variables + * + * Hots starting with a leading dot like `.foo.com` are also matched against domain and all its subdomains like `bar.foo.com` + */ + noProxy?: string | string[]; +}; export declare const createProxy: (opts?: ProxyOptions) => { agent: http.Agent | https.Agent | undefined; diff --git a/src/proxy.ts b/src/proxy.ts index 0105e1d..8dab3af 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -1,7 +1,7 @@ import * as http from "node:http"; import * as https from "node:https"; import { URL } from "node:url"; -import { ProxyAgent as UndiciProxyAgent } from "undici"; +import { Agent as _UndiciAgent, ProxyAgent as _UndiciProxyAgent } from "undici"; import { Agent, AgentConnectOpts } from "agent-base"; import { HttpProxyAgent } from "http-proxy-agent"; import { HttpsProxyAgent } from "https-proxy-agent"; @@ -23,10 +23,16 @@ export function createProxy(opts: ProxyOptions = {}) { }; } - const nodeAgent = new NodeProxyAgent({ uri }); + const _noProxy = opts.noProxy || process.env.no_proxy || process.env.NO_PROXY; + const noProxy = typeof _noProxy === "string" ? _noProxy.split(",") : _noProxy; + + const nodeAgent = new NodeProxyAgent({ uri, noProxy }); // https://undici.nodejs.org/#/docs/api/ProxyAgent - const undiciAgent = new UndiciProxyAgent({ uri }); + const undiciAgent = new UndiciProxyAgent({ + uri, + noProxy, + }); return { agent: nodeAgent, @@ -45,9 +51,47 @@ export const fetch = createFetch({}); // Utils // ---------------------------------------------- -export function debug(...args: any[]) { - if (process.env.debug) { - debug("[node-fetch-native] [proxy]", ...args); +function debug(...args: any[]) { + if (process.env.DEBUG) { + console.debug("[node-fetch-native] [proxy]", ...args); + } +} + +function bypassProxy(host: string, noProxy: string[]) { + if (!noProxy) { + return false; + } + for (const _host of noProxy) { + if (_host === host || (_host[0] === "." && host.endsWith(_host.slice(1)))) { + return true; + } + } + return false; +} + +// ---------------------------------------------- +// Undici Agent +// ---------------------------------------------- + +// https://github.com/nodejs/undici/blob/main/lib/proxy-agent.js + +class UndiciProxyAgent extends _UndiciProxyAgent { + _agent: _UndiciAgent; + + constructor( + private _options: _UndiciProxyAgent.Options & { noProxy: string[] }, + ) { + super(_options); + this._agent = new _UndiciAgent(); + } + + dispatch(options, handler): boolean { + const hostname = new URL(options.origin).hostname; + if (bypassProxy(hostname, this._options.noProxy)) { + debug(`Bypassing proxy for: ${hostname}`); + return this._agent.dispatch(options, handler); + } + return super.dispatch(options, handler); } } @@ -73,15 +117,14 @@ function isValidProtocol(v: string): v is ValidProtocol { return (PROTOCOLS as readonly string[]).includes(v); } -export class NodeProxyAgent extends Agent { +class NodeProxyAgent extends Agent { cache: Map = new Map(); httpAgent: http.Agent; httpsAgent: http.Agent; - constructor(private proxyOptions: { uri: string }) { + constructor(private _options: { uri: string; noProxy: string[] }) { super({}); - debug("Creating new ProxyAgent instance: %o", proxyOptions); this.httpAgent = new http.Agent({}); this.httpsAgent = new https.Agent({}); } @@ -94,33 +137,26 @@ export class NodeProxyAgent extends Agent { ? (isWebSocket ? "wss:" : "https:") : (isWebSocket ? "ws:" : "http:"); - const host = req.getHeader("host"); - const url = new URL(req.path, `${protocol}//${host}`).href; - const proxy = this.proxyOptions.uri; + const host = req.getHeader("host") as string; - if (!proxy) { - debug("Proxy not enabled for URL: %o", url); + if (bypassProxy(host, this._options.noProxy)) { return opts.secureEndpoint ? this.httpsAgent : this.httpAgent; } - debug("Request URL: %o", url); - debug("Proxy URL: %o", proxy); - // Attempt to get a cached `http.Agent` instance first - const cacheKey = `${protocol}+${proxy}`; + const cacheKey = `${protocol}+${this._options.uri}`; let agent = this.cache.get(cacheKey); - if (agent) { - debug("Cache hit for proxy URL: %o", proxy); - } else { - const proxyUrl = new URL(proxy); + if (!agent) { + const proxyUrl = new URL(this._options.uri); const proxyProto = proxyUrl.protocol.replace(":", ""); if (!isValidProtocol(proxyProto)) { - throw new Error(`Unsupported protocol for proxy URL: ${proxy}`); + throw new Error( + `Unsupported protocol for proxy URL: ${this._options.uri}`, + ); } const Ctor = proxies[proxyProto][opts.secureEndpoint || isWebSocket ? 1 : 0]; - // @ts-expect-error meh… - agent = new Ctor(proxy, this.connectOpts); + agent = new (Ctor as any)(this._options.uri, this._options); this.cache.set(cacheKey, agent); } From ed16d526fe2564224a7e1ec07819063c08e686db Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 24 Dec 2023 20:30:10 +0100 Subject: [PATCH 2/3] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6dcf6d2..96e00f7 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Sometimes you want to explicitly use none native (`node-fetch`) implementation o You have two ways to do this: -- Set `FORCE_NODE_FETCH` environment variable before starting the application. +- Set the `FORCE_NODE_FETCH` environment variable before starting the application. - Import from `node-fetch-native/node` ## Polyfill support @@ -111,9 +111,9 @@ Node.js has no built-in support for HTTP Proxies for fetch (see [nodejs/undici#1 This package bundles a compact and simple proxy-supported solution for both Node.js versions without native fetch using [HTTP Agent](https://github.com/TooTallNate/proxy-agents/tree/main/packages/proxy-agent) and versions with native fetch using [Undici Proxy Agent](https://undici.nodejs.org/#/docs/api/ProxyAgent). -By default, `https_proxy`, `http_proxy`, `HTTPS_PROXY`, and `HTTP_PROXY` environment variables will be checked and used (in order) for the proxy and if not any of them are set, the proxy will be disabled. You can override it using `url` option passed to `createFetch` and `createProxy` utils. +By default, `https_proxy`, `http_proxy`, `HTTPS_PROXY`, and `HTTP_PROXY` environment variables will be checked and used (in order) for the proxy and if not any of them are set, the proxy will be disabled. You can override it using the `url` option passed to `createFetch` and `createProxy` utils. -By default, `no_proxy` and `NO_PROXY` environment variables will be checked and used for the (comma seperated) list of hosts to ignore proxy for. You can override it using `noProxy` option passed to `createFetch` and `createProxy` utils. The enties staring with a dot will be used to check domain and also any subdomain. +By default, `no_proxy` and `NO_PROXY` environment variables will be checked and used for the (comma-separated) list of hosts to ignore the proxy for. You can override it using the `noProxy` option passed to `createFetch` and `createProxy` utils. The entries starting with a dot will be used to check the domain and also any subdomain. > [!NOTE] > Using export conditions, this utility adds proxy support for Node.js and for other runtimes, it will simply return native fetch. @@ -133,7 +133,7 @@ console.log(await fetch("https://icanhazip.com").then((r) => r.text()); ### `createFetch` utility -You can use `createFetch` utility to intantiate a `fetch` instance with custom proxy options. +You can use the `createFetch` utility to instantiate a `fetch` instance with custom proxy options. ```ts import { createFetch } from "node-fetch-native/proxy"; From ad69b0609726e48acba99363007970c8a54ced5b Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Sun, 24 Dec 2023 20:30:35 +0100 Subject: [PATCH 3/3] Update proxy.d.ts --- lib/proxy.d.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/proxy.d.ts b/lib/proxy.d.ts index 8bdb103..81228d5 100644 --- a/lib/proxy.d.ts +++ b/lib/proxy.d.ts @@ -11,11 +11,11 @@ export type ProxyOptions = { url?: string; /** - * List of hosts to skip proxy for (comma seperated or array of strings) + * List of hosts to skip proxy for (comma separated or array of strings) * * Default is read from `no_proxy` or `NO_PROXY` environment variables * - * Hots starting with a leading dot like `.foo.com` are also matched against domain and all its subdomains like `bar.foo.com` + * Hots starting with a leading dot, like `.foo.com` are also matched against domain and all its subdomains like `bar.foo.com` */ noProxy?: string | string[]; };