Merge pull request #3432 from NomicFoundation/improve-proxy-support

Improve proxy support

.changeset/dry-guests-count.md

@@ -0,0 +1,5 @@
+---
+"@nomiclabs/hardhat-etherscan": patch
+---
+
+Added support for the `http_proxy` environment variable. When this variable is set, `hardhat-etherscan` will use the given proxy to send the verification requests.

.changeset/few-flies-drum.md

@@ -0,0 +1,11 @@
+---
+"hardhat": patch
+---
+
+Added support for the `http_proxy` environment variable. When this variable is set, Hardhat will send its requests through the given proxy for things like JSON-RPC requests, mainnet forking and downloading compilers.
+
+We also removed support for the `HTTP_PROXY` and `HTTPS_PROXY` environment variables, since `http_proxy` is the most commonly used environment variable for this kind of thing. Those variables could only be used for downloading compilers.
+
+Finally, we also added support for `no_proxy`, which accepts a comma separated list of hosts or `"*"`. Any host included in this list will not be proxied.
+
+Note that requests to `"localhost"` or `"127.0.0.1"` are never proxied.

docs/src/content/hardhat-runner/docs/troubleshooting/common-problems.md

@@ -13,3 +13,9 @@ npx hardhat --max-memory 4096 compile
 ```
 
 If you find yourself using this all the time, you can set it with an environment variable in your `.bashrc` (if using bash) or `.zshrc` (if using zsh): `export HARDHAT_MAX_MEMORY=4096`.
+
+## Using Hardhat with a proxy server
+
+Hardhat supports the `http_proxy` environment variable. When this variable is set, Hardhat will send its requests through the given proxy for things like JSON-RPC requests, mainnet forking and downloading compilers.
+
+There's also support for the `no_proxy` variable, which accepts a comma separated list of hosts or `"*"`. Any host included in this list will not be proxied. Note that requests to `"localhost"` or `"127.0.0.1"` are never proxied.

packages/hardhat-core/package.json

@@ -90,7 +90,6 @@
     "ethers": "^5.0.0",
     "mocha": "^10.0.0",
     "prettier": "2.4.1",
-    "proxy": "^1.0.2",
     "rimraf": "^3.0.2",
     "sinon": "^9.0.0",
     "time-require": "^0.1.2",

packages/hardhat-core/src/internal/core/providers/http.ts

@@ -1,4 +1,4 @@
-import type { Dispatcher, Pool as PoolT } from "undici";
+import type * as Undici from "undici";
 
 import { EventEmitter } from "events";
 
@@ -17,6 +17,7 @@ import {
 import { getHardhatVersion } from "../../util/packageInfo";
 import { HardhatError } from "../errors";
 import { ERRORS } from "../errors-list";
+import { shouldUseProxy } from "../../util/proxy";
 
 import { ProviderError } from "./errors";
 
@@ -33,7 +34,7 @@ const hardhatVersion = getHardhatVersion();
 
 export class HttpProvider extends EventEmitter implements EIP1193Provider {
   private _nextRequestId = 1;
-  private _dispatcher: Dispatcher;
+  private _dispatcher: Undici.Dispatcher;
   private _path: string;
   private _authHeader: string | undefined;
 
@@ -42,11 +43,11 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider {
     private readonly _networkName: string,
     private readonly _extraHeaders: { [name: string]: string } = {},
     private readonly _timeout = 20000,
-    client: Dispatcher | undefined = undefined
+    client: Undici.Dispatcher | undefined = undefined
   ) {
     super();
 
-    const { Pool } = require("undici") as { Pool: typeof PoolT };
+    const { Pool, ProxyAgent } = require("undici") as typeof Undici;
 
     const url = new URL(this._url);
     this._path = url.pathname;
@@ -59,6 +60,10 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider {
           ).toString("base64")}`;
     try {
       this._dispatcher = client ?? new Pool(url.origin);
+
+      if (process.env.http_proxy !== undefined && shouldUseProxy(url.origin)) {
+        this._dispatcher = new ProxyAgent(process.env.http_proxy);
+      }
     } catch (e) {
       if (e instanceof TypeError && e.message === "Invalid URL") {
         e.message += ` ${url.origin}`;
@@ -163,10 +168,13 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider {
     request: JsonRpcRequest | JsonRpcRequest[],
     retryNumber = 0
   ): Promise<JsonRpcResponse | JsonRpcResponse[]> {
+    const { request: sendRequest } = await import("undici");
+    const url = new URL(this._url);
+
     try {
-      const response = await this._dispatcher.request({
+      const response = await sendRequest(url, {
+        dispatcher: this._dispatcher,
         method: "POST",
-        path: this._path,
         body: JSON.stringify(request),
         maxRedirections: 10,
         headersTimeout:
@@ -194,8 +202,6 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider {
           return await this._retry(request, seconds, retryNumber);
         }
 
-        const url = new URL(this._url);
-
         // eslint-disable-next-line @nomiclabs/hardhat-internal-rules/only-hardhat-error
         throw new ProviderError(
           `Too Many Requests error received from ${url.hostname}`,
@@ -255,12 +261,12 @@ export class HttpProvider extends EventEmitter implements EIP1193Provider {
     return true;
   }
 
-  private _isRateLimitResponse(response: Dispatcher.ResponseData) {
+  private _isRateLimitResponse(response: Undici.Dispatcher.ResponseData) {
     return response.statusCode === TOO_MANY_REQUEST_STATUS;
   }
 
   private _getRetryAfterSeconds(
-    response: Dispatcher.ResponseData
+    response: Undici.Dispatcher.ResponseData
   ): number | undefined {
     const header = response.headers["retry-after"];
 

packages/hardhat-core/src/internal/util/download.ts

@@ -1,9 +1,12 @@
+import type { Dispatcher } from "undici";
+
 import fs from "fs";
 import fsExtra from "fs-extra";
 import path from "path";
 import util from "util";
 
 import { getHardhatVersion } from "./packageInfo";
+import { shouldUseProxy } from "./proxy";
 
 const TEMP_FILE_PREFIX = "tmp-";
 
@@ -27,23 +30,18 @@ export async function download(
   const { getGlobalDispatcher, ProxyAgent, request } = await import("undici");
   const streamPipeline = util.promisify(pipeline);
 
-  function chooseDispatcher() {
-    if (process.env.HTTPS_PROXY !== undefined) {
-      return new ProxyAgent(process.env.HTTPS_PROXY);
-    }
-
-    if (process.env.HTTP_PROXY !== undefined) {
-      return new ProxyAgent(process.env.HTTP_PROXY);
-    }
-
-    return getGlobalDispatcher();
+  let dispatcher: Dispatcher;
+  if (process.env.http_proxy !== undefined && shouldUseProxy(url)) {
+    dispatcher = new ProxyAgent(process.env.http_proxy);
+  } else {
+    dispatcher = getGlobalDispatcher();
   }
 
   const hardhatVersion = getHardhatVersion();
 
   // Fetch the url
   const response = await request(url, {
-    dispatcher: chooseDispatcher(),
+    dispatcher,
     headersTimeout: timeoutMillis,
     maxRedirections: 10,
     method: "GET",
@@ -59,6 +57,9 @@ export async function download(
 
     await streamPipeline(response.body, fs.createWriteStream(tmpFilePath));
     return fsExtra.move(tmpFilePath, filePath, { overwrite: true });
+  } else {
+    // undici's response bodies must always be consumed to prevent leaks
+    await response.body.text();
   }
 
   // eslint-disable-next-line @nomiclabs/hardhat-internal-rules/only-hardhat-error

packages/hardhat-core/src/internal/util/proxy.ts

@@ -0,0 +1,18 @@
+export function shouldUseProxy(url: string): boolean {
+  const { hostname } = new URL(url);
+  const noProxy = process.env.NO_PROXY;
+
+  if (hostname === "localhost" || hostname === "127.0.0.1" || noProxy === "*") {
+    return false;
+  }
+
+  if (noProxy !== undefined && noProxy !== "") {
+    const noProxyList = noProxy.split(",");
+
+    if (noProxyList.includes(hostname)) {
+      return false;
+    }
+  }
+
+  return true;
+}

packages/hardhat-core/test/internal/util/download.ts

@@ -1,9 +1,6 @@
 import { assert } from "chai";
 import fsExtra from "fs-extra";
 import path from "path";
-// @ts-ignore
-// eslint-disable-next-line  import/no-extraneous-dependencies
-import Proxy from "proxy";
 
 import { download } from "../../../src/internal/util/download";
 import { useTmpDir } from "../../helpers/fs";
@@ -24,76 +21,3 @@ describe("Compiler List download", function () {
     });
   });
 });
-
-describe("Compiler List download with proxy", function () {
-  let env: typeof process.env;
-  let proxy: any;
-  let proxyPort: number;
-
-  useTmpDir("compiler-downloader");
-
-  before(function (done) {
-    // Setup Proxy Server
-    proxy = new Proxy();
-    proxy.listen(function () {
-      proxyPort = proxy.address().port;
-      done();
-    });
-  });
-
-  describe("Compilers list download with HTTPS_PROXY", function () {
-    before(function () {
-      // Save the Environment Settings and Set
-      env = process.env;
-      process.env.HTTPS_PROXY = `http://127.0.0.1:${proxyPort}`;
-    });
-
-    it("Should call download with the right params", async function () {
-      const compilersDir = this.tmpDir;
-      const downloadPath = path.join(compilersDir, "downloadedCompilerProxy");
-      const expectedUrl = `http://solc-bin.ethereum.org/wasm/list.json`;
-
-      // download the file
-      await download(expectedUrl, downloadPath);
-      // Assert that the file exists
-      assert.isTrue(await fsExtra.pathExists(downloadPath));
-    });
-
-    after(function () {
-      // restoring everything back to the environment
-      process.env = env;
-    });
-  });
-
-  describe("Compilers list download with HTTP_PROXY", function () {
-    before(function () {
-      // Save the Environment Settings and Set
-      env = process.env;
-      process.env.HTTP_PROXY = `http://127.0.0.1:${proxyPort}`;
-    });
-
-    it("Should call download with the right params", async function () {
-      const compilersDir = this.tmpDir;
-      const downloadPath = path.join(compilersDir, "downloadedCompilerProxy");
-      const expectedUrl = `http://solc-bin.ethereum.org/wasm/list.json`;
-
-      // download the file
-      await download(expectedUrl, downloadPath);
-      // Assert that the file exists
-      assert.isTrue(await fsExtra.pathExists(downloadPath));
-    });
-
-    after(function () {
-      // restoring everything back to the environment
-      process.env = env;
-    });
-  });
-
-  after(function (done) {
-    // Shutdown Proxy Server
-    proxy.once("close", function () {
-      done();
-    });
-    proxy.close();
-  });
-});

packages/hardhat-etherscan/src/etherscan/EtherscanService.ts

@@ -2,6 +2,7 @@ import { NomicLabsHardhatPluginError } from "hardhat/plugins";
 import { Dispatcher } from "undici";
 
 import { pluginName } from "../constants";
+import { sendGetRequest, sendPostRequest } from "../undici";
 
 import {
   EtherscanCheckStatusRequest,
@@ -19,18 +20,11 @@ export async function verifyContract(
   url: string,
   req: EtherscanVerifyRequest
 ): Promise<EtherscanResponse> {
-  const { request } = await import("undici");
   const parameters = new URLSearchParams({ ...req });
-  const method: Dispatcher.HttpMethod = "POST";
-  const requestDetails = {
-    method,
-    headers: { "Content-Type": "application/x-www-form-urlencoded" },
-    body: parameters.toString(),
-  };
 
   let response: Dispatcher.ResponseData;
   try {
-    response = await request(url, requestDetails);
+    response = await sendPostRequest(new URL(url), parameters.toString());
   } catch (error: any) {
     throw new NomicLabsHardhatPluginError(
       pluginName,
@@ -84,10 +78,9 @@ export async function getVerificationStatus(
   const urlWithQuery = new URL(url);
   urlWithQuery.search = parameters.toString();
 
-  const { request } = await import("undici");
   let response;
   try {
-    response = await request(urlWithQuery, { method: "GET" });
+    response = await sendGetRequest(urlWithQuery);
 
     if (!(response.statusCode >= 200 && response.statusCode <= 299)) {
       // This could be always interpreted as JSON if there were any such guarantee in the Etherscan API.

packages/hardhat-etherscan/src/solc/version.ts

@@ -1,6 +1,7 @@
 import { NomicLabsHardhatPluginError } from "hardhat/plugins";
 
 import { pluginName } from "../constants";
+import { sendGetRequest } from "../undici";
 
 const COMPILERS_LIST_URL = "https://solc-bin.ethereum.org/bin/list.json";
 
@@ -29,9 +30,8 @@ export async function getLongVersion(shortVersion: string): Promise<string> {
 
 export async function getVersions(): Promise<CompilersList> {
   try {
-    const { request } = await import("undici");
     // It would be better to query an etherscan API to get this list but there's no such API yet.
-    const response = await request(COMPILERS_LIST_URL, { method: "GET" });
+    const response = await sendGetRequest(new URL(COMPILERS_LIST_URL));
 
     if (!(response.statusCode >= 200 && response.statusCode <= 299)) {
       const responseText = await response.body.text();

packages/hardhat-etherscan/src/undici.ts

@@ -0,0 +1,38 @@
+import type * as Undici from "undici";
+
+function getDispatcher(): Undici.Dispatcher {
+  const { ProxyAgent, getGlobalDispatcher } =
+    require("undici") as typeof Undici;
+  if (process.env.http_proxy !== undefined) {
+    return new ProxyAgent(process.env.http_proxy);
+  }
+
+  return getGlobalDispatcher();
+}
+
+export async function sendGetRequest(
+  url: URL
+): Promise<Undici.Dispatcher.ResponseData> {
+  const { request } = await import("undici");
+  const dispatcher = getDispatcher();
+
+  return request(url, {
+    dispatcher,
+    method: "GET",
+  });
+}
+
+export async function sendPostRequest(
+  url: URL,
+  body: string
+): Promise<Undici.Dispatcher.ResponseData> {
+  const { request } = await import("undici");
+  const dispatcher = getDispatcher();
+
+  return request(url, {
+    dispatcher,
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body,
+  });
+}