From 5c5285978b0919e38223cc89abbbc4506689b444 Mon Sep 17 00:00:00 2001 From: MarcMcIntosh Date: Wed, 3 May 2023 17:05:06 +0200 Subject: [PATCH 1/4] fix(http-connect): try wrapping fetch so that it's passed and agent that has keepAlive enabled --- .../manager/src/auth/PrismicAuthManager.ts | 2 +- .../manager/src/lib/checkIsURLAccessible.ts | 2 +- packages/manager/src/lib/fetch.ts | 38 +++++++++++++++++++ .../lib/fetchGitHubReleaseBodyForRelease.ts | 2 +- .../src/lib/fetchNPMPackageVersions.ts | 2 +- .../customTypes/CustomTypesManager.ts | 2 +- .../PrismicRepositoryManager.ts | 2 +- .../screenshots/ScreenshotsManager.ts | 2 +- .../managers/simulator/SimulatorManager.ts | 2 +- .../src/managers/slices/SlicesManager.ts | 2 +- 10 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 packages/manager/src/lib/fetch.ts diff --git a/packages/manager/src/auth/PrismicAuthManager.ts b/packages/manager/src/auth/PrismicAuthManager.ts index 04b2163e31..c06620bffe 100644 --- a/packages/manager/src/auth/PrismicAuthManager.ts +++ b/packages/manager/src/auth/PrismicAuthManager.ts @@ -5,7 +5,7 @@ import * as os from "node:os"; import * as http from "node:http"; import * as h3 from "h3"; -import fetch from "node-fetch"; +import fetch from "../lib/fetch"; import cookie from "cookie"; import cors from "cors"; import getPort from "get-port"; diff --git a/packages/manager/src/lib/checkIsURLAccessible.ts b/packages/manager/src/lib/checkIsURLAccessible.ts index 7e0a6988a0..aaf577cc13 100644 --- a/packages/manager/src/lib/checkIsURLAccessible.ts +++ b/packages/manager/src/lib/checkIsURLAccessible.ts @@ -1,4 +1,4 @@ -import fetch from "node-fetch"; +import fetch from "./fetch"; export const checkIsURLAccessible = async (url: string): Promise => { const res = await fetch(url); diff --git a/packages/manager/src/lib/fetch.ts b/packages/manager/src/lib/fetch.ts new file mode 100644 index 0000000000..c20bb01050 --- /dev/null +++ b/packages/manager/src/lib/fetch.ts @@ -0,0 +1,38 @@ +import http from "node:http"; +import https from "node:https"; +import nodeFetch, { RequestInfo, RequestInit } from "node-fetch"; + +export * from "node-fetch"; + +const httpAgent = new http.Agent({ + keepAlive: true, +}); +const httpsAgent = new https.Agent({ + keepAlive: true, +}); + +const options: RequestInit = { + agent: function (_parsedURL) { + if (_parsedURL.protocol == "http:") { + return httpAgent; + } else { + return httpsAgent; + } + }, +}; + +/** + * Wrapper around node-fetch that passes an user-agent with the keepAlive option + * enabled + */ +export default function fetch( + url: URL | RequestInfo, + init?: RequestInit, +): ReturnType { + const opts = { + ...options, + ...init, + }; + + return nodeFetch(url, opts); +} diff --git a/packages/manager/src/lib/fetchGitHubReleaseBodyForRelease.ts b/packages/manager/src/lib/fetchGitHubReleaseBodyForRelease.ts index d39549e468..14383c20bc 100644 --- a/packages/manager/src/lib/fetchGitHubReleaseBodyForRelease.ts +++ b/packages/manager/src/lib/fetchGitHubReleaseBodyForRelease.ts @@ -1,5 +1,5 @@ import * as t from "io-ts"; -import fetch from "node-fetch"; +import fetch from "./fetch"; import pLimit from "p-limit"; import { decode } from "./decode"; diff --git a/packages/manager/src/lib/fetchNPMPackageVersions.ts b/packages/manager/src/lib/fetchNPMPackageVersions.ts index b719279032..78383ef95e 100644 --- a/packages/manager/src/lib/fetchNPMPackageVersions.ts +++ b/packages/manager/src/lib/fetchNPMPackageVersions.ts @@ -1,5 +1,5 @@ import * as t from "io-ts"; -import fetch from "node-fetch"; +import fetch from "./fetch"; import { decode } from "./decode"; diff --git a/packages/manager/src/managers/customTypes/CustomTypesManager.ts b/packages/manager/src/managers/customTypes/CustomTypesManager.ts index 49e6ed8059..37bae1b9bf 100644 --- a/packages/manager/src/managers/customTypes/CustomTypesManager.ts +++ b/packages/manager/src/managers/customTypes/CustomTypesManager.ts @@ -1,5 +1,5 @@ import * as t from "io-ts"; -import fetch from "node-fetch"; +import fetch from "../../lib/fetch"; import * as prismicCustomTypesClient from "@prismicio/custom-types-client"; import { CustomType } from "@prismicio/types-internal/lib/customtypes"; import { diff --git a/packages/manager/src/managers/prismicRepository/PrismicRepositoryManager.ts b/packages/manager/src/managers/prismicRepository/PrismicRepositoryManager.ts index 8d6c06dff7..0c29d21e20 100644 --- a/packages/manager/src/managers/prismicRepository/PrismicRepositoryManager.ts +++ b/packages/manager/src/managers/prismicRepository/PrismicRepositoryManager.ts @@ -1,5 +1,5 @@ import * as t from "io-ts"; -import fetch, { Response } from "node-fetch"; +import fetch, { Response } from "../../lib/fetch"; import { fold } from "fp-ts/Either"; import { decode } from "../../lib/decode"; diff --git a/packages/manager/src/managers/screenshots/ScreenshotsManager.ts b/packages/manager/src/managers/screenshots/ScreenshotsManager.ts index a238045fb5..ab62a69d06 100644 --- a/packages/manager/src/managers/screenshots/ScreenshotsManager.ts +++ b/packages/manager/src/managers/screenshots/ScreenshotsManager.ts @@ -1,6 +1,6 @@ import * as t from "io-ts"; import { fileTypeFromBuffer } from "file-type"; -import fetch, { FormData, Blob, Response } from "node-fetch"; +import fetch, { FormData, Blob, Response } from "../../lib/fetch"; // puppeteer is lazy-loaded in captureSliceSimulatorScreenshot import type { BrowserContext, Viewport } from "puppeteer"; diff --git a/packages/manager/src/managers/simulator/SimulatorManager.ts b/packages/manager/src/managers/simulator/SimulatorManager.ts index 239d9d28d0..318daddcd6 100644 --- a/packages/manager/src/managers/simulator/SimulatorManager.ts +++ b/packages/manager/src/managers/simulator/SimulatorManager.ts @@ -1,6 +1,6 @@ import * as t from "io-ts"; import { HookError } from "@slicemachine/plugin-kit"; -import fetch from "node-fetch"; +import fetch from "../../lib/fetch"; import { DecodeError } from "../../lib/DecodeError"; import { assertPluginsInitialized } from "../../lib/assertPluginsInitialized"; diff --git a/packages/manager/src/managers/slices/SlicesManager.ts b/packages/manager/src/managers/slices/SlicesManager.ts index be6db98a61..fbe5e46bf1 100644 --- a/packages/manager/src/managers/slices/SlicesManager.ts +++ b/packages/manager/src/managers/slices/SlicesManager.ts @@ -1,5 +1,5 @@ import * as t from "io-ts"; -import fetch from "node-fetch"; +import fetch from "../../lib/fetch"; import * as prismicCustomTypesClient from "@prismicio/custom-types-client"; import { SharedSliceContent } from "@prismicio/types-internal/lib/content"; import { From 4f1ab5596e09729442426d76cf5e152a448e48c9 Mon Sep 17 00:00:00 2001 From: MarcMcIntosh Date: Wed, 3 May 2023 18:30:07 +0200 Subject: [PATCH 2/4] refactor(fetch): remove sufixed _ --- packages/manager/src/lib/fetch.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/manager/src/lib/fetch.ts b/packages/manager/src/lib/fetch.ts index c20bb01050..cb38d2c53f 100644 --- a/packages/manager/src/lib/fetch.ts +++ b/packages/manager/src/lib/fetch.ts @@ -12,8 +12,8 @@ const httpsAgent = new https.Agent({ }); const options: RequestInit = { - agent: function (_parsedURL) { - if (_parsedURL.protocol == "http:") { + agent: function (parsedURL) { + if (parsedURL.protocol == "http:") { return httpAgent; } else { return httpsAgent; From 76de968e22e4fd0e6a39dc04b8beffc6f739792d Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 3 May 2023 09:41:53 -1000 Subject: [PATCH 3/4] refactor: `src/lib/fetch.ts` --- packages/manager/src/lib/fetch.ts | 66 +++++++++++++++++-------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/packages/manager/src/lib/fetch.ts b/packages/manager/src/lib/fetch.ts index cb38d2c53f..64d6ad2b99 100644 --- a/packages/manager/src/lib/fetch.ts +++ b/packages/manager/src/lib/fetch.ts @@ -1,38 +1,46 @@ -import http from "node:http"; -import https from "node:https"; -import nodeFetch, { RequestInfo, RequestInit } from "node-fetch"; +// This temporary wrapper around `node-fetch` fixes an issue where quick +// consecutive network requests causes failed requests. +// +// See https://github.com/node-fetch/node-fetch/issues/1735 for more details. +// +// TODO: Remove this wrapper and replace all imports with `node-fetch` if https://github.com/node-fetch/node-fetch/pull/1736 is merged. + +import * as http from "node:http"; +import * as https from "node:https"; +import baseFetch from "node-fetch"; export * from "node-fetch"; -const httpAgent = new http.Agent({ - keepAlive: true, -}); -const httpsAgent = new https.Agent({ - keepAlive: true, -}); +/** + * The default HTTP Agent with `keepAlive: true` used in `fetch()` requests. + */ +const DEFAULT_HTTP_AGENT = new http.Agent({ keepAlive: true }); -const options: RequestInit = { - agent: function (parsedURL) { - if (parsedURL.protocol == "http:") { - return httpAgent; - } else { - return httpsAgent; - } - }, -}; +/** + * The default HTTPS Agent with `keepAlive: true` used in `fetch()` requests. + */ +const DEFAULT_HTTPS_AGENT = new https.Agent({ keepAlive: true }); /** - * Wrapper around node-fetch that passes an user-agent with the keepAlive option - * enabled + * Patched `fetch()` from `node-fetch` that fixes a bug where quick consecutive + * network requests cause failed requests. + * + * Use this `fetch()` in place of `node-fetch`'s `fetch()`. + * + * @remarks + * `fetch()` is patched by setting an HTTP/HTTPS Agent with `keepAlive: true`. + * If you need to assign an Agent, be sure to retain the `keepAlive: true` + * option. */ -export default function fetch( - url: URL | RequestInfo, - init?: RequestInit, -): ReturnType { - const opts = { - ...options, +const fetch: typeof baseFetch = (url, init) => { + return baseFetch(url, { + agent: (parsedURL) => { + return parsedURL.protocol === "http:" + ? DEFAULT_HTTP_AGENT + : DEFAULT_HTTPS_AGENT; + }, ...init, - }; + }); +}; - return nodeFetch(url, opts); -} +export default fetch; From 5a62eff5b8c33dd6fbcefd5c30529e4e068b46a8 Mon Sep 17 00:00:00 2001 From: Baptiste Morelle Date: Thu, 4 May 2023 10:15:12 +0200 Subject: [PATCH 4/4] fixing a small typo --- packages/manager/src/lib/fetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/manager/src/lib/fetch.ts b/packages/manager/src/lib/fetch.ts index 64d6ad2b99..091ee8e66f 100644 --- a/packages/manager/src/lib/fetch.ts +++ b/packages/manager/src/lib/fetch.ts @@ -1,5 +1,5 @@ // This temporary wrapper around `node-fetch` fixes an issue where quick -// consecutive network requests causes failed requests. +// consecutive network requests cause failed requests. // // See https://github.com/node-fetch/node-fetch/issues/1735 for more details. //