diff --git a/src/connectors/http-utils.ts b/src/connectors/http-utils.ts index e467493..2ff063f 100644 --- a/src/connectors/http-utils.ts +++ b/src/connectors/http-utils.ts @@ -1,5 +1,6 @@ import { getLogger } from "../logger.js"; import { FetchError } from "../errors.js"; +import { loadConfig } from "../config.js"; export interface RetryConfig { maxRetries?: number; @@ -9,6 +10,7 @@ export interface RetryConfig { /** * Fetch wrapper with retry logic for 429 (rate-limit) and 5xx responses. * Uses Retry-After header when available, otherwise exponential backoff. + * Respects `indexing.allowSelfSignedCerts` config for corporate TLS. */ export async function fetchWithRetry( url: string, @@ -19,37 +21,53 @@ export async function fetchWithRetry( const baseDelay = retryConfig?.baseDelay ?? 1000; const log = getLogger(); - for (let attempt = 0; attempt <= maxRetries; attempt++) { - const response = await fetch(url, options); + const config = loadConfig(); + const prevTls = process.env["NODE_TLS_REJECT_UNAUTHORIZED"]; + if (config.indexing.allowSelfSignedCerts) { + process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = "0"; + } - if (response.status === 429 || (response.status >= 500 && response.status < 600)) { - if (attempt >= maxRetries) { - const body = await response.text().catch(() => ""); - throw new FetchError(`HTTP ${response.status} after ${maxRetries + 1} attempts: ${body}`); - } + try { + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const response = await fetch(url, options); + + if (response.status === 429 || (response.status >= 500 && response.status < 600)) { + if (attempt >= maxRetries) { + const body = await response.text().catch(() => ""); + throw new FetchError(`HTTP ${response.status} after ${maxRetries + 1} attempts: ${body}`); + } - let delayMs = baseDelay * 2 ** attempt; - if (response.status === 429) { - const retryAfter = response.headers.get("Retry-After"); - if (retryAfter) { - const parsed = Number(retryAfter); - if (!Number.isNaN(parsed)) { - delayMs = parsed * 1000; + let delayMs = baseDelay * 2 ** attempt; + if (response.status === 429) { + const retryAfter = response.headers.get("Retry-After"); + if (retryAfter) { + const parsed = Number(retryAfter); + if (!Number.isNaN(parsed)) { + delayMs = parsed * 1000; + } } } + + log.warn( + { status: response.status, attempt: attempt + 1, delayMs }, + "Retrying after transient error", + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + continue; } - log.warn( - { status: response.status, attempt: attempt + 1, delayMs }, - "Retrying after transient error", - ); - await new Promise((resolve) => setTimeout(resolve, delayMs)); - continue; + return response; } - return response; + // Unreachable, but satisfies TypeScript + throw new FetchError("fetchWithRetry: unexpected code path"); + } finally { + if (config.indexing.allowSelfSignedCerts) { + if (prevTls === undefined) { + delete process.env["NODE_TLS_REJECT_UNAUTHORIZED"]; + } else { + process.env["NODE_TLS_REJECT_UNAUTHORIZED"] = prevTls; + } + } } - - // Unreachable, but satisfies TypeScript - throw new FetchError("fetchWithRetry: unexpected code path"); } diff --git a/tests/unit/http-utils.test.ts b/tests/unit/http-utils.test.ts index 4bfa214..b39f97e 100644 --- a/tests/unit/http-utils.test.ts +++ b/tests/unit/http-utils.test.ts @@ -15,6 +15,13 @@ vi.mock("../../src/logger.js", () => ({ }), })); +// Mock config +vi.mock("../../src/config.js", () => ({ + loadConfig: () => ({ + indexing: { allowSelfSignedCerts: false, allowPrivateUrls: false, maxDocumentSize: 104857600 }, + }), +})); + const { fetchWithRetry } = await import("../../src/connectors/http-utils.js"); function jsonResponse(data: unknown, status = 200, headers?: Record): Response {