From c424e90ac4aee486678ea85d70804afb7e8d8d71 Mon Sep 17 00:00:00 2001 From: Thibault Ducret Date: Wed, 19 Nov 2025 23:39:19 +0100 Subject: [PATCH 1/3] feat: Replace agentkeepalive with native Node.js HTTP agents for proxy support Replace the agentkeepalive dependency with Node.js native http.Agent and https.Agent to enable HTTP proxy support. The native agents support the HTTP_PROXY and HTTPS_PROXY environment variables out of the box, allowing the client to work seamlessly with proxy configurations. This change maintains the same keep-alive functionality while removing an external dependency and adding proxy support capability. --- package-lock.json | 23 +---------------------- package.json | 1 - src/http_client.ts | 15 +++++++++------ 3 files changed, 10 insertions(+), 29 deletions(-) diff --git a/package-lock.json b/package-lock.json index 321a39dc..02462127 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,6 @@ "@apify/log": "^2.2.6", "@apify/utilities": "^2.23.2", "@crawlee/types": "^3.3.0", - "agentkeepalive": "^4.2.1", "ansi-colors": "^4.1.1", "async-retry": "^1.3.3", "axios": "^1.6.7", @@ -4472,18 +4471,6 @@ "node": ">= 14" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", @@ -8856,15 +8843,6 @@ "node": ">=10.17.0" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -10940,6 +10918,7 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, "license": "MIT" }, "node_modules/nanoid": { diff --git a/package.json b/package.json index b737c3f1..5ec5df42 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,6 @@ "@apify/log": "^2.2.6", "@apify/utilities": "^2.23.2", "@crawlee/types": "^3.3.0", - "agentkeepalive": "^4.2.1", "ansi-colors": "^4.1.1", "async-retry": "^1.3.3", "axios": "^1.6.7", diff --git a/src/http_client.ts b/src/http_client.ts index d8dbb9ba..6e2b70ce 100644 --- a/src/http_client.ts +++ b/src/http_client.ts @@ -1,6 +1,7 @@ +import http from 'node:http'; +import https from 'node:https'; import os from 'node:os'; -import KeepAliveAgent from 'agentkeepalive'; import type { RetryFunction } from 'async-retry'; import retry from 'async-retry'; import type { AxiosError, AxiosInstance, AxiosRequestConfig, AxiosResponse, InternalAxiosRequestConfig } from 'axios'; @@ -32,9 +33,9 @@ export class HttpClient { timeoutMillis: number; - httpAgent?: KeepAliveAgent; + httpAgent?: http.Agent; - httpsAgent?: KeepAliveAgent.HttpsAgent; + httpsAgent?: https.Agent; axios: AxiosInstance; @@ -58,11 +59,13 @@ export class HttpClient { // is for inactive sockets, sometimes the platform would take // long to process requests and the socket would time-out // while waiting for the response. - const agentOpts = { + const agentOptions: http.AgentOptions = { + keepAlive: true, timeout: this.timeoutMillis, }; - this.httpAgent = new KeepAliveAgent(agentOpts); - this.httpsAgent = new KeepAliveAgent.HttpsAgent(agentOpts); + + this.httpAgent = new http.Agent(agentOptions); + this.httpsAgent = new https.Agent(agentOptions); } this.axios = axios.create({ From 4fd44039ff6cd59a8504833f9524cc781b04b821 Mon Sep 17 00:00:00 2001 From: Thibault Ducret Date: Thu, 20 Nov 2025 11:40:51 +0100 Subject: [PATCH 2/3] feat: Enhance HTTP agent with agentkeepalive-inspired optimizations Improved native Node.js HTTP/HTTPS agent configuration with: - Disabled Nagle's algorithm (setNoDelay) for lower latency - LIFO socket scheduling to reuse warm connections - Free socket timeout (15s) to prevent connection leaks - Configurable socket pool limits (256 max sockets) - Better keepalive management with timeoutMillis --- src/http_client.ts | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/http_client.ts b/src/http_client.ts index 6e2b70ce..539cd499 100644 --- a/src/http_client.ts +++ b/src/http_client.ts @@ -54,18 +54,37 @@ export class HttpClient { if (isNode()) { // We want to keep sockets alive for better performance. - // It's important to set the user's timeout also here and not only - // on the axios instance, because even though this timeout - // is for inactive sockets, sometimes the platform would take - // long to process requests and the socket would time-out - // while waiting for the response. - const agentOptions: http.AgentOptions = { + // Enhanced agent configuration based on agentkeepalive best practices: + // - Nagle's algorithm disabled for lower latency + // - Free socket timeout to prevent socket leaks + // - LIFO scheduling to reuse recent sockets + // - Socket TTL for connection freshness + const agentOptions: http.AgentOptions & { scheduling?: 'lifo' | 'fifo' } = { keepAlive: true, + // Timeout for inactive sockets + // Prevents socket leaks from idle connections timeout: this.timeoutMillis, + // Keep alive timeout for free sockets (15 seconds) + // Node.js will close unused sockets after this period + keepAliveMsecs: 15_000, + // Maximum number of sockets per host + maxSockets: 256, + maxFreeSockets: 256, + // LIFO scheduling - reuse most recently used sockets for better performance + scheduling: 'lifo', }; this.httpAgent = new http.Agent(agentOptions); this.httpsAgent = new https.Agent(agentOptions); + + // Disable Nagle's algorithm for lower latency + // This sends data immediately instead of buffering small packets + const setNoDelay = (socket: any) => { + socket.setNoDelay(true); + }; + + this.httpAgent.on('socket', setNoDelay); + this.httpsAgent.on('socket', setNoDelay); } this.axios = axios.create({ From f986aa45020bc32079658e131d565ec25d8c6d85 Mon Sep 17 00:00:00 2001 From: Thibault Ducret Date: Thu, 20 Nov 2025 14:30:12 +0100 Subject: [PATCH 3/3] chore: Exclude node:http and node:https from browser bundle Add node:http and node:https to rsbuild externals configuration to prevent bundling these Node.js-only modules in the browser build, reducing bundle size by ~250kB. --- rsbuild.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rsbuild.config.ts b/rsbuild.config.ts index b83bd377..e3b7a5b5 100644 --- a/rsbuild.config.ts +++ b/rsbuild.config.ts @@ -31,6 +31,8 @@ export default defineConfig({ externals: { 'node:util': 'node:util', 'node:zlib': 'node:zlib', + 'node:http': 'node:http', + 'node:https': 'node:https', crypto: 'node:crypto', }, },