diff --git a/CHANGELOG.md b/CHANGELOG.md index b3558216..9b62d57b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [5.15.0](https://github.com/SocketDev/socket-lib/releases/tag/v5.15.0) - 2026-04-06 + +### Added — http-request + +- `stream` option on `HttpRequestOptions` — resolves with `HttpResponse` immediately after headers arrive, leaving `rawResponse` unconsumed for piping to files +- `headers`, `ok`, `status`, `statusText` fields on `HttpDownloadResult` + +### Changed — http-request + +- `httpDownload` now uses `httpRequest` with `stream: true` internally, eliminating ~120 lines of duplicated HTTP plumbing + ## [5.14.0](https://github.com/SocketDev/socket-lib/releases/tag/v5.14.0) - 2026-04-06 ### Added — http-request diff --git a/package.json b/package.json index 47a168f7..276e46c8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@socketsecurity/lib", - "version": "5.14.0", + "version": "5.15.0", "packageManager": "pnpm@10.33.0", "license": "MIT", "description": "Core utilities and infrastructure for Socket.dev security tools", diff --git a/src/http-request.ts b/src/http-request.ts index fc78b56d..008713a7 100644 --- a/src/http-request.ts +++ b/src/http-request.ts @@ -348,6 +348,20 @@ export interface HttpRequestOptions { * }) * ``` */ + /** + * When true, resolve with an HttpResponse whose body is NOT buffered. + * The `rawResponse` property contains the unconsumed IncomingResponse + * stream for piping to files or other destinations. + * + * `body`, `text()`, `json()`, and `arrayBuffer()` return empty/zero + * values since the stream has not been read. + * + * Incompatible with `maxResponseSize` (size enforcement requires + * reading the body). + * + * @default false + */ + stream?: boolean | undefined throwOnError?: boolean | undefined /** * Request timeout in milliseconds. @@ -823,26 +837,18 @@ export interface HttpDownloadOptions { * Result of a successful file download. */ export interface HttpDownloadResult { - /** - * Absolute path where the file was saved. - * - * @example - * ```ts - * const result = await httpDownload('https://example.com/file.zip', '/tmp/file.zip') - * console.log(`Downloaded to: ${result.path}`) - * ``` - */ + /** HTTP response headers from the final response (after redirects). */ + headers: IncomingHttpHeaders + /** Whether the download succeeded (status 200-299). Always true on success (non-2xx throws). */ + ok: true + /** Absolute path where the file was saved. */ path: string - /** - * Total size of downloaded file in bytes. - * - * @example - * ```ts - * const result = await httpDownload('https://example.com/file.zip', '/tmp/file.zip') - * console.log(`Downloaded ${result.size} bytes`) - * ``` - */ + /** Total size of downloaded file in bytes. */ size: number + /** HTTP status code from the final response (after redirects). */ + status: number + /** HTTP status message from the final response (after redirects). */ + statusText: string } /** @@ -983,7 +989,7 @@ export async function fetchChecksums( } /** - * Single download attempt (used internally by httpDownload with retry logic). + * Single download attempt using httpRequestAttempt with stream: true. * @private */ async function httpDownloadAttempt( @@ -1000,178 +1006,70 @@ async function httpDownloadAttempt( timeout = 120_000, } = { __proto__: null, ...options } as HttpDownloadOptions - return await new Promise((resolve, reject) => { - const parsedUrl = new URL(url) - const isHttps = parsedUrl.protocol === 'https:' - const httpModule = isHttps ? getHttps() : getHttp() - - const requestOptions: Record = { - headers: { - 'User-Agent': 'socket-registry/1.0', - ...headers, - }, - hostname: parsedUrl.hostname, - method: 'GET', - path: parsedUrl.pathname + parsedUrl.search, - port: parsedUrl.port, - timeout, - } - - // Pass custom CA certificates for TLS connections. - if (ca && isHttps) { - requestOptions['ca'] = ca - } - - const { createWriteStream } = getFs() - - let fileStream: ReturnType | undefined - let streamClosed = false - - const closeStream = () => { - if (!streamClosed && fileStream) { - streamClosed = true - fileStream.close() - } - } - - /* c8 ignore start - External HTTP/HTTPS download request */ - const request = httpModule.request( - requestOptions, - (res: IncomingResponse) => { - // Handle redirects - if ( - followRedirects && - res.statusCode && - res.statusCode >= 300 && - res.statusCode < 400 && - res.headers.location - ) { - if (maxRedirects <= 0) { - reject( - new Error( - `Too many redirects (exceeded maximum: ${maxRedirects})`, - ), - ) - return - } - - // Follow redirect - const redirectUrl = res.headers.location.startsWith('http') - ? res.headers.location - : new URL(res.headers.location, url).toString() - - // Reject HTTPS-to-HTTP downgrade redirects. - const redirectParsed = new URL(redirectUrl) - if (isHttps && redirectParsed.protocol !== 'https:') { - reject( - new Error( - `Redirect from HTTPS to HTTP is not allowed: ${redirectUrl}`, - ), - ) - return - } - - resolve( - httpDownloadAttempt(redirectUrl, destPath, { - ca, - followRedirects, - headers, - maxRedirects: maxRedirects - 1, - onProgress, - timeout, - }), - ) - return - } - - // Check status code - if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) { - closeStream() - reject( - new Error( - `Download failed: HTTP ${res.statusCode} ${res.statusMessage}`, - ), - ) - return - } - - const totalSize = Number.parseInt( - res.headers['content-length'] || '0', - 10, - ) - let downloadedSize = 0 - - // Create write stream - fileStream = createWriteStream(destPath) + const response = await httpRequestAttempt(url, { + ca, + followRedirects, + headers, + maxRedirects, + method: 'GET', + stream: true, + timeout, + }) - fileStream.on('error', (error: Error) => { - closeStream() - const err = new Error(`Failed to write file: ${error.message}`, { - cause: error, - }) - reject(err) - }) + if (!response.ok) { + throw new Error( + `Download failed: HTTP ${response.status} ${response.statusText}`, + ) + } - res.on('data', (chunk: Buffer) => { - downloadedSize += chunk.length - if (onProgress && totalSize > 0) { - onProgress(downloadedSize, totalSize) - } - }) + const res = response.rawResponse + if (!res) { + throw new Error('Stream response missing rawResponse') + } - res.on('end', () => { - fileStream?.close(() => { - streamClosed = true - resolve({ - path: destPath, - size: downloadedSize, - }) - }) - }) + const { createWriteStream } = getFs() + const totalSize = Number.parseInt( + (response.headers['content-length'] as string) || '0', + 10, + ) - res.on('error', (error: Error) => { - closeStream() - reject(error) - }) + return await new Promise((resolve, reject) => { + let downloadedSize = 0 + const fileStream = createWriteStream(destPath) - // Pipe response to file - res.pipe(fileStream) - }, - ) + fileStream.on('error', (error: Error) => { + fileStream.close() + reject( + new Error(`Failed to write file: ${error.message}`, { cause: error }), + ) + }) - request.on('error', (error: Error) => { - closeStream() - const code = (error as NodeJS.ErrnoException).code - let message = `HTTP download failed for ${url}: ${error.message}\n` - - if (code === 'ENOTFOUND') { - message += - 'DNS lookup failed. Check the hostname and your network connection.' - } else if (code === 'ECONNREFUSED') { - message += - 'Connection refused. Verify the server is running and accessible.' - } else if (code === 'ETIMEDOUT') { - message += - 'Request timed out. Check your network or increase the timeout value.' - } else if (code === 'ECONNRESET') { - message += - 'Connection reset. The server may have closed the connection unexpectedly.' - } else { - message += - 'Check your network connection and verify the URL is correct.' + res.on('data', (chunk: Buffer) => { + downloadedSize += chunk.length + if (onProgress && totalSize > 0) { + onProgress(downloadedSize, totalSize) } + }) - reject(new Error(message, { cause: error })) + res.on('end', () => { + fileStream.close(() => { + resolve({ + headers: response.headers, + ok: true, + path: destPath, + size: downloadedSize, + status: response.status, + statusText: response.statusText, + }) + }) }) - request.on('timeout', () => { - request.destroy() - closeStream() - reject(new Error(`Download timed out after ${timeout}ms`)) + res.on('error', (error: Error) => { + fileStream.close() + reject(error) }) - request.end() - /* c8 ignore stop */ + res.pipe(fileStream) }) } @@ -1231,6 +1129,7 @@ async function httpRequestAttempt( maxRedirects = 5, maxResponseSize, method = 'GET', + stream = false, timeout = 30_000, } = { __proto__: null, ...options } as HttpRequestOptions @@ -1370,12 +1269,42 @@ async function httpRequestAttempt( maxRedirects: maxRedirects - 1, maxResponseSize, method, + stream, timeout, }), ) return } + // Stream mode: resolve immediately with unconsumed response. + if (stream) { + const status = res.statusCode || 0 + const statusText = res.statusMessage || '' + const ok = status >= 200 && status < 300 + + emitResponse({ + headers: res.headers, + status, + statusText, + }) + + const emptyBody = Buffer.alloc(0) + resolveOnce({ + arrayBuffer: () => emptyBody.buffer as ArrayBuffer, + body: emptyBody, + headers: res.headers, + json: () => { + throw new Error('Cannot parse JSON from a streaming response') + }, + ok, + rawResponse: res, + status, + statusText, + text: () => '', + }) + return + } + const chunks: Buffer[] = [] let totalBytes = 0 @@ -1645,8 +1574,8 @@ export async function httpDownload( await fs.promises.rename(tempPath, destPath) return { + ...result, path: destPath, - size: result.size, } } catch (e) { lastError = e as Error @@ -1816,6 +1745,7 @@ export async function httpRequest( onRetry, retries = 0, retryDelay = 1000, + stream = false, throwOnError = false, timeout = 30_000, } = { __proto__: null, ...options } as HttpRequestOptions @@ -1846,6 +1776,7 @@ export async function httpRequest( maxRedirects, maxResponseSize, method, + stream, timeout, } diff --git a/test/unit/http-request.test.mts b/test/unit/http-request.test.mts index 4e98344b..e753491b 100644 --- a/test/unit/http-request.test.mts +++ b/test/unit/http-request.test.mts @@ -561,6 +561,20 @@ describe('http-request', () => { }, 'httpDownload-basic-') }) + it('should include response metadata in result', async () => { + await runWithTempDir(async tmpDir => { + const destPath = path.join(tmpDir, 'metadata.txt') + const result = await httpDownload(`${httpBaseUrl}/download`, destPath) + + expect(result.ok).toBe(true) + expect(result.status).toBe(200) + expect(result.statusText).toBe('OK') + expect(result.headers).toBeDefined() + expect(result.headers['content-type']).toBe('text/plain') + expect(result.headers['content-length']).toBeDefined() + }, 'httpDownload-metadata-') + }) + it('should track download progress', async () => { await runWithTempDir(async tmpDir => { const destPath = path.join(tmpDir, 'progress.txt') @@ -684,7 +698,7 @@ describe('http-request', () => { retries: 2, retryDelay: 10, }), - ).rejects.toThrow(/HTTP download failed/) + ).rejects.toThrow(/request failed/) expect(attemptCount).toBe(3) }, 'httpDownload-fail-') @@ -2082,6 +2096,86 @@ abc123def456789012345678901234567890123456789012345678901234abcd }) }) + describe('stream option', () => { + it('should resolve immediately with unconsumed rawResponse', async () => { + const response = await httpRequest(`${httpBaseUrl}/text`, { + stream: true, + }) + + expect(response.ok).toBe(true) + expect(response.status).toBe(200) + expect(response.statusText).toBe('OK') + expect(response.headers['content-type']).toBe('text/plain') + expect(response.rawResponse).toBeDefined() + // Body not buffered in stream mode. + expect(response.body.length).toBe(0) + expect(response.text()).toBe('') + + // Read the stream manually. + const chunks: Buffer[] = [] + for await (const chunk of response.rawResponse!) { + chunks.push(chunk as Buffer) + } + expect(Buffer.concat(chunks).toString('utf8')).toBe('Plain text response') + }) + + it('should follow redirects in stream mode', async () => { + const response = await httpRequest(`${httpBaseUrl}/redirect`, { + stream: true, + }) + + expect(response.ok).toBe(true) + expect(response.status).toBe(200) + expect(response.rawResponse).toBeDefined() + + const chunks: Buffer[] = [] + for await (const chunk of response.rawResponse!) { + chunks.push(chunk as Buffer) + } + expect(Buffer.concat(chunks).toString('utf8')).toBe('Plain text response') + }) + + it('should handle non-2xx in stream mode', async () => { + const response = await httpRequest(`${httpBaseUrl}/not-found`, { + stream: true, + }) + + expect(response.ok).toBe(false) + expect(response.status).toBe(404) + expect(response.rawResponse).toBeDefined() + }) + + it('should throw on json() in stream mode', async () => { + const response = await httpRequest(`${httpBaseUrl}/json`, { + stream: true, + }) + expect(() => response.json()).toThrow( + 'Cannot parse JSON from a streaming response', + ) + }) + + it('should pipe to a file via stream mode', async () => { + await runWithTempDir(async tmpDir => { + const response = await httpRequest(`${httpBaseUrl}/download`, { + stream: true, + }) + expect(response.ok).toBe(true) + + const destPath = path.join(tmpDir, 'streamed.txt') + const { createWriteStream } = await import('node:fs') + await new Promise((resolve, reject) => { + const ws = createWriteStream(destPath) + ws.on('error', reject) + ws.on('close', resolve) + response.rawResponse!.pipe(ws) + }) + + const content = await fs.readFile(destPath, 'utf8') + expect(content).toBe('Download test content') + }, 'stream-pipe-') + }) + }) + describe('enrichErrorMessage', () => { it('should enrich each known error code', () => { const cases: Array<[string, string]> = [ diff --git a/test/unit/releases-github.test.mts b/test/unit/releases-github.test.mts index 5099b890..283bf47a 100644 --- a/test/unit/releases-github.test.mts +++ b/test/unit/releases-github.test.mts @@ -860,7 +860,14 @@ describe('releases/github', () => { // Create a dummy binary file const content = '#!/bin/bash\necho "test"' await fs.writeFile(outputPath, content, 'utf8') - return { path: outputPath, size: content.length } + return { + headers: {}, + ok: true as const, + path: outputPath, + size: content.length, + status: 200, + statusText: 'OK', + } }) // First download - creates cache @@ -964,7 +971,14 @@ describe('releases/github', () => { // Create a dummy binary file const content = '#!/bin/bash\necho "test"' await fs.writeFile(outputPath, content, 'utf8') - return { path: outputPath, size: content.length } + return { + headers: {}, + ok: true as const, + path: outputPath, + size: content.length, + status: 200, + statusText: 'OK', + } }) // First download