From 3c23eea5a096f6f8ea0edf3e2a27e1caca88acf9 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sun, 11 Apr 2021 07:20:28 +0200 Subject: [PATCH] Got 12 improvements (#1667) --- .gitignore | 1 + benchmark/index.ts | 72 +- package.json | 7 +- source/as-promise/create-rejection.ts | 31 - source/as-promise/index.ts | 145 +- source/as-promise/normalize-arguments.ts | 98 - source/as-promise/parse-body.ts | 33 - source/as-promise/types.ts | 293 +-- source/core/calculate-retry-delay.ts | 11 +- source/core/errors.ts | 162 ++ source/core/index.ts | 2732 +++++----------------- source/core/options.ts | 2197 +++++++++++++++++ source/core/parse-link-header.ts | 44 + source/core/response.ts | 152 ++ source/core/{utils => }/timed-out.ts | 2 +- source/core/utils/dns-ip-version.ts | 20 - source/core/utils/get-buffer.ts | 21 - source/core/utils/is-client-request.ts | 8 + source/core/utils/is-response-ok.ts | 8 - source/create.ts | 261 +-- source/index.ts | 145 +- source/types.ts | 171 +- source/utils/deep-freeze.ts | 11 - source/utils/deprecation-warning.ts | 14 - test/agent.ts | 9 +- test/arguments.ts | 153 +- test/cache.ts | 10 +- test/cancel.ts | 51 +- test/create.ts | 76 +- test/error.ts | 60 +- test/headers.ts | 2 +- test/helpers.ts | 2 +- test/helpers/slow-data-stream.ts | 12 +- test/helpers/with-server.ts | 20 +- test/hooks.ts | 102 +- test/http.ts | 58 +- test/https.ts | 78 +- test/merge-instances.ts | 25 +- test/normalize-arguments.ts | 102 +- test/pagination.ts | 95 +- test/parse-link-header.ts | 114 + test/post.ts | 45 +- test/promise.ts | 2 +- test/redirects.ts | 21 +- test/response-parse.ts | 18 +- test/retry.ts | 56 +- test/stream.ts | 12 +- test/timeout.ts | 153 +- test/unix-socket.ts | 2 +- tsconfig.json | 3 +- 50 files changed, 4106 insertions(+), 3814 deletions(-) delete mode 100644 source/as-promise/create-rejection.ts delete mode 100644 source/as-promise/normalize-arguments.ts delete mode 100644 source/as-promise/parse-body.ts create mode 100644 source/core/errors.ts create mode 100644 source/core/options.ts create mode 100644 source/core/parse-link-header.ts create mode 100644 source/core/response.ts rename source/core/{utils => }/timed-out.ts (99%) delete mode 100644 source/core/utils/dns-ip-version.ts delete mode 100644 source/core/utils/get-buffer.ts create mode 100644 source/core/utils/is-client-request.ts delete mode 100644 source/core/utils/is-response-ok.ts delete mode 100644 source/utils/deep-freeze.ts delete mode 100644 source/utils/deprecation-warning.ts create mode 100644 test/parse-link-header.ts diff --git a/.gitignore b/.gitignore index b06258d44..bd4471e12 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ yarn.lock coverage .nyc_output dist +*.0x diff --git a/benchmark/index.ts b/benchmark/index.ts index 9a1e27d43..67a4499da 100644 --- a/benchmark/index.ts +++ b/benchmark/index.ts @@ -5,9 +5,8 @@ import * as Benchmark from 'benchmark'; import fetch from 'node-fetch'; import * as request from 'request'; import got from '../source/index'; -import Request, {kIsNormalizedAlready} from '../source/core/index'; - -const {normalizeArguments} = Request; +import Request from '../source/core/index'; +import Options, {OptionsInit} from '../source/core/options'; // Configuration const httpsAgent = new https.Agent({ @@ -15,21 +14,22 @@ const httpsAgent = new https.Agent({ rejectUnauthorized: false }); -const url = new URL('https://127.0.0.1:8080'); +const url = new URL('https://127.0.0.1:8081'); const urlString = url.toString(); -const gotOptions = { +const gotOptions: OptionsInit & {isStream?: true} = { agent: { https: httpsAgent }, - https: { + httpsOptions: { rejectUnauthorized: false }, - retry: 0 + retry: { + limit: 0 + } }; -const normalizedGotOptions = normalizeArguments(url, gotOptions); -normalizedGotOptions[kIsNormalizedAlready] = true; +const normalizedGotOptions = new Options(url, gotOptions); const requestOptions = { strictSSL: false, @@ -80,6 +80,7 @@ suite.add('got - promise', { defer: true, fn: async (deferred: {resolve: () => void}) => { const stream = new Request(url, gotOptions); + void stream.flush(); stream.resume().once('end', () => { deferred.resolve(); }); @@ -88,6 +89,7 @@ suite.add('got - promise', { defer: true, fn: async (deferred: {resolve: () => void}) => { const stream = new Request(undefined as any, normalizedGotOptions); + void stream.flush(); stream.resume().once('end', () => { deferred.resolve(); }); @@ -169,7 +171,8 @@ const internalBenchmark = (): void => { const internalSuite = new Benchmark.Suite(); internalSuite.add('got - normalize options', { fn: () => { - normalizeArguments(url, gotOptions); + // eslint-disable-next-line no-new + new Options(url, gotOptions); } }).on('cycle', (event: Benchmark.Event) => { console.log(String(event.target)); @@ -179,17 +182,48 @@ const internalBenchmark = (): void => { }; // Results (i7-7700k, CPU governor: performance): + +// e9359d3fa0cb40324f2b84364408b3f9f7ff2cee (Rewrite Got #1051) - unknown Node.js version +// got - promise x 3,092 ops/sec ±5.25% (73 runs sampled) +// got - stream x 4,313 ops/sec ±5.61% (72 runs sampled) +// got - promise core x 6,756 ops/sec ±5.32% (80 runs sampled) +// got - stream core x 6,863 ops/sec ±4.68% (76 runs sampled) +// got - stream core - normalized options x 7,960 ops/sec ±3.83% (81 runs sampled) + +// b927e2d028ecc023bf7eff2702ffb5c72016a85a (Fix bugs, increase coverage, update benchmark results) - unknown Node.js version +// got - promise x 3,204 ops/sec ±5.27% (73 runs sampled) +// got - stream x 5,045 ops/sec ±3.85% (77 runs sampled) +// got - promise core x 6,499 ops/sec ±3.67% (77 runs sampled) +// got - stream core x 7,047 ops/sec ±2.32% (83 runs sampled) +// got - stream core - normalized options x 7,313 ops/sec ±2.79% (85 runs sampled) + +// 7e8898e9095e7da52e4ff342606cfd1dc5186f54 (Merge PromisableRequest into Request) - unknown Node.js version // got - promise x 3,003 ops/sec ±6.26% (70 runs sampled) // got - stream x 3,538 ops/sec ±5.86% (67 runs sampled) // got - core x 5,828 ops/sec ±3.11% (79 runs sampled) // got - core - normalized options x 7,596 ops/sec ±1.60% (85 runs sampled) -// request - callback x 6,530 ops/sec ±6.84% (72 runs sampled) -// request - stream x 7,348 ops/sec ±3.62% (78 runs sampled) -// node-fetch - promise x 6,284 ops/sec ±5.50% (76 runs sampled) -// node-fetch - stream x 7,746 ops/sec ±3.32% (80 runs sampled) -// axios - promise x 6,301 ops/sec ±6.24% (77 runs sampled) -// axios - stream x 8,605 ops/sec ±2.73% (87 runs sampled) -// https - stream x 10,477 ops/sec ±3.64% (80 runs sampled) -// Fastest is https - stream -// got - normalize options x 90,974 ops/sec ±0.57% (93 runs sampled) +// [main] - Node.js v15.10.0 +// got - promise x 3,201 ops/sec ±5.24% (67 runs sampled) +// got - stream x 3,633 ops/sec ±4.06% (74 runs sampled) +// got - core x 4,382 ops/sec ±3.26% (77 runs sampled) +// got - core - normalized options x 5,470 ops/sec ±3.70% (78 runs sampled) + +// v12 - Node.js v15.10.0 +// got - promise x 3,492 ops/sec ±5.13% (71 runs sampled) +// got - stream x 4,772 ops/sec ±1.52% (84 runs sampled) +// got - core x 4,990 ops/sec ±1.14% (83 runs sampled) +// got - core - normalized options x 5,386 ops/sec ±0.52% (87 runs sampled) + +// got - normalize options x 117,810 ops/sec ±0.36% (97 runs sampled) + +// ================================================================================ + +// request - callback x 6,448 ops/sec ±5.76% (67 runs sampled) +// request - stream x 7,115 ops/sec ±2.85% (83 runs sampled) +// node-fetch - promise x 6,236 ops/sec ±5.56% (75 runs sampled) +// node-fetch - stream x 7,225 ops/sec ±2.10% (81 runs sampled) +// axios - promise x 5,620 ops/sec ±3.13% (78 runs sampled) +// axios - stream x 7,244 ops/sec ±3.31% (80 runs sampled) +// https - stream x 8,588 ops/sec ±5.50% (61 runs sampled) +// Fastest is https - stream diff --git a/package.json b/package.json index 888728c2d..ad739ab52 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,8 @@ "cacheable-lookup": "^6.0.0", "cacheable-request": "^7.0.1", "decompress-response": "^6.0.0", - "http2-wrapper": "^2.0.0", + "get-stream": "^6.0.0", + "http2-wrapper": "^2.0.1", "lowercase-keys": "^2.0.0", "p-cancelable": "^2.0.0", "responselike": "^2.0.0" @@ -78,7 +79,6 @@ "delay": "^5.0.0", "express": "^4.17.1", "form-data": "^4.0.0", - "get-stream": "^6.0.0", "nock": "^13.0.7", "node-fetch": "^2.6.1", "np": "^7.4.0", @@ -137,7 +137,8 @@ "import/no-anonymous-default-export": "off", "@typescript-eslint/no-implicit-any-catch": "off", "unicorn/import-index": "off", - "import/no-useless-path-segments": "off" + "import/no-useless-path-segments": "off", + "import/no-named-as-default": "off" } }, "runkitExampleFilename": "./documentation/examples/runkit-example.js" diff --git a/source/as-promise/create-rejection.ts b/source/as-promise/create-rejection.ts deleted file mode 100644 index 358e1fe76..000000000 --- a/source/as-promise/create-rejection.ts +++ /dev/null @@ -1,31 +0,0 @@ -import {CancelableRequest, BeforeErrorHook, RequestError} from './types'; - -export default function createRejection(error: Error, ...beforeErrorGroups: Array): CancelableRequest { - const promise = (async () => { - if (error instanceof RequestError) { - try { - for (const hooks of beforeErrorGroups) { - if (hooks) { - for (const hook of hooks) { - // eslint-disable-next-line no-await-in-loop - error = await hook(error as RequestError); - } - } - } - } catch (error_) { - error = error_; - } - } - - throw error; - })() as CancelableRequest; - - const returnPromise = (): CancelableRequest => promise; - - promise.json = returnPromise; - promise.text = returnPromise; - promise.buffer = returnPromise; - promise.on = returnPromise; - - return promise; -} diff --git a/source/as-promise/index.ts b/source/as-promise/index.ts index 4d152b153..86f0e250a 100644 --- a/source/as-promise/index.ts +++ b/source/as-promise/index.ts @@ -2,18 +2,17 @@ import {EventEmitter} from 'events'; import is from '@sindresorhus/is'; import * as PCancelable from 'p-cancelable'; import { - NormalizedOptions, - CancelableRequest, - Response, RequestError, HTTPError, - CancelError -} from './types'; -import parseBody from './parse-body'; -import Request from '../core/index'; + RetryError +} from '../core/errors'; +import {CancelError} from './types'; +import Request from '../core'; +import {parseBody, isResponseOk} from '../core/response'; import proxyEvents from '../core/utils/proxy-events'; -import getBuffer from '../core/utils/get-buffer'; -import {isResponseOk} from '../core/utils/is-response-ok'; +import type Options from '../core/options'; +import type {Response} from '../core/response'; +import type {CancelableRequest} from './types'; const proxiedRequestEvents = [ 'request', @@ -23,65 +22,51 @@ const proxiedRequestEvents = [ 'downloadProgress' ]; -export default function asPromise(normalizedOptions: NormalizedOptions): CancelableRequest { +const supportedCompressionAlgorithms = new Set(['gzip', 'deflate', 'br']); + +export default function asPromise(firstRequest: Request): CancelableRequest { let globalRequest: Request; let globalResponse: Response; + let normalizedOptions: Options; const emitter = new EventEmitter(); const promise = new PCancelable((resolve, reject, onCancel) => { + onCancel(() => { + globalRequest.destroy(); + }); + + onCancel.shouldReject = false; + onCancel(() => { + reject(new CancelError(globalRequest)); + }); + const makeRequest = (retryCount: number): void => { - const request = new Request(undefined, normalizedOptions); + // Errors when a new request is made after the promise settles. + // Used to detect a race condition. + // See https://github.com/sindresorhus/got/issues/1489 + onCancel(() => {}); + + const request = firstRequest ?? new Request(undefined, undefined, normalizedOptions); request.retryCount = retryCount; request._noPipe = true; - onCancel(() => { - request.destroy(); - }); - - onCancel.shouldReject = false; - onCancel(() => { - reject(new CancelError(request)); - }); - globalRequest = request; request.once('response', async (response: Response) => { - response.retryCount = retryCount; - - if (response.request.aborted) { - // Canceled while downloading - will throw a `CancelError` or `TimeoutError` error - return; - } - - // Download body - let rawBody; - try { - rawBody = await getBuffer(request); - response.rawBody = rawBody; - } catch { - // The same error is caught below. - // See request.once('error') - return; - } - - if (request._isAboutToError) { - return; - } - // Parse body const contentEncoding = (response.headers['content-encoding'] ?? '').toLowerCase(); - const isCompressed = ['gzip', 'deflate', 'br'].includes(contentEncoding); + const isCompressed = supportedCompressionAlgorithms.has(contentEncoding); const {options} = request; if (isCompressed && !options.decompress) { - response.body = rawBody; + response.body = response.rawBody; } else { try { response.body = parseBody(response, options.responseType, options.parseJson, options.encoding); } catch (error) { // Fallback to `utf8` - response.body = rawBody.toString(); + response.body = response.rawBody.toString(); if (isResponseOk(response)) { request._beforeError(error); @@ -91,40 +76,32 @@ export default function asPromise(normalizedOptions: NormalizedOptions): Canc } try { - for (const [index, hook] of options.hooks.afterResponse.entries()) { + const hooks = options.hooks.afterResponse; + + // TODO: `xo` should detect if `index` is being used for something else + // eslint-disable-next-line unicorn/no-for-loop + for (let index = 0; index < hooks.length; index++) { + const hook = hooks[index]; + // @ts-expect-error TS doesn't notice that CancelableRequest is a Promise // eslint-disable-next-line no-await-in-loop response = await hook(response, async (updatedOptions): CancelableRequest => { - const typedOptions = Request.normalizeArguments(undefined, { - ...updatedOptions, - retry: { - calculateDelay: () => 0 - }, - throwHttpErrors: false, - resolveBodyOnly: false - }, options); + options.merge(updatedOptions); + options.prefixUrl = ''; - // Remove any further hooks for that request, because we'll call them anyway. - // The loop continues. We don't want duplicates (asPromise recursion). - typedOptions.hooks.afterResponse = typedOptions.hooks.afterResponse.slice(0, index); - - for (const hook of typedOptions.hooks.beforeRetry) { - // eslint-disable-next-line no-await-in-loop - await hook(typedOptions); + if (updatedOptions.url) { + options.url = updatedOptions.url; } - const promise: CancelableRequest = asPromise(typedOptions); - - onCancel(() => { - promise.catch(() => {}); - promise.cancel(); - }); + // Remove any further hooks for that request, because we'll call them anyway. + // The loop continues. We don't want duplicates (asPromise recursion). + options.hooks.afterResponse = options.hooks.afterResponse.slice(0, index); - return promise; + throw new RetryError(request); }); } } catch (error) { - request._beforeError(new RequestError(error.message, error, request)); + request._beforeError(error); return; } @@ -135,6 +112,7 @@ export default function asPromise(normalizedOptions: NormalizedOptions): Canc return; } + request.destroy(); resolve(request.options.resolveBodyOnly ? response.body as T : response as unknown as T); }); @@ -147,6 +125,8 @@ export default function asPromise(normalizedOptions: NormalizedOptions): Canc if (error instanceof HTTPError && !options.throwHttpErrors) { const {response} = error; + + request.destroy(); resolve(request.options.resolveBodyOnly ? response.body as T : response as unknown as T); return; } @@ -156,18 +136,29 @@ export default function asPromise(normalizedOptions: NormalizedOptions): Canc request.once('error', onError); - const previousBody = request.options.body; + const previousBody = request.options?.body; request.once('retry', (newRetryCount: number, error: RequestError) => { - if (previousBody === error.request?.options.body && is.nodeStream(error.request?.options.body)) { + // @ts-expect-error + firstRequest = undefined; + + const newBody = request.options.body; + + if (previousBody === newBody && is.nodeStream(newBody)) { onError(error); return; } + // This is needed! We need to reuse `request.options` because they can get modified! + // For example, by calling `promise.json()`. + normalizedOptions = request.options; + makeRequest(newRetryCount); }); proxyEvents(request, emitter, proxiedRequestEvents); + + void request.flush(); }; makeRequest(0); @@ -178,7 +169,7 @@ export default function asPromise(normalizedOptions: NormalizedOptions): Canc return promise; }; - const shortcut = (responseType: NormalizedOptions['responseType']): CancelableRequest => { + const shortcut = (responseType: Options['responseType']): CancelableRequest => { const newPromise = (async () => { // Wait until downloading has ended await promise; @@ -194,10 +185,12 @@ export default function asPromise(normalizedOptions: NormalizedOptions): Canc }; promise.json = () => { - const {headers} = globalRequest.options; + if (globalRequest.options) { + const {headers} = globalRequest.options; - if (!globalRequest.writableFinished && headers.accept === undefined) { - headers.accept = 'application/json'; + if (!globalRequest.writableFinished && !('accept' in headers)) { + headers.accept = 'application/json'; + } } return shortcut('json'); @@ -208,5 +201,3 @@ export default function asPromise(normalizedOptions: NormalizedOptions): Canc return promise; } - -export * from './types'; diff --git a/source/as-promise/normalize-arguments.ts b/source/as-promise/normalize-arguments.ts deleted file mode 100644 index a0e5525a7..000000000 --- a/source/as-promise/normalize-arguments.ts +++ /dev/null @@ -1,98 +0,0 @@ -import is, {assert} from '@sindresorhus/is'; -import { - Options, - NormalizedOptions, - Defaults, - Method -} from './types'; - -const normalizeArguments = (options: NormalizedOptions, defaults?: Defaults): NormalizedOptions => { - if (is.null_(options.encoding)) { - throw new TypeError('To get a Buffer, set `options.responseType` to `buffer` instead'); - } - - assert.any([is.string, is.undefined], options.encoding); - assert.any([is.boolean, is.undefined], options.resolveBodyOnly); - assert.any([is.boolean, is.undefined], options.methodRewriting); - assert.any([is.boolean, is.undefined], options.isStream); - assert.any([is.string, is.undefined], options.responseType); - - // `options.responseType` - if (options.responseType === undefined) { - options.responseType = 'text'; - } - - // `options.retry` - const {retry} = options; - - if (defaults) { - options.retry = {...defaults.retry}; - } else { - options.retry = { - calculateDelay: retryObject => retryObject.computedValue, - limit: 0, - methods: [], - statusCodes: [], - errorCodes: [], - maxRetryAfter: undefined - }; - } - - if (is.object(retry)) { - options.retry = { - ...options.retry, - ...retry - }; - - options.retry.methods = [...new Set(options.retry.methods.map(method => method.toUpperCase() as Method))]; - options.retry.statusCodes = [...new Set(options.retry.statusCodes)]; - options.retry.errorCodes = [...new Set(options.retry.errorCodes)]; - } else if (is.number(retry)) { - options.retry.limit = retry; - } - - if (is.undefined(options.retry.maxRetryAfter)) { - options.retry.maxRetryAfter = Math.min( - // TypeScript is not smart enough to handle `.filter(x => is.number(x))`. - // eslint-disable-next-line unicorn/no-array-callback-reference - ...[options.timeout.request, options.timeout.connect].filter(is.number) - ); - } - - // `options.pagination` - if (is.object(options.pagination)) { - if (defaults) { - (options as Options).pagination = { - ...defaults.pagination, - ...options.pagination - }; - } - - const {pagination} = options; - - if (!is.function_(pagination.transform)) { - throw new Error('`options.pagination.transform` must be implemented'); - } - - if (!is.function_(pagination.shouldContinue)) { - throw new Error('`options.pagination.shouldContinue` must be implemented'); - } - - if (!is.function_(pagination.filter)) { - throw new TypeError('`options.pagination.filter` must be implemented'); - } - - if (!is.function_(pagination.paginate)) { - throw new Error('`options.pagination.paginate` must be implemented'); - } - } - - // JSON mode - if (options.responseType === 'json' && options.headers.accept === undefined) { - options.headers.accept = 'application/json'; - } - - return options; -}; - -export default normalizeArguments; diff --git a/source/as-promise/parse-body.ts b/source/as-promise/parse-body.ts deleted file mode 100644 index 204a8518e..000000000 --- a/source/as-promise/parse-body.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { - ResponseType, - ParseError, - Response, - ParseJsonFunction -} from './types'; - -const parseBody = (response: Response, responseType: ResponseType, parseJson: ParseJsonFunction, encoding?: BufferEncoding): unknown => { - const {rawBody} = response; - - try { - if (responseType === 'text') { - return rawBody.toString(encoding); - } - - if (responseType === 'json') { - return rawBody.length === 0 ? '' : parseJson(rawBody.toString()); - } - - if (responseType === 'buffer') { - return rawBody; - } - - throw new ParseError({ - message: `Unknown body type '${responseType as string}'`, - name: 'Error' - }, response); - } catch (error) { - throw new ParseError(error, response); - } -}; - -export default parseBody; diff --git a/source/as-promise/types.ts b/source/as-promise/types.ts index e9d27443f..2c1c86935 100644 --- a/source/as-promise/types.ts +++ b/source/as-promise/types.ts @@ -1,289 +1,10 @@ import * as PCancelable from 'p-cancelable'; -import Request, { - Options, - Response, - RequestError, - RequestEvents -} from '../core/index'; - -/** -All parsing methods supported by Got. -*/ -export type ResponseType = 'json' | 'buffer' | 'text'; - -export interface PaginateData { - response: Response; - currentItems: ElementType[]; - allItems: ElementType[]; -} - -export interface FilterData { - item: ElementType; - currentItems: ElementType[]; - allItems: ElementType[]; -} - -export interface PaginationOptions { - /** - All options accepted by `got.paginate()`. - */ - pagination?: { - /** - A function that transform [`Response`](#response) into an array of items. - This is where you should do the parsing. - - @default response => JSON.parse(response.body) - */ - transform?: (response: Response) => Promise | ElementType[]; - - /** - Checks whether the item should be emitted or not. - - @default ({item, currentItems, allItems}) => true - */ - filter?: (data: FilterData) => boolean; - - /** - The function takes an object with the following properties: - - `response` - The current response object. - - `currentItems` - Items from the current response. - - `allItems` - An empty array, unless `pagination.stackAllItems` is set to `true`, in which case, it's an array of the emitted items. - - It should return an object representing Got options pointing to the next page. The options are merged automatically with the previous request, therefore the options returned `pagination.paginate(...)` must reflect changes only. If there are no more pages, `false` should be returned. - - @example - ``` - const got = require('got'); - - (async () => { - const limit = 10; - - const items = got.paginate('https://example.com/items', { - searchParams: { - limit, - offset: 0 - }, - pagination: { - paginate: ({response, currentItems}) => { - const previousSearchParams = response.request.options.searchParams; - const previousOffset = previousSearchParams.get('offset'); - - if (currentItems.length < limit) { - return false; - } - - return { - searchParams: { - ...previousSearchParams, - offset: Number(previousOffset) + limit, - } - }; - } - } - }); - - console.log('Items from all pages:', items); - })(); - ``` - */ - paginate?: (data: PaginateData) => Options | false; - - /** - Checks whether the pagination should continue. - - For example, if you need to stop **before** emitting an entry with some flag, you should use `({item}) => !item.flag`. - - If you want to stop **after** emitting the entry, you should use - `({item, allItems}) => allItems.some(item => item.flag)` instead. - - @default ({item, currentItems, allItems}) => true - */ - shouldContinue?: (data: FilterData) => boolean; - - /** - The maximum amount of items that should be emitted. - - @default Infinity - */ - countLimit?: number; - - /** - Milliseconds to wait before the next request is triggered. - - @default 0 - */ - backoff?: number; - /** - The maximum amount of request that should be triggered. - Retries on failure are not counted towards this limit. - - For example, it can be helpful during development to avoid an infinite number of requests. - - @default 10000 - */ - requestLimit?: number; - - /** - Defines how the property `allItems` in `pagination.paginate`, `pagination.filter` and `pagination.shouldContinue` is managed. - - By default, the property `allItems` is always an empty array. This setting can be helpful to save on memory usage when working with a large dataset. - - When set to `true`, the property `allItems` is an array of the emitted items. - - @default false - */ - stackAllItems?: boolean; - }; -} - -export type AfterResponseHook = (response: Response, retryWithMergedOptions: (options: Options) => CancelableRequest) => Response | CancelableRequest | Promise>; - -// These should be merged into Options in core/index.ts -export namespace PromiseOnly { - export interface Hooks { - /** - Called with [response object](#response) and a retry function. - Calling the retry function will trigger `beforeRetry` hooks. - - Each function should return the response. - This is especially useful when you want to refresh an access token. - - __Note__: When using streams, this hook is ignored. - - @example - ``` - const got = require('got'); - - const instance = got.extend({ - hooks: { - afterResponse: [ - (response, retryWithMergedOptions) => { - if (response.statusCode === 401) { // Unauthorized - const updatedOptions = { - headers: { - token: getNewToken() // Refresh the access token - } - }; - - // Save for further requests - instance.defaults.options = got.mergeOptions(instance.defaults.options, updatedOptions); - - // Make a new retry - return retryWithMergedOptions(updatedOptions); - } - - // No changes otherwise - return response; - } - ], - beforeRetry: [ - (options, error, retryCount) => { - // This will be called on `retryWithMergedOptions(...)` - } - ] - }, - mutableDefaults: true - }); - ``` - */ - afterResponse?: AfterResponseHook[]; - } - - export interface Options extends PaginationOptions { - /** - The parsing method. - - The promise also has `.text()`, `.json()` and `.buffer()` methods which return another Got promise for the parsed body. - - It's like setting the options to `{responseType: 'json', resolveBodyOnly: true}` but without affecting the main Got promise. - - __Note__: When using streams, this option is ignored. - - @example - ``` - (async () => { - const responsePromise = got(url); - const bufferPromise = responsePromise.buffer(); - const jsonPromise = responsePromise.json(); - - const [response, buffer, json] = Promise.all([responsePromise, bufferPromise, jsonPromise]); - // `response` is an instance of Got Response - // `buffer` is an instance of Buffer - // `json` is an object - })(); - ``` - - @example - ``` - // This - const body = await got(url).json(); - - // is semantically the same as this - const body = await got(url, {responseType: 'json', resolveBodyOnly: true}); - ``` - */ - responseType?: ResponseType; - - /** - When set to `true` the promise will return the Response body instead of the Response object. - - @default false - */ - resolveBodyOnly?: boolean; - - /** - Returns a `Stream` instead of a `Promise`. - This is equivalent to calling `got.stream(url, options?)`. - - @default false - */ - isStream?: boolean; - - /** - [Encoding](https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings) to be used on `setEncoding` of the response data. - - To get a [`Buffer`](https://nodejs.org/api/buffer.html), you need to set `responseType` to `buffer` instead. - Don't set this option to `null`. - - __Note__: This doesn't affect streams! Instead, you need to do `got.stream(...).setEncoding(encoding)`. - - @default 'utf-8' - */ - encoding?: BufferEncoding; - } - - export interface NormalizedOptions { - responseType: ResponseType; - resolveBodyOnly: boolean; - isStream: boolean; - encoding?: BufferEncoding; - pagination?: Required['pagination']>; - } - - export interface Defaults { - responseType: ResponseType; - resolveBodyOnly: boolean; - isStream: boolean; - pagination?: Required['pagination']>; - } - - export type HookEvent = 'afterResponse'; -} - -/** -An error to be thrown when server response code is 2xx, and parsing body fails. -Includes a `response` property. -*/ -export class ParseError extends RequestError { - declare readonly response: Response; - - constructor(error: Error, response: Response) { - const {options} = response.request; - - super(`${error.message} in "${options.url.toString()}"`, error, response.request); - this.name = 'ParseError'; - } -} +import {RequestError} from '../core/errors'; +// eslint-disable-next-line import/no-duplicates +import type Request from '../core'; +// eslint-disable-next-line import/no-duplicates +import type {RequestEvents} from '../core'; +import type {Response} from '../core/response'; /** An error to be thrown when the request is aborted with `.cancel()`. @@ -306,5 +27,3 @@ export interface CancelableRequest CancelableRequest; text: () => CancelableRequest; } - -export * from '../core'; diff --git a/source/core/calculate-retry-delay.ts b/source/core/calculate-retry-delay.ts index 6c9085ff0..a75b5ed41 100644 --- a/source/core/calculate-retry-delay.ts +++ b/source/core/calculate-retry-delay.ts @@ -1,10 +1,12 @@ -import {RetryFunction} from './index'; +import type {RetryFunction} from './options'; type Returns unknown, V> = (...args: Parameters) => V; -export const retryAfterStatusCodes: ReadonlySet = new Set([413, 429, 503]); +const calculateRetryDelay: Returns = ({attemptCount, retryOptions, error, retryAfter, computedValue}) => { + if (error.name === 'RetryError') { + return 1; + } -const calculateRetryDelay: Returns = ({attemptCount, retryOptions, error, retryAfter}) => { if (attemptCount > retryOptions.limit) { return 0; } @@ -18,7 +20,8 @@ const calculateRetryDelay: Returns = ({attemptCount, retr if (error.response) { if (retryAfter) { - if (retryOptions.maxRetryAfter === undefined || retryAfter > retryOptions.maxRetryAfter) { + // In this case `computedValue` is `options.request.timeout` + if (retryAfter > computedValue) { return 0; } diff --git a/source/core/errors.ts b/source/core/errors.ts new file mode 100644 index 000000000..de8b02e62 --- /dev/null +++ b/source/core/errors.ts @@ -0,0 +1,162 @@ +import is from '@sindresorhus/is'; +import type Options from './options'; +import type {Timings} from '@szmarczak/http-timer'; +import type {TimeoutError as TimedOutTimeoutError} from './timed-out'; +import type Request from '.'; +import type {PlainResponse, Response} from './response'; + +type Error = NodeJS.ErrnoException; + +// A hacky check to prevent circular references. +function isRequest(x: unknown): x is Request { + return is.object(x) && '_onResponse' in x; +} + +/** +An error to be thrown when a request fails. +Contains a `code` property with error class code, like `ECONNREFUSED`. +*/ +export class RequestError extends Error { + code?: string; + stack!: string; + declare readonly options: Options; + readonly response?: Response; + readonly request?: Request; + readonly timings?: Timings; + + constructor(message: string, error: Partial, self: Request | Options) { + super(message); + Error.captureStackTrace(this, this.constructor); + + this.name = 'RequestError'; + this.code = error.code; + + if (isRequest(self)) { + Object.defineProperty(this, 'request', { + enumerable: false, + value: self + }); + + Object.defineProperty(this, 'response', { + enumerable: false, + value: self.response + }); + + this.options = self.options; + } else { + this.options = self; + } + + this.timings = this.request?.timings; + + // Recover the original stacktrace + if (is.string(error.stack) && is.string(this.stack)) { + const indexOfMessage = this.stack.indexOf(this.message) + this.message.length; + const thisStackTrace = this.stack.slice(indexOfMessage).split('\n').reverse(); + const errorStackTrace = error.stack.slice(error.stack.indexOf(error.message!) + error.message!.length).split('\n').reverse(); + + // Remove duplicated traces + while (errorStackTrace.length > 0 && errorStackTrace[0] === thisStackTrace[0]) { + thisStackTrace.shift(); + } + + this.stack = `${this.stack.slice(0, indexOfMessage)}${thisStackTrace.reverse().join('\n')}${errorStackTrace.reverse().join('\n')}`; + } + } +} + +/** +An error to be thrown when the server redirects you more than ten times. +Includes a `response` property. +*/ +export class MaxRedirectsError extends RequestError { + declare readonly response: Response; + declare readonly request: Request; + declare readonly timings: Timings; + + constructor(request: Request) { + super(`Redirected ${request.options.maxRedirects} times. Aborting.`, {}, request); + this.name = 'MaxRedirectsError'; + } +} + +/** +An error to be thrown when the server response code is not 2xx nor 3xx if `options.followRedirect` is `true`, but always except for 304. +Includes a `response` property. +*/ +export class HTTPError extends RequestError { + declare readonly response: Response; + declare readonly request: Request; + declare readonly timings: Timings; + + constructor(response: PlainResponse) { + super(`Response code ${response.statusCode} (${response.statusMessage!})`, {}, response.request); + this.name = 'HTTPError'; + } +} + +/** +An error to be thrown when a cache method fails. +For example, if the database goes down or there's a filesystem error. +*/ +export class CacheError extends RequestError { + declare readonly request: Request; + + constructor(error: Error, request: Request) { + super(error.message, error, request); + this.name = 'CacheError'; + } +} + +/** +An error to be thrown when the request body is a stream and an error occurs while reading from that stream. +*/ +export class UploadError extends RequestError { + declare readonly request: Request; + + constructor(error: Error, request: Request) { + super(error.message, error, request); + this.name = 'UploadError'; + } +} + +/** +An error to be thrown when the request is aborted due to a timeout. +Includes an `event` and `timings` property. +*/ +export class TimeoutError extends RequestError { + declare readonly request: Request; + readonly timings: Timings; + readonly event: string; + + constructor(error: TimedOutTimeoutError, timings: Timings, request: Request) { + super(error.message, error, request); + this.name = 'TimeoutError'; + this.event = error.event; + this.timings = timings; + } +} + +/** +An error to be thrown when reading from response stream fails. +*/ +export class ReadError extends RequestError { + declare readonly request: Request; + declare readonly response: Response; + declare readonly timings: Timings; + + constructor(error: Error, request: Request) { + super(error.message, error, request); + this.name = 'ReadError'; + } +} + +/** +An error which always triggers a new retry when thrown. +*/ +export class RetryError extends RequestError { + constructor(request: Request) { + super('Retrying', {}, request); + this.name = 'RetryError'; + } +} diff --git a/source/core/index.ts b/source/core/index.ts index 5dae15bb9..a5041a579 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -1,938 +1,38 @@ -import {promisify} from 'util'; import {Duplex, Writable, Readable} from 'stream'; -import {ReadStream} from 'fs'; import {URL, URLSearchParams} from 'url'; -import {Socket} from 'net'; -import {SecureContextOptions, DetailedPeerCertificate} from 'tls'; import * as http from 'http'; -import {ClientRequest, RequestOptions, IncomingMessage, ServerResponse, request as httpRequest} from 'http'; -import * as https from 'https'; -import timer, {ClientRequestWithTimings, Timings, IncomingMessageWithTimings} from '@szmarczak/http-timer'; -import CacheableLookup from 'cacheable-lookup'; +import {ServerResponse} from 'http'; +import timer from '@szmarczak/http-timer'; import * as CacheableRequest from 'cacheable-request'; import decompressResponse = require('decompress-response'); -import http2wrapper = require('http2-wrapper'); -import lowercaseKeys = require('lowercase-keys'); -import ResponseLike = require('responselike'); -import is, {assert} from '@sindresorhus/is'; +import is from '@sindresorhus/is'; import applyDestroyPatch from './utils/apply-destroy-patch'; import getBodySize from './utils/get-body-size'; import isFormData from './utils/is-form-data'; import proxyEvents from './utils/proxy-events'; -import timedOut, {Delays, TimeoutError as TimedOutTimeoutError} from './utils/timed-out'; +import timedOut, {TimeoutError as TimedOutTimeoutError} from './timed-out'; import urlToOptions from './utils/url-to-options'; -import optionsToUrl, {URLOptions} from './utils/options-to-url'; import WeakableMap from './utils/weakable-map'; -import getBuffer from './utils/get-buffer'; -import {DnsLookupIpVersion, isDnsLookupIpVersion, dnsLookupIpVersionToFamily} from './utils/dns-ip-version'; -import {isResponseOk} from './utils/is-response-ok'; -import deprecationWarning from '../utils/deprecation-warning'; -import normalizePromiseArguments from '../as-promise/normalize-arguments'; -import {PromiseOnly} from '../as-promise/types'; +import {buffer as getBuffer} from 'get-stream'; import calculateRetryDelay from './calculate-retry-delay'; - -const [major, minor] = process.versions.node.split('.').map(x => Number(x)) as [number, number, number]; - -let globalDnsCache: CacheableLookup; - -type HttpRequestFunction = typeof httpRequest; -type Error = NodeJS.ErrnoException; - -const kRequest = Symbol('request'); -const kResponse = Symbol('response'); -const kResponseSize = Symbol('responseSize'); -const kDownloadedSize = Symbol('downloadedSize'); -const kBodySize = Symbol('bodySize'); -const kUploadedSize = Symbol('uploadedSize'); -const kServerResponsesPiped = Symbol('serverResponsesPiped'); -const kUnproxyEvents = Symbol('unproxyEvents'); -const kIsFromCache = Symbol('isFromCache'); -const kCancelTimeouts = Symbol('cancelTimeouts'); -const kStartedReading = Symbol('startedReading'); -const kStopReading = Symbol('stopReading'); -const kTriggerRead = Symbol('triggerRead'); -const kBody = Symbol('body'); -const kJobs = Symbol('jobs'); -const kOriginalResponse = Symbol('originalResponse'); -const kRetryTimeout = Symbol('retryTimeout'); -export const kIsNormalizedAlready = Symbol('isNormalizedAlready'); - -const supportsBrotli = is.string((process.versions as any).brotli); - -export interface Agents { - http?: http.Agent; - https?: https.Agent; - http2?: unknown; -} - -export const withoutBody: ReadonlySet = new Set(['GET', 'HEAD']); - -export interface ToughCookieJar { - getCookieString: ((currentUrl: string, options: Record, cb: (error: Error | null, cookies: string) => void) => void) - & ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => void); - setCookie: ((cookieOrString: unknown, currentUrl: string, options: Record, cb: (error: Error | null, cookie: unknown) => void) => void) - & ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void); -} - -export interface PromiseCookieJar { - getCookieString: (url: string) => Promise; - setCookie: (rawCookie: string, url: string) => Promise; -} - -/** -All available HTTP request methods provided by Got. -*/ -export type Method = - | 'GET' - | 'POST' - | 'PUT' - | 'PATCH' - | 'HEAD' - | 'DELETE' - | 'OPTIONS' - | 'TRACE' - | 'get' - | 'post' - | 'put' - | 'patch' - | 'head' - | 'delete' - | 'options' - | 'trace'; - -type Promisable = T | Promise; - -export type InitHook = (options: Options) => void; -export type BeforeRequestHook = (options: NormalizedOptions) => Promisable; -export type BeforeRedirectHook = (options: NormalizedOptions, response: Response) => Promisable; -export type BeforeErrorHook = (error: RequestError) => Promisable; -export type BeforeRetryHook = (options: NormalizedOptions, error?: RequestError, retryCount?: number) => void | Promise; - -interface PlainHooks { - /** - Called with plain request options, right before their normalization. - This is especially useful in conjunction with `got.extend()` when the input needs custom handling. - - __Note #1__: This hook must be synchronous! - - __Note #2__: Errors in this hook will be converted into an instances of `RequestError`. - - __Note #3__: The options object may not have a `url` property. - To modify it, use a `beforeRequest` hook instead. - - @default [] - */ - init?: InitHook[]; - - /** - Called with normalized request options. - Got will make no further changes to the request before it is sent. - This is especially useful in conjunction with `got.extend()` when you want to create an API client that, for example, uses HMAC-signing. - - @default [] - */ - beforeRequest?: BeforeRequestHook[]; - - /** - Called with normalized request options and the redirect response. - Got will make no further changes to the request. - This is especially useful when you want to avoid dead sites. - - @default [] - - @example - ``` - const got = require('got'); - - got('https://example.com', { - hooks: { - beforeRedirect: [ - (options, response) => { - if (options.hostname === 'deadSite') { - options.hostname = 'fallbackSite'; - } - } - ] - } - }); - ``` - */ - beforeRedirect?: BeforeRedirectHook[]; - - /** - Called with an `Error` instance. - The error is passed to the hook right before it's thrown. - This is especially useful when you want to have more detailed errors. - - __Note__: Errors thrown while normalizing input options are thrown directly and not part of this hook. - - @default [] - - @example - ``` - const got = require('got'); - - got('https://api.github.com/some-endpoint', { - hooks: { - beforeError: [ - error => { - const {response} = error; - if (response && response.body) { - error.name = 'GitHubError'; - error.message = `${response.body.message} (${response.statusCode})`; - } - - return error; - } - ] - } - }); - ``` - */ - beforeError?: BeforeErrorHook[]; - - /** - Called with normalized request options, the error and the retry count. - Got will make no further changes to the request. - This is especially useful when some extra work is required before the next try. - - __Note__: When using streams, this hook is ignored. - __Note__: When retrying in a `afterResponse` hook, all remaining `beforeRetry` hooks will be called without the `error` and `retryCount` arguments. - - @default [] - - @example - ``` - const got = require('got'); - - got.post('https://example.com', { - hooks: { - beforeRetry: [ - (options, error, retryCount) => { - if (error.response.statusCode === 413) { // Payload too large - options.body = getNewBody(); - } - } - ] - } - }); - ``` - */ - beforeRetry?: BeforeRetryHook[]; -} - -/** -All available hook of Got. -*/ -export interface Hooks extends PromiseOnly.Hooks, PlainHooks {} - -type PlainHookEvent = 'init' | 'beforeRequest' | 'beforeRedirect' | 'beforeError' | 'beforeRetry'; - -/** -All hook events acceptable by Got. -*/ -export type HookEvent = PromiseOnly.HookEvent | PlainHookEvent; - -export const knownHookEvents: HookEvent[] = [ - 'init', - 'beforeRequest', - 'beforeRedirect', - 'beforeError', - 'beforeRetry', - - // Promise-Only - 'afterResponse' -]; - -type AcceptableResponse = IncomingMessageWithTimings | ResponseLike; -type AcceptableRequestResult = AcceptableResponse | ClientRequest | Promise | undefined; - -export type RequestFunction = (url: URL, options: RequestOptions, callback?: (response: AcceptableResponse) => void) => AcceptableRequestResult; - -export type Headers = Record; - -type CacheableRequestFunction = ( - options: string | URL | RequestOptions, - cb?: (response: ServerResponse | ResponseLike) => void -) => CacheableRequest.Emitter; - -type CheckServerIdentityFunction = (hostname: string, certificate: DetailedPeerCertificate) => Error | void; -export type ParseJsonFunction = (text: string) => unknown; -export type StringifyJsonFunction = (object: unknown) => string; - -interface RealRequestOptions extends https.RequestOptions { - checkServerIdentity: CheckServerIdentityFunction; -} - -export interface RetryObject { - attemptCount: number; - retryOptions: RequiredRetryOptions; - error: TimeoutError | RequestError; - computedValue: number; - retryAfter?: number; -} - -export type RetryFunction = (retryObject: RetryObject) => number | Promise; - -/** -An object representing `limit`, `calculateDelay`, `methods`, `statusCodes`, `maxRetryAfter` and `errorCodes` fields for maximum retry count, retry handler, allowed methods, allowed status codes, maximum [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) time and allowed error codes. - -Delays between retries counts with function `1000 * Math.pow(2, retry) + Math.random() * 100`, where `retry` is attempt number (starts from 1). - -The `calculateDelay` property is a `function` that receives an object with `attemptCount`, `retryOptions`, `error` and `computedValue` properties for current retry count, the retry options, error and default computed value. -The function must return a delay in milliseconds (or a Promise resolving with it) (`0` return value cancels retry). - -By default, it retries *only* on the specified methods, status codes, and on these network errors: -- `ETIMEDOUT`: One of the [timeout](#timeout) limits were reached. -- `ECONNRESET`: Connection was forcibly closed by a peer. -- `EADDRINUSE`: Could not bind to any free port. -- `ECONNREFUSED`: Connection was refused by the server. -- `EPIPE`: The remote side of the stream being written has been closed. -- `ENOTFOUND`: Couldn't resolve the hostname to an IP address. -- `ENETUNREACH`: No internet connection. -- `EAI_AGAIN`: DNS lookup timed out. - -__Note__: If `maxRetryAfter` is set to `undefined`, it will use `options.timeout`. -__Note__: If [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) header is greater than `maxRetryAfter`, it will cancel the request. -*/ -export interface RequiredRetryOptions { - limit: number; - methods: Method[]; - statusCodes: number[]; - errorCodes: string[]; - calculateDelay: RetryFunction; - maxRetryAfter?: number; -} - -export interface CacheOptions { - shared?: boolean; - cacheHeuristic?: number; - immutableMinTimeToLive?: number; - ignoreCargoCult?: boolean; -} - -interface PlainOptions extends URLOptions { - /** - Custom request function. - The main purpose of this is to [support HTTP2 using a wrapper](https://github.com/szmarczak/http2-wrapper). - - @default http.request | https.request - */ - request?: RequestFunction; - - /** - An object representing `http`, `https` and `http2` keys for [`http.Agent`](https://nodejs.org/api/http.html#http_class_http_agent), [`https.Agent`](https://nodejs.org/api/https.html#https_class_https_agent) and [`http2wrapper.Agent`](https://github.com/szmarczak/http2-wrapper#new-http2agentoptions) instance. - This is necessary because a request to one protocol might redirect to another. - In such a scenario, Got will switch over to the right protocol agent for you. - - If a key is not present, it will default to a global agent. - - @example - ``` - const got = require('got'); - const HttpAgent = require('agentkeepalive'); - const {HttpsAgent} = HttpAgent; - - got('https://sindresorhus.com', { - agent: { - http: new HttpAgent(), - https: new HttpsAgent() - } - }); - ``` - */ - agent?: Agents | false; - - /** - Decompress the response automatically. - This will set the `accept-encoding` header to `gzip, deflate, br` on Node.js 11.7.0+ or `gzip, deflate` for older Node.js versions, unless you set it yourself. - - Brotli (`br`) support requires Node.js 11.7.0 or later. - - If this is disabled, a compressed response is returned as a `Buffer`. - This may be useful if you want to handle decompression yourself or stream the raw compressed data. - - @default true - */ - decompress?: boolean; - - /** - Milliseconds to wait for the server to end the response before aborting the request with `got.TimeoutError` error (a.k.a. `request` property). - By default, there's no timeout. - - This also accepts an `object` with the following fields to constrain the duration of each phase of the request lifecycle: - - - `lookup` starts when a socket is assigned and ends when the hostname has been resolved. - Does not apply when using a Unix domain socket. - - `connect` starts when `lookup` completes (or when the socket is assigned if lookup does not apply to the request) and ends when the socket is connected. - - `secureConnect` starts when `connect` completes and ends when the handshaking process completes (HTTPS only). - - `socket` starts when the socket is connected. See [request.setTimeout](https://nodejs.org/api/http.html#http_request_settimeout_timeout_callback). - - `response` starts when the request has been written to the socket and ends when the response headers are received. - - `send` starts when the socket is connected and ends with the request has been written to the socket. - - `request` starts when the request is initiated and ends when the response's end event fires. - */ - timeout?: Delays | number; - - /** - When specified, `prefixUrl` will be prepended to `url`. - The prefix can be any valid URL, either relative or absolute. - A trailing slash `/` is optional - one will be added automatically. - - __Note__: `prefixUrl` will be ignored if the `url` argument is a URL instance. - - __Note__: Leading slashes in `input` are disallowed when using this option to enforce consistency and avoid confusion. - For example, when the prefix URL is `https://example.com/foo` and the input is `/bar`, there's ambiguity whether the resulting URL would become `https://example.com/foo/bar` or `https://example.com/bar`. - The latter is used by browsers. - - __Tip__: Useful when used with `got.extend()` to create niche-specific Got instances. - - __Tip__: You can change `prefixUrl` using hooks as long as the URL still includes the `prefixUrl`. - If the URL doesn't include it anymore, it will throw. - - @example - ``` - const got = require('got'); - - (async () => { - await got('unicorn', {prefixUrl: 'https://cats.com'}); - //=> 'https://cats.com/unicorn' - - const instance = got.extend({ - prefixUrl: 'https://google.com' - }); - - await instance('unicorn', { - hooks: { - beforeRequest: [ - options => { - options.prefixUrl = 'https://cats.com'; - } - ] - } - }); - //=> 'https://cats.com/unicorn' - })(); - ``` - */ - prefixUrl?: string | URL; - - /** - __Note #1__: The `body` option cannot be used with the `json` or `form` option. - - __Note #2__: If you provide this option, `got.stream()` will be read-only. - - __Note #3__: If you provide a payload with the `GET` or `HEAD` method, it will throw a `TypeError` unless the method is `GET` and the `allowGetBody` option is set to `true`. - - __Note #4__: This option is not enumerable and will not be merged with the instance defaults. - - The `content-length` header will be automatically set if `body` is a `string` / `Buffer` / [`form-data` instance](https://github.com/form-data/form-data), and `content-length` and `transfer-encoding` are not manually set in `options.headers`. - - Since Got 12, the `content-length` is not automatically set when `body` is a `fs.createReadStream`. - */ - body?: string | Buffer | Readable; - - /** - The form body is converted to a query string using [`(new URLSearchParams(object)).toString()`](https://nodejs.org/api/url.html#url_constructor_new_urlsearchparams_obj). - - If the `Content-Type` header is not present, it will be set to `application/x-www-form-urlencoded`. - - __Note #1__: If you provide this option, `got.stream()` will be read-only. - - __Note #2__: This option is not enumerable and will not be merged with the instance defaults. - */ - form?: Record; - - /** - JSON body. If the `Content-Type` header is not set, it will be set to `application/json`. - - __Note #1__: If you provide this option, `got.stream()` will be read-only. - - __Note #2__: This option is not enumerable and will not be merged with the instance defaults. - */ - json?: Record; - - /** - The URL to request, as a string, a [`https.request` options object](https://nodejs.org/api/https.html#https_https_request_options_callback), or a [WHATWG `URL`](https://nodejs.org/api/url.html#url_class_url). - - Properties from `options` will override properties in the parsed `url`. - - If no protocol is specified, it will throw a `TypeError`. - - __Note__: The query string is **not** parsed as search params. - - @example - ``` - got('https://example.com/?query=a b'); //=> https://example.com/?query=a%20b - got('https://example.com/', {searchParams: {query: 'a b'}}); //=> https://example.com/?query=a+b - - // The query string is overridden by `searchParams` - got('https://example.com/?query=a b', {searchParams: {query: 'a b'}}); //=> https://example.com/?query=a+b - ``` - */ - url?: string | URL; - - /** - Cookie support. You don't have to care about parsing or how to store them. - - __Note__: If you provide this option, `options.headers.cookie` will be overridden. - */ - cookieJar?: PromiseCookieJar | ToughCookieJar; - - /** - Ignore invalid cookies instead of throwing an error. - Only useful when the `cookieJar` option has been set. Not recommended. - - @default false - */ - ignoreInvalidCookies?: boolean; - - /** - Query string that will be added to the request URL. - This will override the query string in `url`. - - If you need to pass in an array, you can do it using a `URLSearchParams` instance. - - @example - ``` - const got = require('got'); - - const searchParams = new URLSearchParams([['key', 'a'], ['key', 'b']]); - - got('https://example.com', {searchParams}); - - console.log(searchParams.toString()); - //=> 'key=a&key=b' - ``` - */ - searchParams?: string | Record | URLSearchParams; - - /** - An instance of [`CacheableLookup`](https://github.com/szmarczak/cacheable-lookup) used for making DNS lookups. - Useful when making lots of requests to different *public* hostnames. - - `CacheableLookup` uses `dns.resolver4(..)` and `dns.resolver6(...)` under the hood and fall backs to `dns.lookup(...)` when the first two fail, which may lead to additional delay. - - __Note__: This should stay disabled when making requests to internal hostnames such as `localhost`, `database.local` etc. - - @default false - */ - dnsCache?: CacheableLookup | boolean; - - /** - User data. `context` is shallow merged and enumerable. If it contains non-enumerable properties they will NOT be merged. - - @example - ``` - const got = require('got'); - - const instance = got.extend({ - hooks: { - beforeRequest: [ - options => { - if (!options.context || !options.context.token) { - throw new Error('Token required'); - } - - options.headers.token = options.context.token; - } - ] - } - }); - - (async () => { - const context = { - token: 'secret' - }; - - const response = await instance('https://httpbin.org/headers', {context}); - - // Let's see the headers - console.log(response.body); - })(); - ``` - */ - context?: Record; - - /** - Hooks allow modifications during the request lifecycle. - Hook functions may be async and are run serially. - */ - hooks?: Hooks; - - /** - Defines if redirect responses should be followed automatically. - - Note that if a `303` is sent by the server in response to any request type (`POST`, `DELETE`, etc.), Got will automatically request the resource pointed to in the location header via `GET`. - This is in accordance with [the spec](https://tools.ietf.org/html/rfc7231#section-6.4.4). - - @default true - */ - followRedirect?: boolean; - - /** - If exceeded, the request will be aborted and a `MaxRedirectsError` will be thrown. - - @default 10 - */ - maxRedirects?: number; - - /** - A cache adapter instance for storing cached response data. - - @default false - */ - cache?: string | CacheableRequest.StorageAdapter | false; - - /** - Determines if a `got.HTTPError` is thrown for unsuccessful responses. - - If this is disabled, requests that encounter an error status code will be resolved with the `response` instead of throwing. - This may be useful if you are checking for resource availability and are expecting error responses. - - @default true - */ - throwHttpErrors?: boolean; - - username?: string; - - password?: string; - - /** - If set to `true`, Got will additionally accept HTTP2 requests. - - It will choose either HTTP/1.1 or HTTP/2 depending on the ALPN protocol. - - __Note__: This option requires Node.js 15.10.0 or newer as HTTP/2 support on older Node.js versions is very buggy. - - __Note__: Overriding `options.request` will disable HTTP2 support. - - @default false - - @example - ``` - const got = require('got'); - - (async () => { - const {headers} = await got('https://nghttp2.org/httpbin/anything', {http2: true}); - console.log(headers.via); - //=> '2 nghttpx' - })(); - ``` - */ - http2?: boolean; - - /** - Set this to `true` to allow sending body for the `GET` method. - However, the [HTTP/2 specification](https://tools.ietf.org/html/rfc7540#section-8.1.3) says that `An HTTP GET request includes request header fields and no payload body`, therefore when using the HTTP/2 protocol this option will have no effect. - This option is only meant to interact with non-compliant servers when you have no other choice. - - __Note__: The [RFC 7321](https://tools.ietf.org/html/rfc7231#section-4.3.1) doesn't specify any particular behavior for the GET method having a payload, therefore __it's considered an [anti-pattern](https://en.wikipedia.org/wiki/Anti-pattern)__. - - @default false - */ - allowGetBody?: boolean; - - lookup?: CacheableLookup['lookup']; - - /** - Request headers. - - Existing headers will be overwritten. Headers set to `undefined` will be omitted. - - @default {} - */ - headers?: Headers; - - /** - Specifies if the redirects should be [rewritten as `GET`](https://tools.ietf.org/html/rfc7231#section-6.4). - - If `false`, when sending a POST request and receiving a `302`, it will resend the body to the new location using the same HTTP method (`POST` in this case). - - @default false - */ - methodRewriting?: boolean; - - /** - Indicates which DNS record family to use. - - Values: - - `auto`: IPv4 (if present) or IPv6 - - `ipv4`: Only IPv4 - - `ipv6`: Only IPv6 - - __Note__: If you are using the undocumented option `family`, `dnsLookupIpVersion` will override it. - - @default 'auto' - */ - dnsLookupIpVersion?: DnsLookupIpVersion; - - /** - A function used to parse JSON responses. - - @example - ``` - const got = require('got'); - const Bourne = require('@hapi/bourne'); - - (async () => { - const parsed = await got('https://example.com', { - parseJson: text => Bourne.parse(text) - }).json(); - - console.log(parsed); - })(); - ``` - */ - parseJson?: ParseJsonFunction; - - /** - A function used to stringify the body of JSON requests. - - @example - ``` - const got = require('got'); - - (async () => { - await got.post('https://example.com', { - stringifyJson: object => JSON.stringify(object, (key, value) => { - if (key.startsWith('_')) { - return; - } - - return value; - }), - json: { - some: 'payload', - _ignoreMe: 1234 - } - }); - })(); - ``` - - @example - ``` - const got = require('got'); - - (async () => { - await got.post('https://example.com', { - stringifyJson: object => JSON.stringify(object, (key, value) => { - if (typeof value === 'number') { - return value.toString(); - } - - return value; - }), - json: { - some: 'payload', - number: 1 - } - }); - })(); - ``` - */ - stringifyJson?: StringifyJsonFunction; - - /** - An object representing `limit`, `calculateDelay`, `methods`, `statusCodes`, `maxRetryAfter` and `errorCodes` fields for maximum retry count, retry handler, allowed methods, allowed status codes, maximum [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) time and allowed error codes. - - Delays between retries counts with function `1000 * Math.pow(2, retry) + Math.random() * 100`, where `retry` is attempt number (starts from 1). - - The `calculateDelay` property is a `function` that receives an object with `attemptCount`, `retryOptions`, `error` and `computedValue` properties for current retry count, the retry options, error and default computed value. - The function must return a delay in milliseconds (or a Promise resolving with it) (`0` return value cancels retry). - - By default, it retries *only* on the specified methods, status codes, and on these network errors: - - - `ETIMEDOUT`: One of the [timeout](#timeout) limits were reached. - - `ECONNRESET`: Connection was forcibly closed by a peer. - - `EADDRINUSE`: Could not bind to any free port. - - `ECONNREFUSED`: Connection was refused by the server. - - `EPIPE`: The remote side of the stream being written has been closed. - - `ENOTFOUND`: Couldn't resolve the hostname to an IP address. - - `ENETUNREACH`: No internet connection. - - `EAI_AGAIN`: DNS lookup timed out. - - __Note__: If `maxRetryAfter` is set to `undefined`, it will use `options.timeout`. - __Note__: If [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) header is greater than `maxRetryAfter`, it will cancel the request. - */ - retry?: Partial | number; - - // From `http.RequestOptions` - /** - The IP address used to send the request from. - */ - localAddress?: string; - - socketPath?: string; - - /** - The HTTP method used to make the request. - - @default 'GET' - */ - method?: Method; - - createConnection?: (options: http.RequestOptions, oncreate: (error: Error, socket: Socket) => void) => Socket; - - // From `http-cache-semantics` - cacheOptions?: CacheOptions; - - // TODO: remove when Got 12 gets released - /** - If set to `false`, all invalid SSL certificates will be ignored and no error will be thrown. - - If set to `true`, it will throw an error whenever an invalid SSL certificate is detected. - - We strongly recommend to have this set to `true` for security reasons. - - @default true - - @example - ``` - const got = require('got'); - - (async () => { - // Correct: - await got('https://example.com', {rejectUnauthorized: true}); - - // You can disable it when developing an HTTPS app: - await got('https://localhost', {rejectUnauthorized: false}); - - // Never do this: - await got('https://example.com', {rejectUnauthorized: false}); - })(); - ``` - */ - rejectUnauthorized?: boolean; // Here for backwards compatibility - - /** - Options for the advanced HTTPS API. - */ - https?: HTTPSOptions; -} - -export interface Options extends PromiseOnly.Options, PlainOptions {} - -export interface HTTPSOptions { - // From `http.RequestOptions` and `tls.CommonConnectionOptions` - rejectUnauthorized?: https.RequestOptions['rejectUnauthorized']; - - // From `tls.ConnectionOptions` - checkServerIdentity?: CheckServerIdentityFunction; - - // From `tls.SecureContextOptions` - /** - Override the default Certificate Authorities ([from Mozilla](https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReport)). - - @example - ``` - // Single Certificate Authority - got('https://example.com', { - https: { - certificateAuthority: fs.readFileSync('./my_ca.pem') - } - }); - ``` - */ - certificateAuthority?: SecureContextOptions['ca']; - - /** - Private keys in [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) format. - - [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) allows the option of private keys being encrypted. - Encrypted keys will be decrypted with `options.https.passphrase`. - - Multiple keys with different passphrases can be provided as an array of `{pem: , passphrase: }` - */ - key?: SecureContextOptions['key']; - - /** - [Certificate chains](https://en.wikipedia.org/wiki/X.509#Certificate_chains_and_cross-certification) in [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) format. - - One cert chain should be provided per private key (`options.https.key`). - - When providing multiple cert chains, they do not have to be in the same order as their private keys in `options.https.key`. - - If the intermediate certificates are not provided, the peer will not be able to validate the certificate, and the handshake will fail. - */ - certificate?: SecureContextOptions['cert']; - - /** - The passphrase to decrypt the `options.https.key` (if different keys have different passphrases refer to `options.https.key` documentation). - */ - passphrase?: SecureContextOptions['passphrase']; - pfx?: SecureContextOptions['pfx']; -} - -interface NormalizedPlainOptions extends PlainOptions { - method: Method; - url: URL; - timeout: Delays; - prefixUrl: string; - ignoreInvalidCookies: boolean; - decompress: boolean; - searchParams?: URLSearchParams; - cookieJar?: PromiseCookieJar; - headers: Headers; - context: Record; - hooks: Required; - followRedirect: boolean; - maxRedirects: number; - cache?: string | CacheableRequest.StorageAdapter; - throwHttpErrors: boolean; - dnsCache?: CacheableLookup; - http2: boolean; - allowGetBody: boolean; - rejectUnauthorized: boolean; - lookup?: CacheableLookup['lookup']; - methodRewriting: boolean; - username: string; - password: string; - parseJson: ParseJsonFunction; - stringifyJson: StringifyJsonFunction; - retry: RequiredRetryOptions; - cacheOptions: CacheOptions; - [kRequest]: HttpRequestFunction; - [kIsNormalizedAlready]?: boolean; -} - -export interface NormalizedOptions extends PromiseOnly.NormalizedOptions, NormalizedPlainOptions {} - -interface PlainDefaults { - timeout: Delays; - prefixUrl: string; - method: Method; - ignoreInvalidCookies: boolean; - decompress: boolean; - context: Record; - cookieJar?: PromiseCookieJar | ToughCookieJar; - dnsCache?: CacheableLookup; - headers: Headers; - hooks: Required; - followRedirect: boolean; - maxRedirects: number; - cache?: string | CacheableRequest.StorageAdapter; - throwHttpErrors: boolean; - http2: boolean; - allowGetBody: boolean; - https?: HTTPSOptions; - methodRewriting: boolean; - parseJson: ParseJsonFunction; - stringifyJson: StringifyJsonFunction; - retry: RequiredRetryOptions; - - // Optional - agent?: Agents | false; - request?: RequestFunction; - searchParams?: URLSearchParams; - lookup?: CacheableLookup['lookup']; - localAddress?: string; - createConnection?: Options['createConnection']; - - // From `http-cache-semantics` - cacheOptions: CacheOptions; -} - -export interface Defaults extends PromiseOnly.Defaults, PlainDefaults {} +import Options, {OptionsError} from './options'; +import {isResponseOk, Response} from './response'; +import isClientRequest from './utils/is-client-request'; +import { + RequestError, + ReadError, + MaxRedirectsError, + HTTPError, + TimeoutError, + UploadError, + CacheError +} from './errors'; +import type {ClientRequestWithTimings, Timings, IncomingMessageWithTimings} from '@szmarczak/http-timer'; +import type {ClientRequest, RequestOptions, IncomingMessage} from 'http'; +import type {Socket} from 'net'; +import type ResponseLike = require('responselike'); +import type {PlainResponse} from './response'; +import type {OptionsInit, PromiseCookieJar, NativeRequestOptions, RetryOptions} from './options'; export interface Progress { percent: number; @@ -940,108 +40,13 @@ export interface Progress { total?: number; } -export interface PlainResponse extends IncomingMessageWithTimings { - /** - The original request URL. - */ - requestUrl: string; - - /** - The redirect URLs. - */ - redirectUrls: string[]; - - /** - - `options` - The Got options that were set on this request. - - __Note__: This is not a [http.ClientRequest](https://nodejs.org/api/http.html#http_class_http_clientrequest). - */ - request: Request; - - /** - The remote IP address. - - This is hopefully a temporary limitation, see [lukechilds/cacheable-request#86](https://github.com/lukechilds/cacheable-request/issues/86). - - __Note__: Not available when the response is cached. - */ - ip?: string; - - /** - Whether the response was retrieved from the cache. - */ - isFromCache: boolean; - - /** - The status code of the response. - */ - statusCode: number; - - /** - The request URL or the final URL after redirects. - */ - url: string; - - /** - The object contains the following properties: - - - `start` - Time when the request started. - - `socket` - Time when a socket was assigned to the request. - - `lookup` - Time when the DNS lookup finished. - - `connect` - Time when the socket successfully connected. - - `secureConnect` - Time when the socket securely connected. - - `upload` - Time when the request finished uploading. - - `response` - Time when the request fired `response` event. - - `end` - Time when the response fired `end` event. - - `error` - Time when the request fired `error` event. - - `abort` - Time when the request fired `abort` event. - - `phases` - - `wait` - `timings.socket - timings.start` - - `dns` - `timings.lookup - timings.socket` - - `tcp` - `timings.connect - timings.lookup` - - `tls` - `timings.secureConnect - timings.connect` - - `request` - `timings.upload - (timings.secureConnect || timings.connect)` - - `firstByte` - `timings.response - timings.upload` - - `download` - `timings.end - timings.response` - - `total` - `(timings.end || timings.error || timings.abort) - timings.start` - - If something has not been measured yet, it will be `undefined`. - - __Note__: The time is a `number` representing the milliseconds elapsed since the UNIX epoch. - */ - timings: Timings; - - /** - The number of times the request was retried. - */ - retryCount: number; - - // Defined only if request errored - /** - The raw result of the request. - */ - rawBody?: Buffer; - - /** - The result of the request. - */ - body?: unknown; -} +type Error = NodeJS.ErrnoException; -// For Promise support -export interface Response extends PlainResponse { - /** - The result of the request. - */ - body: T; +const supportsBrotli = is.string(process.versions.brotli); - /** - The raw result of the request. - */ - rawBody: Buffer; -} +const methodsWithoutBody: ReadonlySet = new Set(['GET', 'HEAD']); -export interface RequestEvents { +export type GotEventFunction = /** `request` event to get the request object of the request. @@ -1053,7 +58,7 @@ export interface RequestEvents { .on('request', request => setTimeout(() => request.destroy(), 50)); ``` */ - on: ((name: 'request', listener: (request: http.ClientRequest) => void) => T) + ((name: 'request', listener: (request: ClientRequest) => void) => T) /** The `response` event to get the response object of the final request. @@ -1063,7 +68,7 @@ export interface RequestEvents { /** The `redirect` event to get the response object of a redirect. The second argument is options for the next request to the redirect location. */ - & ((name: 'redirect', listener: (response: R, nextOptions: N) => void) => T) + & ((name: 'redirect', listener: (response: R, nextOptions: N) => void) => T) /** Progress events for uploading (sending a request) and downloading (receiving a response). @@ -1103,278 +108,70 @@ export interface RequestEvents { See `got.options.retry` for more information. */ & ((name: 'retry', listener: (retryCount: number, error: RequestError) => void) => T); -} -function validateSearchParameters(searchParameters: Record): asserts searchParameters is Record { - // eslint-disable-next-line guard-for-in - for (const key in searchParameters) { - const value = searchParameters[key]; - - if (!is.string(value) && !is.number(value) && !is.boolean(value) && !is.null_(value) && !is.undefined(value)) { - throw new TypeError(`The \`searchParams\` value '${String(value)}' must be a string, number, boolean or null`); - } - } +export interface RequestEvents { + on: GotEventFunction; + once: GotEventFunction; } -function isClientRequest(clientRequest: unknown): clientRequest is ClientRequest { - return is.object(clientRequest) && !('statusCode' in clientRequest); -} +export type CacheableRequestFunction = ( + options: string | URL | NativeRequestOptions, + cb?: (response: ServerResponse | ResponseLike) => void +) => CacheableRequest.Emitter; const cacheableStore = new WeakableMap(); -const waitForOpenFile = async (file: ReadStream): Promise => new Promise((resolve, reject) => { - const onError = (error: Error): void => { - reject(error); - }; - - // Node.js 12 has incomplete types - if (!(file as any).pending) { - resolve(); - } - - file.once('error', onError); - file.once('ready', () => { - file.off('error', onError); - resolve(); - }); -}); - const redirectCodes: ReadonlySet = new Set([300, 301, 302, 303, 304, 307, 308]); -type NonEnumerableProperty = 'body' | 'json' | 'form'; -const nonEnumerableProperties: NonEnumerableProperty[] = [ - 'body', - 'json', - 'form' -]; - -export const setNonEnumerableProperties = (sources: Array, to: Options): void => { - // Non enumerable properties shall not be merged - const properties: Partial<{[Key in NonEnumerableProperty]: any}> = {}; - - for (const source of sources) { - if (!source) { - continue; - } - - for (const name of nonEnumerableProperties) { - if (!(name in source)) { - continue; - } - - properties[name] = { - writable: true, - configurable: true, - enumerable: false, - // @ts-expect-error TS doesn't see the check above - value: source[name] - }; - } - } - - Object.defineProperties(to, properties); -}; - -/** -An error to be thrown when a request fails. -Contains a `code` property with error class code, like `ECONNREFUSED`. -*/ -export class RequestError extends Error { - code?: string; - stack!: string; - declare readonly options: NormalizedOptions; - readonly response?: Response; - readonly request?: Request; - readonly timings?: Timings; - - constructor(message: string, error: Partial, self: Request | NormalizedOptions) { - super(message); - Error.captureStackTrace(this, this.constructor); - - this.name = 'RequestError'; - this.code = error.code; - - if (self instanceof Request) { - Object.defineProperty(this, 'request', { - enumerable: false, - value: self - }); - - Object.defineProperty(this, 'response', { - enumerable: false, - value: self[kResponse] - }); - - Object.defineProperty(this, 'options', { - // This fails because of TS 3.7.2 useDefineForClassFields - // Ref: https://github.com/microsoft/TypeScript/issues/34972 - enumerable: false, - value: self.options - }); - } else { - Object.defineProperty(this, 'options', { - // This fails because of TS 3.7.2 useDefineForClassFields - // Ref: https://github.com/microsoft/TypeScript/issues/34972 - enumerable: false, - value: self - }); - } - - this.timings = this.request?.timings; - - // Recover the original stacktrace - if (is.string(error.stack) && is.string(this.stack)) { - const indexOfMessage = this.stack.indexOf(this.message) + this.message.length; - const thisStackTrace = this.stack.slice(indexOfMessage).split('\n').reverse(); - const errorStackTrace = error.stack.slice(error.stack.indexOf(error.message!) + error.message!.length).split('\n').reverse(); - - // Remove duplicated traces - while (errorStackTrace.length > 0 && errorStackTrace[0] === thisStackTrace[0]) { - thisStackTrace.shift(); - } - - this.stack = `${this.stack.slice(0, indexOfMessage)}${thisStackTrace.reverse().join('\n')}${errorStackTrace.reverse().join('\n')}`; - } - } -} - -/** -An error to be thrown when the server redirects you more than ten times. -Includes a `response` property. -*/ -export class MaxRedirectsError extends RequestError { - declare readonly response: Response; - declare readonly request: Request; - declare readonly timings: Timings; - - constructor(request: Request) { - super(`Redirected ${request.options.maxRedirects} times. Aborting.`, {}, request); - this.name = 'MaxRedirectsError'; - } -} - -/** -An error to be thrown when the server response code is not 2xx nor 3xx if `options.followRedirect` is `true`, but always except for 304. -Includes a `response` property. -*/ -export class HTTPError extends RequestError { - declare readonly response: Response; - declare readonly request: Request; - declare readonly timings: Timings; - - constructor(response: Response) { - super(`Response code ${response.statusCode} (${response.statusMessage!})`, {}, response.request); - this.name = 'HTTPError'; - } -} -/** -An error to be thrown when a cache method fails. -For example, if the database goes down or there's a filesystem error. -*/ -export class CacheError extends RequestError { - declare readonly request: Request; - - constructor(error: Error, request: Request) { - super(error.message, error, request); - this.name = 'CacheError'; - } -} - -/** -An error to be thrown when the request body is a stream and an error occurs while reading from that stream. -*/ -export class UploadError extends RequestError { - declare readonly request: Request; - - constructor(error: Error, request: Request) { - super(error.message, error, request); - this.name = 'UploadError'; - } -} - -/** -An error to be thrown when the request is aborted due to a timeout. -Includes an `event` and `timings` property. -*/ -export class TimeoutError extends RequestError { - declare readonly request: Request; - readonly timings: Timings; - readonly event: string; - - constructor(error: TimedOutTimeoutError, timings: Timings, request: Request) { - super(error.message, error, request); - this.name = 'TimeoutError'; - this.event = error.event; - this.timings = timings; - } -} - -/** -An error to be thrown when reading from response stream fails. -*/ -export class ReadError extends RequestError { - declare readonly request: Request; - declare readonly response: Response; - declare readonly timings: Timings; - - constructor(error: Error, request: Request) { - super(error.message, error, request); - this.name = 'ReadError'; - } -} - -/** -An error to be thrown when given an unsupported protocol. -*/ -export class UnsupportedProtocolError extends RequestError { - constructor(options: NormalizedOptions) { - super(`Unsupported protocol "${options.url.protocol}"`, {}, options); - this.name = 'UnsupportedProtocolError'; - } -} - const proxiedRequestEvents = [ 'socket', 'connect', 'continue', 'information', - 'upgrade', - 'timeout' + 'upgrade' ]; +const noop = () => {}; + export default class Request extends Duplex implements RequestEvents { ['constructor']: typeof Request; - declare [kUnproxyEvents]: () => void; - declare _cannotHaveBody: boolean; - [kDownloadedSize]: number; - [kUploadedSize]: number; - [kStopReading]: boolean; - [kTriggerRead]: boolean; - [kBody]: Options['body']; - [kJobs]: Array<() => void>; - [kRetryTimeout]?: NodeJS.Timeout; - [kBodySize]?: number; - [kServerResponsesPiped]: Set; - [kIsFromCache]?: boolean; - [kStartedReading]?: boolean; - [kCancelTimeouts]?: () => void; - [kResponseSize]?: number; - [kResponse]?: IncomingMessageWithTimings; - [kOriginalResponse]?: IncomingMessageWithTimings; - [kRequest]?: ClientRequest; _noPipe?: boolean; - declare options: NormalizedOptions; - declare requestUrl: string; - requestInitialized: boolean; - redirects: string[]; + // @ts-expect-error TypeScript doesn't check try/catch inside constructors. Dang. + options: Options; + response?: PlainResponse; + requestUrl?: URL; + redirectUrls: URL[]; retryCount: number; - constructor(url: string | URL | undefined, options: Options = {}, defaults?: Defaults) { + declare private _requestOptions: NativeRequestOptions; + + private _stopRetry: () => void; + private _downloadedSize: number; + private _uploadedSize: number; + private _stopReading: boolean; + private _startedReading: boolean; + private readonly _pipedServerResponses: Set; + private _request?: ClientRequest; + private _responseSize?: number; + private _bodySize?: number; + private _unproxyEvents: () => void; + private _isFromCache?: boolean; + private _cannotHaveBody: boolean; + private _triggerRead: boolean; + declare private _jobs: Array<() => void>; + private _cancelTimeouts: () => void; + private _nativeResponse?: IncomingMessageWithTimings; + private _flushed: boolean; + private _aborted: boolean; + + // We need this because `this._request` if `undefined` when using cache + private _requestInitialized: boolean; + + constructor(url: string | URL | OptionsInit | undefined, options?: OptionsInit, defaults?: Options) { super({ - // This must be false, to enable throwing after destroy - // It is used for retry logic in Promise API + // Don't destroy immediately, as the error may be emitted on unsuccessful retry autoDestroy: false, // It needs to be zero because we're just proxying the data to another stream highWaterMark: 0 @@ -1383,16 +180,25 @@ export default class Request extends Duplex implements RequestEvents { // TODO: Remove this when targeting Node.js 14 applyDestroyPatch(this); - this[kDownloadedSize] = 0; - this[kUploadedSize] = 0; - this.requestInitialized = false; - this[kServerResponsesPiped] = new Set(); - this.redirects = []; - this[kStopReading] = false; - this[kTriggerRead] = false; - this[kJobs] = []; + this._downloadedSize = 0; + this._uploadedSize = 0; + this._stopReading = false; + this._startedReading = false; + this._pipedServerResponses = new Set(); + this._cannotHaveBody = false; + this._unproxyEvents = noop; + this._triggerRead = false; + this._cancelTimeouts = noop; + this._jobs = []; + this._flushed = false; + this._requestInitialized = false; + this._aborted = false; + + this.redirectUrls = []; this.retryCount = 0; + this._stopRetry = noop; + const unlockWrite = (): void => { this._unlockWrite(); }; @@ -1418,470 +224,318 @@ export default class Request extends Duplex implements RequestEvents { }); this.on('pipe', source => { - if (source instanceof IncomingMessage) { - this.options.headers = { - ...source.headers, - ...this.options.headers - }; + if (source.headers) { + Object.assign(this.options.headers, source.headers); } }); - const {json, body, form} = options; - if (json || body || form) { - this._lockWrite(); - } - - if (kIsNormalizedAlready in options) { - this.options = options as NormalizedOptions; - } else { - try { - // @ts-expect-error Common TypeScript bug saying that `this.constructor` is not accessible - this.options = this.constructor.normalizeArguments(url, options, defaults); - } catch (error) { - // TODO: Move this to `_destroy()` - if (is.nodeStream(options.body)) { - options.body.destroy(); - } - - this.destroy(error); - return; - } - } - - (async () => { - try { - if (this.options.body instanceof ReadStream) { - await waitForOpenFile(this.options.body); - } - - const {url: normalizedURL} = this.options; - - if (!normalizedURL) { - throw new TypeError('Missing `url` property'); - } - - this.requestUrl = normalizedURL.toString(); - decodeURI(this.requestUrl); - - await this._finalizeBody(); - await this._makeRequest(); - - if (this.destroyed) { - this[kRequest]?.destroy(); - return; - } - - // Queued writes etc. - for (const job of this[kJobs]) { - job(); - } - - // Prevent memory leak - this[kJobs].length = 0; - - this.requestInitialized = true; - } catch (error) { - if (error instanceof RequestError) { - this._beforeError(error); - return; - } - - // This is a workaround for https://github.com/nodejs/node/issues/33335 - if (!this.destroyed) { - this.destroy(error); - } - } - })(); - } - - static normalizeArguments(url?: string | URL, options?: Options, defaults?: Defaults): NormalizedOptions { - const rawOptions = options; - - if (is.object(url) && !is.urlInstance(url)) { - options = {...defaults, ...(url as Options), ...options}; - } else { - if (url && options && options.url !== undefined) { - throw new TypeError('The `url` option is mutually exclusive with the `input` argument'); - } + try { + this.options = new Options(url, options, defaults); - options = {...defaults, ...options}; + if (!this.options.url) { + if (this.options.prefixUrl === '') { + throw new TypeError('Missing `url` property'); + } - if (url !== undefined) { - options.url = url; + this.options.url = ''; } - if (is.urlInstance(options.url)) { - options.url = new URL(options.url.toString()); + this.requestUrl = this.options.url as URL; + } catch (error) { + const {options} = error as OptionsError; + if (options) { + this.options = options; } - } - // TODO: Deprecate URL options in Got 12. + this.flush = async () => { + this.flush = async () => {}; - // Support extend-specific options - if (options.cache === false) { - options.cache = undefined; - } - - if (options.dnsCache === false) { - options.dnsCache = undefined; - } + this.destroy(error); + }; - // Nice type assertions - assert.any([is.string, is.undefined], options.method); - assert.any([is.object, is.undefined], options.headers); - assert.any([is.string, is.urlInstance, is.undefined], options.prefixUrl); - assert.any([is.object, is.undefined], options.cookieJar); - assert.any([is.object, is.string, is.undefined], options.searchParams); - assert.any([is.object, is.string, is.undefined], options.cache); - assert.any([is.object, is.number, is.undefined], options.timeout); - assert.any([is.object, is.undefined], options.context); - assert.any([is.object, is.undefined], options.hooks); - assert.any([is.boolean, is.undefined], options.decompress); - assert.any([is.boolean, is.undefined], options.ignoreInvalidCookies); - assert.any([is.boolean, is.undefined], options.followRedirect); - assert.any([is.number, is.undefined], options.maxRedirects); - assert.any([is.boolean, is.undefined], options.throwHttpErrors); - assert.any([is.boolean, is.undefined], options.http2); - assert.any([is.boolean, is.undefined], options.allowGetBody); - assert.any([is.string, is.undefined], options.localAddress); - assert.any([isDnsLookupIpVersion, is.undefined], options.dnsLookupIpVersion); - assert.any([is.object, is.undefined], options.https); - assert.any([is.boolean, is.undefined], options.rejectUnauthorized); - - if (options.https) { - assert.any([is.boolean, is.undefined], options.https.rejectUnauthorized); - assert.any([is.function_, is.undefined], options.https.checkServerIdentity); - assert.any([is.string, is.object, is.array, is.undefined], options.https.certificateAuthority); - assert.any([is.string, is.object, is.array, is.undefined], options.https.key); - assert.any([is.string, is.object, is.array, is.undefined], options.https.certificate); - assert.any([is.string, is.undefined], options.https.passphrase); - assert.any([is.string, is.buffer, is.array, is.undefined], options.https.pfx); + return; } - assert.any([is.object, is.undefined], options.cacheOptions); - - // `options.method` - if (is.string(options.method)) { - options.method = options.method.toUpperCase() as Method; - } else { - options.method = 'GET'; + const {json, body, form} = this.options; + if (json || body || form) { + this._lockWrite(); } - // `options.headers` - if (options.headers === defaults?.headers) { - options.headers = {...options.headers}; - } else { - options.headers = lowercaseKeys({...(defaults?.headers), ...options.headers}); - } + // Important! If you replace `body` in a handler with another stream, make sure it's readable first. + // The below is run only once. + if (is.nodeStream(body)) { + body.once('error', error => { + if (this._flushed) { + this._beforeError(new UploadError(error, this)); + } else { + this.flush = async () => { + this.flush = async () => {}; - // Disallow legacy `url.Url` - if ('slashes' in options) { - throw new TypeError('The legacy `url.Url` has been deprecated. Use `URL` instead.'); + this._beforeError(new UploadError(error, this)); + }; + } + }); } + } - // `options.auth` - if ('auth' in options) { - throw new TypeError('Parameter `auth` is deprecated. Use `username` / `password` instead.'); + async flush() { + if (this._flushed) { + return; } - // `options.searchParams` - if ('searchParams' in options && options.searchParams && options.searchParams !== defaults?.searchParams) { - let searchParameters: URLSearchParams; + this._flushed = true; - if (is.string(options.searchParams) || (options.searchParams instanceof URLSearchParams)) { - searchParameters = new URLSearchParams(options.searchParams); - } else { - validateSearchParameters(options.searchParams); + try { + await this._finalizeBody(); - searchParameters = new URLSearchParams(); + if (this.destroyed) { + return; + } - // eslint-disable-next-line guard-for-in - for (const key in options.searchParams) { - const value = options.searchParams[key]; + await this._makeRequest(); - if (value === null) { - searchParameters.append(key, ''); - } else if (value !== undefined) { - searchParameters.append(key, value as string); - } - } + if (this.destroyed) { + this._request?.destroy(); + return; } - // `normalizeArguments()` is also used to merge options - for (const [key, value] of defaults?.searchParams ?? []) { - // Only use default if one isn't already defined - if (!searchParameters.has(key)) { - searchParameters.append(key, value); - } + // Queued writes etc. + for (const job of this._jobs) { + job(); } - options.searchParams = searchParameters; + // Prevent memory leak + this._jobs.length = 0; + + this._requestInitialized = true; + } catch (error) { + this._beforeError(error); } + } - // `options.username` & `options.password` - options.username = options.username ?? ''; - options.password = options.password ?? ''; + _beforeError(error: Error): void { + if (this._stopReading) { + return; + } - // `options.prefixUrl` & `options.url` - if (is.undefined(options.prefixUrl)) { - options.prefixUrl = defaults?.prefixUrl ?? ''; - } else { - options.prefixUrl = options.prefixUrl.toString(); + const {response, options} = this; + const attemptCount = this.retryCount + (error.name === 'RetryError' ? 0 : 1); - if (options.prefixUrl !== '' && !options.prefixUrl.endsWith('/')) { - options.prefixUrl += '/'; - } + this._stopReading = true; + + if (!(error instanceof RequestError)) { + error = new RequestError(error.message, error, this); } - if (is.string(options.url)) { - if (options.url.startsWith('/')) { - throw new Error('`input` must not start with a slash when using `prefixUrl`'); - } + const typedError = error as RequestError; - options.url = optionsToUrl(options.prefixUrl + options.url, options as Options & {searchParams?: URLSearchParams}); - } else if ((is.undefined(options.url) && options.prefixUrl !== '') || options.protocol) { - options.url = optionsToUrl(options.prefixUrl, options as Options & {searchParams?: URLSearchParams}); - } + void (async () => { + if (response && !response.rawBody) { + // @types/node has incorrect typings. `setEncoding` accepts `null` as well. + response.setEncoding(this.readableEncoding!); - if (options.url) { - if ('port' in options) { - delete options.port; - } + const success = await this._setRawBody(response); - if ('protocol' in options) { - delete options.protocol; + if (success) { + response.body = response.rawBody!.toString(); + } } - // Make it possible to change `options.prefixUrl` - let {prefixUrl} = options; - Object.defineProperty(options, 'prefixUrl', { - set: (value: string) => { - const url = options!.url as URL; + if (this.listenerCount('retry') !== 0) { + let backoff: number; - if (!url.href.startsWith(value)) { - throw new Error(`Cannot change \`prefixUrl\` from ${prefixUrl} to ${value}: ${url.href}`); - } + try { + let retryAfter; + if (response && 'retry-after' in response.headers) { + retryAfter = Number(response.headers['retry-after']); + if (Number.isNaN(retryAfter)) { + retryAfter = Date.parse(response.headers['retry-after']!) - Date.now(); - options!.url = new URL(value + url.href.slice(prefixUrl.length)); - prefixUrl = value; - }, - get: () => prefixUrl - }); + if (retryAfter <= 0) { + retryAfter = 1; + } + } else { + retryAfter *= 1000; + } + } - // Support UNIX sockets - let {protocol} = options.url; + const retryOptions = options.retry as RetryOptions; - if (protocol === 'unix:') { - protocol = 'http:'; + backoff = await retryOptions.calculateDelay({ + attemptCount, + retryOptions, + error: typedError, + retryAfter, + computedValue: calculateRetryDelay({ + attemptCount, + retryOptions, + error: typedError, + retryAfter, + computedValue: retryOptions.maxRetryAfter ?? options.timeout.request ?? Number.POSITIVE_INFINITY + }) + }); + } catch (error_) { + void this._error(new RequestError(error_.message, error_, this)); + return; + } - options.url = new URL(`http://unix${options.url.pathname}${options.url.search}`); - } + if (backoff) { + await new Promise(resolve => { + const timeout = setTimeout(resolve, backoff); + this._stopRetry = () => { + clearTimeout(timeout); + resolve(); + }; + }); - // Set search params - if (options.searchParams) { - // eslint-disable-next-line @typescript-eslint/no-base-to-string - options.url.search = options.searchParams.toString(); - } + // Something forced us to abort the retry + if (this.destroyed) { + return; + } - // Protocol check - if (protocol !== 'http:' && protocol !== 'https:') { - throw new UnsupportedProtocolError(options as NormalizedOptions); - } + try { + for (const hook of this.options.hooks.beforeRetry) { + // eslint-disable-next-line no-await-in-loop + await hook(typedError); + } + } catch (error_) { + void this._error(new RequestError(error_.message, error, this)); + return; + } - // Update `username` - if (options.username === '') { - options.username = options.url.username; - } else { - options.url.username = options.username; - } + // Something forced us to abort the retry + if (this.destroyed) { + return; + } - // Update `password` - if (options.password === '') { - options.password = options.url.password; - } else { - options.url.password = options.password; + this.destroy(); + this.emit('retry', this.retryCount + 1, error); + return; + } } - } - - // `options.cookieJar` - const {cookieJar} = options; - if (cookieJar) { - let {setCookie, getCookieString} = cookieJar; - assert.function_(setCookie); - assert.function_(getCookieString); + void this._error(typedError); + })(); + } - /* istanbul ignore next: Horrible `tough-cookie` v3 check */ - if (setCookie.length === 4 && getCookieString.length === 0) { - setCookie = promisify(setCookie.bind(options.cookieJar)); - getCookieString = promisify(getCookieString.bind(options.cookieJar)); + _read(): void { + this._triggerRead = true; - options.cookieJar = { - setCookie, - getCookieString: getCookieString as PromiseCookieJar['getCookieString'] - }; + const {response} = this; + if (response && !this._stopReading) { + // We cannot put this in the `if` above + // because `.read()` also triggers the `end` event + if (response.readableLength) { + this._triggerRead = false; } - } - - // `options.cache` - const {cache} = options; - if (cache && !cacheableStore.has(cache)) { - cacheableStore.set(cache, new CacheableRequest( - ((requestOptions: RequestOptions, handler?: (response: IncomingMessageWithTimings) => void): ClientRequest => { - const result = (requestOptions as Pick)[kRequest](requestOptions, handler); - - // TODO: remove this when `cacheable-request` supports async request functions. - if (is.promise(result)) { - // @ts-expect-error - // We only need to implement the error handler in order to support HTTP2 caching. - // The result will be a promise anyway. - result.once = (event: string, handler: (reason: unknown) => void) => { - if (event === 'error') { - result.catch(handler); - } else if (event === 'abort') { - // The empty catch is needed here in case when - // it rejects before it's `await`ed in `_makeRequest`. - (async () => { - try { - const request = (await result) as ClientRequest; - request.once('abort', handler); - } catch {} - })(); - } else { - /* istanbul ignore next: safety check */ - throw new Error(`Unknown HTTP2 promise event: ${event}`); - } - return result; - }; - } + let data; + while ((data = response.read()) !== null) { + this._downloadedSize += data.length; + this._startedReading = true; - return result; - }) as HttpRequestFunction, - cache as CacheableRequest.StorageAdapter - )); - } + const progress = this.downloadProgress; - // `options.cacheOptions` - options.cacheOptions = {...options.cacheOptions}; + if (progress.percent < 1) { + this.emit('downloadProgress', progress); + } - // `options.dnsCache` - if (options.dnsCache === true) { - if (!globalDnsCache) { - globalDnsCache = new CacheableLookup(); + this.push(data); } - - options.dnsCache = globalDnsCache; - } else if (!is.undefined(options.dnsCache) && !options.dnsCache.lookup) { - throw new TypeError(`Parameter \`dnsCache\` must be a CacheableLookup instance or a boolean, got ${is(options.dnsCache)}`); } + } - // `options.timeout` - if (is.number(options.timeout)) { - options.timeout = {request: options.timeout}; - } else if (defaults && options.timeout !== defaults.timeout) { - options.timeout = { - ...defaults.timeout, - ...options.timeout - }; + // Node.js 12 has incorrect types, so the encoding must be a string + _write(chunk: any, encoding: string | undefined, callback: (error?: Error | null) => void): void { + const write = (): void => { + this._writeRequest(chunk, encoding as BufferEncoding, callback); + }; + + if (this._requestInitialized) { + write(); } else { - options.timeout = {...options.timeout}; + this._jobs.push(write); } + } - // `options.context` - options.context = {...defaults?.context, ...options.context}; - - // `options.hooks` - const areHooksDefault = options.hooks === defaults?.hooks; - options.hooks = {...options.hooks}; - - for (const event of knownHookEvents) { - if (event in options.hooks) { - if (is.array(options.hooks[event])) { - // See https://github.com/microsoft/TypeScript/issues/31445#issuecomment-576929044 - (options.hooks as any)[event] = [...options.hooks[event]!]; - } else { - throw new TypeError(`Parameter \`${event}\` must be an Array, got ${is(options.hooks[event])}`); - } - } else { - options.hooks[event] = []; + _final(callback: (error?: Error | null) => void): void { + const endRequest = (): void => { + // We need to check if `this._request` is present, + // because it isn't when we use cache. + if (!this._request || this._request.destroyed) { + callback(); + return; } - } - if (defaults && !areHooksDefault) { - for (const event of knownHookEvents) { - const defaultHooks = defaults.hooks[event]; + this._request.end((error?: Error | null) => { + if (!error) { + this._bodySize = this._uploadedSize; - if (defaultHooks.length > 0) { - // See https://github.com/microsoft/TypeScript/issues/31445#issuecomment-576929044 - (options.hooks as any)[event] = [ - ...defaults.hooks[event], - ...options.hooks[event]! - ]; + this.emit('uploadProgress', this.uploadProgress); + this._request!.emit('upload-complete'); } - } - } - // DNS options - if ('family' in options) { - deprecationWarning('"options.family" was never documented, please use "options.dnsLookupIpVersion"'); - } + callback(error); + }); + }; - // HTTPS options - if (defaults?.https) { - options.https = {...defaults.https, ...options.https}; + if (this._requestInitialized) { + endRequest(); + } else { + this._jobs.push(endRequest); } + } - if ('rejectUnauthorized' in options) { - deprecationWarning('"options.rejectUnauthorized" is now deprecated, please use "options.https.rejectUnauthorized"'); - } + _destroy(error: Error | null, callback: (error: Error | null) => void): void { + this._stopReading = true; + this.flush = async () => {}; - if ('checkServerIdentity' in options) { - deprecationWarning('"options.checkServerIdentity" was never documented, please use "options.https.checkServerIdentity"'); - } + // Prevent further retries + this._stopRetry(); + this._cancelTimeouts(); - if ('ca' in options) { - deprecationWarning('"options.ca" was never documented, please use "options.https.certificateAuthority"'); + if (this.options) { + const {body} = this.options; + if (is.nodeStream(body)) { + body.destroy(); + } } - if ('key' in options) { - deprecationWarning('"options.key" was never documented, please use "options.https.key"'); + // TODO: Remove the next `if` when targeting Node.js 14. + if (this._request && !this.response?.complete) { + this._request.destroy(); } - if ('cert' in options) { - deprecationWarning('"options.cert" was never documented, please use "options.https.certificate"'); + if (error !== null && !is.undefined(error) && !(error instanceof RequestError)) { + error = new RequestError(error.message, error, this); } - if ('passphrase' in options) { - deprecationWarning('"options.passphrase" was never documented, please use "options.https.passphrase"'); - } + callback(error); + } - if ('pfx' in options) { - deprecationWarning('"options.pfx" was never documented, please use "options.https.pfx"'); + pipe(destination: T, options?: {end?: boolean}): T { + if (this._startedReading) { + throw new Error('Failed to pipe. The response has been emitted already.'); } - // Other options - if ('followRedirects' in options) { - throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.'); + if (destination instanceof ServerResponse) { + this._pipedServerResponses.add(destination); } - if (options.agent) { - for (const key in options.agent) { - if (key !== 'http' && key !== 'https' && key !== 'http2') { - throw new TypeError(`Expected the \`options.agent\` properties to be \`http\`, \`https\` or \`http2\`, got \`${key}\``); - } - } - } + return super.pipe(destination, options); + } - options.maxRedirects = options.maxRedirects ?? 0; + unpipe(destination: T): this { + if (destination instanceof ServerResponse) { + this._pipedServerResponses.delete(destination); + } - // Set non-enumerable properties - setNonEnumerableProperties([defaults, rawOptions], options); + super.unpipe(destination); - return normalizePromiseArguments(options as NormalizedOptions, defaults); + return this; } - _lockWrite(): void { + private _lockWrite(): void { const onLockedWrite = (): never => { throw new TypeError('The payload has been already provided'); }; @@ -1890,85 +544,68 @@ export default class Request extends Duplex implements RequestEvents { this.end = onLockedWrite; } - _unlockWrite(): void { + private _unlockWrite(): void { this.write = super.write; this.end = super.end; } - async _finalizeBody(): Promise { + private async _finalizeBody(): Promise { const {options} = this; const {headers} = options; const isForm = !is.undefined(options.form); const isJSON = !is.undefined(options.json); const isBody = !is.undefined(options.body); - const hasPayload = isForm || isJSON || isBody; - const cannotHaveBody = withoutBody.has(options.method) && !(options.method === 'GET' && options.allowGetBody); + const cannotHaveBody = methodsWithoutBody.has(options.method) && !(options.method === 'GET' && options.allowGetBody); this._cannotHaveBody = cannotHaveBody; - if (hasPayload) { + if (isForm || isJSON || isBody) { if (cannotHaveBody) { throw new TypeError(`The \`${options.method}\` method cannot be used with a body`); } - if ([isBody, isForm, isJSON].filter(isTrue => isTrue).length > 1) { - throw new TypeError('The `body`, `json` and `form` options are mutually exclusive'); - } - - if ( - isBody && - !(options.body instanceof Readable) && - !is.string(options.body) && - !is.buffer(options.body) && - !isFormData(options.body) - ) { - throw new TypeError('The `body` option must be a stream.Readable, string or Buffer'); - } + // Serialize body + const noContentType = !is.string(headers['content-type']); - if (isForm && !is.object(options.form)) { - throw new TypeError('The `form` option must be an Object'); - } + if (isBody) { + // Special case for https://github.com/form-data/form-data + if (isFormData(options.body) && noContentType) { + headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`; + } + } else if (isForm) { + if (noContentType) { + headers['content-type'] = 'application/x-www-form-urlencoded'; + } - { - // Serialize body - const noContentType = !is.string(headers['content-type']); + const {form} = options; + options.form = undefined; - if (isBody) { - // Special case for https://github.com/form-data/form-data - if (isFormData(options.body) && noContentType) { - headers['content-type'] = `multipart/form-data; boundary=${options.body.getBoundary()}`; - } + options.body = (new URLSearchParams(form as Record)).toString(); + } else { + if (noContentType) { + headers['content-type'] = 'application/json'; + } - this[kBody] = options.body; - } else if (isForm) { - if (noContentType) { - headers['content-type'] = 'application/x-www-form-urlencoded'; - } + const {json} = options; + options.json = undefined; - this[kBody] = (new URLSearchParams(options.form as Record)).toString(); - } else { - if (noContentType) { - headers['content-type'] = 'application/json'; - } + options.body = options.stringifyJson(json); + } - this[kBody] = options.stringifyJson(options.json); - } + const uploadBodySize = await getBodySize(options.body, options.headers); - const uploadBodySize = await getBodySize(this[kBody], options.headers); - - // See https://tools.ietf.org/html/rfc7230#section-3.3.2 - // A user agent SHOULD send a Content-Length in a request message when - // no Transfer-Encoding is sent and the request method defines a meaning - // for an enclosed payload body. For example, a Content-Length header - // field is normally sent in a POST request even when the value is 0 - // (indicating an empty payload body). A user agent SHOULD NOT send a - // Content-Length header field when the request message does not contain - // a payload body and the method semantics do not anticipate such a - // body. - if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding']) && !cannotHaveBody && !is.undefined(uploadBodySize)) { - headers['content-length'] = String(uploadBodySize); - } + // See https://tools.ietf.org/html/rfc7230#section-3.3.2 + // A user agent SHOULD send a Content-Length in a request message when + // no Transfer-Encoding is sent and the request method defines a meaning + // for an enclosed payload body. For example, a Content-Length header + // field is normally sent in a POST request even when the value is 0 + // (indicating an empty payload body). A user agent SHOULD NOT send a + // Content-Length header field when the request message does not contain + // a payload body and the method semantics do not anticipate such a + // body. + if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding']) && !cannotHaveBody && !is.undefined(uploadBodySize)) { + headers['content-length'] = String(uploadBodySize); } } else if (cannotHaveBody) { this._lockWrite(); @@ -1976,42 +613,53 @@ export default class Request extends Duplex implements RequestEvents { this._unlockWrite(); } - this[kBodySize] = Number(headers['content-length']) || undefined; + if (options.responseType === 'json' && !('accept' in options.headers)) { + options.headers.accept = 'application/json'; + } + + this._bodySize = Number(headers['content-length']) || undefined; } - async _onResponseBase(response: IncomingMessageWithTimings): Promise { + private async _onResponseBase(response: IncomingMessageWithTimings): Promise { + // This will be called e.g. when using cache so we need to check if this request has been aborted. + if (this.isAborted) { + return; + } + const {options} = this; const {url} = options; - this[kOriginalResponse] = response; + this._nativeResponse = response; if (options.decompress) { response = decompressResponse(response); } const statusCode = response.statusCode!; - const typedResponse = response as Response; + const typedResponse = response as PlainResponse; typedResponse.statusMessage = typedResponse.statusMessage ? typedResponse.statusMessage : http.STATUS_CODES[statusCode]; - typedResponse.url = options.url.toString(); - typedResponse.requestUrl = this.requestUrl; - typedResponse.redirectUrls = this.redirects; + typedResponse.url = options.url!.toString(); + typedResponse.requestUrl = this.requestUrl!; + typedResponse.redirectUrls = this.redirectUrls; typedResponse.request = this; - typedResponse.isFromCache = (response as any).fromCache ?? false; + typedResponse.isFromCache = (this._nativeResponse as any).fromCache ?? false; typedResponse.ip = this.ip; typedResponse.retryCount = this.retryCount; - this[kIsFromCache] = typedResponse.isFromCache; + this._isFromCache = typedResponse.isFromCache; - this[kResponseSize] = Number(response.headers['content-length']) || undefined; - this[kResponse] = response; + this._responseSize = Number(response.headers['content-length']) || undefined; + this.response = typedResponse; response.once('end', () => { - this[kResponseSize] = this[kDownloadedSize]; + this._responseSize = this._downloadedSize; this.emit('downloadProgress', this.downloadProgress); }); response.once('error', (error: Error) => { + this._aborted = true; + // Force clean-up, because some packages don't do this. // TODO: Fix decompress-response response.destroy(); @@ -2020,6 +668,8 @@ export default class Request extends Duplex implements RequestEvents { }); response.once('aborted', () => { + this._aborted = true; + this._beforeError(new ReadError({ name: 'Error', message: 'The server aborted pending request', @@ -2031,7 +681,9 @@ export default class Request extends Duplex implements RequestEvents { const rawCookies = response.headers['set-cookie']; if (is.object(options.cookieJar) && rawCookies) { - let promises: Array> = rawCookies.map(async (rawCookie: string) => options.cookieJar!.setCookie(rawCookie, url.toString())); + let promises: Array> = rawCookies.map(async (rawCookie: string) => { + return (options.cookieJar as PromiseCookieJar).setCookie(rawCookie, url!.toString()); + }); if (options.ignoreInvalidCookies) { promises = promises.map(async p => p.catch(() => {})); @@ -2045,88 +697,82 @@ export default class Request extends Duplex implements RequestEvents { } } + // The above is running a promise, therefore we need to check if this request has been aborted yet again. + if (this.isAborted) { + return; + } + if (options.followRedirect && response.headers.location && redirectCodes.has(statusCode)) { // We're being redirected, we don't care about the response. // It'd be best to abort the request, but we can't because // we would have to sacrifice the TCP connection. We don't want that. response.resume(); - if (this[kRequest]) { - this[kCancelTimeouts]!(); + this._cancelTimeouts(); + this._unproxyEvents(); - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete this[kRequest]; - this[kUnproxyEvents](); + if (this.redirectUrls.length >= options.maxRedirects) { + this._beforeError(new MaxRedirectsError(this)); + return; } - const shouldBeGet = statusCode === 303 && options.method !== 'GET' && options.method !== 'HEAD'; - if (shouldBeGet || options.methodRewriting) { - // Server responded with "see other", indicating that the resource exists at another location, - // and the client should request it from that location via GET or HEAD. - options.method = 'GET'; - - if ('body' in options) { - delete options.body; - } + this._request = undefined; - if ('json' in options) { - delete options.json; - } + const updatedOptions = new Options(undefined, undefined, this.options); - if ('form' in options) { - delete options.form; - } + const shouldBeGet = statusCode === 303 && updatedOptions.method !== 'GET' && updatedOptions.method !== 'HEAD'; + if (shouldBeGet || updatedOptions.methodRewriting) { + // Server responded with "see other", indicating that the resource exists at another location, + // and the client should request it from that location via GET or HEAD. + updatedOptions.method = 'GET'; - this[kBody] = undefined; - delete options.headers['content-length']; - } + updatedOptions.body = undefined; + updatedOptions.json = undefined; + updatedOptions.form = undefined; - if (this.redirects.length >= options.maxRedirects) { - this._beforeError(new MaxRedirectsError(this)); - return; + delete updatedOptions.headers['content-length']; } try { - // Do not remove. See https://github.com/sindresorhus/got/pull/214 + // We need this in order to support UTF-8 const redirectBuffer = Buffer.from(response.headers.location, 'binary').toString(); - - // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604 const redirectUrl = new URL(redirectBuffer, url); - const redirectString = redirectUrl.toString(); - decodeURI(redirectString); // Redirecting to a different site, clear sensitive data. - if (redirectUrl.hostname !== url.hostname || redirectUrl.port !== url.port) { - if ('host' in options.headers) { - delete options.headers.host; + if (redirectUrl.hostname !== (url as URL).hostname || redirectUrl.port !== (url as URL).port) { + if ('host' in updatedOptions.headers) { + delete updatedOptions.headers.host; } - if ('cookie' in options.headers) { - delete options.headers.cookie; + if ('cookie' in updatedOptions.headers) { + delete updatedOptions.headers.cookie; } - if ('authorization' in options.headers) { - delete options.headers.authorization; + if ('authorization' in updatedOptions.headers) { + delete updatedOptions.headers.authorization; } - if (options.username || options.password) { - options.username = ''; - options.password = ''; + if (updatedOptions.username || updatedOptions.password) { + updatedOptions.username = ''; + updatedOptions.password = ''; } } else { - redirectUrl.username = options.username; - redirectUrl.password = options.password; + redirectUrl.username = updatedOptions.username; + redirectUrl.password = updatedOptions.password; } - this.redirects.push(redirectString); - options.url = redirectUrl; + this.redirectUrls.push(redirectUrl); + updatedOptions.prefixUrl = ''; + updatedOptions.url = redirectUrl; - for (const hook of options.hooks.beforeRedirect) { + for (const hook of updatedOptions.hooks.beforeRedirect) { // eslint-disable-next-line no-await-in-loop - await hook(options, typedResponse); + await hook(updatedOptions, typedResponse); } - this.emit('redirect', typedResponse, options); + this.emit('redirect', updatedOptions, typedResponse); + + this.options = updatedOptions; await this._makeRequest(); } catch (error) { @@ -2143,7 +789,7 @@ export default class Request extends Duplex implements RequestEvents { } response.on('readable', () => { - if (this[kTriggerRead]) { + if (this._triggerRead) { this._read(); } }); @@ -2160,9 +806,19 @@ export default class Request extends Duplex implements RequestEvents { this.push(null); }); + if (this._noPipe) { + const success = await this._setRawBody(); + + if (success) { + this.emit('response', response); + } + + return; + } + this.emit('response', response); - for (const destination of this[kServerResponsesPiped]) { + for (const destination of this._pipedServerResponses) { if (destination.headersSent) { continue; } @@ -2181,7 +837,28 @@ export default class Request extends Duplex implements RequestEvents { } } - async _onResponse(response: IncomingMessageWithTimings): Promise { + private async _setRawBody(from: Readable = this): Promise { + if (from.readableEnded) { + return false; + } + + try { + // Errors are emitted via the `error` event + const rawBody = await getBuffer(from); + + // On retry Request is destroyed with no error, therefore the above will successfully resolve. + // So in order to check if this was really successfull, we need to check if it has been properly ended. + if (!this.isAborted) { + this.response!.rawBody = rawBody; + + return true; + } + } catch {} + + return false; + } + + private async _onResponse(response: IncomingMessageWithTimings): Promise { try { await this._onResponseBase(response); } catch (error) { @@ -2190,13 +867,13 @@ export default class Request extends Duplex implements RequestEvents { } } - _onRequest(request: ClientRequest): void { + private _onRequest(request: ClientRequest): void { const {options} = this; const {timeout, url} = options; timer(request); - this[kCancelTimeouts] = timedOut(request, timeout, url); + this._cancelTimeouts = timedOut(request, timeout, url as URL); const responseEventName = options.cache ? 'cacheableResponse' : 'response'; @@ -2205,6 +882,8 @@ export default class Request extends Duplex implements RequestEvents { }); request.once('error', (error: Error) => { + this._aborted = true; + // Force clean-up, because some packages (e.g. nock) don't do this. request.destroy(); @@ -2213,24 +892,27 @@ export default class Request extends Duplex implements RequestEvents { error = error instanceof TimedOutTimeoutError ? new TimeoutError(error, this.timings!, this) : new RequestError(error.message, error, this); - this._beforeError(error as RequestError); + this._beforeError(error); }); - this[kUnproxyEvents] = proxyEvents(request, this, proxiedRequestEvents); + this._unproxyEvents = proxyEvents(request, this, proxiedRequestEvents); - this[kRequest] = request; + this._request = request; this.emit('uploadProgress', this.uploadProgress); + this._sendBody(); + + this.emit('request', request); + } + + private _sendBody() { // Send body - const body = this[kBody]; - const currentRequest = this.redirects.length === 0 ? this : request; + const {body} = this.options; + const currentRequest = this.redirectUrls.length === 0 ? this : this._request ?? this; if (is.nodeStream(body)) { body.pipe(currentRequest); - body.once('error', (error: NodeJS.ErrnoException) => { - this._beforeError(new UploadError(error, this)); - }); } else { this._unlockWrite(); @@ -2245,20 +927,53 @@ export default class Request extends Duplex implements RequestEvents { this._lockWrite(); } } + } - this.emit('request', request); + private _prepareCache(cache: string | CacheableRequest.StorageAdapter) { + if (!cacheableStore.has(cache)) { + cacheableStore.set(cache, new CacheableRequest( + ((requestOptions: RequestOptions, handler?: (response: IncomingMessageWithTimings) => void): ClientRequest => { + const result = (requestOptions as any)._request(requestOptions, handler); + + // TODO: remove this when `cacheable-request` supports async request functions. + if (is.promise(result)) { + // @ts-expect-error + // We only need to implement the error handler in order to support HTTP2 caching. + // The result will be a promise anyway. + // eslint-disable-next-line @typescript-eslint/promise-function-async + result.once = (event: string, handler: (reason: unknown) => void) => { + if (event === 'error') { + result.catch(handler); + } else if (event === 'abort') { + // The empty catch is needed here in case when + // it rejects before it's `await`ed in `_makeRequest`. + (async () => { + try { + const request = (await result) as ClientRequest; + request.once('abort', handler); + } catch {} + })(); + } else { + /* istanbul ignore next: safety check */ + throw new Error(`Unknown HTTP2 promise event: ${event}`); + } + + return result; + }; + } + + return result; + }) as typeof http.request, + cache as CacheableRequest.StorageAdapter + )); + } } - async _createCacheableRequest(url: URL, options: RequestOptions): Promise { + private async _createCacheableRequest(url: URL, options: RequestOptions): Promise { return new Promise((resolve, reject) => { // TODO: Remove `utils/url-to-options.ts` when `cacheable-request` is fixed Object.assign(options, urlToOptions(url)); - // `http-cache-semantics` checks this - // TODO: Fix this ignore. - // @ts-expect-error - delete (options as unknown as NormalizedOptions).url; - let request: ClientRequest | Promise; // This is ugly @@ -2273,9 +988,6 @@ export default class Request extends Duplex implements RequestEvents { resolve(response as unknown as ResponseLike); }); - // Restore options - (options as unknown as NormalizedOptions).url = url; - cacheRequest.once('error', reject); cacheRequest.once('request', async (requestOrPromise: ClientRequest | Promise) => { request = requestOrPromise; @@ -2284,10 +996,10 @@ export default class Request extends Duplex implements RequestEvents { }); } - async _makeRequest(): Promise { + private async _makeRequest(): Promise { const {options} = this; - const {headers} = options; + const cookieJar = options.cookieJar as PromiseCookieJar | undefined; for (const key in headers) { if (is.undefined(headers[key])) { @@ -2303,184 +1015,59 @@ export default class Request extends Duplex implements RequestEvents { } // Set cookies - if (options.cookieJar) { - const cookieString: string = await options.cookieJar.getCookieString(options.url.toString()); + if (cookieJar) { + const cookieString: string = await cookieJar.getCookieString(options.url!.toString()); if (is.nonEmptyString(cookieString)) { - options.headers.cookie = cookieString; + headers.cookie = cookieString; } } + // Reset `prefixUrl` + options.prefixUrl = ''; + + let request = options.getRequestFunction(); + for (const hook of options.hooks.beforeRequest) { // eslint-disable-next-line no-await-in-loop const result = await hook(options); if (!is.undefined(result)) { // @ts-expect-error Skip the type mismatch to support abstract responses - options.request = () => result; + request = () => result; break; } } - if (options.body && this[kBody] !== options.body) { - this[kBody] = options.body; - } - - const {agent, request, timeout, url} = options; - - if (options.dnsCache && !('lookup' in options)) { - options.lookup = options.dnsCache.lookup; - } - - // UNIX sockets - if (url.hostname === 'unix') { - const matches = /(?.+?):(?.+)/.exec(`${url.pathname}${url.search}`); - - if (matches?.groups) { - const {socketPath, path} = matches.groups; - - Object.assign(options, { - socketPath, - path, - host: '' - }); - } - } - - const isHttps = url.protocol === 'https:'; + const url = options.url as URL; - // Fallback function - let fallbackFn: HttpRequestFunction; - if (options.http2) { - if (major < 15 || (major === 15 && minor < 10)) { - throw new Error('To use the `http2` option, install Node.js 15.10.0 or above'); - } + this._requestOptions = options.createNativeRequestOptions() as NativeRequestOptions; - // @ts-expect-error TS bug? - fallbackFn = http2wrapper.auto; - } else { - fallbackFn = isHttps ? https.request : http.request; + if (options.cache) { + (this._requestOptions as any)._request = request; + (this._requestOptions as any).cache = options.cache; + this._prepareCache(options.cache as CacheableRequest.StorageAdapter); } - const realFn = options.request ?? fallbackFn; - // Cache support - const fn = options.cache ? this._createCacheableRequest : realFn; - - // Pass an agent directly when HTTP2 is disabled - if (agent && !options.http2) { - (options as unknown as RequestOptions).agent = agent[isHttps ? 'https' : 'http']; - } - - // Prepare plain HTTP request options - options[kRequest] = realFn as HttpRequestFunction; - delete options.request; - // TODO: Fix this ignore. - // @ts-expect-error - delete options.timeout; - - const requestOptions = options as unknown as (RealRequestOptions & CacheOptions); - requestOptions.shared = options.cacheOptions?.shared; - requestOptions.cacheHeuristic = options.cacheOptions?.cacheHeuristic; - requestOptions.immutableMinTimeToLive = options.cacheOptions?.immutableMinTimeToLive; - requestOptions.ignoreCargoCult = options.cacheOptions?.ignoreCargoCult; - - // If `dnsLookupIpVersion` is not present do not override `family` - if (options.dnsLookupIpVersion !== undefined) { - try { - requestOptions.family = dnsLookupIpVersionToFamily(options.dnsLookupIpVersion); - } catch { - throw new Error('Invalid `dnsLookupIpVersion` option value'); - } - } - - // HTTPS options remapping - if (options.https) { - if ('rejectUnauthorized' in options.https) { - requestOptions.rejectUnauthorized = options.https.rejectUnauthorized; - } - - if (options.https.checkServerIdentity) { - requestOptions.checkServerIdentity = options.https.checkServerIdentity; - } - - if (options.https.certificateAuthority) { - requestOptions.ca = options.https.certificateAuthority; - } - - if (options.https.certificate) { - requestOptions.cert = options.https.certificate; - } - - if (options.https.key) { - requestOptions.key = options.https.key; - } - - if (options.https.passphrase) { - requestOptions.passphrase = options.https.passphrase; - } - - if (options.https.pfx) { - requestOptions.pfx = options.https.pfx; - } - } + const fn = options.cache ? this._createCacheableRequest : request; try { - let requestOrResponse = await fn(url, requestOptions); + let requestOrResponse = await fn(url, this._requestOptions); + // Fallback if (is.undefined(requestOrResponse)) { - requestOrResponse = fallbackFn(url, requestOptions); - } - - // Restore options - options.request = request; - options.timeout = timeout; - options.agent = agent; - - // HTTPS options restore - if (options.https) { - if ('rejectUnauthorized' in options.https) { - delete requestOptions.rejectUnauthorized; - } - - if (options.https.checkServerIdentity) { - // @ts-expect-error - This one will be removed when we remove the alias. - delete requestOptions.checkServerIdentity; - } - - if (options.https.certificateAuthority) { - delete requestOptions.ca; - } - - if (options.https.certificate) { - delete requestOptions.cert; - } - - if (options.https.key) { - delete requestOptions.key; - } - - if (options.https.passphrase) { - delete requestOptions.passphrase; - } - - if (options.https.pfx) { - delete requestOptions.pfx; - } + requestOrResponse = await options.getFallbackRequestFunction()(url, this._requestOptions); } - if (isClientRequest(requestOrResponse)) { + if (isClientRequest(requestOrResponse!)) { this._onRequest(requestOrResponse); - - // Emit the response after the stream has been ended } else if (this.writable) { this.once('finish', () => { void this._onResponse(requestOrResponse as IncomingMessageWithTimings); }); - this._unlockWrite(); - this.end(); - this._lockWrite(); + this._sendBody(); } else { void this._onResponse(requestOrResponse as IncomingMessageWithTimings); } @@ -2489,11 +1076,11 @@ export default class Request extends Duplex implements RequestEvents { throw new CacheError(error, this); } - throw new RequestError(error.message, error, this); + throw error; } } - async _error(error: RequestError): Promise { + private async _error(error: RequestError): Promise { try { for (const hook of this.options.hooks.beforeError) { // eslint-disable-next-line no-await-in-loop @@ -2506,150 +1093,15 @@ export default class Request extends Duplex implements RequestEvents { this.destroy(error); } - _beforeError(error: Error): void { - if (this[kStopReading]) { - return; - } - - const {options} = this; - const retryCount = this.retryCount + 1; - - this[kStopReading] = true; - - if (!(error instanceof RequestError)) { - error = new RequestError(error.message, error, this); - } - - const typedError = error as RequestError; - const {response} = typedError; - - void (async () => { - if (response && !response.body) { - response.setEncoding((this as any)._readableState.encoding); - - try { - response.rawBody = await getBuffer(response); - response.body = response.rawBody.toString(); - } catch {} - } - - if (this.listenerCount('retry') !== 0) { - let backoff: number; - - try { - let retryAfter; - if (response && 'retry-after' in response.headers) { - retryAfter = Number(response.headers['retry-after']); - if (Number.isNaN(retryAfter)) { - retryAfter = Date.parse(response.headers['retry-after']!) - Date.now(); - - if (retryAfter <= 0) { - retryAfter = 1; - } - } else { - retryAfter *= 1000; - } - } - - backoff = await options.retry.calculateDelay({ - attemptCount: retryCount, - retryOptions: options.retry, - error: typedError, - retryAfter, - computedValue: calculateRetryDelay({ - attemptCount: retryCount, - retryOptions: options.retry, - error: typedError, - retryAfter, - computedValue: 0 - }) - }); - } catch (error_) { - void this._error(new RequestError(error_.message, error_, this)); - return; - } - - if (backoff) { - const retry = async (): Promise => { - try { - for (const hook of this.options.hooks.beforeRetry) { - // eslint-disable-next-line no-await-in-loop - await hook(this.options, typedError, retryCount); - } - } catch (error_) { - void this._error(new RequestError(error_.message, error, this)); - return; - } - - // Something forced us to abort the retry - if (this.destroyed) { - return; - } - - this.destroy(); - this.emit('retry', retryCount, error); - }; - - this[kRetryTimeout] = setTimeout(retry, backoff); - return; - } - } - - void this._error(typedError); - })(); - } - - _read(): void { - this[kTriggerRead] = true; - - const response = this[kResponse]; - if (response && !this[kStopReading]) { - // We cannot put this in the `if` above - // because `.read()` also triggers the `end` event - if (response.readableLength) { - this[kTriggerRead] = false; - } - - let data; - while ((data = response.read()) !== null) { - this[kDownloadedSize] += data.length; - this[kStartedReading] = true; - - const progress = this.downloadProgress; - - if (progress.percent < 1) { - this.emit('downloadProgress', progress); - } - - this.push(data); - } - } - } - - // Node.js 12 has incorrect types, so the encoding must be a string - _write(chunk: any, encoding: string | undefined, callback: (error?: Error | null) => void): void { - const write = (): void => { - this._writeRequest(chunk, encoding as BufferEncoding, callback); - }; - - if (this.requestInitialized) { - write(); - } else { - this[kJobs].push(write); - } - } - - _writeRequest(chunk: any, encoding: BufferEncoding | undefined, callback: (error?: Error | null) => void): void { - if (this[kRequest]!.destroyed) { + private _writeRequest(chunk: any, encoding: BufferEncoding | undefined, callback: (error?: Error | null) => void): void { + if (!this._request || this._request.destroyed) { // Probably the `ClientRequest` instance will throw return; } - // TODO: What happens if it's from cache? Then this[kRequest] won't be defined. - - this[kRequest]!.write(chunk, encoding!, (error?: Error | null) => { + this._request.write(chunk, encoding!, (error?: Error | null) => { if (!error) { - this[kUploadedSize] += Buffer.byteLength(chunk, encoding); + this._uploadedSize += Buffer.byteLength(chunk, encoding); const progress = this.uploadProgress; @@ -2662,65 +1114,6 @@ export default class Request extends Duplex implements RequestEvents { }); } - _final(callback: (error?: Error | null) => void): void { - const endRequest = (): void => { - // We need to check if `this[kRequest]` is present, - // because it isn't when we use cache. - if (!(kRequest in this)) { - callback(); - return; - } - - if (this[kRequest]!.destroyed) { - callback(); - return; - } - - this[kRequest]!.end((error?: Error | null) => { - if (!error) { - this[kBodySize] = this[kUploadedSize]; - - this.emit('uploadProgress', this.uploadProgress); - this[kRequest]!.emit('upload-complete'); - } - - callback(error); - }); - }; - - if (this.requestInitialized) { - endRequest(); - } else { - this[kJobs].push(endRequest); - } - } - - _destroy(error: Error | null, callback: (error: Error | null) => void): void { - this[kStopReading] = true; - - // Prevent further retries - clearTimeout(this[kRetryTimeout]!); - - if (kRequest in this) { - this[kCancelTimeouts]!(); - - // TODO: Remove the next `if` when targeting Node.js 14. - if (!this[kResponse]?.complete) { - this[kRequest]!.destroy(); - } - } - - if (error !== null && !is.undefined(error) && !(error instanceof RequestError)) { - error = new RequestError(error.message, error, this); - } - - callback(error); - } - - get _isAboutToError() { - return this[kStopReading]; - } - /** The remote IP address. */ @@ -2731,12 +1124,12 @@ export default class Request extends Duplex implements RequestEvents { /** Indicates whether the request has been aborted or not. */ - get aborted(): boolean { - return (this[kRequest]?.destroyed ?? this.destroyed) && !(this[kOriginalResponse]?.complete); + get isAborted(): boolean { + return this._aborted; } get socket(): Socket | undefined { - return this[kRequest]?.socket ?? undefined; + return this._request?.socket ?? undefined; } /** @@ -2744,9 +1137,9 @@ export default class Request extends Duplex implements RequestEvents { */ get downloadProgress(): Progress { let percent; - if (this[kResponseSize]) { - percent = this[kDownloadedSize] / this[kResponseSize]!; - } else if (this[kResponseSize] === this[kDownloadedSize]) { + if (this._responseSize) { + percent = this._downloadedSize / this._responseSize; + } else if (this._responseSize === this._downloadedSize) { percent = 1; } else { percent = 0; @@ -2754,8 +1147,8 @@ export default class Request extends Duplex implements RequestEvents { return { percent, - transferred: this[kDownloadedSize], - total: this[kResponseSize] + transferred: this._downloadedSize, + total: this._responseSize }; } @@ -2764,9 +1157,9 @@ export default class Request extends Duplex implements RequestEvents { */ get uploadProgress(): Progress { let percent; - if (this[kBodySize]) { - percent = this[kUploadedSize] / this[kBodySize]!; - } else if (this[kBodySize] === this[kUploadedSize]) { + if (this._bodySize) { + percent = this._uploadedSize / this._bodySize; + } else if (this._bodySize === this._uploadedSize) { percent = 1; } else { percent = 0; @@ -2774,8 +1167,8 @@ export default class Request extends Duplex implements RequestEvents { return { percent, - transferred: this[kUploadedSize], - total: this[kBodySize] + transferred: this._uploadedSize, + total: this._bodySize }; } @@ -2807,35 +1200,18 @@ export default class Request extends Duplex implements RequestEvents { __Note__: The time is a `number` representing the milliseconds elapsed since the UNIX epoch. */ get timings(): Timings | undefined { - return (this[kRequest] as ClientRequestWithTimings)?.timings; + return (this._request as ClientRequestWithTimings)?.timings; } /** Whether the response was retrieved from the cache. */ get isFromCache(): boolean | undefined { - return this[kIsFromCache]; - } - - pipe(destination: T, options?: {end?: boolean}): T { - if (this[kStartedReading]) { - throw new Error('Failed to pipe. The response has been emitted already.'); - } - - if (destination instanceof ServerResponse) { - this[kServerResponsesPiped].add(destination); - } - - return super.pipe(destination, options); + return this._isFromCache; } - unpipe(destination: T): this { - if (destination instanceof ServerResponse) { - this[kServerResponsesPiped].delete(destination); - } - - super.unpipe(destination); - - return this; + get reusedSocket(): boolean | undefined { + // @ts-expect-error `@types/node` has incomplete types + return this._request.reusedSocket; } } diff --git a/source/core/options.ts b/source/core/options.ts new file mode 100644 index 000000000..ee7f23ab5 --- /dev/null +++ b/source/core/options.ts @@ -0,0 +1,2197 @@ +import {promisify, inspect} from 'util'; +import {URL, URLSearchParams} from 'url'; +import {checkServerIdentity} from 'tls'; +import {request as httpRequest} from 'http'; +import {request as httpsRequest} from 'https'; +import http2wrapper = require('http2-wrapper'); +import CacheableLookup from 'cacheable-lookup'; +import lowercaseKeys = require('lowercase-keys'); +import is, {assert} from '@sindresorhus/is'; +import parseLinkHeader from './parse-link-header'; +import type {Readable} from 'stream'; +import type {Socket} from 'net'; +import type {SecureContextOptions, DetailedPeerCertificate} from 'tls'; +import type { + Agent as HttpAgent, + ClientRequest +} from 'http'; +import type { + RequestOptions as HttpsRequestOptions, + Agent as HttpsAgent +} from 'https'; +import type {InspectOptions} from 'util'; +import type CacheableRequest = require('cacheable-request'); +import type ResponseLike = require('responselike'); +import type {IncomingMessageWithTimings} from '@szmarczak/http-timer'; +import type {Delays} from './timed-out'; +import type {RequestError} from './errors'; +import type {PlainResponse, Response} from './response'; +import type {CancelableRequest} from '../as-promise/types'; + +const [major, minor] = process.versions.node.split('.').map(v => Number(v)) as [number, number, number]; + +export type DnsLookupIpVersion = undefined | 4 | 6; + +type Except = Pick>; + +export type NativeRequestOptions = HttpsRequestOptions & CacheOptions & {checkServerIdentity?: CheckServerIdentityFunction}; + +type AcceptableResponse = IncomingMessageWithTimings | ResponseLike; +type AcceptableRequestResult = AcceptableResponse | ClientRequest | Promise | undefined; +export type RequestFunction = (url: URL, options: NativeRequestOptions, callback?: (response: AcceptableResponse) => void) => AcceptableRequestResult; + +export interface Agents { + http?: HttpAgent | false; + https?: HttpsAgent | false; + http2?: unknown | false; +} + +export type Headers = Record; + +export interface ToughCookieJar { + getCookieString: ((currentUrl: string, options: Record, cb: (error: Error | null, cookies: string) => void) => void) + & ((url: string, callback: (error: Error | null, cookieHeader: string) => void) => void); + setCookie: ((cookieOrString: unknown, currentUrl: string, options: Record, cb: (error: Error | null, cookie: unknown) => void) => void) + & ((rawCookie: string, url: string, callback: (error: Error | null, result: unknown) => void) => void); +} + +export interface PromiseCookieJar { + getCookieString: (url: string) => Promise; + setCookie: (rawCookie: string, url: string) => Promise; +} + +type Promisable = T | Promise; + +export type InitHook = (init: OptionsInit, self: Options) => void; +export type BeforeRequestHook = (options: Options) => Promisable; +export type BeforeRedirectHook = (updatedOptions: Options, plainResponse: PlainResponse) => Promisable; +export type BeforeErrorHook = (error: RequestError) => Promisable; +export type BeforeRetryHook = (error: RequestError) => Promisable; +export type AfterResponseHook = (response: Response, retryWithMergedOptions: (options: OptionsInit) => never) => Response | CancelableRequest | Promise>; + +/** +All available hooks of Got. +*/ +export interface Hooks { + /** + Called with plain request options, right before their normalization. + This is especially useful in conjunction with `got.extend()` when the input needs custom handling. + + __Note #1__: This hook must be synchronous! + + __Note #2__: Errors in this hook will be converted into an instances of `RequestError`. + + __Note #3__: The options object may not have a `url` property. + To modify it, use a `beforeRequest` hook instead. + + @default [] + */ + init: InitHook[]; + + /** + Called with normalized request options. + Got will make no further changes to the request before it is sent. + This is especially useful in conjunction with `got.extend()` when you want to create an API client that, for example, uses HMAC-signing. + + @default [] + */ + beforeRequest: BeforeRequestHook[]; + + /** + Called with normalized request options and the redirect response. + Got will make no further changes to the request. + This is especially useful when you want to avoid dead sites. + + @default [] + + @example + ``` + const got = require('got'); + + got('https://example.com', { + hooks: { + beforeRedirect: [ + (options, response) => { + if (options.hostname === 'deadSite') { + options.hostname = 'fallbackSite'; + } + } + ] + } + }); + ``` + */ + beforeRedirect: BeforeRedirectHook[]; + + /** + Called with an `Error` instance. + The error is passed to the hook right before it's thrown. + This is especially useful when you want to have more detailed errors. + + __Note__: Errors thrown while normalizing input options are thrown directly and not part of this hook. + + @default [] + + @example + ``` + const got = require('got'); + + got('https://api.github.com/some-endpoint', { + hooks: { + beforeError: [ + error => { + const {response} = error; + if (response && response.body) { + error.name = 'GitHubError'; + error.message = `${response.body.message} (${response.statusCode})`; + } + + return error; + } + ] + } + }); + ``` + */ + beforeError: BeforeErrorHook[]; + + /** + Called with normalized request options, the error and the retry count. + Got will make no further changes to the request. + This is especially useful when some extra work is required before the next try. + + __Note__: When using streams, this hook is ignored. + __Note__: When retrying in a `afterResponse` hook, all remaining `beforeRetry` hooks will be called without the `error` and `retryCount` arguments. + + @default [] + + @example + ``` + const got = require('got'); + + got.post('https://example.com', { + hooks: { + beforeRetry: [ + (options, error, retryCount) => { + if (error.response.statusCode === 413) { // Payload too large + options.body = getNewBody(); + } + } + ] + } + }); + ``` + */ + beforeRetry: BeforeRetryHook[]; + + /** + Called with [response object](#response) and a retry function. + Calling the retry function will trigger `beforeRetry` hooks. + + Each function should return the response. + This is especially useful when you want to refresh an access token. + + __Note__: When using streams, this hook is ignored. + + @example + ``` + const got = require('got'); + + const instance = got.extend({ + hooks: { + afterResponse: [ + (response, retryWithMergedOptions) => { + if (response.statusCode === 401) { // Unauthorized + const updatedOptions = { + headers: { + token: getNewToken() // Refresh the access token + } + }; + + // Save for further requests + instance.defaults.options = got.mergeOptions(instance.defaults.options, updatedOptions); + + // Make a new retry + return retryWithMergedOptions(updatedOptions); + } + + // No changes otherwise + return response; + } + ], + beforeRetry: [ + (options, error, retryCount) => { + // This will be called on `retryWithMergedOptions(...)` + } + ] + }, + mutableDefaults: true + }); + ``` + */ + afterResponse: AfterResponseHook[]; +} + +export type ParseJsonFunction = (text: string) => unknown; +export type StringifyJsonFunction = (object: unknown) => string; + +/** +All available HTTP request methods provided by Got. +*/ +export type Method = + | 'GET' + | 'POST' + | 'PUT' + | 'PATCH' + | 'HEAD' + | 'DELETE' + | 'OPTIONS' + | 'TRACE' + | 'get' + | 'post' + | 'put' + | 'patch' + | 'head' + | 'delete' + | 'options' + | 'trace'; + +export interface RetryObject { + attemptCount: number; + retryOptions: RetryOptions; + error: RequestError; + computedValue: number; + retryAfter?: number; +} + +export type RetryFunction = (retryObject: RetryObject) => number | Promise; + +/** +An object representing `limit`, `calculateDelay`, `methods`, `statusCodes`, `maxRetryAfter` and `errorCodes` fields for maximum retry count, retry handler, allowed methods, allowed status codes, maximum [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) time and allowed error codes. + +Delays between retries counts with function `1000 * Math.pow(2, retry) + Math.random() * 100`, where `retry` is attempt number (starts from 1). + +The `calculateDelay` property is a `function` that receives an object with `attemptCount`, `retryOptions`, `error` and `computedValue` properties for current retry count, the retry options, error and default computed value. +The function must return a delay in milliseconds (or a Promise resolving with it) (`0` return value cancels retry). + +By default, it retries *only* on the specified methods, status codes, and on these network errors: +- `ETIMEDOUT`: One of the [timeout](#timeout) limits were reached. +- `ECONNRESET`: Connection was forcibly closed by a peer. +- `EADDRINUSE`: Could not bind to any free port. +- `ECONNREFUSED`: Connection was refused by the server. +- `EPIPE`: The remote side of the stream being written has been closed. +- `ENOTFOUND`: Couldn't resolve the hostname to an IP address. +- `ENETUNREACH`: No internet connection. +- `EAI_AGAIN`: DNS lookup timed out. + +__Note__: If `maxRetryAfter` is set to `undefined`, it will use `options.timeout`. +__Note__: If [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) header is greater than `maxRetryAfter`, it will cancel the request. +*/ +export interface RetryOptions { + limit: number; + methods: Method[]; + statusCodes: number[]; + errorCodes: string[]; + calculateDelay: RetryFunction; + maxRetryAfter?: number; +} + +export type CreateConnectionFunction = (options: NativeRequestOptions, oncreate: (error: Error, socket: Socket) => void) => Socket; +export type CheckServerIdentityFunction = (hostname: string, certificate: DetailedPeerCertificate) => Error | void; + +export interface CacheOptions { + shared?: boolean; + cacheHeuristic?: number; + immutableMinTimeToLive?: number; + ignoreCargoCult?: boolean; +} + +export interface HttpsOptions { + alpnProtocols?: string[]; + + // From `http.RequestOptions` and `tls.CommonConnectionOptions` + rejectUnauthorized?: NativeRequestOptions['rejectUnauthorized']; + + // From `tls.ConnectionOptions` + checkServerIdentity?: CheckServerIdentityFunction; + + // From `tls.SecureContextOptions` + /** + Override the default Certificate Authorities ([from Mozilla](https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReport)). + + @example + ``` + // Single Certificate Authority + got('https://example.com', { + https: { + certificateAuthority: fs.readFileSync('./my_ca.pem') + } + }); + ``` + */ + certificateAuthority?: SecureContextOptions['ca']; + + /** + Private keys in [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) format. + + [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) allows the option of private keys being encrypted. + Encrypted keys will be decrypted with `options.https.passphrase`. + + Multiple keys with different passphrases can be provided as an array of `{pem: , passphrase: }` + */ + key?: SecureContextOptions['key']; + + /** + [Certificate chains](https://en.wikipedia.org/wiki/X.509#Certificate_chains_and_cross-certification) in [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) format. + + One cert chain should be provided per private key (`options.https.key`). + + When providing multiple cert chains, they do not have to be in the same order as their private keys in `options.https.key`. + + If the intermediate certificates are not provided, the peer will not be able to validate the certificate, and the handshake will fail. + */ + certificate?: SecureContextOptions['cert']; + + /** + The passphrase to decrypt the `options.https.key` (if different keys have different passphrases refer to `options.https.key` documentation). + */ + passphrase?: SecureContextOptions['passphrase']; + pfx?: SecureContextOptions['pfx']; +} + +export interface PaginateData { + response: Response; + currentItems: ElementType[]; + allItems: ElementType[]; +} + +export interface FilterData { + item: ElementType; + currentItems: ElementType[]; + allItems: ElementType[]; +} + +/** +All options accepted by `got.paginate()`. +*/ +export interface PaginationOptions { + /** + A function that transform [`Response`](#response) into an array of items. + This is where you should do the parsing. + + @default response => JSON.parse(response.body) + */ + transform?: (response: Response) => Promise | ElementType[]; + + /** + Checks whether the item should be emitted or not. + + @default ({item, currentItems, allItems}) => true + */ + filter?: (data: FilterData) => boolean; + + /** + The function takes an object with the following properties: + - `response` - The current response object. + - `currentItems` - Items from the current response. + - `allItems` - An empty array, unless `pagination.stackAllItems` is set to `true`, in which case, it's an array of the emitted items. + + It should return an object representing Got options pointing to the next page. The options are merged automatically with the previous request, therefore the options returned `pagination.paginate(...)` must reflect changes only. If there are no more pages, `false` should be returned. + + @example + ``` + const got = require('got'); + + (async () => { + const limit = 10; + + const items = got.paginate('https://example.com/items', { + searchParameters: { + limit, + offset: 0 + }, + pagination: { + paginate: ({response, currentItems}) => { + const previousSearchParams = response.request.options.searchParameters; + const previousOffset = previousSearchParams.get('offset'); + + if (currentItems.length < limit) { + return false; + } + + return { + searchParameters: { + ...previousSearchParams, + offset: Number(previousOffset) + limit, + } + }; + } + } + }); + + console.log('Items from all pages:', items); + })(); + ``` + */ + paginate?: (data: PaginateData) => OptionsInit | false; + + /** + Checks whether the pagination should continue. + + For example, if you need to stop **before** emitting an entry with some flag, you should use `({item}) => !item.flag`. + + If you want to stop **after** emitting the entry, you should use + `({item, allItems}) => allItems.some(item => item.flag)` instead. + + @default ({item, currentItems, allItems}) => true + */ + shouldContinue?: (data: FilterData) => boolean; + + /** + The maximum amount of items that should be emitted. + + @default Infinity + */ + countLimit?: number; + + /** + Milliseconds to wait before the next request is triggered. + + @default 0 + */ + backoff?: number; + /** + The maximum amount of request that should be triggered. + Retries on failure are not counted towards this limit. + + For example, it can be helpful during development to avoid an infinite number of requests. + + @default 10000 + */ + requestLimit?: number; + + /** + Defines how the property `allItems` in `pagination.paginate`, `pagination.filter` and `pagination.shouldContinue` is managed. + + By default, the property `allItems` is always an empty array. This setting can be helpful to save on memory usage when working with a large dataset. + + When set to `true`, the property `allItems` is an array of the emitted items. + + @default false + */ + stackAllItems?: boolean; +} + +export type SearchParameters = Record; + +function validateSearchParameters(searchParameters: Record): asserts searchParameters is Record { + // eslint-disable-next-line guard-for-in + for (const key in searchParameters) { + const value = searchParameters[key]; + + assert.any([is.string, is.number, is.boolean, is.null_, is.undefined], value); + } +} + +/** +All parsing methods supported by Got. +*/ +export type ResponseType = 'json' | 'buffer' | 'text'; + +export type InternalsType = Except; + +export type OptionsError = NodeJS.ErrnoException & {options: Options}; + +export type OptionsInit = + Except, 'hooks' | 'retry'> + & { + hooks?: Partial; + retry?: Partial; + }; + +const globalCache = new Map(); +let globalDnsCache: CacheableLookup; + +const getGlobalDnsCache = (): CacheableLookup => { + if (globalDnsCache) { + return globalDnsCache; + } + + globalDnsCache = new CacheableLookup(); + return globalDnsCache; +}; + +const defaultInternals: Options['_internals'] = { + request: undefined, + agent: { + http: undefined, + https: undefined, + http2: undefined + }, + decompress: true, + timeout: { + connect: undefined, + lookup: undefined, + read: undefined, + request: undefined, + response: undefined, + secureConnect: undefined, + send: undefined, + socket: undefined + }, + prefixUrl: '', + body: undefined, + form: undefined, + json: undefined, + cookieJar: undefined, + ignoreInvalidCookies: false, + searchParameters: undefined, + dnsLookup: undefined, + dnsCache: undefined, + context: {}, + hooks: { + init: [], + beforeRequest: [], + beforeError: [], + beforeRedirect: [], + beforeRetry: [], + afterResponse: [] + }, + followRedirect: true, + maxRedirects: 10, + cache: undefined, + throwHttpErrors: true, + username: '', + password: '', + http2: false, + allowGetBody: false, + headers: { + 'user-agent': 'got (https://github.com/sindresorhus/got)' + }, + methodRewriting: false, + dnsLookupIpVersion: undefined, + parseJson: JSON.parse, + stringifyJson: JSON.stringify, + retry: { + limit: 2, + methods: [ + 'GET', + 'PUT', + 'HEAD', + 'DELETE', + 'OPTIONS', + 'TRACE' + ], + statusCodes: [ + 408, + 413, + 429, + 500, + 502, + 503, + 504, + 521, + 522, + 524 + ], + errorCodes: [ + 'ETIMEDOUT', + 'ECONNRESET', + 'EADDRINUSE', + 'ECONNREFUSED', + 'EPIPE', + 'ENOTFOUND', + 'ENETUNREACH', + 'EAI_AGAIN' + ], + maxRetryAfter: undefined, + calculateDelay: ({computedValue}) => computedValue + }, + localAddress: undefined, + method: 'GET', + createConnection: undefined, + cacheOptions: { + shared: undefined, + cacheHeuristic: undefined, + immutableMinTimeToLive: undefined, + ignoreCargoCult: undefined + }, + httpsOptions: { + alpnProtocols: undefined, + rejectUnauthorized: undefined, + checkServerIdentity: undefined, + certificateAuthority: undefined, + key: undefined, + certificate: undefined, + passphrase: undefined, + pfx: undefined + }, + encoding: undefined, + resolveBodyOnly: false, + isStream: false, + responseType: 'text', + url: undefined, + pagination: { + transform: (response: Response) => { + if (response.request.options.responseType === 'json') { + return response.body; + } + + return JSON.parse(response.body as string); + }, + paginate: ({response}) => { + if (typeof response.headers.link !== 'string') { + return false; + } + + const parsed = parseLinkHeader(response.headers.link); + const next = parsed.find(entry => entry.parameters.rel === 'next' || entry.parameters.rel === '"next"'); + + if (next) { + return {url: next.reference}; + } + + return false; + }, + filter: () => true, + shouldContinue: () => true, + countLimit: Number.POSITIVE_INFINITY, + backoff: 0, + requestLimit: 10000, + stackAllItems: false + }, + setHost: true, + maxHeaderSize: undefined +}; + +const cloneInternals = (internals: typeof defaultInternals): typeof defaultInternals => { + const {hooks, retry} = internals; + + const result = { + ...internals, + context: {...internals.context}, + cacheOptions: {...internals.cacheOptions}, + httpsOptions: {...internals.httpsOptions}, + agent: {...internals.agent}, + headers: {...internals.headers}, + retry: { + ...retry, + errorCodes: [...retry.errorCodes!], + methods: [...retry.methods!], + statusCodes: [...retry.statusCodes!] + }, + timeout: {...internals.timeout}, + hooks: { + init: [...hooks.init], + beforeRequest: [...hooks.beforeRequest], + beforeError: [...hooks.beforeError], + beforeRedirect: [...hooks.beforeRedirect], + beforeRetry: [...hooks.beforeRetry], + afterResponse: [...hooks.afterResponse] + }, + searchParameters: internals.searchParameters ? new URLSearchParams(internals.searchParameters as URLSearchParams) : undefined, + pagination: {...internals.pagination} + }; + + if (result.url !== undefined) { + result.prefixUrl = ''; + } + + return result; +}; + +const descriptor = { + _unixOptions: { + value: undefined, + enumerable: false, + writable: true, + configurable: false + }, + _merging: { + value: false, + enumerable: false, + writable: true, + configurable: false + }, + _internals: { + value: undefined as (Options['_internals'] | undefined), + enumerable: false, + writable: false, + configurable: false + }, + _init: { + value: undefined as (Options['_init'] | undefined), + enumerable: false, + writable: true, + configurable: false + } +}; + +const init = (options: OptionsInit, withOptions: OptionsInit, self: Options): void => { + const initHooks = options.hooks?.init; + if (initHooks) { + for (const hook of initHooks) { + hook(withOptions, self); + } + } +}; + +export default class Options { + // TODO: Remove `declare` when targeting Node.js 14 + declare private _unixOptions?: NativeRequestOptions; + declare private _internals: InternalsType; + declare private _merging: boolean; + declare private readonly _init: OptionsInit[]; + + constructor(input?: string | URL | OptionsInit, options?: OptionsInit, defaults?: Options | OptionsInit) { + assert.any([is.string, is.urlInstance, is.object, is.undefined], input); + assert.any([is.object, is.undefined], options); + assert.any([is.object, is.undefined], defaults); + + if (input instanceof Options || options instanceof Options) { + throw new TypeError('The defaults must be passed as the third argument'); + } + + // TODO: Switch to `this.key = value` when targeting Node.js 14 + descriptor._internals.value = cloneInternals((defaults as Options)?._internals ?? defaults ?? defaultInternals); + descriptor._init.value = [...((defaults as Options)?._init ?? [])]; + Object.defineProperties(this, descriptor); + + try { + if (is.plainObject(input)) { + this.merge(input); + this.merge(options); + this.url = input.url; + } else { + this.merge(options); + + if (options?.url !== undefined) { + if (input === undefined) { + this.url = options.url; + } else { + throw new TypeError('The `url` option is mutually exclusive with the `input` argument'); + } + } else if (input !== undefined) { + this.url = input; + } + } + } catch (error) { + (error as OptionsError).options = this; + + throw error; + } + } + + merge(options?: OptionsInit | Options) { + if (!options) { + return; + } + + if (options instanceof Options) { + for (const init of options._init) { + this.merge(init); + } + + return; + } + + init(this, options, this); + init(options, options, this); + this._init.push(options); + + // This is way much faster than cloning ^_^ + Object.freeze(options); + Object.freeze(options.hooks); + Object.freeze(options.httpsOptions); + Object.freeze(options.cacheOptions); + Object.freeze(options.agent); + Object.freeze(options.headers); + Object.freeze(options.timeout); + Object.freeze(options.retry); + Object.freeze(options.hooks); + Object.freeze(options.context); + + this._merging = true; + + try { + for (const key in options) { + // `got.extend()` options + if (key === 'mutableDefaults' || key === 'handlers') { + continue; + } + + // Never merge `url` + if (key === 'url') { + continue; + } + + if (!(key in this)) { + throw new Error(`Unexpected option: ${key}`); + } + + // @ts-expect-error Type 'unknown' is not assignable to type 'never'. + this[key as keyof Options] = options[key as keyof Options]; + } + } finally { + this._merging = false; + } + } + + /** + Custom request function. + The main purpose of this is to [support HTTP2 using a wrapper](https://github.com/szmarczak/http2-wrapper). + + @default http.request | https.request + */ + get request(): RequestFunction | undefined { + return this._internals.request; + } + + set request(value: RequestFunction | undefined) { + assert.any([is.function_, is.undefined], value); + + this._internals.request = value; + } + + /** + An object representing `http`, `https` and `http2` keys for [`http.Agent`](https://nodejs.org/api/http.html#http_class_http_agent), [`https.Agent`](https://nodejs.org/api/https.html#https_class_https_agent) and [`http2wrapper.Agent`](https://github.com/szmarczak/http2-wrapper#new-http2agentoptions) instance. + This is necessary because a request to one protocol might redirect to another. + In such a scenario, Got will switch over to the right protocol agent for you. + + If a key is not present, it will default to a global agent. + + @example + ``` + const got = require('got'); + const HttpAgent = require('agentkeepalive'); + const {HttpsAgent} = HttpAgent; + + got('https://sindresorhus.com', { + agent: { + http: new HttpAgent(), + https: new HttpsAgent() + } + }); + ``` + */ + get agent(): Agents { + return this._internals.agent; + } + + set agent(value: Agents) { + assert.plainObject(value); + + // eslint-disable-next-line guard-for-in + for (const key in value) { + if (!(key in this._internals.agent)) { + throw new TypeError(`Unexpected agent option: ${key}`); + } + + assert.any([is.object, is.undefined], value[key]); + } + + if (this._merging) { + Object.assign(this._internals.agent, value); + } else { + this._internals.agent = {...value}; + } + } + + /** + Decompress the response automatically. + This will set the `accept-encoding` header to `gzip, deflate, br` on Node.js 11.7.0+ or `gzip, deflate` for older Node.js versions, unless you set it yourself. + + Brotli (`br`) support requires Node.js 11.7.0 or later. + + If this is disabled, a compressed response is returned as a `Buffer`. + This may be useful if you want to handle decompression yourself or stream the raw compressed data. + + @default true + */ + get decompress(): boolean { + return this._internals.decompress; + } + + set decompress(value: boolean) { + assert.boolean(value); + + this._internals.decompress = value; + } + + /** + Milliseconds to wait for the server to end the response before aborting the request with `got.TimeoutError` error (a.k.a. `request` property). + By default, there's no timeout. + + This also accepts an `object` with the following fields to constrain the duration of each phase of the request lifecycle: + + - `lookup` starts when a socket is assigned and ends when the hostname has been resolved. + Does not apply when using a Unix domain socket. + - `connect` starts when `lookup` completes (or when the socket is assigned if lookup does not apply to the request) and ends when the socket is connected. + - `secureConnect` starts when `connect` completes and ends when the handshaking process completes (HTTPS only). + - `socket` starts when the socket is connected. See [request.setTimeout](https://nodejs.org/api/http.html#http_request_settimeout_timeout_callback). + - `response` starts when the request has been written to the socket and ends when the response headers are received. + - `send` starts when the socket is connected and ends with the request has been written to the socket. + - `request` starts when the request is initiated and ends when the response's end event fires. + */ + get timeout(): Delays { + // We always return `Delays` here. + // It has to be `Delays | number`, otherwise TypeScript will error because the getter and the setter have incompatible types. + return this._internals.timeout; + } + + set timeout(value: Delays) { + assert.plainObject(value); + + // eslint-disable-next-line guard-for-in + for (const key in value) { + if (!(key in this._internals.timeout)) { + throw new Error(`Unexpected timeout option: ${key}`); + } + + assert.any([is.number, is.undefined], value[key]); + } + + if (this._merging) { + Object.assign(this._internals.timeout, value); + } else { + this._internals.timeout = {...value}; + } + } + + /** + When specified, `prefixUrl` will be prepended to `url`. + The prefix can be any valid URL, either relative or absolute. + A trailing slash `/` is optional - one will be added automatically. + + __Note__: `prefixUrl` will be ignored if the `url` argument is a URL instance. + + __Note__: Leading slashes in `input` are disallowed when using this option to enforce consistency and avoid confusion. + For example, when the prefix URL is `https://example.com/foo` and the input is `/bar`, there's ambiguity whether the resulting URL would become `https://example.com/foo/bar` or `https://example.com/bar`. + The latter is used by browsers. + + __Tip__: Useful when used with `got.extend()` to create niche-specific Got instances. + + __Tip__: You can change `prefixUrl` using hooks as long as the URL still includes the `prefixUrl`. + If the URL doesn't include it anymore, it will throw. + + @example + ``` + const got = require('got'); + + (async () => { + await got('unicorn', {prefixUrl: 'https://cats.com'}); + //=> 'https://cats.com/unicorn' + + const instance = got.extend({ + prefixUrl: 'https://google.com' + }); + + await instance('unicorn', { + hooks: { + beforeRequest: [ + options => { + options.prefixUrl = 'https://cats.com'; + } + ] + } + }); + //=> 'https://cats.com/unicorn' + })(); + ``` + */ + get prefixUrl(): string | URL { + // We always return `string` here. + // It has to be `string | URL`, otherwise TypeScript will error because the getter and the setter have incompatible types. + return this._internals.prefixUrl; + } + + set prefixUrl(value: string | URL) { + assert.any([is.string, is.urlInstance], value); + + if (value === '') { + this._internals.prefixUrl = ''; + return; + } + + value = value.toString(); + + if (!value.endsWith('/')) { + value += '/'; + } + + if (this._internals.prefixUrl && this._internals.url) { + const {href} = this._internals.url as URL; + + (this._internals.url as URL).href = value + href.slice((this._internals.prefixUrl as string).length); + } + + this._internals.prefixUrl = value; + } + + /** + __Note #1__: The `body` option cannot be used with the `json` or `form` option. + + __Note #2__: If you provide this option, `got.stream()` will be read-only. + + __Note #3__: If you provide a payload with the `GET` or `HEAD` method, it will throw a `TypeError` unless the method is `GET` and the `allowGetBody` option is set to `true`. + + __Note #4__: This option is not enumerable and will not be merged with the instance defaults. + + The `content-length` header will be automatically set if `body` is a `string` / `Buffer` / [`form-data` instance](https://github.com/form-data/form-data), and `content-length` and `transfer-encoding` are not manually set in `options.headers`. + + Since Got 12, the `content-length` is not automatically set when `body` is a `fs.createReadStream`. + */ + get body(): string | Buffer | Readable | undefined { + return this._internals.body; + } + + set body(value: string | Buffer | Readable | undefined) { + assert.any([is.string, is.buffer, is.nodeStream, is.undefined], value); + + if (is.nodeStream(value)) { + assert.truthy(value.readable); + } + + if (value !== undefined) { + assert.undefined(this._internals.form); + assert.undefined(this._internals.json); + } + + this._internals.body = value; + } + + /** + The form body is converted to a query string using [`(new URLSearchParams(object)).toString()`](https://nodejs.org/api/url.html#url_constructor_new_urlsearchparams_obj). + + If the `Content-Type` header is not present, it will be set to `application/x-www-form-urlencoded`. + + __Note #1__: If you provide this option, `got.stream()` will be read-only. + + __Note #2__: This option is not enumerable and will not be merged with the instance defaults. + */ + get form(): Record | undefined { + return this._internals.form; + } + + set form(value: Record | undefined) { + assert.any([is.plainObject, is.undefined], value); + + if (value !== undefined) { + assert.undefined(this._internals.body); + assert.undefined(this._internals.json); + } + + this._internals.form = value; + } + + /** + JSON body. If the `Content-Type` header is not set, it will be set to `application/json`. + + __Note #1__: If you provide this option, `got.stream()` will be read-only. + + __Note #2__: This option is not enumerable and will not be merged with the instance defaults. + */ + get json(): Record | undefined { + return this._internals.json; + } + + set json(value: Record | undefined) { + assert.any([is.object, is.undefined], value); + + if (value !== undefined) { + assert.undefined(this._internals.body); + assert.undefined(this._internals.form); + } + + this._internals.json = value; + } + + /** + The URL to request, as a string, a [`https.request` options object](https://nodejs.org/api/https.html#https_https_request_options_callback), or a [WHATWG `URL`](https://nodejs.org/api/url.html#url_class_url). + + Properties from `options` will override properties in the parsed `url`. + + If no protocol is specified, it will throw a `TypeError`. + + __Note__: The query string is **not** parsed as search params. + + @example + ``` + got('https://example.com/?query=a b'); //=> https://example.com/?query=a%20b + got('https://example.com/', {searchParameters: {query: 'a b'}}); //=> https://example.com/?query=a+b + + // The query string is overridden by `searchParameters` + got('https://example.com/?query=a b', {searchParameters: {query: 'a b'}}); //=> https://example.com/?query=a+b + ``` + */ + get url(): string | URL | undefined { + return this._internals.url; + } + + set url(value: string | URL | undefined) { + assert.any([is.string, is.urlInstance, is.undefined], value); + + if (value === undefined) { + this._internals.url = undefined; + return; + } + + // eslint-disable-next-line @typescript-eslint/prefer-string-starts-ends-with + if (is.string(value) && value[0] === '/') { + throw new Error('`url` must not start with a slash'); + } + + const urlString = `${this.prefixUrl as string}${value.toString()}`; + const url = new URL(urlString); + this._internals.url = url; + decodeURI(urlString); + + if (url.protocol === 'unix:') { + url.href = `http://unix${url.pathname}${url.search}`; + } + + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error(`Unsupported protocol: ${url.protocol}`); + } + + if (this._internals.username) { + url.username = this._internals.username; + this._internals.username = ''; + } + + if (this._internals.password) { + url.password = this._internals.password; + this._internals.password = ''; + } + + if (this._internals.searchParameters) { + url.search = (this._internals.searchParameters as URLSearchParams).toString(); + this._internals.searchParameters = undefined; + } + + if (url.hostname === 'unix') { + const matches = /(?.+?):(?.+)/.exec(`${url.pathname}${url.search}`); + + if (matches?.groups) { + const {socketPath, path} = matches.groups; + + this._unixOptions = { + socketPath, + path, + host: '' + }; + } else { + this._unixOptions = undefined; + } + + return; + } + + this._unixOptions = undefined; + } + + /** + Cookie support. You don't have to care about parsing or how to store them. + + __Note__: If you provide this option, `options.headers.cookie` will be overridden. + */ + get cookieJar(): PromiseCookieJar | ToughCookieJar | undefined { + return this._internals.cookieJar; + } + + set cookieJar(value: PromiseCookieJar | ToughCookieJar | undefined) { + assert.any([is.object, is.undefined], value); + + if (value === undefined) { + this._internals.cookieJar = undefined; + return; + } + + let {setCookie, getCookieString} = value; + + assert.function_(setCookie); + assert.function_(getCookieString); + + /* istanbul ignore next: Horrible `tough-cookie` v3 check */ + if (setCookie.length === 4 && getCookieString.length === 0) { + setCookie = promisify(setCookie.bind(value)); + getCookieString = promisify(getCookieString.bind(value)); + + this._internals.cookieJar = { + setCookie, + getCookieString: getCookieString as PromiseCookieJar['getCookieString'] + }; + } else { + this._internals.cookieJar = value; + } + } + + /** + Ignore invalid cookies instead of throwing an error. + Only useful when the `cookieJar` option has been set. Not recommended. + + @default false + */ + get ignoreInvalidCookies(): boolean { + return this._internals.ignoreInvalidCookies; + } + + set ignoreInvalidCookies(value: boolean) { + assert.boolean(value); + + this._internals.ignoreInvalidCookies = value; + } + + /** + Query string that will be added to the request URL. + This will override the query string in `url`. + + If you need to pass in an array, you can do it using a `URLSearchParams` instance. + + @example + ``` + const got = require('got'); + + const searchParameters = new URLSearchParams([['key', 'a'], ['key', 'b']]); + + got('https://example.com', {searchParameters}); + + console.log(searchParameters.toString()); + //=> 'key=a&key=b' + ``` + */ + get searchParameters(): string | SearchParameters | URLSearchParams | undefined { + if (this._internals.url) { + return (this._internals.url as URL).searchParams; + } + + return this._internals.searchParameters; + } + + set searchParameters(value: string | SearchParameters | URLSearchParams | undefined) { + assert.any([is.string, is.object, is.undefined], value); + + const url = this._internals.url as URL; + + if (value === undefined) { + this._internals.searchParameters = undefined; + + if (url) { + url.search = ''; + } + + return; + } + + let searchParameters = (this.searchParameters ?? new URLSearchParams()) as URLSearchParams; + let updated; + + if (is.string(value) || (value instanceof URLSearchParams)) { + updated = new URLSearchParams(value); + } else { + validateSearchParameters(value); + + updated = new URLSearchParams(); + + // eslint-disable-next-line guard-for-in + for (const key in value) { + const entry = value[key]; + + if (entry === null) { + updated.append(key, ''); + } else if (entry !== undefined) { + updated.append(key, entry as string); + } + } + } + + if (this._merging) { + // eslint-disable-next-line unicorn/no-array-for-each + updated.forEach((value, key) => { + searchParameters.set(key, value); + }); + } else { + searchParameters = updated; + } + + if (!url) { + this._internals.searchParameters = searchParameters; + } + } + + get dnsLookup(): CacheableLookup['lookup'] | undefined { + return this._internals.dnsLookup; + } + + set dnsLookup(value: CacheableLookup['lookup'] | undefined) { + assert.any([is.function_, is.undefined], value); + + this._internals.dnsLookup = value; + } + + /** + An instance of [`CacheableLookup`](https://github.com/szmarczak/cacheable-lookup) used for making DNS lookups. + Useful when making lots of requests to different *public* hostnames. + + `CacheableLookup` uses `dns.resolver4(..)` and `dns.resolver6(...)` under the hood and fall backs to `dns.lookup(...)` when the first two fail, which may lead to additional delay. + + __Note__: This should stay disabled when making requests to internal hostnames such as `localhost`, `database.local` etc. + + @default false + */ + get dnsCache(): CacheableLookup | boolean | undefined { + return this._internals.dnsCache; + } + + set dnsCache(value: CacheableLookup | boolean | undefined) { + assert.any([is.object, is.boolean, is.undefined], value); + + if (value === true) { + this._internals.dnsCache = getGlobalDnsCache(); + } else if (value === false) { + this._internals.dnsCache = undefined; + } else { + this._internals.dnsCache = value; + } + } + + /** + User data. `context` is shallow merged and enumerable. If it contains non-enumerable properties they will NOT be merged. + + @example + ``` + const got = require('got'); + + const instance = got.extend({ + hooks: { + beforeRequest: [ + options => { + if (!options.context || !options.context.token) { + throw new Error('Token required'); + } + + options.headers.token = options.context.token; + } + ] + } + }); + + (async () => { + const context = { + token: 'secret' + }; + + const response = await instance('https://httpbin.org/headers', {context}); + + // Let's see the headers + console.log(response.body); + })(); + ``` + */ + get context(): Record { + return this._internals.context; + } + + set context(value: Record) { + assert.object(value); + + if (this._merging) { + Object.assign(this._internals.context, value); + } else { + this._internals.context = {...value}; + } + } + + /** + Hooks allow modifications during the request lifecycle. + Hook functions may be async and are run serially. + */ + get hooks(): Hooks { + return this._internals.hooks; + } + + set hooks(value: Hooks) { + assert.object(value); + + // eslint-disable-next-line guard-for-in + for (const knownHookEvent in value) { + if (!(knownHookEvent in this._internals.hooks)) { + throw new Error(`Unexpected hook event: ${knownHookEvent}`); + } + + const hooks: unknown[] = value[knownHookEvent as keyof Hooks]; + assert.any([is.array, is.undefined], hooks); + + for (const hook of hooks) { + assert.function_(hook); + } + + if (this._merging) { + if (hooks) { + // @ts-expect-error FIXME + this._internals.hooks[knownHookEvent].push(...hooks); + } + } else if (hooks) { + // @ts-expect-error FIXME + this._internals.hooks[knownHookEvent] = [...hooks]; + } else { + // @ts-expect-error FIXME + this._internals.hooks[knownHookEvent] = []; + } + } + } + + /** + Defines if redirect responses should be followed automatically. + + Note that if a `303` is sent by the server in response to any request type (`POST`, `DELETE`, etc.), Got will automatically request the resource pointed to in the location header via `GET`. + This is in accordance with [the spec](https://tools.ietf.org/html/rfc7231#section-6.4.4). + + @default true + */ + get followRedirect(): boolean { + return this._internals.followRedirect; + } + + set followRedirect(value: boolean) { + assert.boolean(value); + + this._internals.followRedirect = value; + } + + get followRedirects() { + throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.'); + } + + set followRedirects(_value: unknown) { + throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.'); + } + + /** + If exceeded, the request will be aborted and a `MaxRedirectsError` will be thrown. + + @default 10 + */ + get maxRedirects(): number { + return this._internals.maxRedirects; + } + + set maxRedirects(value: number) { + assert.number(value); + + this._internals.maxRedirects = value; + } + + /** + A cache adapter instance for storing cached response data. + + @default false + */ + get cache(): string | CacheableRequest.StorageAdapter | boolean | undefined { + return this._internals.cache; + } + + set cache(value: string | CacheableRequest.StorageAdapter | boolean | undefined) { + assert.any([is.object, is.string, is.boolean, is.undefined], value); + + if (value === true) { + this._internals.cache = globalCache; + } else if (value === false) { + this._internals.cache = undefined; + } else { + this._internals.cache = value; + } + } + + /** + Determines if a `got.HTTPError` is thrown for unsuccessful responses. + + If this is disabled, requests that encounter an error status code will be resolved with the `response` instead of throwing. + This may be useful if you are checking for resource availability and are expecting error responses. + + @default true + */ + get throwHttpErrors(): boolean { + return this._internals.throwHttpErrors; + } + + set throwHttpErrors(value: boolean) { + assert.boolean(value); + + this._internals.throwHttpErrors = value; + } + + get username(): string { + const url = this._internals.url as URL; + + if (url) { + return url.username; + } + + return this._internals.username; + } + + set username(value: string) { + assert.string(value); + + const url = this._internals.url as URL; + + if (url) { + url.username = value; + } else { + this._internals.username = value; + } + } + + get password(): string { + const url = this._internals.url as URL; + + if (url) { + return url.password; + } + + return this._internals.password; + } + + set password(value: string) { + assert.string(value); + + const url = this._internals.url as URL; + + if (url) { + url.password = value; + } else { + this._internals.password = value; + } + } + + /** + If set to `true`, Got will additionally accept HTTP2 requests. + + It will choose either HTTP/1.1 or HTTP/2 depending on the ALPN protocol. + + __Note__: This option requires Node.js 15.10.0 or newer as HTTP/2 support on older Node.js versions is very buggy. + + __Note__: Overriding `options.request` will disable HTTP2 support. + + @default false + + @example + ``` + const got = require('got'); + + (async () => { + const {headers} = await got('https://nghttp2.org/httpbin/anything', {http2: true}); + console.log(headers.via); + //=> '2 nghttpx' + })(); + ``` + */ + get http2(): boolean { + return this._internals.http2; + } + + set http2(value: boolean) { + assert.boolean(value); + + this._internals.http2 = value; + } + + /** + Set this to `true` to allow sending body for the `GET` method. + However, the [HTTP/2 specification](https://tools.ietf.org/html/rfc7540#section-8.1.3) says that `An HTTP GET request includes request header fields and no payload body`, therefore when using the HTTP/2 protocol this option will have no effect. + This option is only meant to interact with non-compliant servers when you have no other choice. + + __Note__: The [RFC 7321](https://tools.ietf.org/html/rfc7231#section-4.3.1) doesn't specify any particular behavior for the GET method having a payload, therefore __it's considered an [anti-pattern](https://en.wikipedia.org/wiki/Anti-pattern)__. + + @default false + */ + get allowGetBody(): boolean { + return this._internals.allowGetBody; + } + + set allowGetBody(value: boolean) { + assert.boolean(value); + + this._internals.allowGetBody = value; + } + + /** + Request headers. + + Existing headers will be overwritten. Headers set to `undefined` will be omitted. + + @default {} + */ + get headers(): Headers { + return this._internals.headers; + } + + set headers(value: Headers) { + assert.plainObject(value); + + if (this._merging) { + Object.assign(this._internals.headers, lowercaseKeys(value)); + } else { + this._internals.headers = lowercaseKeys(value); + } + } + + /** + Specifies if the redirects should be [rewritten as `GET`](https://tools.ietf.org/html/rfc7231#section-6.4). + + If `false`, when sending a POST request and receiving a `302`, it will resend the body to the new location using the same HTTP method (`POST` in this case). + + @default false + */ + get methodRewriting(): boolean { + return this._internals.methodRewriting; + } + + set methodRewriting(value: boolean) { + assert.boolean(value); + + this._internals.methodRewriting = value; + } + + /** + Indicates which DNS record family to use. + + Values: + - `undefined`: IPv4 (if present) or IPv6 + - `4`: Only IPv4 + - `6`: Only IPv6 + + @default undefined + */ + get dnsLookupIpVersion(): DnsLookupIpVersion { + return this._internals.dnsLookupIpVersion; + } + + set dnsLookupIpVersion(value: DnsLookupIpVersion) { + if (value !== undefined && value !== 4 && value !== 6) { + throw new TypeError(`Invalid DNS lookup IP version: ${value as string}`); + } + + this._internals.dnsLookupIpVersion = value; + } + + /** + A function used to parse JSON responses. + + @example + ``` + const got = require('got'); + const Bourne = require('@hapi/bourne'); + + (async () => { + const parsed = await got('https://example.com', { + parseJson: text => Bourne.parse(text) + }).json(); + + console.log(parsed); + })(); + ``` + */ + get parseJson(): ParseJsonFunction { + return this._internals.parseJson; + } + + set parseJson(value: ParseJsonFunction) { + assert.function_(value); + + this._internals.parseJson = value; + } + + /** + A function used to stringify the body of JSON requests. + + @example + ``` + const got = require('got'); + + (async () => { + await got.post('https://example.com', { + stringifyJson: object => JSON.stringify(object, (key, value) => { + if (key.startsWith('_')) { + return; + } + + return value; + }), + json: { + some: 'payload', + _ignoreMe: 1234 + } + }); + })(); + ``` + + @example + ``` + const got = require('got'); + + (async () => { + await got.post('https://example.com', { + stringifyJson: object => JSON.stringify(object, (key, value) => { + if (typeof value === 'number') { + return value.toString(); + } + + return value; + }), + json: { + some: 'payload', + number: 1 + } + }); + })(); + ``` + */ + get stringifyJson(): StringifyJsonFunction { + return this._internals.stringifyJson; + } + + set stringifyJson(value: StringifyJsonFunction) { + assert.function_(value); + + this._internals.stringifyJson = value; + } + + /** + An object representing `limit`, `calculateDelay`, `methods`, `statusCodes`, `maxRetryAfter` and `errorCodes` fields for maximum retry count, retry handler, allowed methods, allowed status codes, maximum [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) time and allowed error codes. + + Delays between retries counts with function `1000 * Math.pow(2, retry) + Math.random() * 100`, where `retry` is attempt number (starts from 1). + + The `calculateDelay` property is a `function` that receives an object with `attemptCount`, `retryOptions`, `error` and `computedValue` properties for current retry count, the retry options, error and default computed value. + The function must return a delay in milliseconds (or a Promise resolving with it) (`0` return value cancels retry). + + By default, it retries *only* on the specified methods, status codes, and on these network errors: + + - `ETIMEDOUT`: One of the [timeout](#timeout) limits were reached. + - `ECONNRESET`: Connection was forcibly closed by a peer. + - `EADDRINUSE`: Could not bind to any free port. + - `ECONNREFUSED`: Connection was refused by the server. + - `EPIPE`: The remote side of the stream being written has been closed. + - `ENOTFOUND`: Couldn't resolve the hostname to an IP address. + - `ENETUNREACH`: No internet connection. + - `EAI_AGAIN`: DNS lookup timed out. + + __Note__: If `maxRetryAfter` is set to `undefined`, it will use `options.timeout`. + __Note__: If [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) header is greater than `maxRetryAfter`, it will cancel the request. + */ + get retry(): Partial { + return this._internals.retry; + } + + set retry(value: Partial) { + assert.plainObject(value); + + assert.any([is.function_, is.undefined], value.calculateDelay); + assert.any([is.number, is.undefined], value.maxRetryAfter); + assert.any([is.number, is.undefined], value.limit); + assert.any([is.array, is.undefined], value.methods); + assert.any([is.array, is.undefined], value.statusCodes); + assert.any([is.array, is.undefined], value.errorCodes); + + for (const key in value) { + if (!(key in this._internals.retry)) { + throw new Error(`Unexpected retry option: ${key}`); + } + } + + if (this._merging) { + Object.assign(this._internals.retry, value); + } else { + this._internals.retry = {...value}; + } + + const {retry} = this._internals; + + retry.methods = [...new Set(retry.methods!.map(method => method.toUpperCase() as Method))]; + retry.statusCodes = [...new Set(retry.statusCodes)]; + retry.errorCodes = [...new Set(retry.errorCodes)]; + } + + /** + From `http.RequestOptions`. + + The IP address used to send the request from. + */ + get localAddress(): string | undefined { + return this._internals.localAddress; + } + + set localAddress(value: string | undefined) { + assert.any([is.string, is.undefined], value); + + this._internals.localAddress = value; + } + + /** + The HTTP method used to make the request. + + @default 'GET' + */ + get method(): Method { + return this._internals.method; + } + + set method(value: Method) { + assert.any([is.string, is.undefined], value); + + this._internals.method = value.toUpperCase() as Method; + } + + get createConnection(): CreateConnectionFunction | undefined { + return this._internals.createConnection; + } + + set createConnection(value: CreateConnectionFunction | undefined) { + assert.any([is.function_, is.undefined], value); + + this._internals.createConnection = value; + } + + /** + From `http-cache-semantics` + + @default {} + */ + get cacheOptions(): CacheOptions { + return this._internals.cacheOptions; + } + + set cacheOptions(value: CacheOptions) { + assert.plainObject(value); + + assert.any([is.boolean, is.undefined], value.shared); + assert.any([is.number, is.undefined], value.cacheHeuristic); + assert.any([is.number, is.undefined], value.immutableMinTimeToLive); + assert.any([is.boolean, is.undefined], value.ignoreCargoCult); + + for (const key in value) { + if (!(key in this._internals.cacheOptions)) { + throw new Error(`Cache option \`${key}\` does not exist`); + } + } + + if (this._merging) { + Object.assign(this._internals.cacheOptions, value); + } else { + this._internals.cacheOptions = {...value}; + } + } + + /** + Options for the advanced HTTPS API. + */ + get httpsOptions(): HttpsOptions { + return this._internals.httpsOptions; + } + + set httpsOptions(value: HttpsOptions) { + assert.plainObject(value); + + assert.any([is.boolean, is.undefined], value.rejectUnauthorized); + assert.any([is.function_, is.undefined], value.checkServerIdentity); + assert.any([is.string, is.object, is.array, is.undefined], value.certificateAuthority); + assert.any([is.string, is.object, is.array, is.undefined], value.key); + assert.any([is.string, is.object, is.array, is.undefined], value.certificate); + assert.any([is.string, is.undefined], value.passphrase); + assert.any([is.string, is.buffer, is.array, is.undefined], value.pfx); + assert.any([is.array, is.undefined], value.alpnProtocols); + + for (const key in value) { + if (!(key in this._internals.httpsOptions)) { + throw new Error(`HTTPS option \`${key}\` does not exist`); + } + } + + if (this._merging) { + Object.assign(this._internals.httpsOptions, value); + } else { + this._internals.httpsOptions = {...value}; + } + } + + /** + [Encoding](https://nodejs.org/api/buffer.html#buffer_buffers_and_character_encodings) to be used on `setEncoding` of the response data. + + To get a [`Buffer`](https://nodejs.org/api/buffer.html), you need to set `responseType` to `buffer` instead. + Don't set this option to `null`. + + __Note__: This doesn't affect streams! Instead, you need to do `got.stream(...).setEncoding(encoding)`. + + @default 'utf-8' + */ + get encoding(): BufferEncoding | undefined { + return this._internals.encoding; + } + + set encoding(value: BufferEncoding | undefined) { + if (value === null) { + throw new TypeError('To get a Buffer, set `options.responseType` to `buffer` instead'); + } + + assert.any([is.string, is.undefined], value); + + this._internals.encoding = value; + } + + /** + When set to `true` the promise will return the Response body instead of the Response object. + + @default false + */ + get resolveBodyOnly(): boolean { + return this._internals.resolveBodyOnly; + } + + set resolveBodyOnly(value: boolean) { + assert.boolean(value); + + this._internals.resolveBodyOnly = value; + } + + /** + Returns a `Stream` instead of a `Promise`. + This is equivalent to calling `got.stream(url, options?)`. + + @default false + */ + get isStream(): boolean { + return this._internals.isStream; + } + + set isStream(value: boolean) { + assert.boolean(value); + + this._internals.isStream = value; + } + + /** + The parsing method. + + The promise also has `.text()`, `.json()` and `.buffer()` methods which return another Got promise for the parsed body. + + It's like setting the options to `{responseType: 'json', resolveBodyOnly: true}` but without affecting the main Got promise. + + __Note__: When using streams, this option is ignored. + + @example + ``` + (async () => { + const responsePromise = got(url); + const bufferPromise = responsePromise.buffer(); + const jsonPromise = responsePromise.json(); + + const [response, buffer, json] = Promise.all([responsePromise, bufferPromise, jsonPromise]); + // `response` is an instance of Got Response + // `buffer` is an instance of Buffer + // `json` is an object + })(); + ``` + + @example + ``` + // This + const body = await got(url).json(); + + // is semantically the same as this + const body = await got(url, {responseType: 'json', resolveBodyOnly: true}); + ``` + */ + get responseType(): ResponseType { + return this._internals.responseType; + } + + set responseType(value: ResponseType) { + if (value === undefined) { + this._internals.responseType = 'text'; + return; + } + + if (value !== 'text' && value !== 'buffer' && value !== 'json') { + throw new Error(`Invalid \`responseType\` option: ${value as string}`); + } + + this._internals.responseType = value; + } + + get pagination(): PaginationOptions { + return this._internals.pagination; + } + + set pagination(value: PaginationOptions) { + assert.object(value); + + if (this._merging) { + Object.assign(this._internals.pagination, value); + } else { + this._internals.pagination = value; + } + } + + get auth() { + throw new Error('Parameter `auth` is deprecated. Use `username` / `password` instead.'); + } + + set auth(_value: unknown) { + throw new Error('Parameter `auth` is deprecated. Use `username` / `password` instead.'); + } + + get setHost() { + return this._internals.setHost; + } + + set setHost(value: boolean) { + assert.boolean(value); + + this._internals.setHost = value; + } + + get maxHeaderSize() { + return this._internals.maxHeaderSize; + } + + set maxHeaderSize(value: number | undefined) { + assert.any([is.number, is.undefined], value); + + this._internals.maxHeaderSize = value; + } + + toJSON() { + return {...this._internals}; + } + + [Symbol.for('nodejs.util.inspect.custom')](_depth: number, options: InspectOptions) { + return inspect(this._internals, options); + } + + createNativeRequestOptions() { + const internals = this._internals; + const url = internals.url as URL; + + let agent; + if (url.protocol === 'https:') { + agent = internals.http2 ? internals.agent : internals.agent.https; + } else { + agent = internals.agent.http; + } + + const {httpsOptions} = internals; + + return { + ...internals.cacheOptions, + ...this._unixOptions, + ca: httpsOptions.certificateAuthority, + cert: httpsOptions.certificate, + key: httpsOptions.key, + passphrase: httpsOptions.passphrase, + pfx: httpsOptions.pfx, + rejectUnauthorized: httpsOptions.rejectUnauthorized, + checkServerIdentity: httpsOptions.checkServerIdentity ?? checkServerIdentity, + lookup: internals.dnsLookup ?? (internals.dnsCache as CacheableLookup | undefined)?.lookup, + family: internals.dnsLookupIpVersion, + agent, + setHost: internals.setHost, + method: internals.method, + maxHeaderSize: internals.maxHeaderSize, + localAddress: internals.localAddress, + headers: internals.headers, + createConnection: internals.createConnection + }; + } + + getRequestFunction() { + const url = this._internals.url as (URL | undefined); + const {request} = this._internals; + + if (!request && url) { + return this.getFallbackRequestFunction(); + } + + return request!; + } + + getFallbackRequestFunction() { + const url = this._internals.url as (URL | undefined); + + if (url!.protocol === 'https:') { + if (this._internals.http2) { + if (major < 15 || (major === 15 && minor < 10)) { + throw new Error('To use the `http2` option, install Node.js 15.10.0 or above'); + } + + return http2wrapper.auto as RequestFunction; + } + + return httpsRequest; + } + + return httpRequest; + } + + freeze() { + const options = this._internals; + + Object.freeze(options); + Object.freeze(options.hooks); + Object.freeze(options.httpsOptions); + Object.freeze(options.cacheOptions); + Object.freeze(options.agent); + Object.freeze(options.headers); + Object.freeze(options.timeout); + Object.freeze(options.retry); + Object.freeze(options.hooks); + Object.freeze(options.context); + } +} + +// It's user responsibility to make sensitive data in `context` non-enumerable +const nonEnumerableProperties = new Set([ + // Functions + 'constructor', + 'merge', + 'tryMerge', + 'createNativeRequestOptions', + 'getRequestFunction', + 'getFallbackRequestFunction', + 'freeze', + + // Payload + 'body', + 'form', + 'json', + + // Getters that always throw + 'auth', + 'followRedirects', + + // May contain sensitive data + 'username', + 'password', + 'headers', + 'searchParameters', + 'url', + + // Privates + '_unixOptions', + '_internals', + '_merging', + '_init' +]); + +// We want all the properties to be enumerable, so people instead doing +// `util.inspect(options, {getters: true, showHidden: true})` +// can do just `util.inspect(options, {getters: true})`. +const propertyDescriptors: PropertyDescriptorMap = {}; +const keys = Object.getOwnPropertyNames(Options.prototype).filter(property => !nonEnumerableProperties.has(property)); +const makeEnumerable = {enumerable: true}; + +for (const key of keys) { + propertyDescriptors[key] = makeEnumerable; +} + +Object.defineProperties(Options.prototype, propertyDescriptors); diff --git a/source/core/parse-link-header.ts b/source/core/parse-link-header.ts new file mode 100644 index 000000000..e8d206a7f --- /dev/null +++ b/source/core/parse-link-header.ts @@ -0,0 +1,44 @@ +export default function parseLinkHeader(link: string) { + const parsed = []; + + const items = link.split(','); + + for (const item of items) { + // https://tools.ietf.org/html/rfc5988#section-5 + const [rawUriReference, ...rawLinkParameters] = item.split(';') as [string, ...string[]]; + const trimmedUriReference = rawUriReference.trim(); + + // eslint-disable-next-line @typescript-eslint/prefer-string-starts-ends-with + if (trimmedUriReference[0] !== '<' || trimmedUriReference[trimmedUriReference.length - 1] !== '>') { + throw new Error(`Invalid format of the Link header reference: ${trimmedUriReference}`); + } + + const reference = trimmedUriReference.slice(1, -1); + const parameters: Record = {}; + + if (rawLinkParameters.length === 0) { + throw new Error(`Unexpected end of Link header parameters: ${rawLinkParameters.join(';')}`); + } + + for (const rawParameter of rawLinkParameters) { + const trimmedRawParameter = rawParameter.trim(); + const center = trimmedRawParameter.indexOf('='); + + if (center === -1) { + throw new Error(`Failed to parse Link header: ${link}`); + } + + const name = trimmedRawParameter.slice(0, center).trim(); + const value = trimmedRawParameter.slice(center + 1).trim(); + + parameters[name] = value; + } + + parsed.push({ + reference, + parameters + }); + } + + return parsed; +} diff --git a/source/core/response.ts b/source/core/response.ts new file mode 100644 index 000000000..c5c10adda --- /dev/null +++ b/source/core/response.ts @@ -0,0 +1,152 @@ +import {RequestError} from './errors'; +import type {IncomingMessageWithTimings, Timings} from '@szmarczak/http-timer'; +import type Request from '.'; +import type {ParseJsonFunction, ResponseType} from './options'; + +export interface PlainResponse extends IncomingMessageWithTimings { + /** + The original request URL. + */ + requestUrl: URL; + + /** + The redirect URLs. + */ + redirectUrls: URL[]; + + /** + - `options` - The Got options that were set on this request. + + __Note__: This is not a [http.ClientRequest](https://nodejs.org/api/http.html#http_class_http_clientrequest). + */ + request: Request; + + /** + The remote IP address. + + This is hopefully a temporary limitation, see [lukechilds/cacheable-request#86](https://github.com/lukechilds/cacheable-request/issues/86). + + __Note__: Not available when the response is cached. + */ + ip?: string; + + /** + Whether the response was retrieved from the cache. + */ + isFromCache: boolean; + + /** + The status code of the response. + */ + statusCode: number; + + /** + The request URL or the final URL after redirects. + */ + url: string; + + /** + The object contains the following properties: + + - `start` - Time when the request started. + - `socket` - Time when a socket was assigned to the request. + - `lookup` - Time when the DNS lookup finished. + - `connect` - Time when the socket successfully connected. + - `secureConnect` - Time when the socket securely connected. + - `upload` - Time when the request finished uploading. + - `response` - Time when the request fired `response` event. + - `end` - Time when the response fired `end` event. + - `error` - Time when the request fired `error` event. + - `abort` - Time when the request fired `abort` event. + - `phases` + - `wait` - `timings.socket - timings.start` + - `dns` - `timings.lookup - timings.socket` + - `tcp` - `timings.connect - timings.lookup` + - `tls` - `timings.secureConnect - timings.connect` + - `request` - `timings.upload - (timings.secureConnect || timings.connect)` + - `firstByte` - `timings.response - timings.upload` + - `download` - `timings.end - timings.response` + - `total` - `(timings.end || timings.error || timings.abort) - timings.start` + + If something has not been measured yet, it will be `undefined`. + + __Note__: The time is a `number` representing the milliseconds elapsed since the UNIX epoch. + */ + timings: Timings; + + /** + The number of times the request was retried. + */ + retryCount: number; + + // Defined only if request errored + /** + The raw result of the request. + */ + rawBody?: Buffer; + + /** + The result of the request. + */ + body?: unknown; +} + +// For Promise support +export interface Response extends PlainResponse { + /** + The result of the request. + */ + body: T; + + /** + The raw result of the request. + */ + rawBody: Buffer; +} + +export const isResponseOk = (response: PlainResponse): boolean => { + const {statusCode} = response; + const limitStatusCode = response.request.options.followRedirect ? 299 : 399; + + return (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; +}; + +/** +An error to be thrown when server response code is 2xx, and parsing body fails. +Includes a `response` property. +*/ +export class ParseError extends RequestError { + declare readonly response: Response; + + constructor(error: Error, response: Response) { + const {options} = response.request; + + super(`${error.message} in "${options.url!.toString()}"`, error, response.request); + this.name = 'ParseError'; + } +} + +export const parseBody = (response: Response, responseType: ResponseType, parseJson: ParseJsonFunction, encoding?: BufferEncoding): unknown => { + const {rawBody} = response; + + try { + if (responseType === 'text') { + return rawBody.toString(encoding); + } + + if (responseType === 'json') { + return rawBody.length === 0 ? '' : parseJson(rawBody.toString()); + } + + if (responseType === 'buffer') { + return rawBody; + } + + throw new ParseError({ + message: `Unknown body type '${responseType as string}'`, + name: 'Error' + }, response); + } catch (error) { + throw new ParseError(error, response); + } +}; diff --git a/source/core/utils/timed-out.ts b/source/core/timed-out.ts similarity index 99% rename from source/core/utils/timed-out.ts rename to source/core/timed-out.ts index 8b4ddb167..c5a9c8f1d 100644 --- a/source/core/utils/timed-out.ts +++ b/source/core/timed-out.ts @@ -1,6 +1,6 @@ import * as net from 'net'; import {ClientRequest, IncomingMessage} from 'http'; -import unhandler from './unhandle'; +import unhandler from './utils/unhandle'; const reentry: unique symbol = Symbol('reentry'); const noop = (): void => {}; diff --git a/source/core/utils/dns-ip-version.ts b/source/core/utils/dns-ip-version.ts deleted file mode 100644 index 7ea331aaf..000000000 --- a/source/core/utils/dns-ip-version.ts +++ /dev/null @@ -1,20 +0,0 @@ -export type DnsLookupIpVersion = 'auto' | 'ipv4' | 'ipv6'; -type DnsIpFamily = 0 | 4 | 6; - -const conversionTable = { - auto: 0, - ipv4: 4, - ipv6: 6 -}; - -export const isDnsLookupIpVersion = (value: any): boolean => { - return value in conversionTable; -}; - -export const dnsLookupIpVersionToFamily = (dnsLookupIpVersion: DnsLookupIpVersion): DnsIpFamily => { - if (isDnsLookupIpVersion(dnsLookupIpVersion)) { - return conversionTable[dnsLookupIpVersion] as DnsIpFamily; - } - - throw new Error('Invalid DNS lookup IP version'); -}; diff --git a/source/core/utils/get-buffer.ts b/source/core/utils/get-buffer.ts deleted file mode 100644 index d62a2202c..000000000 --- a/source/core/utils/get-buffer.ts +++ /dev/null @@ -1,21 +0,0 @@ -import {Readable} from 'stream'; - -// TODO: Update https://github.com/sindresorhus/get-stream - -const getBuffer = async (stream: Readable) => { - const chunks = []; - let length = 0; - - for await (const chunk of stream) { - chunks.push(chunk); - length += Buffer.byteLength(chunk); - } - - if (Buffer.isBuffer(chunks[0])) { - return Buffer.concat(chunks, length); - } - - return Buffer.from(chunks.join('')); -}; - -export default getBuffer; diff --git a/source/core/utils/is-client-request.ts b/source/core/utils/is-client-request.ts new file mode 100644 index 000000000..85eb454b0 --- /dev/null +++ b/source/core/utils/is-client-request.ts @@ -0,0 +1,8 @@ +import type {Writable, Readable} from 'stream'; +import type {ClientRequest} from 'http'; + +function isClientRequest(clientRequest: Writable | Readable): clientRequest is ClientRequest { + return (clientRequest as Writable).writable && !(clientRequest as Writable).writableEnded; +} + +export default isClientRequest; diff --git a/source/core/utils/is-response-ok.ts b/source/core/utils/is-response-ok.ts deleted file mode 100644 index 866745f29..000000000 --- a/source/core/utils/is-response-ok.ts +++ /dev/null @@ -1,8 +0,0 @@ -import {Response} from '../index'; - -export const isResponseOk = (response: Response): boolean => { - const {statusCode} = response; - const limitStatusCode = response.request.options.followRedirect ? 299 : 399; - - return (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304; -}; diff --git a/source/create.ts b/source/create.ts index 0da5e64af..ea132a6d7 100644 --- a/source/create.ts +++ b/source/create.ts @@ -1,34 +1,11 @@ import {URL} from 'url'; -import is from '@sindresorhus/is'; -import asPromise, { - // Response - Response, - - // Options - Options, - NormalizedOptions, - - // Hooks - InitHook, - - // Errors - ParseError, - RequestError, - CacheError, - ReadError, - HTTPError, - MaxRedirectsError, - TimeoutError, - UnsupportedProtocolError, - UploadError, - CancelError -} from './as-promise'; +import is, {assert} from '@sindresorhus/is'; +import asPromise from './as-promise'; import { GotReturn, ExtendOptions, Got, HTTPAlias, - HandlerFunction, InstanceDefaults, GotPaginate, GotStream, @@ -36,42 +13,16 @@ import { OptionsWithPagination, StreamOptions } from './types'; -import createRejection from './as-promise/create-rejection'; -import Request, {kIsNormalizedAlready, setNonEnumerableProperties, Defaults} from './core/index'; -import deepFreeze from './utils/deep-freeze'; - -const errors = { - RequestError, - CacheError, - ReadError, - HTTPError, - MaxRedirectsError, - TimeoutError, - ParseError, - CancelError, - UnsupportedProtocolError, - UploadError -}; +import Request from './core/index'; +import {Response} from './core/response'; +import Options, {OptionsInit} from './core/options'; +import type {CancelableRequest} from './as-promise/types'; // The `delay` package weighs 10KB (!) const delay = async (ms: number) => new Promise(resolve => { setTimeout(resolve, ms); }); -const {normalizeArguments} = Request; - -const mergeOptions = (...sources: Options[]): NormalizedOptions => { - let mergedOptions: NormalizedOptions | undefined; - - for (const source of sources) { - mergedOptions = normalizeArguments(undefined, source, mergedOptions); - } - - return mergedOptions!; -}; - -const getPromiseOrStream = (options: NormalizedOptions): GotReturn => options.isStream ? new Request(undefined, options) : asPromise(options); - const isGotInstance = (value: Got | ExtendOptions): value is Got => ( 'defaults' in value && 'options' in value.defaults ); @@ -85,145 +36,117 @@ const aliases: readonly HTTPAlias[] = [ 'delete' ]; -export const defaultHandler: HandlerFunction = (options, next) => next(options); - -const callInitHooks = (hooks: InitHook[] | undefined, options?: Options): void => { - if (hooks) { - for (const hook of hooks) { - hook(options!); - } - } -}; - const create = (defaults: InstanceDefaults): Got => { - // Proxy properties from next handlers - defaults._rawHandlers = defaults.handlers; - defaults.handlers = defaults.handlers.map(fn => ((options, next) => { - // This will be assigned by assigning result - let root!: ReturnType; - - const result = fn(options, newOptions => { - root = next(newOptions); - return root; - }); + defaults = { + options: new Options(undefined, undefined, defaults.options), + handlers: [...defaults.handlers], + mutableDefaults: defaults.mutableDefaults + }; - if (result !== root && !options.isStream && root) { - const typedResult = result as Promise; + Object.defineProperty(defaults, 'mutableDefaults', { + enumerable: true, + configurable: false, + writable: false + }); - const {then: promiseThen, catch: promiseCatch, finally: promiseFianlly} = typedResult; - Object.setPrototypeOf(typedResult, Object.getPrototypeOf(root)); - Object.defineProperties(typedResult, Object.getOwnPropertyDescriptors(root)); + // Got interface + const got: Got = ((url: string | URL | OptionsInit | undefined, options?: OptionsInit, defaultOptions: Options = defaults.options as Options): GotReturn => { + const request = new Request(url, options, defaultOptions); + let promise: CancelableRequest | undefined; + + const lastHandler = (normalized: Options): GotReturn => { + // Note: `options` is `undefined` when `new Options(...)` fails + request.options = normalized; + request._noPipe = !normalized.isStream; + void request.flush(); + + if (normalized.isStream) { + return request; + } - // These should point to the new promise - // eslint-disable-next-line promise/prefer-await-to-then - typedResult.then = promiseThen; - typedResult.catch = promiseCatch; - typedResult.finally = promiseFianlly; - } + if (!promise) { + promise = asPromise(request); + } - return result; - })); + return promise; + }; - // Got interface - const got: Got = ((url: string | URL, options: Options = {}, _defaults?: Defaults): GotReturn => { let iteration = 0; - const iterateHandlers = (newOptions: NormalizedOptions): GotReturn => { - // TODO: Remove the `!`. This could probably be simplified to not use index access. - return defaults.handlers[iteration++]!( - newOptions, - iteration === defaults.handlers.length ? getPromiseOrStream : iterateHandlers - ) as GotReturn; - }; + const iterateHandlers = (newOptions: Options): GotReturn => { + const handler = defaults.handlers[iteration++] ?? lastHandler; - // TODO: Throw an error in Got 12. - // TODO: Remove this in Got 13. - if (is.plainObject(url)) { - const mergedOptions = { - ...url as Options, - ...options - }; + const result = handler(newOptions, iterateHandlers) as GotReturn; - setNonEnumerableProperties([url as Options, options], mergedOptions); + if (is.promise(result) && !request.options.isStream) { + if (!promise) { + promise = asPromise(request); + } - options = mergedOptions; - url = undefined as any; - } + if (result !== promise) { + const descriptors = Object.getOwnPropertyDescriptors(promise); - try { - // Call `init` hooks - let initHookError: Error | undefined; - try { - callInitHooks(defaults.options.hooks.init, options); - callInitHooks(options.hooks?.init, options); - } catch (error) { - initHookError = error; - } + for (const key in descriptors) { + if (key in result) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete descriptors[key]; + } + } - // Normalize options & call handlers - const normalizedOptions = normalizeArguments(url, options, _defaults ?? defaults.options); - normalizedOptions[kIsNormalizedAlready] = true; + Object.defineProperties(result, descriptors); - if (initHookError) { - throw new RequestError(initHookError.message, initHookError, normalizedOptions); + result.cancel = promise.cancel; + } } - return iterateHandlers(normalizedOptions); - } catch (error) { - if (options.isStream) { - throw error; - } else { - return createRejection(error, defaults.options.hooks.beforeError, options.hooks?.beforeError); - } - } + return result; + }; + + return iterateHandlers(request.options); }) as Got; got.extend = (...instancesOrOptions) => { - const optionsArray: Options[] = [defaults.options]; - let handlers: HandlerFunction[] = [...defaults._rawHandlers!]; - let isMutableDefaults: boolean | undefined; + const options = new Options(undefined, undefined, defaults.options); + const handlers = [...defaults.handlers]; + + let mutableDefaults: boolean | undefined; for (const value of instancesOrOptions) { if (isGotInstance(value)) { - optionsArray.push(value.defaults.options); - handlers.push(...value.defaults._rawHandlers!); - isMutableDefaults = value.defaults.mutableDefaults; + options.merge(value.defaults.options); + handlers.push(...value.defaults.handlers); + mutableDefaults = value.defaults.mutableDefaults; } else { - optionsArray.push(value); + options.merge(value); - if ('handlers' in value) { - handlers.push(...value.handlers!); + if (value.handlers) { + handlers.push(...value.handlers); } - isMutableDefaults = value.mutableDefaults; + mutableDefaults = value.mutableDefaults; } } - handlers = handlers.filter(handler => handler !== defaultHandler); - - if (handlers.length === 0) { - handlers.push(defaultHandler); - } - return create({ - options: mergeOptions(...optionsArray), + options, handlers, - mutableDefaults: Boolean(isMutableDefaults) + mutableDefaults: Boolean(mutableDefaults) }); }; // Pagination const paginateEach = (async function * (url: string | URL, options?: OptionsWithPagination): AsyncIterableIterator { - // TODO: Remove this `@ts-expect-error` when upgrading to TypeScript 4. - // Error: Argument of type 'Merge> | undefined' is not assignable to parameter of type 'Options | undefined'. - // @ts-expect-error - let normalizedOptions = normalizeArguments(url, options, defaults.options); + let normalizedOptions = new Options(url, options as OptionsInit, defaults.options); normalizedOptions.resolveBodyOnly = false; - const pagination = normalizedOptions.pagination!; + const {pagination} = normalizedOptions; - if (!is.object(pagination)) { - throw new TypeError('`options.pagination` must be implemented'); - } + assert.function_(pagination.transform); + assert.function_(pagination.shouldContinue); + assert.function_(pagination.filter); + assert.function_(pagination.paginate); + assert.number(pagination.countLimit); + assert.number(pagination.requestLimit); + assert.number(pagination.backoff); const allItems: T[] = []; let {countLimit} = pagination; @@ -235,8 +158,6 @@ const create = (defaults: InstanceDefaults): Got => { await delay(pagination.backoff); } - // @ts-expect-error FIXME! - // TODO: Throw when response is not an instance of Response // eslint-disable-next-line no-await-in-loop const response = (await got(undefined, undefined, normalizedOptions)) as Response; @@ -276,8 +197,13 @@ const create = (defaults: InstanceDefaults): Got => { if (optionsToMerge === response.request.options) { normalizedOptions = response.request.options; - } else if (optionsToMerge !== undefined) { - normalizedOptions = normalizeArguments(undefined, optionsToMerge, normalizedOptions); + } else { + normalizedOptions.merge(optionsToMerge); + + if (optionsToMerge.url !== undefined) { + normalizedOptions.prefixUrl = ''; + normalizedOptions.url = optionsToMerge.url; + } } numberOfRequests++; @@ -311,18 +237,19 @@ const create = (defaults: InstanceDefaults): Got => { }) as GotStream; } - Object.assign(got, errors); + if (!defaults.mutableDefaults) { + Object.freeze(defaults.handlers); + (defaults.options as Options).freeze(); + } + Object.defineProperty(got, 'defaults', { - value: defaults.mutableDefaults ? defaults : deepFreeze(defaults), - writable: defaults.mutableDefaults, - configurable: defaults.mutableDefaults, + value: defaults, + writable: false, + configurable: false, enumerable: true }); - got.mergeOptions = mergeOptions; - return got; }; export default create; -export * from './types'; diff --git a/source/index.ts b/source/index.ts index 79334b4f3..092b7ac1a 100644 --- a/source/index.ts +++ b/source/index.ts @@ -1,136 +1,25 @@ -import {URL} from 'url'; -import {Response, Options} from './as-promise'; -import create, {defaultHandler, InstanceDefaults} from './create'; +import create from './create'; +import {InstanceDefaults} from './types'; +import Options from './core/options'; const defaults: InstanceDefaults = { - options: { - method: 'GET', - retry: { - limit: 2, - methods: [ - 'GET', - 'PUT', - 'HEAD', - 'DELETE', - 'OPTIONS', - 'TRACE' - ], - statusCodes: [ - 408, - 413, - 429, - 500, - 502, - 503, - 504, - 521, - 522, - 524 - ], - errorCodes: [ - 'ETIMEDOUT', - 'ECONNRESET', - 'EADDRINUSE', - 'ECONNREFUSED', - 'EPIPE', - 'ENOTFOUND', - 'ENETUNREACH', - 'EAI_AGAIN' - ], - maxRetryAfter: undefined, - calculateDelay: ({computedValue}) => computedValue - }, - timeout: {}, - headers: { - 'user-agent': 'got (https://github.com/sindresorhus/got)' - }, - hooks: { - init: [], - beforeRequest: [], - beforeRedirect: [], - beforeRetry: [], - beforeError: [], - afterResponse: [] - }, - cache: undefined, - dnsCache: undefined, - decompress: true, - throwHttpErrors: true, - followRedirect: true, - isStream: false, - responseType: 'text', - resolveBodyOnly: false, - maxRedirects: 10, - prefixUrl: '', - methodRewriting: false, - ignoreInvalidCookies: false, - context: {}, - // TODO: Set this to `true` for Got 13. - http2: false, - allowGetBody: false, - https: undefined, - pagination: { - transform: (response: Response) => { - if (response.request.options.responseType === 'json') { - return response.body; - } - - return JSON.parse(response.body as string); - }, - paginate: ({response}) => { - if (!Reflect.has(response.headers, 'link')) { - return false; - } - - const items = (response.headers.link as string).split(','); - - let next: string | undefined; - for (const item of items) { - // TODO: Give these more semantic names. - const [first, second] = item.split(';'); - - if (first && second?.includes('next')) { - next = first - .trim() - .slice(1, -1); - - break; - } - } - - if (next) { - const options: Options = { - url: new URL(next) - }; - - return options; - } - - return false; - }, - filter: () => true, - shouldContinue: () => true, - countLimit: Number.POSITIVE_INFINITY, - backoff: 0, - requestLimit: 10000, - stackAllItems: false - }, - parseJson: (text: string) => JSON.parse(text), - stringifyJson: (object: unknown) => JSON.stringify(object), - cacheOptions: {} - }, - handlers: [defaultHandler], + options: new Options(), + handlers: [], mutableDefaults: false }; const got = create(defaults); export default got; - -// For CommonJS default export support -module.exports = got; -module.exports.default = got; -module.exports.__esModule = true; // Workaround for TS issue: https://github.com/sindresorhus/got/pull/1267 - -export * from './create'; -export * from './as-promise'; +export {got}; + +export {default as Options} from './core/options'; +export * from './core/options'; +export * from './core/response'; +export * from './core/index'; +export * from './core/errors'; +export {default as calculateRetryDelay} from './core/calculate-retry-delay'; +export * from './as-promise/types'; +export * from './types'; +export {default as create} from './create'; +export {default as parseLinkHeader} from './core/parse-link-header'; diff --git a/source/types.ts b/source/types.ts index 8e62cf87c..89e6ab939 100644 --- a/source/types.ts +++ b/source/types.ts @@ -1,28 +1,11 @@ -import {URL} from 'url'; -import {CancelError} from 'p-cancelable'; -import { - // Request & Response - CancelableRequest, - Response, - - // Options - Options, - NormalizedOptions, - Defaults as DefaultOptions, - PaginationOptions, - - // Errors - ParseError, - RequestError, - CacheError, - ReadError, - HTTPError, - MaxRedirectsError, - TimeoutError, - UnsupportedProtocolError, - UploadError -} from './as-promise'; -import Request from './core/index'; +import type {URL} from 'url'; +import type {CancelableRequest} from './as-promise/types'; +import type {Response} from './core/response'; +// eslint-disable-next-line import/no-duplicates +import type Options from './core/options'; +// eslint-disable-next-line import/no-duplicates +import type {PaginationOptions, OptionsInit, InternalsType} from './core/options'; +import type Request from './core/index'; // `type-fest` utilities type Except = Pick>; @@ -33,9 +16,9 @@ Defaults for each Got instance. */ export interface InstanceDefaults { /** - An object containing the default options of Got. + An object containing the default options of Got. */ - options: DefaultOptions; + options: InternalsType | Options; /** An array of functions. You execute them directly by calling `got()`. @@ -53,8 +36,6 @@ export interface InstanceDefaults { @default false */ mutableDefaults: boolean; - - _rawHandlers?: HandlerFunction[]; } /** @@ -66,12 +47,12 @@ export type GotReturn = Request | CancelableRequest; A function to handle options and returns a Request object. It acts sort of like a "global hook", and will be called before any actual request is made. */ -export type HandlerFunction = (options: NormalizedOptions, next: (options: NormalizedOptions) => T) => T | Promise; +export type HandlerFunction = (options: Options, next: (options: Options) => T) => T | Promise; /** The options available for `got.extend()`. */ -export interface ExtendOptions extends Options { +export interface ExtendOptions extends OptionsInit { /** An array of functions. You execute them directly by calling `got()`. They are some sort of "global hooks" - these functions are called first. @@ -90,15 +71,15 @@ export interface ExtendOptions extends Options { mutableDefaults?: boolean; } -export type OptionsOfTextResponseBody = Merge; -export type OptionsOfJSONResponseBody = Merge; -export type OptionsOfBufferResponseBody = Merge; -export type OptionsOfUnknownResponseBody = Merge; -export type StrictOptions = Except; -export type StreamOptions = Merge; +export type OptionsOfTextResponseBody = Merge; +export type OptionsOfJSONResponseBody = Merge; +export type OptionsOfBufferResponseBody = Merge; +export type OptionsOfUnknownResponseBody = Merge; +export type StrictOptions = Except; +export type StreamOptions = Merge; type ResponseBodyOnly = {resolveBodyOnly: true}; -export type OptionsWithPagination = Merge>; +export type OptionsWithPagination = Merge}>; /** An instance of `got.paginate`. @@ -184,14 +165,17 @@ export interface GotRequestFunction { (options: (Merge)): CancelableRequest; // `asStream` usage - (url: string | URL, options?: Merge): Request; + (url: string | URL, options?: Merge): Request; - (options: Merge): Request; + (options: Merge): Request; // Fallback - (url: string | URL, options?: Options): CancelableRequest | Request; + (url: string | URL, options?: OptionsInit): CancelableRequest | Request; + + (options: OptionsInit): CancelableRequest | Request; - (options: Options): CancelableRequest | Request; + // Internal usage + (url: undefined, options: undefined, defaults: Options): CancelableRequest | Request; } /** @@ -205,10 +189,9 @@ export type HTTPAlias = | 'head' | 'delete'; -interface GotStreamFunction { - (url: string | URL, options?: Merge): Request; - (options?: Merge): Request; -} +type GotStreamFunction = + ((url?: string | URL, options?: Merge) => Request) & + ((options?: Merge) => Request); /** An instance of `got.stream()`. @@ -261,62 +244,6 @@ export interface Got extends Record, GotRequestFu */ defaults: InstanceDefaults; - /** - An error to be thrown when a cache method fails. - For example, if the database goes down or there's a filesystem error. - */ - CacheError: typeof CacheError; - - /** - An error to be thrown when a request fails. - Contains a `code` property with error class code, like `ECONNREFUSED`. - */ - RequestError: typeof RequestError; - - /** - An error to be thrown when reading from response stream fails. - */ - ReadError: typeof ReadError; - - /** - An error to be thrown when server response code is 2xx, and parsing body fails. - Includes a `response` property. - */ - ParseError: typeof ParseError; - - /** - An error to be thrown when the server response code is not 2xx nor 3xx if `options.followRedirect` is `true`, but always except for 304. - Includes a `response` property. - */ - HTTPError: typeof HTTPError; - - /** - An error to be thrown when the server redirects you more than ten times. - Includes a `response` property. - */ - MaxRedirectsError: typeof MaxRedirectsError; - - /** - An error to be thrown when given an unsupported protocol. - */ - UnsupportedProtocolError: typeof UnsupportedProtocolError; - - /** - An error to be thrown when the request is aborted due to a timeout. - Includes an `event` and `timings` property. - */ - TimeoutError: typeof TimeoutError; - - /** - An error to be thrown when the request body is a stream and an error occurs while reading from that stream. - */ - UploadError: typeof UploadError; - - /** - An error to be thrown when the request is aborted with `.cancel()`. - */ - CancelError: typeof CancelError; - /** Configure a new `got` instance with default `options`. The `options` are merged with the parent instance's `defaults.options` using `got.mergeOptions`. @@ -346,44 +273,4 @@ export interface Got extends Record, GotRequestFu ``` */ extend: (...instancesOrOptions: Array) => Got; - - /** - Merges multiple `got` instances into the parent. - */ - mergeInstances: (parent: Got, ...instances: Got[]) => Got; - - /** - Extends parent options. - Avoid using [object spread](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_syntax#Spread_in_object_literals) as it doesn't work recursively. - - Options are deeply merged to a new object. The value of each key is determined as follows: - - - If the new property is not defined, the old value is used. - - If the new property is explicitly set to `undefined`: - - If the parent property is a plain `object`, the parent value is deeply cloned. - - Otherwise, `undefined` is used. - - If the parent value is an instance of `URLSearchParams`: - - If the new value is a `string`, an `object` or an instance of `URLSearchParams`, a new `URLSearchParams` instance is created. - The values are merged using [`urlSearchParams.append(key, value)`](https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams/append). - The keys defined in the new value override the keys defined in the parent value. - - Otherwise, the only available value is `undefined`. - - If the new property is a plain `object`: - - If the parent property is a plain `object` too, both values are merged recursively into a new `object`. - - Otherwise, only the new value is deeply cloned. - - If the new property is an `Array`, it overwrites the old one with a deep clone of the new property. - - Properties that are not enumerable, such as `body`, `json`, and `form`, will not be merged. - - Otherwise, the new value is assigned to the key. - - **Note:** Only Got options are merged! Custom user options should be defined via [`options.context`](#context). - - @example - ``` - const a = {headers: {cat: 'meow', wolf: ['bark', 'wrrr']}}; - const b = {headers: {cow: 'moo', wolf: ['auuu']}}; - - {...a, ...b} // => {headers: {cow: 'moo', wolf: ['auuu']}} - got.mergeOptions(a, b) // => {headers: {cat: 'meow', cow: 'moo', wolf: ['auuu']}} - ``` - */ - mergeOptions: (...sources: Options[]) => NormalizedOptions; } diff --git a/source/utils/deep-freeze.ts b/source/utils/deep-freeze.ts deleted file mode 100644 index 2b1f45b0d..000000000 --- a/source/utils/deep-freeze.ts +++ /dev/null @@ -1,11 +0,0 @@ -import is from '@sindresorhus/is'; - -export default function deepFreeze>(object: T): Readonly { - for (const value of Object.values(object)) { - if (is.plainObject(value) || is.array(value)) { - deepFreeze(value); - } - } - - return Object.freeze(object); -} diff --git a/source/utils/deprecation-warning.ts b/source/utils/deprecation-warning.ts deleted file mode 100644 index 928f0a2ff..000000000 --- a/source/utils/deprecation-warning.ts +++ /dev/null @@ -1,14 +0,0 @@ -const alreadyWarned: Set = new Set(); - -export default (message: string) => { - if (alreadyWarned.has(message)) { - return; - } - - alreadyWarned.add(message); - - // @ts-expect-error Missing types. - process.emitWarning(`Got: ${message}`, { - type: 'DeprecationWarning' - }); -}; diff --git a/test/agent.ts b/test/agent.ts index 3085398be..49fdd4244 100644 --- a/test/agent.ts +++ b/test/agent.ts @@ -20,7 +20,7 @@ test('non-object agent option works with http', withServer, async (t, server, go const {agent, spy} = createAgentSpy(HttpAgent); t.truthy((await got({ - https: { + httpsOptions: { rejectUnauthorized: false }, agent: { @@ -41,7 +41,7 @@ test('non-object agent option works with https', withHttpsServer(), async (t, se const {agent, spy} = createAgentSpy(HttpsAgent); t.truthy((await got({ - https: { + httpsOptions: { rejectUnauthorized: false }, agent: { @@ -172,7 +172,6 @@ test('socket connect listener cleaned up after request', withHttpsServer(), asyn }); const {body} = await got({ - prefixUrl: 'http://127.0.0.1:3000', agent: { http: agent }, @@ -198,7 +197,9 @@ test('socket connect listener cleaned up after request', withHttpsServer(), asyn ] }, // Disable automatic retries, manual retries are allowed - retry: 0 + retry: { + limit: 0 + } }); t.is(body, 'ok'); diff --git a/test/arguments.ts b/test/arguments.ts index 8c8ca181c..061b6d7b7 100644 --- a/test/arguments.ts +++ b/test/arguments.ts @@ -2,7 +2,7 @@ import {parse, URL, URLSearchParams} from 'url'; import test from 'ava'; import {Handler} from 'express'; import * as pEvent from 'p-event'; -import got, {StrictOptions} from '../source/index'; +import got, {Options, StrictOptions} from '../source/index'; import withServer, {withBodyParsingServer} from './helpers/with-server'; const echoUrl: Handler = (request, response) => { @@ -21,7 +21,7 @@ test('`url` is required', async t => { await t.throwsAsync( got(''), { - message: 'No URL protocol specified' + message: 'Invalid URL: ' } ); @@ -30,7 +30,7 @@ test('`url` is required', async t => { url: '' }), { - message: 'No URL protocol specified' + message: 'Invalid URL: ' } ); }); @@ -51,15 +51,16 @@ test('throws if no arguments provided', async t => { }); }); +test('throws if the url option is missing', async t => { + await t.throwsAsync(got({}), { + message: 'Missing `url` property' + }); +}); + test('throws an error if the protocol is not specified', async t => { await t.throwsAsync(got('example.com'), { - instanceOf: TypeError, message: 'Invalid URL: example.com' }); - - await t.throwsAsync(got({}), { - message: 'Missing `url` property' - }); }); test('properly encodes query string', withServer, async (t, server, got) => { @@ -96,45 +97,33 @@ test('methods are normalized', withServer, async (t, server, got) => { await instance('test', {method: 'post'}); }); -test.failing('throws an error when legacy URL is passed', withServer, async (t, server) => { +test('throws an error when legacy URL is passed', withServer, async (t, server) => { server.get('/test', echoUrl); await t.throwsAsync( // @ts-expect-error Error tests - got(parse(`${server.url}/test`)), - {message: 'The legacy `url.Url` has been deprecated. Use `URL` instead.'} + got(parse(`${server.url}/test`)) ); + // TODO: Assert message above. + await t.throwsAsync( got({ protocol: 'http:', hostname: 'localhost', port: server.port - }), - {message: 'The legacy `url.Url` has been deprecated. Use `URL` instead.'} + } as any), + {message: 'Unexpected option: protocol'} ); }); -test('accepts legacy URL options', withServer, async (t, server) => { - server.get('/test', echoUrl); - - const {body: secondBody} = await got({ - protocol: 'http:', - hostname: 'localhost', - port: server.port, - pathname: '/test' - }); - - t.is(secondBody, '/test'); -}); - test('overrides `searchParams` from options', withServer, async (t, server, got) => { server.get('/', echoUrl); const {body} = await got( '?drop=this', { - searchParams: { + searchParameters: { test: 'wow' } } @@ -147,7 +136,7 @@ test('does not duplicate `searchParams`', withServer, async (t, server, got) => server.get('/', echoUrl); const instance = got.extend({ - searchParams: new URLSearchParams({foo: '123'}) + searchParameters: new URLSearchParams({foo: '123'}) }); const body = await instance('?bar=456').text(); @@ -159,7 +148,7 @@ test('escapes `searchParams` parameter values', withServer, async (t, server, go server.get('/', echoUrl); const {body} = await got({ - searchParams: { + searchParameters: { test: 'it’s ok' } }); @@ -171,14 +160,14 @@ test('the `searchParams` option can be a URLSearchParams', withServer, async (t, server.get('/', echoUrl); const searchParameters = new URLSearchParams({test: 'wow'}); - const {body} = await got({searchParams: searchParameters}); + const {body} = await got({searchParameters}); t.is(body, '/?test=wow'); }); test('ignores empty searchParams object', withServer, async (t, server, got) => { server.get('/test', echoUrl); - t.is((await got('test', {searchParams: {}})).requestUrl, `${server.url}/test`); + t.is((await got('test', {searchParameters: {}})).requestUrl.toString(), `${server.url}/test`); }); test('throws when passing body with a non payload method', async t => { @@ -190,11 +179,10 @@ test('throws when passing body with a non payload method', async t => { test('`allowGetBody` option', withServer, async (t, server, got) => { server.get('/test', echoUrl); - const url = new URL(`${server.url}/test`); - await t.notThrowsAsync(got(url, {body: 'asdf', allowGetBody: true})); + await t.notThrowsAsync(got('test', {body: 'asdf', allowGetBody: true})); }); -test('WHATWG URL support', withServer, async (t, server, got) => { +test('WHATWG URL support', withServer, async (t, server) => { server.get('/test', echoUrl); const url = new URL(`${server.url}/test`); @@ -222,41 +210,42 @@ test('can omit `url` option if using `prefixUrl`', withServer, async (t, server, await t.notThrowsAsync(got({})); }); -test('throws TypeError when `options.hooks` is not an object', async t => { +test('throws when `options.hooks` is not an object', async t => { await t.throwsAsync( // @ts-expect-error Error tests got('https://example.com', {hooks: 'not object'}), { - message: 'Expected value which is `predicate returns truthy for any value`, received value of type `Array`.' + message: 'Expected value which is `Object`, received value of type `string`.' } ); }); -test('throws TypeError when known `options.hooks` value is not an array', async t => { +test('throws when known `options.hooks` value is not an array', async t => { await t.throwsAsync( // @ts-expect-error Error tests - got('https://example.com', {hooks: {beforeRequest: {}}}), - { - message: 'Parameter `beforeRequest` must be an Array, got Object' - } + got('https://example.com', {hooks: {beforeRequest: {}}}) ); + + // TODO: Assert message above. }); -test('throws TypeError when known `options.hooks` array item is not a function', async t => { +test('throws when known `options.hooks` array item is not a function', async t => { await t.throwsAsync( // @ts-expect-error Error tests got('https://example.com', {hooks: {beforeRequest: [{}]}}), { - message: 'hook is not a function' + message: 'Expected value which is `Function`, received value of type `Object`.' } ); }); -test('allows extra keys in `options.hooks`', withServer, async (t, server, got) => { +test('does not allow extra keys in `options.hooks`', withServer, async (t, server, got) => { server.get('/test', echoUrl); - // @ts-expect-error We do not allow extra keys in hooks but this won't throw - await t.notThrowsAsync(got('test', {hooks: {extra: []}})); + // @ts-expect-error Error tests + await t.throwsAsync(got('test', {hooks: {extra: []}}), { + message: 'Unexpected hook event: extra' + }); }); test('`prefixUrl` option works', withServer, async (t, server, got) => { @@ -300,34 +289,18 @@ test('`prefixUrl` can be changed if the URL contains the old one', withServer, a t.is(body, '/'); }); -test('throws if cannot change `prefixUrl`', async t => { - const instanceA = got.extend({ - prefixUrl: 'https://example.com', - handlers: [ - (options, next) => { - options.url = new URL('https://google.pl'); - options.prefixUrl = 'https://example.com'; - return next(options); - } - ] - }); - - await t.throwsAsync(instanceA(''), {message: 'Cannot change `prefixUrl` from https://example.com/ to https://example.com: https://google.pl/'}); -}); - -test('throws if the `searchParams` value is invalid', async t => { +test('throws if the `searchParameters` value is invalid', async t => { await t.throwsAsync(got('https://example.com', { - searchParams: { + searchParameters: { // @ts-expect-error Error tests foo: [] } - }), { - instanceOf: TypeError, - message: 'The `searchParams` value \'\' must be a string, number, boolean or null' - }); + })); + + // TODO: Assert message above. }); -test('`context` option is enumerable', withServer, async (t, server, got) => { +test.failing('`context` option is enumerable', withServer, async (t, server, got) => { server.get('/', echoUrl); const context = { @@ -347,20 +320,20 @@ test('`context` option is enumerable', withServer, async (t, server, got) => { }); }); -test('`context` option is accessible when using hooks', withServer, async (t, server, got) => { +test('`context` option is accessible when using hooks', withServer, async (t, server) => { server.get('/', echoUrl); const context = { foo: 'bar' }; - await got({ + await got(server.url, { context, hooks: { beforeRequest: [ options => { t.deepEqual(options.context, context); - t.true({}.propertyIsEnumerable.call(options, 'context')); + t.false({}.propertyIsEnumerable.call(options, 'context')); } ] } @@ -375,7 +348,7 @@ test('`context` option is accessible when extending instances', t => { const instance = got.extend({context}); t.deepEqual(instance.defaults.options.context, context); - t.true({}.propertyIsEnumerable.call(instance.defaults.options, 'context')); + t.false({}.propertyIsEnumerable.call(instance.defaults.options, 'context')); }); test('`context` option is shallow merged', t => { @@ -390,7 +363,7 @@ test('`context` option is shallow merged', t => { const instance1 = got.extend({context}); t.deepEqual(instance1.defaults.options.context, context); - t.true({}.propertyIsEnumerable.call(instance1.defaults.options, 'context')); + t.false({}.propertyIsEnumerable.call(instance1.defaults.options, 'context')); const instance2 = instance1.extend({context: context2}); @@ -419,13 +392,15 @@ test('throws a helpful error when passing `followRedirects`', async t => { test('merges `searchParams` instances', t => { const instance = got.extend({ - searchParams: new URLSearchParams('a=1') + searchParameters: new URLSearchParams('a=1') }, { - searchParams: new URLSearchParams('b=2') + searchParameters: new URLSearchParams('b=2') }); - t.is(instance.defaults.options.searchParams!.get('a'), '1'); - t.is(instance.defaults.options.searchParams!.get('b'), '2'); + const searchParameters = instance.defaults.options.searchParameters as URLSearchParams; + + t.is(searchParameters.get('a'), '1'); + t.is(searchParameters.get('b'), '2'); }); test('throws a helpful error when passing `auth`', async t => { @@ -439,7 +414,7 @@ test('throws a helpful error when passing `auth`', async t => { test('throws on leading slashes', async t => { await t.throwsAsync(got('/asdf', {prefixUrl: 'https://example.com'}), { - message: '`input` must not start with a slash when using `prefixUrl`' + message: '`url` must not start with a slash' }); }); @@ -447,7 +422,9 @@ test('throws on invalid `dnsCache` option', async t => { await t.throwsAsync(got('https://example.com', { // @ts-expect-error Error tests dnsCache: 123 - }), {message: 'Parameter `dnsCache` must be a CacheableLookup instance or a boolean, got number'}); + })); + + // TODO: Assert message above. }); test('throws on invalid `agent` option', async t => { @@ -456,7 +433,7 @@ test('throws on invalid `agent` option', async t => { // @ts-expect-error Error tests asdf: 123 } - }), {message: 'Expected the `options.agent` properties to be `http`, `https` or `http2`, got `asdf`'}); + }), {message: 'Unexpected agent option: asdf'}); }); test('fallbacks to native http if `request(...)` returns undefined', withServer, async (t, server, got) => { @@ -490,20 +467,20 @@ test('does not throw on frozen options', withServer, async (t, server, got) => { }); test('encodes query string included in input', t => { - const {url} = got.mergeOptions({ + const {url} = new Options({ url: new URL('https://example.com/?a=b c') }); - t.is(url.search, '?a=b%20c'); + t.is(url!.search, '?a=b%20c'); }); test('normalizes search params included in options', t => { - const {url} = got.mergeOptions({ + const {url} = new Options({ url: new URL('https://example.com'), - searchParams: 'a=b c' + searchParameters: 'a=b c' }); - t.is(url.search, '?a=b+c'); + t.is(url!.search, '?a=b+c'); }); test('reuse options while using init hook', withServer, async (t, server, got) => { @@ -537,7 +514,9 @@ test('allowGetBody sends json payload', withBodyParsingServer, async (t, server, const {statusCode} = await got({ allowGetBody: true, json: {hello: 'world'}, - retry: 0, + retry: { + limit: 0 + }, throwHttpErrors: false }); t.is(statusCode, 200); @@ -552,7 +531,7 @@ test('no URL pollution', withServer, async (t, server) => { hooks: { beforeRequest: [ options => { - options.url.pathname = '/ok'; + (options.url as URL).pathname = '/ok'; } ] } diff --git a/test/cache.ts b/test/cache.ts index c21b46d77..cccf244a8 100644 --- a/test/cache.ts +++ b/test/cache.ts @@ -6,7 +6,7 @@ import * as getStream from 'get-stream'; import {Handler} from 'express'; import CacheableLookup from 'cacheable-lookup'; import * as delay from 'delay'; -import got, {Response} from '../source/index'; +import got, {CacheError, Response} from '../source/index'; import withServer from './helpers/with-server'; const cacheEndpoint: Handler = (_request, response) => { @@ -112,7 +112,7 @@ test('cached response has got options', withServer, async (t, server, got) => { t.is(secondResponse.request.options.username, options.username); }); -test('cache error throws `got.CacheError`', withServer, async (t, server, got) => { +test('cache error throws `CacheError`', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok'); }); @@ -120,7 +120,7 @@ test('cache error throws `got.CacheError`', withServer, async (t, server, got) = const cache = {}; // @ts-expect-error Error tests - await t.throwsAsync(got({cache}), {instanceOf: got.CacheError}); + await t.throwsAsync(got({cache}), {instanceOf: CacheError}); }); test('doesn\'t cache response when received HTTP error', withServer, async (t, server, got) => { @@ -256,7 +256,9 @@ test('decompresses cached responses', withServer, async (t, server, got) => { cache, responseType: 'json', decompress: true, - retry: 2 + retry: { + limit: 2 + } })); } diff --git a/test/cancel.ts b/test/cancel.ts index a030675bb..afc7637f9 100644 --- a/test/cancel.ts +++ b/test/cancel.ts @@ -6,7 +6,7 @@ import * as delay from 'delay'; import * as pEvent from 'p-event'; import * as getStream from 'get-stream'; import {Handler} from 'express'; -import got, {CancelError} from '../source/index'; +import got, {CancelError, TimeoutError} from '../source/index'; import slowDataStream from './helpers/slow-data-stream'; import {GlobalClock} from './helpers/types'; import {ExtendedHttpTestServer} from './helpers/create-http-test-server'; @@ -47,7 +47,7 @@ const prepareServer = (server: ExtendedHttpTestServer, clock: GlobalClock): {emi return {emitter, promise}; }; -const downloadHandler = (clock: GlobalClock): Handler => (_request, response) => { +const downloadHandler = (clock?: GlobalClock): Handler => (_request, response) => { response.writeHead(200, { 'transfer-encoding': 'chunked' }); @@ -87,7 +87,9 @@ test.serial('cleans up request timeouts', withServer, async (t, server, got) => server.get('/', () => {}); const gotPromise = got({ - timeout: 10, + timeout: { + request: 10 + }, retry: { calculateDelay: ({computedValue}) => { process.nextTick(() => { @@ -138,7 +140,7 @@ test.serial('cancels in-progress request with timeout', withServerAndFakeTimers, }); body.push('1'); - const gotPromise = got.post('abort', {body, timeout: 10000}); + const gotPromise = got.post('abort', {body, timeout: {request: 10000}}); // Wait for the connection to be established before canceling emitter.once('connection', () => { @@ -190,7 +192,7 @@ test('recover from cancellation using error instance', async t => { // Canceled before connection started const p = got('http://example.com'); const recover = p.catch((error: Error) => { - if (error instanceof got.CancelError) { + if (error instanceof CancelError) { return; } @@ -208,21 +210,25 @@ test.serial('throws on incomplete (canceled) response - promise', withServerAndF await t.throwsAsync( got({ timeout: {request: 500}, - retry: 0 + retry: { + limit: 0 + } }), - {instanceOf: got.TimeoutError} + {instanceOf: TimeoutError} ); }); -test.serial('throws on incomplete (canceled) response - promise #2', withServerAndFakeTimers, async (t, server, got, clock) => { - server.get('/', downloadHandler(clock)); +// TODO: Use `fakeTimers` here +test.serial('throws on incomplete (canceled) response - promise #2', withServer, async (t, server, got) => { + server.get('/', downloadHandler()); - const promise = got('').on('response', () => { - clock.tick(500); + const promise = got(''); + + setTimeout(() => { promise.cancel(); - }); + }, 500); - await t.throwsAsync(promise, {instanceOf: got.CancelError}); + await t.throwsAsync(promise, {instanceOf: CancelError}); }); test.serial('throws on incomplete (canceled) response - stream', withServerAndFakeTimers, async (t, server, got, clock) => { @@ -238,7 +244,6 @@ test.serial('throws on incomplete (canceled) response - stream', withServerAndFa await t.throwsAsync(getStream(stream), {message: errorString}); }); -// Note: it will throw, but the response is loaded already. test('throws when canceling cached request', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.setHeader('Cache-Control', 'public, max-age=60'); @@ -248,24 +253,8 @@ test('throws when canceling cached request', withServer, async (t, server, got) const cache = new Map(); await got({cache}); - const promise = got({cache}).on('response', () => { - promise.cancel(); - }); - - await t.throwsAsync(promise, {instanceOf: got.CancelError}); -}); - -test('throws when canceling cached request #2', withServer, async (t, server, got) => { - server.get('/', (_request, response) => { - response.setHeader('Cache-Control', 'public, max-age=60'); - response.end(Date.now().toString()); - }); - - const cache = new Map(); - await got({cache}); - const promise = got({cache}); promise.cancel(); - await t.throwsAsync(promise, {instanceOf: got.CancelError}); + await t.throwsAsync(promise, {instanceOf: CancelError}); }); diff --git a/test/create.ts b/test/create.ts index 9107ad6d8..e0c3dc5ac 100644 --- a/test/create.ts +++ b/test/create.ts @@ -7,6 +7,8 @@ import got, { BeforeRequestHook, Headers, Hooks, + Options, + OptionsInit, RequestFunction } from '../source/index'; import withServer from './helpers/with-server'; @@ -81,21 +83,16 @@ test('custom headers (extend)', withServer, async (t, server, got) => { }); test('extend overwrites arrays with a deep clone', t => { - const beforeRequest = [0]; + const x = () => {}; + const y = () => {}; + + const beforeRequest = [x]; const a = got.extend({hooks: {beforeRequest} as unknown as Hooks}); - beforeRequest[0] = 1; - t.deepEqual(a.defaults.options.hooks.beforeRequest, [0] as unknown as BeforeRequestHook[]); + beforeRequest[0] = y; + t.deepEqual(a.defaults.options.hooks.beforeRequest, [x] as unknown as BeforeRequestHook[]); t.not(a.defaults.options.hooks.beforeRequest, beforeRequest as unknown as BeforeRequestHook[]); }); -test('extend keeps the old value if the new one is undefined', t => { - const a = got.extend({headers: undefined}); - t.deepEqual( - a.defaults.options.headers, - got.defaults.options.headers - ); -}); - test('hooks are merged on got.extend()', t => { const hooksA = [() => {}]; const hooksB = [() => {}]; @@ -123,25 +120,25 @@ test('no tampering with defaults', t => { t.is(got.defaults.options.prefixUrl, ''); }); -test('can set defaults to `got.mergeOptions(...)`', t => { +test('can set defaults to `new Options(...)`', t => { const instance = got.extend({ mutableDefaults: true, followRedirect: false }); t.notThrows(() => { - instance.defaults.options = got.mergeOptions(instance.defaults.options, { - followRedirect: true - }); + instance.defaults.options = new Options({ + followRedirect: false + }, undefined, instance.defaults.options); }); - t.true(instance.defaults.options.followRedirect); + t.false(instance.defaults.options.followRedirect); t.notThrows(() => { - instance.defaults.options = got.mergeOptions({}); + instance.defaults.options = new Options({}); }); - t.is(instance.defaults.options.followRedirect, undefined); + t.true(instance.defaults.options.followRedirect); }); test('can set mutable defaults using got.extend', t => { @@ -172,18 +169,21 @@ test('only plain objects are freezed', withServer, async (t, server, got) => { }); }); -test('defaults are cloned on instance creation', t => { - const options = {foo: 'bar', hooks: {beforeRequest: [() => {}]}}; +// eslint-disable-next-line ava/no-skip-test +test.skip('defaults are cloned on instance creation', t => { + const options: OptionsInit = {hooks: {beforeRequest: [() => {}]}}; const instance = got.extend(options); + const context = { + foo: {} + }; t.notThrows(() => { - options.foo = 'foo'; - delete options.hooks.beforeRequest[0]; + options.context = context; + delete options.hooks!.beforeRequest![0]; }); - // @ts-expect-error This IS correct - t.not(options.foo, instance.defaults.options.foo); - t.not(options.hooks.beforeRequest, instance.defaults.options.hooks.beforeRequest); + t.not(options.context!.foo, instance.defaults.options.context.foo); + t.not(options.hooks!.beforeRequest, instance.defaults.options.hooks.beforeRequest); }); test('ability to pass a custom request method', withServer, async (t, server, got) => { @@ -236,17 +236,23 @@ test('should pass an options object into an initialization hook after .extend', server.get('/', echoHeaders); + let first = true; + const instance = got.extend({ hooks: { init: [ options => { - t.deepEqual(options, {}); + if (!first) { + t.deepEqual(options, {}); + } + + first = false; } ] } }); - await instance(''); + await instance('', {}); }); test('hooks aren\'t overriden when merging options', withServer, async (t, server, got) => { @@ -301,12 +307,18 @@ test('async handlers', withServer, async (t, server, got) => { const instance = got.extend({ handlers: [ - async (options, next) => { - const result = await next(options); - // @ts-expect-error Manual tests - result.modified = true; + (options, next) => { + if (options.isStream) { + return next(options); + } + + return (async () => { + const result = await next(options); + // @ts-expect-error Manual tests + result.modified = true; - return result; + return result; + })(); } ] }); diff --git a/test/error.ts b/test/error.ts index 93e9672bc..4073185fa 100644 --- a/test/error.ts +++ b/test/error.ts @@ -1,3 +1,4 @@ +import {URL} from 'url'; import {promisify} from 'util'; import * as net from 'net'; import * as http from 'http'; @@ -7,6 +8,7 @@ import * as getStream from 'get-stream'; import is from '@sindresorhus/is'; import got, {RequestError, HTTPError, TimeoutError} from '../source/index'; import withServer from './helpers/with-server'; +import Request from '../source/core'; const pStreamPipeline = promisify(stream.pipeline); @@ -22,12 +24,8 @@ test('properties', withServer, async (t, server, got) => { t.truthy(error); t.truthy(error.response); t.truthy(error.options); - t.false({}.propertyIsEnumerable.call(error, 'options')); + t.true({}.propertyIsEnumerable.call(error, 'options')); t.false({}.propertyIsEnumerable.call(error, 'response')); - // This fails because of TS 3.7.2 useDefineForClassFields - // Class fields will always be initialized, even though they are undefined - // A test to check for undefined is in place below - // t.false({}.hasOwnProperty.call(error, 'code')); t.is(error.code, undefined); t.is(error.message, 'Response code 404 (Not Found)'); t.deepEqual(error.options.url, url); @@ -36,18 +34,18 @@ test('properties', withServer, async (t, server, got) => { }); test('catches dns errors', async t => { - const error = await t.throwsAsync(got('http://doesntexist', {retry: 0})); + const error = await t.throwsAsync(got('http://doesntexist', {retry: {limit: 0}})); t.truthy(error); t.regex(error.message, /ENOTFOUND|EAI_AGAIN/); - t.is(error.options.url.host, 'doesntexist'); + t.is((error.options.url as URL).host, 'doesntexist'); t.is(error.options.method, 'GET'); }); test('`options.body` form error message', async t => { // @ts-expect-error Error tests - await t.throwsAsync(got.post('https://example.com', {body: Buffer.from('test'), form: ''}), { - message: 'The `body`, `json` and `form` options are mutually exclusive' - }); + await t.throwsAsync(got.post('https://example.com', {body: Buffer.from('test'), form: ''}) + // {message: 'The `body`, `json` and `form` options are mutually exclusive'} + ); }); test('no plain object restriction on json body', withServer, async (t, server, got) => { @@ -104,12 +102,15 @@ test('contains Got options', withServer, async (t, server, got) => { response.end(); }); - const options: {agent: false} = { - agent: false - }; + const options = { + context: { + foo: 'bar' + } + } as const; - const error = await t.throwsAsync(got(options)); - t.is(error.options.agent, options.agent); + const error = await t.throwsAsync(got(options)); + t.is(error.response.statusCode, 404); + t.is(error.options.context.foo, options.context.foo); }); test('empty status message is overriden by the default one', withServer, async (t, server, got) => { @@ -129,7 +130,7 @@ test('`http.request` error', async t => { throw new TypeError('The header content contains invalid characters'); } }), { - instanceOf: got.RequestError, + instanceOf: RequestError, message: 'The header content contains invalid characters' }); }); @@ -163,7 +164,7 @@ test('`http.request` pipe error', async t => { }, throwHttpErrors: false }), { - instanceOf: got.RequestError, + instanceOf: RequestError, message }); }); @@ -175,23 +176,28 @@ test('`http.request` error through CacheableRequest', async t => { }, cache: new Map() }), { - instanceOf: got.RequestError, + instanceOf: RequestError, message: 'The header content contains invalid characters' }); }); -test('errors are thrown directly when options.isStream is true', t => { - t.throws(() => { - // @ts-expect-error Error tests - void got('https://example.com', {isStream: true, hooks: false}); - }, { - message: 'Expected value which is `predicate returns truthy for any value`, received value of type `Array`.' +test('returns a stream even if normalization fails', async t => { + const stream = got('https://example.com', { + isStream: true, + // @ts-expect-error + hooks: false + }) as unknown as Request; + + await t.throwsAsync(getStream(stream), { + message: 'Expected value which is `Object`, received value of type `boolean`.' }); }); test('normalization errors using convenience methods', async t => { const url = 'undefined/https://example.com'; - await t.throwsAsync(got(url).json().text().buffer(), {message: `Invalid URL: ${url}`}); + await t.throwsAsync(got(url).json(), {message: `Invalid URL: ${url}`}); + await t.throwsAsync(got(url).text(), {message: `Invalid URL: ${url}`}); + await t.throwsAsync(got(url).buffer(), {message: `Invalid URL: ${url}`}); }); test('errors can have request property', withServer, async (t, server, got) => { @@ -213,7 +219,9 @@ test('promise does not hang on timeout on HTTP error', withServer, async (t, ser }); await t.throwsAsync(got({ - timeout: 100 + timeout: { + request: 100 + } }), { instanceOf: TimeoutError }); diff --git a/test/headers.ts b/test/headers.ts index d4a5b1518..5e4a2e6f9 100644 --- a/test/headers.ts +++ b/test/headers.ts @@ -199,7 +199,7 @@ test('throws on null value headers', async t => { await t.throwsAsync(got({ url: 'https://example.com', headers: { - // @ts-expect-error Testing purposes + // @ts-expect-error For testing purposes 'user-agent': null } }), { diff --git a/test/helpers.ts b/test/helpers.ts index 1ed05e181..efb92b171 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -17,5 +17,5 @@ test('works', withServer, async (t, server) => { const error = await t.throwsAsync(got.get(`${server.url}/404`), {instanceOf: HTTPError}); t.is(error.response.body, 'not found'); - await t.throwsAsync(got.get('.com', {retry: 0}), {message: 'Invalid URL: .com'}); + await t.throwsAsync(got.get('.com', {retry: {limit: 0}}), {message: 'Invalid URL: .com'}); }); diff --git a/test/helpers/slow-data-stream.ts b/test/helpers/slow-data-stream.ts index 3f8c592e8..2af45076b 100644 --- a/test/helpers/slow-data-stream.ts +++ b/test/helpers/slow-data-stream.ts @@ -1,14 +1,20 @@ import {Readable} from 'stream'; import {Clock} from '@sinonjs/fake-timers'; +import delay = require('delay'); -export default (clock: Clock): Readable => { +export default (clock?: Clock): Readable => { let i = 0; return new Readable({ - read() { + async read() { + if (clock) { + clock.tick(100); + } else { + await delay(100); + } + if (i++ < 10) { this.push('data\n'.repeat(100)); - clock.tick(100); } else { this.push(null); } diff --git a/test/helpers/with-server.ts b/test/helpers/with-server.ts index fe50ddb54..7ca72a1f8 100644 --- a/test/helpers/with-server.ts +++ b/test/helpers/with-server.ts @@ -6,7 +6,7 @@ import * as tempy from 'tempy'; import createHttpsTestServer, {ExtendedHttpsTestServer, HttpsServerOptions} from './create-https-test-server'; import createHttpTestServer, {ExtendedHttpTestServer, HttpServerOptions} from './create-http-test-server'; import * as FakeTimers from '@sinonjs/fake-timers'; -import got, {InstanceDefaults, Got} from '../../source/index'; +import got, {Got, ExtendOptions} from '../../source/index'; import {ExtendedHttpServer, GlobalClock, InstalledClock} from './types'; export type RunTestWithServer = (t: test.ExecutionContext, server: ExtendedHttpTestServer, got: Got, clock: GlobalClock) => Promise | void; @@ -23,9 +23,10 @@ const generateHook = ({install, options: testServerOptions}: {install?: boolean; } as any }); - const options: InstanceDefaults = { - // @ts-expect-error Augmenting for test detection - avaTest: t.title, + const options: ExtendOptions = { + context: { + avaTest: t.title + }, handlers: [ (options, next) => { const result = next(options); @@ -66,8 +67,9 @@ const generateHttpsHook = (options?: HttpsServerOptions, installFakeTimer = fals const server = await createHttpsTestServer(options); const preparedGot = got.extend({ - // @ts-expect-error Augmenting for test detection - avaTest: t.title, + context: { + avaTest: t.title + }, handlers: [ (options, next) => { const result = next(options); @@ -83,7 +85,7 @@ const generateHttpsHook = (options?: HttpsServerOptions, installFakeTimer = fals } ], prefixUrl: server.url, - https: { + httpsOptions: { certificateAuthority: (server as any).caCert, rejectUnauthorized: true } @@ -112,8 +114,8 @@ export const withSocketServer: test.Macro<[RunTestWithSocket]> = async (t, run) server.socketPath = socketPath; - // @ts-expect-error - TS 4.1 bug. - await promisify(server.listen.bind(server))(socketPath); + // @ts-expect-error TypeScript doesn't accept `callback` with no arguments + await promisify(server.listen.bind(server))(socketPath); try { await run(t, server); diff --git a/test/hooks.ts b/test/hooks.ts index f7775405c..84c57dccb 100644 --- a/test/hooks.ts +++ b/test/hooks.ts @@ -81,7 +81,7 @@ test('catches init thrown errors', async t => { }); test('passes init thrown errors to beforeError hooks (promise-only)', async t => { - t.plan(2); + t.plan(1); await t.throwsAsync(got('https://example.com', { hooks: { @@ -100,21 +100,6 @@ test('passes init thrown errors to beforeError hooks (promise-only)', async t => }); }); -test('passes init thrown errors to beforeError hooks (promise-only) - beforeError rejection', async t => { - const message = 'foo, bar!'; - - await t.throwsAsync(got('https://example.com', { - hooks: { - init: [() => { - throw error; - }], - beforeError: [() => { - throw new Error(message); - }] - } - }), {message}); -}); - test('catches beforeRequest thrown errors', async t => { await t.throwsAsync(got('https://example.com', { hooks: { @@ -287,17 +272,21 @@ test('init from defaults is called with options', withServer, async (t, server, const context = {}; + let count = 0; + const instance = got.extend({ hooks: { init: [ options => { - t.is(options.context, context); + count += options.context ? 1 : 0; } ] } }); await instance({context}); + + t.is(count, 1); }); test('init allows modifications', withServer, async (t, server, got) => { @@ -326,8 +315,9 @@ test('beforeRequest is called with options', withServer, async (t, server, got) hooks: { beforeRequest: [ options => { - t.is(options.url.pathname, '/'); - t.is(options.url.hostname, 'localhost'); + const url = options.url as URL; + t.is(url.pathname, '/'); + t.is(url.hostname, 'localhost'); } ] } @@ -382,8 +372,9 @@ test('beforeRedirect is called with options and response', withServer, async (t, hooks: { beforeRedirect: [ (options, response) => { - t.is(options.url.pathname, '/'); - t.is(options.url.hostname, 'localhost'); + const url = options.url as URL; + t.is(url.pathname, '/'); + t.is(url.hostname, 'localhost'); t.is(response.statusCode, 302); t.is(new URL(response.url).pathname, '/redirect'); @@ -411,24 +402,29 @@ test('beforeRedirect allows modifications', withServer, async (t, server, got) = t.is(body.foo, 'bar'); }); -test('beforeRetry is called with options', withServer, async (t, server, got) => { +test('beforeRetry is called with options', withServer, async (t, server) => { server.get('/', echoHeaders); server.get('/retry', retryEndpoint); const context = {}; await got('retry', { + prefixUrl: server.url, responseType: 'json', - retry: 1, + retry: { + limit: 1 + }, throwHttpErrors: false, context, hooks: { beforeRetry: [ - (options, error, retryCount) => { - t.is(options.url.hostname, 'localhost'); + error => { + const {options} = error; + const {retryCount} = error.request!; + t.is((options.url as URL).hostname, 'localhost'); t.deepEqual(options.context, context); t.truthy(error); - t.true(retryCount! >= 1); + t.is(retryCount, 0); } ] } @@ -443,7 +439,7 @@ test('beforeRetry allows modifications', withServer, async (t, server, got) => { responseType: 'json', hooks: { beforeRetry: [ - options => { + ({options}) => { options.headers.foo = 'bar'; } ] @@ -476,7 +472,7 @@ test('beforeRetry allows stream body if different from original', withServer, as }, hooks: { beforeRetry: [ - options => { + ({options}) => { const form = generateBody(); options.body = form; options.headers['content-type'] = `multipart/form-data; boundary=${form.getBoundary()}`; @@ -562,9 +558,7 @@ test('afterResponse allows to retry without losing the port', withServer, async }); const {statusCode} = await got({ - protocol: 'http:', - hostname: server.hostname, - port: server.port, + url: server.url, hooks: { afterResponse: [ (response, retryWithMergedOptions) => { @@ -665,7 +659,9 @@ test('no infinity loop when retrying on afterResponse', withServer, async (t, se }); await t.throwsAsync(got({ - retry: 0, + retry: { + limit: 0 + }, hooks: { afterResponse: [ (_response, retryWithMergedOptions) => { @@ -677,7 +673,7 @@ test('no infinity loop when retrying on afterResponse', withServer, async (t, se } ] } - }), {instanceOf: got.HTTPError, message: 'Response code 401 (Unauthorized)'}); + }), {instanceOf: HTTPError, message: 'Response code 401 (Unauthorized)'}); }); test('throws on afterResponse retry failure', withServer, async (t, server, got) => { @@ -694,7 +690,9 @@ test('throws on afterResponse retry failure', withServer, async (t, server, got) }); await t.throwsAsync(got({ - retry: 1, + retry: { + limit: 1 + }, hooks: { afterResponse: [ (response, retryWithMergedOptions) => { @@ -710,10 +708,10 @@ test('throws on afterResponse retry failure', withServer, async (t, server, got) } ] } - }), {instanceOf: got.HTTPError, message: 'Response code 500 (Internal Server Error)'}); + }), {instanceOf: HTTPError, message: 'Response code 500 (Internal Server Error)'}); }); -test('doesn\'t throw on afterResponse retry HTTP failure if throwHttpErrors is false', withServer, async (t, server, got) => { +test('does not throw on afterResponse retry HTTP failure if throwHttpErrors is false', withServer, async (t, server, got) => { let didVisit401then500: boolean; server.get('/', (_request, response) => { if (didVisit401then500) { @@ -728,7 +726,9 @@ test('doesn\'t throw on afterResponse retry HTTP failure if throwHttpErrors is f const {statusCode} = await got({ throwHttpErrors: false, - retry: 1, + retry: { + limit: 1 + }, hooks: { afterResponse: [ (response, retryWithMergedOptions) => { @@ -845,9 +845,9 @@ test('does not break on `afterResponse` hook with JSON mode', withServer, async afterResponse: [ (response, retryWithMergedOptions) => { if (response.statusCode === 404) { - const url = new URL('/foobar', response.url); - - return retryWithMergedOptions({url}); + return retryWithMergedOptions({ + url: new URL('/foobar', response.url) + }); } return response; @@ -865,7 +865,7 @@ test('catches HTTPErrors', withServer, async (t, _server, got) => { hooks: { beforeError: [ error => { - t.true(error instanceof got.HTTPError); + t.true(error instanceof HTTPError); return error; } ] @@ -877,7 +877,9 @@ test('timeout can be modified using a hook', withServer, async (t, server, got) server.get('/', () => {}); await t.throwsAsync(got({ - timeout: 1000, + timeout: { + request: 1000 + }, hooks: { beforeRequest: [ options => { @@ -885,7 +887,9 @@ test('timeout can be modified using a hook', withServer, async (t, server, got) } ] }, - retry: 0 + retry: { + limit: 1 + } }), {message: 'Timeout awaiting \'request\' for 500ms'}); }); @@ -927,7 +931,9 @@ test('beforeError emits valid promise `HTTPError`s', async t => { } ] }, - retry: 0 + retry: { + limit: 0 + } }); await t.throwsAsync(instance('https://ValidHTTPErrors.com')); @@ -946,7 +952,9 @@ test('hooks are not duplicated', withServer, async (t, _server, got) => { } ] }, - retry: 0 + retry: { + limit: 0 + } }), {message: 'Response code 404 (Not Found)'}); t.is(calls, 1); @@ -975,7 +983,9 @@ test('async afterResponse allows to retry with allowGetBody and json payload', w } ] }, - retry: 0, + retry: { + limit: 0 + }, throwHttpErrors: false }); diff --git a/test/http.ts b/test/http.ts index d4d91aef3..20df96d6f 100644 --- a/test/http.ts +++ b/test/http.ts @@ -6,7 +6,7 @@ import {Handler} from 'express'; import * as nock from 'nock'; import * as getStream from 'get-stream'; import pEvent from 'p-event'; -import got, {HTTPError, UnsupportedProtocolError, CancelableRequest, ReadError} from '../source/index'; +import got, {HTTPError, ReadError, RequestError} from '../source/index'; import withServer from './helpers/with-server'; const IPv6supported = Object.values(os.networkInterfaces()).some(iface => iface?.some(addr => !addr.internal && addr.family === 'IPv6')); @@ -53,8 +53,8 @@ test('response has `requestUrl` property', withServer, async (t, server, got) => response.end(); }); - t.is((await got('')).requestUrl, `${server.url}/`); - t.is((await got('empty')).requestUrl, `${server.url}/empty`); + t.is((await got('')).requestUrl.toString(), `${server.url}/`); + t.is((await got('empty')).requestUrl.toString(), `${server.url}/empty`); }); test('http errors have `response` property', withServer, async (t, server, got) => { @@ -92,8 +92,8 @@ test('doesn\'t throw if `options.throwHttpErrors` is false', withServer, async ( test('invalid protocol throws', async t => { await t.throwsAsync(got('c:/nope.com').json(), { - instanceOf: UnsupportedProtocolError, - message: 'Unsupported protocol "c:"' + instanceOf: RequestError, + message: 'Unsupported protocol: c:' }); }); @@ -136,8 +136,8 @@ test('`searchParams` option', withServer, async (t, server, got) => { response.end('recent'); }); - t.is((await got({searchParams: {recent: true}})).body, 'recent'); - t.is((await got({searchParams: 'recent=true'})).body, 'recent'); + t.is((await got({searchParameters: {recent: true}})).body, 'recent'); + t.is((await got({searchParameters: 'recent=true'})).body, 'recent'); }); test('response contains url', withServer, async (t, server, got) => { @@ -193,7 +193,7 @@ test('socket destroyed by the server throws ECONNRESET', withServer, async (t, s request.socket.destroy(); }); - await t.throwsAsync(got('', {retry: 0}), { + await t.throwsAsync(got('', {retry: {limit: 0}}), { code: 'ECONNRESET' }); }); @@ -237,7 +237,7 @@ test('statusMessage fallback', async t => { const {statusMessage} = await got('http://statusMessageFallback', { throwHttpErrors: false, - retry: 0 + retry: {limit: 0} }); t.is(statusMessage, STATUS_CODES[503]); @@ -253,7 +253,9 @@ test('does not destroy completed requests', withServer, async (t, server, got) = agent: { http: new Agent({keepAlive: true}) }, - retry: 0 + retry: { + limit: 0 + } }; const stream = got.stream(options); @@ -294,7 +296,7 @@ test('DNS auto', withServer, async (t, server, got) => { server.get('/ok', echoIp); const response = await got('ok', { - dnsLookupIpVersion: 'auto' + dnsLookupIpVersion: undefined }); t.true(isIPv4(response.body)); @@ -304,7 +306,7 @@ test('DNS IPv4', withServer, async (t, server, got) => { server.get('/ok', echoIp); const response = await got('ok', { - dnsLookupIpVersion: 'ipv4' + dnsLookupIpVersion: 4 }); t.true(isIPv4(response.body)); @@ -315,7 +317,7 @@ testIPv6('DNS IPv6', withServer, async (t, server, got) => { server.get('/ok', echoIp); const response = await got('ok', { - dnsLookupIpVersion: 'ipv6' + dnsLookupIpVersion: 6 }); t.true(isIPv6(response.body)); @@ -329,34 +331,16 @@ test('invalid `dnsLookupIpVersion`', withServer, async (t, server, got) => { } as any)); }); -test.serial('deprecated `family` option', withServer, async (t, server, got) => { +test('deprecated `family` option', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok'); }); - await new Promise(resolve => { - let request: CancelableRequest; - (async () => { - const warning = await pEvent(process, 'warning'); - t.is(warning.name, 'DeprecationWarning'); - request!.cancel(); - resolve(); - })(); - - (async () => { - request = got({ - family: '4' - } as any); - - try { - await request; - t.fail(); - } catch { - t.true(request!.isCanceled); - } - - resolve(); - })(); + await t.throwsAsync(got({ + // @ts-expect-error + family: 4 + }), { + message: 'Unexpected option: family' }); }); diff --git a/test/https.ts b/test/https.ts index 126730681..58dfb0af9 100644 --- a/test/https.ts +++ b/test/https.ts @@ -3,7 +3,7 @@ import {DetailedPeerCertificate} from 'tls'; import pEvent from 'p-event'; import * as pify from 'pify'; import * as pem from 'pem'; -import got, {CancelableRequest} from '../source/index'; +import got from '../source/index'; import {withHttpsServer} from './helpers/with-server'; const createPrivateKey = pify(pem.createPrivateKey); @@ -17,7 +17,7 @@ test('https request without ca', withHttpsServer(), async (t, server, got) => { }); t.truthy((await got({ - https: { + httpsOptions: { certificateAuthority: [], rejectUnauthorized: false } @@ -78,7 +78,7 @@ test('https request with `checkServerIdentity` OK', withHttpsServer(), async (t, }); const {body} = await got({ - https: { + httpsOptions: { checkServerIdentity: (hostname: string, certificate: DetailedPeerCertificate) => { t.is(hostname, 'localhost'); t.is(certificate.subject.CN, 'localhost'); @@ -96,7 +96,7 @@ test('https request with `checkServerIdentity` NOT OK', withHttpsServer(), async }); const promise = got({ - https: { + httpsOptions: { checkServerIdentity: (hostname: string, certificate: DetailedPeerCertificate) => { t.is(hostname, 'localhost'); t.is(certificate.subject.CN, 'localhost'); @@ -158,7 +158,7 @@ test('http2', async t => { return; } - t.fail(error); + t.fail(error.stack); } }); @@ -167,29 +167,11 @@ test.serial('deprecated `rejectUnauthorized` option', withHttpsServer(), async ( response.end('ok'); }); - await new Promise(resolve => { - let request: CancelableRequest; - (async () => { - const warning = await pEvent(process, 'warning'); - t.is(warning.name, 'DeprecationWarning'); - request!.cancel(); - resolve(); - })(); - - (async () => { - request = got({ - rejectUnauthorized: false - }); - - try { - await request; - t.fail(); - } catch { - t.true(request!.isCanceled); - } - - resolve(); - })(); + await t.throwsAsync(got({ + // @ts-expect-error + rejectUnauthorized: false + }), { + message: 'Unexpected option: rejectUnauthorized' }); }); @@ -204,7 +186,7 @@ test.serial('non-deprecated `rejectUnauthorized` option', withHttpsServer(), asy })(); await got({ - https: { + httpsOptions: { rejectUnauthorized: false } }); @@ -212,32 +194,6 @@ test.serial('non-deprecated `rejectUnauthorized` option', withHttpsServer(), asy t.pass(); }); -test.serial('no double deprecated warning', withHttpsServer(), async (t, server, got) => { - server.get('/', (_request, response) => { - response.end('ok'); - }); - - (async () => { - const warning = await pEvent(process, 'warning'); - t.is(warning.name, 'DeprecationWarning'); - })(); - - await got({ - rejectUnauthorized: false - }); - - (async () => { - const warning = await pEvent(process, 'warning'); - t.not(warning.name, 'DeprecationWarning'); - })(); - - await got({ - rejectUnauthorized: false - }); - - t.pass(); -}); - test('client certificate', withHttpsServer(), async (t, server, got) => { server.get('/', (request, response) => { const peerCertificate = (request.socket as any).getPeerCertificate(true); @@ -261,7 +217,7 @@ test('client certificate', withHttpsServer(), async (t, server, got) => { const clientCert = clientResult.certificate; const response: any = await got({ - https: { + httpsOptions: { key: clientKey, certificate: clientCert } @@ -294,7 +250,7 @@ test('invalid client certificate (self-signed)', withHttpsServer(), async (t, se const clientCert = clientResult.certificate; const response: any = await got({ - https: { + httpsOptions: { key: clientKey, certificate: clientCert } @@ -334,7 +290,7 @@ test('invalid client certificate (other CA)', withHttpsServer(), async (t, serve const clientCert = clientResult.certificate; const response: any = await got({ - https: { + httpsOptions: { key: clientKey, certificate: clientCert } @@ -382,7 +338,7 @@ test('key passphrase', withHttpsServer(), async (t, server, got) => { const clientCert = clientResult.certificate; const response: any = await got({ - https: { + httpsOptions: { key: clientKey, passphrase: 'randomPassword', certificate: clientCert @@ -431,7 +387,7 @@ test('invalid key passphrase', withHttpsServer(), async (t, server, got) => { const clientCert = clientResult.certificate; const request = got({ - https: { + httpsOptions: { key: clientKey, passphrase: 'wrongPassword', certificate: clientCert @@ -468,7 +424,7 @@ test('client certificate PFX', withHttpsServer(), async (t, server, got) => { const {pkcs12} = await createPkcs12(clientKey, clientCert, 'randomPassword'); const response: any = await got({ - https: { + httpsOptions: { pfx: pkcs12, passphrase: 'randomPassword' } diff --git a/test/merge-instances.ts b/test/merge-instances.ts index 091f0cd48..79192658b 100644 --- a/test/merge-instances.ts +++ b/test/merge-instances.ts @@ -1,6 +1,6 @@ import test from 'ava'; import {Handler} from 'express'; -import got, {BeforeRequestHook, Got, Headers, NormalizedOptions} from '../source/index'; +import got, {BeforeRequestHook, Got, Headers} from '../source/index'; import withServer from './helpers/with-server'; const echoHeaders: Handler = (request, response) => { @@ -100,7 +100,7 @@ test('hooks are merged', t => { test('default handlers are not duplicated', t => { const instance = got.extend(got); - t.is(instance.defaults.handlers.length, 1); + t.is(instance.defaults.handlers.length, 0); }); test('URL is not polluted', withServer, async (t, server, got) => { @@ -118,40 +118,40 @@ test('URL is not polluted', withServer, async (t, server, got) => { }); test('merging instances with HTTPS options', t => { - const instanceA = got.extend({https: { + const instanceA = got.extend({httpsOptions: { rejectUnauthorized: true, certificate: 'FIRST' }}); - const instanceB = got.extend({https: { + const instanceB = got.extend({httpsOptions: { certificate: 'SECOND' }}); const merged = instanceA.extend(instanceB); - t.true(merged.defaults.options.https?.rejectUnauthorized); - t.is(merged.defaults.options.https?.certificate, 'SECOND'); + t.true(merged.defaults.options.httpsOptions.rejectUnauthorized); + t.is(merged.defaults.options.httpsOptions.certificate, 'SECOND'); }); test('merging instances with HTTPS options undefined', t => { - const instanceA = got.extend({https: { + const instanceA = got.extend({httpsOptions: { rejectUnauthorized: true, certificate: 'FIRST' }}); - const instanceB = got.extend({https: { + const instanceB = got.extend({httpsOptions: { certificate: undefined }}); const merged = instanceA.extend(instanceB); - t.true(merged.defaults.options.https?.rejectUnauthorized); - t.is(merged.defaults.options.https?.certificate, undefined); + t.true(merged.defaults.options.httpsOptions.rejectUnauthorized); + t.is(merged.defaults.options.httpsOptions.certificate, undefined); }); test('accepts options for promise API', t => { got.extend({ hooks: { beforeRequest: [ - (options: NormalizedOptions): void => { + options => { options.responseType = 'buffer'; } ] @@ -169,9 +169,8 @@ test('merging `prefixUrl`', t => { const mergedAonB = instanceB.extend(instanceA); const mergedBonA = instanceA.extend(instanceB); - t.is(mergedAonB.defaults.options.prefixUrl, ''); + t.is(mergedAonB.defaults.options.prefixUrl, prefixUrl); t.is(mergedBonA.defaults.options.prefixUrl, prefixUrl); t.is(instanceB.extend({}).defaults.options.prefixUrl, prefixUrl); - t.is(instanceB.extend({prefixUrl: undefined}).defaults.options.prefixUrl, prefixUrl); }); diff --git a/test/normalize-arguments.ts b/test/normalize-arguments.ts index 096fbdb94..a3ef8acbb 100644 --- a/test/normalize-arguments.ts +++ b/test/normalize-arguments.ts @@ -1,29 +1,31 @@ import {URL, URLSearchParams} from 'url'; import test from 'ava'; -import got from '../source/index'; +import got, {Options} from '../source/index'; test('should merge options replacing responseType', t => { const responseType = 'json'; - const options = got.mergeOptions(got.defaults.options, { + const options = new Options({ responseType - }); + }, undefined, got.defaults.options); t.is(options.responseType, responseType); }); test('no duplicated searchParams values', t => { - const options = got.mergeOptions(got.defaults.options, { - searchParams: 'string=true&noDuplication=true' + const options = new Options({ + searchParameters: 'string=true&noDuplication=true' }, { - searchParams: new URLSearchParams({ + searchParameters: new URLSearchParams({ instance: 'true', noDuplication: 'true' }) }); - t.is(options.searchParams?.get('string'), 'true'); - t.is(options.searchParams?.get('instance'), 'true'); - t.is(options.searchParams?.getAll('noDuplication').length, 1); + const searchParameters = options.searchParameters as URLSearchParams; + + t.is(searchParameters.get('string'), 'true'); + t.is(searchParameters.get('instance'), 'true'); + t.is(searchParameters.getAll('noDuplication').length, 1); }); test('should copy non-numerable properties', t => { @@ -31,37 +33,14 @@ test('should copy non-numerable properties', t => { json: {hello: '123'} }; - const merged = got.mergeOptions(got.defaults.options, options); - const mergedTwice = got.mergeOptions(got.defaults.options, merged); + const merged = new Options(options, undefined, got.defaults.options); + const mergedTwice = new Options(undefined, undefined, merged); t.is(mergedTwice.json, options.json); }); -test('should replace URLs', t => { - const options = got.mergeOptions({ - url: new URL('http://localhost:41285'), - searchParams: new URLSearchParams('page=0') - }, { - url: 'http://localhost:41285/?page=1', - searchParams: undefined - }); - - const otherOptions = got.mergeOptions({ - url: new URL('http://localhost:41285'), - searchParams: { - page: 0 - } - }, { - url: 'http://localhost:41285/?page=1', - searchParams: undefined - }); - - t.is(options.url.href, 'http://localhost:41285/?page=1'); - t.is(otherOptions.url.href, 'http://localhost:41285/?page=1'); -}); - test('should get username and password from the URL', t => { - const options = got.mergeOptions({ + const options = new Options({ url: 'http://user:pass@localhost:41285' }); @@ -70,7 +49,7 @@ test('should get username and password from the URL', t => { }); test('should get username and password from the options', t => { - const options = got.mergeOptions({ + const options = new Options({ url: 'http://user:pass@localhost:41285', username: 'user_OPT', password: 'pass_OPT' @@ -81,7 +60,7 @@ test('should get username and password from the options', t => { }); test('should get username and password from the merged options', t => { - const options = got.mergeOptions( + const options = new Options( { url: 'http://user:pass@localhost:41285' }, @@ -96,23 +75,60 @@ test('should get username and password from the merged options', t => { }); test('null value in search params means empty', t => { - const options = got.mergeOptions({ + const options = new Options({ url: new URL('http://localhost'), - searchParams: { + searchParameters: { foo: null } }); - t.is(options.url.href, 'http://localhost/?foo='); + t.is((options.url as URL).href, 'http://localhost/?foo='); }); test('undefined value in search params means it does not exist', t => { - const options = got.mergeOptions({ + const options = new Options({ url: new URL('http://localhost'), - searchParams: { + searchParameters: { foo: undefined } }); - t.is(options.url.href, 'http://localhost/'); + t.is((options.url as URL).href, 'http://localhost/'); +}); + +test('prefixUrl alone does not set url', t => { + const options = new Options({ + prefixUrl: 'https://example.com' + }); + + t.is(options.url, undefined); +}); + +test('maxRetryAfter is calculated seperately from request timeout', t => { + const options = new Options({ + timeout: { + request: 1000 + }, + retry: { + maxRetryAfter: undefined + } + }); + + t.is(options.retry.maxRetryAfter, undefined); + + options.merge({ + timeout: { + request: 2000 + } + }); + + t.is(options.retry.maxRetryAfter, undefined); + + options.merge({ + retry: { + maxRetryAfter: 300 + } + }); + + t.is(options.retry.maxRetryAfter, 300); }); diff --git a/test/pagination.ts b/test/pagination.ts index 6843d4bbb..9af8f57b6 100644 --- a/test/pagination.ts +++ b/test/pagination.ts @@ -148,12 +148,14 @@ test('custom paginate function using allItems', withServer, async (t, server, go const result = await got.paginate.all({ pagination: { - paginate: ({allItems}) => { + paginate: ({allItems, response}) => { if (allItems.length === 2) { return false; } - return {path: '/?page=3'}; + return { + url: new URL('/?page=3', response.url) + }; }, stackAllItems: true } @@ -167,12 +169,14 @@ test('custom paginate function using currentItems', withServer, async (t, server const result = await got.paginate.all({ pagination: { - paginate: ({currentItems}) => { + paginate: ({currentItems, response}) => { if (currentItems[0] === 3) { return false; } - return {path: '/?page=3'}; + return { + url: new URL('/?page=3', response.url) + }; } } }); @@ -245,27 +249,15 @@ test('`countLimit` works', withServer, async (t, server, got) => { t.deepEqual(results, [1]); }); -test('throws if no `pagination` option', async t => { - const iterator = got.extend({ - pagination: false as any - }).paginate('', { - prefixUrl: 'https://example.com' - }); - - await t.throwsAsync(iterator.next(), { - message: '`options.pagination` must be implemented' - }); -}); - test('throws if the `pagination` option does not have `transform` property', async t => { const iterator = got.paginate('', { pagination: {...resetPagination}, prefixUrl: 'https://example.com' }); - await t.throwsAsync(iterator.next(), { - message: '`options.pagination.transform` must be implemented' - }); + await t.throwsAsync(iterator.next() + // {message: '`options.pagination.transform` must be implemented'} + ); }); test('throws if the `pagination` option does not have `shouldContinue` property', async t => { @@ -277,9 +269,9 @@ test('throws if the `pagination` option does not have `shouldContinue` property' prefixUrl: 'https://example.com' }); - await t.throwsAsync(iterator.next(), { - message: '`options.pagination.shouldContinue` must be implemented' - }); + await t.throwsAsync(iterator.next() + // {message: '`options.pagination.shouldContinue` must be implemented'} + ); }); test('throws if the `pagination` option does not have `filter` property', async t => { @@ -293,9 +285,9 @@ test('throws if the `pagination` option does not have `filter` property', async prefixUrl: 'https://example.com' }); - await t.throwsAsync(iterator.next(), { - message: '`options.pagination.filter` must be implemented' - }); + await t.throwsAsync(iterator.next() + // {message: '`options.pagination.filter` must be implemented'} + ); }); test('throws if the `pagination` option does not have `paginate` property', async t => { @@ -309,9 +301,9 @@ test('throws if the `pagination` option does not have `paginate` property', asyn prefixUrl: 'https://example.com' }); - await t.throwsAsync(iterator.next(), { - message: '`options.pagination.paginate` must be implemented' - }); + await t.throwsAsync(iterator.next() + // {message: '`options.pagination.paginate` must be implemented'} + ); }); test('ignores the `resolveBodyOnly` option', withServer, async (t, server, got) => { @@ -336,7 +328,9 @@ test('allowGetBody sends json payload with .paginate()', withBodyParsingServer, const iterator = got.paginate({ allowGetBody: true, json: {hello: 'world'}, - retry: 0 + retry: { + limit: 0 + } }); const results: number[] = []; @@ -407,7 +401,9 @@ test('allowGetBody sends correct json payload with .paginate()', withServer, asy const iterator = got.paginate({ allowGetBody: true, - retry: 0, + retry: { + limit: 0 + }, json: {body}, pagination: { paginate: () => { @@ -500,7 +496,7 @@ test('`stackAllItems` set to true', withServer, async (t, server, got) => { itemCount += 1; t.is(allItems.length, itemCount); - return got.defaults.options.pagination!.paginate({response, currentItems, allItems}); + return got.defaults.options.pagination.paginate!({response, currentItems, allItems}); } } }); @@ -527,7 +523,7 @@ test('`stackAllItems` set to false', withServer, async (t, server, got) => { paginate: ({response, currentItems, allItems}) => { t.is(allItems.length, 0); - return got.defaults.options.pagination!.paginate({response, allItems, currentItems}); + return got.defaults.options.pagination.paginate!({response, allItems, currentItems}); } } }); @@ -552,7 +548,7 @@ test('next url in json response', withServer, async (t, server, got) => { } const all = await got.paginate.all('', { - searchParams: { + searchParameters: { page: 0 }, responseType: 'json', @@ -570,7 +566,7 @@ test('next url in json response', withServer, async (t, server, got) => { return { url: next, prefixUrl: '', - searchParams: undefined + searchParameters: undefined }; } } @@ -584,7 +580,7 @@ test('next url in json response', withServer, async (t, server, got) => { ]); }); -test('pagination using searchParams', withServer, async (t, server, got) => { +test('pagination using searchParameters', withServer, async (t, server, got) => { server.get('/', (request, response) => { const parameters = new URLSearchParams(request.url.slice(2)); const page = Number(parameters.get('page') ?? 0); @@ -601,7 +597,7 @@ test('pagination using searchParams', withServer, async (t, server, got) => { } const all = await got.paginate.all('', { - searchParams: { + searchParameters: { page: 0 }, responseType: 'json', @@ -611,14 +607,15 @@ test('pagination using searchParams', withServer, async (t, server, got) => { }, paginate: ({response}) => { const {next} = response.body; - const previousPage = Number(response.request.options.searchParams!.get('page')); + const searchParameters = response.request.options.searchParameters as URLSearchParams; + const previousPage = Number(searchParameters.get('page')); if (!next) { return false; } return { - searchParams: { + searchParameters: { page: previousPage + 1 } }; @@ -634,7 +631,7 @@ test('pagination using searchParams', withServer, async (t, server, got) => { ]); }); -test('pagination using extended searchParams', withServer, async (t, server, got) => { +test('pagination using extended searchParameters', withServer, async (t, server, got) => { server.get('/', (request, response) => { const parameters = new URLSearchParams(request.url.slice(2)); const page = Number(parameters.get('page') ?? 0); @@ -651,13 +648,13 @@ test('pagination using extended searchParams', withServer, async (t, server, got } const client = got.extend({ - searchParams: { + searchParameters: { limit: 10 } }); const all = await client.paginate.all('', { - searchParams: { + searchParameters: { page: 0 }, responseType: 'json', @@ -667,14 +664,15 @@ test('pagination using extended searchParams', withServer, async (t, server, got }, paginate: ({response}) => { const {next} = response.body; - const previousPage = Number(response.request.options.searchParams!.get('page')); + const searchParameters = response.request.options.searchParameters as URLSearchParams; + const previousPage = Number(searchParameters.get('page')); if (!next) { return false; } return { - searchParams: { + searchParameters: { page: previousPage + 1 } }; @@ -682,10 +680,9 @@ test('pagination using extended searchParams', withServer, async (t, server, got } }); - t.deepEqual(all, [ - '/?page=0&limit=10', - '/?page=1&limit=10', - '/?page=2&limit=10', - '/?page=3&limit=10' - ]); + t.is(all.length, 4); + + for (let i = 0; i < 4; i++) { + t.true(all[i] === `/?page=${i}&limit=10` || all[i] === `/?limit=10&page=${i}`); + } }); diff --git a/test/parse-link-header.ts b/test/parse-link-header.ts new file mode 100644 index 000000000..6514db66d --- /dev/null +++ b/test/parse-link-header.ts @@ -0,0 +1,114 @@ +import test from 'ava'; +import parseLinkHeader from '../source/core/parse-link-header'; + +test('works as expected', t => { + t.deepEqual( + parseLinkHeader( + '; rel="preconnect", ; rel="preconnect", ; rel="preconnect"' + ), + [ + { + reference: 'https://one.example.com', + parameters: {rel: '"preconnect"'} + }, + { + reference: 'https://two.example.com', + parameters: {rel: '"preconnect"'} + }, + { + reference: 'https://three.example.com', + parameters: {rel: '"preconnect"'} + } + ] + ); + + t.deepEqual( + parseLinkHeader( + '; rel="previous"; title="previous chapter"' + ), + [ + { + reference: 'https://one.example.com', + parameters: {rel: '"previous"', title: '"previous chapter"'} + } + ] + ); + + t.deepEqual( + parseLinkHeader('; rel="http://example.net/foo"'), + [ + { + reference: '/', + parameters: {rel: '"http://example.net/foo"'} + } + ] + ); + + t.deepEqual( + parseLinkHeader('; rel="copyright"; anchor="#foo"'), + [ + { + reference: '/terms', + parameters: {rel: '"copyright"', anchor: '"#foo"'} + } + ] + ); + + t.deepEqual(parseLinkHeader(`; + rel="previous"; title*=UTF-8'de'letztes%20Kapitel, + ; + rel="next"; title*=UTF-8'de'n%c3%a4chstes%20Kapitel`), [ + { + reference: '/TheBook/chapter2', + parameters: { + rel: '"previous"', + // eslint-disable-next-line @typescript-eslint/quotes + 'title*': `UTF-8'de'letztes%20Kapitel` + } + }, + { + reference: '/TheBook/chapter4', + parameters: { + rel: '"next"', + // eslint-disable-next-line @typescript-eslint/quotes + 'title*': `UTF-8'de'n%c3%a4chstes%20Kapitel` + } + } + ]); + + t.throws(() => parseLinkHeader('https://bad.example; rel="preconnect"'), { + message: 'Invalid format of the Link header reference: https://bad.example' + }); + + t.throws(() => parseLinkHeader('https://bad.example; rel'), { + message: 'Invalid format of the Link header reference: https://bad.example' + }); + + t.throws(() => parseLinkHeader('https://bad.example'), { + message: 'Invalid format of the Link header reference: https://bad.example' + }); + + t.throws(() => parseLinkHeader(''), { + message: 'Invalid format of the Link header reference: ' + }); + + t.throws(() => parseLinkHeader('; rel'), { + message: 'Failed to parse Link header: ; rel' + }); + + t.throws(() => parseLinkHeader(''), { + message: 'Unexpected end of Link header parameters: ' + }); + + t.throws(() => parseLinkHeader('<>'), { + message: 'Unexpected end of Link header parameters: ' + }); + + t.throws(() => parseLinkHeader(' parseLinkHeader('https://bad.example>'), { + message: 'Invalid format of the Link header reference: https://bad.example>' + }); +}); diff --git a/test/post.ts b/test/post.ts index bde08b645..572583521 100644 --- a/test/post.ts +++ b/test/post.ts @@ -37,10 +37,7 @@ test('GET can have body with option allowGetBody', withServer, async (t, server, test('invalid body', async t => { await t.throwsAsync( // @ts-expect-error Error tests - got.post('https://example.com', {body: {}}), - { - message: 'The `body` option must be a stream.Readable, string or Buffer' - } + got.post('https://example.com', {body: {}}) ); }); @@ -75,14 +72,12 @@ test('sends plain objects as forms', withServer, async (t, server, got) => { t.is(body, 'such=wow'); }); -test('does NOT support sending arrays as forms', withServer, async (t, server, got) => { +test('does not support sending arrays as forms', withServer, async (t, server, got) => { server.post('/', defaultEndpoint); await t.throwsAsync(got.post({ form: ['such', 'wow'] - }), { - message: 'Each query pair must be an iterable [name, value] tuple' - }); + })); }); test('sends plain objects as JSON', withServer, async (t, server, got) => { @@ -212,9 +207,7 @@ test('`content-type` header is not overriden when object in `options.body`', wit test('throws when form body is not a plain object or array', async t => { // @ts-expect-error Manual test - await t.throwsAsync(got.post('https://example.com', {form: 'such=wow'}), { - message: 'The `form` option must be an Object' - }); + await t.throwsAsync(got.post('https://example.com', {form: 'such=wow'})); }); // See https://github.com/sindresorhus/got/issues/897 @@ -238,36 +231,22 @@ test('the `body` payload is not touched', withServer, async (t, server, got) => server.post('/', defaultEndpoint); const buffer = Buffer.from('Hello, Got!'); + // @ts-expect-error + buffer.context = {foo: 'bar'}; - await got.post({ - body: buffer, - hooks: { - beforeRequest: [ - options => { - t.is(options.body, buffer); - } - ] - } - }); + const body = await got.post({body: buffer}).text(); + t.is(body, 'Hello, Got!'); }); test('the `form` payload is not touched', withServer, async (t, server, got) => { server.post('/', defaultEndpoint); - const object = { - foo: 'bar' + const form = { + context: true }; - await got.post({ - form: object, - hooks: { - beforeRequest: [ - options => { - t.is(options.form, object); - } - ] - } - }); + const body = await got.post({form}).text(); + t.is(body, 'context=true'); }); test('DELETE method sends plain objects as JSON', withServer, async (t, server, got) => { diff --git a/test/promise.ts b/test/promise.ts index bca0ea8c0..b15ec2b73 100644 --- a/test/promise.ts +++ b/test/promise.ts @@ -23,7 +23,7 @@ test('emits response event as promise', withServer, async (t, server, got) => { await got('').json().on('response', (response: Response) => { t.true(response instanceof IncomingMessage); - t.true(response.readable); + t.false(response.readable); t.is(response.statusCode, 200); t.is(response.ip, '127.0.0.1'); }); diff --git a/test/redirects.ts b/test/redirects.ts index cfe5088d9..20521feca 100644 --- a/test/redirects.ts +++ b/test/redirects.ts @@ -33,7 +33,7 @@ test('follows redirect', withServer, async (t, server, got) => { const {body, redirectUrls} = await got('finite'); t.is(body, 'reached'); - t.deepEqual(redirectUrls, [`${server.url}/`]); + t.deepEqual(redirectUrls.map(x => String(x)), [`${server.url}/`]); }); test('follows 307, 308 redirect', withServer, async (t, server, got) => { @@ -83,7 +83,7 @@ test('throws on endless redirects - default behavior', withServer, async (t, ser const error = await t.throwsAsync(got(''), {message: 'Redirected 10 times. Aborting.'}); - t.deepEqual(error.response.redirectUrls, Array.from({length: 10}).fill(`${server.url}/`)); + t.deepEqual(error.response.redirectUrls.map(x => String(x)), Array.from({length: 10}).fill(`${server.url}/`)); }); test('custom `maxRedirects` option', withServer, async (t, server, got) => { @@ -96,7 +96,7 @@ test('custom `maxRedirects` option', withServer, async (t, server, got) => { const error = await t.throwsAsync(got('', {maxRedirects: 5}), {message: 'Redirected 5 times. Aborting.'}); - t.deepEqual(error.response.redirectUrls, Array.from({length: 5}).fill(`${server.url}/`)); + t.deepEqual(error.response.redirectUrls.map(x => String(x)), Array.from({length: 5}).fill(`${server.url}/`)); }); test('searchParams are not breaking redirects', withServer, async (t, server, got) => { @@ -111,7 +111,7 @@ test('searchParams are not breaking redirects', withServer, async (t, server, go response.end(); }); - t.is((await got('relativeSearchParam', {searchParams: 'bang=1'})).body, 'reached'); + t.is((await got('relativeSearchParam', {searchParameters: 'bang=1'})).body, 'reached'); }); test('redirects GET and HEAD requests', withServer, async (t, server, got) => { @@ -123,7 +123,7 @@ test('redirects GET and HEAD requests', withServer, async (t, server, got) => { }); await t.throwsAsync(got.get(''), { - instanceOf: got.MaxRedirectsError + instanceOf: MaxRedirectsError }); }); @@ -136,7 +136,7 @@ test('redirects POST requests', withServer, async (t, server, got) => { }); await t.throwsAsync(got.post({body: 'wow'}), { - instanceOf: got.MaxRedirectsError + instanceOf: MaxRedirectsError }); }); @@ -253,7 +253,7 @@ test('redirect response contains old url', withServer, async (t, server, got) => server.get('/finite', finiteHandler); const {requestUrl} = await got('finite'); - t.is(requestUrl, `${server.url}/finite`); + t.is(requestUrl.toString(), `${server.url}/finite`); }); test('redirect response contains UTF-8 with binary encoding', withServer, async (t, server, got) => { @@ -495,12 +495,7 @@ test('correct port on redirect', withServer, async (t, server1, got) => { response.end('SERVER2'); }); - const response = await got({ - protocol: 'http:', - hostname: server1.hostname, - port: server1.port, - pathname: '/redirect' - }); + const response = await got(`${server1.url}/redirect`, {prefixUrl: ''}); t.is(response.body, 'SERVER2'); }); diff --git a/test/response-parse.ts b/test/response-parse.ts index 7a336dbaf..39e7ce388 100644 --- a/test/response-parse.ts +++ b/test/response-parse.ts @@ -87,9 +87,7 @@ test('throws an error on invalid response type', withServer, async (t, server, g // @ts-expect-error Error tests const error = await t.throwsAsync(got({responseType: 'invalid'})); - t.regex(error.message, /^Unknown body type 'invalid'/); - t.true(error.message.includes(error.options.url.hostname)); - t.is(error.options.url.pathname, '/'); + t.is(error.message, 'Invalid `responseType` option: invalid'); }); test('wraps parsing errors', withServer, async (t, server, got) => { @@ -97,9 +95,9 @@ test('wraps parsing errors', withServer, async (t, server, got) => { response.end('/'); }); - const error = await t.throwsAsync(got({responseType: 'json'}), {instanceOf: got.ParseError}); - t.true(error.message.includes(error.options.url.hostname)); - t.is(error.options.url.pathname, '/'); + const error = await t.throwsAsync(got({responseType: 'json'}), {instanceOf: ParseError}); + t.true(error.message.includes((error.options.url as URL).hostname)); + t.is((error.options.url as URL).pathname, '/'); }); test('parses non-200 responses', withServer, async (t, server, got) => { @@ -108,7 +106,7 @@ test('parses non-200 responses', withServer, async (t, server, got) => { response.end(jsonResponse); }); - const error = await t.throwsAsync(got({responseType: 'json', retry: 0}), {instanceOf: HTTPError}); + const error = await t.throwsAsync(got({responseType: 'json', retry: {limit: 0}}), {instanceOf: HTTPError}); t.deepEqual(error.response.body, dog); }); @@ -118,13 +116,13 @@ test('ignores errors on invalid non-200 responses', withServer, async (t, server response.end('Internal error'); }); - const error = await t.throwsAsync(got({responseType: 'json', retry: 0}), { - instanceOf: got.HTTPError, + const error = await t.throwsAsync(got({responseType: 'json', retry: {limit: 0}}), { + instanceOf: HTTPError, message: 'Response code 500 (Internal Server Error)' }); t.is(error.response.body, 'Internal error'); - t.is(error.options.url.pathname, '/'); + t.is((error.options.url as URL).pathname, '/'); }); test('parse errors have `response` property', withServer, async (t, server, got) => { diff --git a/test/retry.ts b/test/retry.ts index db031a3f3..1a06cbe3e 100644 --- a/test/retry.ts +++ b/test/retry.ts @@ -7,7 +7,7 @@ import is from '@sindresorhus/is'; import {Handler} from 'express'; import * as getStream from 'get-stream'; import * as pEvent from 'p-event'; -import got, {HTTPError} from '../source/index'; +import got, {HTTPError, TimeoutError} from '../source/index'; import withServer from './helpers/with-server'; const retryAfterOn413 = 2; @@ -95,7 +95,7 @@ test('setting to `0` disables retrying', async t => { return createSocketTimeoutStream(); } }), { - instanceOf: got.TimeoutError, + instanceOf: TimeoutError, message: `Timeout awaiting 'socket' for ${socketTimeout}ms` }); }); @@ -173,6 +173,10 @@ test('custom error codes', async t => { emitter.abort = () => {}; emitter.end = () => {}; emitter.destroy = () => {}; + // @ts-expect-error + emitter.writable = true; + // @ts-expect-error + emitter.writableEnded = false; const error = new Error('Snap!'); (error as Error & {code: typeof errorCode}).code = errorCode; @@ -212,7 +216,9 @@ test('respects 413 Retry-After', withServer, async (t, server, got) => { const {statusCode, body} = await got({ throwHttpErrors: false, - retry: 1 + retry: { + limit: 1 + } }); t.is(statusCode, 413); t.true(Number(body) >= retryAfterOn413 * 1000); @@ -232,7 +238,9 @@ test('respects 413 Retry-After with RFC-1123 timestamp', withServer, async (t, s const {statusCode, body} = await got({ throwHttpErrors: false, - retry: 1 + retry: { + limit: 1 + } }); t.is(statusCode, 413); t.true(Date.now() >= Date.parse(body)); @@ -288,7 +296,9 @@ test('retries on 503 without Retry-After header', withServer, async (t, server, const {retryCount} = await got({ throwHttpErrors: false, - retry: 1 + retry: { + limit: 1 + } }); t.is(retryCount, 1); }); @@ -298,9 +308,11 @@ test('doesn\'t retry on streams', withServer, async (t, server, got) => { // @ts-expect-error Error tests const stream = got.stream({ - timeout: 1, + timeout: { + request: 1 + }, retry: { - retries: () => { + calculateDelay: () => { t.fail('Retries on streams'); } } @@ -323,7 +335,9 @@ test('doesn\'t retry when set to 0', withServer, async (t, server, got) => { const {statusCode, retryCount} = await got({ throwHttpErrors: false, - retry: 0 + retry: { + limit: 0 + } }); t.is(statusCode, 413); t.is(retryCount, 0); @@ -333,7 +347,9 @@ test('works when defaults.options.retry is a number', withServer, async (t, serv server.get('/', handler413); const instance = got.extend({ - retry: 2 + retry: { + limit: 2 + } }); const {retryCount} = await instance({ @@ -359,7 +375,9 @@ test('does not retry on POST', withServer, async (t, server, got) => { server.post('/', () => {}); await t.throwsAsync(got.post({ - timeout: 200, + timeout: { + request: 200 + }, hooks: { beforeRetry: [ () => { @@ -367,7 +385,7 @@ test('does not retry on POST', withServer, async (t, server, got) => { } ] } - }), {instanceOf: got.TimeoutError}); + }), {instanceOf: TimeoutError}); }); test('does not break on redirect', withServer, async (t, server, got) => { @@ -531,3 +549,19 @@ test('promise does not retry when body is a stream', withServer, async (t, serve t.is(response.retryCount, 0); }); + +test('reuses request options on retry', withServer, async (t, server, got) => { + let first = true; + server.get('/', (request, response) => { + if (first) { + first = false; + return; + } + + response.end(JSON.stringify(request.headers)); + }); + + const {body: {accept}, retryCount} = await got('', {timeout: {request: 1000}, responseType: 'json'}); + t.is(retryCount, 1); + t.is(accept, 'application/json'); +}); diff --git a/test/stream.ts b/test/stream.ts index 214882e88..f592ea56e 100644 --- a/test/stream.ts +++ b/test/stream.ts @@ -9,7 +9,7 @@ import * as getStream from 'get-stream'; import * as pEvent from 'p-event'; import * as FormData from 'form-data'; import is from '@sindresorhus/is'; -import got, {RequestError} from '../source/index'; +import got, {HTTPError, RequestError} from '../source/index'; import withServer from './helpers/with-server'; import delay = require('delay'); @@ -132,7 +132,7 @@ test('has redirect event', withServer, async (t, server, got) => { server.get('/redirect', redirectHandler); const stream = got.stream('redirect'); - const {headers} = await pEvent(stream, 'redirect'); + const [_updatedOptions, {headers}] = await pEvent(stream, 'redirect', {multiArgs: true}); t.is(headers.location, '/'); await getStream(stream); @@ -150,7 +150,7 @@ test('has error event', withServer, async (t, server, got) => { const stream = got.stream(''); await t.throwsAsync(pEvent(stream, 'response'), { - instanceOf: got.HTTPError, + instanceOf: HTTPError, message: 'Response code 404 (Not Found)' }); }); @@ -183,7 +183,7 @@ test('redirect response contains old url', withServer, async (t, server, got) => server.get('/redirect', redirectHandler); const {requestUrl} = await pEvent(got.stream('redirect'), 'response'); - t.is(requestUrl, `${server.url}/redirect`); + t.is(requestUrl.toString(), `${server.url}/redirect`); }); test('check for pipe method', withServer, (t, server, got) => { @@ -442,7 +442,9 @@ if (Number.parseInt(process.versions.node.split('.')[0]!, 10) <= 12) { await t.notThrowsAsync(new Promise((resolve, reject) => { got.stream({ - timeout: 100, + timeout: { + request: 100 + }, hooks: { beforeError: [ async error => { diff --git a/test/timeout.ts b/test/timeout.ts index a49f9f9dd..152b6280f 100644 --- a/test/timeout.ts +++ b/test/timeout.ts @@ -11,7 +11,7 @@ import CacheableLookup from 'cacheable-lookup'; import {Handler} from 'express'; import * as pEvent from 'p-event'; import got, {TimeoutError} from '../source/index'; -import timedOut from '../source/core/utils/timed-out'; +import timedOut from '../source/core/timed-out'; import slowDataStream from './helpers/slow-data-stream'; import {GlobalClock} from './helpers/types'; import withServer, {withServerAndFakeTimers, withHttpsServer} from './helpers/with-server'; @@ -21,7 +21,7 @@ const pStreamPipeline = promisify(stream.pipeline); const requestDelay = 800; const errorMatcher = { - instanceOf: got.TimeoutError, + instanceOf: TimeoutError, code: 'ETIMEDOUT' }; @@ -37,7 +37,7 @@ const defaultHandler = (clock: GlobalClock): Handler => (request, response) => { }); }; -const downloadHandler = (clock: GlobalClock): Handler => (_request, response) => { +const downloadHandler = (clock?: GlobalClock): Handler => (_request, response) => { response.writeHead(200, { 'transfer-encoding': 'chunked' }); @@ -63,8 +63,12 @@ test.serial('timeout option', withServerAndFakeTimers, async (t, server, got, cl await t.throwsAsync( got({ - timeout: 1, - retry: 0 + timeout: { + request: 1 + }, + retry: { + limit: 0 + } }), { ...errorMatcher, @@ -79,7 +83,9 @@ test.serial('timeout option as object', withServerAndFakeTimers, async (t, serve await t.throwsAsync( got({ timeout: {request: 1}, - retry: 0 + retry: { + limit: 0 + } }), { ...errorMatcher, @@ -92,7 +98,9 @@ test.serial('socket timeout', async t => { await t.throwsAsync( got('https://example.com', { timeout: {socket: 1}, - retry: 0, + retry: { + limit: 0 + }, request: () => { const stream = new PassThroughStream(); // @ts-expect-error Mocking the behaviour of a ClientRequest @@ -108,7 +116,7 @@ test.serial('socket timeout', async t => { } }), { - instanceOf: got.TimeoutError, + instanceOf: TimeoutError, code: 'ETIMEDOUT', message: 'Timeout awaiting \'socket\' for 1ms' } @@ -122,7 +130,9 @@ test.serial('send timeout', withServerAndFakeTimers, async (t, server, got, cloc got.post({ timeout: {send: 1}, body: new stream.PassThrough(), - retry: 0 + retry: { + limit: 0 + } }).on('request', request => { request.once('socket', socket => { socket.once('connect', () => { @@ -151,7 +161,9 @@ test.serial('send timeout (keepalive)', withServerAndFakeTimers, async (t, serve http: keepAliveAgent }, timeout: {send: 1}, - retry: 0, + retry: { + limit: 0 + }, body: slowDataStream(clock) }).on('request', (request: http.ClientRequest) => { request.once('socket', socket => { @@ -175,7 +187,9 @@ test.serial('response timeout', withServerAndFakeTimers, async (t, server, got, await t.throwsAsync( got({ timeout: {response: 1}, - retry: 0 + retry: { + limit: 0 + } }), { ...errorMatcher, @@ -188,20 +202,24 @@ test.serial('response timeout unaffected by slow upload', withServerAndFakeTimer server.post('/', defaultHandler(clock)); await t.notThrowsAsync(got.post({ - retry: 0, + retry: { + limit: 0 + }, body: slowDataStream(clock) })); }); -test.serial('response timeout unaffected by slow download', withServerAndFakeTimers, async (t, server, got, clock) => { - server.get('/', downloadHandler(clock)); +test.serial('response timeout unaffected by slow download', withServer, async (t, server, got) => { + server.get('/', downloadHandler()); await t.notThrowsAsync(got({ timeout: {response: 200}, - retry: 0 + retry: { + limit: 0 + } })); - clock.tick(100); + await delay(100); }); test.serial('response timeout (keepalive)', withServerAndFakeTimers, async (t, server, got, clock) => { @@ -217,7 +235,9 @@ test.serial('response timeout (keepalive)', withServerAndFakeTimers, async (t, s http: keepAliveAgent }, timeout: {response: 1}, - retry: 0 + retry: { + limit: 0 + } }).on('request', (request: http.ClientRequest) => { request.once('socket', socket => { t.false(socket.connecting); @@ -246,7 +266,9 @@ test.serial('connect timeout', withServerAndFakeTimers, async (t, _server, got, return socket; }, timeout: {connect: 1}, - retry: 0 + retry: { + limit: 0 + } }).on('request', (request: http.ClientRequest) => { request.on('socket', () => { clock.runAll(); @@ -270,7 +292,9 @@ test.serial('connect timeout (ip address)', withServerAndFakeTimers, async (t, _ return socket; }, timeout: {connect: 1}, - retry: 0 + retry: { + limit: 0 + } }).on('request', (request: http.ClientRequest) => { request.on('socket', () => { clock.runAll(); @@ -300,7 +324,9 @@ test.serial('secureConnect timeout', withHttpsServer({}, true), async (t, _serve return socket; }, timeout: {secureConnect: 0}, - retry: 0 + retry: { + limit: 0 + } }).on('request', (request: http.ClientRequest) => { request.on('socket', () => { clock!.runAll(); @@ -320,8 +346,10 @@ test('secureConnect timeout not breached', withServer, async (t, server, got) => await t.notThrowsAsync(got({ timeout: {secureConnect: 200}, - retry: 0, - https: { + retry: { + limit: 0 + }, + httpsOptions: { rejectUnauthorized: false } })); @@ -332,9 +360,11 @@ test.serial('lookup timeout', withServerAndFakeTimers, async (t, server, got, cl await t.throwsAsync( got({ - lookup: () => {}, + dnsLookup: () => {}, timeout: {lookup: 1}, - retry: 0 + retry: { + limit: 0 + } }).on('request', (request: http.ClientRequest) => { request.on('socket', () => { clock.runAll(); @@ -359,7 +389,7 @@ test.serial('lookup timeout no error (ip address)', withServerAndFakeTimers, asy await t.notThrowsAsync(got({ url: `http://127.0.0.1:${server.port}`, timeout: {lookup: 1}, - retry: 0 + retry: {limit: 0} })); }); @@ -373,7 +403,7 @@ test.serial('lookup timeout no error (keepalive)', withServerAndFakeTimers, asyn await t.notThrowsAsync(got({ agent: {http: keepAliveAgent}, timeout: {lookup: 1}, - retry: 0 + retry: {limit: 0} }).on('request', (request: http.ClientRequest) => { request.once('connect', () => { t.fail('connect event fired, invalidating test'); @@ -388,7 +418,9 @@ test.serial('retries on timeout', withServer, async (t, server, got) => { let hasTried = false; await t.throwsAsync(got({ - timeout: 1, + timeout: { + request: 1 + }, retry: { calculateDelay: () => { if (hasTried) { @@ -411,8 +443,8 @@ test.serial('timeout with streams', withServerAndFakeTimers, async (t, server, g server.get('/', defaultHandler(clock)); const stream = got.stream({ - timeout: 0, - retry: 0 + timeout: {request: 0}, + retry: {limit: 0} }); await t.throwsAsync(pEvent(stream, 'response'), {code: 'ETIMEDOUT'}); }); @@ -421,7 +453,9 @@ test.serial('no error emitted when timeout is not breached (stream)', withServer server.get('/', defaultHandler(clock)); const stream = got.stream({ - retry: 0, + retry: { + limit: 0 + }, timeout: { request: requestDelay * 2 } @@ -434,7 +468,9 @@ test.serial('no error emitted when timeout is not breached (promise)', withServe server.get('/', defaultHandler(clock)); await t.notThrowsAsync(got({ - retry: 0, + retry: { + limit: 0 + }, timeout: { request: requestDelay * 2 } @@ -445,16 +481,23 @@ test.serial('no unhandled `socket hung up` errors', withServerAndFakeTimers, asy server.get('/', defaultHandler(clock)); await t.throwsAsync( - got({retry: 0, timeout: requestDelay / 2}), - {instanceOf: got.TimeoutError} + got({ + retry: { + limit: 0 + }, + timeout: { + request: requestDelay / 2 + } + }), + {instanceOf: TimeoutError} ); }); // TODO: use fakeTimers here test.serial('no unhandled timeout errors', withServer, async (t, _server, got) => { await t.throwsAsync(got({ - retry: 0, - timeout: 100, + retry: {limit: 0}, + timeout: {request: 100}, request: (...args: any[]) => { // @ts-expect-error const result = http.request(...args); @@ -477,7 +520,9 @@ test.serial('no unhandled timeout errors #2', withServer, async (t, server, got) }); const gotPromise = got('', { - timeout: 20, + timeout: { + request: 20 + }, retry: { calculateDelay: ({computedValue}) => { if (computedValue) { @@ -524,7 +569,7 @@ test.serial('no more timeouts after an error', withServer, async (t, _server, go }; await t.throwsAsync(got(`http://${Date.now()}.dev`, { - retry: 1, + retry: {limit: 1}, timeout: { lookup: 1, connect: 1, @@ -534,7 +579,7 @@ test.serial('no more timeouts after an error', withServer, async (t, _server, go send: 1, request: 1 } - }), {instanceOf: got.TimeoutError}); + }), {instanceOf: TimeoutError}); await delay(100); @@ -547,7 +592,7 @@ test.serial('socket timeout is canceled on error', withServerAndFakeTimers, asyn const promise = got({ timeout: {socket: 50}, - retry: 0 + retry: {limit: 0} }).on('request', (request: http.ClientRequest) => { request.destroy(new Error(message)); }); @@ -615,14 +660,14 @@ test.serial('doesn\'t throw on early lookup', withServerAndFakeTimers, async (t, timeout: { lookup: 1 }, - retry: 0, + retry: {limit: 0}, // @ts-expect-error - lookup: (...[_hostname, options, callback]: Parameters) => { + dnsLookup: (...[_hostname, options, callback]: Parameters) => { if (typeof options === 'function') { callback = options; } - // @ts-expect-error This should be fixed in upstream + // @ts-expect-error Invalid types upstream. callback(null, '127.0.0.1', 4); } })); @@ -635,8 +680,8 @@ test.serial('no unhandled `Premature close` error', withServer, async (t, server }); await t.throwsAsync(got({ - timeout: 10, - retry: 0 + timeout: {request: 10}, + retry: {limit: 0} }), {message: 'Timeout awaiting \'request\' for 10ms'}); await delay(20); @@ -652,7 +697,9 @@ test.serial('`read` timeout - promise', withServer, async (t, server, got) => { timeout: { read: 10 }, - retry: 0 + retry: { + limit: 0 + } }), {message: 'Timeout awaiting \'read\' for 10ms'}); }); @@ -680,8 +727,12 @@ test.serial('cancelling the request removes timeouts', withServer, async (t, ser }); const promise = got({ - timeout: 500, - retry: 0 + timeout: { + request: 500 + }, + retry: { + limit: 0 + } }).on('downloadProgress', () => { promise.cancel(); }).on('request', request => { @@ -702,8 +753,12 @@ test('timeouts are emitted ASAP', async t => { const marginOfError = process.env.CI ? 200 : 100; const error = await t.throwsAsync(got('http://192.0.2.1/test', { - retry: 0, - timeout + retry: { + limit: 0 + }, + timeout: { + request: timeout + } }), {instanceOf: TimeoutError}); t.true(error.timings.phases.total! < (timeout + marginOfError)); diff --git a/test/unix-socket.ts b/test/unix-socket.ts index 3198d5ad1..75d8af659 100644 --- a/test/unix-socket.ts +++ b/test/unix-socket.ts @@ -32,7 +32,7 @@ if (process.platform !== 'win32') { test('throws on invalid URL', async t => { try { - await got('unix:', {retry: 0}); + await got('unix:', {retry: {limit: 0}}); } catch (error) { t.regex(error.code, /ENOTFOUND|EAI_AGAIN/); } diff --git a/tsconfig.json b/tsconfig.json index 272fe16ee..416a6935f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,8 +6,7 @@ "lib": [ "es2019" ], - "useDefineForClassFields": false, - // "allowSyntheticDefaultImports": true, + "allowSyntheticDefaultImports": false, // "esModuleInterop": true // "module": "es2020" },