diff --git a/lib/util/clone.ts b/lib/util/clone.ts index 435e4b050914de..d1c9530e7e597a 100644 --- a/lib/util/clone.ts +++ b/lib/util/clone.ts @@ -1,5 +1,5 @@ import safeStringify from 'fast-safe-stringify'; -export function clone(input: T): T { +export function clone(input: T = null): T { return JSON.parse(safeStringify(input)); } diff --git a/lib/util/got/__snapshots__/index.spec.ts.snap b/lib/util/got/__snapshots__/index.spec.ts.snap index d3e89bf57f2d4b..fc49174cb79403 100644 --- a/lib/util/got/__snapshots__/index.spec.ts.snap +++ b/lib/util/got/__snapshots__/index.spec.ts.snap @@ -1,18 +1,14 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`util/got/index uses basic auth 1`] = ` +exports[`util/got/index gets 1`] = ` Object { "body": Object {}, "options": Object { - "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, - "gotTimeout": Object { - "request": 60000, - }, "hash": "", "headers": Object { "accept": "application/json", @@ -27,6 +23,7 @@ Object { "beforeRetry": Array [], "init": Array [], }, + "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, @@ -43,24 +40,21 @@ Object { "search": "", "stream": false, "throwHttpErrors": true, + "useCache": false, "useElectronNet": false, }, } `; -exports[`util/got/index uses basic auth 2`] = ` +exports[`util/got/index gets 2`] = ` Object { "body": Object {}, "options": Object { - "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, - "gotTimeout": Object { - "request": 60000, - }, "hash": "", "headers": Object { "accept": "application/json", @@ -75,10 +69,11 @@ Object { "beforeRetry": Array [], "init": Array [], }, + "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, - "method": "GET", + "method": "HEAD", "path": "/some", "pathname": "/some", "protocol": "https:", @@ -96,20 +91,23 @@ Object { } `; -exports[`util/got/index uses bearer auth 1`] = ` +exports[`util/got/index uses basic auth 1`] = ` Object { "body": Object {}, "options": Object { + "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, + "gotTimeout": Object { + "request": 60000, + }, "hash": "", "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", - "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -141,20 +139,23 @@ Object { } `; -exports[`util/got/index uses bearer auth 2`] = ` +exports[`util/got/index uses basic auth 2`] = ` Object { "body": Object {}, "options": Object { + "auth": ":test", "baseUrl": "https://api.github.com/", "cache": false, "decompress": true, "followRedirect": true, "form": false, + "gotTimeout": Object { + "request": 60000, + }, "hash": "", "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", - "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -186,7 +187,7 @@ Object { } `; -exports[`util/got/index uses no cache 1`] = ` +exports[`util/got/index uses bearer auth 1`] = ` Object { "body": Object {}, "options": Object { @@ -199,6 +200,7 @@ Object { "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", + "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -209,7 +211,6 @@ Object { "beforeRetry": Array [], "init": Array [], }, - "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, @@ -226,13 +227,12 @@ Object { "search": "", "stream": false, "throwHttpErrors": true, - "useCache": false, "useElectronNet": false, }, } `; -exports[`util/got/index uses no cache 2`] = ` +exports[`util/got/index uses bearer auth 2`] = ` Object { "body": Object {}, "options": Object { @@ -245,6 +245,7 @@ Object { "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", + "authorization": "Bearer XXX", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", }, "hooks": Object { @@ -255,11 +256,10 @@ Object { "beforeRetry": Array [], "init": Array [], }, - "hostType": "github", "hostname": "api.github.com", "href": "https://api.github.com/some", "json": true, - "method": "HEAD", + "method": "GET", "path": "/some", "pathname": "/some", "protocol": "https:", diff --git a/lib/util/got/cache-get.ts b/lib/util/got/cache-get.ts deleted file mode 100644 index c49ee5d9bf5f5e..00000000000000 --- a/lib/util/got/cache-get.ts +++ /dev/null @@ -1,53 +0,0 @@ -import crypto from 'crypto'; -import { logger } from '../../logger'; -import * as runCache from '../cache/run'; -import { clone } from '../clone'; -import { create } from './util'; - -// With this caching, it means every GET request is cached during each repository run - -function cloneBody(response: any): any { - return { - ...response, - body: clone(response.body), - }; -} - -export default create({ - options: {}, - handler: (options, next) => { - if (options.stream) { - return next(options); - } - if (!['github', 'npm'].includes(options.hostType)) { - return next(options).then(cloneBody); - } - if (options.method === 'GET') { - const cacheKey = crypto - .createHash('md5') - .update( - 'got-' + - JSON.stringify({ href: options.href, headers: options.headers }) - ) - .digest('hex'); - if (options.useCache === false) { - logger.trace('GET cache skipped: ' + options.href); - } else { - const cachedGot = runCache.get(cacheKey); - // istanbul ignore if - if (cachedGot) { - logger.trace('GET cache hit: ' + options.href); - return cachedGot; - } - logger.trace('GET cache miss: ' + options.href); - } - const promisedRes = next(options).catch((err) => { - runCache.set(cacheKey, null); - throw err; - }); - runCache.set(cacheKey, promisedRes); - return promisedRes.then(cloneBody); - } - return next(options); - }, -}); diff --git a/lib/util/got/index.spec.ts b/lib/util/got/index.spec.ts index 6b9d09317dc02a..a1352e64b41903 100644 --- a/lib/util/got/index.spec.ts +++ b/lib/util/got/index.spec.ts @@ -69,7 +69,7 @@ describe(getName(__filename), () => { expect(req.isDone()).toBe(true); }); - it('uses no cache', async () => { + it('gets', async () => { const req = mock({}) .head('/some') .reply(200, {}) @@ -93,29 +93,4 @@ describe(getName(__filename), () => { expect(req.isDone()).toBe(true); }); - - it('streams no cache', async () => { - const req = mock(); - - const stream = api.stream('/some', { - baseUrl, - }); - expect(stream).toBeDefined(); - - let data = ''; - - stream.on('data', (c) => { - data += c; - }); - - const done = new Promise((resolve, reject) => { - stream.on('end', resolve); - stream.on('error', reject); - }); - - await done; - - expect(data).toBe('{}'); - expect(req.isDone()).toBe(true); - }); }); diff --git a/lib/util/got/index.ts b/lib/util/got/index.ts index 66ee31e67442a9..c385a9efeeafd0 100644 --- a/lib/util/got/index.ts +++ b/lib/util/got/index.ts @@ -1,6 +1,5 @@ import got from 'got'; import auth from './auth'; -import cacheGet from './cache-get'; import hostRules from './host-rules'; import { mergeInstances } from './util'; @@ -11,6 +10,6 @@ export * from './common'; * - Cache all GET requests for the lifetime of the repo * */ -export const api = mergeInstances(got, cacheGet, hostRules, auth); +export const api = mergeInstances(got, hostRules, auth); export default api; diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index e8f284088cbed5..ff254197ceda02 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -1,4 +1,8 @@ +import crypto from 'crypto'; import URL from 'url'; +import { GotPromise } from 'got'; +import * as runCache from '../cache/run'; +import { clone } from '../clone'; import got from '../got'; interface OutgoingHttpHeaders { @@ -28,10 +32,21 @@ export interface HttpResponse { headers: any; } +async function cloneResponse( + promisedResponse: GotPromise +): Promise> { + const response = await promisedResponse; + // clone body and headers so that the cached result doesn't get accidentally mutated + return { + body: clone(response.body), + headers: clone(response.headers), + }; +} + export class Http { constructor(private hostType: string, private options?: HttpOptions) {} - protected async request( + protected request( requestUrl: string | URL, httpOptions?: InternalHttpOptions ): Promise | null> { @@ -72,8 +87,26 @@ export class Http { process.env.RENOVATE_USER_AGENT || 'https://github.com/renovatebot/renovate', }; - const res = await got(url, options); - return { body: res.body, headers: res.headers }; + + // Cache GET requests unless useCache=false + let promisedRes: GotPromise; + if (options.method === 'get') { + const cacheKey = crypto + .createHash('md5') + .update('got-' + JSON.stringify({ url, headers: options.headers })) + .digest('hex'); + if (options.useCache !== false) { + // check cache unless bypassing it + promisedRes = runCache.get(cacheKey); + } + if (promisedRes === undefined) { + // cache miss OR cache bypass + promisedRes = got(url, options); + } + runCache.set(cacheKey, promisedRes); // always set + return cloneResponse(promisedRes); + } + return cloneResponse(got(url, options)); } get(url: string, options: HttpOptions = {}): Promise {