feat(internal): request caching in http module (#6497)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
renovatebot · Jun 12, 2020 · 5d624ed · 5d624ed
1 parent 21028a7
commit 5d624ed
Show file tree

Hide file tree

Showing 6 changed files with 60 additions and 106 deletions.
diff --git a/lib/util/clone.ts b/lib/util/clone.ts
@@ -1,5 +1,5 @@
 import safeStringify from 'fast-safe-stringify';
 
-export function clone<T>(input: T): T {
+export function clone<T>(input: T = null): T {
   return JSON.parse(safeStringify(input));
 }
diff --git a/lib/util/got/__snapshots__/index.spec.ts.snap b/lib/util/got/__snapshots__/index.spec.ts.snap
@@ -1,18 +1,14 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`util/got/index uses  basic auth 1`] = `
+exports[`util/got/index gets 1`] = `
 Object {
   "body": Object {},
   "options": Object {
-    "auth": ":test",
     "baseUrl": "https://api.github.com/",
     "cache": false,
     "decompress": true,
     "followRedirect": true,
     "form": false,
-    "gotTimeout": Object {
-      "request": 60000,
-    },
     "hash": "",
     "headers": Object {
       "accept": "application/json",
@@ -27,6 +23,7 @@ Object {
       "beforeRetry": Array [],
       "init": Array [],
     },
+    "hostType": "github",
     "hostname": "api.github.com",
     "href": "https://api.github.com/some",
     "json": true,
@@ -43,24 +40,21 @@ Object {
     "search": "",
     "stream": false,
     "throwHttpErrors": true,
+    "useCache": false,
     "useElectronNet": false,
   },
 }
 `;
 
-exports[`util/got/index uses  basic auth 2`] = `
+exports[`util/got/index gets 2`] = `
 Object {
   "body": Object {},
   "options": Object {
-    "auth": ":test",
     "baseUrl": "https://api.github.com/",
     "cache": false,
     "decompress": true,
     "followRedirect": true,
     "form": false,
-    "gotTimeout": Object {
-      "request": 60000,
-    },
     "hash": "",
     "headers": Object {
       "accept": "application/json",
@@ -75,10 +69,11 @@ Object {
       "beforeRetry": Array [],
       "init": Array [],
     },
+    "hostType": "github",
     "hostname": "api.github.com",
     "href": "https://api.github.com/some",
     "json": true,
-    "method": "GET",
+    "method": "HEAD",
     "path": "/some",
     "pathname": "/some",
     "protocol": "https:",
@@ -96,20 +91,23 @@ Object {
 }
 `;
 
-exports[`util/got/index uses  bearer auth 1`] = `
+exports[`util/got/index uses  basic auth 1`] = `
 Object {
   "body": Object {},
   "options": Object {
+    "auth": ":test",
     "baseUrl": "https://api.github.com/",
     "cache": false,
     "decompress": true,
     "followRedirect": true,
     "form": false,
+    "gotTimeout": Object {
+      "request": 60000,
+    },
     "hash": "",
     "headers": Object {
       "accept": "application/json",
       "accept-encoding": "gzip, deflate",
-      "authorization": "Bearer XXX",
       "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
     },
     "hooks": Object {
@@ -141,20 +139,23 @@ Object {
 }
 `;
 
-exports[`util/got/index uses  bearer auth 2`] = `
+exports[`util/got/index uses  basic auth 2`] = `
 Object {
   "body": Object {},
   "options": Object {
+    "auth": ":test",
     "baseUrl": "https://api.github.com/",
     "cache": false,
     "decompress": true,
     "followRedirect": true,
     "form": false,
+    "gotTimeout": Object {
+      "request": 60000,
+    },
     "hash": "",
     "headers": Object {
       "accept": "application/json",
       "accept-encoding": "gzip, deflate",
-      "authorization": "Bearer XXX",
       "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
     },
     "hooks": Object {
@@ -186,7 +187,7 @@ Object {
 }
 `;
 
-exports[`util/got/index uses no cache 1`] = `
+exports[`util/got/index uses  bearer auth 1`] = `
 Object {
   "body": Object {},
   "options": Object {
@@ -199,6 +200,7 @@ Object {
     "headers": Object {
       "accept": "application/json",
       "accept-encoding": "gzip, deflate",
+      "authorization": "Bearer XXX",
       "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
     },
     "hooks": Object {
@@ -209,7 +211,6 @@ Object {
       "beforeRetry": Array [],
       "init": Array [],
     },
-    "hostType": "github",
     "hostname": "api.github.com",
     "href": "https://api.github.com/some",
     "json": true,
@@ -226,13 +227,12 @@ Object {
     "search": "",
     "stream": false,
     "throwHttpErrors": true,
-    "useCache": false,
     "useElectronNet": false,
   },
 }
 `;
 
-exports[`util/got/index uses no cache 2`] = `
+exports[`util/got/index uses  bearer auth 2`] = `
 Object {
   "body": Object {},
   "options": Object {
@@ -245,6 +245,7 @@ Object {
     "headers": Object {
       "accept": "application/json",
       "accept-encoding": "gzip, deflate",
+      "authorization": "Bearer XXX",
       "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
     },
     "hooks": Object {
@@ -255,11 +256,10 @@ Object {
       "beforeRetry": Array [],
       "init": Array [],
     },
-    "hostType": "github",
     "hostname": "api.github.com",
     "href": "https://api.github.com/some",
     "json": true,
-    "method": "HEAD",
+    "method": "GET",
     "path": "/some",
     "pathname": "/some",
     "protocol": "https:",

diff --git a/lib/util/got/cache-get.ts b/lib/util/got/cache-get.ts
diff --git a/lib/util/got/index.spec.ts b/lib/util/got/index.spec.ts
@@ -69,7 +69,7 @@ describe(getName(__filename), () => {
     expect(req.isDone()).toBe(true);
   });
 
-  it('uses no cache', async () => {
+  it('gets', async () => {
     const req = mock({})
       .head('/some')
       .reply(200, {})
@@ -93,29 +93,4 @@ describe(getName(__filename), () => {
 
     expect(req.isDone()).toBe(true);
   });
-
-  it('streams no cache', async () => {
-    const req = mock();
-
-    const stream = api.stream('/some', {
-      baseUrl,
-    });
-    expect(stream).toBeDefined();
-
-    let data = '';
-
-    stream.on('data', (c) => {
-      data += c;
-    });
-
-    const done = new Promise((resolve, reject) => {
-      stream.on('end', resolve);
-      stream.on('error', reject);
-    });
-
-    await done;
-
-    expect(data).toBe('{}');
-    expect(req.isDone()).toBe(true);
-  });
 });
diff --git a/lib/util/got/index.ts b/lib/util/got/index.ts
@@ -1,6 +1,5 @@
 import got from 'got';
 import auth from './auth';
-import cacheGet from './cache-get';
 import hostRules from './host-rules';
 import { mergeInstances } from './util';
 
@@ -11,6 +10,6 @@ export * from './common';
  *  - Cache all GET requests for the lifetime of the repo
  *
  */
-export const api = mergeInstances(got, cacheGet, hostRules, auth);
+export const api = mergeInstances(got, hostRules, auth);
 
 export default api;
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
@@ -1,4 +1,8 @@
+import crypto from 'crypto';
 import URL from 'url';
+import { GotPromise } from 'got';
+import * as runCache from '../cache/run';
+import { clone } from '../clone';
 import got from '../got';
 
 interface OutgoingHttpHeaders {
@@ -28,10 +32,21 @@ export interface HttpResponse<T = string> {
   headers: any;
 }
 
+async function cloneResponse<T>(
+  promisedResponse: GotPromise<any>
+): Promise<HttpResponse<T>> {
+  const response = await promisedResponse;
+  // clone body and headers so that the cached result doesn't get accidentally mutated
+  return {
+    body: clone<T>(response.body),
+    headers: clone(response.headers),
+  };
+}
+
 export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
   constructor(private hostType: string, private options?: HttpOptions) {}
 
-  protected async request<T>(
+  protected request<T>(
     requestUrl: string | URL,
     httpOptions?: InternalHttpOptions
   ): Promise<HttpResponse<T> | null> {
@@ -72,8 +87,26 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
         process.env.RENOVATE_USER_AGENT ||
         'https://github.com/renovatebot/renovate',
     };
-    const res = await got(url, options);
-    return { body: res.body, headers: res.headers };
+
+    // Cache GET requests unless useCache=false
+    let promisedRes: GotPromise<any>;
+    if (options.method === 'get') {
+      const cacheKey = crypto
+        .createHash('md5')
+        .update('got-' + JSON.stringify({ url, headers: options.headers }))
+        .digest('hex');
+      if (options.useCache !== false) {
+        // check cache unless bypassing it
+        promisedRes = runCache.get(cacheKey);
+      }
+      if (promisedRes === undefined) {
+        // cache miss OR cache bypass
+        promisedRes = got(url, options);
+      }
+      runCache.set(cacheKey, promisedRes); // always set
+      return cloneResponse<T>(promisedRes);
+    }
+    return cloneResponse<T>(got(url, options));
   }
 
   get(url: string, options: HttpOptions = {}): Promise<HttpResponse> {