feat(github): Remember GraphQL optimal page size (#13047)

Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/util/cache/repository/types.ts

@@ -31,6 +31,11 @@ export interface BranchCache {
   upgrades: BranchUpgradeCache[];
 }
 
+export interface GithubGraphqlPageCache {
+  pageLastResizedAt: string;
+  pageSize: number;
+}
+
 export interface Cache {
   configFileName?: string;
   semanticCommits?: 'enabled' | 'disabled';
@@ -40,4 +45,9 @@ export interface Cache {
   init?: RepoInitConfig;
   scan?: Record<string, BaseBranchCache>;
   lastPlatformAutomergeFailure?: string;
+  platform?: {
+    github?: {
+      graphqlPageCache?: Record<string, GithubGraphqlPageCache>;
+    };
+  };
 }

lib/util/http/__snapshots__/github.spec.ts.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
+exports[`util/http/github GraphQL expands items count on timeout 1`] = `
 Array [
   Object {
     "graphql": Object {
@@ -42,14 +42,14 @@ Array [
         },
       },
       "variables": Object {
-        "count": 100,
+        "count": 84,
         "cursor": null,
       },
     },
     "headers": Object {
       "accept": "application/vnd.github.v3+json",
       "accept-encoding": "gzip, deflate, br",
-      "content-length": "494",
+      "content-length": "493",
       "content-type": "application/json",
       "host": "api.github.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
@@ -97,14 +97,129 @@ Array [
         },
       },
       "variables": Object {
-        "count": 100,
+        "count": 84,
         "cursor": "cursor1",
       },
     },
     "headers": Object {
       "accept": "application/vnd.github.v3+json",
       "accept-encoding": "gzip, deflate, br",
-      "content-length": "499",
+      "content-length": "498",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "__vars": Object {
+          "$count": "Int",
+          "$cursor": "String",
+          "$name": "String!",
+          "$owner": "String!",
+        },
+        "repository": Object {
+          "__args": Object {
+            "name": "$name",
+            "owner": "$name",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "$cursor",
+              "filterBy": Object {
+                "createdBy": "someone",
+              },
+              "first": "$count",
+              "orderBy": Object {
+                "direction": "DESC",
+                "field": "UPDATED_AT",
+              },
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+      "variables": Object {
+        "count": 84,
+        "cursor": "cursor2",
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "content-length": "498",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+]
+`;
+
+exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
+Array [
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "__vars": Object {
+          "$count": "Int",
+          "$cursor": "String",
+          "$name": "String!",
+          "$owner": "String!",
+        },
+        "repository": Object {
+          "__args": Object {
+            "name": "$name",
+            "owner": "$name",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "$cursor",
+              "filterBy": Object {
+                "createdBy": "someone",
+              },
+              "first": "$count",
+              "orderBy": Object {
+                "direction": "DESC",
+                "field": "UPDATED_AT",
+              },
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+      "variables": Object {
+        "count": 50,
+        "cursor": null,
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "content-length": "493",
       "content-type": "application/json",
       "host": "api.github.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
@@ -207,7 +322,62 @@ Array [
         },
       },
       "variables": Object {
-        "count": 50,
+        "count": 25,
+        "cursor": "cursor1",
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "content-length": "498",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "__vars": Object {
+          "$count": "Int",
+          "$cursor": "String",
+          "$name": "String!",
+          "$owner": "String!",
+        },
+        "repository": Object {
+          "__args": Object {
+            "name": "$name",
+            "owner": "$name",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "$cursor",
+              "filterBy": Object {
+                "createdBy": "someone",
+              },
+              "first": "$count",
+              "orderBy": Object {
+                "direction": "DESC",
+                "field": "UPDATED_AT",
+              },
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+      "variables": Object {
+        "count": 25,
         "cursor": "cursor2",
       },
     },

lib/util/http/github.spec.ts

@@ -1,4 +1,6 @@
+import { DateTime } from 'luxon';
 import * as httpMock from '../../../test/http-mock';
+import { mocked } from '../../../test/util';
 import {
   EXTERNAL_HOST_ERROR,
   PLATFORM_BAD_CREDENTIALS,
@@ -7,9 +9,14 @@ import {
   REPOSITORY_CHANGED,
 } from '../../constants/error-messages';
 import { id as GITHUB_RELEASES_ID } from '../../datasource/github-releases';
+import * as _repositoryCache from '../../util/cache/repository';
+import type { Cache } from '../../util/cache/repository/types';
 import * as hostRules from '../host-rules';
 import { GithubHttp, setBaseUrl } from './github';
 
+jest.mock('../../util/cache/repository');
+const repositoryCache = mocked(_repositoryCache);
+
 const githubApiHost = 'https://api.github.com';
 
 const graphqlQuery = `
@@ -40,10 +47,14 @@ query(
 
 describe('util/http/github', () => {
   let githubApi: GithubHttp;
+  let repoCache: Cache = {};
+
   beforeEach(() => {
     githubApi = new GithubHttp();
     setBaseUrl(githubApiHost);
     jest.resetAllMocks();
+    repoCache = {};
+    repositoryCache.getCache.mockReturnValue(repoCache);
   });
 
   afterEach(() => {
@@ -474,6 +485,15 @@ describe('util/http/github', () => {
       expect(items).toHaveLength(2);
     });
     it('shrinks items count on 50x', async () => {
+      repoCache.platform ??= {};
+      repoCache.platform.github ??= {};
+      repoCache.platform.github.graphqlPageCache = {
+        testItem: {
+          pageLastResizedAt: DateTime.local().toISO(),
+          pageSize: 50,
+        },
+      };
+
       httpMock
         .scope(githubApiHost)
         .post('/graphql')
@@ -488,11 +508,42 @@ describe('util/http/github', () => {
       const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
       expect(items).toHaveLength(3);
 
+      expect(
+        repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
+      ).toBe(25);
+
       const trace = httpMock.getTrace();
       expect(trace).toHaveLength(4);
       expect(trace).toMatchSnapshot();
     });
+    it('expands items count on timeout', async () => {
+      repoCache.platform ??= {};
+      repoCache.platform.github ??= {};
+      repoCache.platform.github.graphqlPageCache = {
+        testItem: {
+          pageLastResizedAt: DateTime.local()
+            .minus({ hours: 24, seconds: 1 })
+            .toISO(),
+          pageSize: 42,
+        },
+      };
+
+      httpMock
+        .scope(githubApiHost)
+        .post('/graphql')
+        .reply(200, page1)
+        .post('/graphql')
+        .reply(200, page2)
+        .post('/graphql')
+        .reply(200, page3);
 
+      const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
+      expect(items).toHaveLength(3);
+      expect(
+        repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
+      ).toBe(84);
+      expect(httpMock.getTrace()).toMatchSnapshot();
+    });
     it('continues to iterate with a lower page size on error 502', async () => {
       httpMock
         .scope(githubApiHost)
@@ -507,8 +558,41 @@ describe('util/http/github', () => {
 
       const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
       expect(items).toHaveLength(3);
+
+      const trace = httpMock.getTrace();
+      expect(trace).toHaveLength(4);
     });
+    it('removes cache record once expanded to the maximum', async () => {
+      repoCache.platform ??= {};
+      repoCache.platform.github ??= {};
+      repoCache.platform.github.graphqlPageCache = {
+        testItem: {
+          pageLastResizedAt: DateTime.local()
+            .minus({ hours: 24, seconds: 1 })
+            .toISO(),
+          pageSize: 50,
+        },
+      };
+
+      httpMock
+        .scope(githubApiHost)
+        .post('/graphql')
+        .reply(200, page1)
+        .post('/graphql')
+        .reply(200, page2)
+        .post('/graphql')
+        .reply(200, page3);
+
+      const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
+      expect(items).toHaveLength(3);
 
+      expect(
+        repoCache?.platform?.github?.graphqlPageCache?.testItem
+      ).toBeUndefined();
+
+      const trace = httpMock.getTrace();
+      expect(trace).toHaveLength(3);
+    });
     it('throws on 50x if count < 10', async () => {
       httpMock.scope(githubApiHost).post('/graphql').reply(500);
       await expect(