feat(github): Remember GraphQL optimal page size (#13047)

Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/util/cache/repository/types.ts

@@ -31,6 +31,11 @@ export interface BranchCache {
   upgrades: BranchUpgradeCache[];
 }
 
+export interface GithubGraphqlPageCache {
+  pageLastResizedAt: string;
+  pageSize: number;
+}
+
 export interface Cache {
   configFileName?: string;
   semanticCommits?: 'enabled' | 'disabled';
@@ -40,4 +45,9 @@ export interface Cache {
   init?: RepoInitConfig;
   scan?: Record<string, BaseBranchCache>;
   lastPlatformAutomergeFailure?: string;
+  platform?: {
+    github?: {
+      graphqlPageCache?: Record<string, GithubGraphqlPageCache>;
+    };
+  };
 }

lib/util/http/__snapshots__/github.spec.ts.snap

@@ -1,5 +1,175 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`util/http/github GraphQL expands items count on timeout 1`] = `
+Array [
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "__vars": Object {
+          "$count": "Int",
+          "$cursor": "String",
+          "$name": "String!",
+          "$owner": "String!",
+        },
+        "repository": Object {
+          "__args": Object {
+            "name": "$name",
+            "owner": "$name",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "$cursor",
+              "filterBy": Object {
+                "createdBy": "someone",
+              },
+              "first": "$count",
+              "orderBy": Object {
+                "direction": "DESC",
+                "field": "UPDATED_AT",
+              },
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+      "variables": Object {
+        "count": 84,
+        "cursor": null,
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "content-length": "493",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "__vars": Object {
+          "$count": "Int",
+          "$cursor": "String",
+          "$name": "String!",
+          "$owner": "String!",
+        },
+        "repository": Object {
+          "__args": Object {
+            "name": "$name",
+            "owner": "$name",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "$cursor",
+              "filterBy": Object {
+                "createdBy": "someone",
+              },
+              "first": "$count",
+              "orderBy": Object {
+                "direction": "DESC",
+                "field": "UPDATED_AT",
+              },
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+      "variables": Object {
+        "count": 84,
+        "cursor": "cursor1",
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "content-length": "498",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "__vars": Object {
+          "$count": "Int",
+          "$cursor": "String",
+          "$name": "String!",
+          "$owner": "String!",
+        },
+        "repository": Object {
+          "__args": Object {
+            "name": "$name",
+            "owner": "$name",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "$cursor",
+              "filterBy": Object {
+                "createdBy": "someone",
+              },
+              "first": "$count",
+              "orderBy": Object {
+                "direction": "DESC",
+                "field": "UPDATED_AT",
+              },
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+      "variables": Object {
+        "count": 84,
+        "cursor": "cursor2",
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "content-length": "498",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+]
+`;
+
 exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
 Array [
   Object {
@@ -42,69 +212,14 @@ Array [
         },
       },
       "variables": Object {
-        "count": 100,
+        "count": 50,
         "cursor": null,
       },
     },
     "headers": Object {
       "accept": "application/vnd.github.v3+json",
       "accept-encoding": "gzip, deflate, br",
-      "content-length": "494",
+      "content-length": "493",
-      "content-type": "application/json",
-      "host": "api.github.com",
-      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
-    },
-    "method": "POST",
-    "url": "https://api.github.com/graphql",
-  },
-  Object {
-    "graphql": Object {
-      "query": Object {
-        "__vars": Object {
-          "$count": "Int",
-          "$cursor": "String",
-          "$name": "String!",
-          "$owner": "String!",
-        },
-        "repository": Object {
-          "__args": Object {
-            "name": "$name",
-            "owner": "$name",
-          },
-          "testItem": Object {
-            "__args": Object {
-              "after": "$cursor",
-              "filterBy": Object {
-                "createdBy": "someone",
-              },
-              "first": "$count",
-              "orderBy": Object {
-                "direction": "DESC",
-                "field": "UPDATED_AT",
-              },
-            },
-            "nodes": Object {
-              "body": null,
-              "number": null,
-              "state": null,
-              "title": null,
-            },
-            "pageInfo": Object {
-              "endCursor": null,
-              "hasNextPage": null,
-            },
-          },
-        },
-      },
-      "variables": Object {
-        "count": 100,
-        "cursor": "cursor1",
-      },
-    },
-    "headers": Object {
-      "accept": "application/vnd.github.v3+json",
-      "accept-encoding": "gzip, deflate, br",
-      "content-length": "499",
       "content-type": "application/json",
       "host": "api.github.com",
       "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
@@ -207,7 +322,62 @@ Array [
         },
       },
       "variables": Object {
-        "count": 50,
+        "count": 25,
+        "cursor": "cursor1",
+      },
+    },
+    "headers": Object {
+      "accept": "application/vnd.github.v3+json",
+      "accept-encoding": "gzip, deflate, br",
+      "content-length": "498",
+      "content-type": "application/json",
+      "host": "api.github.com",
+      "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+    },
+    "method": "POST",
+    "url": "https://api.github.com/graphql",
+  },
+  Object {
+    "graphql": Object {
+      "query": Object {
+        "__vars": Object {
+          "$count": "Int",
+          "$cursor": "String",
+          "$name": "String!",
+          "$owner": "String!",
+        },
+        "repository": Object {
+          "__args": Object {
+            "name": "$name",
+            "owner": "$name",
+          },
+          "testItem": Object {
+            "__args": Object {
+              "after": "$cursor",
+              "filterBy": Object {
+                "createdBy": "someone",
+              },
+              "first": "$count",
+              "orderBy": Object {
+                "direction": "DESC",
+                "field": "UPDATED_AT",
+              },
+            },
+            "nodes": Object {
+              "body": null,
+              "number": null,
+              "state": null,
+              "title": null,
+            },
+            "pageInfo": Object {
+              "endCursor": null,
+              "hasNextPage": null,
+            },
+          },
+        },
+      },
+      "variables": Object {
+        "count": 25,
         "cursor": "cursor2",
       },
     },

lib/util/http/github.spec.ts

@@ -1,4 +1,6 @@
+import { DateTime } from 'luxon';
 import * as httpMock from '../../../test/http-mock';
+import { mocked } from '../../../test/util';
 import {
   EXTERNAL_HOST_ERROR,
   PLATFORM_BAD_CREDENTIALS,
@@ -7,9 +9,14 @@ import {
   REPOSITORY_CHANGED,
 } from '../../constants/error-messages';
 import { id as GITHUB_RELEASES_ID } from '../../datasource/github-releases';
+import * as _repositoryCache from '../../util/cache/repository';
+import type { Cache } from '../../util/cache/repository/types';
 import * as hostRules from '../host-rules';
 import { GithubHttp, setBaseUrl } from './github';
 
+jest.mock('../../util/cache/repository');
+const repositoryCache = mocked(_repositoryCache);
+
 const githubApiHost = 'https://api.github.com';
 
 const graphqlQuery = `
@@ -40,10 +47,14 @@ query(
 
 describe('util/http/github', () => {
   let githubApi: GithubHttp;
+  let repoCache: Cache = {};
+
   beforeEach(() => {
     githubApi = new GithubHttp();
     setBaseUrl(githubApiHost);
     jest.resetAllMocks();
+    repoCache = {};
+    repositoryCache.getCache.mockReturnValue(repoCache);
   });
 
   afterEach(() => {
@@ -474,6 +485,15 @@ describe('util/http/github', () => {
       expect(items).toHaveLength(2);
     });
     it('shrinks items count on 50x', async () => {
+      repoCache.platform ??= {};
+      repoCache.platform.github ??= {};
+      repoCache.platform.github.graphqlPageCache = {
+        testItem: {
+          pageLastResizedAt: DateTime.local().toISO(),
+          pageSize: 50,
+        },
+      };
+
       httpMock
         .scope(githubApiHost)
         .post('/graphql')
@@ -488,11 +508,42 @@ describe('util/http/github', () => {
       const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
       expect(items).toHaveLength(3);
 
+      expect(
+        repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
+      ).toBe(25);
+
       const trace = httpMock.getTrace();
       expect(trace).toHaveLength(4);
       expect(trace).toMatchSnapshot();
     });
+    it('expands items count on timeout', async () => {
+      repoCache.platform ??= {};
+      repoCache.platform.github ??= {};
+      repoCache.platform.github.graphqlPageCache = {
+        testItem: {
+          pageLastResizedAt: DateTime.local()
+            .minus({ hours: 24, seconds: 1 })
+            .toISO(),
+          pageSize: 42,
+        },
+      };
 
+      httpMock
+        .scope(githubApiHost)
+        .post('/graphql')
+        .reply(200, page1)
+        .post('/graphql')
+        .reply(200, page2)
+        .post('/graphql')
+        .reply(200, page3);
+
+      const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
+      expect(items).toHaveLength(3);
+      expect(
+        repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
+      ).toBe(84);
+      expect(httpMock.getTrace()).toMatchSnapshot();
+    });
     it('continues to iterate with a lower page size on error 502', async () => {
       httpMock
         .scope(githubApiHost)
@@ -507,8 +558,41 @@ describe('util/http/github', () => {
 
       const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
       expect(items).toHaveLength(3);
-    });
 
+      const trace = httpMock.getTrace();
+      expect(trace).toHaveLength(4);
+    });
+    it('removes cache record once expanded to the maximum', async () => {
+      repoCache.platform ??= {};
+      repoCache.platform.github ??= {};
+      repoCache.platform.github.graphqlPageCache = {
+        testItem: {
+          pageLastResizedAt: DateTime.local()
+            .minus({ hours: 24, seconds: 1 })
+            .toISO(),
+          pageSize: 50,
+        },
+      };
+
+      httpMock
+        .scope(githubApiHost)
+        .post('/graphql')
+        .reply(200, page1)
+        .post('/graphql')
+        .reply(200, page2)
+        .post('/graphql')
+        .reply(200, page3);
+
+      const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
+      expect(items).toHaveLength(3);
+
+      expect(
+        repoCache?.platform?.github?.graphqlPageCache?.testItem
+      ).toBeUndefined();
+
+      const trace = httpMock.getTrace();
+      expect(trace).toHaveLength(3);
+    });
     it('throws on 50x if count < 10', async () => {
       httpMock.scope(githubApiHost).post('/graphql').reply(500);
       await expect(

lib/util/http/github.ts

@@ -1,4 +1,5 @@
 import is from '@sindresorhus/is';
+import { DateTime } from 'luxon';
 import pAll from 'p-all';
 import { PlatformId } from '../../constants';
 import {
@@ -9,6 +10,7 @@ import {
 } from '../../constants/error-messages';
 import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
+import { getCache } from '../../util/cache/repository';
 import { maskToken } from '../mask';
 import { range } from '../range';
 import { regEx } from '../regex';
@@ -180,6 +182,77 @@ function constructAcceptString(input?: any): string {
   return acceptStrings.join(', ');
 }
 
+const MAX_GRAPHQL_PAGE_SIZE = 100;
+
+function getGraphqlPageSize(
+  fieldName: string,
+  defaultPageSize = MAX_GRAPHQL_PAGE_SIZE
+): number {
+  const cache = getCache();
+  const graphqlPageCache = cache?.platform?.github?.graphqlPageCache;
+  const cachedRecord = graphqlPageCache?.[fieldName];
+
+  if (graphqlPageCache && cachedRecord) {
+    logger.debug(
+      { fieldName, ...cachedRecord },
+      'GraphQL page size: found cached value'
+    );
+
+    const oldPageSize = cachedRecord.pageSize;
+
+    const now = DateTime.local();
+    const then = DateTime.fromISO(cachedRecord.pageLastResizedAt);
+    const expiry = then.plus({ hours: 24 });
+    if (now > expiry) {
+      const newPageSize = Math.min(oldPageSize * 2, MAX_GRAPHQL_PAGE_SIZE);
+      if (newPageSize < MAX_GRAPHQL_PAGE_SIZE) {
+        const timestamp = now.toISO();
+
+        logger.debug(
+          { fieldName, oldPageSize, newPageSize, timestamp },
+          'GraphQL page size: expanding'
+        );
+
+        cachedRecord.pageLastResizedAt = timestamp;
+        cachedRecord.pageSize = newPageSize;
+      } else {
+        logger.debug(
+          { fieldName, oldPageSize, newPageSize },
+          'GraphQL page size: expanded to default page size'
+        );
+
+        delete graphqlPageCache[fieldName];
+      }
+
+      return newPageSize;
+    }
+
+    return oldPageSize;
+  }
+
+  return defaultPageSize;
+}
+
+function setGraphqlPageSize(fieldName: string, newPageSize: number): void {
+  const oldPageSize = getGraphqlPageSize(fieldName);
+  if (newPageSize !== oldPageSize) {
+    const now = DateTime.local();
+    const pageLastResizedAt = now.toISO();
+    logger.debug(
+      { fieldName, oldPageSize, newPageSize, timestamp: pageLastResizedAt },
+      'GraphQL page size: shrinking'
+    );
+    const cache = getCache();
+    cache.platform ??= {};
+    cache.platform.github ??= {};
+    cache.platform.github.graphqlPageCache ??= {};
+    cache.platform.github.graphqlPageCache[fieldName] = {
+      pageLastResizedAt,
+      pageSize: newPageSize,
+    };
+  }
+}
+
 export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
   constructor(
     hostType: string = PlatformId.Github,
@@ -263,7 +336,7 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
   ): Promise<GithubGraphqlResponse<T> | null> {
     const path = 'graphql';
 
-    const { paginate, count = 100, cursor = null } = options;
+    const { paginate, count = MAX_GRAPHQL_PAGE_SIZE, cursor = null } = options;
     let { variables } = options;
     if (paginate) {
       variables = {
@@ -308,8 +381,10 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
     const { paginate = true } = options;
 
     let optimalCount: null | number = null;
-    const initialCount = options.count ?? 100;
+    let count = getGraphqlPageSize(
-    let count = initialCount;
+      fieldName,
+      options.count ?? MAX_GRAPHQL_PAGE_SIZE
+    );
     let limit = options.limit ?? 1000;
     let cursor: string | null = null;
 
@@ -362,17 +437,8 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
       }
     }
 
-    // See: https://github.com/renovatebot/renovate/issues/12703
+    if (optimalCount && optimalCount < MAX_GRAPHQL_PAGE_SIZE) {
-    // istanbul ignore if
+      setGraphqlPageSize(fieldName, optimalCount);
-    if (
-      optimalCount &&
-      optimalCount < initialCount && // log only shrinked results
-      baseUrl === githubBaseUrl
-    ) {
-      logger.debug(
-        { fieldName, optimalCount },
-        'Successful GraphQL query with shrinked pagination size'
-      );
     }
 
     return result;