mirror of
https://github.com/renovatebot/renovate.git
synced 2025-01-11 22:46:27 +00:00
feat(github): Remember GraphQL optimal page size (#13047)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
This commit is contained in:
Sergei Zharinov
GitHub
parent
3242ce7bcd
commit
3b14ef2869
4 changed files with 403 additions and 73 deletions
10
lib/util/cache/repository/types.ts
vendored
10
lib/util/cache/repository/types.ts
vendored
|
|
@ -31,6 +31,11 @@ export interface BranchCache {
|
|||
upgrades: BranchUpgradeCache[];
|
||||
}
|
||||
|
||||
export interface GithubGraphqlPageCache {
|
||||
pageLastResizedAt: string;
|
||||
pageSize: number;
|
||||
}
|
||||
|
||||
export interface Cache {
|
||||
configFileName?: string;
|
||||
semanticCommits?: 'enabled' | 'disabled';
|
||||
|
|
@ -40,4 +45,9 @@ export interface Cache {
|
|||
init?: RepoInitConfig;
|
||||
scan?: Record<string, BaseBranchCache>;
|
||||
lastPlatformAutomergeFailure?: string;
|
||||
platform?: {
|
||||
github?: {
|
||||
graphqlPageCache?: Record<string, GithubGraphqlPageCache>;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,175 @@
|
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`util/http/github GraphQL expands items count on timeout 1`] = `
|
||||
Array [
|
||||
Object {
|
||||
"graphql": Object {
|
||||
"query": Object {
|
||||
"__vars": Object {
|
||||
"$count": "Int",
|
||||
"$cursor": "String",
|
||||
"$name": "String!",
|
||||
"$owner": "String!",
|
||||
},
|
||||
"repository": Object {
|
||||
"__args": Object {
|
||||
"name": "$name",
|
||||
"owner": "$name",
|
||||
},
|
||||
"testItem": Object {
|
||||
"__args": Object {
|
||||
"after": "$cursor",
|
||||
"filterBy": Object {
|
||||
"createdBy": "someone",
|
||||
},
|
||||
"first": "$count",
|
||||
"orderBy": Object {
|
||||
"direction": "DESC",
|
||||
"field": "UPDATED_AT",
|
||||
},
|
||||
},
|
||||
"nodes": Object {
|
||||
"body": null,
|
||||
"number": null,
|
||||
"state": null,
|
||||
"title": null,
|
||||
},
|
||||
"pageInfo": Object {
|
||||
"endCursor": null,
|
||||
"hasNextPage": null,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"variables": Object {
|
||||
"count": 84,
|
||||
"cursor": null,
|
||||
},
|
||||
},
|
||||
"headers": Object {
|
||||
"accept": "application/vnd.github.v3+json",
|
||||
"accept-encoding": "gzip, deflate, br",
|
||||
"content-length": "493",
|
||||
"content-type": "application/json",
|
||||
"host": "api.github.com",
|
||||
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "https://api.github.com/graphql",
|
||||
},
|
||||
Object {
|
||||
"graphql": Object {
|
||||
"query": Object {
|
||||
"__vars": Object {
|
||||
"$count": "Int",
|
||||
"$cursor": "String",
|
||||
"$name": "String!",
|
||||
"$owner": "String!",
|
||||
},
|
||||
"repository": Object {
|
||||
"__args": Object {
|
||||
"name": "$name",
|
||||
"owner": "$name",
|
||||
},
|
||||
"testItem": Object {
|
||||
"__args": Object {
|
||||
"after": "$cursor",
|
||||
"filterBy": Object {
|
||||
"createdBy": "someone",
|
||||
},
|
||||
"first": "$count",
|
||||
"orderBy": Object {
|
||||
"direction": "DESC",
|
||||
"field": "UPDATED_AT",
|
||||
},
|
||||
},
|
||||
"nodes": Object {
|
||||
"body": null,
|
||||
"number": null,
|
||||
"state": null,
|
||||
"title": null,
|
||||
},
|
||||
"pageInfo": Object {
|
||||
"endCursor": null,
|
||||
"hasNextPage": null,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"variables": Object {
|
||||
"count": 84,
|
||||
"cursor": "cursor1",
|
||||
},
|
||||
},
|
||||
"headers": Object {
|
||||
"accept": "application/vnd.github.v3+json",
|
||||
"accept-encoding": "gzip, deflate, br",
|
||||
"content-length": "498",
|
||||
"content-type": "application/json",
|
||||
"host": "api.github.com",
|
||||
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "https://api.github.com/graphql",
|
||||
},
|
||||
Object {
|
||||
"graphql": Object {
|
||||
"query": Object {
|
||||
"__vars": Object {
|
||||
"$count": "Int",
|
||||
"$cursor": "String",
|
||||
"$name": "String!",
|
||||
"$owner": "String!",
|
||||
},
|
||||
"repository": Object {
|
||||
"__args": Object {
|
||||
"name": "$name",
|
||||
"owner": "$name",
|
||||
},
|
||||
"testItem": Object {
|
||||
"__args": Object {
|
||||
"after": "$cursor",
|
||||
"filterBy": Object {
|
||||
"createdBy": "someone",
|
||||
},
|
||||
"first": "$count",
|
||||
"orderBy": Object {
|
||||
"direction": "DESC",
|
||||
"field": "UPDATED_AT",
|
||||
},
|
||||
},
|
||||
"nodes": Object {
|
||||
"body": null,
|
||||
"number": null,
|
||||
"state": null,
|
||||
"title": null,
|
||||
},
|
||||
"pageInfo": Object {
|
||||
"endCursor": null,
|
||||
"hasNextPage": null,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"variables": Object {
|
||||
"count": 84,
|
||||
"cursor": "cursor2",
|
||||
},
|
||||
},
|
||||
"headers": Object {
|
||||
"accept": "application/vnd.github.v3+json",
|
||||
"accept-encoding": "gzip, deflate, br",
|
||||
"content-length": "498",
|
||||
"content-type": "application/json",
|
||||
"host": "api.github.com",
|
||||
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "https://api.github.com/graphql",
|
||||
},
|
||||
]
|
||||
`;
|
||||
|
||||
exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
|
||||
Array [
|
||||
Object {
|
||||
|
|
@ -42,69 +212,14 @@ Array [
|
|||
},
|
||||
},
|
||||
"variables": Object {
|
||||
"count": 100,
|
||||
"count": 50,
|
||||
"cursor": null,
|
||||
},
|
||||
},
|
||||
"headers": Object {
|
||||
"accept": "application/vnd.github.v3+json",
|
||||
"accept-encoding": "gzip, deflate, br",
|
||||
"content-length": "494",
|
||||
"content-type": "application/json",
|
||||
"host": "api.github.com",
|
||||
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "https://api.github.com/graphql",
|
||||
},
|
||||
Object {
|
||||
"graphql": Object {
|
||||
"query": Object {
|
||||
"__vars": Object {
|
||||
"$count": "Int",
|
||||
"$cursor": "String",
|
||||
"$name": "String!",
|
||||
"$owner": "String!",
|
||||
},
|
||||
"repository": Object {
|
||||
"__args": Object {
|
||||
"name": "$name",
|
||||
"owner": "$name",
|
||||
},
|
||||
"testItem": Object {
|
||||
"__args": Object {
|
||||
"after": "$cursor",
|
||||
"filterBy": Object {
|
||||
"createdBy": "someone",
|
||||
},
|
||||
"first": "$count",
|
||||
"orderBy": Object {
|
||||
"direction": "DESC",
|
||||
"field": "UPDATED_AT",
|
||||
},
|
||||
},
|
||||
"nodes": Object {
|
||||
"body": null,
|
||||
"number": null,
|
||||
"state": null,
|
||||
"title": null,
|
||||
},
|
||||
"pageInfo": Object {
|
||||
"endCursor": null,
|
||||
"hasNextPage": null,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"variables": Object {
|
||||
"count": 100,
|
||||
"cursor": "cursor1",
|
||||
},
|
||||
},
|
||||
"headers": Object {
|
||||
"accept": "application/vnd.github.v3+json",
|
||||
"accept-encoding": "gzip, deflate, br",
|
||||
"content-length": "499",
|
||||
"content-length": "493",
|
||||
"content-type": "application/json",
|
||||
"host": "api.github.com",
|
||||
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
|
||||
|
|
@ -207,7 +322,62 @@ Array [
|
|||
},
|
||||
},
|
||||
"variables": Object {
|
||||
"count": 50,
|
||||
"count": 25,
|
||||
"cursor": "cursor1",
|
||||
},
|
||||
},
|
||||
"headers": Object {
|
||||
"accept": "application/vnd.github.v3+json",
|
||||
"accept-encoding": "gzip, deflate, br",
|
||||
"content-length": "498",
|
||||
"content-type": "application/json",
|
||||
"host": "api.github.com",
|
||||
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
|
||||
},
|
||||
"method": "POST",
|
||||
"url": "https://api.github.com/graphql",
|
||||
},
|
||||
Object {
|
||||
"graphql": Object {
|
||||
"query": Object {
|
||||
"__vars": Object {
|
||||
"$count": "Int",
|
||||
"$cursor": "String",
|
||||
"$name": "String!",
|
||||
"$owner": "String!",
|
||||
},
|
||||
"repository": Object {
|
||||
"__args": Object {
|
||||
"name": "$name",
|
||||
"owner": "$name",
|
||||
},
|
||||
"testItem": Object {
|
||||
"__args": Object {
|
||||
"after": "$cursor",
|
||||
"filterBy": Object {
|
||||
"createdBy": "someone",
|
||||
},
|
||||
"first": "$count",
|
||||
"orderBy": Object {
|
||||
"direction": "DESC",
|
||||
"field": "UPDATED_AT",
|
||||
},
|
||||
},
|
||||
"nodes": Object {
|
||||
"body": null,
|
||||
"number": null,
|
||||
"state": null,
|
||||
"title": null,
|
||||
},
|
||||
"pageInfo": Object {
|
||||
"endCursor": null,
|
||||
"hasNextPage": null,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"variables": Object {
|
||||
"count": 25,
|
||||
"cursor": "cursor2",
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import { DateTime } from 'luxon';
|
||||
import * as httpMock from '../../../test/http-mock';
|
||||
import { mocked } from '../../../test/util';
|
||||
import {
|
||||
EXTERNAL_HOST_ERROR,
|
||||
PLATFORM_BAD_CREDENTIALS,
|
||||
|
|
@ -7,9 +9,14 @@ import {
|
|||
REPOSITORY_CHANGED,
|
||||
} from '../../constants/error-messages';
|
||||
import { id as GITHUB_RELEASES_ID } from '../../datasource/github-releases';
|
||||
import * as _repositoryCache from '../../util/cache/repository';
|
||||
import type { Cache } from '../../util/cache/repository/types';
|
||||
import * as hostRules from '../host-rules';
|
||||
import { GithubHttp, setBaseUrl } from './github';
|
||||
|
||||
jest.mock('../../util/cache/repository');
|
||||
const repositoryCache = mocked(_repositoryCache);
|
||||
|
||||
const githubApiHost = 'https://api.github.com';
|
||||
|
||||
const graphqlQuery = `
|
||||
|
|
@ -40,10 +47,14 @@ query(
|
|||
|
||||
describe('util/http/github', () => {
|
||||
let githubApi: GithubHttp;
|
||||
let repoCache: Cache = {};
|
||||
|
||||
beforeEach(() => {
|
||||
githubApi = new GithubHttp();
|
||||
setBaseUrl(githubApiHost);
|
||||
jest.resetAllMocks();
|
||||
repoCache = {};
|
||||
repositoryCache.getCache.mockReturnValue(repoCache);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
|
|
@ -474,6 +485,15 @@ describe('util/http/github', () => {
|
|||
expect(items).toHaveLength(2);
|
||||
});
|
||||
it('shrinks items count on 50x', async () => {
|
||||
repoCache.platform ??= {};
|
||||
repoCache.platform.github ??= {};
|
||||
repoCache.platform.github.graphqlPageCache = {
|
||||
testItem: {
|
||||
pageLastResizedAt: DateTime.local().toISO(),
|
||||
pageSize: 50,
|
||||
},
|
||||
};
|
||||
|
||||
httpMock
|
||||
.scope(githubApiHost)
|
||||
.post('/graphql')
|
||||
|
|
@ -488,11 +508,42 @@ describe('util/http/github', () => {
|
|||
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
|
||||
expect(items).toHaveLength(3);
|
||||
|
||||
expect(
|
||||
repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
|
||||
).toBe(25);
|
||||
|
||||
const trace = httpMock.getTrace();
|
||||
expect(trace).toHaveLength(4);
|
||||
expect(trace).toMatchSnapshot();
|
||||
});
|
||||
it('expands items count on timeout', async () => {
|
||||
repoCache.platform ??= {};
|
||||
repoCache.platform.github ??= {};
|
||||
repoCache.platform.github.graphqlPageCache = {
|
||||
testItem: {
|
||||
pageLastResizedAt: DateTime.local()
|
||||
.minus({ hours: 24, seconds: 1 })
|
||||
.toISO(),
|
||||
pageSize: 42,
|
||||
},
|
||||
};
|
||||
|
||||
httpMock
|
||||
.scope(githubApiHost)
|
||||
.post('/graphql')
|
||||
.reply(200, page1)
|
||||
.post('/graphql')
|
||||
.reply(200, page2)
|
||||
.post('/graphql')
|
||||
.reply(200, page3);
|
||||
|
||||
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
|
||||
expect(items).toHaveLength(3);
|
||||
expect(
|
||||
repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
|
||||
).toBe(84);
|
||||
expect(httpMock.getTrace()).toMatchSnapshot();
|
||||
});
|
||||
it('continues to iterate with a lower page size on error 502', async () => {
|
||||
httpMock
|
||||
.scope(githubApiHost)
|
||||
|
|
@ -507,8 +558,41 @@ describe('util/http/github', () => {
|
|||
|
||||
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
|
||||
expect(items).toHaveLength(3);
|
||||
});
|
||||
|
||||
const trace = httpMock.getTrace();
|
||||
expect(trace).toHaveLength(4);
|
||||
});
|
||||
it('removes cache record once expanded to the maximum', async () => {
|
||||
repoCache.platform ??= {};
|
||||
repoCache.platform.github ??= {};
|
||||
repoCache.platform.github.graphqlPageCache = {
|
||||
testItem: {
|
||||
pageLastResizedAt: DateTime.local()
|
||||
.minus({ hours: 24, seconds: 1 })
|
||||
.toISO(),
|
||||
pageSize: 50,
|
||||
},
|
||||
};
|
||||
|
||||
httpMock
|
||||
.scope(githubApiHost)
|
||||
.post('/graphql')
|
||||
.reply(200, page1)
|
||||
.post('/graphql')
|
||||
.reply(200, page2)
|
||||
.post('/graphql')
|
||||
.reply(200, page3);
|
||||
|
||||
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
|
||||
expect(items).toHaveLength(3);
|
||||
|
||||
expect(
|
||||
repoCache?.platform?.github?.graphqlPageCache?.testItem
|
||||
).toBeUndefined();
|
||||
|
||||
const trace = httpMock.getTrace();
|
||||
expect(trace).toHaveLength(3);
|
||||
});
|
||||
it('throws on 50x if count < 10', async () => {
|
||||
httpMock.scope(githubApiHost).post('/graphql').reply(500);
|
||||
await expect(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import is from '@sindresorhus/is';
|
||||
import { DateTime } from 'luxon';
|
||||
import pAll from 'p-all';
|
||||
import { PlatformId } from '../../constants';
|
||||
import {
|
||||
|
|
@ -9,6 +10,7 @@ import {
|
|||
} from '../../constants/error-messages';
|
||||
import { logger } from '../../logger';
|
||||
import { ExternalHostError } from '../../types/errors/external-host-error';
|
||||
import { getCache } from '../../util/cache/repository';
|
||||
import { maskToken } from '../mask';
|
||||
import { range } from '../range';
|
||||
import { regEx } from '../regex';
|
||||
|
|
@ -180,6 +182,77 @@ function constructAcceptString(input?: any): string {
|
|||
return acceptStrings.join(', ');
|
||||
}
|
||||
|
||||
const MAX_GRAPHQL_PAGE_SIZE = 100;
|
||||
|
||||
function getGraphqlPageSize(
|
||||
fieldName: string,
|
||||
defaultPageSize = MAX_GRAPHQL_PAGE_SIZE
|
||||
): number {
|
||||
const cache = getCache();
|
||||
const graphqlPageCache = cache?.platform?.github?.graphqlPageCache;
|
||||
const cachedRecord = graphqlPageCache?.[fieldName];
|
||||
|
||||
if (graphqlPageCache && cachedRecord) {
|
||||
logger.debug(
|
||||
{ fieldName, ...cachedRecord },
|
||||
'GraphQL page size: found cached value'
|
||||
);
|
||||
|
||||
const oldPageSize = cachedRecord.pageSize;
|
||||
|
||||
const now = DateTime.local();
|
||||
const then = DateTime.fromISO(cachedRecord.pageLastResizedAt);
|
||||
const expiry = then.plus({ hours: 24 });
|
||||
if (now > expiry) {
|
||||
const newPageSize = Math.min(oldPageSize * 2, MAX_GRAPHQL_PAGE_SIZE);
|
||||
if (newPageSize < MAX_GRAPHQL_PAGE_SIZE) {
|
||||
const timestamp = now.toISO();
|
||||
|
||||
logger.debug(
|
||||
{ fieldName, oldPageSize, newPageSize, timestamp },
|
||||
'GraphQL page size: expanding'
|
||||
);
|
||||
|
||||
cachedRecord.pageLastResizedAt = timestamp;
|
||||
cachedRecord.pageSize = newPageSize;
|
||||
} else {
|
||||
logger.debug(
|
||||
{ fieldName, oldPageSize, newPageSize },
|
||||
'GraphQL page size: expanded to default page size'
|
||||
);
|
||||
|
||||
delete graphqlPageCache[fieldName];
|
||||
}
|
||||
|
||||
return newPageSize;
|
||||
}
|
||||
|
||||
return oldPageSize;
|
||||
}
|
||||
|
||||
return defaultPageSize;
|
||||
}
|
||||
|
||||
function setGraphqlPageSize(fieldName: string, newPageSize: number): void {
|
||||
const oldPageSize = getGraphqlPageSize(fieldName);
|
||||
if (newPageSize !== oldPageSize) {
|
||||
const now = DateTime.local();
|
||||
const pageLastResizedAt = now.toISO();
|
||||
logger.debug(
|
||||
{ fieldName, oldPageSize, newPageSize, timestamp: pageLastResizedAt },
|
||||
'GraphQL page size: shrinking'
|
||||
);
|
||||
const cache = getCache();
|
||||
cache.platform ??= {};
|
||||
cache.platform.github ??= {};
|
||||
cache.platform.github.graphqlPageCache ??= {};
|
||||
cache.platform.github.graphqlPageCache[fieldName] = {
|
||||
pageLastResizedAt,
|
||||
pageSize: newPageSize,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
|
||||
constructor(
|
||||
hostType: string = PlatformId.Github,
|
||||
|
|
@ -263,7 +336,7 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
|
|||
): Promise<GithubGraphqlResponse<T> | null> {
|
||||
const path = 'graphql';
|
||||
|
||||
const { paginate, count = 100, cursor = null } = options;
|
||||
const { paginate, count = MAX_GRAPHQL_PAGE_SIZE, cursor = null } = options;
|
||||
let { variables } = options;
|
||||
if (paginate) {
|
||||
variables = {
|
||||
|
|
@ -308,8 +381,10 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
|
|||
const { paginate = true } = options;
|
||||
|
||||
let optimalCount: null | number = null;
|
||||
const initialCount = options.count ?? 100;
|
||||
let count = initialCount;
|
||||
let count = getGraphqlPageSize(
|
||||
fieldName,
|
||||
options.count ?? MAX_GRAPHQL_PAGE_SIZE
|
||||
);
|
||||
let limit = options.limit ?? 1000;
|
||||
let cursor: string | null = null;
|
||||
|
||||
|
|
@ -362,17 +437,8 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
|
|||
}
|
||||
}
|
||||
|
||||
// See: https://github.com/renovatebot/renovate/issues/12703
|
||||
// istanbul ignore if
|
||||
if (
|
||||
optimalCount &&
|
||||
optimalCount < initialCount && // log only shrinked results
|
||||
baseUrl === githubBaseUrl
|
||||
) {
|
||||
logger.debug(
|
||||
{ fieldName, optimalCount },
|
||||
'Successful GraphQL query with shrinked pagination size'
|
||||
);
|
||||
if (optimalCount && optimalCount < MAX_GRAPHQL_PAGE_SIZE) {
|
||||
setGraphqlPageSize(fieldName, optimalCount);
|
||||
}
|
||||
|
||||
return result;
|
||||
|
|
|
|||
Loading…
Reference in a new issue