Skip to content
Snippets Groups Projects
Unverified Commit 3b14ef28 authored by Sergei Zharinov's avatar Sergei Zharinov Committed by GitHub
Browse files

feat(github): Remember GraphQL optimal page size (#13047) 


Co-authored-by: default avatarRhys Arkins <rhys@arkins.net>
parent 3242ce7b
No related merge requests found
Hide whitespace changes
Inline Side-by-side
lib/util/cache/repository/types.ts
+ 10
− 0
View file @ 3b14ef28
......@@ -31,6 +31,11 @@ export interface BranchCache {
upgrades: BranchUpgradeCache[];
}
export interface GithubGraphqlPageCache {
pageLastResizedAt: string;
pageSize: number;
}
export interface Cache {
configFileName?: string;
semanticCommits?: 'enabled' | 'disabled';
......@@ -40,4 +45,9 @@ export interface Cache {
init?: RepoInitConfig;
scan?: Record<string, BaseBranchCache>;
lastPlatformAutomergeFailure?: string;
platform?: {
github?: {
graphqlPageCache?: Record<string, GithubGraphqlPageCache>;
};
};
}
lib/util/http/__snapshots__/github.spec.ts.snap
+ 176
− 6
View file @ 3b14ef28
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
exports[`util/http/github GraphQL expands items count on timeout 1`] = `
Array [
Object {
"graphql": Object {
......@@ -42,14 +42,14 @@ Array [
},
},
"variables": Object {
"count": 100,
"count": 84,
"cursor": null,
},
},
"headers": Object {
"accept": "application/vnd.github.v3+json",
"accept-encoding": "gzip, deflate, br",
"content-length": "494",
"content-length": "493",
"content-type": "application/json",
"host": "api.github.com",
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
......@@ -97,14 +97,129 @@ Array [
},
},
"variables": Object {
"count": 100,
"count": 84,
"cursor": "cursor1",
},
},
"headers": Object {
"accept": "application/vnd.github.v3+json",
"accept-encoding": "gzip, deflate, br",
"content-length": "499",
"content-length": "498",
"content-type": "application/json",
"host": "api.github.com",
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
},
"method": "POST",
"url": "https://api.github.com/graphql",
},
Object {
"graphql": Object {
"query": Object {
"__vars": Object {
"$count": "Int",
"$cursor": "String",
"$name": "String!",
"$owner": "String!",
},
"repository": Object {
"__args": Object {
"name": "$name",
"owner": "$name",
},
"testItem": Object {
"__args": Object {
"after": "$cursor",
"filterBy": Object {
"createdBy": "someone",
},
"first": "$count",
"orderBy": Object {
"direction": "DESC",
"field": "UPDATED_AT",
},
},
"nodes": Object {
"body": null,
"number": null,
"state": null,
"title": null,
},
"pageInfo": Object {
"endCursor": null,
"hasNextPage": null,
},
},
},
},
"variables": Object {
"count": 84,
"cursor": "cursor2",
},
},
"headers": Object {
"accept": "application/vnd.github.v3+json",
"accept-encoding": "gzip, deflate, br",
"content-length": "498",
"content-type": "application/json",
"host": "api.github.com",
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
},
"method": "POST",
"url": "https://api.github.com/graphql",
},
]
`;
exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
Array [
Object {
"graphql": Object {
"query": Object {
"__vars": Object {
"$count": "Int",
"$cursor": "String",
"$name": "String!",
"$owner": "String!",
},
"repository": Object {
"__args": Object {
"name": "$name",
"owner": "$name",
},
"testItem": Object {
"__args": Object {
"after": "$cursor",
"filterBy": Object {
"createdBy": "someone",
},
"first": "$count",
"orderBy": Object {
"direction": "DESC",
"field": "UPDATED_AT",
},
},
"nodes": Object {
"body": null,
"number": null,
"state": null,
"title": null,
},
"pageInfo": Object {
"endCursor": null,
"hasNextPage": null,
},
},
},
},
"variables": Object {
"count": 50,
"cursor": null,
},
},
"headers": Object {
"accept": "application/vnd.github.v3+json",
"accept-encoding": "gzip, deflate, br",
"content-length": "493",
"content-type": "application/json",
"host": "api.github.com",
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
......@@ -207,7 +322,62 @@ Array [
},
},
"variables": Object {
"count": 50,
"count": 25,
"cursor": "cursor1",
},
},
"headers": Object {
"accept": "application/vnd.github.v3+json",
"accept-encoding": "gzip, deflate, br",
"content-length": "498",
"content-type": "application/json",
"host": "api.github.com",
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
},
"method": "POST",
"url": "https://api.github.com/graphql",
},
Object {
"graphql": Object {
"query": Object {
"__vars": Object {
"$count": "Int",
"$cursor": "String",
"$name": "String!",
"$owner": "String!",
},
"repository": Object {
"__args": Object {
"name": "$name",
"owner": "$name",
},
"testItem": Object {
"__args": Object {
"after": "$cursor",
"filterBy": Object {
"createdBy": "someone",
},
"first": "$count",
"orderBy": Object {
"direction": "DESC",
"field": "UPDATED_AT",
},
},
"nodes": Object {
"body": null,
"number": null,
"state": null,
"title": null,
},
"pageInfo": Object {
"endCursor": null,
"hasNextPage": null,
},
},
},
},
"variables": Object {
"count": 25,
"cursor": "cursor2",
},
},
......
lib/util/http/github.spec.ts
+ 84
− 0
View file @ 3b14ef28
import { DateTime } from 'luxon';
import * as httpMock from '../../../test/http-mock';
import { mocked } from '../../../test/util';
import {
EXTERNAL_HOST_ERROR,
PLATFORM_BAD_CREDENTIALS,
......@@ -7,9 +9,14 @@ import {
REPOSITORY_CHANGED,
} from '../../constants/error-messages';
import { id as GITHUB_RELEASES_ID } from '../../datasource/github-releases';
import * as _repositoryCache from '../../util/cache/repository';
import type { Cache } from '../../util/cache/repository/types';
import * as hostRules from '../host-rules';
import { GithubHttp, setBaseUrl } from './github';
jest.mock('../../util/cache/repository');
const repositoryCache = mocked(_repositoryCache);
const githubApiHost = 'https://api.github.com';
const graphqlQuery = `
......@@ -40,10 +47,14 @@ query(
describe('util/http/github', () => {
let githubApi: GithubHttp;
let repoCache: Cache = {};
beforeEach(() => {
githubApi = new GithubHttp();
setBaseUrl(githubApiHost);
jest.resetAllMocks();
repoCache = {};
repositoryCache.getCache.mockReturnValue(repoCache);
});
afterEach(() => {
......@@ -474,6 +485,15 @@ describe('util/http/github', () => {
expect(items).toHaveLength(2);
});
it('shrinks items count on 50x', async () => {
repoCache.platform ??= {};
repoCache.platform.github ??= {};
repoCache.platform.github.graphqlPageCache = {
testItem: {
pageLastResizedAt: DateTime.local().toISO(),
pageSize: 50,
},
};
httpMock
.scope(githubApiHost)
.post('/graphql')
......@@ -488,11 +508,42 @@ describe('util/http/github', () => {
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
expect(items).toHaveLength(3);
expect(
repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
).toBe(25);
const trace = httpMock.getTrace();
expect(trace).toHaveLength(4);
expect(trace).toMatchSnapshot();
});
it('expands items count on timeout', async () => {
repoCache.platform ??= {};
repoCache.platform.github ??= {};
repoCache.platform.github.graphqlPageCache = {
testItem: {
pageLastResizedAt: DateTime.local()
.minus({ hours: 24, seconds: 1 })
.toISO(),
pageSize: 42,
},
};
httpMock
.scope(githubApiHost)
.post('/graphql')
.reply(200, page1)
.post('/graphql')
.reply(200, page2)
.post('/graphql')
.reply(200, page3);
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
expect(items).toHaveLength(3);
expect(
repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
).toBe(84);
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('continues to iterate with a lower page size on error 502', async () => {
httpMock
.scope(githubApiHost)
......@@ -507,8 +558,41 @@ describe('util/http/github', () => {
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
expect(items).toHaveLength(3);
const trace = httpMock.getTrace();
expect(trace).toHaveLength(4);
});
it('removes cache record once expanded to the maximum', async () => {
repoCache.platform ??= {};
repoCache.platform.github ??= {};
repoCache.platform.github.graphqlPageCache = {
testItem: {
pageLastResizedAt: DateTime.local()
.minus({ hours: 24, seconds: 1 })
.toISO(),
pageSize: 50,
},
};
httpMock
.scope(githubApiHost)
.post('/graphql')
.reply(200, page1)
.post('/graphql')
.reply(200, page2)
.post('/graphql')
.reply(200, page3);
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
expect(items).toHaveLength(3);
expect(
repoCache?.platform?.github?.graphqlPageCache?.testItem
).toBeUndefined();
const trace = httpMock.getTrace();
expect(trace).toHaveLength(3);
});
it('throws on 50x if count < 10', async () => {
httpMock.scope(githubApiHost).post('/graphql').reply(500);
await expect(
......
lib/util/http/github.ts
+ 80
− 14
View file @ 3b14ef28
import is from '@sindresorhus/is';
import { DateTime } from 'luxon';
import pAll from 'p-all';
import { PlatformId } from '../../constants';
import {
......@@ -9,6 +10,7 @@ import {
} from '../../constants/error-messages';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import { getCache } from '../../util/cache/repository';
import { maskToken } from '../mask';
import { range } from '../range';
import { regEx } from '../regex';
......@@ -180,6 +182,77 @@ function constructAcceptString(input?: any): string {
return acceptStrings.join(', ');
}
const MAX_GRAPHQL_PAGE_SIZE = 100;
function getGraphqlPageSize(
fieldName: string,
defaultPageSize = MAX_GRAPHQL_PAGE_SIZE
): number {
const cache = getCache();
const graphqlPageCache = cache?.platform?.github?.graphqlPageCache;
const cachedRecord = graphqlPageCache?.[fieldName];
if (graphqlPageCache && cachedRecord) {
logger.debug(
{ fieldName, ...cachedRecord },
'GraphQL page size: found cached value'
);
const oldPageSize = cachedRecord.pageSize;
const now = DateTime.local();
const then = DateTime.fromISO(cachedRecord.pageLastResizedAt);
const expiry = then.plus({ hours: 24 });
if (now > expiry) {
const newPageSize = Math.min(oldPageSize * 2, MAX_GRAPHQL_PAGE_SIZE);
if (newPageSize < MAX_GRAPHQL_PAGE_SIZE) {
const timestamp = now.toISO();
logger.debug(
{ fieldName, oldPageSize, newPageSize, timestamp },
'GraphQL page size: expanding'
);
cachedRecord.pageLastResizedAt = timestamp;
cachedRecord.pageSize = newPageSize;
} else {
logger.debug(
{ fieldName, oldPageSize, newPageSize },
'GraphQL page size: expanded to default page size'
);
delete graphqlPageCache[fieldName];
}
return newPageSize;
}
return oldPageSize;
}
return defaultPageSize;
}
function setGraphqlPageSize(fieldName: string, newPageSize: number): void {
const oldPageSize = getGraphqlPageSize(fieldName);
if (newPageSize !== oldPageSize) {
const now = DateTime.local();
const pageLastResizedAt = now.toISO();
logger.debug(
{ fieldName, oldPageSize, newPageSize, timestamp: pageLastResizedAt },
'GraphQL page size: shrinking'
);
const cache = getCache();
cache.platform ??= {};
cache.platform.github ??= {};
cache.platform.github.graphqlPageCache ??= {};
cache.platform.github.graphqlPageCache[fieldName] = {
pageLastResizedAt,
pageSize: newPageSize,
};
}
}
export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
constructor(
hostType: string = PlatformId.Github,
......@@ -263,7 +336,7 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
): Promise<GithubGraphqlResponse<T> | null> {
const path = 'graphql';
const { paginate, count = 100, cursor = null } = options;
const { paginate, count = MAX_GRAPHQL_PAGE_SIZE, cursor = null } = options;
let { variables } = options;
if (paginate) {
variables = {
......@@ -308,8 +381,10 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
const { paginate = true } = options;
let optimalCount: null | number = null;
const initialCount = options.count ?? 100;
let count = initialCount;
let count = getGraphqlPageSize(
fieldName,
options.count ?? MAX_GRAPHQL_PAGE_SIZE
);
let limit = options.limit ?? 1000;
let cursor: string | null = null;
......@@ -362,17 +437,8 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
}
}
// See: https://github.com/renovatebot/renovate/issues/12703
// istanbul ignore if
if (
optimalCount &&
optimalCount < initialCount && // log only shrinked results
baseUrl === githubBaseUrl
) {
logger.debug(
{ fieldName, optimalCount },
'Successful GraphQL query with shrinked pagination size'
);
if (optimalCount && optimalCount < MAX_GRAPHQL_PAGE_SIZE) {
setGraphqlPageSize(fieldName, optimalCount);
}
return result;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment