From 3b14ef286903135c39f735b8ce62ff379c3fd2ab Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Tue, 18 Jan 2022 18:36:44 +0300
Subject: [PATCH] feat(github): Remember GraphQL optimal page size (#13047)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
lib/util/cache/repository/types.ts | 10 +
.../http/__snapshots__/github.spec.ts.snap | 182 +++++++++++++++++-
lib/util/http/github.spec.ts | 84 ++++++++
lib/util/http/github.ts | 94 +++++++--
4 files changed, 350 insertions(+), 20 deletions(-)
diff --git a/lib/util/cache/repository/types.ts b/lib/util/cache/repository/types.ts
index d3e7d073ba..5f1ce36796 100644
--- a/lib/util/cache/repository/types.ts
+++ b/lib/util/cache/repository/types.ts
@@ -31,6 +31,11 @@ export interface BranchCache {
upgrades: BranchUpgradeCache[];
}
+export interface GithubGraphqlPageCache {
+ pageLastResizedAt: string;
+ pageSize: number;
+}
+
export interface Cache {
configFileName?: string;
semanticCommits?: 'enabled' | 'disabled';
@@ -40,4 +45,9 @@ export interface Cache {
init?: RepoInitConfig;
scan?: Record<string, BaseBranchCache>;
lastPlatformAutomergeFailure?: string;
+ platform?: {
+ github?: {
+ graphqlPageCache?: Record<string, GithubGraphqlPageCache>;
+ };
+ };
}
diff --git a/lib/util/http/__snapshots__/github.spec.ts.snap b/lib/util/http/__snapshots__/github.spec.ts.snap
index c555f9ce02..30e484a0a3 100644
--- a/lib/util/http/__snapshots__/github.spec.ts.snap
+++ b/lib/util/http/__snapshots__/github.spec.ts.snap
@@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
-exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
+exports[`util/http/github GraphQL expands items count on timeout 1`] = `
Array [
Object {
"graphql": Object {
@@ -42,14 +42,14 @@ Array [
},
},
"variables": Object {
- "count": 100,
+ "count": 84,
"cursor": null,
},
},
"headers": Object {
"accept": "application/vnd.github.v3+json",
"accept-encoding": "gzip, deflate, br",
- "content-length": "494",
+ "content-length": "493",
"content-type": "application/json",
"host": "api.github.com",
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
@@ -97,14 +97,129 @@ Array [
},
},
"variables": Object {
- "count": 100,
+ "count": 84,
"cursor": "cursor1",
},
},
"headers": Object {
"accept": "application/vnd.github.v3+json",
"accept-encoding": "gzip, deflate, br",
- "content-length": "499",
+ "content-length": "498",
+ "content-type": "application/json",
+ "host": "api.github.com",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "POST",
+ "url": "https://api.github.com/graphql",
+ },
+ Object {
+ "graphql": Object {
+ "query": Object {
+ "__vars": Object {
+ "$count": "Int",
+ "$cursor": "String",
+ "$name": "String!",
+ "$owner": "String!",
+ },
+ "repository": Object {
+ "__args": Object {
+ "name": "$name",
+ "owner": "$name",
+ },
+ "testItem": Object {
+ "__args": Object {
+ "after": "$cursor",
+ "filterBy": Object {
+ "createdBy": "someone",
+ },
+ "first": "$count",
+ "orderBy": Object {
+ "direction": "DESC",
+ "field": "UPDATED_AT",
+ },
+ },
+ "nodes": Object {
+ "body": null,
+ "number": null,
+ "state": null,
+ "title": null,
+ },
+ "pageInfo": Object {
+ "endCursor": null,
+ "hasNextPage": null,
+ },
+ },
+ },
+ },
+ "variables": Object {
+ "count": 84,
+ "cursor": "cursor2",
+ },
+ },
+ "headers": Object {
+ "accept": "application/vnd.github.v3+json",
+ "accept-encoding": "gzip, deflate, br",
+ "content-length": "498",
+ "content-type": "application/json",
+ "host": "api.github.com",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "POST",
+ "url": "https://api.github.com/graphql",
+ },
+]
+`;
+
+exports[`util/http/github GraphQL shrinks items count on 50x 1`] = `
+Array [
+ Object {
+ "graphql": Object {
+ "query": Object {
+ "__vars": Object {
+ "$count": "Int",
+ "$cursor": "String",
+ "$name": "String!",
+ "$owner": "String!",
+ },
+ "repository": Object {
+ "__args": Object {
+ "name": "$name",
+ "owner": "$name",
+ },
+ "testItem": Object {
+ "__args": Object {
+ "after": "$cursor",
+ "filterBy": Object {
+ "createdBy": "someone",
+ },
+ "first": "$count",
+ "orderBy": Object {
+ "direction": "DESC",
+ "field": "UPDATED_AT",
+ },
+ },
+ "nodes": Object {
+ "body": null,
+ "number": null,
+ "state": null,
+ "title": null,
+ },
+ "pageInfo": Object {
+ "endCursor": null,
+ "hasNextPage": null,
+ },
+ },
+ },
+ },
+ "variables": Object {
+ "count": 50,
+ "cursor": null,
+ },
+ },
+ "headers": Object {
+ "accept": "application/vnd.github.v3+json",
+ "accept-encoding": "gzip, deflate, br",
+ "content-length": "493",
"content-type": "application/json",
"host": "api.github.com",
"user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
@@ -207,7 +322,62 @@ Array [
},
},
"variables": Object {
- "count": 50,
+ "count": 25,
+ "cursor": "cursor1",
+ },
+ },
+ "headers": Object {
+ "accept": "application/vnd.github.v3+json",
+ "accept-encoding": "gzip, deflate, br",
+ "content-length": "498",
+ "content-type": "application/json",
+ "host": "api.github.com",
+ "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+ },
+ "method": "POST",
+ "url": "https://api.github.com/graphql",
+ },
+ Object {
+ "graphql": Object {
+ "query": Object {
+ "__vars": Object {
+ "$count": "Int",
+ "$cursor": "String",
+ "$name": "String!",
+ "$owner": "String!",
+ },
+ "repository": Object {
+ "__args": Object {
+ "name": "$name",
+ "owner": "$name",
+ },
+ "testItem": Object {
+ "__args": Object {
+ "after": "$cursor",
+ "filterBy": Object {
+ "createdBy": "someone",
+ },
+ "first": "$count",
+ "orderBy": Object {
+ "direction": "DESC",
+ "field": "UPDATED_AT",
+ },
+ },
+ "nodes": Object {
+ "body": null,
+ "number": null,
+ "state": null,
+ "title": null,
+ },
+ "pageInfo": Object {
+ "endCursor": null,
+ "hasNextPage": null,
+ },
+ },
+ },
+ },
+ "variables": Object {
+ "count": 25,
"cursor": "cursor2",
},
},
diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts
index 1a36c4efcf..cbb70841a3 100644
--- a/lib/util/http/github.spec.ts
+++ b/lib/util/http/github.spec.ts
@@ -1,4 +1,6 @@
+import { DateTime } from 'luxon';
import * as httpMock from '../../../test/http-mock';
+import { mocked } from '../../../test/util';
import {
EXTERNAL_HOST_ERROR,
PLATFORM_BAD_CREDENTIALS,
@@ -7,9 +9,14 @@ import {
REPOSITORY_CHANGED,
} from '../../constants/error-messages';
import { id as GITHUB_RELEASES_ID } from '../../datasource/github-releases';
+import * as _repositoryCache from '../../util/cache/repository';
+import type { Cache } from '../../util/cache/repository/types';
import * as hostRules from '../host-rules';
import { GithubHttp, setBaseUrl } from './github';
+jest.mock('../../util/cache/repository');
+const repositoryCache = mocked(_repositoryCache);
+
const githubApiHost = 'https://api.github.com';
const graphqlQuery = `
@@ -40,10 +47,14 @@ query(
describe('util/http/github', () => {
let githubApi: GithubHttp;
+ let repoCache: Cache = {};
+
beforeEach(() => {
githubApi = new GithubHttp();
setBaseUrl(githubApiHost);
jest.resetAllMocks();
+ repoCache = {};
+ repositoryCache.getCache.mockReturnValue(repoCache);
});
afterEach(() => {
@@ -474,6 +485,15 @@ describe('util/http/github', () => {
expect(items).toHaveLength(2);
});
it('shrinks items count on 50x', async () => {
+ repoCache.platform ??= {};
+ repoCache.platform.github ??= {};
+ repoCache.platform.github.graphqlPageCache = {
+ testItem: {
+ pageLastResizedAt: DateTime.local().toISO(),
+ pageSize: 50,
+ },
+ };
+
httpMock
.scope(githubApiHost)
.post('/graphql')
@@ -488,11 +508,42 @@ describe('util/http/github', () => {
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
expect(items).toHaveLength(3);
+ expect(
+ repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
+ ).toBe(25);
+
const trace = httpMock.getTrace();
expect(trace).toHaveLength(4);
expect(trace).toMatchSnapshot();
});
+ it('expands items count on timeout', async () => {
+ repoCache.platform ??= {};
+ repoCache.platform.github ??= {};
+ repoCache.platform.github.graphqlPageCache = {
+ testItem: {
+ pageLastResizedAt: DateTime.local()
+ .minus({ hours: 24, seconds: 1 })
+ .toISO(),
+ pageSize: 42,
+ },
+ };
+
+ httpMock
+ .scope(githubApiHost)
+ .post('/graphql')
+ .reply(200, page1)
+ .post('/graphql')
+ .reply(200, page2)
+ .post('/graphql')
+ .reply(200, page3);
+ const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
+ expect(items).toHaveLength(3);
+ expect(
+ repoCache?.platform?.github?.graphqlPageCache?.testItem?.pageSize
+ ).toBe(84);
+ expect(httpMock.getTrace()).toMatchSnapshot();
+ });
it('continues to iterate with a lower page size on error 502', async () => {
httpMock
.scope(githubApiHost)
@@ -507,8 +558,41 @@ describe('util/http/github', () => {
const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
expect(items).toHaveLength(3);
+
+ const trace = httpMock.getTrace();
+ expect(trace).toHaveLength(4);
});
+ it('removes cache record once expanded to the maximum', async () => {
+ repoCache.platform ??= {};
+ repoCache.platform.github ??= {};
+ repoCache.platform.github.graphqlPageCache = {
+ testItem: {
+ pageLastResizedAt: DateTime.local()
+ .minus({ hours: 24, seconds: 1 })
+ .toISO(),
+ pageSize: 50,
+ },
+ };
+
+ httpMock
+ .scope(githubApiHost)
+ .post('/graphql')
+ .reply(200, page1)
+ .post('/graphql')
+ .reply(200, page2)
+ .post('/graphql')
+ .reply(200, page3);
+
+ const items = await githubApi.queryRepoField(graphqlQuery, 'testItem');
+ expect(items).toHaveLength(3);
+ expect(
+ repoCache?.platform?.github?.graphqlPageCache?.testItem
+ ).toBeUndefined();
+
+ const trace = httpMock.getTrace();
+ expect(trace).toHaveLength(3);
+ });
it('throws on 50x if count < 10', async () => {
httpMock.scope(githubApiHost).post('/graphql').reply(500);
await expect(
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 7e31cb9b3a..d281e3911d 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -1,4 +1,5 @@
import is from '@sindresorhus/is';
+import { DateTime } from 'luxon';
import pAll from 'p-all';
import { PlatformId } from '../../constants';
import {
@@ -9,6 +10,7 @@ import {
} from '../../constants/error-messages';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
+import { getCache } from '../../util/cache/repository';
import { maskToken } from '../mask';
import { range } from '../range';
import { regEx } from '../regex';
@@ -180,6 +182,77 @@ function constructAcceptString(input?: any): string {
return acceptStrings.join(', ');
}
+const MAX_GRAPHQL_PAGE_SIZE = 100;
+
+function getGraphqlPageSize(
+ fieldName: string,
+ defaultPageSize = MAX_GRAPHQL_PAGE_SIZE
+): number {
+ const cache = getCache();
+ const graphqlPageCache = cache?.platform?.github?.graphqlPageCache;
+ const cachedRecord = graphqlPageCache?.[fieldName];
+
+ if (graphqlPageCache && cachedRecord) {
+ logger.debug(
+ { fieldName, ...cachedRecord },
+ 'GraphQL page size: found cached value'
+ );
+
+ const oldPageSize = cachedRecord.pageSize;
+
+ const now = DateTime.local();
+ const then = DateTime.fromISO(cachedRecord.pageLastResizedAt);
+ const expiry = then.plus({ hours: 24 });
+ if (now > expiry) {
+ const newPageSize = Math.min(oldPageSize * 2, MAX_GRAPHQL_PAGE_SIZE);
+ if (newPageSize < MAX_GRAPHQL_PAGE_SIZE) {
+ const timestamp = now.toISO();
+
+ logger.debug(
+ { fieldName, oldPageSize, newPageSize, timestamp },
+ 'GraphQL page size: expanding'
+ );
+
+ cachedRecord.pageLastResizedAt = timestamp;
+ cachedRecord.pageSize = newPageSize;
+ } else {
+ logger.debug(
+ { fieldName, oldPageSize, newPageSize },
+ 'GraphQL page size: expanded to default page size'
+ );
+
+ delete graphqlPageCache[fieldName];
+ }
+
+ return newPageSize;
+ }
+
+ return oldPageSize;
+ }
+
+ return defaultPageSize;
+}
+
+function setGraphqlPageSize(fieldName: string, newPageSize: number): void {
+ const oldPageSize = getGraphqlPageSize(fieldName);
+ if (newPageSize !== oldPageSize) {
+ const now = DateTime.local();
+ const pageLastResizedAt = now.toISO();
+ logger.debug(
+ { fieldName, oldPageSize, newPageSize, timestamp: pageLastResizedAt },
+ 'GraphQL page size: shrinking'
+ );
+ const cache = getCache();
+ cache.platform ??= {};
+ cache.platform.github ??= {};
+ cache.platform.github.graphqlPageCache ??= {};
+ cache.platform.github.graphqlPageCache[fieldName] = {
+ pageLastResizedAt,
+ pageSize: newPageSize,
+ };
+ }
+}
+
export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
constructor(
hostType: string = PlatformId.Github,
@@ -263,7 +336,7 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
): Promise<GithubGraphqlResponse<T> | null> {
const path = 'graphql';
- const { paginate, count = 100, cursor = null } = options;
+ const { paginate, count = MAX_GRAPHQL_PAGE_SIZE, cursor = null } = options;
let { variables } = options;
if (paginate) {
variables = {
@@ -308,8 +381,10 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
const { paginate = true } = options;
let optimalCount: null | number = null;
- const initialCount = options.count ?? 100;
- let count = initialCount;
+ let count = getGraphqlPageSize(
+ fieldName,
+ options.count ?? MAX_GRAPHQL_PAGE_SIZE
+ );
let limit = options.limit ?? 1000;
let cursor: string | null = null;
@@ -362,17 +437,8 @@ export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
}
}
- // See: https://github.com/renovatebot/renovate/issues/12703
- // istanbul ignore if
- if (
- optimalCount &&
- optimalCount < initialCount && // log only shrinked results
- baseUrl === githubBaseUrl
- ) {
- logger.debug(
- { fieldName, optimalCount },
- 'Successful GraphQL query with shrinked pagination size'
- );
+ if (optimalCount && optimalCount < MAX_GRAPHQL_PAGE_SIZE) {
+ setGraphqlPageSize(fieldName, optimalCount);
}
return result;
--
GitLab