From 23a334c7e34c0a9eba48697f6094dd52e615000b Mon Sep 17 00:00:00 2001 From: Rhys Arkins <rhys@arkins.net> Date: Mon, 22 Jan 2024 12:49:40 +0100 Subject: [PATCH] feat(cache): etag caching for github GET (#26788) Co-authored-by: Michael Kriese <michael.kriese@visualon.de> --- lib/modules/platform/github/index.ts | 14 +++++++--- lib/modules/platform/github/pr.ts | 1 + lib/util/cache/repository/types.ts | 8 ++++++ lib/util/http/index.spec.ts | 31 ++++++++++++++++++++--- lib/util/http/index.ts | 38 ++++++++++++++++++++++++++++ lib/util/http/types.ts | 2 ++ 6 files changed, 88 insertions(+), 6 deletions(-) diff --git a/lib/modules/platform/github/index.ts b/lib/modules/platform/github/index.ts index c2513c1000..e1f37380ba 100644 --- a/lib/modules/platform/github/index.ts +++ b/lib/modules/platform/github/index.ts @@ -35,7 +35,10 @@ import type { import * as hostRules from '../../../util/host-rules'; import * as githubHttp from '../../../util/http/github'; import type { GithubHttpOptions } from '../../../util/http/github'; -import type { HttpResponse } from '../../../util/http/types'; +import type { + HttpResponse, + InternalHttpOptions, +} from '../../../util/http/types'; import { coerceObject } from '../../../util/object'; import { regEx } from '../../../util/regex'; import { sanitize } from '../../../util/sanitize'; @@ -316,11 +319,15 @@ export async function getRawFile( branchOrTag?: string, ): Promise<string | null> { const repo = repoName ?? config.repository; + const httpOptions: InternalHttpOptions = { + // Only cache response if it's from the same repo + repoCache: repo === config.repository, + }; let url = `repos/${repo}/contents/${fileName}`; if (branchOrTag) { url += `?ref=` + branchOrTag; } - const res = await githubApi.getJson<{ content: string }>(url); + const res = await githubApi.getJson<{ content: string }>(url, httpOptions); const buf = res.body.content; const str = fromBase64(buf); return str; @@ -1220,7 +1227,7 @@ export async function getIssue( const issueBody = ( await githubApi.getJson<{ body: string }>( `repos/${config.parentRepo ?? config.repository}/issues/${number}`, - { memCache: useCache }, + { memCache: useCache, repoCache: true }, ) ).body.body; return { @@ -1306,6 +1313,7 @@ export async function ensureIssue({ `repos/${config.parentRepo ?? config.repository}/issues/${ issue.number }`, + { repoCache: true }, ) ).body.body; if ( diff --git a/lib/modules/platform/github/pr.ts b/lib/modules/platform/github/pr.ts index 432abe9965..2cd6bb9359 100644 --- a/lib/modules/platform/github/pr.ts +++ b/lib/modules/platform/github/pr.ts @@ -67,6 +67,7 @@ export async function getPrCache( if (pageIdx === 1 && isInitial) { // Speed up initial fetch opts.paginate = true; + opts.repoCache = true; } const perPage = isInitial ? 100 : 20; diff --git a/lib/util/cache/repository/types.ts b/lib/util/cache/repository/types.ts index eabbaba5f2..68d915d30d 100644 --- a/lib/util/cache/repository/types.ts +++ b/lib/util/cache/repository/types.ts @@ -8,6 +8,7 @@ import type { BitbucketPrCacheData } from '../../../modules/platform/bitbucket/t import type { GiteaPrCacheData } from '../../../modules/platform/gitea/types'; import type { RepoInitConfig } from '../../../workers/repository/init/types'; import type { PrBlockedBy } from '../../../workers/types'; +import type { HttpResponse } from '../../http/types'; export interface BaseBranchCache { sha: string; // branch commit sha @@ -124,8 +125,15 @@ export interface BranchCache { result?: string; } +export interface HttpCache { + etag: string; + httpResponse: HttpResponse<unknown>; + timeStamp: string; +} + export interface RepoCacheData { configFileName?: string; + httpCache?: Record<string, HttpCache>; semanticCommits?: 'enabled' | 'disabled'; branches?: BranchCache[]; init?: RepoInitConfig; diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts index eb9aadd63b..a48769c6aa 100644 --- a/lib/util/http/index.spec.ts +++ b/lib/util/http/index.spec.ts @@ -76,13 +76,38 @@ describe('util/http/index', () => { }, }) .get('/') - .reply(200, '{ "test": true }'); - expect(await http.getJson('http://renovate.com')).toEqual({ + .reply(200, '{ "test": true }', { etag: 'abc123' }); + expect( + await http.getJson('http://renovate.com', { repoCache: true }), + ).toEqual({ authorization: false, body: { test: true, }, - headers: {}, + headers: { + etag: 'abc123', + }, + statusCode: 200, + }); + + httpMock + .scope(baseUrl, { + reqheaders: { + accept: 'application/json', + }, + }) + .get('/') + .reply(304, '', { etag: 'abc123' }); + expect( + await http.getJson('http://renovate.com', { repoCache: true }), + ).toEqual({ + authorization: false, + body: { + test: true, + }, + headers: { + etag: 'abc123', + }, statusCode: 200, }); }); diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index c675e378f5..40881e4309 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -7,6 +7,7 @@ import { pkg } from '../../expose.cjs'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as memCache from '../cache/memory'; +import { getCache } from '../cache/repository'; import { clone } from '../clone'; import { hash } from '../hash'; import { type AsyncResult, Result } from '../result'; @@ -163,6 +164,8 @@ export class Http<Opts extends HttpOptions = HttpOptions> { httpOptions, ); + logger.trace(`HTTP request: ${options.method.toUpperCase()} ${url}`); + const etagCache = httpOptions.etagCache && options.method === 'get' ? httpOptions.etagCache @@ -210,6 +213,16 @@ export class Http<Opts extends HttpOptions = HttpOptions> { // istanbul ignore else: no cache tests if (!resPromise) { + if (httpOptions.repoCache) { + const cachedEtag = getCache().httpCache?.[url]?.etag; + if (cachedEtag) { + logger.debug(`Using cached etag for ${url}`); + options.headers = { + ...options.headers, + 'If-None-Match': cachedEtag, + }; + } + } const startTime = Date.now(); const httpTask: GotTask<T> = () => { const queueDuration = Date.now() - startTime; @@ -243,6 +256,31 @@ export class Http<Opts extends HttpOptions = HttpOptions> { const deepCopyNeeded = !!memCacheKey && res.statusCode !== 304; const resCopy = copyResponse(res, deepCopyNeeded); resCopy.authorization = !!options?.headers?.authorization; + if (httpOptions.repoCache) { + const cache = getCache(); + cache.httpCache ??= {}; + if (resCopy.statusCode === 200 && resCopy.headers?.etag) { + logger.debug( + `Saving response to cache: ${url} with etag ${resCopy.headers.etag}`, + ); + cache.httpCache[url] = { + etag: resCopy.headers.etag, + httpResponse: copyResponse(res, deepCopyNeeded), + timeStamp: new Date().toISOString(), + }; + } + if (resCopy.statusCode === 304 && cache.httpCache[url]?.httpResponse) { + logger.debug( + `Using cached response: ${url} with etag ${resCopy.headers.etag} from ${cache.httpCache[url].timeStamp}`, + ); + const cacheCopy = copyResponse( + cache.httpCache[url].httpResponse, + deepCopyNeeded, + ); + cacheCopy.authorization = !!options?.headers?.authorization; + return cacheCopy as HttpResponse<T>; + } + } return resCopy; } catch (err) { const { abortOnError, abortIgnoreStatusCodes } = options; diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts index 5d3d2770a2..5969aeade4 100644 --- a/lib/util/http/types.ts +++ b/lib/util/http/types.ts @@ -65,6 +65,7 @@ export interface HttpOptions { token?: string; memCache?: boolean; + repoCache?: boolean; } export interface EtagCache<T = any> { @@ -81,6 +82,7 @@ export interface InternalHttpOptions extends HttpOptions { responseType?: 'json' | 'buffer'; method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head'; parseJson?: ParseJsonFunction; + repoCache?: boolean; } export interface HttpHeaders extends IncomingHttpHeaders { -- GitLab