From 8596967060d32b789838e10248c9664facdb964e Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Tue, 19 Mar 2024 13:34:05 -0300
Subject: [PATCH] feat(http): Switch to pluggable HTTP cache implementation
(#27966)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
lib/modules/platform/github/index.ts | 26 ++++++----
lib/modules/platform/github/pr.ts | 3 +-
lib/util/http/github.ts | 2 +-
lib/util/http/index.spec.ts | 76 ++--------------------------
lib/util/http/index.ts | 65 +-----------------------
lib/util/http/types.ts | 18 -------
6 files changed, 24 insertions(+), 166 deletions(-)
diff --git a/lib/modules/platform/github/index.ts b/lib/modules/platform/github/index.ts
index def3a90e63..a57f44b4f0 100644
--- a/lib/modules/platform/github/index.ts
+++ b/lib/modules/platform/github/index.ts
@@ -35,12 +35,10 @@ import type {
LongCommitSha,
} from '../../../util/git/types';
import * as hostRules from '../../../util/host-rules';
+import { repoCacheProvider } from '../../../util/http/cache/repository-http-cache-provider';
import * as githubHttp from '../../../util/http/github';
import type { GithubHttpOptions } from '../../../util/http/github';
-import type {
- HttpResponse,
- InternalHttpOptions,
-} from '../../../util/http/types';
+import type { HttpResponse } from '../../../util/http/types';
import { coerceObject } from '../../../util/object';
import { regEx } from '../../../util/regex';
import { sanitize } from '../../../util/sanitize';
@@ -312,7 +310,7 @@ async function getBranchProtection(
}
const res = await githubApi.getJson<BranchProtection>(
`repos/${config.repository}/branches/${escapeHash(branchName)}/protection`,
- { repoCache: true },
+ { cacheProvider: repoCacheProvider },
);
return res.body;
}
@@ -323,11 +321,14 @@ export async function getRawFile(
branchOrTag?: string,
): Promise<string | null> {
const repo = repoName ?? config.repository;
+
+ // only use cache for the same org
+ const httpOptions: GithubHttpOptions = {};
const isSameOrg = repo?.split('/')?.[0] === config.repositoryOwner;
- const httpOptions: InternalHttpOptions = {
- // Only cache response if it's from the same org
- repoCache: isSameOrg,
- };
+ if (isSameOrg) {
+ httpOptions.cacheProvider = repoCacheProvider;
+ }
+
let url = `repos/${repo}/contents/${fileName}`;
if (branchOrTag) {
url += `?ref=` + branchOrTag;
@@ -1239,7 +1240,10 @@ export async function getIssue(
const repo = config.parentRepo ?? config.repository;
const { body: issue } = await githubApi.getJson(
`repos/${repo}/issues/${number}`,
- { memCache: useCache, repoCache: true },
+ {
+ memCache: useCache,
+ cacheProvider: repoCacheProvider,
+ },
Issue,
);
return issue;
@@ -1320,7 +1324,7 @@ export async function ensureIssue({
body: { body: issueBody },
} = await githubApi.getJson(
`repos/${repo}/issues/${issue.number}`,
- { repoCache: true },
+ { cacheProvider: repoCacheProvider },
Issue,
);
if (
diff --git a/lib/modules/platform/github/pr.ts b/lib/modules/platform/github/pr.ts
index e7152a3488..66b37aa342 100644
--- a/lib/modules/platform/github/pr.ts
+++ b/lib/modules/platform/github/pr.ts
@@ -2,6 +2,7 @@ import is from '@sindresorhus/is';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
import { getCache } from '../../../util/cache/repository';
+import { repoCacheProvider } from '../../../util/http/cache/repository-http-cache-provider';
import type { GithubHttp, GithubHttpOptions } from '../../../util/http/github';
import { parseLinkHeader } from '../../../util/url';
import { ApiCache } from './api-cache';
@@ -64,7 +65,7 @@ export async function getPrCache(
while (needNextPageFetch && needNextPageSync) {
const opts: GithubHttpOptions = { paginate: false };
if (pageIdx === 1) {
- opts.repoCache = true;
+ opts.cacheProvider = repoCacheProvider;
if (isInitial) {
// Speed up initial fetch
opts.paginate = true;
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 97f20590ce..67276d3c50 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -339,7 +339,7 @@ export class GithubHttp extends Http<GithubHttpOptions> {
nextUrl.searchParams.set('page', String(pageNumber));
return this.request<T>(
nextUrl,
- { ...opts, paginate: false, repoCache: false },
+ { ...opts, paginate: false, cacheProvider: undefined },
okToRetry,
);
},
diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts
index d3e4bb3b00..3c1b37bb75 100644
--- a/lib/util/http/index.spec.ts
+++ b/lib/util/http/index.spec.ts
@@ -6,6 +6,7 @@ import {
HOST_DISABLED,
} from '../../constants/error-messages';
import * as memCache from '../cache/memory';
+import { resetCache } from '../cache/repository';
import * as hostRules from '../host-rules';
import * as queue from './queue';
import * as throttle from './throttle';
@@ -22,6 +23,7 @@ describe('util/http/index', () => {
hostRules.clear();
queue.clear();
throttle.clear();
+ resetCache();
});
it('get', async () => {
@@ -77,84 +79,16 @@ describe('util/http/index', () => {
})
.get('/')
.reply(200, '{ "test": true }', { etag: 'abc123' });
- expect(
- await http.getJson('http://renovate.com', { repoCache: true }),
- ).toEqual({
- authorization: false,
- body: {
- test: true,
- },
- headers: {
- etag: 'abc123',
- },
- statusCode: 200,
- });
-
- httpMock
- .scope(baseUrl, {
- reqheaders: {
- accept: 'application/json',
- },
- })
- .get('/')
- .reply(304, '', { etag: 'abc123' });
- expect(
- await http.getJson('http://renovate.com', { repoCache: true }),
- ).toEqual({
- authorization: false,
- body: {
- test: true,
- },
- headers: {
- etag: 'abc123',
- },
- statusCode: 200,
- });
- });
- it('uses last-modified header for caching', async () => {
- httpMock
- .scope(baseUrl, {
- reqheaders: {
- accept: 'application/json',
- },
- })
- .get('/')
- .reply(200, '{ "test": true }', {
- 'last-modified': 'Sun, 18 Feb 2024 18:00:05 GMT',
- });
- expect(
- await http.getJson('http://renovate.com', { repoCache: true }),
- ).toEqual({
- authorization: false,
- body: {
- test: true,
- },
- headers: {
- 'last-modified': 'Sun, 18 Feb 2024 18:00:05 GMT',
- },
- statusCode: 200,
- });
+ const res = await http.getJson('http://renovate.com');
- httpMock
- .scope(baseUrl, {
- reqheaders: {
- accept: 'application/json',
- },
- })
- .get('/')
- .reply(304, '', {
- 'last-modified': 'Sun, 18 Feb 2024 18:00:05 GMT',
- });
- expect(
- await http.getJson('http://renovate.com', { repoCache: true }),
- ).toEqual({
+ expect(res).toEqual({
authorization: false,
body: {
test: true,
},
headers: {
- 'last-modified': 'Sun, 18 Feb 2024 18:00:05 GMT',
+ etag: 'abc123',
},
statusCode: 200,
});
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index af3fc83150..1b084542ba 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -8,14 +8,9 @@ import { pkg } from '../../expose.cjs';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as memCache from '../cache/memory';
-import { getCache } from '../cache/repository';
import { hash } from '../hash';
import { type AsyncResult, Result } from '../result';
-import {
- HttpCacheStats,
- type HttpRequestStatsDataPoint,
- HttpStats,
-} from '../stats';
+import { type HttpRequestStatsDataPoint, HttpStats } from '../stats';
import { resolveBaseUrl } from '../url';
import { applyAuthorization, removeAuthorization } from './auth';
import { hooks } from './hooks';
@@ -27,7 +22,6 @@ import type {
GotJSONOptions,
GotOptions,
GotTask,
- HttpCache,
HttpOptions,
HttpResponse,
InternalHttpOptions,
@@ -198,30 +192,6 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
// istanbul ignore else: no cache tests
if (!resPromise) {
- if (httpOptions.repoCache) {
- const responseCache = getCache().httpCache?.[url] as
- | HttpCache
- | undefined;
- // Prefer If-Modified-Since over If-None-Match
- if (responseCache?.['lastModified']) {
- logger.debug(
- `http cache: trying cached Last-Modified "${responseCache?.['lastModified']}" for ${url}`,
- );
- options.headers = {
- ...options.headers,
- 'If-Modified-Since': responseCache['lastModified'],
- };
- } else if (responseCache?.etag) {
- logger.debug(
- `http cache: trying cached etag "${responseCache.etag}" for ${url}`,
- );
- options.headers = {
- ...options.headers,
- 'If-None-Match': responseCache.etag,
- };
- }
- }
-
if (options.cacheProvider) {
await options.cacheProvider.setCacheHeaders(url, options);
}
@@ -256,39 +226,6 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
const resCopy = copyResponse(res, deepCopyNeeded);
resCopy.authorization = !!options?.headers?.authorization;
- if (httpOptions.repoCache) {
- const cache = getCache();
- cache.httpCache ??= {};
- if (
- resCopy.statusCode === 200 &&
- (resCopy.headers?.etag ?? resCopy.headers['last-modified'])
- ) {
- logger.debug(
- `http cache: saving ${url} (etag=${resCopy.headers.etag}, lastModified=${resCopy.headers['last-modified']})`,
- );
- HttpCacheStats.incRemoteMisses(url);
- cache.httpCache[url] = {
- etag: resCopy.headers.etag,
- httpResponse: copyResponse(res, deepCopyNeeded),
- lastModified: resCopy.headers['last-modified'],
- timeStamp: new Date().toISOString(),
- };
- }
- const httpCache = cache.httpCache[url] as HttpCache | undefined;
- if (resCopy.statusCode === 304 && httpCache) {
- logger.debug(
- `http cache: Using cached response: ${url} from ${httpCache.timeStamp}`,
- );
- HttpCacheStats.incRemoteHits(url);
- const cacheCopy = copyResponse(
- httpCache.httpResponse,
- deepCopyNeeded,
- );
- cacheCopy.authorization = !!options?.headers?.authorization;
- return cacheCopy as HttpResponse<T>;
- }
- }
-
if (options.cacheProvider) {
return await options.cacheProvider.wrapResponse(url, resCopy);
}
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
index ceaf2768bc..396ccbfa8c 100644
--- a/lib/util/http/types.ts
+++ b/lib/util/http/types.ts
@@ -66,10 +66,6 @@ export interface HttpOptions {
token?: string;
memCache?: boolean;
- /**
- * @deprecated
- */
- repoCache?: boolean;
cacheProvider?: HttpCacheProvider;
}
@@ -78,10 +74,6 @@ export interface InternalHttpOptions extends HttpOptions {
responseType?: 'json' | 'buffer';
method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
parseJson?: ParseJsonFunction;
- /**
- * @deprecated
- */
- repoCache?: boolean;
}
export interface HttpHeaders extends IncomingHttpHeaders {
@@ -97,13 +89,3 @@ export interface HttpResponse<T = string> {
export type Task<T> = () => Promise<T>;
export type GotTask<T> = Task<HttpResponse<T>>;
-
-/**
- * @deprecated
- */
-export interface HttpCache {
- etag?: string;
- httpResponse: HttpResponse<unknown>;
- lastModified?: string;
- timeStamp: string;
-}
--
GitLab