diff --git a/lib/util/cache/repository/types.ts b/lib/util/cache/repository/types.ts
index 7859e03cfa93b9982da83f419ae9f00a857bbf6f..4d0a68003590ff4b9984b3fb38c2e4f6c280d79e 100644
--- a/lib/util/cache/repository/types.ts
+++ b/lib/util/cache/repository/types.ts
@@ -6,7 +6,6 @@ import type {
import type { PackageFile } from '../../../modules/manager/types';
import type { RepoInitConfig } from '../../../workers/repository/init/types';
import type { PrBlockedBy } from '../../../workers/types';
-import type { HttpResponse } from '../../http/types';
export interface BaseBranchCache {
sha: string; // branch commit sha
@@ -123,16 +122,9 @@ export interface BranchCache {
result?: string;
}
-export interface HttpCache {
- etag?: string;
- httpResponse: HttpResponse<unknown>;
- lastModified?: string;
- timeStamp: string;
-}
-
export interface RepoCacheData {
configFileName?: string;
- httpCache?: Record<string, HttpCache>;
+ httpCache?: Record<string, unknown>;
semanticCommits?: 'enabled' | 'disabled';
branches?: BranchCache[];
init?: RepoInitConfig;
diff --git a/lib/util/http/cache/abstract-http-cache-provider.ts b/lib/util/http/cache/abstract-http-cache-provider.ts
new file mode 100644
index 0000000000000000000000000000000000000000..938e60b836dca613ff3c490bf02238165a62dca0
--- /dev/null
+++ b/lib/util/http/cache/abstract-http-cache-provider.ts
@@ -0,0 +1,94 @@
+import { logger } from '../../../logger';
+import { HttpCacheStats } from '../../stats';
+import type { GotOptions, HttpResponse } from '../types';
+import { copyResponse } from '../util';
+import { HttpCacheSchema } from './schema';
+import type { HttpCache, HttpCacheProvider } from './types';
+
+export abstract class AbstractHttpCacheProvider implements HttpCacheProvider {
+ protected abstract load(url: string): Promise<unknown>;
+ protected abstract persist(url: string, data: HttpCache): Promise<void>;
+
+ async get(url: string): Promise<HttpCache | null> {
+ const cache = await this.load(url);
+ const httpCache = HttpCacheSchema.parse(cache);
+ if (!httpCache) {
+ return null;
+ }
+
+ return httpCache as HttpCache;
+ }
+
+ async setCacheHeaders<T extends Pick<GotOptions, 'headers'>>(
+ url: string,
+ opts: T,
+ ): Promise<void> {
+ const httpCache = await this.get(url);
+ if (!httpCache) {
+ return;
+ }
+
+ opts.headers ??= {};
+
+ if (httpCache.etag) {
+ opts.headers['If-None-Match'] = httpCache.etag;
+ }
+
+ if (httpCache.lastModified) {
+ opts.headers['If-Modified-Since'] = httpCache.lastModified;
+ }
+ }
+
+ async wrapResponse<T>(
+ url: string,
+ resp: HttpResponse<T>,
+ ): Promise<HttpResponse<T>> {
+ if (resp.statusCode === 200) {
+ const etag = resp.headers?.['etag'];
+ const lastModified = resp.headers?.['last-modified'];
+
+ HttpCacheStats.incRemoteMisses(url);
+
+ const httpResponse = copyResponse(resp, true);
+ const timestamp = new Date().toISOString();
+
+ const newHttpCache = HttpCacheSchema.parse({
+ etag,
+ lastModified,
+ httpResponse,
+ timestamp,
+ });
+ if (newHttpCache) {
+ logger.debug(
+ `http cache: saving ${url} (etag=${etag}, lastModified=${lastModified})`,
+ );
+ await this.persist(url, newHttpCache as HttpCache);
+ } else {
+ logger.debug(`http cache: failed to persist cache for ${url}`);
+ }
+
+ return resp;
+ }
+
+ if (resp.statusCode === 304) {
+ const httpCache = await this.get(url);
+ if (!httpCache) {
+ return resp;
+ }
+
+ const timestamp = httpCache.timestamp;
+ logger.debug(
+ `http cache: Using cached response: ${url} from ${timestamp}`,
+ );
+ HttpCacheStats.incRemoteHits(url);
+ const cachedResp = copyResponse(
+ httpCache.httpResponse as HttpResponse<T>,
+ true,
+ );
+ cachedResp.authorization = resp.authorization;
+ return cachedResp;
+ }
+
+ return resp;
+ }
+}
diff --git a/lib/util/http/cache/repository-http-cache-provider.spec.ts b/lib/util/http/cache/repository-http-cache-provider.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..a8acbba4b1178605a3778416e86a039ae3cc0cd4
--- /dev/null
+++ b/lib/util/http/cache/repository-http-cache-provider.spec.ts
@@ -0,0 +1,152 @@
+import { Http } from '..';
+import * as httpMock from '../../../../test/http-mock';
+import { logger } from '../../../../test/util';
+import { getCache, resetCache } from '../../cache/repository';
+import { repoCacheProvider } from './repository-http-cache-provider';
+
+describe('util/http/cache/repository-http-cache-provider', () => {
+ beforeEach(() => {
+ resetCache();
+ });
+
+ const http = new Http('test', {
+ cacheProvider: repoCacheProvider,
+ });
+
+ it('reuses data with etag', async () => {
+ const scope = httpMock.scope('https://example.com');
+
+ scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+ const res1 = await http.getJson('https://example.com/foo/bar');
+ expect(res1).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: false,
+ });
+
+ scope.get('/foo/bar').reply(304);
+ const res2 = await http.getJson('https://example.com/foo/bar');
+ expect(res2).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: false,
+ });
+ });
+
+ it('reuses data with last-modified', async () => {
+ const scope = httpMock.scope('https://example.com');
+
+ scope
+ .get('/foo/bar')
+ .reply(
+ 200,
+ { msg: 'Hello, world!' },
+ { 'last-modified': 'Mon, 01 Jan 2000 00:00:00 GMT' },
+ );
+ const res1 = await http.getJson('https://example.com/foo/bar');
+ expect(res1).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: false,
+ });
+
+ scope.get('/foo/bar').reply(304);
+ const res2 = await http.getJson('https://example.com/foo/bar');
+ expect(res2).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: false,
+ });
+ });
+
+ it('uses older cache format', async () => {
+ const repoCache = getCache();
+ repoCache.httpCache = {
+ 'https://example.com/foo/bar': {
+ etag: '123',
+ lastModified: 'Mon, 01 Jan 2000 00:00:00 GMT',
+ httpResponse: { statusCode: 200, body: { msg: 'Hello, world!' } },
+ timeStamp: new Date().toISOString(),
+ },
+ };
+ httpMock.scope('https://example.com').get('/foo/bar').reply(304);
+
+ const res = await http.getJson('https://example.com/foo/bar');
+
+ expect(res).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: false,
+ });
+ });
+
+ it('reports if cache could not be persisted', async () => {
+ httpMock
+ .scope('https://example.com')
+ .get('/foo/bar')
+ .reply(200, { msg: 'Hello, world!' });
+
+ await http.getJson('https://example.com/foo/bar');
+
+ expect(logger.logger.debug).toHaveBeenCalledWith(
+ 'http cache: failed to persist cache for https://example.com/foo/bar',
+ );
+ });
+
+ it('handles abrupt cache reset', async () => {
+ const scope = httpMock.scope('https://example.com');
+
+ scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+ const res1 = await http.getJson('https://example.com/foo/bar');
+ expect(res1).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: false,
+ });
+
+ resetCache();
+
+ scope.get('/foo/bar').reply(304);
+ const res2 = await http.getJson('https://example.com/foo/bar');
+ expect(res2).toMatchObject({
+ statusCode: 304,
+ authorization: false,
+ });
+ });
+
+ it('bypasses for statuses other than 200 and 304', async () => {
+ const scope = httpMock.scope('https://example.com');
+ scope.get('/foo/bar').reply(203);
+
+ const res = await http.getJson('https://example.com/foo/bar');
+
+ expect(res).toMatchObject({
+ statusCode: 203,
+ authorization: false,
+ });
+ });
+
+ it('supports authorization', async () => {
+ const scope = httpMock.scope('https://example.com');
+
+ scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+ const res1 = await http.getJson('https://example.com/foo/bar', {
+ headers: { authorization: 'Bearer 123' },
+ });
+ expect(res1).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: true,
+ });
+
+ scope.get('/foo/bar').reply(304);
+ const res2 = await http.getJson('https://example.com/foo/bar', {
+ headers: { authorization: 'Bearer 123' },
+ });
+ expect(res2).toMatchObject({
+ statusCode: 200,
+ body: { msg: 'Hello, world!' },
+ authorization: true,
+ });
+ });
+});
diff --git a/lib/util/http/cache/repository-http-cache-provider.ts b/lib/util/http/cache/repository-http-cache-provider.ts
new file mode 100644
index 0000000000000000000000000000000000000000..9cf9c8cfa3b0c1353e32302e04177784e8f8c2ff
--- /dev/null
+++ b/lib/util/http/cache/repository-http-cache-provider.ts
@@ -0,0 +1,20 @@
+import { getCache } from '../../cache/repository';
+import { AbstractHttpCacheProvider } from './abstract-http-cache-provider';
+import type { HttpCache } from './types';
+
+export class RepositoryHttpCacheProvider extends AbstractHttpCacheProvider {
+ override load(url: string): Promise<unknown> {
+ const cache = getCache();
+ cache.httpCache ??= {};
+ return Promise.resolve(cache.httpCache[url]);
+ }
+
+ override persist(url: string, data: HttpCache): Promise<void> {
+ const cache = getCache();
+ cache.httpCache ??= {};
+ cache.httpCache[url] = data;
+ return Promise.resolve();
+ }
+}
+
+export const repoCacheProvider = new RepositoryHttpCacheProvider();
diff --git a/lib/util/http/cache/schema.ts b/lib/util/http/cache/schema.ts
new file mode 100644
index 0000000000000000000000000000000000000000..d1d71fda9bc61aae8a2e559f3a63e4b2e9f9efdc
--- /dev/null
+++ b/lib/util/http/cache/schema.ts
@@ -0,0 +1,34 @@
+import { z } from 'zod';
+
+const invalidFieldsMsg =
+ 'Cache object should have `etag` or `lastModified` fields';
+
+export const HttpCacheSchema = z
+ .object({
+ // TODO: remove this migration part during the Christmas eve 2024
+ timeStamp: z.string().optional(),
+ timestamp: z.string().optional(),
+ })
+ .passthrough()
+ .transform((data) => {
+ if (data.timeStamp) {
+ data.timestamp = data.timeStamp;
+ delete data.timeStamp;
+ }
+ return data;
+ })
+ .pipe(
+ z
+ .object({
+ etag: z.string().optional(),
+ lastModified: z.string().optional(),
+ httpResponse: z.unknown(),
+ timestamp: z.string(),
+ })
+ .refine(
+ ({ etag, lastModified }) => etag ?? lastModified,
+ invalidFieldsMsg,
+ ),
+ )
+ .nullable()
+ .catch(null);
diff --git a/lib/util/http/cache/types.ts b/lib/util/http/cache/types.ts
new file mode 100644
index 0000000000000000000000000000000000000000..1159f58028762f813597c1b54133122d7110d764
--- /dev/null
+++ b/lib/util/http/cache/types.ts
@@ -0,0 +1,17 @@
+import type { GotOptions, HttpResponse } from '../types';
+
+export interface HttpCache {
+ etag?: string;
+ lastModified?: string;
+ httpResponse: unknown;
+ timestamp: string;
+}
+
+export interface HttpCacheProvider {
+ setCacheHeaders<T extends Pick<GotOptions, 'headers'>>(
+ url: string,
+ opts: T,
+ ): Promise<void>;
+
+ wrapResponse<T>(url: string, resp: HttpResponse<T>): Promise<HttpResponse<T>>;
+}
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index fbf92cb044439e8aca2eb583f3765c2e9e60e8bb..af3fc8315096c9bd672008c3d9913f924027c9ba 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -1,3 +1,4 @@
+import is from '@sindresorhus/is';
import merge from 'deepmerge';
import got, { Options, RequestError } from 'got';
import type { SetRequired } from 'type-fest';
@@ -8,7 +9,6 @@ import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as memCache from '../cache/memory';
import { getCache } from '../cache/repository';
-import { clone } from '../clone';
import { hash } from '../hash';
import { type AsyncResult, Result } from '../result';
import {
@@ -27,12 +27,14 @@ import type {
GotJSONOptions,
GotOptions,
GotTask,
+ HttpCache,
HttpOptions,
HttpResponse,
InternalHttpOptions,
} from './types';
// TODO: refactor code to remove this (#9651)
import './legacy';
+import { copyResponse } from './util';
export { RequestError as HttpError };
@@ -49,26 +51,6 @@ type JsonArgs<
schema?: Schema;
};
-// Copying will help to avoid circular structure
-// and mutation of the cached response.
-function copyResponse<T>(
- response: HttpResponse<T>,
- deep: boolean,
-): HttpResponse<T> {
- const { body, statusCode, headers } = response;
- return deep
- ? {
- statusCode,
- body: body instanceof Buffer ? (body.subarray() as T) : clone<T>(body),
- headers: clone(headers),
- }
- : {
- statusCode,
- body,
- headers,
- };
-}
-
function applyDefaultHeaders(options: Options): void {
const renovateVersion = pkg.version;
options.headers = {
@@ -142,13 +124,17 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
options: HttpOptions = {},
) {
const retryLimit = process.env.NODE_ENV === 'test' ? 0 : 2;
- this.options = merge<GotOptions>(options, {
- context: { hostType },
- retry: {
- limit: retryLimit,
- maxRetryAfter: 0, // Don't rely on `got` retry-after handling, just let it fail and then we'll handle it
+ this.options = merge<GotOptions>(
+ options,
+ {
+ context: { hostType },
+ retry: {
+ limit: retryLimit,
+ maxRetryAfter: 0, // Don't rely on `got` retry-after handling, just let it fail and then we'll handle it
+ },
},
- });
+ { isMergeableObject: is.plainObject },
+ );
}
protected getThrottle(url: string): Throttle | null {
@@ -164,13 +150,14 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
url = resolveBaseUrl(httpOptions.baseUrl, url);
}
- let options = merge<SetRequired<GotOptions, 'method'>, GotOptions>(
+ let options = merge<SetRequired<GotOptions, 'method'>, InternalHttpOptions>(
{
method: 'get',
...this.options,
hostType: this.hostType,
},
httpOptions,
+ { isMergeableObject: is.plainObject },
);
logger.trace(`HTTP request: ${options.method.toUpperCase()} ${url}`);
@@ -212,7 +199,9 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
// istanbul ignore else: no cache tests
if (!resPromise) {
if (httpOptions.repoCache) {
- const responseCache = getCache().httpCache?.[url];
+ const responseCache = getCache().httpCache?.[url] as
+ | HttpCache
+ | undefined;
// Prefer If-Modified-Since over If-None-Match
if (responseCache?.['lastModified']) {
logger.debug(
@@ -232,6 +221,11 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
};
}
}
+
+ if (options.cacheProvider) {
+ await options.cacheProvider.setCacheHeaders(url, options);
+ }
+
const startTime = Date.now();
const httpTask: GotTask<T> = () => {
const queueMs = Date.now() - startTime;
@@ -261,6 +255,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
const deepCopyNeeded = !!memCacheKey && res.statusCode !== 304;
const resCopy = copyResponse(res, deepCopyNeeded);
resCopy.authorization = !!options?.headers?.authorization;
+
if (httpOptions.repoCache) {
const cache = getCache();
cache.httpCache ??= {};
@@ -279,19 +274,25 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
timeStamp: new Date().toISOString(),
};
}
- if (resCopy.statusCode === 304 && cache.httpCache[url]?.httpResponse) {
+ const httpCache = cache.httpCache[url] as HttpCache | undefined;
+ if (resCopy.statusCode === 304 && httpCache) {
logger.debug(
- `http cache: Using cached response: ${url} from ${cache.httpCache[url].timeStamp}`,
+ `http cache: Using cached response: ${url} from ${httpCache.timeStamp}`,
);
HttpCacheStats.incRemoteHits(url);
const cacheCopy = copyResponse(
- cache.httpCache[url].httpResponse,
+ httpCache.httpResponse,
deepCopyNeeded,
);
cacheCopy.authorization = !!options?.headers?.authorization;
return cacheCopy as HttpResponse<T>;
}
}
+
+ if (options.cacheProvider) {
+ return await options.cacheProvider.wrapResponse(url, resCopy);
+ }
+
return resCopy;
} catch (err) {
const { abortOnError, abortIgnoreStatusCodes } = options;
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
index c09fce3aa9972ee44acdf142fff962294c17fb2b..ceaf2768bcb983f166aa91a24c602c9162311727 100644
--- a/lib/util/http/types.ts
+++ b/lib/util/http/types.ts
@@ -4,6 +4,7 @@ import type {
OptionsOfJSONResponseBody,
ParseJsonFunction,
} from 'got';
+import type { HttpCacheProvider } from './cache/types';
export type GotContextOptions = {
authType?: string;
@@ -65,7 +66,11 @@ export interface HttpOptions {
token?: string;
memCache?: boolean;
+ /**
+ * @deprecated
+ */
repoCache?: boolean;
+ cacheProvider?: HttpCacheProvider;
}
export interface InternalHttpOptions extends HttpOptions {
@@ -73,6 +78,9 @@ export interface InternalHttpOptions extends HttpOptions {
responseType?: 'json' | 'buffer';
method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
parseJson?: ParseJsonFunction;
+ /**
+ * @deprecated
+ */
repoCache?: boolean;
}
@@ -89,3 +97,13 @@ export interface HttpResponse<T = string> {
export type Task<T> = () => Promise<T>;
export type GotTask<T> = Task<HttpResponse<T>>;
+
+/**
+ * @deprecated
+ */
+export interface HttpCache {
+ etag?: string;
+ httpResponse: HttpResponse<unknown>;
+ lastModified?: string;
+ timeStamp: string;
+}
diff --git a/lib/util/http/util.ts b/lib/util/http/util.ts
new file mode 100644
index 0000000000000000000000000000000000000000..aa0a045c27ff8583e3563fd9f36263e3fca70a73
--- /dev/null
+++ b/lib/util/http/util.ts
@@ -0,0 +1,22 @@
+import { clone } from '../clone';
+import type { HttpResponse } from './types';
+
+// Copying will help to avoid circular structure
+// and mutation of the cached response.
+export function copyResponse<T>(
+ response: HttpResponse<T>,
+ deep: boolean,
+): HttpResponse<T> {
+ const { body, statusCode, headers } = response;
+ return deep
+ ? {
+ statusCode,
+ body: body instanceof Buffer ? (body.subarray() as T) : clone<T>(body),
+ headers: clone(headers),
+ }
+ : {
+ statusCode,
+ body,
+ headers,
+ };
+}