From e89be688a59ee232c67db40edc566ac4b0c21aa2 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov <zharinov@users.noreply.github.com> Date: Sun, 14 Apr 2024 11:16:47 -0300 Subject: [PATCH] feat(http): Cleanup for HTTP cache (#28381) Co-authored-by: Rhys Arkins <rhys@arkins.net> --- docs/usage/self-hosted-configuration.md | 10 +++ lib/config/global.ts | 1 + lib/config/options/index.ts | 8 +++ lib/config/types.ts | 1 + lib/util/cache/repository/http-cache.spec.ts | 75 ++++++++++++++++++++ lib/util/cache/repository/http-cache.ts | 33 +++++++++ lib/util/cache/repository/impl/base.ts | 3 + 7 files changed, 131 insertions(+) create mode 100644 lib/util/cache/repository/http-cache.spec.ts create mode 100644 lib/util/cache/repository/http-cache.ts diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index af593128fb..88c3aee431 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -666,6 +666,16 @@ Use the `extends` field instead of this if, for example, you need the ability fo When Renovate resolves `globalExtends` it does not fully process the configuration. This means that Renovate does not have the authentication it needs to fetch private things. +## httpCacheTtlDays + +This option sets the number of days that Renovate will cache HTTP responses. +The default value is 90 days. +Value of `0` means no caching. + +<!-- prettier-ignore --> +!!! warning + When you set `httpCacheTtlDays` to `0`, Renovate will remove the cached HTTP data. + ## includeMirrors By default, Renovate does not autodiscover repositories that are mirrors. diff --git a/lib/config/global.ts b/lib/config/global.ts index e59c0ae0ca..21b2b56cc2 100644 --- a/lib/config/global.ts +++ b/lib/config/global.ts @@ -32,6 +32,7 @@ export class GlobalConfig { 'gitTimeout', 'platform', 'endpoint', + 'httpCacheTtlDays', ]; private static config: RepoGlobalConfig = {}; diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index d2994100d0..1b18948a1e 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2981,6 +2981,14 @@ const options: RenovateOptions[] = [ default: null, supportedPlatforms: ['github'], }, + { + name: 'httpCacheTtlDays', + description: 'Maximum duration in days to keep HTTP cache entries.', + type: 'integer', + stage: 'repository', + default: 90, + globalOnly: true, + }, ]; export function getOptions(): RenovateOptions[] { diff --git a/lib/config/types.ts b/lib/config/types.ts index b34d914add..6c68f35093 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -158,6 +158,7 @@ export interface RepoGlobalConfig { presetCachePersistence?: boolean; privateKey?: string; privateKeyOld?: string; + httpCacheTtlDays?: number; } export interface LegacyAdminConfig { diff --git a/lib/util/cache/repository/http-cache.spec.ts b/lib/util/cache/repository/http-cache.spec.ts new file mode 100644 index 0000000000..2d68f5e89d --- /dev/null +++ b/lib/util/cache/repository/http-cache.spec.ts @@ -0,0 +1,75 @@ +import { DateTime, Settings } from 'luxon'; +import { GlobalConfig } from '../../../config/global'; +import { cleanupHttpCache } from './http-cache'; + +describe('util/cache/repository/http-cache', () => { + beforeEach(() => { + const now = DateTime.fromISO('2024-04-12T12:00:00.000Z').valueOf(); + Settings.now = () => now; + GlobalConfig.reset(); + }); + + it('should not throw if cache is not a valid HttpCache', () => { + expect(() => cleanupHttpCache('not a valid cache')).not.toThrow(); + }); + + it('should remove expired items from the cache', () => { + const now = DateTime.now(); + const expiredItemTimestamp = now.minus({ days: 91 }).toISO(); + const cache = { + httpCache: { + 'http://example.com/foo': { + timestamp: expiredItemTimestamp, + etag: 'abc', + lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT', + httpResponse: {}, + }, + 'http://example.com/bar': { + timestamp: now.toISO(), + etag: 'abc', + lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT', + httpResponse: {}, + }, + }, + }; + + cleanupHttpCache(cache); + + expect(cache).toEqual({ + httpCache: { + 'http://example.com/bar': { + timestamp: now.toISO(), + etag: 'abc', + httpResponse: {}, + lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT', + }, + }, + }); + }); + + it('should remove all items if ttlDays is not configured', () => { + GlobalConfig.set({ httpCacheTtlDays: 0 }); + + const now = DateTime.now(); + const cache = { + httpCache: { + 'http://example.com/foo': { + timestamp: now.toISO(), + etag: 'abc', + lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT', + httpResponse: {}, + }, + 'http://example.com/bar': { + timestamp: now.toISO(), + etag: 'abc', + lastModified: 'Mon, 01 Jan 2024 00:00:00 GMT', + httpResponse: {}, + }, + }, + }; + + cleanupHttpCache(cache); + + expect(cache).toEqual({}); + }); +}); diff --git a/lib/util/cache/repository/http-cache.ts b/lib/util/cache/repository/http-cache.ts new file mode 100644 index 0000000000..36121766a0 --- /dev/null +++ b/lib/util/cache/repository/http-cache.ts @@ -0,0 +1,33 @@ +import is from '@sindresorhus/is'; +import { DateTime } from 'luxon'; +import { GlobalConfig } from '../../../config/global'; +import { logger } from '../../../logger'; +import { HttpCacheSchema } from '../../http/cache/schema'; + +export function cleanupHttpCache(cacheData: unknown): void { + if (!is.plainObject(cacheData) || !is.plainObject(cacheData['httpCache'])) { + logger.warn('cleanupHttpCache: invalid cache data'); + return; + } + const httpCache = cacheData['httpCache']; + + const ttlDays = GlobalConfig.get('httpCacheTtlDays', 90); + if (ttlDays === 0) { + logger.trace('cleanupHttpCache: zero value received, removing the cache'); + delete cacheData['httpCache']; + return; + } + + const now = DateTime.now(); + for (const [url, item] of Object.entries(httpCache)) { + const parsed = HttpCacheSchema.safeParse(item); + if (parsed.success && parsed.data) { + const item = parsed.data; + const expiry = DateTime.fromISO(item.timestamp).plus({ days: ttlDays }); + if (expiry < now) { + logger.debug(`http cache: removing expired cache for ${url}`); + delete httpCache[url]; + } + } + } +} diff --git a/lib/util/cache/repository/impl/base.ts b/lib/util/cache/repository/impl/base.ts index 59a78649c5..2fbb6ef896 100644 --- a/lib/util/cache/repository/impl/base.ts +++ b/lib/util/cache/repository/impl/base.ts @@ -5,6 +5,7 @@ import { compressToBase64, decompressFromBase64 } from '../../../compress'; import { hash } from '../../../hash'; import { safeStringify } from '../../../stringify'; import { CACHE_REVISION } from '../common'; +import { cleanupHttpCache } from '../http-cache'; import { RepoCacheRecord, RepoCacheV13 } from '../schema'; import type { RepoCache, RepoCacheData } from '../types'; @@ -70,6 +71,8 @@ export abstract class RepoCacheBase implements RepoCache { } async save(): Promise<void> { + cleanupHttpCache(this.data); + const jsonStr = safeStringify(this.data); const hashedJsonStr = hash(jsonStr); if (hashedJsonStr === this.oldHash) { -- GitLab