diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts
index 33f5da96829bda8dba06208cd37d27c55f60c7fe..0848f5fdfc01d699681f2c1ea1d6fbf4ccb59caf 100644
--- a/lib/modules/datasource/npm/get.ts
+++ b/lib/modules/datasource/npm/get.ts
@@ -10,6 +10,7 @@ import * as packageCache from '../../../util/cache/package';
import type { Http } from '../../../util/http';
import type { HttpOptions } from '../../../util/http/types';
import { regEx } from '../../../util/regex';
+import { HttpCacheStats } from '../../../util/stats';
import { joinUrlParts } from '../../../util/url';
import type { Release, ReleaseResult } from '../types';
import type { CachedReleaseResult, NpmResponse } from './types';
@@ -91,10 +92,13 @@ export async function getDependency(
);
if (softExpireAt.isValid && softExpireAt > DateTime.local()) {
logger.trace('Cached result is not expired - reusing');
+ HttpCacheStats.incLocalHits(packageUrl);
delete cachedResult.cacheData;
return cachedResult;
}
+
logger.trace('Cached result is soft expired');
+ HttpCacheStats.incLocalMisses(packageUrl);
} else {
logger.trace(
`Package cache for npm package "${packageName}" is from an old revision - discarding`,
@@ -127,6 +131,7 @@ export async function getDependency(
const raw = await http.getJson<NpmResponse>(packageUrl, options);
if (cachedResult?.cacheData && raw.statusCode === 304) {
logger.trace(`Cached npm result for ${packageName} is revalidated`);
+ HttpCacheStats.incRemoteHits(packageUrl);
cachedResult.cacheData.softExpireAt = softExpireAt;
await packageCache.set(
cacheNamespace,
@@ -137,6 +142,7 @@ export async function getDependency(
delete cachedResult.cacheData;
return cachedResult;
}
+ HttpCacheStats.incRemoteMisses(packageUrl);
const etag = raw.headers.etag;
const res = raw.body;
if (!res.versions || !Object.keys(res.versions).length) {
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index 107f4726344928519ba7899cee26b291dff72ef7..d6139238eff4d78833fffa9390ca10a08cf5fb4c 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -11,7 +11,11 @@ import { getCache } from '../cache/repository';
import { clone } from '../clone';
import { hash } from '../hash';
import { type AsyncResult, Result } from '../result';
-import { type HttpRequestStatsDataPoint, HttpStats } from '../stats';
+import {
+ HttpCacheStats,
+ type HttpRequestStatsDataPoint,
+ HttpStats,
+} from '../stats';
import { resolveBaseUrl } from '../url';
import { applyAuthorization, removeAuthorization } from './auth';
import { hooks } from './hooks';
@@ -279,6 +283,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
logger.debug(
`http cache: saving ${url} (etag=${resCopy.headers.etag}, lastModified=${resCopy.headers['last-modified']})`,
);
+ HttpCacheStats.incRemoteMisses(url);
cache.httpCache[url] = {
etag: resCopy.headers.etag,
httpResponse: copyResponse(res, deepCopyNeeded),
@@ -290,6 +295,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
logger.debug(
`http cache: Using cached response: ${url} from ${cache.httpCache[url].timeStamp}`,
);
+ HttpCacheStats.incRemoteHits(url);
const cacheCopy = copyResponse(
cache.httpCache[url].httpResponse,
deepCopyNeeded,
diff --git a/lib/util/stats.spec.ts b/lib/util/stats.spec.ts
index f23bfe35d6e3a18ce719a3882dcca5a51c9a3ee0..4ea3c275855af6641b2556737100a0fbd53099cc 100644
--- a/lib/util/stats.spec.ts
+++ b/lib/util/stats.spec.ts
@@ -1,6 +1,7 @@
import { logger } from '../../test/util';
import * as memCache from './cache/memory';
import {
+ HttpCacheStats,
HttpStats,
LookupStats,
PackageCacheStats,
@@ -455,4 +456,79 @@ describe('util/stats', () => {
});
});
});
+
+ describe('HttpCacheStats', () => {
+ it('returns empty data', () => {
+ const res = HttpCacheStats.getData();
+ expect(res).toEqual({});
+ });
+
+ it('ignores wrong url', () => {
+ HttpCacheStats.incLocalHits('<invalid>');
+ expect(HttpCacheStats.getData()).toEqual({});
+ });
+
+ it('writes data points', () => {
+ HttpCacheStats.incLocalHits('https://example.com/foo');
+ HttpCacheStats.incLocalHits('https://example.com/foo');
+ HttpCacheStats.incLocalMisses('https://example.com/foo');
+ HttpCacheStats.incLocalMisses('https://example.com/bar');
+ HttpCacheStats.incRemoteHits('https://example.com/bar');
+ HttpCacheStats.incRemoteMisses('https://example.com/bar');
+
+ const res = HttpCacheStats.getData();
+
+ expect(res).toEqual({
+ 'https://example.com/bar': {
+ localHits: 0,
+ localMisses: 1,
+ localTotal: 1,
+ remoteHits: 1,
+ remoteMisses: 1,
+ remoteTotal: 2,
+ },
+ 'https://example.com/foo': {
+ localHits: 2,
+ localMisses: 1,
+ localTotal: 3,
+ remoteHits: 0,
+ remoteMisses: 0,
+ remoteTotal: 0,
+ },
+ });
+ });
+
+ it('prints report', () => {
+ HttpCacheStats.incLocalHits('https://example.com/foo');
+ HttpCacheStats.incLocalHits('https://example.com/foo');
+ HttpCacheStats.incLocalMisses('https://example.com/foo');
+ HttpCacheStats.incLocalMisses('https://example.com/bar');
+ HttpCacheStats.incRemoteHits('https://example.com/bar');
+ HttpCacheStats.incRemoteMisses('https://example.com/bar');
+
+ HttpCacheStats.report();
+
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ const [data, msg] = logger.logger.debug.mock.calls[0];
+ expect(msg).toBe('HTTP cache statistics');
+ expect(data).toEqual({
+ 'https://example.com/bar': {
+ localHits: 0,
+ localMisses: 1,
+ localTotal: 1,
+ remoteHits: 1,
+ remoteMisses: 1,
+ remoteTotal: 2,
+ },
+ 'https://example.com/foo': {
+ localHits: 2,
+ localMisses: 1,
+ localTotal: 3,
+ remoteHits: 0,
+ remoteMisses: 0,
+ remoteTotal: 0,
+ },
+ });
+ });
+ });
});
diff --git a/lib/util/stats.ts b/lib/util/stats.ts
index 6508f49d83455121270a4a71240f8962ee538612..c0e6996e360db5d415d4d69cdd8985245f90f8a9 100644
--- a/lib/util/stats.ts
+++ b/lib/util/stats.ts
@@ -235,3 +235,99 @@ export class HttpStats {
logger.debug({ urls, hosts, requests }, 'HTTP statistics');
}
}
+
+interface HttpCacheHostStatsData {
+ localHits: number;
+ localMisses: number;
+ localTotal: number;
+ remoteHits: number;
+ remoteMisses: number;
+ remoteTotal: number;
+}
+
+type HttpCacheStatsData = Record<string, HttpCacheHostStatsData>;
+
+export class HttpCacheStats {
+ static getData(): HttpCacheStatsData {
+ return memCache.get<HttpCacheStatsData>('http-cache-stats') ?? {};
+ }
+
+ static read(key: string): HttpCacheHostStatsData {
+ return (
+ this.getData()?.[key] ?? {
+ localHits: 0,
+ localMisses: 0,
+ localTotal: 0,
+ remoteHits: 0,
+ remoteMisses: 0,
+ remoteTotal: 0,
+ }
+ );
+ }
+
+ static write(key: string, data: HttpCacheHostStatsData): void {
+ const stats = memCache.get<HttpCacheStatsData>('http-cache-stats') ?? {};
+ stats[key] = data;
+ memCache.set('http-cache-stats', stats);
+ }
+
+ static getBaseUrl(url: string): string | null {
+ const parsedUrl = parseUrl(url);
+ if (!parsedUrl) {
+ logger.debug({ url }, 'Failed to parse URL during cache stats');
+ return null;
+ }
+ const { origin, pathname } = parsedUrl;
+ const baseUrl = `${origin}${pathname}`;
+ return baseUrl;
+ }
+
+ static incLocalHits(url: string): void {
+ const baseUrl = HttpCacheStats.getBaseUrl(url);
+ if (baseUrl) {
+ const host = baseUrl;
+ const stats = HttpCacheStats.read(host);
+ stats.localHits += 1;
+ stats.localTotal += 1;
+ HttpCacheStats.write(host, stats);
+ }
+ }
+
+ static incLocalMisses(url: string): void {
+ const baseUrl = HttpCacheStats.getBaseUrl(url);
+ if (baseUrl) {
+ const host = baseUrl;
+ const stats = HttpCacheStats.read(host);
+ stats.localMisses += 1;
+ stats.localTotal += 1;
+ HttpCacheStats.write(host, stats);
+ }
+ }
+
+ static incRemoteHits(url: string): void {
+ const baseUrl = HttpCacheStats.getBaseUrl(url);
+ if (baseUrl) {
+ const host = baseUrl;
+ const stats = HttpCacheStats.read(host);
+ stats.remoteHits += 1;
+ stats.remoteTotal += 1;
+ HttpCacheStats.write(host, stats);
+ }
+ }
+
+ static incRemoteMisses(url: string): void {
+ const baseUrl = HttpCacheStats.getBaseUrl(url);
+ if (baseUrl) {
+ const host = baseUrl;
+ const stats = HttpCacheStats.read(host);
+ stats.remoteMisses += 1;
+ stats.remoteTotal += 1;
+ HttpCacheStats.write(host, stats);
+ }
+ }
+
+ static report(): void {
+ const stats = HttpCacheStats.getData();
+ logger.debug(stats, 'HTTP cache statistics');
+ }
+}
diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts
index 01c0658eb3e79b423fb94193fd019d6a5dfbd25b..fa7fa880e833ebf904eb4d4c356b6c24bc169f39 100644
--- a/lib/workers/repository/index.ts
+++ b/lib/workers/repository/index.ts
@@ -19,7 +19,12 @@ import { clearDnsCache, printDnsStats } from '../../util/http/dns';
import * as queue from '../../util/http/queue';
import * as throttle from '../../util/http/throttle';
import { addSplit, getSplits, splitInit } from '../../util/split';
-import { HttpStats, LookupStats, PackageCacheStats } from '../../util/stats';
+import {
+ HttpCacheStats,
+ HttpStats,
+ LookupStats,
+ PackageCacheStats,
+} from '../../util/stats';
import { setBranchCache } from './cache';
import { extractRepoProblems } from './common';
import { ensureDependencyDashboard } from './dependency-dashboard';
@@ -126,6 +131,7 @@ export async function renovateRepository(
logger.debug(splits, 'Repository timing splits (milliseconds)');
PackageCacheStats.report();
HttpStats.report();
+ HttpCacheStats.report();
LookupStats.report();
printDnsStats();
clearDnsCache();