diff --git a/lib/modules/datasource/index.ts b/lib/modules/datasource/index.ts
index 006a0023c28ffe6f2bae39eae80d30c1ab70f80d..3815c69b6f90fd5e8152564188a3d5c6afb2ea98 100644
--- a/lib/modules/datasource/index.ts
+++ b/lib/modules/datasource/index.ts
@@ -9,6 +9,7 @@ import * as memCache from '../../util/cache/memory';
import * as packageCache from '../../util/cache/package';
import { clone } from '../../util/clone';
import { AsyncResult, Result } from '../../util/result';
+import { DatasourceCacheStats } from '../../util/stats';
import { trimTrailingSlash } from '../../util/url';
import datasources from './api';
import {
@@ -69,16 +70,22 @@ async function getRegistryReleases(
cacheNamespace,
cacheKey,
);
+
// istanbul ignore if
if (cachedResult) {
logger.trace({ cacheKey }, 'Returning cached datasource response');
+ DatasourceCacheStats.hit(datasource.id, registryUrl, config.packageName);
return cachedResult;
}
+
+ DatasourceCacheStats.miss(datasource.id, registryUrl, config.packageName);
}
+
const res = await datasource.getReleases({ ...config, registryUrl });
if (res?.releases.length) {
res.registryUrl ??= registryUrl;
}
+
// cache non-null responses unless marked as private
if (datasource.caching && res) {
const cachePrivatePackages = GlobalConfig.get(
@@ -89,8 +96,12 @@ async function getRegistryReleases(
logger.trace({ cacheKey }, 'Caching datasource response');
const cacheMinutes = 15;
await packageCache.set(cacheNamespace, cacheKey, res, cacheMinutes);
+ DatasourceCacheStats.set(datasource.id, registryUrl, config.packageName);
+ } else {
+ DatasourceCacheStats.skip(datasource.id, registryUrl, config.packageName);
}
}
+
return res;
}
diff --git a/lib/util/stats.spec.ts b/lib/util/stats.spec.ts
index cd8325f3a84eb962def0244773c13163b74ff2c3..d479ea48e47d2952bf1f8cabb0fc4b7a1c3dc46d 100644
--- a/lib/util/stats.spec.ts
+++ b/lib/util/stats.spec.ts
@@ -1,6 +1,7 @@
import { logger } from '../../test/util';
import * as memCache from './cache/memory';
import {
+ DatasourceCacheStats,
HttpCacheStats,
HttpStats,
LookupStats,
@@ -230,6 +231,60 @@ describe('util/stats', () => {
});
});
+ describe('DatasourceCacheStats', () => {
+ it('collects data points', () => {
+ DatasourceCacheStats.hit('crate', 'https://foo.example.com', 'foo');
+ DatasourceCacheStats.miss('maven', 'https://bar.example.com', 'bar');
+ DatasourceCacheStats.set('npm', 'https://baz.example.com', 'baz');
+ DatasourceCacheStats.skip('rubygems', 'https://qux.example.com', 'qux');
+
+ const report = DatasourceCacheStats.getReport();
+
+ expect(report).toEqual({
+ long: {
+ crate: {
+ 'https://foo.example.com': { foo: { read: 'hit' } },
+ },
+ maven: {
+ 'https://bar.example.com': { bar: { read: 'miss' } },
+ },
+ npm: {
+ 'https://baz.example.com': { baz: { write: 'set' } },
+ },
+ rubygems: {
+ 'https://qux.example.com': { qux: { write: 'skip' } },
+ },
+ },
+ short: {
+ crate: {
+ 'https://foo.example.com': { hit: 1, miss: 0, set: 0, skip: 0 },
+ },
+ maven: {
+ 'https://bar.example.com': { hit: 0, miss: 1, set: 0, skip: 0 },
+ },
+ npm: {
+ 'https://baz.example.com': { hit: 0, miss: 0, set: 1, skip: 0 },
+ },
+ rubygems: {
+ 'https://qux.example.com': { hit: 0, miss: 0, set: 0, skip: 1 },
+ },
+ },
+ });
+ });
+
+ it('reports', () => {
+ DatasourceCacheStats.hit('crate', 'https://foo.example.com', 'foo');
+ DatasourceCacheStats.miss('maven', 'https://bar.example.com', 'bar');
+ DatasourceCacheStats.set('npm', 'https://baz.example.com', 'baz');
+ DatasourceCacheStats.skip('rubygems', 'https://qux.example.com', 'qux');
+
+ DatasourceCacheStats.report();
+
+ expect(logger.logger.trace).toHaveBeenCalledTimes(1);
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+ });
+
describe('HttpStats', () => {
it('returns empty report', () => {
const res = HttpStats.getReport();
diff --git a/lib/util/stats.ts b/lib/util/stats.ts
index c250020bd4f7418de533bec3e98523653a9f1f90..ecd933f924940849f99edaa64e2e78c72bd3f52e 100644
--- a/lib/util/stats.ts
+++ b/lib/util/stats.ts
@@ -105,6 +105,145 @@ export class PackageCacheStats {
}
}
+interface DatasourceCacheDataPoint {
+ datasource: string;
+ registryUrl: string;
+ packageName: string;
+ action: 'hit' | 'miss' | 'set' | 'skip';
+}
+
+export interface DatasourceCacheReport {
+ long: {
+ [datasource in string]: {
+ [registryUrl in string]: {
+ [packageName in string]: {
+ read?: 'hit' | 'miss';
+ write?: 'set' | 'skip';
+ };
+ };
+ };
+ };
+ short: {
+ [datasource in string]: {
+ [registryUrl in string]: {
+ hit: number;
+ miss: number;
+ set: number;
+ skip: number;
+ };
+ };
+ };
+}
+
+export class DatasourceCacheStats {
+ private static getData(): DatasourceCacheDataPoint[] {
+ return (
+ memCache.get<DatasourceCacheDataPoint[]>('datasource-cache-stats') ?? []
+ );
+ }
+
+ private static setData(data: DatasourceCacheDataPoint[]): void {
+ memCache.set('datasource-cache-stats', data);
+ }
+
+ static hit(
+ datasource: string,
+ registryUrl: string,
+ packageName: string,
+ ): void {
+ const data = this.getData();
+ data.push({ datasource, registryUrl, packageName, action: 'hit' });
+ this.setData(data);
+ }
+
+ static miss(
+ datasource: string,
+ registryUrl: string,
+ packageName: string,
+ ): void {
+ const data = this.getData();
+ data.push({ datasource, registryUrl, packageName, action: 'miss' });
+ this.setData(data);
+ }
+
+ static set(
+ datasource: string,
+ registryUrl: string,
+ packageName: string,
+ ): void {
+ const data = this.getData();
+ data.push({ datasource, registryUrl, packageName, action: 'set' });
+ this.setData(data);
+ }
+
+ static skip(
+ datasource: string,
+ registryUrl: string,
+ packageName: string,
+ ): void {
+ const data = this.getData();
+ data.push({ datasource, registryUrl, packageName, action: 'skip' });
+ this.setData(data);
+ }
+
+ static getReport(): DatasourceCacheReport {
+ const data = this.getData();
+ const result: DatasourceCacheReport = { long: {}, short: {} };
+ for (const { datasource, registryUrl, packageName, action } of data) {
+ result.long[datasource] ??= {};
+ result.long[datasource][registryUrl] ??= {};
+ result.long[datasource][registryUrl] ??= {};
+ result.long[datasource][registryUrl][packageName] ??= {};
+
+ result.short[datasource] ??= {};
+ result.short[datasource][registryUrl] ??= {
+ hit: 0,
+ miss: 0,
+ set: 0,
+ skip: 0,
+ };
+
+ if (action === 'hit') {
+ result.long[datasource][registryUrl][packageName].read = 'hit';
+ result.short[datasource][registryUrl].hit += 1;
+ continue;
+ }
+
+ if (action === 'miss') {
+ result.long[datasource][registryUrl][packageName].read = 'miss';
+ result.short[datasource][registryUrl].miss += 1;
+ continue;
+ }
+
+ if (action === 'set') {
+ result.long[datasource][registryUrl][packageName].write = 'set';
+ result.short[datasource][registryUrl].set += 1;
+ continue;
+ }
+
+ if (action === 'skip') {
+ result.long[datasource][registryUrl][packageName].write = 'skip';
+ result.short[datasource][registryUrl].skip += 1;
+ continue;
+ }
+ }
+
+ return result;
+ }
+
+ static report(): void {
+ const { long, short } = this.getReport();
+
+ if (Object.keys(short).length > 0) {
+ logger.debug(short, 'Datasource cache statistics');
+ }
+
+ if (Object.keys(long).length > 0) {
+ logger.trace(long, 'Datasource cache detailed statistics');
+ }
+ }
+}
+
export interface HttpRequestStatsDataPoint {
method: string;
url: string;
diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts
index 9b05fd8420af5b86a61e4d7165c75eb602620510..6a5addcc0889e263247a6a43dcb24f53bbedfb3a 100644
--- a/lib/workers/repository/index.ts
+++ b/lib/workers/repository/index.ts
@@ -20,6 +20,7 @@ import * as queue from '../../util/http/queue';
import * as throttle from '../../util/http/throttle';
import { addSplit, getSplits, splitInit } from '../../util/split';
import {
+ DatasourceCacheStats,
HttpCacheStats,
HttpStats,
LookupStats,
@@ -134,6 +135,7 @@ export async function renovateRepository(
const splits = getSplits();
logger.debug(splits, 'Repository timing splits (milliseconds)');
PackageCacheStats.report();
+ DatasourceCacheStats.report();
HttpStats.report();
HttpCacheStats.report();
LookupStats.report();