diff --git a/lib/util/cache/repository/types.ts b/lib/util/cache/repository/types.ts
index 4a8c39478b1f12e51ad3de4eaf83579a31a3370b..de6af74256607b4ffe9c0249163b12847b899e23 100644
--- a/lib/util/cache/repository/types.ts
+++ b/lib/util/cache/repository/types.ts
@@ -8,6 +8,7 @@ import type { RepoInitConfig } from '../../../workers/repository/init/types';
export interface BaseBranchCache {
sha: string; // branch commit sha
configHash: string; // object hash of config
+ extractionFingerprints: Record<string, string | undefined>; // matching manager fingerprints
packageFiles: Record<string, PackageFile[]>; // extract result
}
diff --git a/lib/workers/repository/extract/index.spec.ts b/lib/workers/repository/extract/index.spec.ts
index 9c2a6bcef18b2b2c00e5a0b9698d66dff630be71..4b39cf0be52636debc4fc9df048eda1f521d9bfc 100644
--- a/lib/workers/repository/extract/index.spec.ts
+++ b/lib/workers/repository/extract/index.spec.ts
@@ -30,7 +30,7 @@ describe('workers/repository/extract/index', () => {
config.enabledManagers = ['npm'];
managerFiles.getManagerPackageFiles.mockResolvedValue([{} as never]);
const res = await extractAllDependencies(config);
- expect(res).toEqual({ packageFiles: { npm: [{}] } });
+ expect(res).toMatchObject({ packageFiles: { npm: [{}] } });
});
it('warns if no packages found for a enabled manager', async () => {
diff --git a/lib/workers/repository/extract/index.ts b/lib/workers/repository/extract/index.ts
index e3e5e18ee7c731a5b28a8d9f9060eb5c656f0ce4..57395bf4075eb9ff91b927829d680314bdb0e2c1 100644
--- a/lib/workers/repository/extract/index.ts
+++ b/lib/workers/repository/extract/index.ts
@@ -2,7 +2,7 @@ import is from '@sindresorhus/is';
import { getManagerConfig, mergeChildConfig } from '../../../config';
import type { ManagerConfig, RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
-import { getManagerList } from '../../../modules/manager';
+import { getManagerList, hashMap } from '../../../modules/manager';
import { getFileList } from '../../../util/git';
import type { ExtractResult, WorkerExtractConfig } from '../../types';
import { getMatchingFiles } from './file-match';
@@ -43,8 +43,15 @@ export async function extractAllDependencies(
const extractResult: ExtractResult = {
packageFiles: {},
+ extractionFingerprints: {},
};
+ // Store the fingerprint of all managers which match any file (even if they do not find any dependencies)
+ // The cached result needs to be invalidated if the fingerprint of any matching manager changes
+ for (const { manager } of extractList) {
+ extractResult.extractionFingerprints[manager] = hashMap.get(manager);
+ }
+
const extractResults = await Promise.all(
extractList.map(async (managerConfig) => {
const packageFiles = await getManagerPackageFiles(managerConfig);
diff --git a/lib/workers/repository/process/extract-update.spec.ts b/lib/workers/repository/process/extract-update.spec.ts
index f67d0b65aac7bb3c15172feee9f4a55a048096de..42380976007b7e4ba7cb69917bb47a5a536e87f5 100644
--- a/lib/workers/repository/process/extract-update.spec.ts
+++ b/lib/workers/repository/process/extract-update.spec.ts
@@ -87,6 +87,7 @@ describe('workers/repository/process/extract-update', () => {
master: {
sha: '123test',
configHash: fingerprint(generateFingerprintConfig(config)),
+ extractionFingerprints: {},
packageFiles,
},
},
@@ -99,19 +100,23 @@ describe('workers/repository/process/extract-update', () => {
});
describe('isCacheExtractValid()', () => {
- let cachedExtract: BaseBranchCache = undefined as never;
+ let cachedExtract: BaseBranchCache;
- it('undefined cache', () => {
- expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
- expect(logger.logger.debug).toHaveBeenCalledTimes(0);
- });
-
- it('partial cache', () => {
+ beforeEach(() => {
cachedExtract = {
sha: 'sha',
configHash: undefined as never,
+ extractionFingerprints: {},
packageFiles: {},
};
+ });
+
+ it('undefined cache', () => {
+ expect(isCacheExtractValid('sha', 'hash', undefined)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledTimes(0);
+ });
+
+ it('partial cache', () => {
expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
expect(logger.logger.debug).toHaveBeenCalledTimes(0);
});
@@ -126,7 +131,6 @@ describe('workers/repository/process/extract-update', () => {
});
it('config change', () => {
- cachedExtract.sha = 'sha';
cachedExtract.configHash = 'hash';
expect(isCacheExtractValid('sha', 'new_hash', cachedExtract)).toBe(false);
expect(logger.logger.debug).toHaveBeenCalledWith(
@@ -135,8 +139,30 @@ describe('workers/repository/process/extract-update', () => {
expect(logger.logger.debug).toHaveBeenCalledTimes(1);
});
+ it('invalid if no extractionFingerprints', () => {
+ cachedExtract.configHash = 'hash';
+ const { extractionFingerprints, ...restOfCache } = cachedExtract;
+ expect(
+ isCacheExtractValid(
+ 'sha',
+ 'hash',
+ restOfCache as never as BaseBranchCache
+ )
+ ).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledWith(
+ 'Cached extract is missing extractionFingerprints, so cannot be used'
+ );
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+
+ it('invalid if changed fingerprints', () => {
+ cachedExtract.configHash = 'hash';
+ cachedExtract.extractionFingerprints = { npm: 'old-fingerprint' };
+ expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+
it('valid cache and config', () => {
- cachedExtract.sha = 'sha';
cachedExtract.configHash = 'hash';
expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(true);
expect(logger.logger.debug).toHaveBeenCalledWith(
diff --git a/lib/workers/repository/process/extract-update.ts b/lib/workers/repository/process/extract-update.ts
index 029636be8b333929f51584bf8345ffc494158647..763aadd539ff2484bb43a4497b9fd1c0fed884cd 100644
--- a/lib/workers/repository/process/extract-update.ts
+++ b/lib/workers/repository/process/extract-update.ts
@@ -1,6 +1,7 @@
import is from '@sindresorhus/is';
import type { RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
+import { hashMap } from '../../../modules/manager';
import type { PackageFile } from '../../../modules/manager/types';
import { getCache } from '../../../util/cache/repository';
import type { BaseBranchCache } from '../../../util/cache/repository/types';
@@ -80,6 +81,27 @@ export function isCacheExtractValid(
logger.debug('Cached extract result cannot be used due to config change');
return false;
}
+ if (!cachedExtract.extractionFingerprints) {
+ logger.debug(
+ 'Cached extract is missing extractionFingerprints, so cannot be used'
+ );
+ return false;
+ }
+ const changedManagers = new Set();
+ for (const [manager, fingerprint] of Object.entries(
+ cachedExtract.extractionFingerprints
+ )) {
+ if (fingerprint !== hashMap.get(manager)) {
+ changedManagers.add(manager);
+ }
+ }
+ if (changedManagers.size > 0) {
+ logger.debug(
+ { changedManagers: [...changedManagers] },
+ 'Manager fingerprint(s) have changed, extract cache cannot be reused'
+ );
+ return false;
+ }
logger.debug(
`Cached extract for sha=${baseBranchSha} is valid and can be used`
);
@@ -114,12 +136,14 @@ export async function extract(
}
} else {
await checkoutBranch(baseBranch!);
- const extractResult = await extractAllDependencies(config);
- packageFiles = extractResult?.packageFiles;
+ const extractResult = (await extractAllDependencies(config)) || {};
+ packageFiles = extractResult.packageFiles;
+ const { extractionFingerprints } = extractResult;
// TODO: fix types (#7154)
cache.scan[baseBranch!] = {
sha: baseBranchSha!,
configHash,
+ extractionFingerprints,
packageFiles,
};
// Clean up cached branch extracts
diff --git a/lib/workers/types.ts b/lib/workers/types.ts
index fea5a440cd7f566a1cfc7bcb43aa930cb60a2f29..8236081cb5cd44cb3f7937b4df3daab277cdc11d 100644
--- a/lib/workers/types.ts
+++ b/lib/workers/types.ts
@@ -188,5 +188,6 @@ export interface UpgradeFingerprintConfig {
}
export interface ExtractResult {
+ extractionFingerprints: Record<string, string | undefined>;
packageFiles: Record<string, PackageFile[]>;
}