From cf6be1719e6cb4c4fdb4b6253578df7033657fee Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Wed, 1 Feb 2023 09:12:45 +0100
Subject: [PATCH] feat(cache): retain fingerprints for all matched managers
(#20138)
---
lib/util/cache/repository/types.ts | 1 +
lib/workers/repository/extract/index.spec.ts | 2 +-
lib/workers/repository/extract/index.ts | 9 +++-
.../repository/process/extract-update.spec.ts | 44 +++++++++++++++----
.../repository/process/extract-update.ts | 28 +++++++++++-
lib/workers/types.ts | 1 +
6 files changed, 72 insertions(+), 13 deletions(-)
diff --git a/lib/util/cache/repository/types.ts b/lib/util/cache/repository/types.ts
index 4a8c39478b..de6af74256 100644
--- a/lib/util/cache/repository/types.ts
+++ b/lib/util/cache/repository/types.ts
@@ -8,6 +8,7 @@ import type { RepoInitConfig } from '../../../workers/repository/init/types';
export interface BaseBranchCache {
sha: string; // branch commit sha
configHash: string; // object hash of config
+ extractionFingerprints: Record<string, string | undefined>; // matching manager fingerprints
packageFiles: Record<string, PackageFile[]>; // extract result
}
diff --git a/lib/workers/repository/extract/index.spec.ts b/lib/workers/repository/extract/index.spec.ts
index 9c2a6bcef1..4b39cf0be5 100644
--- a/lib/workers/repository/extract/index.spec.ts
+++ b/lib/workers/repository/extract/index.spec.ts
@@ -30,7 +30,7 @@ describe('workers/repository/extract/index', () => {
config.enabledManagers = ['npm'];
managerFiles.getManagerPackageFiles.mockResolvedValue([{} as never]);
const res = await extractAllDependencies(config);
- expect(res).toEqual({ packageFiles: { npm: [{}] } });
+ expect(res).toMatchObject({ packageFiles: { npm: [{}] } });
});
it('warns if no packages found for a enabled manager', async () => {
diff --git a/lib/workers/repository/extract/index.ts b/lib/workers/repository/extract/index.ts
index e3e5e18ee7..57395bf407 100644
--- a/lib/workers/repository/extract/index.ts
+++ b/lib/workers/repository/extract/index.ts
@@ -2,7 +2,7 @@ import is from '@sindresorhus/is';
import { getManagerConfig, mergeChildConfig } from '../../../config';
import type { ManagerConfig, RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
-import { getManagerList } from '../../../modules/manager';
+import { getManagerList, hashMap } from '../../../modules/manager';
import { getFileList } from '../../../util/git';
import type { ExtractResult, WorkerExtractConfig } from '../../types';
import { getMatchingFiles } from './file-match';
@@ -43,8 +43,15 @@ export async function extractAllDependencies(
const extractResult: ExtractResult = {
packageFiles: {},
+ extractionFingerprints: {},
};
+ // Store the fingerprint of all managers which match any file (even if they do not find any dependencies)
+ // The cached result needs to be invalidated if the fingerprint of any matching manager changes
+ for (const { manager } of extractList) {
+ extractResult.extractionFingerprints[manager] = hashMap.get(manager);
+ }
+
const extractResults = await Promise.all(
extractList.map(async (managerConfig) => {
const packageFiles = await getManagerPackageFiles(managerConfig);
diff --git a/lib/workers/repository/process/extract-update.spec.ts b/lib/workers/repository/process/extract-update.spec.ts
index f67d0b65aa..4238097600 100644
--- a/lib/workers/repository/process/extract-update.spec.ts
+++ b/lib/workers/repository/process/extract-update.spec.ts
@@ -87,6 +87,7 @@ describe('workers/repository/process/extract-update', () => {
master: {
sha: '123test',
configHash: fingerprint(generateFingerprintConfig(config)),
+ extractionFingerprints: {},
packageFiles,
},
},
@@ -99,19 +100,23 @@ describe('workers/repository/process/extract-update', () => {
});
describe('isCacheExtractValid()', () => {
- let cachedExtract: BaseBranchCache = undefined as never;
+ let cachedExtract: BaseBranchCache;
- it('undefined cache', () => {
- expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
- expect(logger.logger.debug).toHaveBeenCalledTimes(0);
- });
-
- it('partial cache', () => {
+ beforeEach(() => {
cachedExtract = {
sha: 'sha',
configHash: undefined as never,
+ extractionFingerprints: {},
packageFiles: {},
};
+ });
+
+ it('undefined cache', () => {
+ expect(isCacheExtractValid('sha', 'hash', undefined)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledTimes(0);
+ });
+
+ it('partial cache', () => {
expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
expect(logger.logger.debug).toHaveBeenCalledTimes(0);
});
@@ -126,7 +131,6 @@ describe('workers/repository/process/extract-update', () => {
});
it('config change', () => {
- cachedExtract.sha = 'sha';
cachedExtract.configHash = 'hash';
expect(isCacheExtractValid('sha', 'new_hash', cachedExtract)).toBe(false);
expect(logger.logger.debug).toHaveBeenCalledWith(
@@ -135,8 +139,30 @@ describe('workers/repository/process/extract-update', () => {
expect(logger.logger.debug).toHaveBeenCalledTimes(1);
});
+ it('invalid if no extractionFingerprints', () => {
+ cachedExtract.configHash = 'hash';
+ const { extractionFingerprints, ...restOfCache } = cachedExtract;
+ expect(
+ isCacheExtractValid(
+ 'sha',
+ 'hash',
+ restOfCache as never as BaseBranchCache
+ )
+ ).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledWith(
+ 'Cached extract is missing extractionFingerprints, so cannot be used'
+ );
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+
+ it('invalid if changed fingerprints', () => {
+ cachedExtract.configHash = 'hash';
+ cachedExtract.extractionFingerprints = { npm: 'old-fingerprint' };
+ expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+
it('valid cache and config', () => {
- cachedExtract.sha = 'sha';
cachedExtract.configHash = 'hash';
expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(true);
expect(logger.logger.debug).toHaveBeenCalledWith(
diff --git a/lib/workers/repository/process/extract-update.ts b/lib/workers/repository/process/extract-update.ts
index 029636be8b..763aadd539 100644
--- a/lib/workers/repository/process/extract-update.ts
+++ b/lib/workers/repository/process/extract-update.ts
@@ -1,6 +1,7 @@
import is from '@sindresorhus/is';
import type { RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
+import { hashMap } from '../../../modules/manager';
import type { PackageFile } from '../../../modules/manager/types';
import { getCache } from '../../../util/cache/repository';
import type { BaseBranchCache } from '../../../util/cache/repository/types';
@@ -80,6 +81,27 @@ export function isCacheExtractValid(
logger.debug('Cached extract result cannot be used due to config change');
return false;
}
+ if (!cachedExtract.extractionFingerprints) {
+ logger.debug(
+ 'Cached extract is missing extractionFingerprints, so cannot be used'
+ );
+ return false;
+ }
+ const changedManagers = new Set();
+ for (const [manager, fingerprint] of Object.entries(
+ cachedExtract.extractionFingerprints
+ )) {
+ if (fingerprint !== hashMap.get(manager)) {
+ changedManagers.add(manager);
+ }
+ }
+ if (changedManagers.size > 0) {
+ logger.debug(
+ { changedManagers: [...changedManagers] },
+ 'Manager fingerprint(s) have changed, extract cache cannot be reused'
+ );
+ return false;
+ }
logger.debug(
`Cached extract for sha=${baseBranchSha} is valid and can be used`
);
@@ -114,12 +136,14 @@ export async function extract(
}
} else {
await checkoutBranch(baseBranch!);
- const extractResult = await extractAllDependencies(config);
- packageFiles = extractResult?.packageFiles;
+ const extractResult = (await extractAllDependencies(config)) || {};
+ packageFiles = extractResult.packageFiles;
+ const { extractionFingerprints } = extractResult;
// TODO: fix types (#7154)
cache.scan[baseBranch!] = {
sha: baseBranchSha!,
configHash,
+ extractionFingerprints,
packageFiles,
};
// Clean up cached branch extracts
diff --git a/lib/workers/types.ts b/lib/workers/types.ts
index fea5a440cd..8236081cb5 100644
--- a/lib/workers/types.ts
+++ b/lib/workers/types.ts
@@ -188,5 +188,6 @@ export interface UpgradeFingerprintConfig {
}
export interface ExtractResult {
+ extractionFingerprints: Record<string, string | undefined>;
packageFiles: Record<string, PackageFile[]>;
}
--
GitLab