diff --git a/lib/config/types.ts b/lib/config/types.ts
index 7603028afe57188c664879f659e35423838a8af6..ee1824dd92968c2e4e4c2426a577f87f0f0b006d 100644
--- a/lib/config/types.ts
+++ b/lib/config/types.ts
@@ -181,7 +181,7 @@ export interface RegexManagerTemplates {
export interface RegExManager extends RegexManagerTemplates {
fileMatch: string[];
matchStrings: string[];
- matchStringsStrategy?: string;
+ matchStringsStrategy?: MatchStringsStrategy;
autoReplaceStringTemplate?: string;
}
@@ -231,6 +231,7 @@ export interface RenovateConfig
defaultRegistryUrls?: string[];
registryUrls?: string[] | null;
+ registryAliases?: Record<string, string>;
repoIsOnboarded?: boolean;
repoIsActivated?: boolean;
@@ -246,6 +247,7 @@ export interface RenovateConfig
secrets?: Record<string, string>;
constraints?: Record<string, string>;
+ skipInstalls?: boolean;
}
export interface AllConfig
diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts
index 0191d6a42104fd1cdbe819327ff4d4854662e610..90794370797457aa64d0b632c920eea383a37eb9 100644
--- a/lib/config/validation.spec.ts
+++ b/lib/config/validation.spec.ts
@@ -550,7 +550,7 @@ describe('config/validation', () => {
registryAliases: {
sample: {
example1: 'http://www.example.com',
- },
+ } as unknown as string, // intentional incorrect config to check error message
},
};
const { warnings, errors } = await configValidation.validateConfig(
diff --git a/lib/workers/repository/extract/extract-fingerprint-config.spec.ts b/lib/workers/repository/extract/extract-fingerprint-config.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..e18da13569c136193067bac5b80aa39d264a6113
--- /dev/null
+++ b/lib/workers/repository/extract/extract-fingerprint-config.spec.ts
@@ -0,0 +1,115 @@
+import { mergeChildConfig } from '../../../config';
+import { getConfig } from '../../../config/defaults';
+import { getManagerList } from '../../../modules/manager';
+import { generateFingerprintConfig } from './extract-fingerprint-config';
+
+describe('workers/repository/extract/extract-fingerprint-config', () => {
+ it('filter with enabledManagers', () => {
+ const config = mergeChildConfig(getConfig(), {
+ registryAliases: {
+ stable: 'http://some.link', // intentionally placing the field incorrectly
+ },
+ ignorePaths: ['ignore-path-1'],
+ includePaths: ['include-path-1'],
+ npm: {
+ fileMatch: ['hero.json'],
+ ignorePaths: ['ignore-path-2'],
+ includePaths: ['include-path-2'],
+ registryAliases: {
+ notStable: 'http://some.link.2',
+ },
+ },
+ enabledManagers: ['npm', 'regex'],
+ regexManagers: [
+ {
+ fileMatch: ['js', '***$}{]]['],
+ matchStrings: ['^(?<depName>foo)(?<currentValue>bar)$'],
+ datasourceTemplate: 'maven',
+ versioningTemplate: 'gradle',
+ },
+ ],
+ });
+
+ const fingerprintConfig = generateFingerprintConfig(config);
+
+ expect(fingerprintConfig.managerList).toEqual(new Set(['npm', 'regex']));
+ expect(
+ fingerprintConfig.managers.find((manager) => manager.manager === 'npm')
+ ).toEqual({
+ enabled: true,
+ fileList: [],
+ fileMatch: ['(^|/)package\\.json$', 'hero.json'],
+ ignorePaths: ['ignore-path-2'],
+ includePaths: ['include-path-2'],
+ manager: 'npm',
+ npmrc: null,
+ npmrcMerge: false,
+ registryAliases: {
+ notStable: 'http://some.link.2',
+ },
+ skipInstalls: null,
+ });
+ expect(
+ fingerprintConfig.managers.find((manager) => manager.manager === 'regex')
+ ).toEqual({
+ fileMatch: ['js', '***$}{]]['],
+ ignorePaths: ['ignore-path-1'],
+ includePaths: ['include-path-1'],
+ fileList: [],
+ matchStrings: ['^(?<depName>foo)(?<currentValue>bar)$'],
+ datasourceTemplate: 'maven',
+ versioningTemplate: 'gradle',
+ enabled: true,
+ manager: 'regex',
+ npmrc: null,
+ npmrcMerge: false,
+ registryAliases: {
+ stable: 'http://some.link',
+ },
+ skipInstalls: null,
+ });
+ });
+
+ it('filter with all managers enabled', () => {
+ const config = mergeChildConfig(getConfig(), {
+ npmrc: 'some-string',
+ npmrcMerge: true,
+ npm: { fileMatch: ['hero.json'] },
+ });
+ const fingerprintConfig = generateFingerprintConfig(config);
+ expect(fingerprintConfig.managerList).toEqual(new Set(getManagerList()));
+ expect(
+ fingerprintConfig.managers.find((manager) => manager.manager === 'npm')
+ ).toEqual({
+ enabled: true,
+ fileList: [],
+ fileMatch: ['(^|/)package\\.json$', 'hero.json'],
+ ignorePaths: ['**/node_modules/**', '**/bower_components/**'],
+ includePaths: [],
+ manager: 'npm',
+ npmrc: 'some-string',
+ npmrcMerge: true,
+ registryAliases: {},
+ skipInstalls: null,
+ });
+ expect(
+ fingerprintConfig.managers.find(
+ (manager) => manager.manager === 'dockerfile'
+ )
+ ).toEqual({
+ enabled: true,
+ fileList: [],
+ fileMatch: ['(^|/|\\.)Dockerfile$', '(^|/)Dockerfile[^/]*$'],
+ ignorePaths: ['**/node_modules/**', '**/bower_components/**'],
+ includePaths: [],
+ manager: 'dockerfile',
+ npmrc: 'some-string',
+ npmrcMerge: true,
+ registryAliases: {},
+ skipInstalls: null,
+ });
+ expect(
+ fingerprintConfig.managers.find((manager) => manager.manager === 'regex')
+ ).toBeUndefined();
+ });
+});
diff --git a/lib/workers/repository/extract/extract-fingerprint-config.ts b/lib/workers/repository/extract/extract-fingerprint-config.ts
new file mode 100644
index 0000000000000000000000000000000000000000..f7089bc7d003f86bdd3e65ca4f459626c18c6b69
--- /dev/null
+++ b/lib/workers/repository/extract/extract-fingerprint-config.ts
@@ -0,0 +1,84 @@
+import { getManagerConfig, mergeChildConfig } from '../../../config';
+import type {
+ RegexManagerTemplates,
+ RenovateConfig,
+} from '../../../config/types';
+import { getManagerList } from '../../../modules/manager';
+import { validMatchFields } from '../../../modules/manager/regex/utils';
+import type { CustomExtractConfig } from '../../../modules/manager/types';
+import type { WorkerExtractConfig } from '../../types';
+
+export interface FingerprintExtractConfig {
+ managerList: Set<string>;
+ managers: WorkerExtractConfig[];
+}
+
+function getRegexManagerFields(
+ config: WorkerExtractConfig
+): CustomExtractConfig {
+ const regexFields = {} as CustomExtractConfig;
+ for (const field of validMatchFields.map(
+ (f) => `${f}Template` as keyof RegexManagerTemplates
+ )) {
+ if (config[field]) {
+ regexFields[field] = config[field];
+ }
+ }
+
+ return {
+ autoReplaceStringTemplate: config.autoReplaceStringTemplate,
+ matchStrings: config.matchStrings,
+ matchStringsStrategy: config.matchStringsStrategy,
+ ...regexFields,
+ };
+}
+
+function getFilteredManagerConfig(
+ config: WorkerExtractConfig
+): WorkerExtractConfig {
+ return {
+ ...(config.manager === 'regex' && getRegexManagerFields(config)),
+ manager: config.manager,
+ fileMatch: config.fileMatch,
+ npmrc: config.npmrc,
+ npmrcMerge: config.npmrcMerge,
+ enabled: config.enabled,
+ ignorePaths: config.ignorePaths ?? [],
+ includePaths: config.includePaths ?? [],
+ skipInstalls: config.skipInstalls,
+ registryAliases: config.registryAliases,
+ fileList: [],
+ };
+}
+
+export function generateFingerprintConfig(
+ config: RenovateConfig
+): FingerprintExtractConfig {
+ const managerExtractConfigs: WorkerExtractConfig[] = [];
+ let managerList: Set<string>;
+ const { enabledManagers } = config;
+ if (enabledManagers?.length) {
+ managerList = new Set(enabledManagers);
+ } else {
+ managerList = new Set(getManagerList());
+ }
+
+ for (const manager of managerList) {
+ const managerConfig = getManagerConfig(config, manager);
+ if (manager === 'regex') {
+ for (const regexManager of config.regexManagers ?? []) {
+ managerExtractConfigs.push({
+ ...mergeChildConfig(managerConfig, regexManager),
+ fileList: [],
+ });
+ }
+ } else {
+ managerExtractConfigs.push({ ...managerConfig, fileList: [] });
+ }
+ }
+
+ return {
+ managerList,
+ managers: managerExtractConfigs.map(getFilteredManagerConfig),
+ };
+}
diff --git a/lib/workers/repository/process/extract-update.spec.ts b/lib/workers/repository/process/extract-update.spec.ts
index de4eebf437da5f7d9161a6b39cc55e7f20d2e749..f67d0b65aac7bb3c15172feee9f4a55a048096de 100644
--- a/lib/workers/repository/process/extract-update.spec.ts
+++ b/lib/workers/repository/process/extract-update.spec.ts
@@ -1,9 +1,11 @@
-import { git, mocked } from '../../../../test/util';
+import { git, logger, mocked } from '../../../../test/util';
import type { PackageFile } from '../../../modules/manager/types';
import * as _repositoryCache from '../../../util/cache/repository';
+import type { BaseBranchCache } from '../../../util/cache/repository/types';
import { fingerprint } from '../../../util/fingerprint';
+import { generateFingerprintConfig } from '../extract/extract-fingerprint-config';
import * as _branchify from '../updates/branchify';
-import { extract, lookup, update } from './extract-update';
+import { extract, isCacheExtractValid, lookup, update } from './extract-update';
jest.mock('./write');
jest.mock('./sort');
@@ -59,6 +61,13 @@ describe('workers/repository/process/extract-update', () => {
baseBranches: ['master', 'dev'],
repoIsOnboarded: true,
suppressNotifications: ['deprecationWarningIssues'],
+ enabledManagers: ['npm'],
+ javascript: {
+ labels: ['js'],
+ },
+ npm: {
+ addLabels: 'npm',
+ },
};
git.checkoutBranch.mockResolvedValueOnce('123test');
repositoryCache.getCache.mockReturnValueOnce({ scan: {} });
@@ -77,7 +86,7 @@ describe('workers/repository/process/extract-update', () => {
scan: {
master: {
sha: '123test',
- configHash: fingerprint(config),
+ configHash: fingerprint(generateFingerprintConfig(config)),
packageFiles,
},
},
@@ -88,4 +97,52 @@ describe('workers/repository/process/extract-update', () => {
expect(res).toEqual(packageFiles);
});
});
+
+ describe('isCacheExtractValid()', () => {
+ let cachedExtract: BaseBranchCache = undefined as never;
+
+ it('undefined cache', () => {
+ expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledTimes(0);
+ });
+
+ it('partial cache', () => {
+ cachedExtract = {
+ sha: 'sha',
+ configHash: undefined as never,
+ packageFiles: {},
+ };
+ expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledTimes(0);
+ });
+
+ it('sha mismatch', () => {
+ cachedExtract.configHash = 'hash';
+ expect(isCacheExtractValid('new_sha', 'hash', cachedExtract)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledWith(
+ `Cached extract result cannot be used due to base branch SHA change (old=sha, new=new_sha)`
+ );
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+
+ it('config change', () => {
+ cachedExtract.sha = 'sha';
+ cachedExtract.configHash = 'hash';
+ expect(isCacheExtractValid('sha', 'new_hash', cachedExtract)).toBe(false);
+ expect(logger.logger.debug).toHaveBeenCalledWith(
+ 'Cached extract result cannot be used due to config change'
+ );
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+
+ it('valid cache and config', () => {
+ cachedExtract.sha = 'sha';
+ cachedExtract.configHash = 'hash';
+ expect(isCacheExtractValid('sha', 'hash', cachedExtract)).toBe(true);
+ expect(logger.logger.debug).toHaveBeenCalledWith(
+ 'Cached extract for sha=sha is valid and can be used'
+ );
+ expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+ });
+ });
});
diff --git a/lib/workers/repository/process/extract-update.ts b/lib/workers/repository/process/extract-update.ts
index 44894838d2a509bef33d6b33edcf579be55b3fe4..07550b2ffe3bfb879be98f5293c283d32a1db904 100644
--- a/lib/workers/repository/process/extract-update.ts
+++ b/lib/workers/repository/process/extract-update.ts
@@ -1,14 +1,15 @@
-// TODO #7154
import is from '@sindresorhus/is';
import type { RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
import type { PackageFile } from '../../../modules/manager/types';
import { getCache } from '../../../util/cache/repository';
+import type { BaseBranchCache } from '../../../util/cache/repository/types';
import { checkGithubToken as ensureGithubToken } from '../../../util/check-token';
import { fingerprint } from '../../../util/fingerprint';
import { checkoutBranch, getBranchCommit } from '../../../util/git';
import type { BranchConfig } from '../../types';
import { extractAllDependencies } from '../extract';
+import { generateFingerprintConfig } from '../extract/extract-fingerprint-config';
import { branchifyUpgrades } from '../updates/branchify';
import { raiseDeprecationWarnings } from './deprecated';
import { fetchUpdates } from './fetch';
@@ -61,6 +62,30 @@ function extractStats(
return stats;
}
+export function isCacheExtractValid(
+ baseBranchSha: string,
+ configHash: string,
+ cachedExtract?: BaseBranchCache
+): boolean {
+ if (!(cachedExtract?.sha && cachedExtract.configHash)) {
+ return false;
+ }
+ if (cachedExtract.sha !== baseBranchSha) {
+ logger.debug(
+ `Cached extract result cannot be used due to base branch SHA change (old=${cachedExtract.sha}, new=${baseBranchSha})`
+ );
+ return false;
+ }
+ if (cachedExtract.configHash !== configHash) {
+ logger.debug('Cached extract result cannot be used due to config change');
+ return false;
+ }
+ logger.debug(
+ `Cached extract for sha=${baseBranchSha} is valid and can be used`
+ );
+ return true;
+}
+
export async function extract(
config: RenovateConfig
): Promise<Record<string, PackageFile[]>> {
@@ -71,17 +96,9 @@ export async function extract(
const cache = getCache();
cache.scan ||= {};
const cachedExtract = cache.scan[baseBranch!];
- const { packageRules, ...remainingConfig } = config;
- // Calculate hash excluding packageRules, because they're not applied during extract
- const configHash = fingerprint(remainingConfig);
+ const configHash = fingerprint(generateFingerprintConfig(config));
// istanbul ignore if
- if (
- cachedExtract?.sha === baseBranchSha &&
- cachedExtract?.configHash === configHash
- ) {
- logger.debug(
- `Found cached extract for ${baseBranch!} (sha=${baseBranchSha})`
- );
+ if (isCacheExtractValid(baseBranchSha!, configHash, cachedExtract)) {
packageFiles = cachedExtract.packageFiles;
try {
for (const files of Object.values(packageFiles)) {
@@ -98,6 +115,7 @@ export async function extract(
} else {
await checkoutBranch(baseBranch!);
packageFiles = await extractAllDependencies(config);
+ // TODO: fix types (#7154)
cache.scan[baseBranch!] = {
sha: baseBranchSha!,
configHash,
diff --git a/lib/workers/types.ts b/lib/workers/types.ts
index 692058889d6db2078f43627aae860d716bd0afb6..b1900a5977088a68d723e3bad60b3c987cfe46d0 100644
--- a/lib/workers/types.ts
+++ b/lib/workers/types.ts
@@ -2,7 +2,6 @@ import type { Merge } from 'type-fest';
import type {
GroupConfig,
LegacyAdminConfig,
- RegExManager,
RenovateConfig,
RenovateSharedConfig,
ValidationMessage,
@@ -154,11 +153,8 @@ export interface WorkerExtractConfig extends ExtractConfig {
manager: string;
fileList: string[];
fileMatch?: string[];
- updateInternalDeps?: boolean;
includePaths?: string[];
ignorePaths?: string[];
- regexManagers?: RegExManager[];
- enabledManagers?: string[];
enabled?: boolean;
}