diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index c5f1568323691c673ce2e8b6eba1cd5f7bf43085..dae88c9a9e6092193b9a4975e79e84a062cdbad5 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -149,6 +149,12 @@ If this value is set then Renovate will use Redis for its global cache instead o
## repositories
+## repositoryCache
+
+Set this to `"enabled"` to have Renovate maintain a JSON file cache per-repository to speed up extractions. Set to `"reset"` if you ever need to bypass the cache and have it overwritten. JSON files will be stored inside the `cacheDir` beside the existing file-based package cache.
+
+Warning: this is an experimental feature and may be modified or removed in a future non-major release.
+
## requireConfig
## skipInstalls
diff --git a/lib/config/common.ts b/lib/config/common.ts
index f212f835574eb190ddf35b7560ddf1e63305c2e0..a5537c876beb6b44573df57b4a8ddff28dae9420 100644
--- a/lib/config/common.ts
+++ b/lib/config/common.ts
@@ -9,6 +9,8 @@ export type RenovateConfigStage =
| 'branch'
| 'pr';
+export type RepositoryCacheConfig = 'disabled' | 'enabled' | 'reset';
+
export interface GroupConfig extends Record<string, unknown> {
branchName?: string;
branchTopic?: string;
@@ -45,6 +47,8 @@ export interface RenovateSharedConfig {
rebaseLabel?: string;
rebaseWhen?: string;
recreateClosed?: boolean;
+ repository?: string;
+ repositoryCache?: RepositoryCacheConfig;
requiredStatusChecks?: string[];
schedule?: string[];
semanticCommits?: boolean;
diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts
index ca7e6d77a4848e4929523062ebffe417fde10b01..554148d1f9460cd214663885183f5374df0cb5eb 100644
--- a/lib/config/definitions.ts
+++ b/lib/config/definitions.ts
@@ -204,6 +204,15 @@ const options: RenovateOptions[] = [
cli: false,
env: false,
},
+ {
+ name: 'repositoryCache',
+ description: 'Option to do repository extract caching.',
+ admin: true,
+ type: 'string',
+ allowedValues: ['disabled', 'enabled', 'reset'],
+ stage: 'repository',
+ default: 'disabled',
+ },
{
name: 'force',
description:
diff --git a/lib/util/cache/repository/index.spec.ts b/lib/util/cache/repository/index.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..b1b5d69dedb19c507adc5cce51a830b798d08dd4
--- /dev/null
+++ b/lib/util/cache/repository/index.spec.ts
@@ -0,0 +1,39 @@
+import * as _fs from 'fs-extra';
+import { mocked } from '../../../../test/util';
+import * as repositoryCache from '.';
+
+jest.mock('fs-extra');
+
+const fs = mocked(_fs);
+
+describe('lib/util/cache/repository', () => {
+ const config = {
+ cacheDir: '/tmp/renovate/cache/',
+ platform: 'github',
+ repository: 'abc/def',
+ };
+ it('catches and returns', async () => {
+ await repositoryCache.initialize({});
+ expect(fs.readFile.mock.calls).toHaveLength(0);
+ });
+ it('returns if cache not enabled', async () => {
+ await repositoryCache.initialize({
+ ...config,
+ repositoryCache: 'disabled',
+ });
+ expect(fs.readFile.mock.calls).toHaveLength(0);
+ });
+ it('reads from cache and finalizes', async () => {
+ fs.readFile.mockResolvedValueOnce('{}' as any);
+ await repositoryCache.initialize({
+ ...config,
+ repositoryCache: 'enabled',
+ });
+ await repositoryCache.finalize();
+ expect(fs.readFile.mock.calls).toHaveLength(1);
+ expect(fs.outputFile.mock.calls).toHaveLength(1);
+ });
+ it('gets', () => {
+ expect(repositoryCache.getCache()).toEqual({});
+ });
+});
diff --git a/lib/util/cache/repository/index.ts b/lib/util/cache/repository/index.ts
new file mode 100644
index 0000000000000000000000000000000000000000..395316808c4594580e99567885dde40aec9ee0b2
--- /dev/null
+++ b/lib/util/cache/repository/index.ts
@@ -0,0 +1,58 @@
+import * as fs from 'fs-extra';
+import { join } from 'upath';
+import { RenovateConfig, RepositoryCacheConfig } from '../../../config/common';
+import { logger } from '../../../logger';
+import { PackageFile } from '../../../manager/common';
+
+export interface BaseBranchCache {
+ sha: string; // branch commit sha
+ configHash: string; // object hash of config
+ packageFiles: PackageFile[]; // extract result
+}
+
+export interface Cache {
+ init?: {
+ configFile: string;
+ contents: RenovateConfig;
+ };
+ scan?: Record<string, BaseBranchCache>;
+}
+
+let repositoryCache: RepositoryCacheConfig = 'disabled';
+let cacheFileName: string;
+let cache: Cache = Object.create({});
+
+export function getCacheFileName(config: RenovateConfig): string {
+ return join(
+ config.cacheDir,
+ '/renovate/repository/',
+ config.platform,
+ config.repository + '.json'
+ );
+}
+
+export async function initialize(config: RenovateConfig): Promise<void> {
+ try {
+ cacheFileName = getCacheFileName(config);
+ repositoryCache = config.repositoryCache;
+ if (repositoryCache !== 'enabled') {
+ logger.debug('Skipping repository cache');
+ cache = {};
+ return;
+ }
+ cache = JSON.parse(await fs.readFile(cacheFileName, 'utf8'));
+ logger.debug({ cacheFileName }, 'Read repository cache');
+ } catch (err) {
+ logger.debug({ cacheFileName }, 'No repository cache found');
+ }
+}
+
+export function getCache(): Cache {
+ return cache;
+}
+
+export async function finalize(): Promise<void> {
+ if (repositoryCache !== 'disabled') {
+ await fs.outputFile(cacheFileName, JSON.stringify(cache));
+ }
+}
diff --git a/lib/workers/repository/finalise/index.ts b/lib/workers/repository/finalise/index.ts
index 1d893607437ac5c35876dc2dcbcd19a4f9b013f0..c8ff40d7c7755b1763f0c211626e5278c18b9cbe 100644
--- a/lib/workers/repository/finalise/index.ts
+++ b/lib/workers/repository/finalise/index.ts
@@ -1,5 +1,6 @@
import { RenovateConfig } from '../../../config';
import { platform } from '../../../platform';
+import * as repositoryCache from '../../../util/cache/repository';
import { pruneStaleBranches } from './prune';
// istanbul ignore next
@@ -7,6 +8,7 @@ export async function finaliseRepo(
config: RenovateConfig,
branchList: string[]
): Promise<void> {
+ await repositoryCache.finalize();
await pruneStaleBranches(config, branchList);
await platform.ensureIssueClosing(
`Action Required: Fix Renovate Configuration`
diff --git a/lib/workers/repository/init/config.ts b/lib/workers/repository/init/config.ts
index cdc7d8fa1fc14955420cb561a5fa6517918d11cd..c67276686c57fedfb5981fd42828615cbf2de83d 100644
--- a/lib/workers/repository/init/config.ts
+++ b/lib/workers/repository/init/config.ts
@@ -12,6 +12,8 @@ import * as npmApi from '../../../datasource/npm';
import { logger } from '../../../logger';
import { platform } from '../../../platform';
import { ExternalHostError } from '../../../types/errors/external-host-error';
+import { getCache } from '../../../util/cache/repository';
+import { clone } from '../../../util/clone';
import { readLocalFile } from '../../../util/gitfs';
import * as hostRules from '../../../util/host-rules';
import { flattenPackageRules } from './flatten';
@@ -121,6 +123,11 @@ export async function mergeRenovateConfig(
}
logger.debug({ configFile, config: renovateJson }, 'Repository config');
}
+ const cache = getCache();
+ cache.init = {
+ configFile,
+ contents: clone(renovateJson),
+ };
const migratedConfig = await migrateAndValidate(config, renovateJson);
if (migratedConfig.errors.length) {
const error = new Error(CONFIG_VALIDATION);
diff --git a/lib/workers/repository/init/index.ts b/lib/workers/repository/init/index.ts
index f0c901d5055789e59e132a2ababb3802eb913ff7..040cd7f06151267c0d7b00a09ecbca4af8d7434a 100644
--- a/lib/workers/repository/init/index.ts
+++ b/lib/workers/repository/init/index.ts
@@ -2,6 +2,7 @@ import { RenovateConfig } from '../../../config';
import { logger } from '../../../logger';
import { platform } from '../../../platform';
import * as memCache from '../../../util/cache/memory';
+import * as repositoryCache from '../../../util/cache/repository';
import { checkIfConfigured } from '../configured';
import { checkOnboardingBranch } from '../onboarding/branch';
import { initApis } from './apis';
@@ -12,6 +13,7 @@ import { detectVulnerabilityAlerts } from './vulnerability';
export async function initRepo(input: RenovateConfig): Promise<RenovateConfig> {
memCache.init();
+ await repositoryCache.initialize(input);
let config: RenovateConfig = {
...input,
errors: [],
diff --git a/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap b/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap
index 20f00a8ca662d9c9ae99cfc8a35b6e6a547c6c9d..2b515357e01873b2ba8f469cd443195e72334555 100644
--- a/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap
+++ b/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap
@@ -1,6 +1,8 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
-exports[`workers/repository/process/extract-update extract() runs 1`] = `
+exports[`workers/repository/process/extract-update extract() runs with baseBranches 1`] = `undefined`;
+
+exports[`workers/repository/process/extract-update extract() runs with no baseBranches 1`] = `
Object {
"branchList": Array [
"branchName",
diff --git a/lib/workers/repository/process/extract-update.spec.ts b/lib/workers/repository/process/extract-update.spec.ts
index 429cd72ea79ec80a0256d6883a8b650c9165beab..aa5ddeed6323d79bf43a0626ac30db606a55bbda 100644
--- a/lib/workers/repository/process/extract-update.spec.ts
+++ b/lib/workers/repository/process/extract-update.spec.ts
@@ -1,4 +1,6 @@
+import hash from 'object-hash';
import { mocked } from '../../../../test/util';
+import * as _repositoryCache from '../../../util/cache/repository';
import * as _branchify from '../updates/branchify';
import { extract, lookup, update } from './extract-update';
@@ -7,8 +9,10 @@ jest.mock('./sort');
jest.mock('./fetch');
jest.mock('../updates/branchify');
jest.mock('../extract');
+jest.mock('../../../util/cache/repository');
const branchify = mocked(_branchify);
+const repositoryCache = mocked(_repositoryCache);
branchify.branchifyUpgrades.mockResolvedValueOnce({
branches: [{ branchName: 'some-branch', upgrades: [] }],
@@ -17,15 +21,46 @@ branchify.branchifyUpgrades.mockResolvedValueOnce({
describe('workers/repository/process/extract-update', () => {
describe('extract()', () => {
- it('runs', async () => {
+ it('runs with no baseBranches', async () => {
const config = {
repoIsOnboarded: true,
suppressNotifications: ['deprecationWarningIssues'],
};
+ repositoryCache.getCache.mockReturnValueOnce({});
const packageFiles = await extract(config);
const res = await lookup(config, packageFiles);
expect(res).toMatchSnapshot();
await expect(update(config, res.branches)).resolves.not.toThrow();
});
+ it('runs with baseBranches', async () => {
+ const config = {
+ baseBranches: ['master', 'dev'],
+ repoIsOnboarded: true,
+ suppressNotifications: ['deprecationWarningIssues'],
+ };
+ repositoryCache.getCache.mockReturnValueOnce({});
+ const packageFiles = await extract(config);
+ expect(packageFiles).toMatchSnapshot();
+ });
+ it('uses repository cache', async () => {
+ const packageFiles = [];
+ const config = {
+ repoIsOnboarded: true,
+ suppressNotifications: ['deprecationWarningIssues'],
+ baseBranch: 'master',
+ baseBranchSha: 'abc123',
+ };
+ repositoryCache.getCache.mockReturnValueOnce({
+ scan: {
+ master: {
+ sha: config.baseBranchSha,
+ configHash: hash(config).toString(),
+ packageFiles,
+ },
+ },
+ });
+ const res = await extract(config);
+ expect(res).toEqual(packageFiles);
+ });
});
});
diff --git a/lib/workers/repository/process/extract-update.ts b/lib/workers/repository/process/extract-update.ts
index 2c4d9bcba3fd21c206a2f92dbad47fd4d547473b..e27076a4c34ae812e38a444f21ed1cb479e4b523 100644
--- a/lib/workers/repository/process/extract-update.ts
+++ b/lib/workers/repository/process/extract-update.ts
@@ -1,6 +1,9 @@
+import is from '@sindresorhus/is';
+import hash from 'object-hash';
import { RenovateConfig } from '../../../config';
import { logger } from '../../../logger';
import { PackageFile } from '../../../manager/common';
+import { getCache } from '../../../util/cache/repository';
import { BranchConfig } from '../../common';
import { extractAllDependencies } from '../extract';
import { branchifyUpgrades } from '../updates/branchify';
@@ -47,7 +50,36 @@ export async function extract(
config: RenovateConfig
): Promise<Record<string, PackageFile[]>> {
logger.debug('extract()');
- const packageFiles = await extractAllDependencies(config);
+ const { baseBranch, baseBranchSha } = config;
+ let packageFiles;
+ const cache = getCache();
+ const cachedExtract = cache?.scan?.[baseBranch];
+ const configHash = hash(config);
+ // istanbul ignore if
+ if (
+ cachedExtract?.sha === baseBranchSha &&
+ cachedExtract?.configHash === configHash
+ ) {
+ logger.debug({ baseBranch, baseBranchSha }, 'Found cached extract');
+ packageFiles = cachedExtract.packageFiles;
+ } else {
+ packageFiles = await extractAllDependencies(config);
+ cache.scan = cache.scan || Object.create({});
+ cache.scan[baseBranch] = {
+ sha: baseBranchSha,
+ configHash,
+ packageFiles,
+ };
+ // Clean up cached branch extracts
+ const baseBranches = is.nonEmptyArray(config.baseBranches)
+ ? config.baseBranches
+ : [baseBranch];
+ Object.keys(cache.scan).forEach((branchName) => {
+ if (!baseBranches.includes(branchName)) {
+ delete cache.scan[branchName];
+ }
+ });
+ }
const stats = extractStats(packageFiles);
logger.info(
{ baseBranch: config.baseBranch, stats },
diff --git a/package.json b/package.json
index 6532a7305773bfdb391ffe4ac8469a820016de09..f5b5da61f20dcffa7948f63ec73ef349a2d7bd34 100644
--- a/package.json
+++ b/package.json
@@ -151,6 +151,7 @@
"moment-timezone": "0.5.31",
"node-emoji": "1.10.0",
"node-html-parser": "1.2.20",
+ "object-hash": "2.0.3",
"p-all": "2.1.0",
"p-map": "4.0.0",
"parse-diff": "0.7.0",
diff --git a/yarn.lock b/yarn.lock
index 654696d73335bdfe4ccadb747f9393fbcdeebad8..1d76acaad5a4a6a9a6b2793d6fd509adb0440489 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7474,6 +7474,11 @@ object-copy@^0.1.0:
define-property "^0.2.5"
kind-of "^3.0.3"
+object-hash@2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea"
+ integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg==
+
object-inspect@^1.7.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0"