From d70b8c1f2fa999be9684a8ca5da3dda8cd2392ab Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Mon, 29 Jun 2020 15:51:22 +0200
Subject: [PATCH] feat: repositoryCache (#6589)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
docs/usage/self-hosted-configuration.md | 6 ++
lib/config/common.ts | 4 ++
lib/config/definitions.ts | 9 +++
lib/util/cache/repository/index.spec.ts | 39 +++++++++++++
lib/util/cache/repository/index.ts | 58 +++++++++++++++++++
lib/workers/repository/finalise/index.ts | 2 +
lib/workers/repository/init/config.ts | 7 +++
lib/workers/repository/init/index.ts | 2 +
.../__snapshots__/extract-update.spec.ts.snap | 4 +-
.../repository/process/extract-update.spec.ts | 37 +++++++++++-
.../repository/process/extract-update.ts | 34 ++++++++++-
package.json | 1 +
yarn.lock | 5 ++
13 files changed, 205 insertions(+), 3 deletions(-)
create mode 100644 lib/util/cache/repository/index.spec.ts
create mode 100644 lib/util/cache/repository/index.ts
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index c5f1568323..dae88c9a9e 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -149,6 +149,12 @@ If this value is set then Renovate will use Redis for its global cache instead o
## repositories
+## repositoryCache
+
+Set this to `"enabled"` to have Renovate maintain a JSON file cache per-repository to speed up extractions. Set to `"reset"` if you ever need to bypass the cache and have it overwritten. JSON files will be stored inside the `cacheDir` beside the existing file-based package cache.
+
+Warning: this is an experimental feature and may be modified or removed in a future non-major release.
+
## requireConfig
## skipInstalls
diff --git a/lib/config/common.ts b/lib/config/common.ts
index f212f83557..a5537c876b 100644
--- a/lib/config/common.ts
+++ b/lib/config/common.ts
@@ -9,6 +9,8 @@ export type RenovateConfigStage =
| 'branch'
| 'pr';
+export type RepositoryCacheConfig = 'disabled' | 'enabled' | 'reset';
+
export interface GroupConfig extends Record<string, unknown> {
branchName?: string;
branchTopic?: string;
@@ -45,6 +47,8 @@ export interface RenovateSharedConfig {
rebaseLabel?: string;
rebaseWhen?: string;
recreateClosed?: boolean;
+ repository?: string;
+ repositoryCache?: RepositoryCacheConfig;
requiredStatusChecks?: string[];
schedule?: string[];
semanticCommits?: boolean;
diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts
index ca7e6d77a4..554148d1f9 100644
--- a/lib/config/definitions.ts
+++ b/lib/config/definitions.ts
@@ -204,6 +204,15 @@ const options: RenovateOptions[] = [
cli: false,
env: false,
},
+ {
+ name: 'repositoryCache',
+ description: 'Option to do repository extract caching.',
+ admin: true,
+ type: 'string',
+ allowedValues: ['disabled', 'enabled', 'reset'],
+ stage: 'repository',
+ default: 'disabled',
+ },
{
name: 'force',
description:
diff --git a/lib/util/cache/repository/index.spec.ts b/lib/util/cache/repository/index.spec.ts
new file mode 100644
index 0000000000..b1b5d69ded
--- /dev/null
+++ b/lib/util/cache/repository/index.spec.ts
@@ -0,0 +1,39 @@
+import * as _fs from 'fs-extra';
+import { mocked } from '../../../../test/util';
+import * as repositoryCache from '.';
+
+jest.mock('fs-extra');
+
+const fs = mocked(_fs);
+
+describe('lib/util/cache/repository', () => {
+ const config = {
+ cacheDir: '/tmp/renovate/cache/',
+ platform: 'github',
+ repository: 'abc/def',
+ };
+ it('catches and returns', async () => {
+ await repositoryCache.initialize({});
+ expect(fs.readFile.mock.calls).toHaveLength(0);
+ });
+ it('returns if cache not enabled', async () => {
+ await repositoryCache.initialize({
+ ...config,
+ repositoryCache: 'disabled',
+ });
+ expect(fs.readFile.mock.calls).toHaveLength(0);
+ });
+ it('reads from cache and finalizes', async () => {
+ fs.readFile.mockResolvedValueOnce('{}' as any);
+ await repositoryCache.initialize({
+ ...config,
+ repositoryCache: 'enabled',
+ });
+ await repositoryCache.finalize();
+ expect(fs.readFile.mock.calls).toHaveLength(1);
+ expect(fs.outputFile.mock.calls).toHaveLength(1);
+ });
+ it('gets', () => {
+ expect(repositoryCache.getCache()).toEqual({});
+ });
+});
diff --git a/lib/util/cache/repository/index.ts b/lib/util/cache/repository/index.ts
new file mode 100644
index 0000000000..395316808c
--- /dev/null
+++ b/lib/util/cache/repository/index.ts
@@ -0,0 +1,58 @@
+import * as fs from 'fs-extra';
+import { join } from 'upath';
+import { RenovateConfig, RepositoryCacheConfig } from '../../../config/common';
+import { logger } from '../../../logger';
+import { PackageFile } from '../../../manager/common';
+
+export interface BaseBranchCache {
+ sha: string; // branch commit sha
+ configHash: string; // object hash of config
+ packageFiles: PackageFile[]; // extract result
+}
+
+export interface Cache {
+ init?: {
+ configFile: string;
+ contents: RenovateConfig;
+ };
+ scan?: Record<string, BaseBranchCache>;
+}
+
+let repositoryCache: RepositoryCacheConfig = 'disabled';
+let cacheFileName: string;
+let cache: Cache = Object.create({});
+
+export function getCacheFileName(config: RenovateConfig): string {
+ return join(
+ config.cacheDir,
+ '/renovate/repository/',
+ config.platform,
+ config.repository + '.json'
+ );
+}
+
+export async function initialize(config: RenovateConfig): Promise<void> {
+ try {
+ cacheFileName = getCacheFileName(config);
+ repositoryCache = config.repositoryCache;
+ if (repositoryCache !== 'enabled') {
+ logger.debug('Skipping repository cache');
+ cache = {};
+ return;
+ }
+ cache = JSON.parse(await fs.readFile(cacheFileName, 'utf8'));
+ logger.debug({ cacheFileName }, 'Read repository cache');
+ } catch (err) {
+ logger.debug({ cacheFileName }, 'No repository cache found');
+ }
+}
+
+export function getCache(): Cache {
+ return cache;
+}
+
+export async function finalize(): Promise<void> {
+ if (repositoryCache !== 'disabled') {
+ await fs.outputFile(cacheFileName, JSON.stringify(cache));
+ }
+}
diff --git a/lib/workers/repository/finalise/index.ts b/lib/workers/repository/finalise/index.ts
index 1d89360743..c8ff40d7c7 100644
--- a/lib/workers/repository/finalise/index.ts
+++ b/lib/workers/repository/finalise/index.ts
@@ -1,5 +1,6 @@
import { RenovateConfig } from '../../../config';
import { platform } from '../../../platform';
+import * as repositoryCache from '../../../util/cache/repository';
import { pruneStaleBranches } from './prune';
// istanbul ignore next
@@ -7,6 +8,7 @@ export async function finaliseRepo(
config: RenovateConfig,
branchList: string[]
): Promise<void> {
+ await repositoryCache.finalize();
await pruneStaleBranches(config, branchList);
await platform.ensureIssueClosing(
`Action Required: Fix Renovate Configuration`
diff --git a/lib/workers/repository/init/config.ts b/lib/workers/repository/init/config.ts
index cdc7d8fa1f..c67276686c 100644
--- a/lib/workers/repository/init/config.ts
+++ b/lib/workers/repository/init/config.ts
@@ -12,6 +12,8 @@ import * as npmApi from '../../../datasource/npm';
import { logger } from '../../../logger';
import { platform } from '../../../platform';
import { ExternalHostError } from '../../../types/errors/external-host-error';
+import { getCache } from '../../../util/cache/repository';
+import { clone } from '../../../util/clone';
import { readLocalFile } from '../../../util/gitfs';
import * as hostRules from '../../../util/host-rules';
import { flattenPackageRules } from './flatten';
@@ -121,6 +123,11 @@ export async function mergeRenovateConfig(
}
logger.debug({ configFile, config: renovateJson }, 'Repository config');
}
+ const cache = getCache();
+ cache.init = {
+ configFile,
+ contents: clone(renovateJson),
+ };
const migratedConfig = await migrateAndValidate(config, renovateJson);
if (migratedConfig.errors.length) {
const error = new Error(CONFIG_VALIDATION);
diff --git a/lib/workers/repository/init/index.ts b/lib/workers/repository/init/index.ts
index f0c901d505..040cd7f061 100644
--- a/lib/workers/repository/init/index.ts
+++ b/lib/workers/repository/init/index.ts
@@ -2,6 +2,7 @@ import { RenovateConfig } from '../../../config';
import { logger } from '../../../logger';
import { platform } from '../../../platform';
import * as memCache from '../../../util/cache/memory';
+import * as repositoryCache from '../../../util/cache/repository';
import { checkIfConfigured } from '../configured';
import { checkOnboardingBranch } from '../onboarding/branch';
import { initApis } from './apis';
@@ -12,6 +13,7 @@ import { detectVulnerabilityAlerts } from './vulnerability';
export async function initRepo(input: RenovateConfig): Promise<RenovateConfig> {
memCache.init();
+ await repositoryCache.initialize(input);
let config: RenovateConfig = {
...input,
errors: [],
diff --git a/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap b/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap
index 20f00a8ca6..2b515357e0 100644
--- a/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap
+++ b/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap
@@ -1,6 +1,8 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
-exports[`workers/repository/process/extract-update extract() runs 1`] = `
+exports[`workers/repository/process/extract-update extract() runs with baseBranches 1`] = `undefined`;
+
+exports[`workers/repository/process/extract-update extract() runs with no baseBranches 1`] = `
Object {
"branchList": Array [
"branchName",
diff --git a/lib/workers/repository/process/extract-update.spec.ts b/lib/workers/repository/process/extract-update.spec.ts
index 429cd72ea7..aa5ddeed63 100644
--- a/lib/workers/repository/process/extract-update.spec.ts
+++ b/lib/workers/repository/process/extract-update.spec.ts
@@ -1,4 +1,6 @@
+import hash from 'object-hash';
import { mocked } from '../../../../test/util';
+import * as _repositoryCache from '../../../util/cache/repository';
import * as _branchify from '../updates/branchify';
import { extract, lookup, update } from './extract-update';
@@ -7,8 +9,10 @@ jest.mock('./sort');
jest.mock('./fetch');
jest.mock('../updates/branchify');
jest.mock('../extract');
+jest.mock('../../../util/cache/repository');
const branchify = mocked(_branchify);
+const repositoryCache = mocked(_repositoryCache);
branchify.branchifyUpgrades.mockResolvedValueOnce({
branches: [{ branchName: 'some-branch', upgrades: [] }],
@@ -17,15 +21,46 @@ branchify.branchifyUpgrades.mockResolvedValueOnce({
describe('workers/repository/process/extract-update', () => {
describe('extract()', () => {
- it('runs', async () => {
+ it('runs with no baseBranches', async () => {
const config = {
repoIsOnboarded: true,
suppressNotifications: ['deprecationWarningIssues'],
};
+ repositoryCache.getCache.mockReturnValueOnce({});
const packageFiles = await extract(config);
const res = await lookup(config, packageFiles);
expect(res).toMatchSnapshot();
await expect(update(config, res.branches)).resolves.not.toThrow();
});
+ it('runs with baseBranches', async () => {
+ const config = {
+ baseBranches: ['master', 'dev'],
+ repoIsOnboarded: true,
+ suppressNotifications: ['deprecationWarningIssues'],
+ };
+ repositoryCache.getCache.mockReturnValueOnce({});
+ const packageFiles = await extract(config);
+ expect(packageFiles).toMatchSnapshot();
+ });
+ it('uses repository cache', async () => {
+ const packageFiles = [];
+ const config = {
+ repoIsOnboarded: true,
+ suppressNotifications: ['deprecationWarningIssues'],
+ baseBranch: 'master',
+ baseBranchSha: 'abc123',
+ };
+ repositoryCache.getCache.mockReturnValueOnce({
+ scan: {
+ master: {
+ sha: config.baseBranchSha,
+ configHash: hash(config).toString(),
+ packageFiles,
+ },
+ },
+ });
+ const res = await extract(config);
+ expect(res).toEqual(packageFiles);
+ });
});
});
diff --git a/lib/workers/repository/process/extract-update.ts b/lib/workers/repository/process/extract-update.ts
index 2c4d9bcba3..e27076a4c3 100644
--- a/lib/workers/repository/process/extract-update.ts
+++ b/lib/workers/repository/process/extract-update.ts
@@ -1,6 +1,9 @@
+import is from '@sindresorhus/is';
+import hash from 'object-hash';
import { RenovateConfig } from '../../../config';
import { logger } from '../../../logger';
import { PackageFile } from '../../../manager/common';
+import { getCache } from '../../../util/cache/repository';
import { BranchConfig } from '../../common';
import { extractAllDependencies } from '../extract';
import { branchifyUpgrades } from '../updates/branchify';
@@ -47,7 +50,36 @@ export async function extract(
config: RenovateConfig
): Promise<Record<string, PackageFile[]>> {
logger.debug('extract()');
- const packageFiles = await extractAllDependencies(config);
+ const { baseBranch, baseBranchSha } = config;
+ let packageFiles;
+ const cache = getCache();
+ const cachedExtract = cache?.scan?.[baseBranch];
+ const configHash = hash(config);
+ // istanbul ignore if
+ if (
+ cachedExtract?.sha === baseBranchSha &&
+ cachedExtract?.configHash === configHash
+ ) {
+ logger.debug({ baseBranch, baseBranchSha }, 'Found cached extract');
+ packageFiles = cachedExtract.packageFiles;
+ } else {
+ packageFiles = await extractAllDependencies(config);
+ cache.scan = cache.scan || Object.create({});
+ cache.scan[baseBranch] = {
+ sha: baseBranchSha,
+ configHash,
+ packageFiles,
+ };
+ // Clean up cached branch extracts
+ const baseBranches = is.nonEmptyArray(config.baseBranches)
+ ? config.baseBranches
+ : [baseBranch];
+ Object.keys(cache.scan).forEach((branchName) => {
+ if (!baseBranches.includes(branchName)) {
+ delete cache.scan[branchName];
+ }
+ });
+ }
const stats = extractStats(packageFiles);
logger.info(
{ baseBranch: config.baseBranch, stats },
diff --git a/package.json b/package.json
index 6532a73057..f5b5da61f2 100644
--- a/package.json
+++ b/package.json
@@ -151,6 +151,7 @@
"moment-timezone": "0.5.31",
"node-emoji": "1.10.0",
"node-html-parser": "1.2.20",
+ "object-hash": "2.0.3",
"p-all": "2.1.0",
"p-map": "4.0.0",
"parse-diff": "0.7.0",
diff --git a/yarn.lock b/yarn.lock
index 654696d733..1d76acaad5 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -7474,6 +7474,11 @@ object-copy@^0.1.0:
define-property "^0.2.5"
kind-of "^3.0.3"
+object-hash@2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea"
+ integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg==
+
object-inspect@^1.7.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0"
--
GitLab