From 51a43d5e41e0727b618553a688ff554898677d66 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Mon, 4 May 2020 08:27:38 +0200
Subject: [PATCH] feat(internal): cache extractions results (#6118)
---
.../extract/__snapshots__/cache.spec.ts.snap | 3 +
lib/workers/repository/extract/cache.spec.ts | 30 +++++++++
lib/workers/repository/extract/cache.ts | 67 +++++++++++++++++++
lib/workers/repository/extract/index.spec.ts | 8 +++
lib/workers/repository/extract/index.ts | 6 ++
.../onboarding/branch/index.spec.ts | 1 +
6 files changed, 115 insertions(+)
create mode 100644 lib/workers/repository/extract/__snapshots__/cache.spec.ts.snap
create mode 100644 lib/workers/repository/extract/cache.spec.ts
create mode 100644 lib/workers/repository/extract/cache.ts
diff --git a/lib/workers/repository/extract/__snapshots__/cache.spec.ts.snap b/lib/workers/repository/extract/__snapshots__/cache.spec.ts.snap
new file mode 100644
index 0000000000..649f7005f5
--- /dev/null
+++ b/lib/workers/repository/extract/__snapshots__/cache.spec.ts.snap
@@ -0,0 +1,3 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`workers/repository/extract/cache returns a hash 1`] = `"3Jfx5tzsGJEm8HQCeHT/6QL5vzE="`;
diff --git a/lib/workers/repository/extract/cache.spec.ts b/lib/workers/repository/extract/cache.spec.ts
new file mode 100644
index 0000000000..18b16bfa85
--- /dev/null
+++ b/lib/workers/repository/extract/cache.spec.ts
@@ -0,0 +1,30 @@
+import * as cache from './cache';
+
+describe('workers/repository/extract/cache', () => {
+ const config = { baseBranch: 'master', baseBranchSha: 'abc123' };
+ const extractList = [];
+ const extraction = { foo: [] };
+ it('handles missing sha', () => {
+ expect(cache.getExtractHash({}, [])).toBeNull();
+ });
+ it('returns a hash', () => {
+ expect(
+ cache.getExtractHash({ baseBranchSha: 'abc123' }, [])
+ ).toMatchSnapshot();
+ });
+ it('sets a value', async () => {
+ await cache.setCachedExtract(config, extractList, extraction);
+ });
+ it('gets no value', async () => {
+ extractList.push('abc');
+ const res = await cache.getCachedExtract(config, extractList);
+ expect(res).toEqual(null);
+ });
+ it('handles no sha error', async () => {
+ const res = await cache.getCachedExtract(
+ { baseBranch: 'nothing' },
+ extractList
+ );
+ expect(res).toBeNull();
+ });
+});
diff --git a/lib/workers/repository/extract/cache.ts b/lib/workers/repository/extract/cache.ts
new file mode 100644
index 0000000000..ca4baedcb7
--- /dev/null
+++ b/lib/workers/repository/extract/cache.ts
@@ -0,0 +1,67 @@
+import crypto from 'crypto';
+import { RenovateConfig } from '../../../config/common';
+import { logger } from '../../../logger';
+import { PackageFile } from '../../../manager/common';
+
+function getCacheNamespaceKey(
+ config: RenovateConfig
+): { cacheNamespace: string; cacheKey: string } {
+ // Cache extract results per-base branch
+ const { platform, repository, baseBranch } = config;
+ const cacheNamespace = 'repository-extract';
+ const cacheKey = `${platform}/${repository}/${baseBranch}`;
+ return { cacheNamespace, cacheKey };
+}
+
+export function getExtractHash(
+ config: RenovateConfig,
+ extractList: RenovateConfig[]
+): string | null {
+ // A cache is only valid if the following are unchanged:
+ // * base branch SHA
+ // * the list of matching files for each manager
+ if (!config.baseBranchSha) {
+ logger.warn('No baseBranchSha found in config');
+ return null;
+ }
+ return crypto
+ .createHash('sha1')
+ .update(config.baseBranchSha)
+ .update(JSON.stringify(extractList))
+ .digest('base64');
+}
+
+export async function getCachedExtract(
+ config: RenovateConfig,
+ extractList: RenovateConfig[]
+): Promise<Record<string, PackageFile[]> | null> {
+ const { baseBranch } = config;
+ const { cacheNamespace, cacheKey } = getCacheNamespaceKey(config);
+ const cachedExtract = await renovateCache.get(cacheNamespace, cacheKey);
+ // istanbul ignore if
+ if (cachedExtract) {
+ const extractHash = getExtractHash(config, extractList);
+ if (cachedExtract.extractHash === extractHash) {
+ logger.info({ baseBranch }, 'Returning cached extract result');
+ return cachedExtract.extractions;
+ }
+ logger.debug({ baseBranch }, 'Cached extract result does not match');
+ } else {
+ logger.debug({ baseBranch }, 'No cached extract result found');
+ }
+ return null;
+}
+
+export async function setCachedExtract(
+ config: RenovateConfig,
+ extractList: RenovateConfig[],
+ extractions: Record<string, PackageFile[]>
+): Promise<void> {
+ const { baseBranch } = config;
+ logger.debug({ baseBranch }, 'Setting cached extract result');
+ const { cacheNamespace, cacheKey } = getCacheNamespaceKey(config);
+ const extractHash = getExtractHash(config, extractList);
+ const payload = { extractHash, extractions };
+ const cacheMinutes = 24 * 60;
+ await renovateCache.set(cacheNamespace, cacheKey, payload, cacheMinutes);
+}
diff --git a/lib/workers/repository/extract/index.spec.ts b/lib/workers/repository/extract/index.spec.ts
index 37087657b6..45a461a9d6 100644
--- a/lib/workers/repository/extract/index.spec.ts
+++ b/lib/workers/repository/extract/index.spec.ts
@@ -1,10 +1,13 @@
import { defaultConfig, mocked, platform } from '../../../../test/util';
import { RenovateConfig } from '../../../config';
+import * as _cache from './cache';
import * as _managerFiles from './manager-files';
import { extractAllDependencies } from '.';
+jest.mock('./cache');
jest.mock('./manager-files');
+const cache = mocked(_cache);
const managerFiles = mocked(_managerFiles);
describe('workers/repository/extract/index', () => {
@@ -21,6 +24,11 @@ describe('workers/repository/extract/index', () => {
const res = await extractAllDependencies(config);
expect(Object.keys(res).includes('ansible')).toBe(true);
});
+ it('uses cache', async () => {
+ cache.getCachedExtract.mockResolvedValueOnce({} as never);
+ const res = await extractAllDependencies(config);
+ expect(res).toEqual({});
+ });
it('skips non-enabled managers', async () => {
config.enabledManagers = ['npm'];
managerFiles.getManagerPackageFiles.mockResolvedValue([{} as never]);
diff --git a/lib/workers/repository/extract/index.ts b/lib/workers/repository/extract/index.ts
index a27d8efec0..5fde50312d 100644
--- a/lib/workers/repository/extract/index.ts
+++ b/lib/workers/repository/extract/index.ts
@@ -7,6 +7,7 @@ import {
import { logger } from '../../../logger';
import { getManagerList } from '../../../manager';
import { PackageFile } from '../../../manager/common';
+import { getCachedExtract, setCachedExtract } from './cache';
import { getMatchingFiles } from './file-match';
import { getManagerPackageFiles } from './manager-files';
@@ -44,6 +45,10 @@ export async function extractAllDependencies(
}
}
}
+ const cachedExtractions = await getCachedExtract(config, extractList);
+ if (cachedExtractions) {
+ return cachedExtractions;
+ }
const extractResults = await Promise.all(
extractList.map(async (managerConfig) => {
const packageFiles = await getManagerPackageFiles(managerConfig);
@@ -60,5 +65,6 @@ export async function extractAllDependencies(
}
}
logger.debug(`Found ${fileCount} package file(s)`);
+ await setCachedExtract(config, extractList, extractions);
return extractions;
}
diff --git a/lib/workers/repository/onboarding/branch/index.spec.ts b/lib/workers/repository/onboarding/branch/index.spec.ts
index 1ffea415f2..a8a4090537 100644
--- a/lib/workers/repository/onboarding/branch/index.spec.ts
+++ b/lib/workers/repository/onboarding/branch/index.spec.ts
@@ -8,6 +8,7 @@ import { checkOnboardingBranch } from '.';
const rebase: any = _rebase;
jest.mock('../../../../workers/repository/onboarding/branch/rebase');
+jest.mock('../../extract/cache');
describe('workers/repository/onboarding/branch', () => {
describe('checkOnboardingBranch', () => {
--
GitLab