feat(internal): cache extractions results (#6118)

lib/workers/repository/extract/__snapshots__/cache.spec.ts.snap

+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`workers/repository/extract/cache returns a hash 1`] = `"3Jfx5tzsGJEm8HQCeHT/6QL5vzE="`;

lib/workers/repository/extract/cache.spec.ts

+import * as cache from './cache';
+
+describe('workers/repository/extract/cache', () => {
+  const config = { baseBranch: 'master', baseBranchSha: 'abc123' };
+  const extractList = [];
+  const extraction = { foo: [] };
+  it('handles missing sha', () => {
+    expect(cache.getExtractHash({}, [])).toBeNull();
+  });
+  it('returns a hash', () => {
+    expect(
+      cache.getExtractHash({ baseBranchSha: 'abc123' }, [])
+    ).toMatchSnapshot();
+  });
+  it('sets a value', async () => {
+    await cache.setCachedExtract(config, extractList, extraction);
+  });
+  it('gets no value', async () => {
+    extractList.push('abc');
+    const res = await cache.getCachedExtract(config, extractList);
+    expect(res).toEqual(null);
+  });
+  it('handles no sha error', async () => {
+    const res = await cache.getCachedExtract(
+      { baseBranch: 'nothing' },
+      extractList
+    );
+    expect(res).toBeNull();
+  });
+});

lib/workers/repository/extract/cache.ts

+import crypto from 'crypto';
+import { RenovateConfig } from '../../../config/common';
+import { logger } from '../../../logger';
+import { PackageFile } from '../../../manager/common';
+
+function getCacheNamespaceKey(
+  config: RenovateConfig
+): { cacheNamespace: string; cacheKey: string } {
+  // Cache extract results per-base branch
+  const { platform, repository, baseBranch } = config;
+  const cacheNamespace = 'repository-extract';
+  const cacheKey = `${platform}/${repository}/${baseBranch}`;
+  return { cacheNamespace, cacheKey };
+}
+
+export function getExtractHash(
+  config: RenovateConfig,
+  extractList: RenovateConfig[]
+): string | null {
+  // A cache is only valid if the following are unchanged:
+  //  * base branch SHA
+  //  * the list of matching files for each manager
+  if (!config.baseBranchSha) {
+    logger.warn('No baseBranchSha found in config');
+    return null;
+  }
+  return crypto
+    .createHash('sha1')
+    .update(config.baseBranchSha)
+    .update(JSON.stringify(extractList))
+    .digest('base64');
+}
+
+export async function getCachedExtract(
+  config: RenovateConfig,
+  extractList: RenovateConfig[]
+): Promise<Record<string, PackageFile[]> | null> {
+  const { baseBranch } = config;
+  const { cacheNamespace, cacheKey } = getCacheNamespaceKey(config);
+  const cachedExtract = await renovateCache.get(cacheNamespace, cacheKey);
+  // istanbul ignore if
+  if (cachedExtract) {
+    const extractHash = getExtractHash(config, extractList);
+    if (cachedExtract.extractHash === extractHash) {
+      logger.info({ baseBranch }, 'Returning cached extract result');
+      return cachedExtract.extractions;
+    }
+    logger.debug({ baseBranch }, 'Cached extract result does not match');
+  } else {
+    logger.debug({ baseBranch }, 'No cached extract result found');
+  }
+  return null;
+}
+
+export async function setCachedExtract(
+  config: RenovateConfig,
+  extractList: RenovateConfig[],
+  extractions: Record<string, PackageFile[]>
+): Promise<void> {
+  const { baseBranch } = config;
+  logger.debug({ baseBranch }, 'Setting cached extract result');
+  const { cacheNamespace, cacheKey } = getCacheNamespaceKey(config);
+  const extractHash = getExtractHash(config, extractList);
+  const payload = { extractHash, extractions };
+  const cacheMinutes = 24 * 60;
+  await renovateCache.set(cacheNamespace, cacheKey, payload, cacheMinutes);
+}

lib/workers/repository/extract/index.spec.ts

 import { defaultConfig, mocked, platform } from '../../../../test/util';
 import { RenovateConfig } from '../../../config';
+import * as _cache from './cache';
 import * as _managerFiles from './manager-files';
 import { extractAllDependencies } from '.';
 
+jest.mock('./cache');
 jest.mock('./manager-files');
 
+const cache = mocked(_cache);
 const managerFiles = mocked(_managerFiles);
 
 describe('workers/repository/extract/index', () => {
@@ -21,6 +24,11 @@ describe('workers/repository/extract/index', () => {
       const res = await extractAllDependencies(config);
       expect(Object.keys(res).includes('ansible')).toBe(true);
     });
+    it('uses cache', async () => {
+      cache.getCachedExtract.mockResolvedValueOnce({} as never);
+      const res = await extractAllDependencies(config);
+      expect(res).toEqual({});
+    });
     it('skips non-enabled managers', async () => {
       config.enabledManagers = ['npm'];
       managerFiles.getManagerPackageFiles.mockResolvedValue([{} as never]);

lib/workers/repository/extract/index.ts

@@ -7,6 +7,7 @@ import {
 import { logger } from '../../../logger';
 import { getManagerList } from '../../../manager';
 import { PackageFile } from '../../../manager/common';
+import { getCachedExtract, setCachedExtract } from './cache';
 import { getMatchingFiles } from './file-match';
 import { getManagerPackageFiles } from './manager-files';
 
@@ -44,6 +45,10 @@ export async function extractAllDependencies(
       }
     }
   }
+  const cachedExtractions = await getCachedExtract(config, extractList);
+  if (cachedExtractions) {
+    return cachedExtractions;
+  }
   const extractResults = await Promise.all(
     extractList.map(async (managerConfig) => {
       const packageFiles = await getManagerPackageFiles(managerConfig);
@@ -60,5 +65,6 @@ export async function extractAllDependencies(
     }
   }
   logger.debug(`Found ${fileCount} package file(s)`);
+  await setCachedExtract(config, extractList, extractions);
   return extractions;
 }

lib/workers/repository/onboarding/branch/index.spec.ts

@@ -8,6 +8,7 @@ import { checkOnboardingBranch } from '.';
 const rebase: any = _rebase;
 
 jest.mock('../../../../workers/repository/onboarding/branch/rebase');
+jest.mock('../../extract/cache');
 
 describe('workers/repository/onboarding/branch', () => {
   describe('checkOnboardingBranch', () => {