From 92009c658039e2c8ac9b47bd8d97478252b3e2bd Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Mon, 11 May 2020 13:29:39 +0200
Subject: [PATCH] feat(internal): use runtime cache for buffering global cache
requests (#6202)
---
lib/datasource/github-tags/index.spec.ts | 2 +
.../helm/__snapshots__/index.spec.ts.snap | 118 ------------------
lib/datasource/helm/index.spec.ts | 24 +---
lib/datasource/helm/index.ts | 1 +
lib/datasource/nuget/index.spec.ts | 6 +-
lib/datasource/orb/index.spec.ts | 2 +
lib/datasource/ruby-version/index.spec.ts | 2 +
lib/util/cache/global.ts | 14 ++-
lib/workers/pr/changelog/index.spec.ts | 2 +
9 files changed, 30 insertions(+), 141 deletions(-)
diff --git a/lib/datasource/github-tags/index.spec.ts b/lib/datasource/github-tags/index.spec.ts
index 430e3d4a8b..8fb288b979 100644
--- a/lib/datasource/github-tags/index.spec.ts
+++ b/lib/datasource/github-tags/index.spec.ts
@@ -1,5 +1,6 @@
import { api } from '../../platform/github/gh-got-wrapper';
import * as globalCache from '../../util/cache/global';
+import * as runCache from '../../util/cache/run';
import * as _hostRules from '../../util/host-rules';
import * as github from '.';
@@ -16,6 +17,7 @@ describe('datasource/github-tags', () => {
beforeEach(() => {
jest.resetAllMocks();
hostRules.hosts = jest.fn(() => []);
+ runCache.clear();
return globalCache.rmAll();
});
it('returns null if no token', async () => {
diff --git a/lib/datasource/helm/__snapshots__/index.spec.ts.snap b/lib/datasource/helm/__snapshots__/index.spec.ts.snap
index e29bfcf08e..11e2ce569c 100644
--- a/lib/datasource/helm/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/helm/__snapshots__/index.spec.ts.snap
@@ -1,123 +1,5 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
-exports[`datasource/helm getReleases returns list of versions for normal response if index.yaml is cached 1`] = `
-Object {
- "homepage": "https://www.getambassador.io/",
- "name": "ambassador",
- "releases": Array [
- Object {
- "releaseTimestamp": "2019-06-02T08:56:36.119Z",
- "version": "2.7.0",
- },
- Object {
- "releaseTimestamp": "2019-05-31T18:57:01.540Z",
- "version": "2.6.2",
- },
- Object {
- "releaseTimestamp": "2019-05-27T23:28:25.443Z",
- "version": "2.6.1",
- },
- Object {
- "releaseTimestamp": "2019-05-24T10:56:37.781Z",
- "version": "2.6.0",
- },
- Object {
- "releaseTimestamp": "2019-05-20T23:56:32.309Z",
- "version": "2.5.1",
- },
- Object {
- "releaseTimestamp": "2019-05-16T21:56:48.001Z",
- "version": "2.5.0",
- },
- Object {
- "releaseTimestamp": "2019-05-15T07:56:25.942Z",
- "version": "2.4.1",
- },
- Object {
- "releaseTimestamp": "2019-05-09T17:29:35.612Z",
- "version": "2.4.0",
- },
- Object {
- "releaseTimestamp": "2019-05-08T19:26:25.925Z",
- "version": "2.3.1",
- },
- Object {
- "releaseTimestamp": "2019-05-06T21:28:21.342Z",
- "version": "2.3.0",
- },
- Object {
- "releaseTimestamp": "2019-05-06T20:56:39.732Z",
- "version": "2.2.5",
- },
- Object {
- "releaseTimestamp": "2019-05-01T19:56:23.829Z",
- "version": "2.2.4",
- },
- Object {
- "releaseTimestamp": "2019-05-01T14:27:55.475Z",
- "version": "2.2.3",
- },
- Object {
- "releaseTimestamp": "2019-04-29T20:56:20.987Z",
- "version": "2.2.2",
- },
- Object {
- "releaseTimestamp": "2019-04-27T01:56:41.147Z",
- "version": "2.2.1",
- },
- Object {
- "releaseTimestamp": "2019-04-23T19:56:09.380Z",
- "version": "2.2.0",
- },
- Object {
- "releaseTimestamp": "2019-04-23T14:57:59.089Z",
- "version": "2.1.0",
- },
- Object {
- "releaseTimestamp": "2019-04-08T13:26:12.863Z",
- "version": "2.0.2",
- },
- Object {
- "releaseTimestamp": "2019-04-06T13:26:04.916Z",
- "version": "2.0.1",
- },
- Object {
- "releaseTimestamp": "2019-03-22T14:26:18.094Z",
- "version": "2.0.0",
- },
- Object {
- "releaseTimestamp": "2019-03-21T15:25:53.910Z",
- "version": "1.1.5",
- },
- Object {
- "releaseTimestamp": "2019-02-24T04:55:38.454Z",
- "version": "1.1.4",
- },
- Object {
- "releaseTimestamp": "2019-02-22T16:28:27.398Z",
- "version": "1.1.3",
- },
- Object {
- "releaseTimestamp": "2019-02-15T23:56:28.304Z",
- "version": "1.1.2",
- },
- Object {
- "releaseTimestamp": "2019-02-14T16:55:51.519Z",
- "version": "1.1.1",
- },
- Object {
- "releaseTimestamp": "2019-02-14T15:25:43.743Z",
- "version": "1.1.0",
- },
- Object {
- "releaseTimestamp": "2019-02-13T00:56:01.476Z",
- "version": "1.0.0",
- },
- ],
- "sourceUrl": "https://github.com/datawire/ambassador",
-}
-`;
-
exports[`datasource/helm getReleases returns list of versions for normal response if index.yaml is not cached 1`] = `
Object {
"homepage": "https://www.getambassador.io/",
diff --git a/lib/datasource/helm/index.spec.ts b/lib/datasource/helm/index.spec.ts
index 520e12433c..466e870d26 100644
--- a/lib/datasource/helm/index.spec.ts
+++ b/lib/datasource/helm/index.spec.ts
@@ -1,7 +1,8 @@
import fs from 'fs';
import * as globalCache from '../../util/cache/global';
+import * as runCache from '../../util/cache/run';
import _got from '../../util/got';
-import { getReleases, getRepositoryData } from '.';
+import { getReleases } from '.';
const got: any = _got;
@@ -17,6 +18,7 @@ describe('datasource/helm', () => {
describe('getReleases', () => {
beforeEach(() => {
jest.resetAllMocks();
+ runCache.clear();
return globalCache.rmAll();
});
it('returns null if lookupName was not provided', async () => {
@@ -137,25 +139,5 @@ describe('datasource/helm', () => {
expect(releases).not.toBeNull();
expect(releases).toMatchSnapshot();
});
- it('returns list of versions for normal response if index.yaml is cached', async () => {
- const repository = 'example-repository.com';
- const cacheNamespace = 'datasource-helm';
- const cacheKey = repository;
- const cacheMinutes = 10;
- got.mockReturnValueOnce({ body: indexYaml });
- const repositoryData = await getRepositoryData(repository);
- await globalCache.set(
- cacheNamespace,
- cacheKey,
- repositoryData,
- cacheMinutes
- );
- const releases = await getReleases({
- lookupName: 'ambassador',
- registryUrls: [repository],
- });
- expect(releases).not.toBeNull();
- expect(releases).toMatchSnapshot();
- });
});
});
diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts
index e1eed39e5e..1c8df70e3e 100644
--- a/lib/datasource/helm/index.ts
+++ b/lib/datasource/helm/index.ts
@@ -15,6 +15,7 @@ export async function getRepositoryData(
const cacheNamespace = 'datasource-helm';
const cacheKey = repository;
const cachedIndex = await globalCache.get(cacheNamespace, cacheKey);
+ // istanbul ignore if
if (cachedIndex) {
return cachedIndex;
}
diff --git a/lib/datasource/nuget/index.spec.ts b/lib/datasource/nuget/index.spec.ts
index f980d8ee6e..811f49e364 100644
--- a/lib/datasource/nuget/index.spec.ts
+++ b/lib/datasource/nuget/index.spec.ts
@@ -1,5 +1,6 @@
import fs from 'fs';
import * as globalCache from '../../util/cache/global';
+import * as runCache from '../../util/cache/run';
import _got from '../../util/got';
import * as _hostRules from '../../util/host-rules';
import * as nuget from '.';
@@ -83,7 +84,10 @@ const configV3NotNugetOrg = {
};
describe('datasource/nuget', () => {
- beforeEach(() => globalCache.rmAll());
+ beforeEach(() => {
+ runCache.clear();
+ return globalCache.rmAll();
+ });
describe('getReleases', () => {
beforeEach(() => {
jest.resetAllMocks();
diff --git a/lib/datasource/orb/index.spec.ts b/lib/datasource/orb/index.spec.ts
index 52606fce0a..8a2c1a442c 100644
--- a/lib/datasource/orb/index.spec.ts
+++ b/lib/datasource/orb/index.spec.ts
@@ -1,4 +1,5 @@
import * as globalCache from '../../util/cache/global';
+import * as runCache from '../../util/cache/run';
import _got from '../../util/got';
import * as datasource from '.';
@@ -31,6 +32,7 @@ describe('datasource/orb', () => {
describe('getReleases', () => {
beforeEach(() => {
jest.clearAllMocks();
+ runCache.clear();
return globalCache.rmAll();
});
it('returns null for empty result', async () => {
diff --git a/lib/datasource/ruby-version/index.spec.ts b/lib/datasource/ruby-version/index.spec.ts
index f99b8eaef9..abf17da1ec 100644
--- a/lib/datasource/ruby-version/index.spec.ts
+++ b/lib/datasource/ruby-version/index.spec.ts
@@ -1,5 +1,6 @@
import fs from 'fs';
import * as globalCache from '../../util/cache/global';
+import * as runCache from '../../util/cache/run';
import _got from '../../util/got';
import { getReleases } from '.';
@@ -15,6 +16,7 @@ const rubyReleasesHtml = fs.readFileSync(
describe('datasource/gradle', () => {
describe('getReleases', () => {
beforeEach(() => {
+ runCache.clear();
return globalCache.rmAll();
});
it('parses real data', async () => {
diff --git a/lib/util/cache/global.ts b/lib/util/cache/global.ts
index 5f3a8e6db1..326d27925d 100644
--- a/lib/util/cache/global.ts
+++ b/lib/util/cache/global.ts
@@ -1,5 +1,15 @@
+import * as runCache from './run';
+
+function getGlobalKey(namespace: string, key: string): string {
+ return `global%%${namespace}%%${key}`;
+}
+
export function get<T = any>(namespace: string, key: string): Promise<T> {
- return renovateCache.get(namespace, key);
+ const globalKey = getGlobalKey(namespace, key);
+ if (!runCache.get(globalKey)) {
+ runCache.set(globalKey, renovateCache.get(namespace, key));
+ }
+ return runCache.get(globalKey);
}
export function set(
@@ -8,6 +18,8 @@ export function set(
value: any,
minutes: number
): Promise<void> {
+ const globalKey = getGlobalKey(namespace, key);
+ runCache.set(globalKey, value);
return renovateCache.set(namespace, key, value, minutes);
}
diff --git a/lib/workers/pr/changelog/index.spec.ts b/lib/workers/pr/changelog/index.spec.ts
index 788e4e99cb..7ad98cb885 100644
--- a/lib/workers/pr/changelog/index.spec.ts
+++ b/lib/workers/pr/changelog/index.spec.ts
@@ -2,6 +2,7 @@ import { mocked, partial } from '../../../../test/util';
import { PLATFORM_TYPE_GITHUB } from '../../../constants/platforms';
import { api } from '../../../platform/github/gh-got-wrapper';
import * as globalCache from '../../../util/cache/global';
+import * as runCache from '../../../util/cache/run';
import * as hostRules from '../../../util/host-rules';
import * as semverVersioning from '../../../versioning/semver';
import { BranchConfig } from '../../common';
@@ -44,6 +45,7 @@ describe('workers/pr/changelog', () => {
token: 'abc',
});
await globalCache.rmAll();
+ runCache.clear();
});
it('returns null if @types', async () => {
expect(
--
GitLab