From 8ed1eb08d1a4d248d69d3d3e1e4dec168b4c9ee8 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Mon, 25 May 2020 10:23:32 +0200
Subject: [PATCH] feat(cache): redis global cache (#6315)
---
docs/usage/self-hosted-configuration.md | 4 ++
lib/config/common.ts | 1 +
lib/config/definitions.ts | 7 +++
lib/util/cache/global/index.spec.ts | 16 ++++-
lib/util/cache/global/index.ts | 16 ++++-
lib/util/cache/global/redis.ts | 77 +++++++++++++++++++++++
lib/workers/global/index.ts | 3 +-
package.json | 1 +
yarn.lock | 83 +++++++++++++++----------
9 files changed, 168 insertions(+), 40 deletions(-)
create mode 100644 lib/util/cache/global/redis.ts
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index 0afac48edc..b45d28f04a 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -136,6 +136,10 @@ To create the key pair with openssl use the following commands:
Override this object if you wish to change the URLs that Renovate links to, e.g. if you have an internal forum for asking for help.
+## redisUrl
+
+If this value is set then Renovate will use Redis for its global cache instead of the local file system. The global cache is used to store lookup results (e.g. dependency versions and release notes) between repositories and runs. Example url: `redis://localhost`.
+
## repositories
## requireConfig
diff --git a/lib/config/common.ts b/lib/config/common.ts
index b057689f3b..4be8e8a8b3 100644
--- a/lib/config/common.ts
+++ b/lib/config/common.ts
@@ -91,6 +91,7 @@ export interface RenovateAdminConfig {
repositories?: RenovateRepository[];
requireConfig?: boolean;
trustLevel?: 'low' | 'high';
+ redisUrl?: string;
}
export type PostUpgradeTasks = {
diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts
index 4554c8f7fa..eb3eb3791c 100644
--- a/lib/config/definitions.ts
+++ b/lib/config/definitions.ts
@@ -241,6 +241,13 @@ const options: RenovateOptions[] = [
allowedValues: ['auto', 'global', 'docker'],
default: 'auto',
},
+ {
+ name: 'redisUrl',
+ description:
+ 'If defined, this redis url will be used for caching instead of the file system',
+ admin: true,
+ type: 'string',
+ },
{
name: 'baseDir',
description:
diff --git a/lib/util/cache/global/index.spec.ts b/lib/util/cache/global/index.spec.ts
index 897b7a2f1d..d88d80496a 100644
--- a/lib/util/cache/global/index.spec.ts
+++ b/lib/util/cache/global/index.spec.ts
@@ -1,19 +1,29 @@
import { getName } from '../../../../test/util';
-import { get, init, set } from '.';
+import { cleanup, get, init, set } from '.';
jest.mock('./file');
+jest.mock('./redis');
describe(getName(__filename), () => {
it('returns undefined if not initialized', async () => {
expect(await get('test', 'missing-key')).toBeUndefined();
expect(await set('test', 'some-key', 'some-value', 5)).toBeUndefined();
});
- it('sets and gets', async () => {
+ it('sets and gets file', async () => {
global.renovateCache = { get: jest.fn(), set: jest.fn(), rm: jest.fn() };
- init('some-dir');
+ init({ cacheDir: 'some-dir' });
expect(
await set('some-namespace', 'some-key', 'some-value', 1)
).toBeUndefined();
expect(await get('some-namespace', 'unknown-key')).toBeUndefined();
});
+ it('sets and gets redis', async () => {
+ global.renovateCache = { get: jest.fn(), set: jest.fn(), rm: jest.fn() };
+ init({ redisUrl: 'some-url' });
+ expect(
+ await set('some-namespace', 'some-key', 'some-value', 1)
+ ).toBeUndefined();
+ expect(await get('some-namespace', 'unknown-key')).toBeUndefined();
+ expect(cleanup({ redisUrl: 'some-url' })).toBeUndefined();
+ });
});
diff --git a/lib/util/cache/global/index.ts b/lib/util/cache/global/index.ts
index f80727f254..055f01cae9 100644
--- a/lib/util/cache/global/index.ts
+++ b/lib/util/cache/global/index.ts
@@ -1,5 +1,7 @@
+import { RenovateConfig } from '../../../config/common';
import * as runCache from '../run';
import * as fileCache from './file';
+import * as redisCache from './redis';
function getGlobalKey(namespace: string, key: string): string {
return `global%%${namespace}%%${key}`;
@@ -30,6 +32,16 @@ export function set(
return global.renovateCache.set(namespace, key, value, minutes);
}
-export function init(cacheDir: string): void {
- return fileCache.init(cacheDir);
+export function init(config: RenovateConfig): void {
+ if (config.redisUrl) {
+ redisCache.init(config.redisUrl);
+ } else {
+ fileCache.init(config.cacheDir);
+ }
+}
+
+export function cleanup(config: RenovateConfig): void {
+ if (config.redisUrl) {
+ redisCache.end();
+ }
}
diff --git a/lib/util/cache/global/redis.ts b/lib/util/cache/global/redis.ts
new file mode 100644
index 0000000000..abb7497b8c
--- /dev/null
+++ b/lib/util/cache/global/redis.ts
@@ -0,0 +1,77 @@
+/* istanbul ignore file */
+import { IHandyRedis, createHandyClient } from 'handy-redis';
+import { DateTime } from 'luxon';
+import { logger } from '../../../logger';
+
+let client: IHandyRedis | undefined;
+
+function getKey(namespace: string, key: string): string {
+ return `${namespace}-${key}`;
+}
+
+export function end(): void {
+ try {
+ client?.redis?.end(true); // TODO: Why is this not supported by client directly?
+ } catch (err) {
+ logger.warn({ err }, 'Redis cache end failed');
+ }
+}
+
+async function rm(namespace: string, key: string): Promise<void> {
+ logger.trace({ namespace, key }, 'Removing cache entry');
+ await client?.del(getKey(namespace, key));
+}
+
+async function get<T = never>(namespace: string, key: string): Promise<T> {
+ logger.trace(`cache.get(${namespace}, ${key})`);
+ try {
+ const res = await client?.get(getKey(namespace, key));
+ const cachedValue = JSON.parse(res);
+ if (cachedValue) {
+ if (DateTime.local() < DateTime.fromISO(cachedValue.expiry)) {
+ logger.trace({ namespace, key }, 'Returning cached value');
+ return cachedValue.value;
+ }
+ // istanbul ignore next
+ await rm(namespace, key);
+ }
+ } catch (err) {
+ logger.trace({ namespace, key }, 'Cache miss');
+ }
+ return null;
+}
+
+async function set(
+ namespace: string,
+ key: string,
+ value: unknown,
+ ttlMinutes = 5
+): Promise<void> {
+ logger.trace({ namespace, key, ttlMinutes }, 'Saving cached value');
+ await client?.set(
+ getKey(namespace, key),
+ JSON.stringify({
+ value,
+ expiry: DateTime.local().plus({ minutes: ttlMinutes }),
+ }),
+ ['EX', ttlMinutes * 60]
+ );
+}
+
+export function init(url: string): void {
+ if (!url) {
+ return;
+ }
+ logger.debug('Redis cache init');
+ client = createHandyClient({
+ url,
+ retry_strategy: (options) => {
+ if (options.error) {
+ logger.error({ err: options.error }, 'Redis cache error');
+ }
+ // Reconnect after this time
+ return Math.min(options.attempt * 100, 3000);
+ },
+ });
+ global.renovateCache = { get, set, rm };
+}
diff --git a/lib/workers/global/index.ts b/lib/workers/global/index.ts
index 9d0b0757af..c2675d672d 100644
--- a/lib/workers/global/index.ts
+++ b/lib/workers/global/index.ts
@@ -33,7 +33,6 @@ async function setDirectories(input: RenovateConfig): Promise<RenovateConfig> {
logger.debug('Using cacheDir: ' + config.cacheDir);
}
await fs.ensureDir(config.cacheDir);
- globalCache.init(config.cacheDir);
return config;
}
@@ -63,6 +62,7 @@ export async function start(): Promise<0 | 1> {
let config = await getGlobalConfig();
config = await initPlatform(config);
config = await setDirectories(config);
+ globalCache.init(config);
config = await autodiscoverRepositories(config);
limits.init(config);
@@ -85,6 +85,7 @@ export async function start(): Promise<0 | 1> {
await repositoryWorker.renovateRepository(repoConfig);
}
setMeta({});
+ globalCache.cleanup(config);
logger.debug(`Renovate exiting successfully`);
} catch (err) /* istanbul ignore next */ {
if (err.message.startsWith('Init: ')) {
diff --git a/package.json b/package.json
index 208746aa5a..28b858937e 100644
--- a/package.json
+++ b/package.json
@@ -131,6 +131,7 @@
"global-agent": "2.1.8",
"got": "9.6.0",
"handlebars": "4.7.6",
+ "handy-redis": "1.8.1",
"hasha": "5.2.0",
"ini": "1.3.5",
"js-yaml": "3.13.1",
diff --git a/yarn.lock b/yarn.lock
index 878550f77c..1c15dcff6b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1670,6 +1670,13 @@
resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.0.1.tgz#b6e98083f13faa1e5231bfa3bdb1b0feff536b6d"
integrity sha512-boy4xPNEtiw6N3abRhBi/e7hNvy3Tt8E9ZRAQrwAGzoCGZS/1wjo9KY7JHhnfnEsG5wSjDbymCozUM9a3ea7OQ==
+"@types/redis@^2.8.14":
+ version "2.8.21"
+ resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.21.tgz#4bd4a56747ee57156e50a4389ece0f79f1dfeac6"
+ integrity sha512-EcqWrhXnzlo2z7AwZG3jEuwGcs/QZoab1lBbOHRfV/ezzVrczpkFrCoQorWp3dvp7pfOJtAryxBFou0zQFYpDQ==
+ dependencies:
+ "@types/node" "*"
+
"@types/registry-auth-token@3.3.0":
version "3.3.0"
resolved "https://registry.yarnpkg.com/@types/registry-auth-token/-/registry-auth-token-3.3.0.tgz#bfb57ed386d84749c982ec20c804ac119382b285"
@@ -3208,7 +3215,7 @@ debug@^3.1.0:
dependencies:
ms "^2.1.1"
-debuglog@*, debuglog@^1.0.1:
+debuglog@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
integrity sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=
@@ -3319,6 +3326,11 @@ delegates@^1.0.0:
resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=
+denque@^1.4.1:
+ version "1.4.1"
+ resolved "https://registry.yarnpkg.com/denque/-/denque-1.4.1.tgz#6744ff7641c148c3f8a69c307e51235c1f4a37cf"
+ integrity sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==
+
deprecation@^2.0.0, deprecation@^2.3.1:
version "2.3.1"
resolved "https://registry.yarnpkg.com/deprecation/-/deprecation-2.3.1.tgz#6368cbdb40abf3373b525ac87e4a260c3a700919"
@@ -4518,6 +4530,14 @@ handlebars@4.7.6, handlebars@^4.7.6:
optionalDependencies:
uglify-js "^3.1.4"
+handy-redis@1.8.1:
+ version "1.8.1"
+ resolved "https://registry.yarnpkg.com/handy-redis/-/handy-redis-1.8.1.tgz#7e5f0fc63bbe8e7ed7a4641e41d314b81a4cd6a0"
+ integrity sha512-/yA/8l351iLNC/hrbgiz5qrFZNSZYrIVu/lZK6sYPHV8QiKFUb9Xcb1pZjeO7Fe5Z/4Pn6l6KbaQrdkovWqSjQ==
+ dependencies:
+ "@types/redis" "^2.8.14"
+ redis "^3.0.2"
+
har-schema@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
@@ -4790,7 +4810,7 @@ import-local@^3.0.2:
pkg-dir "^4.2.0"
resolve-cwd "^3.0.0"
-imurmurhash@*, imurmurhash@^0.1.4:
+imurmurhash@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
integrity sha1-khi5srkoojixPcT7a21XbyMUU+o=
@@ -6239,11 +6259,6 @@ lockfile@^1.0.4:
dependencies:
signal-exit "^3.0.2"
-lodash._baseindexof@*:
- version "3.1.0"
- resolved "https://registry.yarnpkg.com/lodash._baseindexof/-/lodash._baseindexof-3.1.0.tgz#fe52b53a1c6761e42618d654e4a25789ed61822c"
- integrity sha1-/lK1OhxnYeQmGNZU5KJXie1hgiw=
-
lodash._baseuniq@~4.6.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz#0ebb44e456814af7905c6212fa2c9b2d51b841e8"
@@ -6252,33 +6267,11 @@ lodash._baseuniq@~4.6.0:
lodash._createset "~4.0.0"
lodash._root "~3.0.0"
-lodash._bindcallback@*:
- version "3.0.1"
- resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e"
- integrity sha1-5THCdkTPi1epnhftlbNcdIeJOS4=
-
-lodash._cacheindexof@*:
- version "3.0.2"
- resolved "https://registry.yarnpkg.com/lodash._cacheindexof/-/lodash._cacheindexof-3.0.2.tgz#3dc69ac82498d2ee5e3ce56091bafd2adc7bde92"
- integrity sha1-PcaayCSY0u5ePOVgkbr9Ktx73pI=
-
-lodash._createcache@*:
- version "3.1.2"
- resolved "https://registry.yarnpkg.com/lodash._createcache/-/lodash._createcache-3.1.2.tgz#56d6a064017625e79ebca6b8018e17440bdcf093"
- integrity sha1-VtagZAF2JeeevKa4AY4XRAvc8JM=
- dependencies:
- lodash._getnative "^3.0.0"
-
lodash._createset@~4.0.0:
version "4.0.3"
resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26"
integrity sha1-D0ZZ+7CddRlPqeK4imZE02PJ/iY=
-lodash._getnative@*, lodash._getnative@^3.0.0:
- version "3.9.1"
- resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5"
- integrity sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U=
-
lodash._reinterpolate@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d"
@@ -6324,11 +6317,6 @@ lodash.isstring@^4.0.1:
resolved "https://registry.yarnpkg.com/lodash.isstring/-/lodash.isstring-4.0.1.tgz#d527dfb5456eca7cc9bb95d5daeaf88ba54a5451"
integrity sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=
-lodash.restparam@*:
- version "3.6.1"
- resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805"
- integrity sha1-k2pOMJ7zMKdkXtQUWYbIWuWyCAU=
-
lodash.sortby@^4.7.0:
version "4.7.0"
resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438"
@@ -8401,6 +8389,33 @@ redeyed@~2.1.0:
dependencies:
esprima "~4.0.0"
+redis-commands@^1.5.0:
+ version "1.5.0"
+ resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.5.0.tgz#80d2e20698fe688f227127ff9e5164a7dd17e785"
+ integrity sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==
+
+redis-errors@^1.0.0, redis-errors@^1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad"
+ integrity sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=
+
+redis-parser@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
+ integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
+ dependencies:
+ redis-errors "^1.0.0"
+
+redis@^3.0.2:
+ version "3.0.2"
+ resolved "https://registry.yarnpkg.com/redis/-/redis-3.0.2.tgz#bd47067b8a4a3e6a2e556e57f71cc82c7360150a"
+ integrity sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==
+ dependencies:
+ denque "^1.4.1"
+ redis-commands "^1.5.0"
+ redis-errors "^1.2.0"
+ redis-parser "^3.0.0"
+
regenerate-unicode-properties@^8.2.0:
version "8.2.0"
resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-8.2.0.tgz#e5de7111d655e7ba60c057dbe9ff37c87e65cdec"
--
GitLab