From 039fce8dc6082e2268049939c13dc67585bb1708 Mon Sep 17 00:00:00 2001
From: Gabriel-Ladzaretti
<97394622+Gabriel-Ladzaretti@users.noreply.github.com>
Date: Sun, 21 Aug 2022 15:53:20 +0300
Subject: [PATCH] refactor(repository/cache): add support for adding various
cache clients (#17146)
Co-authored-by: Nabeel Saabna <48175656+nabeelsaabna@users.noreply.github.com>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
docs/usage/self-hosted-configuration.md | 2 +
lib/config/options/index.ts | 10 +++
lib/config/types.ts | 2 +
lib/util/cache/repository/impl/base.ts | 72 ++++++++++++++--
.../cache/repository/impl/cache-factory.ts | 22 +++++
lib/util/cache/repository/impl/local.spec.ts | 34 +++++---
lib/util/cache/repository/impl/local.ts | 86 ++++---------------
lib/util/cache/repository/impl/null.ts | 19 ++++
lib/util/cache/repository/index.ts | 8 +-
lib/util/cache/repository/init.ts | 22 ++---
10 files changed, 175 insertions(+), 102 deletions(-)
create mode 100644 lib/util/cache/repository/impl/cache-factory.ts
create mode 100644 lib/util/cache/repository/impl/null.ts
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index 423019b9f2..650c681856 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -626,6 +626,8 @@ Set this to `"enabled"` to have Renovate maintain a JSON file cache per-reposito
Set to `"reset"` if you ever need to bypass the cache and have it overwritten.
JSON files will be stored inside the `cacheDir` beside the existing file-based package cache.
+## repositoryCacheType
+
## requireConfig
By default, Renovate needs a Renovate config file in each repository where it runs before it will propose any dependency updates.
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index c791195515..a8736faccc 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -229,6 +229,16 @@ const options: RenovateOptions[] = [
default: 'disabled',
experimental: true,
},
+ {
+ name: 'repositoryCacheType',
+ description:
+ 'Set the type of renovate repository cache if repositoryCache is not disabled.',
+ globalOnly: true,
+ type: 'string',
+ stage: 'repository',
+ default: 'local',
+ experimental: true,
+ },
{
name: 'force',
description:
diff --git a/lib/config/types.ts b/lib/config/types.ts
index 4e11fa6e81..65ccc58323 100644
--- a/lib/config/types.ts
+++ b/lib/config/types.ts
@@ -11,6 +11,7 @@ export type RenovateConfigStage =
| 'pr';
export type RepositoryCacheConfig = 'disabled' | 'enabled' | 'reset';
+export type RepositoryCacheType = 'local' | string;
export type DryRunConfig = 'extract' | 'lookup' | 'full';
export type RequiredConfig = 'required' | 'optional' | 'ignored';
@@ -65,6 +66,7 @@ export interface RenovateSharedConfig {
recreateClosed?: boolean;
repository?: string;
repositoryCache?: RepositoryCacheConfig;
+ repositoryCacheType?: RepositoryCacheType;
schedule?: string[];
automergeSchedule?: string[];
semanticCommits?: 'auto' | 'enabled' | 'disabled';
diff --git a/lib/util/cache/repository/impl/base.ts b/lib/util/cache/repository/impl/base.ts
index 449a0801b2..f2950a3286 100644
--- a/lib/util/cache/repository/impl/base.ts
+++ b/lib/util/cache/repository/impl/base.ts
@@ -1,16 +1,74 @@
-import type { RepoCache, RepoCacheData } from '../types';
+import { promisify } from 'util';
+import zlib from 'zlib';
+import hasha from 'hasha';
+import { GlobalConfig } from '../../../../config/global';
+import { logger } from '../../../../logger';
+import {
+ CACHE_REVISION,
+ isValidRev10,
+ isValidRev11,
+ isValidRev12,
+} from '../common';
+import type { RepoCache, RepoCacheData, RepoCacheRecord } from '../types';
-export class RepoCacheBase implements RepoCache {
- protected data: RepoCacheData = {};
+const compress = promisify(zlib.brotliCompress);
+const decompress = promisify(zlib.brotliDecompress);
+
+export abstract class RepoCacheBase implements RepoCache {
+ protected platform = GlobalConfig.get('platform')!;
+ private oldHash: string | null = null;
+ private data: RepoCacheData = {};
+
+ protected constructor(protected readonly repository: string) {}
+
+ protected abstract read(): Promise<string | undefined>;
+
+ protected abstract write(data: RepoCacheRecord): Promise<void>;
- // istanbul ignore next
async load(): Promise<void> {
- await Promise.resolve();
+ try {
+ const oldCache = await this.read();
+
+ if (isValidRev12(oldCache, this.repository)) {
+ const compressed = Buffer.from(oldCache.payload, 'base64');
+ const uncompressed = await decompress(compressed);
+ const jsonStr = uncompressed.toString('utf8');
+ this.data = JSON.parse(jsonStr);
+ this.oldHash = oldCache.hash;
+ logger.debug('Repository cache is valid');
+ return;
+ }
+
+ if (isValidRev11(oldCache, this.repository)) {
+ this.data = oldCache.data;
+ logger.debug('Repository cache is migrated from 11 revision');
+ return;
+ }
+
+ if (isValidRev10(oldCache, this.repository)) {
+ delete oldCache.repository;
+ delete oldCache.revision;
+ this.data = oldCache;
+ logger.debug('Repository cache is migrated from 10 revision');
+ return;
+ }
+
+ logger.debug('Repository cache is invalid');
+ } catch (err) {
+ logger.debug('Error reading repository cache');
+ }
}
- // istanbul ignore next
async save(): Promise<void> {
- await Promise.resolve();
+ const revision = CACHE_REVISION;
+ const repository = this.repository;
+ const jsonStr = JSON.stringify(this.data);
+ const hash = await hasha.async(jsonStr, { algorithm: 'sha256' });
+ if (hash !== this.oldHash) {
+ const compressed = await compress(jsonStr);
+ const payload = compressed.toString('base64');
+ await this.write({ revision, repository, payload, hash });
+ }
}
getData(): RepoCacheData {
diff --git a/lib/util/cache/repository/impl/cache-factory.ts b/lib/util/cache/repository/impl/cache-factory.ts
new file mode 100644
index 0000000000..1ebfa5136b
--- /dev/null
+++ b/lib/util/cache/repository/impl/cache-factory.ts
@@ -0,0 +1,22 @@
+import type { RepositoryCacheType } from '../../../../config/types';
+import { logger } from '../../../../logger';
+import type { RepoCache } from '../types';
+import { RepoCacheLocal } from './local';
+
+export class CacheFactory {
+ static get(
+ repository: string,
+ cacheType: RepositoryCacheType = 'local'
+ ): RepoCache {
+ switch (cacheType) {
+ case 'local':
+ return new RepoCacheLocal(repository);
+ default:
+ logger.warn(
+ { cacheType },
+ `Repository cache type not supported using type "local" instead`
+ );
+ return new RepoCacheLocal(repository);
+ }
+ }
+}
diff --git a/lib/util/cache/repository/impl/local.spec.ts b/lib/util/cache/repository/impl/local.spec.ts
index 38ec21c00a..a3ab46953c 100644
--- a/lib/util/cache/repository/impl/local.spec.ts
+++ b/lib/util/cache/repository/impl/local.spec.ts
@@ -3,9 +3,11 @@ import zlib from 'zlib';
import hasha from 'hasha';
import { fs } from '../../../../../test/util';
import { GlobalConfig } from '../../../../config/global';
+import { logger } from '../../../../logger';
import { CACHE_REVISION } from '../common';
import type { RepoCacheData, RepoCacheRecord } from '../types';
-import { LocalRepoCache } from './local';
+import { CacheFactory } from './cache-factory';
+import { RepoCacheLocal } from './local';
jest.mock('../../../fs');
@@ -20,17 +22,16 @@ async function createCacheRecord(
const hash = hasha(jsonStr, { algorithm: 'sha256' });
const compressed = await compress(jsonStr);
const payload = compressed.toString('base64');
- const record: RepoCacheRecord = { revision, repository, payload, hash };
- return record;
+ return { revision, repository, payload, hash };
}
describe('util/cache/repository/impl/local', () => {
beforeEach(() => {
- GlobalConfig.set({ cacheDir: '/tmp/cache' });
+ GlobalConfig.set({ cacheDir: '/tmp/cache', platform: 'github' });
});
it('returns empty object before any data load', () => {
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
expect(localRepoCache.getData()).toBeEmpty();
});
@@ -38,7 +39,7 @@ describe('util/cache/repository/impl/local', () => {
const data: RepoCacheData = { semanticCommits: 'enabled' };
const cacheRecord = await createCacheRecord(data);
fs.readCacheFile.mockResolvedValue(JSON.stringify(cacheRecord));
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
await localRepoCache.load();
@@ -53,7 +54,7 @@ describe('util/cache/repository/impl/local', () => {
semanticCommits: 'enabled',
})
);
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
await localRepoCache.load();
await localRepoCache.save();
@@ -73,7 +74,7 @@ describe('util/cache/repository/impl/local', () => {
data: { semanticCommits: 'enabled' },
})
);
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
await localRepoCache.load();
await localRepoCache.save();
@@ -94,7 +95,7 @@ describe('util/cache/repository/impl/local', () => {
})
);
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
await localRepoCache.load();
expect(localRepoCache.getData()).toBeEmpty();
@@ -102,7 +103,7 @@ describe('util/cache/repository/impl/local', () => {
it('handles invalid data', async () => {
fs.readCacheFile.mockResolvedValue(JSON.stringify({ foo: 'bar' }));
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
await localRepoCache.load();
@@ -111,7 +112,7 @@ describe('util/cache/repository/impl/local', () => {
it('handles file read error', async () => {
fs.readCacheFile.mockRejectedValue(new Error('unknown error'));
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
await localRepoCache.load();
@@ -123,7 +124,7 @@ describe('util/cache/repository/impl/local', () => {
const cacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
fs.readCacheFile.mockResolvedValueOnce(JSON.stringify(cacheRecord));
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
+ const localRepoCache = CacheFactory.get('some/repo', 'local');
await localRepoCache.load();
expect(localRepoCache.getData()).toEqual({});
@@ -131,9 +132,9 @@ describe('util/cache/repository/impl/local', () => {
it('saves modified cache data to file', async () => {
const oldCacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
+ const cacheType = 'protocol://domain/path';
fs.readCacheFile.mockResolvedValueOnce(JSON.stringify(oldCacheRecord));
- const localRepoCache = new LocalRepoCache('github', 'some/repo');
-
+ const localRepoCache = CacheFactory.get('some/repo', cacheType);
await localRepoCache.load();
const data = localRepoCache.getData();
data.semanticCommits = 'disabled';
@@ -142,6 +143,11 @@ describe('util/cache/repository/impl/local', () => {
const newCacheRecord = await createCacheRecord({
semanticCommits: 'disabled',
});
+ expect(localRepoCache instanceof RepoCacheLocal).toBeTrue();
+ expect(logger.warn).toHaveBeenCalledWith(
+ { cacheType },
+ `Repository cache type not supported using type "local" instead`
+ );
expect(fs.outputCacheFile).toHaveBeenCalledWith(
'/tmp/cache/renovate/repository/github/some/repo.json',
JSON.stringify(newCacheRecord)
diff --git a/lib/util/cache/repository/impl/local.ts b/lib/util/cache/repository/impl/local.ts
index c3edec3e56..6315cc78fb 100644
--- a/lib/util/cache/repository/impl/local.ts
+++ b/lib/util/cache/repository/impl/local.ts
@@ -1,86 +1,38 @@
-import { promisify } from 'util';
-import zlib from 'zlib';
-import hasha from 'hasha';
import upath from 'upath';
import { GlobalConfig } from '../../../../config/global';
import { logger } from '../../../../logger';
import { outputCacheFile, readCacheFile } from '../../../fs';
-import {
- CACHE_REVISION,
- isValidRev10,
- isValidRev11,
- isValidRev12,
-} from '../common';
import type { RepoCacheRecord } from '../types';
import { RepoCacheBase } from './base';
-const compress = promisify(zlib.brotliCompress);
-const decompress = promisify(zlib.brotliDecompress);
-
-export class LocalRepoCache extends RepoCacheBase {
- private oldHash: string | null = null;
-
- constructor(private platform: string, private repository: string) {
- super();
+export class RepoCacheLocal extends RepoCacheBase {
+ constructor(repository: string) {
+ super(repository);
}
- public getCacheFileName(): string {
- const cacheDir = GlobalConfig.get('cacheDir');
- const repoCachePath = '/renovate/repository/';
- const platform = this.platform;
- const fileName = `${this.repository}.json`;
- return upath.join(cacheDir, repoCachePath, platform, fileName);
- }
-
- override async load(): Promise<void> {
+ protected async read(): Promise<string | undefined> {
const cacheFileName = this.getCacheFileName();
+ let data: string | undefined;
try {
- const cacheFileName = this.getCacheFileName();
const rawCache = await readCacheFile(cacheFileName, 'utf8');
- const oldCache = JSON.parse(rawCache);
-
- if (isValidRev12(oldCache, this.repository)) {
- const compressed = Buffer.from(oldCache.payload, 'base64');
- const uncompressed = await decompress(compressed);
- const jsonStr = uncompressed.toString('utf8');
- this.data = JSON.parse(jsonStr);
- this.oldHash = oldCache.hash;
- logger.debug('Repository cache is valid');
- return;
- }
-
- if (isValidRev11(oldCache, this.repository)) {
- this.data = oldCache.data;
- logger.debug('Repository cache is migrated from 11 revision');
- return;
- }
-
- if (isValidRev10(oldCache, this.repository)) {
- delete oldCache.repository;
- delete oldCache.revision;
- this.data = oldCache;
- logger.debug('Repository cache is migrated from 10 revision');
- return;
- }
-
- logger.debug('Repository cache is invalid');
+ data = JSON.parse(rawCache);
} catch (err) {
- logger.debug({ cacheFileName }, 'Repository cache not found');
+ logger.debug({ cacheFileName }, 'Repository local cache not found');
+ throw err;
}
+ return data;
}
- override async save(): Promise<void> {
+ protected async write(data: RepoCacheRecord): Promise<void> {
const cacheFileName = this.getCacheFileName();
- const revision = CACHE_REVISION;
- const repository = this.repository;
- const data = this.getData();
- const jsonStr = JSON.stringify(data);
- const hash = await hasha.async(jsonStr, { algorithm: 'sha256' });
- if (hash !== this.oldHash) {
- const compressed = await compress(jsonStr);
- const payload = compressed.toString('base64');
- const record: RepoCacheRecord = { revision, repository, payload, hash };
- await outputCacheFile(cacheFileName, JSON.stringify(record));
- }
+ await outputCacheFile(cacheFileName, JSON.stringify(data));
+ }
+
+ private getCacheFileName(): string {
+ const cacheDir = GlobalConfig.get('cacheDir');
+ const repoCachePath = '/renovate/repository/';
+ const platform = this.platform;
+ const fileName = `${this.repository}.json`;
+ return upath.join(cacheDir, repoCachePath, platform, fileName);
}
}
diff --git a/lib/util/cache/repository/impl/null.ts b/lib/util/cache/repository/impl/null.ts
new file mode 100644
index 0000000000..99e06623c8
--- /dev/null
+++ b/lib/util/cache/repository/impl/null.ts
@@ -0,0 +1,19 @@
+import type { RepoCache, RepoCacheData } from '../types';
+
+export class RepoCacheNull implements RepoCache {
+ private data: RepoCacheData = {};
+
+ // istanbul ignore next
+ load(): Promise<void> {
+ return Promise.resolve();
+ }
+
+ // istanbul ignore next
+ save(): Promise<void> {
+ return Promise.resolve();
+ }
+
+ getData(): RepoCacheData {
+ return this.data;
+ }
+}
diff --git a/lib/util/cache/repository/index.ts b/lib/util/cache/repository/index.ts
index 2a2ea539b5..1fc4553892 100644
--- a/lib/util/cache/repository/index.ts
+++ b/lib/util/cache/repository/index.ts
@@ -1,10 +1,12 @@
-import { RepoCacheBase } from './impl/base';
+import { RepoCacheNull } from './impl/null';
import type { RepoCache, RepoCacheData } from './types';
-let repoCache: RepoCache = new RepoCacheBase();
+// This will be overwritten with initRepoCache()
+// Used primarily as a placeholder and for testing
+let repoCache: RepoCache = new RepoCacheNull();
export function resetCache(): void {
- setCache(new RepoCacheBase());
+ setCache(new RepoCacheNull());
}
export function setCache(cache: RepoCache): void {
diff --git a/lib/util/cache/repository/init.ts b/lib/util/cache/repository/init.ts
index dbef05bd91..be2999d0b3 100644
--- a/lib/util/cache/repository/init.ts
+++ b/lib/util/cache/repository/init.ts
@@ -1,6 +1,6 @@
-import { GlobalConfig } from '../../../config/global';
import type { RenovateConfig } from '../../../config/types';
-import { LocalRepoCache } from './impl/local';
+import { CacheFactory } from './impl/cache-factory';
+import { RepoCacheNull } from './impl/null';
import { resetCache, setCache } from '.';
/**
@@ -9,24 +9,24 @@ import { resetCache, setCache } from '.';
export async function initRepoCache(config: RenovateConfig): Promise<void> {
resetCache();
- const { platform } = GlobalConfig.get();
- const { repository, repositoryCache } = config;
+ const { repository, repositoryCache, repositoryCacheType: type } = config;
- if (repositoryCache === 'disabled' || !platform || !repository) {
+ if (repositoryCache === 'disabled') {
+ setCache(new RepoCacheNull());
return;
}
if (repositoryCache === 'enabled') {
- const localCache = new LocalRepoCache(platform, repository);
- await localCache.load();
- setCache(localCache);
+ const cache = CacheFactory.get(repository!, type);
+ await cache.load();
+ setCache(cache);
return;
}
if (repositoryCache === 'reset') {
- const localCache = new LocalRepoCache(platform, repository);
- await localCache.save();
- setCache(localCache);
+ const cache = CacheFactory.get(repository!, type);
+ await cache.save();
+ setCache(cache);
return;
}
}
--
GitLab