diff --git a/lib/util/cache/repository/common.ts b/lib/util/cache/repository/common.ts
index ce88e1a1af56e7d68ec801512c9c4bb1a9292b31..11cb26a26ae614ec1d30e68d49309b4e60bee8db 100644
--- a/lib/util/cache/repository/common.ts
+++ b/lib/util/cache/repository/common.ts
@@ -2,30 +2,46 @@ import is from '@sindresorhus/is';
import type { RepoCacheData, RepoCacheRecord } from './types';
// Increment this whenever there could be incompatibilities between old and new cache structure
-export const CACHE_REVISION = 11;
+export const CACHE_REVISION = 12;
-export function isValidCacheRecord(
+export function isValidRev10(
input: unknown,
repo?: string
-): input is RepoCacheRecord {
+): input is RepoCacheData & { repository?: string; revision?: number } {
return (
is.plainObject(input) &&
- is.string(input.repository) &&
is.safeInteger(input.revision) &&
- (!repo || repo === input.repository) &&
- input.revision === CACHE_REVISION
+ input.revision === 10 &&
+ is.string(input.repository) &&
+ (!repo || repo === input.repository)
);
}
-export function canBeMigratedToV11(
+export function isValidRev11(
input: unknown,
repo?: string
-): input is RepoCacheData & { repository?: string; revision?: number } {
+): input is { repository: string; revision: number; data: RepoCacheData } {
return (
is.plainObject(input) &&
+ is.safeInteger(input.revision) &&
+ input.revision === 11 &&
is.string(input.repository) &&
+ is.plainObject(input.data) &&
+ (!repo || repo === input.repository)
+ );
+}
+
+export function isValidRev12(
+ input: unknown,
+ repo?: string
+): input is RepoCacheRecord {
+ return (
+ is.plainObject(input) &&
is.safeInteger(input.revision) &&
+ input.revision === CACHE_REVISION &&
+ is.string(input.repository) &&
(!repo || repo === input.repository) &&
- input.revision === 10
+ is.string(input.payload) &&
+ is.string(input.hash)
);
}
diff --git a/lib/util/cache/repository/impl/local.spec.ts b/lib/util/cache/repository/impl/local.spec.ts
index 2e7158d3e91824a15db904fb7d79799a79eebf86..0d1ad4e4dd5ac316c6d956553c74f222ea03a3d3 100644
--- a/lib/util/cache/repository/impl/local.spec.ts
+++ b/lib/util/cache/repository/impl/local.spec.ts
@@ -1,3 +1,6 @@
+import { promisify } from 'util';
+import zlib from 'zlib';
+import hasha from 'hasha';
import { fs } from '../../../../../test/util';
import { GlobalConfig } from '../../../../config/global';
import { CACHE_REVISION } from '../common';
@@ -6,6 +9,21 @@ import { LocalRepoCache } from './local';
jest.mock('../../../fs');
+const compress = promisify(zlib.brotliCompress);
+
+async function createCacheRecord(
+ data: RepoCacheData,
+ repository = 'some/repo'
+): Promise<RepoCacheRecord> {
+ const revision = CACHE_REVISION;
+ const jsonStr = JSON.stringify(data);
+ const hash = hasha(jsonStr, { algorithm: 'sha256' });
+ const compressed = await compress(jsonStr);
+ const payload = compressed.toString('base64');
+ const record: RepoCacheRecord = { revision, repository, payload, hash };
+ return record;
+}
+
describe('util/cache/repository/impl/local', () => {
beforeEach(() => {
GlobalConfig.set({ cacheDir: '/tmp/cache' });
@@ -16,14 +34,10 @@ describe('util/cache/repository/impl/local', () => {
expect(localRepoCache.getData()).toBeEmpty();
});
- it('loads valid cache from disk', async () => {
+ it('loads previously stored cache from disk', async () => {
const data: RepoCacheData = { semanticCommits: 'enabled' };
- const cache: RepoCacheRecord = {
- repository: 'some/repo',
- revision: CACHE_REVISION,
- data,
- };
- fs.readFile.mockResolvedValue(JSON.stringify(cache));
+ const cacheRecord = await createCacheRecord(data);
+ fs.readFile.mockResolvedValue(JSON.stringify(cacheRecord));
const localRepoCache = new LocalRepoCache('github', 'some/repo');
await localRepoCache.load();
@@ -31,7 +45,7 @@ describe('util/cache/repository/impl/local', () => {
expect(localRepoCache.getData()).toEqual(data);
});
- it('migrates revision from 10 to 11', async () => {
+ it('migrates revision from 10 to 12', async () => {
fs.readFile.mockResolvedValue(
JSON.stringify({
revision: 10,
@@ -40,18 +54,35 @@ describe('util/cache/repository/impl/local', () => {
})
);
const localRepoCache = new LocalRepoCache('github', 'some/repo');
- await localRepoCache.load();
+ await localRepoCache.load();
await localRepoCache.save();
+ const cacheRecord = await createCacheRecord({ semanticCommits: 'enabled' });
expect(fs.outputFile).toHaveBeenCalledWith(
'/tmp/cache/renovate/repository/github/some/repo.json',
+ JSON.stringify(cacheRecord)
+ );
+ });
+
+ it('migrates revision from 11 to 12', async () => {
+ fs.readFile.mockResolvedValue(
JSON.stringify({
- revision: CACHE_REVISION,
+ revision: 11,
repository: 'some/repo',
data: { semanticCommits: 'enabled' },
})
);
+ const localRepoCache = new LocalRepoCache('github', 'some/repo');
+
+ await localRepoCache.load();
+ await localRepoCache.save();
+
+ const cacheRecord = await createCacheRecord({ semanticCommits: 'enabled' });
+ expect(fs.outputFile).toHaveBeenCalledWith(
+ '/tmp/cache/renovate/repository/github/some/repo.json',
+ JSON.stringify(cacheRecord)
+ );
});
it('does not migrate from older revisions to 11', async () => {
@@ -89,13 +120,8 @@ describe('util/cache/repository/impl/local', () => {
});
it('resets if repository does not match', async () => {
- fs.readFile.mockResolvedValueOnce(
- JSON.stringify({
- revision: CACHE_REVISION,
- repository: 'foo/bar',
- data: { semanticCommits: 'enabled' },
- }) as never
- );
+ const cacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
+ fs.readFile.mockResolvedValueOnce(JSON.stringify(cacheRecord) as never);
const localRepoCache = new LocalRepoCache('github', 'some/repo');
await localRepoCache.load();
@@ -103,28 +129,22 @@ describe('util/cache/repository/impl/local', () => {
expect(localRepoCache.getData()).toEqual({});
});
- it('saves cache data to file', async () => {
- fs.readFile.mockResolvedValueOnce(
- JSON.stringify({
- revision: CACHE_REVISION,
- repository: 'some/repo',
- data: { semanticCommits: 'enabled' },
- })
- );
+ it('saves modified cache data to file', async () => {
+ const oldCacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
+ fs.readFile.mockResolvedValueOnce(JSON.stringify(oldCacheRecord));
const localRepoCache = new LocalRepoCache('github', 'some/repo');
- await localRepoCache.load();
+ await localRepoCache.load();
const data = localRepoCache.getData();
data.semanticCommits = 'disabled';
await localRepoCache.save();
+ const newCacheRecord = await createCacheRecord({
+ semanticCommits: 'disabled',
+ });
expect(fs.outputFile).toHaveBeenCalledWith(
'/tmp/cache/renovate/repository/github/some/repo.json',
- JSON.stringify({
- revision: CACHE_REVISION,
- repository: 'some/repo',
- data: { semanticCommits: 'disabled' },
- })
+ JSON.stringify(newCacheRecord)
);
});
});
diff --git a/lib/util/cache/repository/impl/local.ts b/lib/util/cache/repository/impl/local.ts
index 3b41aa4245fb68a390a4d3333a140a64b50f0e2d..ce9a546515d56bce26366bd0c7581eaa1dd659dc 100644
--- a/lib/util/cache/repository/impl/local.ts
+++ b/lib/util/cache/repository/impl/local.ts
@@ -1,21 +1,30 @@
+import { promisify } from 'util';
+import zlib from 'zlib';
+import hasha from 'hasha';
import upath from 'upath';
import { GlobalConfig } from '../../../../config/global';
import { logger } from '../../../../logger';
import { outputFile, readFile } from '../../../fs';
import {
CACHE_REVISION,
- canBeMigratedToV11,
- isValidCacheRecord,
+ isValidRev10,
+ isValidRev11,
+ isValidRev12,
} from '../common';
import type { RepoCacheRecord } from '../types';
import { RepoCacheBase } from './base';
+const compress = promisify(zlib.brotliCompress);
+const decompress = promisify(zlib.brotliDecompress);
+
export class LocalRepoCache extends RepoCacheBase {
+ private oldHash: string | null = null;
+
constructor(private platform: string, private repository: string) {
super();
}
- private getCacheFileName(): string {
+ public getCacheFileName(): string {
const cacheDir = GlobalConfig.get('cacheDir');
const repoCachePath = '/renovate/repository/';
const platform = this.platform;
@@ -29,17 +38,32 @@ export class LocalRepoCache extends RepoCacheBase {
const cacheFileName = this.getCacheFileName();
const rawCache = await readFile(cacheFileName, 'utf8');
const oldCache = JSON.parse(rawCache);
- if (isValidCacheRecord(oldCache, this.repository)) {
- this.data = oldCache.data;
+
+ if (isValidRev12(oldCache, this.repository)) {
+ const compressed = Buffer.from(oldCache.payload, 'base64');
+ const uncompressed = await decompress(compressed);
+ const jsonStr = uncompressed.toString('utf8');
+ this.data = JSON.parse(jsonStr);
+ this.oldHash = oldCache.hash;
logger.debug('Repository cache is valid');
- } else if (canBeMigratedToV11(oldCache, this.repository)) {
+ return;
+ }
+
+ if (isValidRev11(oldCache, this.repository)) {
+ this.data = oldCache.data;
+ logger.debug('Repository cache is migrated from 11 revision');
+ return;
+ }
+
+ if (isValidRev10(oldCache, this.repository)) {
delete oldCache.repository;
delete oldCache.revision;
this.data = oldCache;
- logger.debug('Repository cache is migrated');
- } else {
- logger.debug('Repository cache is invalid');
+ logger.debug('Repository cache is migrated from 10 revision');
+ return;
}
+
+ logger.debug('Repository cache is invalid');
} catch (err) {
logger.debug({ cacheFileName }, 'Repository cache not found');
}
@@ -50,7 +74,13 @@ export class LocalRepoCache extends RepoCacheBase {
const revision = CACHE_REVISION;
const repository = this.repository;
const data = this.getData();
- const record: RepoCacheRecord = { revision, repository, data };
- await outputFile(cacheFileName, JSON.stringify(record));
+ const jsonStr = JSON.stringify(data);
+ const hash = await hasha.async(jsonStr, { algorithm: 'sha256' });
+ if (hash !== this.oldHash) {
+ const compressed = await compress(jsonStr);
+ const payload = compressed.toString('base64');
+ const record: RepoCacheRecord = { revision, repository, payload, hash };
+ await outputFile(cacheFileName, JSON.stringify(record));
+ }
}
}
diff --git a/lib/util/cache/repository/types.ts b/lib/util/cache/repository/types.ts
index ec0f12ffc9a4dc903c55c7ea7c8c2cbfd9b7bda7..79f8cf40b59d43da39647cbea562386a7ba93a40 100644
--- a/lib/util/cache/repository/types.ts
+++ b/lib/util/cache/repository/types.ts
@@ -49,7 +49,8 @@ export interface RepoCacheData {
export interface RepoCacheRecord {
repository: string;
revision: number;
- data: RepoCacheData;
+ payload: string;
+ hash: string;
}
export interface RepoCache {