From 991cc7ac3d132b8f832a58390c047864c3492b73 Mon Sep 17 00:00:00 2001
From: Gabriel-Ladzaretti
<97394622+Gabriel-Ladzaretti@users.noreply.github.com>
Date: Sun, 13 Nov 2022 11:00:06 +0200
Subject: [PATCH] feat(repo/cache): add s3 support for user configured folder
hierarchy (#18865)
Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
---
docs/usage/self-hosted-configuration.md | 5 ++
lib/util/cache/repository/impl/s3.spec.ts | 75 +++++++++++++++++++++--
lib/util/cache/repository/impl/s3.ts | 23 ++++++-
3 files changed, 97 insertions(+), 6 deletions(-)
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index 3873769761..43acd75998 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -669,6 +669,11 @@ Set this to an S3 URI to enable S3 backed repository cache.
AWS_REGION
```
+<!-- prettier-ignore -->
+!!! tip
+ If you're storing the repository cache on Amazon S3 then you may set a folder hierarchy as part of `repositoryCacheType`.
+ For example, `repositoryCacheType: 's3://bucket-name/dir1/.../dirN/'`.
+
## requireConfig
By default, Renovate needs a Renovate config file in each repository where it runs before it will propose any dependency updates.
diff --git a/lib/util/cache/repository/impl/s3.spec.ts b/lib/util/cache/repository/impl/s3.spec.ts
index cfb276a6f3..1b8cf5b031 100644
--- a/lib/util/cache/repository/impl/s3.spec.ts
+++ b/lib/util/cache/repository/impl/s3.spec.ts
@@ -18,21 +18,24 @@ import { RepoCacheS3 } from './s3';
function createGetObjectCommandInput(
repository: string,
- url: string
+ url: string,
+ folder = ''
): GetObjectCommandInput {
+ const platform = GlobalConfig.get('platform')!;
return {
Bucket: parseS3Url(url)?.Bucket,
- Key: `github/${repository}/cache.json`,
+ Key: `${folder}${platform}/${repository}/cache.json`,
};
}
function createPutObjectCommandInput(
repository: string,
url: string,
- data: RepoCacheRecord
+ data: RepoCacheRecord,
+ folder = ''
): PutObjectCommandInput {
return {
- ...createGetObjectCommandInput(repository, url),
+ ...createGetObjectCommandInput(repository, url, folder),
Body: JSON.stringify(data),
ContentType: 'application/json',
};
@@ -76,6 +79,49 @@ describe('util/cache/repository/impl/s3', () => {
expect(logger.debug).toHaveBeenCalledWith('RepoCacheS3.read() - success');
});
+ it('successfully reads from s3://bucket/dir1/.../dirN/', async () => {
+ const json = '{}';
+ const folder = 'dir1/dir2/dir3/';
+ s3Cache = new RepoCacheS3(
+ repository,
+ '0123456789abcdef',
+ `${url}/${folder}`
+ );
+ s3Mock
+ .on(
+ GetObjectCommand,
+ createGetObjectCommandInput(repository, url, folder)
+ )
+ .resolvesOnce({ Body: Readable.from([json]) });
+ await expect(s3Cache.read()).resolves.toBe(json);
+ expect(logger.warn).toHaveBeenCalledTimes(0);
+ expect(logger.error).toHaveBeenCalledTimes(0);
+ expect(logger.debug).toHaveBeenCalledWith('RepoCacheS3.read() - success');
+ });
+
+ it('appends a missing traling slash to pathname when instantiating RepoCacheS3', async () => {
+ const json = '{}';
+ const pathname = 'dir1/dir2/dir3/file.ext';
+ s3Cache = new RepoCacheS3(
+ repository,
+ '0123456789abcdef',
+ `${url}/${pathname}`
+ );
+ s3Mock
+ .on(
+ GetObjectCommand,
+ createGetObjectCommandInput(repository, url, pathname + '/')
+ )
+ .resolvesOnce({ Body: Readable.from([json]) });
+ await expect(s3Cache.read()).resolves.toBe(json);
+ expect(logger.debug).toHaveBeenCalledWith('RepoCacheS3.read() - success');
+ expect(logger.warn).toHaveBeenCalledTimes(1);
+ expect(logger.warn).toHaveBeenCalledWith(
+ { pathname },
+ 'RepoCacheS3.getCacheFolder() - appending missing trailing slash to pathname'
+ );
+ });
+
it('gets an unexpected response from s3', async () => {
s3Mock.on(GetObjectCommand, getObjectCommandInput).resolvesOnce({});
await expect(s3Cache.read()).resolves.toBeNull();
@@ -117,6 +163,27 @@ describe('util/cache/repository/impl/s3', () => {
expect(logger.warn).toHaveBeenCalledTimes(0);
});
+ it('successfully writes to s3://bucket/dir1/.../dirN/', async () => {
+ const putObjectCommandOutput: PutObjectCommandOutput = {
+ $metadata: { attempts: 1, httpStatusCode: 200, totalRetryDelay: 0 },
+ };
+ const folder = 'dir1/dir2/dir3/';
+ s3Cache = new RepoCacheS3(
+ repository,
+ '0123456789abcdef',
+ `${url}/${folder}`
+ );
+ s3Mock
+ .on(
+ PutObjectCommand,
+ createPutObjectCommandInput(repository, url, repoCache, folder)
+ )
+ .resolvesOnce(putObjectCommandOutput);
+ await expect(s3Cache.write(repoCache)).toResolve();
+ expect(logger.warn).toHaveBeenCalledTimes(0);
+ expect(logger.error).toHaveBeenCalledTimes(0);
+ });
+
it('fails to write to s3', async () => {
s3Mock.on(PutObjectCommand, putObjectCommandInput).rejectsOnce(err);
await expect(s3Cache.write(repoCache)).toResolve();
diff --git a/lib/util/cache/repository/impl/s3.ts b/lib/util/cache/repository/impl/s3.ts
index da57ba094a..35f137c979 100644
--- a/lib/util/cache/repository/impl/s3.ts
+++ b/lib/util/cache/repository/impl/s3.ts
@@ -14,10 +14,13 @@ import { RepoCacheBase } from './base';
export class RepoCacheS3 extends RepoCacheBase {
private readonly s3Client;
private readonly bucket;
+ private readonly dir;
constructor(repository: string, fingerprint: string, url: string) {
super(repository, fingerprint);
- this.bucket = parseS3Url(url)?.Bucket;
+ const { Bucket, Key } = parseS3Url(url)!;
+ this.dir = this.getCacheFolder(Key);
+ this.bucket = Bucket;
this.s3Client = getS3Client();
}
@@ -64,7 +67,23 @@ export class RepoCacheS3 extends RepoCacheBase {
}
}
+ private getCacheFolder(pathname: string | undefined): string {
+ if (!pathname) {
+ return '';
+ }
+
+ if (pathname.endsWith('/')) {
+ return pathname;
+ }
+
+ logger.warn(
+ { pathname },
+ 'RepoCacheS3.getCacheFolder() - appending missing trailing slash to pathname'
+ );
+ return pathname + '/';
+ }
+
private getCacheFileName(): string {
- return `${this.platform}/${this.repository}/cache.json`;
+ return `${this.dir}${this.platform}/${this.repository}/cache.json`;
}
}
--
GitLab