diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index 2952c4bfea455d79da4735832830dee080c264ae..e4ca94e93f09c91bbeafad016b52e47bb1ef2230 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -181,6 +181,12 @@ For example: } ``` +## containerbaseDir + +This directory is used to cache downloads when `binarySource=docker` or `binarySource=install`. + +Use this option if you need such downloads to be stored outside of Renovate's regular cache directory (`cacheDir`). + ## customEnvVariables This configuration will be applied after all other environment variables so you can use it to override defaults. diff --git a/lib/config/global.ts b/lib/config/global.ts index db0b80366beab1c61a146be876fb0db1acceda55..8051cd74d6cb6f74fbc76855c179238a4b3f05d6 100644 --- a/lib/config/global.ts +++ b/lib/config/global.ts @@ -10,6 +10,7 @@ export class GlobalConfig { 'allowScripts', 'binarySource', 'cacheDir', + 'containerbaseDir', 'customEnvVariables', 'dockerChildPrefix', 'dockerImagePrefix', diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index a744bc9dde4f192dbea6b09a66114b558d5c147f..f32acf03bfdd11e96cd68e55a3651c25aa9d6172 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -314,6 +314,13 @@ const options: RenovateOptions[] = [ globalOnly: true, type: 'string', }, + { + name: 'containerbaseDir', + description: + 'The directory where Renovate stores its containerbase cache. If left empty, Renovate creates a subdirectory within the `cacheDir`.', + globalOnly: true, + type: 'string', + }, { name: 'customEnvVariables', description: diff --git a/lib/config/types.ts b/lib/config/types.ts index a955ec2ee749b971f3dcce2ef7d37762e8d626c8..06d357c407d098c5b07d2d22ba1eeb1664af87fa 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -87,6 +87,7 @@ export interface GlobalOnlyConfig { autodiscoverFilter?: string; baseDir?: string; cacheDir?: string; + containerbaseDir?: string; detectHostRulesFromEnv?: boolean; forceCli?: boolean; gitNoVerify?: GitNoVerifyOption[]; @@ -126,6 +127,7 @@ export interface RepoGlobalConfig { privateKeyOld?: string; localDir?: string; cacheDir?: string; + containerbaseDir?: string; platform?: string; endpoint?: string; } diff --git a/lib/modules/manager/bundler/artifacts.spec.ts b/lib/modules/manager/bundler/artifacts.spec.ts index 9c4d799b6b2c88dfe3ef9ae06111113cb8bff89d..ae18b3c6b158533be6471ec9ec2624d004f17b05 100644 --- a/lib/modules/manager/bundler/artifacts.spec.ts +++ b/lib/modules/manager/bundler/artifacts.spec.ts @@ -34,6 +34,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; const config: UpdateArtifactsConfig = {}; diff --git a/lib/modules/manager/bundler/common.spec.ts b/lib/modules/manager/bundler/common.spec.ts index 612de5a22f3130d9363244141d3b291109e7cb79..15727c7f5d91ed305cbb5ff593e0ebf240ed102b 100644 --- a/lib/modules/manager/bundler/common.spec.ts +++ b/lib/modules/manager/bundler/common.spec.ts @@ -15,6 +15,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; describe('modules/manager/bundler/common', () => { diff --git a/lib/modules/manager/cargo/artifacts.spec.ts b/lib/modules/manager/cargo/artifacts.spec.ts index a3cfcc774624cf352abc99aa4a0caf68cf8cf76d..a07024f3347f25e1fe4e36b5eb36e6b724377d2a 100644 --- a/lib/modules/manager/cargo/artifacts.spec.ts +++ b/lib/modules/manager/cargo/artifacts.spec.ts @@ -18,6 +18,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; describe('modules/manager/cargo/artifacts', () => { diff --git a/lib/modules/manager/cocoapods/artifacts.spec.ts b/lib/modules/manager/cocoapods/artifacts.spec.ts index 1f0fc0f87f7698133e038f05a82a080196ddbb4f..7e5ac1b8b6068cbd2a275776dea797f273a18729 100644 --- a/lib/modules/manager/cocoapods/artifacts.spec.ts +++ b/lib/modules/manager/cocoapods/artifacts.spec.ts @@ -24,6 +24,7 @@ const config: UpdateArtifactsConfig = {}; const adminConfig: RepoGlobalConfig = { localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; describe('modules/manager/cocoapods/artifacts', () => { diff --git a/lib/modules/manager/composer/artifacts.spec.ts b/lib/modules/manager/composer/artifacts.spec.ts index 6909b90acf31dcc22c28e1b7bc1d15e7b4193183..5626a6508599c04fdbc07cb3fa05627d1b26d770 100644 --- a/lib/modules/manager/composer/artifacts.spec.ts +++ b/lib/modules/manager/composer/artifacts.spec.ts @@ -32,6 +32,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const repoStatus = partial<StatusResult>({ diff --git a/lib/modules/manager/gomod/artifacts.spec.ts b/lib/modules/manager/gomod/artifacts.spec.ts index a8069bc086cf54afa8a61a4028b8948a28ab817f..5279d6a930067f9fc729532e3ba2a4bdba58be74 100644 --- a/lib/modules/manager/gomod/artifacts.spec.ts +++ b/lib/modules/manager/gomod/artifacts.spec.ts @@ -45,6 +45,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const config: UpdateArtifactsConfig = { diff --git a/lib/modules/manager/gradle-wrapper/artifacts.spec.ts b/lib/modules/manager/gradle-wrapper/artifacts.spec.ts index 75b91fea4f17c5757f6fa23c572de78d072d07ef..bde60af989a5150452ef49fd83f5c04524cec81b 100644 --- a/lib/modules/manager/gradle-wrapper/artifacts.spec.ts +++ b/lib/modules/manager/gradle-wrapper/artifacts.spec.ts @@ -32,6 +32,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; const config: UpdateArtifactsConfig = { diff --git a/lib/modules/manager/helmv3/artifacts.spec.ts b/lib/modules/manager/helmv3/artifacts.spec.ts index 2e2d9558a430ffe33b5ae325c5918acccb4f8789..4ce50230d34930f71d51ad5e96197426266c4cee 100644 --- a/lib/modules/manager/helmv3/artifacts.spec.ts +++ b/lib/modules/manager/helmv3/artifacts.spec.ts @@ -20,6 +20,7 @@ const datasource = mocked(_datasource); const adminConfig: RepoGlobalConfig = { localDir: join('/tmp/github/some/repo'), // `join` fixes Windows CI cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const config: UpdateArtifactsConfig = {}; diff --git a/lib/modules/manager/jsonnet-bundler/artifacts.spec.ts b/lib/modules/manager/jsonnet-bundler/artifacts.spec.ts index 29d3fec712b0cfb0de970df432fe2125928d20ba..82ce4032d10c04e368bb95d14acc2becd67fe7ad 100644 --- a/lib/modules/manager/jsonnet-bundler/artifacts.spec.ts +++ b/lib/modules/manager/jsonnet-bundler/artifacts.spec.ts @@ -15,6 +15,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const config: UpdateArtifactsConfig = {}; diff --git a/lib/modules/manager/mix/artifacts.spec.ts b/lib/modules/manager/mix/artifacts.spec.ts index b79b7708387e41020bbb4bc8b5cd36b0bc648861..d43b8c88187655c454bbb7119d3b11d9b1f57415 100644 --- a/lib/modules/manager/mix/artifacts.spec.ts +++ b/lib/modules/manager/mix/artifacts.spec.ts @@ -19,6 +19,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; const config: UpdateArtifactsConfig = {}; diff --git a/lib/modules/manager/nuget/artifacts.spec.ts b/lib/modules/manager/nuget/artifacts.spec.ts index e164e4a2a5418e1d95ea0b6311d9ee8c6b1b06de..c07b622fa4ee6c3aaae3c3becaf0e3e2292c93d6 100644 --- a/lib/modules/manager/nuget/artifacts.spec.ts +++ b/lib/modules/manager/nuget/artifacts.spec.ts @@ -25,6 +25,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const config: UpdateArtifactsConfig = {}; diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts index 6076f30988a7d574b3f1c8884df24804ee0d53a6..1da97714c8b7d72942fdd48265becda5213a5ed6 100644 --- a/lib/modules/manager/pip-compile/artifacts.spec.ts +++ b/lib/modules/manager/pip-compile/artifacts.spec.ts @@ -21,6 +21,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const dockerAdminConfig = { ...adminConfig, binarySource: 'docker' }; diff --git a/lib/modules/manager/pip_requirements/artifacts.spec.ts b/lib/modules/manager/pip_requirements/artifacts.spec.ts index bfc998dde76ee39cfac25ddcdbb42478c52683a8..f06399480828370b91aff64e0e72afb1444a8d1b 100644 --- a/lib/modules/manager/pip_requirements/artifacts.spec.ts +++ b/lib/modules/manager/pip_requirements/artifacts.spec.ts @@ -16,6 +16,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const config: UpdateArtifactsConfig = { constraints: { python: '3.10.2' } }; diff --git a/lib/modules/manager/pipenv/artifacts.spec.ts b/lib/modules/manager/pipenv/artifacts.spec.ts index 155a721abd5eb0063e1bcf47370774b87ca706f0..e83707c4fccb36b1724625b7f2358035cce698d5 100644 --- a/lib/modules/manager/pipenv/artifacts.spec.ts +++ b/lib/modules/manager/pipenv/artifacts.spec.ts @@ -24,6 +24,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const dockerAdminConfig = { ...adminConfig, binarySource: 'docker' }; diff --git a/lib/modules/manager/poetry/artifacts.spec.ts b/lib/modules/manager/poetry/artifacts.spec.ts index 8751b532dc5b6a37a470e2db72b0fd5d63e4e82b..e4de7a1276ab9a5c864419479b166ddb9b3a5775 100644 --- a/lib/modules/manager/poetry/artifacts.spec.ts +++ b/lib/modules/manager/poetry/artifacts.spec.ts @@ -26,6 +26,7 @@ const hostRules = mocked(_hostRules); const adminConfig: RepoGlobalConfig = { localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; const config: UpdateArtifactsConfig = {}; diff --git a/lib/modules/manager/terraform/extract.spec.ts b/lib/modules/manager/terraform/extract.spec.ts index dd6a3f311ea6dce7a22d5bee59df8f527d2e83ff..49072138eb2538396abdb1cfb7d760df15efa2f8 100644 --- a/lib/modules/manager/terraform/extract.spec.ts +++ b/lib/modules/manager/terraform/extract.spec.ts @@ -26,6 +26,7 @@ const adminConfig: RepoGlobalConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/cache'), + containerbaseDir: join('/tmp/cache/containerbase'), }; // auto-mock fs diff --git a/lib/modules/manager/terraform/lockfile/index.spec.ts b/lib/modules/manager/terraform/lockfile/index.spec.ts index a99f29835c3c933f451c97ffa164ce10320f9646..7b709d51d506b9aeddc38a4bc8550bcc245b04cc 100644 --- a/lib/modules/manager/terraform/lockfile/index.spec.ts +++ b/lib/modules/manager/terraform/lockfile/index.spec.ts @@ -20,6 +20,7 @@ const adminConfig = { // `join` fixes Windows CI localDir: join('/tmp/github/some/repo'), cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), }; const validLockfile = Fixtures.get('validLockfile.hcl'); diff --git a/lib/util/exec/docker/index.spec.ts b/lib/util/exec/docker/index.spec.ts index c8f7fe6db0c8c33e00444d8f2485ccc696e49178..22dd2d010ff0bbdb615b4fc5e8b54ec2c4b23939 100644 --- a/lib/util/exec/docker/index.spec.ts +++ b/lib/util/exec/docker/index.spec.ts @@ -252,6 +252,43 @@ describe('util/exec/docker/index', () => { ); }); + it('adds custom containerbaseDir to volumes', async () => { + mockExecAll(); + GlobalConfig.set({ + cacheDir: '/tmp/cache', + containerbaseDir: '/tmp/containerbase', + dockerUser: 'some-user', + }); + const volumes: VolumeOption[] = ['/tmp/foo']; + const res = await generateDockerCommand(commands, preCommands, { + ...dockerOptions, + volumes: [...volumes, ...volumes], + }); + expect(res).toBe( + command( + image, + `-v "/tmp/cache":"/tmp/cache" -v "/tmp/containerbase":"/tmp/containerbase" -v "/tmp/foo":"/tmp/foo"` + ) + ); + }); + + it('adds dedupes default containerbaseDir in volumes', async () => { + mockExecAll(); + GlobalConfig.set({ + cacheDir: '/tmp/cache', + containerbaseDir: '/tmp/cache/containerbase', + dockerUser: 'some-user', + }); + const volumes: VolumeOption[] = ['/tmp/foo']; + const res = await generateDockerCommand(commands, preCommands, { + ...dockerOptions, + volumes: [...volumes, ...volumes], + }); + expect(res).toBe( + command(image, `-v "/tmp/cache":"/tmp/cache" -v "/tmp/foo":"/tmp/foo"`) + ); + }); + it('handles tag parameter', async () => { mockExecAll(); const res = await generateDockerCommand(commands, preCommands, { diff --git a/lib/util/exec/docker/index.ts b/lib/util/exec/docker/index.ts index c57bcfd3a0c334a62d346cee56a9c05bd58d131c..3ea7cbfc63dc938fad24d344381046fb8ab6936a 100644 --- a/lib/util/exec/docker/index.ts +++ b/lib/util/exec/docker/index.ts @@ -217,6 +217,7 @@ export async function generateDockerCommand( const { localDir, cacheDir, + containerbaseDir, dockerUser, dockerChildPrefix, dockerImagePrefix, @@ -230,7 +231,12 @@ export async function generateDockerCommand( result.push(`--user=${dockerUser}`); } - result.push(...prepareVolumes([localDir, cacheDir, ...volumes])); + const volumeDirs: VolumeOption[] = [localDir, cacheDir]; + if (containerbaseDir && cacheDir && !containerbaseDir.startsWith(cacheDir)) { + volumeDirs.push(containerbaseDir); + } + volumeDirs.push(...volumes); + result.push(...prepareVolumes(volumeDirs)); if (envVars) { result.push( diff --git a/lib/util/exec/index.spec.ts b/lib/util/exec/index.spec.ts index 6180248684fe92334714cc7e467c34b561a3b12e..463e690390dcc418c02df5a05d7fef38a4941f16 100644 --- a/lib/util/exec/index.spec.ts +++ b/lib/util/exec/index.spec.ts @@ -30,13 +30,14 @@ describe('util/exec/index', () => { let processEnvOrig: NodeJS.ProcessEnv; const cacheDir = '/tmp/renovate/cache/'; + const containerbaseDir = '/tmp/renovate/cache/containerbase'; const cwd = '/tmp/renovate/github/some/repo/'; const defaultCwd = `-w "${cwd}"`; const defaultCacheVolume = `-v "${cacheDir}":"${cacheDir}"`; const defaultVolumes = `-v "${cwd}":"${cwd}" ${defaultCacheVolume}`; - const globalConfig: RepoGlobalConfig = { cacheDir }; + const globalConfig: RepoGlobalConfig = { cacheDir, containerbaseDir }; beforeEach(() => { dockerModule.resetPrefetchedImages(); diff --git a/lib/util/exec/index.ts b/lib/util/exec/index.ts index 055a99672bd6a92dbc4ca08184c81ba676ed1433..9887a3d4f27b729fedb9d1c06e81ea6c1d8e95ac 100644 --- a/lib/util/exec/index.ts +++ b/lib/util/exec/index.ts @@ -101,12 +101,12 @@ async function prepareRawExec( opts: ExecOptions = {} ): Promise<RawExecArguments> { const { docker } = opts; - const { customEnvVariables, cacheDir, binarySource } = GlobalConfig.get(); + const { customEnvVariables, containerbaseDir, binarySource } = + GlobalConfig.get(); if (binarySource === 'docker' || binarySource === 'install') { - const buildPackCacheDir = upath.join(cacheDir, 'containerbase'); opts.env ??= {}; - opts.env.BUILDPACK_CACHE_DIR = buildPackCacheDir; + opts.env.BUILDPACK_CACHE_DIR = containerbaseDir; } const rawOptions = getRawExecOptions(opts); diff --git a/lib/workers/global/initialize.spec.ts b/lib/workers/global/initialize.spec.ts index 3a7ce466111a2e1b131ca255ec5fc2bde55635e3..91547ffdcb06e940c4d016313e609d61e4a1468b 100644 --- a/lib/workers/global/initialize.spec.ts +++ b/lib/workers/global/initialize.spec.ts @@ -29,5 +29,14 @@ describe('workers/global/initialize', () => { git.validateGitVersion.mockResolvedValueOnce(true); await expect(globalInitialize(config)).toResolve(); }); + + it('supports containerbase cache dir', async () => { + const config: AllConfig = { + binarySource: 'docker', + containerbaseDir: '/tmp/containerbase', + }; + git.validateGitVersion.mockResolvedValueOnce(true); + await expect(globalInitialize(config)).toResolve(); + }); }); }); diff --git a/lib/workers/global/initialize.ts b/lib/workers/global/initialize.ts index dd928c1fa23de528e41bf232a7d05e3022e98401..052b32dc69bac8f20d5733e5a2c5d2b599e8be8c 100644 --- a/lib/workers/global/initialize.ts +++ b/lib/workers/global/initialize.ts @@ -29,7 +29,15 @@ async function setDirectories(input: AllConfig): Promise<AllConfig> { } await fs.ensureDir(config.cacheDir); if (config.binarySource === 'docker' || config.binarySource === 'install') { - await fs.ensureDir(upath.join(config.cacheDir, 'containerbase')); + if (config.containerbaseDir) { + logger.debug( + 'Using configured containerbaseDir: ' + config.containerbaseDir + ); + } else { + config.containerbaseDir = upath.join(config.cacheDir, 'containerbase'); + logger.debug('Using containerbaseDir: ' + config.containerbaseDir); + } + await fs.ensureDir(config.containerbaseDir); } return config; }