From 70700eedae5b28076ab0ce7c29a7526b21c92390 Mon Sep 17 00:00:00 2001 From: Florian Greinacher <florian@greinacher.de> Date: Thu, 4 Nov 2021 09:43:52 +0100 Subject: [PATCH] feat(config)!: make host rule detection configurable and opt-in (#12294) Add configuration option `detectHostRulesFromEnv`. BREAKING CHANGE: `hostRules` are no longer automatically derived from env variables such as `NPM_X_TOKEN`. Set `detectHostRulesFromEnv=true` in config to re-enable same functionality. --- .../usage/getting-started/private-packages.md | 66 +---------------- docs/usage/self-hosted-configuration.md | 66 +++++++++++++++++ lib/config/options/index.ts | 8 ++ lib/config/types.ts | 1 + .../parse/__snapshots__/env.spec.ts.snap | 51 ------------- lib/workers/global/config/parse/env.spec.ts | 73 ------------------- lib/workers/global/config/parse/env.ts | 3 - lib/workers/global/config/parse/index.spec.ts | 14 +++- lib/workers/global/config/parse/index.ts | 5 ++ 9 files changed, 93 insertions(+), 194 deletions(-) diff --git a/docs/usage/getting-started/private-packages.md b/docs/usage/getting-started/private-packages.md index 3f02c4274b..304021c9d3 100644 --- a/docs/usage/getting-started/private-packages.md +++ b/docs/usage/getting-started/private-packages.md @@ -392,68 +392,4 @@ Note: Encrypted values can't be used in the "Admin/Bot config". ### hostRules configuration using environment variables -Self-hosted users can use environment variables to configure the most common types of `hostRules` for authentication. - -The format of the environment variables must follow: - -- Datasource name (e.g. `NPM`, `PYPI`) -- Underscore (`_`) -- `matchHost` -- Underscore (`_`) -- Field name (`TOKEN`, `USER_NAME`, or `PASSWORD`) - -Hyphens (`-`) in datasource or host name must be replaced with double underscores (`__`). -Periods (`.`) in host names must be replaced with a single underscore (`_`). - -Note: the following prefixes cannot be supported for this functionality: `npm_config_`, `npm_lifecycle_`, `npm_package_`. - -#### npmjs registry token example - -`NPM_REGISTRY_NPMJS_ORG_TOKEN=abc123`: - -```json -{ - "hostRules": [ - { - "hostType": "npm", - "matchHost": "registry.npmjs.org", - "token": "abc123" - } - ] -} -``` - -#### GitLab Tags username/password example - -`GITLAB__TAGS_CODE__HOST_COMPANY_COM_USERNAME=bot GITLAB__TAGS_CODE__HOST_COMPANY_COM_PASSWORD=botpass123`: - -```json -{ - "hostRules": [ - { - "hostType": "gitlab-tags", - "matchHost": "code-host.company.com", - "username": "bot", - "password": "botpass123" - } - ] -} -``` - -#### Datasource and credentials only - -You can skip the host part, and use just the datasource and credentials. - -`DOCKER_USERNAME=bot DOCKER_PASSWORD=botpass123`: - -```json -{ - "hostRules": [ - { - "hostType": "docker", - "username": "bot", - "password": "botpass123" - } - ] -} -``` +Self-hosted users can enable the option [`detectHostRulesFromEnv`](../self-hosted-configuration.md#detectHostRulesFromEnv) to configure the most common types of `hostRules` via environment variables. diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index a743db4029..8a90bff426 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -139,6 +139,72 @@ This feature is disabled by default because it may prove surprising or undesirab Currently this capability is supported for the `npm` manager only - specifically the `~/.npmrc` file. If found, it will be imported into `config.npmrc` with `config.npmrcMerge` will be set to `true`. +## detectHostRulesFromEnv + +The format of the environment variables must follow: + +- Datasource name (e.g. `NPM`, `PYPI`) +- Underscore (`_`) +- `matchHost` +- Underscore (`_`) +- Field name (`TOKEN`, `USER_NAME`, or `PASSWORD`) + +Hyphens (`-`) in datasource or host name must be replaced with double underscores (`__`). +Periods (`.`) in host names must be replaced with a single underscore (`_`). + +Note: the following prefixes cannot be supported for this functionality: `npm_config_`, `npm_lifecycle_`, `npm_package_`. + +### npmjs registry token example + +`NPM_REGISTRY_NPMJS_ORG_TOKEN=abc123`: + +```json +{ + "hostRules": [ + { + "hostType": "npm", + "matchHost": "registry.npmjs.org", + "token": "abc123" + } + ] +} +``` + +### GitLab Tags username/password example + +`GITLAB__TAGS_CODE__HOST_COMPANY_COM_USERNAME=bot GITLAB__TAGS_CODE__HOST_COMPANY_COM_PASSWORD=botpass123`: + +```json +{ + "hostRules": [ + { + "hostType": "gitlab-tags", + "matchHost": "code-host.company.com", + "username": "bot", + "password": "botpass123" + } + ] +} +``` + +### Datasource and credentials only + +You can skip the host part, and use just the datasource and credentials. + +`DOCKER_USERNAME=bot DOCKER_PASSWORD=botpass123`: + +```json +{ + "hostRules": [ + { + "hostType": "docker", + "username": "bot", + "password": "botpass123" + } + ] +} +``` + ## dockerChildPrefix Adds a custom prefix to the default Renovate sidecar Docker containers name and label. diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 6dfdde6edf..b76269a135 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -15,6 +15,14 @@ const options: RenovateOptions[] = [ default: false, globalOnly: true, }, + { + name: 'detectHostRulesFromEnv', + description: + 'If true, Renovate tries to detect host rules from environment variables.', + type: 'boolean', + default: false, + globalOnly: true, + }, { name: 'allowPostUpgradeCommandTemplating', description: 'If true allow templating for post-upgrade commands.', diff --git a/lib/config/types.ts b/lib/config/types.ts index fc73d3c998..07cbe71eb1 100644 --- a/lib/config/types.ts +++ b/lib/config/types.ts @@ -74,6 +74,7 @@ export interface GlobalOnlyConfig { autodiscoverFilter?: string; baseDir?: string; cacheDir?: string; + detectHostRulesFromEnv?: boolean; forceCli?: boolean; gitNoVerify?: GitNoVerifyOption[]; gitPrivateKey?: string; diff --git a/lib/workers/global/config/parse/__snapshots__/env.spec.ts.snap b/lib/workers/global/config/parse/__snapshots__/env.spec.ts.snap index 840b3afa04..38bc9f5f9b 100644 --- a/lib/workers/global/config/parse/__snapshots__/env.spec.ts.snap +++ b/lib/workers/global/config/parse/__snapshots__/env.spec.ts.snap @@ -81,54 +81,3 @@ Object { "token": "a gitlab.com token", } `; - -exports[`workers/global/config/parse/env .getConfig(env) supports datasource env token 1`] = ` -Object { - "hostRules": Array [ - Object { - "hostType": "pypi", - "token": "some-token", - }, - ], -} -`; - -exports[`workers/global/config/parse/env .getConfig(env) supports docker username/password 1`] = ` -Object { - "hostRules": Array [ - Object { - "hostType": "docker", - "password": "some-password", - "username": "some-username", - }, - ], -} -`; - -exports[`workers/global/config/parse/env .getConfig(env) supports domain and host names with case insensitivity 1`] = ` -Object { - "hostRules": Array [ - Object { - "hostType": "github-tags", - "matchHost": "github.com", - "token": "some-token", - }, - Object { - "hostType": "pypi", - "matchHost": "my.custom.host", - "password": "some-password", - }, - ], -} -`; - -exports[`workers/global/config/parse/env .getConfig(env) supports password-only 1`] = ` -Object { - "hostRules": Array [ - Object { - "hostType": "npm", - "password": "some-password", - }, - ], -} -`; diff --git a/lib/workers/global/config/parse/env.spec.ts b/lib/workers/global/config/parse/env.spec.ts index fdf3b573f7..432183313c 100644 --- a/lib/workers/global/config/parse/env.spec.ts +++ b/lib/workers/global/config/parse/env.spec.ts @@ -158,79 +158,6 @@ describe('workers/global/config/parse/env', () => { token: 'an Azure DevOps token', }); }); - it('supports docker username/password', () => { - const envParam: NodeJS.ProcessEnv = { - DOCKER_USERNAME: 'some-username', - DOCKER_PASSWORD: 'some-password', - }; - expect(env.getConfig(envParam)).toMatchSnapshot({ - hostRules: [ - { - hostType: 'docker', - password: 'some-password', - username: 'some-username', - }, - ], - }); - }); - it('supports password-only', () => { - const envParam: NodeJS.ProcessEnv = { - NPM_PASSWORD: 'some-password', - }; - expect(env.getConfig(envParam)).toMatchSnapshot({ - hostRules: [{ hostType: 'npm', password: 'some-password' }], - }); - }); - it('supports domain and host names with case insensitivity', () => { - const envParam: NodeJS.ProcessEnv = { - GITHUB__TAGS_GITHUB_COM_TOKEN: 'some-token', - pypi_my_CUSTOM_HOST_passWORD: 'some-password', - }; - const res = env.getConfig(envParam); - expect(res).toMatchSnapshot({ - hostRules: [ - { matchHost: 'github.com', token: 'some-token' }, - { matchHost: 'my.custom.host', password: 'some-password' }, - ], - }); - }); - it('regression test for #10937', () => { - const envParam: NodeJS.ProcessEnv = { - GIT__TAGS_GITLAB_EXAMPLE__DOMAIN_NET_USERNAME: 'some-user', - GIT__TAGS_GITLAB_EXAMPLE__DOMAIN_NET_PASSWORD: 'some-password', - }; - const res = env.getConfig(envParam); - expect(res).toMatchObject({ - hostRules: [ - { - hostType: 'git-tags', - matchHost: 'gitlab.example-domain.net', - password: 'some-password', - username: 'some-user', - }, - ], - }); - }); - it('supports datasource env token', () => { - const envParam: NodeJS.ProcessEnv = { - PYPI_TOKEN: 'some-token', - }; - expect(env.getConfig(envParam)).toMatchSnapshot({ - hostRules: [{ hostType: 'pypi', token: 'some-token' }], - }); - }); - it('rejects incomplete datasource env token', () => { - const envParam: NodeJS.ProcessEnv = { - PYPI_FOO_TOKEN: 'some-token', - }; - expect(env.getConfig(envParam).hostRules).toHaveLength(0); - }); - it('rejects npm env', () => { - const envParam: NodeJS.ProcessEnv = { - npm_package_devDependencies__types_registry_auth_token: '4.2.0', - }; - expect(env.getConfig(envParam).hostRules).toHaveLength(0); - }); it('supports Bitbucket token', () => { const envParam: NodeJS.ProcessEnv = { RENOVATE_PLATFORM: PlatformId.Bitbucket, diff --git a/lib/workers/global/config/parse/env.ts b/lib/workers/global/config/parse/env.ts index 51353ef48b..f7268d74cb 100644 --- a/lib/workers/global/config/parse/env.ts +++ b/lib/workers/global/config/parse/env.ts @@ -3,7 +3,6 @@ import { getOptions } from '../../../../config/options'; import type { AllConfig, RenovateOptions } from '../../../../config/types'; import { PlatformId } from '../../../../constants'; import { logger } from '../../../../logger'; -import { hostRulesFromEnv } from './host-rules-from-env'; function normalizePrefixes( env: NodeJS.ProcessEnv, @@ -118,8 +117,6 @@ export function getConfig(inputEnv: NodeJS.ProcessEnv): AllConfig { }); } - config.hostRules = [...config.hostRules, ...hostRulesFromEnv(env)]; - // These env vars are deprecated and deleted to make sure they're not used const unsupportedEnv = [ 'BITBUCKET_TOKEN', diff --git a/lib/workers/global/config/parse/index.spec.ts b/lib/workers/global/config/parse/index.spec.ts index b69f970372..5fd74b6d39 100644 --- a/lib/workers/global/config/parse/index.spec.ts +++ b/lib/workers/global/config/parse/index.spec.ts @@ -1,23 +1,26 @@ import upath from 'upath'; +import { mocked } from '../../../../../test/util'; import { readFile } from '../../../../util/fs'; import getArgv from './__fixtures__/argv'; +import * as _hostRulesFromEnv from './host-rules-from-env'; jest.mock('../../../../datasource/npm'); jest.mock('../../../../util/fs'); - +jest.mock('./host-rules-from-env'); try { jest.mock('../../config.js'); } catch (err) { // file does not exist } +const { hostRulesFromEnv } = mocked(_hostRulesFromEnv); + describe('workers/global/config/parse/index', () => { describe('.parseConfigs(env, defaultArgv)', () => { let configParser: typeof import('.'); let defaultArgv: string[]; let defaultEnv: NodeJS.ProcessEnv; beforeEach(async () => { - jest.resetModules(); configParser = await import('./index'); defaultArgv = getArgv(); defaultEnv = { @@ -125,5 +128,12 @@ describe('workers/global/config/parse/index', () => { const parsed = await configParser.parseConfigs(defaultEnv, defaultArgv); expect(parsed.npmrc).toBeNull(); }); + + it('parses host rules from env', async () => { + defaultArgv = defaultArgv.concat(['--detect-host-rules-from-env=true']); + hostRulesFromEnv.mockReturnValueOnce([{ matchHost: 'example.org' }]); + const parsed = await configParser.parseConfigs(defaultEnv, defaultArgv); + expect(parsed.hostRules).toContainEqual({ matchHost: 'example.org' }); + }); }); }); diff --git a/lib/workers/global/config/parse/index.ts b/lib/workers/global/config/parse/index.ts index e6b35f742b..35306b784d 100644 --- a/lib/workers/global/config/parse/index.ts +++ b/lib/workers/global/config/parse/index.ts @@ -8,6 +8,7 @@ import { ensureTrailingSlash } from '../../../../util/url'; import * as cliParser from './cli'; import * as envParser from './env'; import * as fileParser from './file'; +import { hostRulesFromEnv } from './host-rules-from-env'; export async function parseConfigs( env: NodeJS.ProcessEnv, @@ -81,6 +82,10 @@ export async function parseConfigs( config = mergeChildConfig(config, globalManagerConfig); } + if (config.detectHostRulesFromEnv) { + const hostRules = hostRulesFromEnv(env); + config.hostRules = [...config.hostRules, ...hostRules]; + } // Get global config logger.trace({ config }, 'Full config'); -- GitLab