From c3e57e7b51c47ac00a3cd11955ca54fa461a6b4a Mon Sep 17 00:00:00 2001
From: Miles Budnek <mbudnek@gmail.com>
Date: Mon, 12 Feb 2024 12:25:14 -0500
Subject: [PATCH] feat(manager/pip-compile): Add support for non-default
package repos to the pip-compile manager (#26853)
Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
---
.../usage/getting-started/private-packages.md | 28 ++++++
.../manager/pip-compile/artifacts.spec.ts | 28 ++++++
lib/modules/manager/pip-compile/artifacts.ts | 23 ++++-
.../manager/pip-compile/common.spec.ts | 99 ++++++++++++++++++-
lib/modules/manager/pip-compile/common.ts | 77 ++++++++++++++-
lib/modules/manager/pip-compile/types.ts | 8 ++
6 files changed, 256 insertions(+), 7 deletions(-)
diff --git a/docs/usage/getting-started/private-packages.md b/docs/usage/getting-started/private-packages.md
index e3090fb0f6..b481c94c07 100644
--- a/docs/usage/getting-started/private-packages.md
+++ b/docs/usage/getting-started/private-packages.md
@@ -461,6 +461,34 @@ verify_ssl = true
name = "pypi"
```
+### pip-compile
+
+The pip-compile manager extracts `--index-url` and `--extra-index-url` directives from its input file.
+Renovate will match those URLs with credentials from matching `hostRules` blocks in its configuration and pass appropriate `PIP_INDEX_URL` or `PIP_EXTRA_INDEX_URL` environment variables `pip-compile` with those credentials while generating the output file.
+
+```title="requirements.in"
+--extra-index-url https://pypi.my.domain/simple
+
+private-package==1.2.3
+```
+
+```json
+{
+ "pip-compile": {
+ "fileMatch": ["requirements.in"]
+ },
+ "hostRules": [
+ {
+ "matchHost": "pypi.my.domain",
+ "username": "myuser",
+ "password": "mypassword"
+ }
+ ]
+}
+```
+
+Note: When credentials are passed to `pip-compile` this way Renovate will also pass the `--no-emit-index-url` flag to avoid leaking plain-text credentials to the output file.
+
### poetry
For every Poetry source, a `hostRules` search is done and then any found credentials are added to env like `POETRY_HTTP_BASIC_X_USERNAME` and `POETRY_HTTP_BASIC_X_PASSWORD`, where `X` represents the normalized name of the source in `pyproject.toml`.
diff --git a/lib/modules/manager/pip-compile/artifacts.spec.ts b/lib/modules/manager/pip-compile/artifacts.spec.ts
index 14a92a54a4..4811afd5af 100644
--- a/lib/modules/manager/pip-compile/artifacts.spec.ts
+++ b/lib/modules/manager/pip-compile/artifacts.spec.ts
@@ -279,16 +279,42 @@ describe('modules/manager/pip-compile/artifacts', () => {
Fixtures.get('requirementsNoHeaders.txt'),
'subdir/requirements.in',
'subdir/requirements.txt',
+ false,
),
).toBe('pip-compile requirements.in');
});
+ it('returns --no-emit-index-url when credentials are present in URLs', () => {
+ expect(
+ constructPipCompileCmd(
+ Fixtures.get('requirementsNoHeaders.txt'),
+ 'subdir/requirements.in',
+ 'subdir/requirements.txt',
+ true,
+ ),
+ ).toBe('pip-compile --no-emit-index-url requirements.in');
+ });
+
it('returns extracted common arguments (like those featured in the README)', () => {
expect(
constructPipCompileCmd(
Fixtures.get('requirementsWithHashes.txt'),
'subdir/requirements.in',
'subdir/requirements.txt',
+ false,
+ ),
+ ).toBe(
+ 'pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --strip-extras --resolver=backtracking --output-file=requirements.txt requirements.in',
+ );
+ });
+
+ it('returns --no-emit-index-url only once when its in the header and credentials are present in URLs', () => {
+ expect(
+ constructPipCompileCmd(
+ Fixtures.get('requirementsWithHashes.txt'),
+ 'subdir/requirements.in',
+ 'subdir/requirements.txt',
+ true,
),
).toBe(
'pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --strip-extras --resolver=backtracking --output-file=requirements.txt requirements.in',
@@ -301,6 +327,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
Fixtures.get('requirementsWithUnknownArguments.txt'),
'subdir/requirements.in',
'subdir/requirements.txt',
+ false,
),
).toBe('pip-compile --generate-hashes requirements.in');
expect(logger.trace).toHaveBeenCalledWith(
@@ -319,6 +346,7 @@ describe('modules/manager/pip-compile/artifacts', () => {
Fixtures.get('requirementsWithExploitingArguments.txt'),
'subdir/requirements.in',
'subdir/requirements.txt',
+ false,
),
).toBe(
'pip-compile --generate-hashes --output-file=requirements.txt requirements.in',
diff --git a/lib/modules/manager/pip-compile/artifacts.ts b/lib/modules/manager/pip-compile/artifacts.ts
index 9e3eeaf645..31633e75fc 100644
--- a/lib/modules/manager/pip-compile/artifacts.ts
+++ b/lib/modules/manager/pip-compile/artifacts.ts
@@ -10,23 +10,30 @@ import {
} from '../../../util/fs';
import { getRepoStatus } from '../../../util/git';
import { regEx } from '../../../util/regex';
+import * as pipRequirements from '../pip_requirements';
import type { UpdateArtifact, UpdateArtifactsResult } from '../types';
import {
constraintLineRegex,
deprecatedAllowedPipArguments,
getExecOptions,
+ getRegistryUrlVarsFromPackageFile,
} from './common';
export function constructPipCompileCmd(
content: string,
inputFileName: string,
outputFileName: string,
+ haveCredentials: boolean,
): string {
const headers = constraintLineRegex.exec(content);
const args = ['pip-compile'];
- if (headers?.groups) {
- logger.debug(`Found pip-compile header: ${headers[0]}`);
- for (const argument of split(headers.groups.arguments)) {
+ if (!!headers?.groups || haveCredentials) {
+ logger.debug(`Found pip-compile header: ${headers?.[0]}`);
+ const headerArguments = split(headers?.groups?.arguments ?? '');
+ if (haveCredentials && !headerArguments.includes('--no-emit-index-url')) {
+ headerArguments.push('--no-emit-index-url');
+ }
+ for (const argument of headerArguments) {
if (deprecatedAllowedPipArguments.includes(argument)) {
args.push(argument);
} else if (argument.startsWith('--output-file=')) {
@@ -93,13 +100,21 @@ export async function updateArtifacts({
if (config.isLockFileMaintenance) {
await deleteLocalFile(outputFileName);
}
+ const packageFile = pipRequirements.extractPackageFile(newInputContent);
+ const registryUrlVars = getRegistryUrlVarsFromPackageFile(packageFile);
const cmd = constructPipCompileCmd(
existingOutput,
inputFileName,
outputFileName,
+ registryUrlVars.haveCredentials,
+ );
+ const execOptions = await getExecOptions(
+ config,
+ inputFileName,
+ registryUrlVars.environmentVars,
);
- const execOptions = await getExecOptions(config, inputFileName);
logger.trace({ cmd }, 'pip-compile command');
+ logger.trace({ env: execOptions.extraEnv }, 'pip-compile extra env vars');
await exec(cmd, execOptions);
const status = await getRepoStatus();
if (!status?.modified.includes(outputFileName)) {
diff --git a/lib/modules/manager/pip-compile/common.spec.ts b/lib/modules/manager/pip-compile/common.spec.ts
index fb9f1e5937..1a1f31e1d0 100644
--- a/lib/modules/manager/pip-compile/common.spec.ts
+++ b/lib/modules/manager/pip-compile/common.spec.ts
@@ -1,4 +1,12 @@
-import { allowedPipOptions, extractHeaderCommand } from './common';
+import { mockDeep } from 'jest-mock-extended';
+import { hostRules } from '../../../../test/util';
+import {
+ allowedPipOptions,
+ extractHeaderCommand,
+ getRegistryUrlVarsFromPackageFile,
+} from './common';
+
+jest.mock('../../../util/host-rules', () => mockDeep());
function getCommandInHeader(command: string) {
return `#
@@ -141,4 +149,93 @@ describe('modules/manager/pip-compile/common', () => {
).toHaveProperty('isCustomCommand', true);
});
});
+
+ describe('getRegistryUrlFlagsFromPackageFile()', () => {
+ it('handles both registryUrls and additionalRegistryUrls', () => {
+ hostRules.find.mockReturnValue({});
+ expect(
+ getRegistryUrlVarsFromPackageFile({
+ deps: [],
+ registryUrls: ['https://example.com/pypi/simple'],
+ additionalRegistryUrls: ['https://example2.com/pypi/simple'],
+ }),
+ ).toEqual({
+ haveCredentials: false,
+ environmentVars: {
+ PIP_INDEX_URL: 'https://example.com/pypi/simple',
+ PIP_EXTRA_INDEX_URL: 'https://example2.com/pypi/simple',
+ },
+ });
+ });
+
+ it('handles multiple additionalRegistryUrls', () => {
+ hostRules.find.mockReturnValue({});
+ expect(
+ getRegistryUrlVarsFromPackageFile({
+ deps: [],
+ additionalRegistryUrls: [
+ 'https://example.com/pypi/simple',
+ 'https://example2.com/pypi/simple',
+ ],
+ }),
+ ).toEqual({
+ haveCredentials: false,
+ environmentVars: {
+ PIP_EXTRA_INDEX_URL:
+ 'https://example.com/pypi/simple https://example2.com/pypi/simple',
+ },
+ });
+ });
+
+ it('uses extra index URLs with no auth', () => {
+ hostRules.find.mockReturnValue({});
+ expect(
+ getRegistryUrlVarsFromPackageFile({
+ deps: [],
+ registryUrls: ['https://example.com/pypi/simple'],
+ }),
+ ).toEqual({
+ haveCredentials: false,
+ environmentVars: {
+ PIP_INDEX_URL: 'https://example.com/pypi/simple',
+ },
+ });
+ });
+
+ it('uses auth from extra index URLs matching host rules', () => {
+ hostRules.find.mockReturnValue({
+ username: 'user',
+ password: 'password',
+ });
+ expect(
+ getRegistryUrlVarsFromPackageFile({
+ deps: [],
+ registryUrls: ['https://example.com/pypi/simple'],
+ }),
+ ).toEqual({
+ haveCredentials: true,
+ environmentVars: {
+ PIP_INDEX_URL: 'https://user:password@example.com/pypi/simple',
+ },
+ });
+ });
+
+ it('handles invalid URLs', () => {
+ hostRules.find.mockReturnValue({});
+ expect(
+ getRegistryUrlVarsFromPackageFile({
+ deps: [],
+ additionalRegistryUrls: [
+ 'https://example.com/pypi/simple',
+ 'this is not a valid URL',
+ ],
+ }),
+ ).toEqual({
+ haveCredentials: false,
+ environmentVars: {
+ PIP_EXTRA_INDEX_URL: 'https://example.com/pypi/simple',
+ },
+ });
+ });
+ });
});
diff --git a/lib/modules/manager/pip-compile/common.ts b/lib/modules/manager/pip-compile/common.ts
index 00bdb8c365..3279f14730 100644
--- a/lib/modules/manager/pip-compile/common.ts
+++ b/lib/modules/manager/pip-compile/common.ts
@@ -1,11 +1,13 @@
import is from '@sindresorhus/is';
import { split } from 'shlex';
import { logger } from '../../../logger';
+import { isNotNullOrUndefined } from '../../../util/array';
import type { ExecOptions } from '../../../util/exec/types';
import { ensureCacheDir } from '../../../util/fs';
+import * as hostRules from '../../../util/host-rules';
import { regEx } from '../../../util/regex';
-import type { UpdateArtifactsConfig } from '../types';
-import type { PipCompileArgs } from './types';
+import type { PackageFileContent, UpdateArtifactsConfig } from '../types';
+import type { GetRegistryUrlVarsResult, PipCompileArgs } from './types';
export function getPythonConstraint(
config: UpdateArtifactsConfig,
@@ -35,6 +37,7 @@ export function getPipToolsConstraint(config: UpdateArtifactsConfig): string {
export async function getExecOptions(
config: UpdateArtifactsConfig,
inputFileName: string,
+ extraEnv: Record<string, string>,
): Promise<ExecOptions> {
const constraint = getPythonConstraint(config);
const pipToolsConstraint = getPipToolsConstraint(config);
@@ -53,6 +56,7 @@ export async function getExecOptions(
],
extraEnv: {
PIP_CACHE_DIR: await ensureCacheDir('pip'),
+ ...extraEnv,
},
};
return execOptions;
@@ -217,3 +221,72 @@ function throwForUnknownOption(arg: string): void {
}
throw new Error(`Option ${arg} not supported (yet)`);
}
+
+function buildRegistryUrl(url: string): URL | null {
+ try {
+ const ret = new URL(url);
+ const hostRule = hostRules.find({ url });
+ if (!ret.username && !ret.password) {
+ ret.username = hostRule.username ?? '';
+ ret.password = hostRule.password ?? '';
+ }
+ return ret;
+ } catch {
+ return null;
+ }
+}
+
+function getRegistryUrlVarFromUrls(
+ varName: keyof GetRegistryUrlVarsResult['environmentVars'],
+ urls: URL[],
+): GetRegistryUrlVarsResult {
+ if (!urls.length) {
+ return {
+ haveCredentials: false,
+ environmentVars: {},
+ };
+ }
+
+ let haveCredentials = false;
+ for (const url of urls) {
+ if (url.username || url.password) {
+ haveCredentials = true;
+ }
+ }
+ const registryUrlsString = urls.map((url) => url.href).join(' ');
+ const ret: GetRegistryUrlVarsResult = {
+ haveCredentials,
+ environmentVars: {},
+ };
+ if (registryUrlsString) {
+ ret.environmentVars[varName] = registryUrlsString;
+ }
+ return ret;
+}
+
+export function getRegistryUrlVarsFromPackageFile(
+ packageFile: PackageFileContent | null,
+): GetRegistryUrlVarsResult {
+ // There should only ever be one element in registryUrls, since pip_requirements gets them from --index-url
+ // flags in the input file, and that only makes sense once
+ const indexUrl = getRegistryUrlVarFromUrls(
+ 'PIP_INDEX_URL',
+ packageFile?.registryUrls
+ ?.map(buildRegistryUrl)
+ .filter(isNotNullOrUndefined) ?? [],
+ );
+ const extraIndexUrls = getRegistryUrlVarFromUrls(
+ 'PIP_EXTRA_INDEX_URL',
+ packageFile?.additionalRegistryUrls
+ ?.map(buildRegistryUrl)
+ .filter(isNotNullOrUndefined) ?? [],
+ );
+
+ return {
+ haveCredentials: indexUrl.haveCredentials || extraIndexUrls.haveCredentials,
+ environmentVars: {
+ ...indexUrl.environmentVars,
+ ...extraIndexUrls.environmentVars,
+ },
+ };
+}
diff --git a/lib/modules/manager/pip-compile/types.ts b/lib/modules/manager/pip-compile/types.ts
index c9e29b55c1..964c1cc9b6 100644
--- a/lib/modules/manager/pip-compile/types.ts
+++ b/lib/modules/manager/pip-compile/types.ts
@@ -1,3 +1,11 @@
+export interface GetRegistryUrlVarsResult {
+ haveCredentials: boolean;
+ environmentVars: {
+ PIP_INDEX_URL?: string;
+ PIP_EXTRA_INDEX_URL?: string;
+ };
+}
+
// managers supported by pip-tools Python package
export type SupportedManagers =
| 'pip_requirements'
--
GitLab