From 529a3a3f11fc7a563d0c81c0bb262c3d1cc2bf12 Mon Sep 17 00:00:00 2001
From: Malte Swart <msw@kialo.com>
Date: Thu, 10 Feb 2022 18:30:50 +0100
Subject: [PATCH] feat(manager/pip-compile): support basic arguments (#14098)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
.../__fixtures__/requirementsNoHeaders.txt | 5 ++
.../requirementsWithExploitingArguments.txt | 9 +++
.../__fixtures__/requirementsWithHashes.txt | 14 +++++
.../requirementsWithUnknownArguments.txt | 9 +++
lib/manager/pip-compile/artifacts.spec.ts | 56 +++++++++++++++++++
lib/manager/pip-compile/artifacts.ts | 49 +++++++++++++++-
lib/manager/pip-compile/readme.md | 7 +++
7 files changed, 147 insertions(+), 2 deletions(-)
create mode 100644 lib/manager/pip-compile/__fixtures__/requirementsNoHeaders.txt
create mode 100644 lib/manager/pip-compile/__fixtures__/requirementsWithExploitingArguments.txt
create mode 100644 lib/manager/pip-compile/__fixtures__/requirementsWithHashes.txt
create mode 100644 lib/manager/pip-compile/__fixtures__/requirementsWithUnknownArguments.txt
diff --git a/lib/manager/pip-compile/__fixtures__/requirementsNoHeaders.txt b/lib/manager/pip-compile/__fixtures__/requirementsNoHeaders.txt
new file mode 100644
index 0000000000..3fc1aa1392
--- /dev/null
+++ b/lib/manager/pip-compile/__fixtures__/requirementsNoHeaders.txt
@@ -0,0 +1,5 @@
+# simple comment
+some-package==0.3.1
+some-other-package==1.0.0
+sphinx
+not_semver==1.9
diff --git a/lib/manager/pip-compile/__fixtures__/requirementsWithExploitingArguments.txt b/lib/manager/pip-compile/__fixtures__/requirementsWithExploitingArguments.txt
new file mode 100644
index 0000000000..930ea670d1
--- /dev/null
+++ b/lib/manager/pip-compile/__fixtures__/requirementsWithExploitingArguments.txt
@@ -0,0 +1,9 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+# pip-compile --generate-hashes --$(curl asdf) --output-file=/etc/shadow requirements.in
+#
+attrs==21.2.0 \
+ --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \
+ --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb
diff --git a/lib/manager/pip-compile/__fixtures__/requirementsWithHashes.txt b/lib/manager/pip-compile/__fixtures__/requirementsWithHashes.txt
new file mode 100644
index 0000000000..38ca9ec1db
--- /dev/null
+++ b/lib/manager/pip-compile/__fixtures__/requirementsWithHashes.txt
@@ -0,0 +1,14 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+# pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in
+#
+attrs==21.2.0 \
+ --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \
+ --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==59.2.0 \
+ --hash=sha256:157d21de9d055ab9e8ea3186d91e7f4f865e11f42deafa952d90842671fc2576 \
+ --hash=sha256:4adde3d1e1c89bde1c643c64d89cdd94cbfd8c75252ee459d4500bccb9c7d05d
diff --git a/lib/manager/pip-compile/__fixtures__/requirementsWithUnknownArguments.txt b/lib/manager/pip-compile/__fixtures__/requirementsWithUnknownArguments.txt
new file mode 100644
index 0000000000..a2f060d334
--- /dev/null
+++ b/lib/manager/pip-compile/__fixtures__/requirementsWithUnknownArguments.txt
@@ -0,0 +1,9 @@
+#
+# This file is autogenerated by pip-compile with python 3.9
+# To update, run:
+#
+# pip-compile --generate-hashes --version requirements.in
+#
+attrs==21.2.0 \
+ --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \
+ --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb
diff --git a/lib/manager/pip-compile/artifacts.spec.ts b/lib/manager/pip-compile/artifacts.spec.ts
index 6b37eb0e07..094a4a1dfc 100644
--- a/lib/manager/pip-compile/artifacts.spec.ts
+++ b/lib/manager/pip-compile/artifacts.spec.ts
@@ -2,9 +2,11 @@ import { exec as _exec } from 'child_process';
import _fs from 'fs-extra';
import { join } from 'upath';
import { envMock, mockExecAll } from '../../../test/exec-util';
+import { Fixtures } from '../../../test/fixtures';
import { git, mocked } from '../../../test/util';
import { GlobalConfig } from '../../config/global';
import type { RepoGlobalConfig } from '../../config/types';
+import { logger } from '../../logger';
import * as docker from '../../util/exec/docker';
import * as _env from '../../util/exec/env';
import type { StatusResult } from '../../util/git/types';
@@ -160,4 +162,58 @@ describe('manager/pip-compile/artifacts', () => {
).not.toBeNull();
expect(execSnapshots).toMatchSnapshot();
});
+
+ describe('constructPipCompileCmd()', () => {
+ it('returns default cmd for garbage', () => {
+ expect(
+ pipCompile.constructPipCompileCmd(
+ Fixtures.get('requirementsNoHeaders.txt'),
+ 'subdir/requirements.in',
+ 'subdir/requirements.txt'
+ )
+ ).toBe('pip-compile requirements.in');
+ });
+
+ it('returns extracted common arguments (like those featured in the README)', () => {
+ expect(
+ pipCompile.constructPipCompileCmd(
+ Fixtures.get('requirementsWithHashes.txt'),
+ 'subdir/requirements.in',
+ 'subdir/requirements.txt'
+ )
+ ).toBe(
+ 'pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in'
+ );
+ });
+
+ it('skips unknown arguments', () => {
+ expect(
+ pipCompile.constructPipCompileCmd(
+ Fixtures.get('requirementsWithUnknownArguments.txt'),
+ 'subdir/requirements.in',
+ 'subdir/requirements.txt'
+ )
+ ).toBe('pip-compile --generate-hashes requirements.in');
+ expect(logger.trace).toHaveBeenCalledWith(
+ { argument: '--version' },
+ 'pip-compile argument is not (yet) supported'
+ );
+ });
+
+ it('skips exploitable subcommands and files', () => {
+ expect(
+ pipCompile.constructPipCompileCmd(
+ Fixtures.get('requirementsWithExploitingArguments.txt'),
+ 'subdir/requirements.in',
+ 'subdir/requirements.txt'
+ )
+ ).toBe(
+ 'pip-compile --generate-hashes --output-file=requirements.txt requirements.in'
+ );
+ expect(logger.warn).toHaveBeenCalledWith(
+ { argument: '--output-file=/etc/shadow' },
+ 'pip-compile was previously executed with an unexpected `--output-file` filename'
+ );
+ });
+ });
});
diff --git a/lib/manager/pip-compile/artifacts.ts b/lib/manager/pip-compile/artifacts.ts
index dd79a9acd2..f2848b7b3c 100644
--- a/lib/manager/pip-compile/artifacts.ts
+++ b/lib/manager/pip-compile/artifacts.ts
@@ -1,5 +1,5 @@
import is from '@sindresorhus/is';
-import { quote } from 'shlex';
+import { quote, split } from 'shlex';
import upath from 'upath';
import { TEMPORARY_ERROR } from '../../constants/error-messages';
import { logger } from '../../logger';
@@ -40,6 +40,47 @@ function getPipToolsConstraint(config: UpdateArtifactsConfig): string {
return '';
}
+const constraintLineRegex = regEx(
+ /^(#.*?\r?\n)+# {4}pip-compile(?<arguments>.*?)\r?\n/
+);
+
+export function constructPipCompileCmd(
+ content: string,
+ inputFileName: string,
+ outputFileName: string
+): string {
+ const headers = constraintLineRegex.exec(content);
+ const args = ['pip-compile'];
+ if (headers) {
+ logger.debug({ header: headers[0] }, 'Found pip-compile header');
+ for (const argument of split(headers.groups.arguments)) {
+ if (['--allow-unsafe', '--generate-hashes'].includes(argument)) {
+ args.push(argument);
+ } else if (argument.startsWith('--output-file=')) {
+ const file = upath.parse(outputFileName).base;
+ if (argument !== `--output-file=${file}`) {
+ // we don't trust the user-supplied output-file argument; use our value here
+ logger.warn(
+ { argument },
+ 'pip-compile was previously executed with an unexpected `--output-file` filename'
+ );
+ }
+ args.push(`--output-file=${file}`);
+ } else if (argument.startsWith('--')) {
+ logger.trace(
+ { argument },
+ 'pip-compile argument is not (yet) supported'
+ );
+ } else {
+ // ignore position argument (.in file)
+ }
+ }
+ }
+ args.push(upath.parse(inputFileName).base);
+
+ return args.map((argument) => quote(argument)).join(' ');
+}
+
export async function updateArtifacts({
packageFileName: inputFileName,
newPackageFileContent: newInputContent,
@@ -59,7 +100,11 @@ export async function updateArtifacts({
if (config.isLockFileMaintenance) {
await deleteLocalFile(outputFileName);
}
- const cmd = `pip-compile ${quote(upath.parse(inputFileName).base)}`;
+ const cmd = constructPipCompileCmd(
+ existingOutput,
+ inputFileName,
+ outputFileName
+ );
const tagConstraint = getPythonConstraint(config);
const pipToolsConstraint = getPipToolsConstraint(config);
const execOptions: ExecOptions = {
diff --git a/lib/manager/pip-compile/readme.md b/lib/manager/pip-compile/readme.md
index 42330b6006..6118ab5303 100644
--- a/lib/manager/pip-compile/readme.md
+++ b/lib/manager/pip-compile/readme.md
@@ -40,3 +40,10 @@ To get Renovate to use another version of Python, add a contraints` rule to the
}
}
```
+
+### `pip-compile` arguments
+
+Renovate reads the `requirements.txt` file and extracts these `pip-compile` arguments:
+
+- `--generate-hashes`
+- `--allow-unsafe`
--
GitLab