From 32b414b5e0f85fb24ed1f13f127c58be03f401f9 Mon Sep 17 00:00:00 2001
From: Roald Storm <RmStorm@users.noreply.github.com>
Date: Mon, 15 Jul 2019 22:36:09 +0200
Subject: [PATCH] fix(pip): Add the ability to handle pip's --extra-index-url
(#4056)
* Add the ability to handle pip's --extra-index-url
As specified in the docs here:
https://pip.pypa.io/en/stable/reference/pip_wheel/#extra-index-url
it's possible to define more index-urls in a single package by using
--extra-index-url this was not correctly handled by renovatebot.
---
lib/manager/pip_requirements/extract.js | 28 +++-
.../__snapshots__/extract.spec.js.snap | 156 ++++++++++++++++++
.../_fixtures/requirements5.txt | 10 ++
.../_fixtures/requirements6.txt | 9 +
test/manager/pip_requirements/extract.spec.js | 53 +++++-
5 files changed, 245 insertions(+), 11 deletions(-)
create mode 100644 test/manager/pip_requirements/_fixtures/requirements5.txt
create mode 100644 test/manager/pip_requirements/_fixtures/requirements6.txt
diff --git a/lib/manager/pip_requirements/extract.js b/lib/manager/pip_requirements/extract.js
index 77e480013a..434cfa7973 100644
--- a/lib/manager/pip_requirements/extract.js
+++ b/lib/manager/pip_requirements/extract.js
@@ -17,16 +17,34 @@ module.exports = {
extractPackageFile,
};
-function extractPackageFile(content) {
+function extractPackageFile(content, _, config) {
logger.trace('pip_requirements.extractPackageFile()');
- let registryUrls;
+ let indexUrl;
+ const extraUrls = [];
content.split('\n').forEach(line => {
if (line.startsWith('--index-url ')) {
- const registryUrl = line.substring('--index-url '.length).split(' ')[0];
- registryUrls = [registryUrl];
+ indexUrl = line.substring('--index-url '.length).split(' ')[0];
+ }
+ if (line.startsWith('--extra-index-url ')) {
+ const extraUrl = line
+ .substring('--extra-index-url '.length)
+ .split(' ')[0];
+ extraUrls.push(extraUrl);
}
});
+ let registryUrls = [];
+ if (indexUrl) {
+ // index url in file takes precedence
+ registryUrls.push(indexUrl);
+ } else if (config.registryUrls && config.registryUrls.length) {
+ // configured registryURls takes next precedence
+ registryUrls = registryUrls.concat(config.registryUrls);
+ } else if (extraUrls.length) {
+ // Use default registry first if extra URLs are present and index URL is not
+ registryUrls.push('https://pypi.org/pypi/');
+ }
+ registryUrls = registryUrls.concat(extraUrls);
const regex = new RegExp(`^${dependencyPattern}$`, 'g');
const deps = content
@@ -64,7 +82,7 @@ function extractPackageFile(content) {
return null;
}
const res = { deps };
- if (registryUrls) {
+ if (registryUrls.length > 0) {
res.registryUrls = registryUrls;
}
return res;
diff --git a/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap b/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap
index 5e78df4c1b..f932fa4a6c 100644
--- a/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap
+++ b/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap
@@ -112,6 +112,162 @@ Array [
]
`;
+exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url 1`] = `
+Object {
+ "deps": Array [
+ Object {
+ "currentValue": "==2.0.12",
+ "datasource": "pypi",
+ "depName": "Django",
+ "fromVersion": "2.0.12",
+ "lineNumber": 4,
+ },
+ Object {
+ "currentValue": "==4.1.1",
+ "datasource": "pypi",
+ "depName": "celery",
+ "fromVersion": "4.1.1",
+ "lineNumber": 5,
+ },
+ Object {
+ "currentValue": " == 3.2.1",
+ "datasource": "pypi",
+ "depName": "foo",
+ "lineNumber": 6,
+ },
+ Object {
+ "currentValue": "==0.3.1",
+ "datasource": "pypi",
+ "depName": "some-package",
+ "fromVersion": "0.3.1",
+ "lineNumber": 7,
+ },
+ Object {
+ "currentValue": "==1.0.0",
+ "datasource": "pypi",
+ "depName": "some-other-package",
+ "fromVersion": "1.0.0",
+ "lineNumber": 8,
+ },
+ Object {
+ "currentValue": "==1.9",
+ "datasource": "pypi",
+ "depName": "not_semver",
+ "fromVersion": "1.9",
+ "lineNumber": 9,
+ },
+ ],
+ "registryUrls": Array [
+ "https://artifactory.company.com/artifactory/api/pypi/python/simple",
+ "http://example.com/private-pypi/",
+ ],
+}
+`;
+
+exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url and defaults without index to config 1`] = `
+Object {
+ "deps": Array [
+ Object {
+ "currentValue": "==2.0.12",
+ "datasource": "pypi",
+ "depName": "Django",
+ "fromVersion": "2.0.12",
+ "lineNumber": 3,
+ },
+ Object {
+ "currentValue": "==4.1.1",
+ "datasource": "pypi",
+ "depName": "celery",
+ "fromVersion": "4.1.1",
+ "lineNumber": 4,
+ },
+ Object {
+ "currentValue": " == 3.2.1",
+ "datasource": "pypi",
+ "depName": "foo",
+ "lineNumber": 5,
+ },
+ Object {
+ "currentValue": "==0.3.1",
+ "datasource": "pypi",
+ "depName": "some-package",
+ "fromVersion": "0.3.1",
+ "lineNumber": 6,
+ },
+ Object {
+ "currentValue": "==1.0.0",
+ "datasource": "pypi",
+ "depName": "some-other-package",
+ "fromVersion": "1.0.0",
+ "lineNumber": 7,
+ },
+ Object {
+ "currentValue": "==1.9",
+ "datasource": "pypi",
+ "depName": "not_semver",
+ "fromVersion": "1.9",
+ "lineNumber": 8,
+ },
+ ],
+ "registryUrls": Array [
+ "AnExistingDefaultUrl",
+ "http://example.com/private-pypi/",
+ ],
+}
+`;
+
+exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url and defaults without index to pypi 1`] = `
+Object {
+ "deps": Array [
+ Object {
+ "currentValue": "==2.0.12",
+ "datasource": "pypi",
+ "depName": "Django",
+ "fromVersion": "2.0.12",
+ "lineNumber": 3,
+ },
+ Object {
+ "currentValue": "==4.1.1",
+ "datasource": "pypi",
+ "depName": "celery",
+ "fromVersion": "4.1.1",
+ "lineNumber": 4,
+ },
+ Object {
+ "currentValue": " == 3.2.1",
+ "datasource": "pypi",
+ "depName": "foo",
+ "lineNumber": 5,
+ },
+ Object {
+ "currentValue": "==0.3.1",
+ "datasource": "pypi",
+ "depName": "some-package",
+ "fromVersion": "0.3.1",
+ "lineNumber": 6,
+ },
+ Object {
+ "currentValue": "==1.0.0",
+ "datasource": "pypi",
+ "depName": "some-other-package",
+ "fromVersion": "1.0.0",
+ "lineNumber": 7,
+ },
+ Object {
+ "currentValue": "==1.9",
+ "datasource": "pypi",
+ "depName": "not_semver",
+ "fromVersion": "1.9",
+ "lineNumber": 8,
+ },
+ ],
+ "registryUrls": Array [
+ "https://pypi.org/pypi/",
+ "http://example.com/private-pypi/",
+ ],
+}
+`;
+
exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extras and complex index url 1`] = `
Object {
"deps": Array [
diff --git a/test/manager/pip_requirements/_fixtures/requirements5.txt b/test/manager/pip_requirements/_fixtures/requirements5.txt
new file mode 100644
index 0000000000..de53ecbede
--- /dev/null
+++ b/test/manager/pip_requirements/_fixtures/requirements5.txt
@@ -0,0 +1,10 @@
+# Repositories
+--index-url https://artifactory.company.com/artifactory/api/pypi/python/simple --trusted-host artifactory.company.com --default-timeout 600
+--extra-index-url http://example.com/private-pypi/
+# Packages
+Django[argon2]==2.0.12
+celery [redis]==4.1.1
+foo [bar] == 3.2.1 # handles extra white space
+some-package==0.3.1
+some-other-package==1.0.0
+not_semver==1.9
diff --git a/test/manager/pip_requirements/_fixtures/requirements6.txt b/test/manager/pip_requirements/_fixtures/requirements6.txt
new file mode 100644
index 0000000000..6325d69ac7
--- /dev/null
+++ b/test/manager/pip_requirements/_fixtures/requirements6.txt
@@ -0,0 +1,9 @@
+# Repositories
+--extra-index-url http://example.com/private-pypi/
+# Packages
+Django[argon2]==2.0.12
+celery [redis]==4.1.1
+foo [bar] == 3.2.1 # handles extra white space
+some-package==0.3.1
+some-other-package==1.0.0
+not_semver==1.9
diff --git a/test/manager/pip_requirements/extract.spec.js b/test/manager/pip_requirements/extract.spec.js
index ac357ea81a..e191eb7cf7 100644
--- a/test/manager/pip_requirements/extract.spec.js
+++ b/test/manager/pip_requirements/extract.spec.js
@@ -21,38 +21,79 @@ const requirements4 = fs.readFileSync(
'utf8'
);
+const requirements5 = fs.readFileSync(
+ 'test/manager/pip_requirements/_fixtures/requirements5.txt',
+ 'utf8'
+);
+
+const requirements6 = fs.readFileSync(
+ 'test/manager/pip_requirements/_fixtures/requirements6.txt',
+ 'utf8'
+);
+
describe('lib/manager/pip_requirements/extract', () => {
describe('extractPackageFile()', () => {
let config;
beforeEach(() => {
- config = {};
+ config = { registryUrls: ['AnExistingDefaultUrl'] };
});
it('returns null for empty', () => {
- expect(extractPackageFile('nothing here', config)).toBeNull();
+ expect(
+ extractPackageFile('nothing here', 'requirements.txt', config)
+ ).toBeNull();
});
it('extracts dependencies', () => {
- const res = extractPackageFile(requirements1, config);
+ const res = extractPackageFile(requirements1, 'unused_file_name', config);
expect(res).toMatchSnapshot();
expect(res.registryUrls).toEqual(['http://example.com/private-pypi/']);
expect(res.deps).toHaveLength(3);
});
it('extracts multiple dependencies', () => {
- const res = extractPackageFile(requirements2, config).deps;
+ const res = extractPackageFile(requirements2, 'unused_file_name', config)
+ .deps;
expect(res).toMatchSnapshot();
expect(res).toHaveLength(5);
});
it('handles comments and commands', () => {
- const res = extractPackageFile(requirements3, config).deps;
+ const res = extractPackageFile(requirements3, 'unused_file_name', config)
+ .deps;
expect(res).toMatchSnapshot();
expect(res).toHaveLength(5);
});
it('handles extras and complex index url', () => {
- const res = extractPackageFile(requirements4, config);
+ const res = extractPackageFile(requirements4, 'unused_file_name', config);
expect(res).toMatchSnapshot();
expect(res.registryUrls).toEqual([
'https://artifactory.company.com/artifactory/api/pypi/python/simple',
]);
expect(res.deps).toHaveLength(3);
});
+ it('handles extra index url', () => {
+ const res = extractPackageFile(requirements5, 'unused_file_name', config);
+ expect(res).toMatchSnapshot();
+ expect(res.registryUrls).toEqual([
+ 'https://artifactory.company.com/artifactory/api/pypi/python/simple',
+ 'http://example.com/private-pypi/',
+ ]);
+ expect(res.deps).toHaveLength(6);
+ });
+ it('handles extra index url and defaults without index to config', () => {
+ const res = extractPackageFile(requirements6, 'unused_file_name', config);
+ expect(res).toMatchSnapshot();
+ expect(res.registryUrls).toEqual([
+ 'AnExistingDefaultUrl',
+ 'http://example.com/private-pypi/',
+ ]);
+ expect(res.deps).toHaveLength(6);
+ });
+ it('handles extra index url and defaults without index to pypi', () => {
+ const res = extractPackageFile(requirements6, 'unused_file_name', {});
+ expect(res).toMatchSnapshot();
+ expect(res.registryUrls).toEqual([
+ 'https://pypi.org/pypi/',
+ 'http://example.com/private-pypi/',
+ ]);
+ expect(res.deps).toHaveLength(6);
+ });
});
});
--
GitLab