fix(pip): Add the ability to handle pip's --extra-index-url (#4056)

* Add the ability to handle pip's --extra-index-url

As specified in the docs here:
https://pip.pypa.io/en/stable/reference/pip_wheel/#extra-index-url
it's possible to define more index-urls in a single package by using
--extra-index-url this was not correctly handled by renovatebot.

lib/manager/pip_requirements/extract.js

@@ -17,16 +17,34 @@ module.exports = {
   extractPackageFile,
 };
 
-function extractPackageFile(content) {
+function extractPackageFile(content, _, config) {
   logger.trace('pip_requirements.extractPackageFile()');
 
-  let registryUrls;
+  let indexUrl;
+  const extraUrls = [];
   content.split('\n').forEach(line => {
     if (line.startsWith('--index-url ')) {
-      const registryUrl = line.substring('--index-url '.length).split(' ')[0];
-      registryUrls = [registryUrl];
+      indexUrl = line.substring('--index-url '.length).split(' ')[0];
+    }
+    if (line.startsWith('--extra-index-url ')) {
+      const extraUrl = line
+        .substring('--extra-index-url '.length)
+        .split(' ')[0];
+      extraUrls.push(extraUrl);
     }
   });
+  let registryUrls = [];
+  if (indexUrl) {
+    // index url in file takes precedence
+    registryUrls.push(indexUrl);
+  } else if (config.registryUrls && config.registryUrls.length) {
+    // configured registryURls takes next precedence
+    registryUrls = registryUrls.concat(config.registryUrls);
+  } else if (extraUrls.length) {
+    // Use default registry first if extra URLs are present and index URL is not
+    registryUrls.push('https://pypi.org/pypi/');
+  }
+  registryUrls = registryUrls.concat(extraUrls);
 
   const regex = new RegExp(`^${dependencyPattern}$`, 'g');
   const deps = content
@@ -64,7 +82,7 @@ function extractPackageFile(content) {
     return null;
   }
   const res = { deps };
-  if (registryUrls) {
+  if (registryUrls.length > 0) {
     res.registryUrls = registryUrls;
   }
   return res;

test/manager/pip_requirements/__snapshots__/extract.spec.js.snap

@@ -112,6 +112,162 @@ Array [
 ]
 `;
 
+exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "==2.0.12",
+      "datasource": "pypi",
+      "depName": "Django",
+      "fromVersion": "2.0.12",
+      "lineNumber": 4,
+    },
+    Object {
+      "currentValue": "==4.1.1",
+      "datasource": "pypi",
+      "depName": "celery",
+      "fromVersion": "4.1.1",
+      "lineNumber": 5,
+    },
+    Object {
+      "currentValue": " == 3.2.1",
+      "datasource": "pypi",
+      "depName": "foo",
+      "lineNumber": 6,
+    },
+    Object {
+      "currentValue": "==0.3.1",
+      "datasource": "pypi",
+      "depName": "some-package",
+      "fromVersion": "0.3.1",
+      "lineNumber": 7,
+    },
+    Object {
+      "currentValue": "==1.0.0",
+      "datasource": "pypi",
+      "depName": "some-other-package",
+      "fromVersion": "1.0.0",
+      "lineNumber": 8,
+    },
+    Object {
+      "currentValue": "==1.9",
+      "datasource": "pypi",
+      "depName": "not_semver",
+      "fromVersion": "1.9",
+      "lineNumber": 9,
+    },
+  ],
+  "registryUrls": Array [
+    "https://artifactory.company.com/artifactory/api/pypi/python/simple",
+    "http://example.com/private-pypi/",
+  ],
+}
+`;
+
+exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url and defaults without index to config 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "==2.0.12",
+      "datasource": "pypi",
+      "depName": "Django",
+      "fromVersion": "2.0.12",
+      "lineNumber": 3,
+    },
+    Object {
+      "currentValue": "==4.1.1",
+      "datasource": "pypi",
+      "depName": "celery",
+      "fromVersion": "4.1.1",
+      "lineNumber": 4,
+    },
+    Object {
+      "currentValue": " == 3.2.1",
+      "datasource": "pypi",
+      "depName": "foo",
+      "lineNumber": 5,
+    },
+    Object {
+      "currentValue": "==0.3.1",
+      "datasource": "pypi",
+      "depName": "some-package",
+      "fromVersion": "0.3.1",
+      "lineNumber": 6,
+    },
+    Object {
+      "currentValue": "==1.0.0",
+      "datasource": "pypi",
+      "depName": "some-other-package",
+      "fromVersion": "1.0.0",
+      "lineNumber": 7,
+    },
+    Object {
+      "currentValue": "==1.9",
+      "datasource": "pypi",
+      "depName": "not_semver",
+      "fromVersion": "1.9",
+      "lineNumber": 8,
+    },
+  ],
+  "registryUrls": Array [
+    "AnExistingDefaultUrl",
+    "http://example.com/private-pypi/",
+  ],
+}
+`;
+
+exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url and defaults without index to pypi 1`] = `
+Object {
+  "deps": Array [
+    Object {
+      "currentValue": "==2.0.12",
+      "datasource": "pypi",
+      "depName": "Django",
+      "fromVersion": "2.0.12",
+      "lineNumber": 3,
+    },
+    Object {
+      "currentValue": "==4.1.1",
+      "datasource": "pypi",
+      "depName": "celery",
+      "fromVersion": "4.1.1",
+      "lineNumber": 4,
+    },
+    Object {
+      "currentValue": " == 3.2.1",
+      "datasource": "pypi",
+      "depName": "foo",
+      "lineNumber": 5,
+    },
+    Object {
+      "currentValue": "==0.3.1",
+      "datasource": "pypi",
+      "depName": "some-package",
+      "fromVersion": "0.3.1",
+      "lineNumber": 6,
+    },
+    Object {
+      "currentValue": "==1.0.0",
+      "datasource": "pypi",
+      "depName": "some-other-package",
+      "fromVersion": "1.0.0",
+      "lineNumber": 7,
+    },
+    Object {
+      "currentValue": "==1.9",
+      "datasource": "pypi",
+      "depName": "not_semver",
+      "fromVersion": "1.9",
+      "lineNumber": 8,
+    },
+  ],
+  "registryUrls": Array [
+    "https://pypi.org/pypi/",
+    "http://example.com/private-pypi/",
+  ],
+}
+`;
+
 exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extras and complex index url 1`] = `
 Object {
   "deps": Array [

test/manager/pip_requirements/_fixtures/requirements5.txt

+# Repositories
+--index-url https://artifactory.company.com/artifactory/api/pypi/python/simple --trusted-host artifactory.company.com --default-timeout 600
+--extra-index-url http://example.com/private-pypi/
+# Packages
+Django[argon2]==2.0.12
+celery [redis]==4.1.1
+foo [bar] == 3.2.1 # handles extra white space
+some-package==0.3.1
+some-other-package==1.0.0
+not_semver==1.9

test/manager/pip_requirements/_fixtures/requirements6.txt

+# Repositories
+--extra-index-url http://example.com/private-pypi/
+# Packages
+Django[argon2]==2.0.12
+celery [redis]==4.1.1
+foo [bar] == 3.2.1 # handles extra white space
+some-package==0.3.1
+some-other-package==1.0.0
+not_semver==1.9

test/manager/pip_requirements/extract.spec.js

@@ -21,38 +21,79 @@ const requirements4 = fs.readFileSync(
   'utf8'
 );
 
+const requirements5 = fs.readFileSync(
+  'test/manager/pip_requirements/_fixtures/requirements5.txt',
+  'utf8'
+);
+
+const requirements6 = fs.readFileSync(
+  'test/manager/pip_requirements/_fixtures/requirements6.txt',
+  'utf8'
+);
+
 describe('lib/manager/pip_requirements/extract', () => {
   describe('extractPackageFile()', () => {
     let config;
     beforeEach(() => {
-      config = {};
+      config = { registryUrls: ['AnExistingDefaultUrl'] };
     });
     it('returns null for empty', () => {
-      expect(extractPackageFile('nothing here', config)).toBeNull();
+      expect(
+        extractPackageFile('nothing here', 'requirements.txt', config)
+      ).toBeNull();
     });
     it('extracts dependencies', () => {
-      const res = extractPackageFile(requirements1, config);
+      const res = extractPackageFile(requirements1, 'unused_file_name', config);
       expect(res).toMatchSnapshot();
       expect(res.registryUrls).toEqual(['http://example.com/private-pypi/']);
       expect(res.deps).toHaveLength(3);
     });
     it('extracts multiple dependencies', () => {
-      const res = extractPackageFile(requirements2, config).deps;
+      const res = extractPackageFile(requirements2, 'unused_file_name', config)
+        .deps;
       expect(res).toMatchSnapshot();
       expect(res).toHaveLength(5);
     });
     it('handles comments and commands', () => {
-      const res = extractPackageFile(requirements3, config).deps;
+      const res = extractPackageFile(requirements3, 'unused_file_name', config)
+        .deps;
       expect(res).toMatchSnapshot();
       expect(res).toHaveLength(5);
     });
     it('handles extras and complex index url', () => {
-      const res = extractPackageFile(requirements4, config);
+      const res = extractPackageFile(requirements4, 'unused_file_name', config);
       expect(res).toMatchSnapshot();
       expect(res.registryUrls).toEqual([
         'https://artifactory.company.com/artifactory/api/pypi/python/simple',
       ]);
       expect(res.deps).toHaveLength(3);
     });
+    it('handles extra index url', () => {
+      const res = extractPackageFile(requirements5, 'unused_file_name', config);
+      expect(res).toMatchSnapshot();
+      expect(res.registryUrls).toEqual([
+        'https://artifactory.company.com/artifactory/api/pypi/python/simple',
+        'http://example.com/private-pypi/',
+      ]);
+      expect(res.deps).toHaveLength(6);
+    });
+    it('handles extra index url and defaults without index to config', () => {
+      const res = extractPackageFile(requirements6, 'unused_file_name', config);
+      expect(res).toMatchSnapshot();
+      expect(res.registryUrls).toEqual([
+        'AnExistingDefaultUrl',
+        'http://example.com/private-pypi/',
+      ]);
+      expect(res.deps).toHaveLength(6);
+    });
+    it('handles extra index url and defaults without index to pypi', () => {
+      const res = extractPackageFile(requirements6, 'unused_file_name', {});
+      expect(res).toMatchSnapshot();
+      expect(res.registryUrls).toEqual([
+        'https://pypi.org/pypi/',
+        'http://example.com/private-pypi/',
+      ]);
+      expect(res.deps).toHaveLength(6);
+    });
   });
 });