diff --git a/lib/manager/pip_requirements/extract.js b/lib/manager/pip_requirements/extract.js index 77e480013a0818fd09f784bd7510e405a19bc805..434cfa797343f108b854184c4361422d23b43f3a 100644 --- a/lib/manager/pip_requirements/extract.js +++ b/lib/manager/pip_requirements/extract.js @@ -17,16 +17,34 @@ module.exports = { extractPackageFile, }; -function extractPackageFile(content) { +function extractPackageFile(content, _, config) { logger.trace('pip_requirements.extractPackageFile()'); - let registryUrls; + let indexUrl; + const extraUrls = []; content.split('\n').forEach(line => { if (line.startsWith('--index-url ')) { - const registryUrl = line.substring('--index-url '.length).split(' ')[0]; - registryUrls = [registryUrl]; + indexUrl = line.substring('--index-url '.length).split(' ')[0]; + } + if (line.startsWith('--extra-index-url ')) { + const extraUrl = line + .substring('--extra-index-url '.length) + .split(' ')[0]; + extraUrls.push(extraUrl); } }); + let registryUrls = []; + if (indexUrl) { + // index url in file takes precedence + registryUrls.push(indexUrl); + } else if (config.registryUrls && config.registryUrls.length) { + // configured registryURls takes next precedence + registryUrls = registryUrls.concat(config.registryUrls); + } else if (extraUrls.length) { + // Use default registry first if extra URLs are present and index URL is not + registryUrls.push('https://pypi.org/pypi/'); + } + registryUrls = registryUrls.concat(extraUrls); const regex = new RegExp(`^${dependencyPattern}$`, 'g'); const deps = content @@ -64,7 +82,7 @@ function extractPackageFile(content) { return null; } const res = { deps }; - if (registryUrls) { + if (registryUrls.length > 0) { res.registryUrls = registryUrls; } return res; diff --git a/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap b/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap index 5e78df4c1b79ac95e7429c544f33a159a340d9a9..f932fa4a6c4fa49511f4b9a05b8ea6ad3805afb2 100644 --- a/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap +++ b/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap @@ -112,6 +112,162 @@ Array [ ] `; +exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "==2.0.12", + "datasource": "pypi", + "depName": "Django", + "fromVersion": "2.0.12", + "lineNumber": 4, + }, + Object { + "currentValue": "==4.1.1", + "datasource": "pypi", + "depName": "celery", + "fromVersion": "4.1.1", + "lineNumber": 5, + }, + Object { + "currentValue": " == 3.2.1", + "datasource": "pypi", + "depName": "foo", + "lineNumber": 6, + }, + Object { + "currentValue": "==0.3.1", + "datasource": "pypi", + "depName": "some-package", + "fromVersion": "0.3.1", + "lineNumber": 7, + }, + Object { + "currentValue": "==1.0.0", + "datasource": "pypi", + "depName": "some-other-package", + "fromVersion": "1.0.0", + "lineNumber": 8, + }, + Object { + "currentValue": "==1.9", + "datasource": "pypi", + "depName": "not_semver", + "fromVersion": "1.9", + "lineNumber": 9, + }, + ], + "registryUrls": Array [ + "https://artifactory.company.com/artifactory/api/pypi/python/simple", + "http://example.com/private-pypi/", + ], +} +`; + +exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url and defaults without index to config 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "==2.0.12", + "datasource": "pypi", + "depName": "Django", + "fromVersion": "2.0.12", + "lineNumber": 3, + }, + Object { + "currentValue": "==4.1.1", + "datasource": "pypi", + "depName": "celery", + "fromVersion": "4.1.1", + "lineNumber": 4, + }, + Object { + "currentValue": " == 3.2.1", + "datasource": "pypi", + "depName": "foo", + "lineNumber": 5, + }, + Object { + "currentValue": "==0.3.1", + "datasource": "pypi", + "depName": "some-package", + "fromVersion": "0.3.1", + "lineNumber": 6, + }, + Object { + "currentValue": "==1.0.0", + "datasource": "pypi", + "depName": "some-other-package", + "fromVersion": "1.0.0", + "lineNumber": 7, + }, + Object { + "currentValue": "==1.9", + "datasource": "pypi", + "depName": "not_semver", + "fromVersion": "1.9", + "lineNumber": 8, + }, + ], + "registryUrls": Array [ + "AnExistingDefaultUrl", + "http://example.com/private-pypi/", + ], +} +`; + +exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extra index url and defaults without index to pypi 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": "==2.0.12", + "datasource": "pypi", + "depName": "Django", + "fromVersion": "2.0.12", + "lineNumber": 3, + }, + Object { + "currentValue": "==4.1.1", + "datasource": "pypi", + "depName": "celery", + "fromVersion": "4.1.1", + "lineNumber": 4, + }, + Object { + "currentValue": " == 3.2.1", + "datasource": "pypi", + "depName": "foo", + "lineNumber": 5, + }, + Object { + "currentValue": "==0.3.1", + "datasource": "pypi", + "depName": "some-package", + "fromVersion": "0.3.1", + "lineNumber": 6, + }, + Object { + "currentValue": "==1.0.0", + "datasource": "pypi", + "depName": "some-other-package", + "fromVersion": "1.0.0", + "lineNumber": 7, + }, + Object { + "currentValue": "==1.9", + "datasource": "pypi", + "depName": "not_semver", + "fromVersion": "1.9", + "lineNumber": 8, + }, + ], + "registryUrls": Array [ + "https://pypi.org/pypi/", + "http://example.com/private-pypi/", + ], +} +`; + exports[`lib/manager/pip_requirements/extract extractPackageFile() handles extras and complex index url 1`] = ` Object { "deps": Array [ diff --git a/test/manager/pip_requirements/_fixtures/requirements5.txt b/test/manager/pip_requirements/_fixtures/requirements5.txt new file mode 100644 index 0000000000000000000000000000000000000000..de53ecbede9cd0d98006cc3e29411a293c7915f6 --- /dev/null +++ b/test/manager/pip_requirements/_fixtures/requirements5.txt @@ -0,0 +1,10 @@ +# Repositories +--index-url https://artifactory.company.com/artifactory/api/pypi/python/simple --trusted-host artifactory.company.com --default-timeout 600 +--extra-index-url http://example.com/private-pypi/ +# Packages +Django[argon2]==2.0.12 +celery [redis]==4.1.1 +foo [bar] == 3.2.1 # handles extra white space +some-package==0.3.1 +some-other-package==1.0.0 +not_semver==1.9 diff --git a/test/manager/pip_requirements/_fixtures/requirements6.txt b/test/manager/pip_requirements/_fixtures/requirements6.txt new file mode 100644 index 0000000000000000000000000000000000000000..6325d69ac788ea51393977d3c98f229ff6e80cb0 --- /dev/null +++ b/test/manager/pip_requirements/_fixtures/requirements6.txt @@ -0,0 +1,9 @@ +# Repositories +--extra-index-url http://example.com/private-pypi/ +# Packages +Django[argon2]==2.0.12 +celery [redis]==4.1.1 +foo [bar] == 3.2.1 # handles extra white space +some-package==0.3.1 +some-other-package==1.0.0 +not_semver==1.9 diff --git a/test/manager/pip_requirements/extract.spec.js b/test/manager/pip_requirements/extract.spec.js index ac357ea81a8297839757eb6da3a18291c1e9315d..e191eb7cf76c9b539ecb2b8e4b853a911d212692 100644 --- a/test/manager/pip_requirements/extract.spec.js +++ b/test/manager/pip_requirements/extract.spec.js @@ -21,38 +21,79 @@ const requirements4 = fs.readFileSync( 'utf8' ); +const requirements5 = fs.readFileSync( + 'test/manager/pip_requirements/_fixtures/requirements5.txt', + 'utf8' +); + +const requirements6 = fs.readFileSync( + 'test/manager/pip_requirements/_fixtures/requirements6.txt', + 'utf8' +); + describe('lib/manager/pip_requirements/extract', () => { describe('extractPackageFile()', () => { let config; beforeEach(() => { - config = {}; + config = { registryUrls: ['AnExistingDefaultUrl'] }; }); it('returns null for empty', () => { - expect(extractPackageFile('nothing here', config)).toBeNull(); + expect( + extractPackageFile('nothing here', 'requirements.txt', config) + ).toBeNull(); }); it('extracts dependencies', () => { - const res = extractPackageFile(requirements1, config); + const res = extractPackageFile(requirements1, 'unused_file_name', config); expect(res).toMatchSnapshot(); expect(res.registryUrls).toEqual(['http://example.com/private-pypi/']); expect(res.deps).toHaveLength(3); }); it('extracts multiple dependencies', () => { - const res = extractPackageFile(requirements2, config).deps; + const res = extractPackageFile(requirements2, 'unused_file_name', config) + .deps; expect(res).toMatchSnapshot(); expect(res).toHaveLength(5); }); it('handles comments and commands', () => { - const res = extractPackageFile(requirements3, config).deps; + const res = extractPackageFile(requirements3, 'unused_file_name', config) + .deps; expect(res).toMatchSnapshot(); expect(res).toHaveLength(5); }); it('handles extras and complex index url', () => { - const res = extractPackageFile(requirements4, config); + const res = extractPackageFile(requirements4, 'unused_file_name', config); expect(res).toMatchSnapshot(); expect(res.registryUrls).toEqual([ 'https://artifactory.company.com/artifactory/api/pypi/python/simple', ]); expect(res.deps).toHaveLength(3); }); + it('handles extra index url', () => { + const res = extractPackageFile(requirements5, 'unused_file_name', config); + expect(res).toMatchSnapshot(); + expect(res.registryUrls).toEqual([ + 'https://artifactory.company.com/artifactory/api/pypi/python/simple', + 'http://example.com/private-pypi/', + ]); + expect(res.deps).toHaveLength(6); + }); + it('handles extra index url and defaults without index to config', () => { + const res = extractPackageFile(requirements6, 'unused_file_name', config); + expect(res).toMatchSnapshot(); + expect(res.registryUrls).toEqual([ + 'AnExistingDefaultUrl', + 'http://example.com/private-pypi/', + ]); + expect(res.deps).toHaveLength(6); + }); + it('handles extra index url and defaults without index to pypi', () => { + const res = extractPackageFile(requirements6, 'unused_file_name', {}); + expect(res).toMatchSnapshot(); + expect(res.registryUrls).toEqual([ + 'https://pypi.org/pypi/', + 'http://example.com/private-pypi/', + ]); + expect(res.deps).toHaveLength(6); + }); }); });