From 517de6f545c2095df20ec2fbe378f9a829da56f1 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Thu, 5 Jul 2018 11:33:50 +0200
Subject: [PATCH] feat: repositoryUrls (#2221)
Adds config option repositoryUrls which can be used by pip to define an alternate host to pypi.
Closes #2181
---
lib/config/definitions.js | 10 +++++++
lib/datasource/pypi.js | 11 ++++++--
lib/manager/pip_requirements/extract.js | 14 +++++++++-
.../__snapshots__/pypi.spec.js.snap | 11 ++++++++
test/datasource/pypi.spec.js | 13 ++++++++++
.../__snapshots__/extract.spec.js.snap | 9 +++++++
website/docs/configuration-options.md | 4 +++
website/docs/python.md | 26 +++++++++++++++++++
8 files changed, 95 insertions(+), 3 deletions(-)
diff --git a/lib/config/definitions.js b/lib/config/definitions.js
index bb844f547f..c45e3f149a 100644
--- a/lib/config/definitions.js
+++ b/lib/config/definitions.js
@@ -320,6 +320,16 @@ const options = [
mergeable: true,
cli: false,
},
+ {
+ name: 'registryUrls',
+ description:
+ 'List of URLs to try for dependency lookup. Package manager-specific',
+ type: 'list',
+ default: null,
+ stage: 'package',
+ cli: false,
+ env: false,
+ },
// depType
{
name: 'ignoreDeps',
diff --git a/lib/datasource/pypi.js b/lib/datasource/pypi.js
index 69da412037..9bdda93008 100644
--- a/lib/datasource/pypi.js
+++ b/lib/datasource/pypi.js
@@ -1,15 +1,22 @@
const got = require('got');
const { isVersion, sortVersions } = require('../versioning')('pep440');
+const url = require('url');
+const is = require('@sindresorhus/is');
module.exports = {
getDependency,
};
-async function getDependency(purl) {
+async function getDependency(purl, config = {}) {
const { fullname: depName } = purl;
+ let hostUrl = 'https://pypi.org/pypi/';
+ if (!is.empty(config.registryUrls)) {
+ [hostUrl] = config.registryUrls;
+ }
+ const lookupUrl = url.resolve(hostUrl, `${depName}/json`);
try {
const dependency = {};
- const rep = await got(`https://pypi.org/pypi/${depName}/json`, {
+ const rep = await got(lookupUrl, {
json: true,
});
const dep = rep && rep.body;
diff --git a/lib/manager/pip_requirements/extract.js b/lib/manager/pip_requirements/extract.js
index b7283f4529..cd3e7a3367 100644
--- a/lib/manager/pip_requirements/extract.js
+++ b/lib/manager/pip_requirements/extract.js
@@ -16,6 +16,14 @@ module.exports = {
function extractDependencies(content) {
logger.debug('pip_requirements.extractDependencies()');
+ let registryUrls;
+ content.split('\n').forEach(line => {
+ if (line.startsWith('--index-url ')) {
+ const registryUrl = line.substring('--index-url '.length);
+ registryUrls = [registryUrl];
+ }
+ });
+
const regex = new RegExp(`^(${packagePattern})(${specifierPattern})$`, 'g');
const deps = content
.split('\n')
@@ -26,13 +34,17 @@ function extractDependencies(content) {
return null;
}
const [, depName, currentValue] = matches;
- return {
+ const dep = {
depName,
currentValue,
lineNumber,
purl: 'pkg:pypi/' + depName,
versionScheme: 'pep440',
};
+ if (registryUrls) {
+ dep.registryUrls = registryUrls;
+ }
+ return dep;
})
.filter(Boolean);
if (!deps.length) {
diff --git a/test/datasource/__snapshots__/pypi.spec.js.snap b/test/datasource/__snapshots__/pypi.spec.js.snap
index 32eece8d04..1ef2c12495 100644
--- a/test/datasource/__snapshots__/pypi.spec.js.snap
+++ b/test/datasource/__snapshots__/pypi.spec.js.snap
@@ -102,3 +102,14 @@ Object {
"releases": Array [],
}
`;
+
+exports[`datasource/pypi getDependency supports custom datasource url 1`] = `
+Array [
+ Array [
+ "https://custom.pypi.net/azure-cli-monitor/json",
+ Object {
+ "json": true,
+ },
+ ],
+]
+`;
diff --git a/test/datasource/pypi.spec.js b/test/datasource/pypi.spec.js
index e8bc3da2d5..bc2d62c51c 100644
--- a/test/datasource/pypi.spec.js
+++ b/test/datasource/pypi.spec.js
@@ -8,6 +8,9 @@ const res1 = fs.readFileSync('test/_fixtures/pypi/azure-cli-monitor.json');
describe('datasource/pypi', () => {
describe('getDependency', () => {
+ beforeEach(() => {
+ jest.resetAllMocks();
+ });
it('returns null for empty result', async () => {
got.mockReturnValueOnce({});
expect(await datasource.getDependency('pkg:pypi/something')).toBeNull();
@@ -26,6 +29,16 @@ describe('datasource/pypi', () => {
await datasource.getDependency('pkg:pypi/azure-cli-monitor')
).toMatchSnapshot();
});
+ it('supports custom datasource url', async () => {
+ got.mockReturnValueOnce({
+ body: JSON.parse(res1),
+ });
+ const config = {
+ registryUrls: ['https://custom.pypi.net/foo'],
+ };
+ await datasource.getDependency('pkg:pypi/azure-cli-monitor', config);
+ expect(got.mock.calls).toMatchSnapshot();
+ });
it('returns non-github home_page', async () => {
got.mockReturnValueOnce({
body: {
diff --git a/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap b/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap
index 7680614897..c366526d88 100644
--- a/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap
+++ b/test/manager/pip_requirements/__snapshots__/extract.spec.js.snap
@@ -7,6 +7,9 @@ Array [
"depName": "some-package",
"lineNumber": 2,
"purl": "pkg:pypi/some-package",
+ "registryUrls": Array [
+ "http://example.com/private-pypi/",
+ ],
"versionScheme": "pep440",
},
Object {
@@ -14,6 +17,9 @@ Array [
"depName": "some-other-package",
"lineNumber": 3,
"purl": "pkg:pypi/some-other-package",
+ "registryUrls": Array [
+ "http://example.com/private-pypi/",
+ ],
"versionScheme": "pep440",
},
Object {
@@ -21,6 +27,9 @@ Array [
"depName": "not_semver",
"lineNumber": 4,
"purl": "pkg:pypi/not_semver",
+ "registryUrls": Array [
+ "http://example.com/private-pypi/",
+ ],
"versionScheme": "pep440",
},
]
diff --git a/website/docs/configuration-options.md b/website/docs/configuration-options.md
index 87ad02bd5e..30bd58f7c4 100644
--- a/website/docs/configuration-options.md
+++ b/website/docs/configuration-options.md
@@ -553,6 +553,10 @@ By default, Renovate will detect if it has proposed an update to a project befor
Typically you shouldn't need to modify this setting.
+## registryUrls
+
+This is only necessary in case you need to manually configure a registry URL to use for datasource lookups. Applies to PyPI (pip) only for now. Supports only one URL for now but is defined as a list for forwards compatibility.
+
## renovateFork
By default, Renovate will skip over any repositories that are forked, even if they contain a `renovate.json`, because that config may have been from the source repository. To enable Renovate on forked repositories, you need to add `renovateFork: true` to your renovate config.
diff --git a/website/docs/python.md b/website/docs/python.md
index c05a9033b1..753f3ad7bc 100644
--- a/website/docs/python.md
+++ b/website/docs/python.md
@@ -28,6 +28,32 @@ The default file matching regex for requirements.txt aims to pick up the most po
}
```
+## Alternate registries
+
+Renovate will default to performing all lookups on pypi.org, but it also supports alternative index URLs. There are two ways to achieve this:
+
+#### index-url in `requirements.txt`
+
+The index URL can be specified in the first line of the file, For example:
+
+```
+--index-url http://example.com/private-pypi/
+some-package==0.3.1
+some-other-package==1.0.0
+```
+
+#### Specify URL in configuration
+
+The configuration option `registryUrls` can be used to configure an alternate index URL. Example:
+
+```json
+ "python": {
+ "registryUrls": ["http://example.com/private-pypi/"]
+ }
+```
+
+Note: an index-url found in the `requirements.txt` will take precedent over a registryUrl configured like the above. To override the URL found in `requirements.txt`, you need to configure it in `packageRules`, as they are applied _after_ package file extraction.
+
## Disabling Python Support
The most direct way to disable all Python support in Renovate is like this:
--
GitLab