From fe40f1ef4862c8ed4b4a987e77fb365859f78748 Mon Sep 17 00:00:00 2001
From: rtaum <rtaum@users.noreply.github.com>
Date: Thu, 7 Feb 2019 08:56:02 +0100
Subject: [PATCH] feat(python): add simple endpoint support (#3125)
Closes #2970
---
lib/datasource/pypi/index.js | 57 +++++++++++++++-
lib/manager/pipenv/extract.js | 4 +-
.../_fixtures/pypi/versions-html-badfile.html | 11 ++++
test/_fixtures/pypi/versions-html.html | 23 +++++++
.../__snapshots__/pypi.spec.js.snap | 58 +++++++++++++---
test/datasource/pypi.spec.js | 66 +++++++++++++++++++
.../pipenv/__snapshots__/extract.spec.js.snap | 18 ++---
test/manager/pipenv/extract.spec.js | 11 ----
8 files changed, 213 insertions(+), 35 deletions(-)
create mode 100644 test/_fixtures/pypi/versions-html-badfile.html
create mode 100644 test/_fixtures/pypi/versions-html.html
diff --git a/lib/datasource/pypi/index.js b/lib/datasource/pypi/index.js
index dae36548a3..124eaefe86 100644
--- a/lib/datasource/pypi/index.js
+++ b/lib/datasource/pypi/index.js
@@ -1,6 +1,6 @@
const url = require('url');
const is = require('@sindresorhus/is');
-
+const { parse } = require('node-html-parser');
const { matches } = require('../../versioning/pep440');
const got = require('../../util/got');
@@ -35,8 +35,14 @@ async function getPkgReleases({ compatibility, lookupName, registryUrls }) {
if (process.env.PIP_INDEX_URL) {
hostUrls = [process.env.PIP_INDEX_URL];
}
- for (const hostUrl of hostUrls) {
- const dep = await getDependency(lookupName, hostUrl, compatibility);
+ for (let hostUrl of hostUrls) {
+ hostUrl += hostUrl.endsWith('/') ? '' : '/';
+ let dep;
+ if (hostUrl.endsWith('/simple/')) {
+ dep = getSimpleDependency(lookupName, hostUrl);
+ } else {
+ dep = await getDependency(lookupName, hostUrl, compatibility);
+ }
if (dep !== null) {
return dep;
}
@@ -91,3 +97,48 @@ async function getDependency(depName, hostUrl, compatibility) {
return null;
}
}
+
+async function getSimpleDependency(depName, hostUrl) {
+ const lookupUrl = url.resolve(hostUrl, `${depName}`);
+ try {
+ const dependency = {};
+ const response = await got(url.parse(lookupUrl), {
+ json: false,
+ });
+ const dep = response && response.body;
+ if (!dep) {
+ logger.debug({ dependency: depName }, 'pip package not found');
+ return null;
+ }
+ const root = parse(dep);
+ const links = root.querySelectorAll('a');
+ const versions = new Set();
+ for (const link of links) {
+ const result = extractVersionFromLinkText(link.text);
+ if (result) {
+ versions.add(result);
+ }
+ }
+ dependency.releases = [];
+ if (versions && versions.size > 0) {
+ dependency.releases = [...versions].map(version => ({
+ version,
+ }));
+ }
+ return dependency;
+ } catch (err) {
+ logger.info(
+ 'pypi dependency not found: ' + depName + '(searching in ' + hostUrl + ')'
+ );
+ return null;
+ }
+}
+
+function extractVersionFromLinkText(text) {
+ const versionRegexp = /\d+(\.\d+)+/;
+ const result = text.match(versionRegexp);
+ if (result && result.length > 0) {
+ return result[0];
+ }
+ return null;
+}
diff --git a/lib/manager/pipenv/extract.js b/lib/manager/pipenv/extract.js
index 17414ab15b..5f6840ce28 100644
--- a/lib/manager/pipenv/extract.js
+++ b/lib/manager/pipenv/extract.js
@@ -25,9 +25,7 @@ function extractPackageFile(content) {
}
let registryUrls;
if (pipfile.source) {
- registryUrls = pipfile.source.map(source =>
- source.url.replace(/simple(\/)?$/, 'pypi/')
- );
+ registryUrls = pipfile.source.map(source => source.url);
}
const deps = [
diff --git a/test/_fixtures/pypi/versions-html-badfile.html b/test/_fixtures/pypi/versions-html-badfile.html
new file mode 100644
index 0000000000..a2155662d8
--- /dev/null
+++ b/test/_fixtures/pypi/versions-html-badfile.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>Links for dj-database-url</title>
+ </head>
+ <body>
+ <h1>Links for dj-database-url</h1>
+ <a href="https://files.pythonhosted.org/packages/04/89/29cdbc86a0890a4f1e46b6f4bb9b7959e461e0202f6a305bd8b586cc1404/dj-database-url-0.1.2.tar.gz#sha256=6169f2c272326e3cced6999effb19013365ea73f6ed6c731efa4e346711d8969">dj-database-url.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/bd/80/f8430a065c09367cd766cdea08f80d11b625944a653f96c2bd02d183355b/dj-database-url-0.1.3.tar.gz#sha256=222744896dcbe939aa940217c940a8a95981be13beb9af639a1da024be4f9411">dj-database-url.tar.gz</a><br/>
+ </body>
+</html>
diff --git a/test/_fixtures/pypi/versions-html.html b/test/_fixtures/pypi/versions-html.html
new file mode 100644
index 0000000000..4872820cc8
--- /dev/null
+++ b/test/_fixtures/pypi/versions-html.html
@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>Links for dj-database-url</title>
+ </head>
+ <body>
+ <h1>Links for dj-database-url</h1>
+ <a href="https://files.pythonhosted.org/packages/04/89/29cdbc86a0890a4f1e46b6f4bb9b7959e461e0202f6a305bd8b586cc1404/dj-database-url-0.1.2.tar.gz#sha256=6169f2c272326e3cced6999effb19013365ea73f6ed6c731efa4e346711d8969">dj-database-url-0.1.2.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/bd/80/f8430a065c09367cd766cdea08f80d11b625944a653f96c2bd02d183355b/dj-database-url-0.1.3.tar.gz#sha256=222744896dcbe939aa940217c940a8a95981be13beb9af639a1da024be4f9411">dj-database-url-0.1.3.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/3e/78/c18103be8b9f06a3cb9ee93adb63f91c16f8c8781c8abb0e5b06acef7106/dj-database-url-0.1.4.tar.gz#sha256=14faa143247e267aefd807490e1e89e3ad9fac0a06c4aee3f9fe328849bd15cf">dj-database-url-0.1.4.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/d6/cb/ab7fc10ea1e3571c34bcd74c3b3090e2d3c378d01ad79550fc967cd84114/dj-database-url-0.2.0.tar.gz#sha256=5edd253ccb407a0bd19e91c4c9bbe164632639767086b4a38f2d20e00010488b">dj-database-url-0.2.0.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/a6/94/72572715f45dd132ddc1e26aa4e8c2b13395759386a541e7f00124ceda11/dj-database-url-0.2.1.tar.gz#sha256=f95c0b2e9e70cc246bd101720e1be492524ecf0dd5ea39241b51ef142faefecc">dj-database-url-0.2.1.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/19/11/2867ccdaa0203ed14d9f725b26b6dd3264c4209a16d5bf0096597ecf9a7a/dj-database-url-0.2.2.tar.gz#sha256=492a7294b85ad8ac1b13be0b7337f381d2d44c4da185f289ab7c26dd765ef6cb">dj-database-url-0.2.2.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/e1/0e/2cceb7afb13cf784e385928530af59b49dd1524a76323e293f18ea28a6de/dj-database-url-0.3.0.tar.gz#sha256=f2e273ed34acbb560962d5cf12917936d8df02297df09bd3089b8546d4584138">dj-database-url-0.3.0.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/ef/b6/9283fcf61ced22bf90e7b4a84ba5b53d126b2c9b0dc9b667347698097026/dj_database_url-0.3.0-py2.py3-none-any.whl#sha256=ca01768fdecde134301f3170743226f60edff5c3935f12437378ebd911506353">dj_database_url-0.3.0-py2.py3-none-any.whl</a><br/>
+ <a href="https://files.pythonhosted.org/packages/64/d9/99774e3f66683ded1d3aa3f66045f671cefc0b550aca4ccfeaeeed4e074a/dj-database-url-0.4.0.tar.gz#sha256=858312abb7b330ea875733a65806a36ad04d7b8451c6ce8835118a2fa10d6870">dj-database-url-0.4.0.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/39/9f/30f937db9f9e7a4e4e3205682af4c34c65d647ff9850897ddfbbf5dc6178/dj-database-url-0.4.1.tar.gz#sha256=7f4c78d2a090df8dfaf56d5d3ff7bbee17360436e4879558317e2314424864cd">dj-database-url-0.4.1.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/c8/4b/b23dbcf4c5711f26e2222bb2e300915c9c8d35e643b0af00c2d8f36c9490/dj-database-url-0.4.2.tar.gz#sha256=a6832d8445ee9d788c5baa48aef8130bf61fdc442f7d9a548424d25cd85c9f08">dj-database-url-0.4.2.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/91/84/50cbfabb91593cff18a37046986f7c2eb69224a694a52ae614711dfa11c6/dj_database_url-0.4.2-py2.py3-none-any.whl#sha256=e16d94c382ea0564c48038fa7fe8d9c890ef1ab1a8ec4cb48e732c124b9482fd">dj_database_url-0.4.2-py2.py3-none-any.whl</a><br/>
+ <a href="https://files.pythonhosted.org/packages/01/c4/98fbf678e810029be8078419f7bba626aafa2e81bc38748757db954c477c/dj-database-url-0.5.0.tar.gz#sha256=4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163">dj-database-url-0.5.0.tar.gz</a><br/>
+ <a href="https://files.pythonhosted.org/packages/d4/a6/4b8578c1848690d0c307c7c0596af2077536c9ef2a04d42b00fabaa7e49d/dj_database_url-0.5.0-py2.py3-none-any.whl#sha256=851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9">dj_database_url-0.5.0-py2.py3-none-any.whl</a><br/>
+ </body>
+</html>
diff --git a/test/datasource/__snapshots__/pypi.spec.js.snap b/test/datasource/__snapshots__/pypi.spec.js.snap
index 4f248bf390..27ee1735f2 100644
--- a/test/datasource/__snapshots__/pypi.spec.js.snap
+++ b/test/datasource/__snapshots__/pypi.spec.js.snap
@@ -1,5 +1,45 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
+exports[`datasource/pypi getPkgReleases process data from simple endpoint 1`] = `
+Object {
+ "releases": Array [
+ Object {
+ "version": "0.1.2",
+ },
+ Object {
+ "version": "0.1.3",
+ },
+ Object {
+ "version": "0.1.4",
+ },
+ Object {
+ "version": "0.2.0",
+ },
+ Object {
+ "version": "0.2.1",
+ },
+ Object {
+ "version": "0.2.2",
+ },
+ Object {
+ "version": "0.3.0",
+ },
+ Object {
+ "version": "0.4.0",
+ },
+ Object {
+ "version": "0.4.1",
+ },
+ Object {
+ "version": "0.4.2",
+ },
+ Object {
+ "version": "0.5.0",
+ },
+ ],
+}
+`;
+
exports[`datasource/pypi getPkgReleases processes real data 1`] = `
Object {
"releases": Array [
@@ -130,9 +170,9 @@ Array [
"hash": null,
"host": "custom.pypi.net",
"hostname": "custom.pypi.net",
- "href": "https://custom.pypi.net/azure-cli-monitor/json",
- "path": "/azure-cli-monitor/json",
- "pathname": "/azure-cli-monitor/json",
+ "href": "https://custom.pypi.net/foo/azure-cli-monitor/json",
+ "path": "/foo/azure-cli-monitor/json",
+ "pathname": "/foo/azure-cli-monitor/json",
"port": null,
"protocol": "https:",
"query": null,
@@ -178,9 +218,9 @@ Array [
"hash": null,
"host": "custom.pypi.net",
"hostname": "custom.pypi.net",
- "href": "https://custom.pypi.net/azure-cli-monitor/json",
- "path": "/azure-cli-monitor/json",
- "pathname": "/azure-cli-monitor/json",
+ "href": "https://custom.pypi.net/foo/azure-cli-monitor/json",
+ "path": "/foo/azure-cli-monitor/json",
+ "pathname": "/foo/azure-cli-monitor/json",
"port": null,
"protocol": "https:",
"query": null,
@@ -197,9 +237,9 @@ Array [
"hash": null,
"host": "second-index",
"hostname": "second-index",
- "href": "https://second-index/azure-cli-monitor/json",
- "path": "/azure-cli-monitor/json",
- "pathname": "/azure-cli-monitor/json",
+ "href": "https://second-index/foo/azure-cli-monitor/json",
+ "path": "/foo/azure-cli-monitor/json",
+ "pathname": "/foo/azure-cli-monitor/json",
"port": null,
"protocol": "https:",
"query": null,
diff --git a/test/datasource/pypi.spec.js b/test/datasource/pypi.spec.js
index d9cb78702a..86701a6dcf 100644
--- a/test/datasource/pypi.spec.js
+++ b/test/datasource/pypi.spec.js
@@ -5,6 +5,10 @@ const datasource = require('../../lib/datasource');
jest.mock('../../lib/util/got');
const res1 = fs.readFileSync('test/_fixtures/pypi/azure-cli-monitor.json');
+const htmlResponse = fs.readFileSync('test/_fixtures/pypi/versions-html.html');
+const badResponse = fs.readFileSync(
+ 'test/_fixtures/pypi/versions-html-badfile.html'
+);
describe('datasource/pypi', () => {
describe('getPkgReleases', () => {
@@ -160,5 +164,67 @@ describe('datasource/pypi', () => {
})
).toMatchSnapshot();
});
+ it('process data from simple endpoint', async () => {
+ got.mockReturnValueOnce({
+ body: htmlResponse + '',
+ });
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ expect(
+ await datasource.getPkgReleases({
+ ...config,
+ compatibility: { python: '2.7' },
+ datasource: 'pypi',
+ depName: 'dj-database-url',
+ })
+ ).toMatchSnapshot();
+ });
+ it('returns null for empty resonse', async () => {
+ got.mockReturnValueOnce({});
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ expect(
+ await datasource.getPkgReleases({
+ ...config,
+ compatibility: { python: '2.7' },
+ datasource: 'pypi',
+ depName: 'dj-database-url',
+ })
+ ).toBeNull();
+ });
+ it('returns null for 404 response from simple endpoint', async () => {
+ got.mockImplementationOnce(() => {
+ throw new Error();
+ });
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ expect(
+ await datasource.getPkgReleases({
+ ...config,
+ compatibility: { python: '2.7' },
+ datasource: 'pypi',
+ depName: 'dj-database-url',
+ })
+ ).toBeNull();
+ });
+ it('returns null for response with no versions', async () => {
+ got.mockReturnValueOnce({
+ body: badResponse + '',
+ });
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ expect(
+ await datasource.getPkgReleases({
+ ...config,
+ compatibility: { python: '2.7' },
+ datasource: 'pypi',
+ depName: 'dj-database-url',
+ })
+ ).toEqual({ releases: [] });
+ });
});
});
diff --git a/test/manager/pipenv/__snapshots__/extract.spec.js.snap b/test/manager/pipenv/__snapshots__/extract.spec.js.snap
index e19291857d..40c46af618 100644
--- a/test/manager/pipenv/__snapshots__/extract.spec.js.snap
+++ b/test/manager/pipenv/__snapshots__/extract.spec.js.snap
@@ -9,7 +9,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
"http://example.com/private-pypi/",
],
},
@@ -20,7 +20,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
"http://example.com/private-pypi/",
],
},
@@ -31,7 +31,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": true,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
"http://example.com/private-pypi/",
],
},
@@ -42,7 +42,7 @@ Array [
"depType": "dev-packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
"http://example.com/private-pypi/",
],
},
@@ -58,7 +58,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
],
},
Object {
@@ -68,7 +68,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
],
},
Object {
@@ -78,7 +78,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
],
},
Object {
@@ -88,7 +88,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
],
},
Object {
@@ -98,7 +98,7 @@ Array [
"depType": "packages",
"pipenvNestedVersion": false,
"registryUrls": Array [
- "https://pypi.org/pypi/",
+ "https://pypi.org/simple",
],
},
]
diff --git a/test/manager/pipenv/extract.spec.js b/test/manager/pipenv/extract.spec.js
index b4d9180ce4..578515708a 100644
--- a/test/manager/pipenv/extract.spec.js
+++ b/test/manager/pipenv/extract.spec.js
@@ -44,16 +44,5 @@ describe('lib/manager/pipenv/extract', () => {
const res = extractPackageFile(content, config).deps;
expect(res[0].registryUrls).toEqual(['source-url', 'other-source-url']);
});
- it('converts simple-API URLs to JSON-API URLs', () => {
- const content =
- '[[source]]\r\nurl = "https://my-pypi/foo/simple/"\r\n' +
- '[[source]]\r\nurl = "https://other-pypi/foo/simple"\r\n' +
- '[packages]\r\nfoo = "==1.0.0"\r\n';
- const res = extractPackageFile(content, config).deps;
- expect(res[0].registryUrls).toEqual([
- 'https://my-pypi/foo/pypi/',
- 'https://other-pypi/foo/pypi/',
- ]);
- });
});
});
--
GitLab