From 2d1c9dfb709c96a2d55a687c31053838a6081740 Mon Sep 17 00:00:00 2001
From: tgroemer <80391116+tgroemer@users.noreply.github.com>
Date: Wed, 17 Nov 2021 17:02:39 +0100
Subject: [PATCH] fix(pypi): normalize simple package lookup (#12544)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
.../__fixtures__/versions-html-hyphens.html | 12 +++
.../versions-html-mixed-case.html | 12 +++
.../versions-html-with-periods.html | 12 +++
lib/datasource/pypi/index.spec.ts | 79 +++++++++++++++++++
lib/datasource/pypi/index.ts | 38 +++++----
5 files changed, 138 insertions(+), 15 deletions(-)
create mode 100644 lib/datasource/pypi/__fixtures__/versions-html-hyphens.html
create mode 100644 lib/datasource/pypi/__fixtures__/versions-html-mixed-case.html
create mode 100644 lib/datasource/pypi/__fixtures__/versions-html-with-periods.html
diff --git a/lib/datasource/pypi/__fixtures__/versions-html-hyphens.html b/lib/datasource/pypi/__fixtures__/versions-html-hyphens.html
new file mode 100644
index 0000000000..64a958fd34
--- /dev/null
+++ b/lib/datasource/pypi/__fixtures__/versions-html-hyphens.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html><head>
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
+ <title>Links for package--with-hyphens</title>
+ </head>
+ <body>
+ <h1>Links for package--with-hyphens</h1>
+ <a href="">package--with-hyphens-2.0.0.tar.gz</a><br>
+ <a href="">package_with_hyphens-2.0.1-py3-none-any.whl</a><br>
+ <a href="">package_with_hyphens-2.0.2-py3-none-any.whl</a><br>
+ <a href="">package--with-hyphens-2.0.2.tar.gz</a><br>
+</body></html>
\ No newline at end of file
diff --git a/lib/datasource/pypi/__fixtures__/versions-html-mixed-case.html b/lib/datasource/pypi/__fixtures__/versions-html-mixed-case.html
new file mode 100644
index 0000000000..a295006601
--- /dev/null
+++ b/lib/datasource/pypi/__fixtures__/versions-html-mixed-case.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html><head>
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
+ <title>Links for PackageWithMixedCase</title>
+ </head>
+ <body>
+ <h1>Links for PackageWithMixedCase</h1>
+ <a href="">PackageWithMixedCase-2.0.0.tar.gz</a><br>
+ <a href="">PackageWithMixedCase-2.0.1-py3-none-any.whl</a><br>
+ <a href="">PackageWithMixedCase-2.0.2-py3-none-any.whl</a><br>
+ <a href="">PackageWithMixedCase-2.0.2.tar.gz</a><br>
+</body></html>
\ No newline at end of file
diff --git a/lib/datasource/pypi/__fixtures__/versions-html-with-periods.html b/lib/datasource/pypi/__fixtures__/versions-html-with-periods.html
new file mode 100644
index 0000000000..846d72d71b
--- /dev/null
+++ b/lib/datasource/pypi/__fixtures__/versions-html-with-periods.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html><head>
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
+ <title>Links for package.with.periods</title>
+ </head>
+ <body>
+ <h1>Links for package.with.periods</h1>
+ <a href="">package.with.periods-2.0.0.tar.gz</a><br>
+ <a href="">package.with.periods-2.0.1-py3-none-any.whl</a><br>
+ <a href="">package.with.periods-2.0.2-py3-none-any.whl</a><br>
+ <a href="">package.with.periods-2.0.2.tar.gz</a><br>
+</body></html>
\ No newline at end of file
diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts
index 8f919d9122..1e95cb1983 100644
--- a/lib/datasource/pypi/index.spec.ts
+++ b/lib/datasource/pypi/index.spec.ts
@@ -12,6 +12,9 @@ const dataRequiresPythonResponse = loadFixture(
'versions-html-data-requires-python.html'
);
const mixedHyphensResponse = loadFixture('versions-html-mixed-hyphens.html');
+const mixedCaseResponse = loadFixture('versions-html-mixed-case.html');
+const withPeriodsResponse = loadFixture('versions-html-with-periods.html');
+const hyphensResponse = loadFixture('versions-html-hyphens.html');
const baseUrl = 'https://pypi.org/pypi';
const datasource = PypiDatasource.id;
@@ -304,6 +307,25 @@ describe('datasource/pypi/index', () => {
});
expect(res.isPrivate).toBeTrue();
});
+ it('process data from simple endpoint with hyphens', async () => {
+ httpMock
+ .scope('https://pypi.org/simple/')
+ .get('/package-with-hyphens/')
+ .reply(200, hyphensResponse);
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ const res = await getPkgReleases({
+ datasource,
+ ...config,
+ depName: 'package--with-hyphens',
+ });
+ expect(res.releases).toMatchObject([
+ { version: '2.0.0' },
+ { version: '2.0.1' },
+ { version: '2.0.2' },
+ ]);
+ });
it('process data from simple endpoint with hyphens replaced with underscores', async () => {
httpMock
.scope('https://pypi.org/simple/')
@@ -322,6 +344,63 @@ describe('datasource/pypi/index', () => {
).toMatchSnapshot();
expect(httpMock.getTrace()).toMatchSnapshot();
});
+ it('process data from simple endpoint with mixed-case characters', async () => {
+ httpMock
+ .scope('https://pypi.org/simple/')
+ .get('/packagewithmixedcase/')
+ .reply(200, mixedCaseResponse);
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ const res = await getPkgReleases({
+ datasource,
+ ...config,
+ depName: 'PackageWithMixedCase',
+ });
+ expect(res.releases).toMatchObject([
+ { version: '2.0.0' },
+ { version: '2.0.1' },
+ { version: '2.0.2' },
+ ]);
+ });
+ it('process data from simple endpoint with mixed-case characters when using lower case dependency name', async () => {
+ httpMock
+ .scope('https://pypi.org/simple/')
+ .get('/packagewithmixedcase/')
+ .reply(200, mixedCaseResponse);
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ const res = await getPkgReleases({
+ datasource,
+ ...config,
+ depName: 'packagewithmixedcase',
+ });
+ expect(res.releases).toMatchObject([
+ { version: '2.0.0' },
+ { version: '2.0.1' },
+ { version: '2.0.2' },
+ ]);
+ });
+ it('process data from simple endpoint with periods', async () => {
+ httpMock
+ .scope('https://pypi.org/simple/')
+ .get('/package-with-periods/')
+ .reply(200, withPeriodsResponse);
+ const config = {
+ registryUrls: ['https://pypi.org/simple/'],
+ };
+ const res = await getPkgReleases({
+ datasource,
+ ...config,
+ depName: 'package.with.periods',
+ });
+ expect(res.releases).toMatchObject([
+ { version: '2.0.0' },
+ { version: '2.0.1' },
+ { version: '2.0.2' },
+ ]);
+ });
it('returns null for empty response', async () => {
httpMock
.scope('https://pypi.org/simple/')
diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts
index bd143aec39..e270568660 100644
--- a/lib/datasource/pypi/index.ts
+++ b/lib/datasource/pypi/index.ts
@@ -73,6 +73,10 @@ export class PypiDatasource extends Datasource {
}
private static normalizeName(input: string): string {
+ return input.toLowerCase().replace(regEx(/_/g), '-');
+ }
+
+ private static normalizeNameForUrlLookup(input: string): string {
return input.toLowerCase().replace(regEx(/(_|\.|-)+/g), '-');
}
@@ -80,7 +84,10 @@ export class PypiDatasource extends Datasource {
packageName: string,
hostUrl: string
): Promise<ReleaseResult | null> {
- const lookupUrl = url.resolve(hostUrl, `${packageName}/json`);
+ const lookupUrl = url.resolve(
+ hostUrl,
+ `${PypiDatasource.normalizeNameForUrlLookup(packageName)}/json`
+ );
const dependency: ReleaseResult = { releases: null };
logger.trace({ lookupUrl }, 'Pypi api got lookup');
const rep = await this.http.getJson<PypiJSON>(lookupUrl);
@@ -164,27 +171,25 @@ export class PypiDatasource extends Datasource {
text: string,
packageName: string
): string | null {
- const srcPrefixes = [
- `${packageName}-`,
- `${packageName.replace(regEx(/-/g), '_')}-`,
- ];
- for (const prefix of srcPrefixes) {
- const suffix = '.tar.gz';
- if (text.startsWith(prefix) && text.endsWith(suffix)) {
- return text.replace(prefix, '').replace(regEx(/\.tar\.gz$/), ''); // TODO #12071
- }
+ // source packages
+ const srcText = PypiDatasource.normalizeName(text);
+ const srcPrefix = `${packageName}-`;
+ const srcSuffix = '.tar.gz';
+ if (srcText.startsWith(srcPrefix) && srcText.endsWith(srcSuffix)) {
+ return srcText.replace(srcPrefix, '').replace(regEx(/\.tar\.gz$/), ''); // TODO #12071
}
// pep-0427 wheel packages
// {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl.
+ const wheelText = text.toLowerCase();
const wheelPrefix = packageName.replace(regEx(/[^\w\d.]+/g), '_') + '-';
const wheelSuffix = '.whl';
if (
- text.startsWith(wheelPrefix) &&
- text.endsWith(wheelSuffix) &&
- text.split('-').length > 2
+ wheelText.startsWith(wheelPrefix) &&
+ wheelText.endsWith(wheelSuffix) &&
+ wheelText.split('-').length > 2
) {
- return text.split('-')[1];
+ return wheelText.split('-')[1];
}
return null;
@@ -210,7 +215,10 @@ export class PypiDatasource extends Datasource {
packageName: string,
hostUrl: string
): Promise<ReleaseResult | null> {
- const lookupUrl = url.resolve(hostUrl, ensureTrailingSlash(packageName));
+ const lookupUrl = url.resolve(
+ hostUrl,
+ ensureTrailingSlash(PypiDatasource.normalizeNameForUrlLookup(packageName))
+ );
const dependency: ReleaseResult = { releases: null };
const response = await this.http.get(lookupUrl);
const dep = response?.body;
--
GitLab