From 20c25d9340a8c2ed655f71a1fd3a78b886db4088 Mon Sep 17 00:00:00 2001
From: Christopher Frieler <christopher@frieler.rocks>
Date: Sun, 31 Oct 2021 08:31:30 +0100
Subject: [PATCH] fix(pypi): Normalize package names before lookup (#12273)
---
lib/datasource/pypi/index.spec.ts | 47 +++++++++++++++++++++++++++++++
lib/datasource/pypi/index.ts | 17 +++++++++--
2 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts
index c335b49217..8f919d9122 100644
--- a/lib/datasource/pypi/index.spec.ts
+++ b/lib/datasource/pypi/index.spec.ts
@@ -172,6 +172,53 @@ describe('datasource/pypi/index', () => {
expect(result.changelogUrl).toBe(info.project_urls.changelog);
expect(httpMock.getTrace()).toMatchSnapshot();
});
+ it('normalizes the package name according to PEP 503', async () => {
+ const expectedHttpCall = httpMock
+ .scope(baseUrl)
+ .get('/not-normalized-package/json')
+ .reply(200, htmlResponse);
+
+ await getPkgReleases({
+ datasource,
+ registryUrls: [baseUrl],
+ depName: 'not_normalized.Package',
+ });
+
+ expect(expectedHttpCall.isDone()).toBeTrue();
+ });
+ it('normalizes the package name according to PEP 503 when falling back to simple endpoint', async () => {
+ httpMock
+ .scope(baseUrl)
+ .get('/not-normalized-package/json')
+ .reply(404, '');
+ const expectedFallbackHttpCall = httpMock
+ .scope(baseUrl)
+ .get('/not-normalized-package/')
+ .reply(200, htmlResponse);
+
+ await getPkgReleases({
+ datasource,
+ registryUrls: [baseUrl],
+ depName: 'not_normalized.Package',
+ });
+
+ expect(expectedFallbackHttpCall.isDone()).toBeTrue();
+ });
+ it('normalizes the package name according to PEP 503 querying a simple endpoint', async () => {
+ const simpleRegistryUrl = 'https://pypi.org/simple/';
+ const expectedHttpCall = httpMock
+ .scope(simpleRegistryUrl)
+ .get('/not-normalized-package/')
+ .reply(200, htmlResponse);
+
+ await getPkgReleases({
+ datasource,
+ registryUrls: [simpleRegistryUrl],
+ depName: 'not_normalized.Package',
+ });
+
+ expect(expectedHttpCall.isDone()).toBeTrue();
+ });
it('respects constraints', async () => {
httpMock
diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts
index ba15c473f9..bd143aec39 100644
--- a/lib/datasource/pypi/index.ts
+++ b/lib/datasource/pypi/index.ts
@@ -36,6 +36,7 @@ export class PypiDatasource extends Datasource {
}: GetReleasesConfig): Promise<ReleaseResult | null> {
let dependency: ReleaseResult = null;
const hostUrl = ensureTrailingSlash(registryUrl);
+ const normalizedLookupName = PypiDatasource.normalizeName(lookupName);
// not all simple indexes use this identifier, but most do
if (hostUrl.endsWith('/simple/') || hostUrl.endsWith('/+simple/')) {
@@ -43,12 +44,15 @@ export class PypiDatasource extends Datasource {
{ lookupName, hostUrl },
'Looking up pypi simple dependency'
);
- dependency = await this.getSimpleDependency(lookupName, hostUrl);
+ dependency = await this.getSimpleDependency(
+ normalizedLookupName,
+ hostUrl
+ );
} else {
logger.trace({ lookupName, hostUrl }, 'Looking up pypi api dependency');
try {
// we need to resolve early here so we can catch any 404s and fallback to a simple lookup
- dependency = await this.getDependency(lookupName, hostUrl);
+ dependency = await this.getDependency(normalizedLookupName, hostUrl);
} catch (err) {
if (err.statusCode !== 404) {
throw err;
@@ -59,12 +63,19 @@ export class PypiDatasource extends Datasource {
{ lookupName, hostUrl },
'Looking up pypi simple dependency via fallback'
);
- dependency = await this.getSimpleDependency(lookupName, hostUrl);
+ dependency = await this.getSimpleDependency(
+ normalizedLookupName,
+ hostUrl
+ );
}
}
return dependency;
}
+ private static normalizeName(input: string): string {
+ return input.toLowerCase().replace(regEx(/(_|\.|-)+/g), '-');
+ }
+
private async getDependency(
packageName: string,
hostUrl: string
--
GitLab