diff --git a/lib/datasource/pypi/__fixtures__/versions-html-hyphens.html b/lib/datasource/pypi/__fixtures__/versions-html-hyphens.html new file mode 100644 index 0000000000000000000000000000000000000000..64a958fd346454e6956953b2a0090aabdbdd2612 --- /dev/null +++ b/lib/datasource/pypi/__fixtures__/versions-html-hyphens.html @@ -0,0 +1,12 @@ +<!DOCTYPE html> +<html><head> +<meta http-equiv="content-type" content="text/html; charset=UTF-8"> + <title>Links for package--with-hyphens</title> + </head> + <body> + <h1>Links for package--with-hyphens</h1> + <a href="">package--with-hyphens-2.0.0.tar.gz</a><br> + <a href="">package_with_hyphens-2.0.1-py3-none-any.whl</a><br> + <a href="">package_with_hyphens-2.0.2-py3-none-any.whl</a><br> + <a href="">package--with-hyphens-2.0.2.tar.gz</a><br> +</body></html> \ No newline at end of file diff --git a/lib/datasource/pypi/__fixtures__/versions-html-mixed-case.html b/lib/datasource/pypi/__fixtures__/versions-html-mixed-case.html new file mode 100644 index 0000000000000000000000000000000000000000..a295006601c998bab29e282da70b71d3a4553039 --- /dev/null +++ b/lib/datasource/pypi/__fixtures__/versions-html-mixed-case.html @@ -0,0 +1,12 @@ +<!DOCTYPE html> +<html><head> +<meta http-equiv="content-type" content="text/html; charset=UTF-8"> + <title>Links for PackageWithMixedCase</title> + </head> + <body> + <h1>Links for PackageWithMixedCase</h1> + <a href="">PackageWithMixedCase-2.0.0.tar.gz</a><br> + <a href="">PackageWithMixedCase-2.0.1-py3-none-any.whl</a><br> + <a href="">PackageWithMixedCase-2.0.2-py3-none-any.whl</a><br> + <a href="">PackageWithMixedCase-2.0.2.tar.gz</a><br> +</body></html> \ No newline at end of file diff --git a/lib/datasource/pypi/__fixtures__/versions-html-with-periods.html b/lib/datasource/pypi/__fixtures__/versions-html-with-periods.html new file mode 100644 index 0000000000000000000000000000000000000000..846d72d71b430b1d0fb2276db2ea818b179cc9f8 --- /dev/null +++ b/lib/datasource/pypi/__fixtures__/versions-html-with-periods.html @@ -0,0 +1,12 @@ +<!DOCTYPE html> +<html><head> +<meta http-equiv="content-type" content="text/html; charset=UTF-8"> + <title>Links for package.with.periods</title> + </head> + <body> + <h1>Links for package.with.periods</h1> + <a href="">package.with.periods-2.0.0.tar.gz</a><br> + <a href="">package.with.periods-2.0.1-py3-none-any.whl</a><br> + <a href="">package.with.periods-2.0.2-py3-none-any.whl</a><br> + <a href="">package.with.periods-2.0.2.tar.gz</a><br> +</body></html> \ No newline at end of file diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts index 8f919d9122aecccd2b2aea3ebb11315b96a80398..1e95cb1983440363f4f99d52f327c5d6085b0ab7 100644 --- a/lib/datasource/pypi/index.spec.ts +++ b/lib/datasource/pypi/index.spec.ts @@ -12,6 +12,9 @@ const dataRequiresPythonResponse = loadFixture( 'versions-html-data-requires-python.html' ); const mixedHyphensResponse = loadFixture('versions-html-mixed-hyphens.html'); +const mixedCaseResponse = loadFixture('versions-html-mixed-case.html'); +const withPeriodsResponse = loadFixture('versions-html-with-periods.html'); +const hyphensResponse = loadFixture('versions-html-hyphens.html'); const baseUrl = 'https://pypi.org/pypi'; const datasource = PypiDatasource.id; @@ -304,6 +307,25 @@ describe('datasource/pypi/index', () => { }); expect(res.isPrivate).toBeTrue(); }); + it('process data from simple endpoint with hyphens', async () => { + httpMock + .scope('https://pypi.org/simple/') + .get('/package-with-hyphens/') + .reply(200, hyphensResponse); + const config = { + registryUrls: ['https://pypi.org/simple/'], + }; + const res = await getPkgReleases({ + datasource, + ...config, + depName: 'package--with-hyphens', + }); + expect(res.releases).toMatchObject([ + { version: '2.0.0' }, + { version: '2.0.1' }, + { version: '2.0.2' }, + ]); + }); it('process data from simple endpoint with hyphens replaced with underscores', async () => { httpMock .scope('https://pypi.org/simple/') @@ -322,6 +344,63 @@ describe('datasource/pypi/index', () => { ).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('process data from simple endpoint with mixed-case characters', async () => { + httpMock + .scope('https://pypi.org/simple/') + .get('/packagewithmixedcase/') + .reply(200, mixedCaseResponse); + const config = { + registryUrls: ['https://pypi.org/simple/'], + }; + const res = await getPkgReleases({ + datasource, + ...config, + depName: 'PackageWithMixedCase', + }); + expect(res.releases).toMatchObject([ + { version: '2.0.0' }, + { version: '2.0.1' }, + { version: '2.0.2' }, + ]); + }); + it('process data from simple endpoint with mixed-case characters when using lower case dependency name', async () => { + httpMock + .scope('https://pypi.org/simple/') + .get('/packagewithmixedcase/') + .reply(200, mixedCaseResponse); + const config = { + registryUrls: ['https://pypi.org/simple/'], + }; + const res = await getPkgReleases({ + datasource, + ...config, + depName: 'packagewithmixedcase', + }); + expect(res.releases).toMatchObject([ + { version: '2.0.0' }, + { version: '2.0.1' }, + { version: '2.0.2' }, + ]); + }); + it('process data from simple endpoint with periods', async () => { + httpMock + .scope('https://pypi.org/simple/') + .get('/package-with-periods/') + .reply(200, withPeriodsResponse); + const config = { + registryUrls: ['https://pypi.org/simple/'], + }; + const res = await getPkgReleases({ + datasource, + ...config, + depName: 'package.with.periods', + }); + expect(res.releases).toMatchObject([ + { version: '2.0.0' }, + { version: '2.0.1' }, + { version: '2.0.2' }, + ]); + }); it('returns null for empty response', async () => { httpMock .scope('https://pypi.org/simple/') diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts index bd143aec398289920c12a944dda84d68ae0e2ef1..e2705686601c9487d59f4d7dffdac1c41e4b570d 100644 --- a/lib/datasource/pypi/index.ts +++ b/lib/datasource/pypi/index.ts @@ -73,6 +73,10 @@ export class PypiDatasource extends Datasource { } private static normalizeName(input: string): string { + return input.toLowerCase().replace(regEx(/_/g), '-'); + } + + private static normalizeNameForUrlLookup(input: string): string { return input.toLowerCase().replace(regEx(/(_|\.|-)+/g), '-'); } @@ -80,7 +84,10 @@ export class PypiDatasource extends Datasource { packageName: string, hostUrl: string ): Promise<ReleaseResult | null> { - const lookupUrl = url.resolve(hostUrl, `${packageName}/json`); + const lookupUrl = url.resolve( + hostUrl, + `${PypiDatasource.normalizeNameForUrlLookup(packageName)}/json` + ); const dependency: ReleaseResult = { releases: null }; logger.trace({ lookupUrl }, 'Pypi api got lookup'); const rep = await this.http.getJson<PypiJSON>(lookupUrl); @@ -164,27 +171,25 @@ export class PypiDatasource extends Datasource { text: string, packageName: string ): string | null { - const srcPrefixes = [ - `${packageName}-`, - `${packageName.replace(regEx(/-/g), '_')}-`, - ]; - for (const prefix of srcPrefixes) { - const suffix = '.tar.gz'; - if (text.startsWith(prefix) && text.endsWith(suffix)) { - return text.replace(prefix, '').replace(regEx(/\.tar\.gz$/), ''); // TODO #12071 - } + // source packages + const srcText = PypiDatasource.normalizeName(text); + const srcPrefix = `${packageName}-`; + const srcSuffix = '.tar.gz'; + if (srcText.startsWith(srcPrefix) && srcText.endsWith(srcSuffix)) { + return srcText.replace(srcPrefix, '').replace(regEx(/\.tar\.gz$/), ''); // TODO #12071 } // pep-0427 wheel packages // {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl. + const wheelText = text.toLowerCase(); const wheelPrefix = packageName.replace(regEx(/[^\w\d.]+/g), '_') + '-'; const wheelSuffix = '.whl'; if ( - text.startsWith(wheelPrefix) && - text.endsWith(wheelSuffix) && - text.split('-').length > 2 + wheelText.startsWith(wheelPrefix) && + wheelText.endsWith(wheelSuffix) && + wheelText.split('-').length > 2 ) { - return text.split('-')[1]; + return wheelText.split('-')[1]; } return null; @@ -210,7 +215,10 @@ export class PypiDatasource extends Datasource { packageName: string, hostUrl: string ): Promise<ReleaseResult | null> { - const lookupUrl = url.resolve(hostUrl, ensureTrailingSlash(packageName)); + const lookupUrl = url.resolve( + hostUrl, + ensureTrailingSlash(PypiDatasource.normalizeNameForUrlLookup(packageName)) + ); const dependency: ReleaseResult = { releases: null }; const response = await this.http.get(lookupUrl); const dep = response?.body;