diff --git a/lib/modules/datasource/pypi/common.ts b/lib/modules/datasource/pypi/common.ts index ea46fddcc7530ff7ae7732ac93da5ab0ccd537ba..17650cb4c12603760a7b569f540a5834d046ff69 100644 --- a/lib/modules/datasource/pypi/common.ts +++ b/lib/modules/datasource/pypi/common.ts @@ -5,3 +5,8 @@ const githubRepoPattern = regEx(/^https?:\/\/github\.com\/[^/]+\/[^/]+$/); export function isGitHubRepo(url: string): boolean { return !url.includes('sponsors') && githubRepoPattern.test(url); } + +// https://packaging.python.org/en/latest/specifications/name-normalization/ +export function normalizeDepName(name: string): string { + return name.replace(/[-_.]+/g, '-').toLowerCase(); +} diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 4aab61635c9cfb8956a8156c197f2ab3ae6ad0fb..9ca84c592f6cee8b4fd72fa39a32855356ca5eef 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -8,7 +8,7 @@ import { ensureTrailingSlash } from '../../../util/url'; import * as pep440 from '../../versioning/pep440'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; -import { isGitHubRepo } from './common'; +import { isGitHubRepo, normalizeDepName } from './common'; import type { PypiJSON, PypiJSONRelease, Releases } from './types'; export class PypiDatasource extends Datasource { @@ -79,17 +79,13 @@ export class PypiDatasource extends Datasource { return input.toLowerCase().replace(regEx(/_/g), '-'); } - private static normalizeNameForUrlLookup(input: string): string { - return input.toLowerCase().replace(regEx(/(_|\.|-)+/g), '-'); - } - private async getDependency( packageName: string, hostUrl: string, ): Promise<ReleaseResult | null> { const lookupUrl = url.resolve( hostUrl, - `${PypiDatasource.normalizeNameForUrlLookup(packageName)}/json`, + `${normalizeDepName(packageName)}/json`, ); const dependency: ReleaseResult = { releases: [] }; logger.trace({ lookupUrl }, 'Pypi api got lookup'); @@ -227,9 +223,7 @@ export class PypiDatasource extends Datasource { ): Promise<ReleaseResult | null> { const lookupUrl = url.resolve( hostUrl, - ensureTrailingSlash( - PypiDatasource.normalizeNameForUrlLookup(packageName), - ), + ensureTrailingSlash(normalizeDepName(packageName)), ); const dependency: ReleaseResult = { releases: [] }; const response = await this.http.get(lookupUrl); diff --git a/lib/modules/manager/pip-compile/extract.spec.ts b/lib/modules/manager/pip-compile/extract.spec.ts index 7a78fcff21557ded18c99a7bb8201c3112ac266b..786fc48a1a9007cb57ba0b762ef516f9b9184ec0 100644 --- a/lib/modules/manager/pip-compile/extract.spec.ts +++ b/lib/modules/manager/pip-compile/extract.spec.ts @@ -1,5 +1,6 @@ import { Fixtures } from '../../../../test/fixtures'; import { fs } from '../../../../test/util'; +import { logger } from '../../../logger'; import { extractAllPackageFiles, extractPackageFile } from '.'; jest.mock('../../../util/fs'); @@ -139,11 +140,64 @@ describe('modules/manager/pip-compile/extract', () => { ['foo==1.0.1'], ), ); - fs.readLocalFile.mockResolvedValueOnce('!@#$'); - fs.readLocalFile.mockResolvedValueOnce(''); + fs.readLocalFile.mockResolvedValueOnce('!@#$'); // malformed.in + fs.readLocalFile.mockResolvedValueOnce(''); // empty.in + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=headerOnly.txt reqs.in', + [], + ), + ); - const lockFiles = ['empty.txt', 'noHeader.txt', 'badSource.txt']; + const lockFiles = [ + 'empty.txt', + 'noHeader.txt', + 'badSource.txt', + 'headerOnly.txt', + ]; const packageFiles = await extractAllPackageFiles({}, lockFiles); expect(packageFiles).toBeNull(); }); + + it('adds lockedVersion to deps in package file', async () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements.txt requirements.in', + ['friendly-bard==1.0.1'], + ), + ); + // also check if normalized name is used + fs.readLocalFile.mockResolvedValueOnce('FrIeNdLy-._.-bArD>=1.0.0'); + + const lockFiles = ['requirements.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + const packageFile = packageFiles!.pop(); + expect(packageFile!.deps).toHaveLength(1); + expect(packageFile!.deps.pop()).toMatchObject({ + currentValue: '>=1.0.0', + depName: 'FrIeNdLy-._.-bArD', + lockedVersion: '1.0.1', + }); + }); + + it('warns if dependency has no locked version', async () => { + fs.readLocalFile.mockResolvedValueOnce( + getSimpleRequirementsFile( + 'pip-compile --output-file=requirements.txt requirements.in', + ['foo==1.0.1'], + ), + ); + fs.readLocalFile.mockResolvedValueOnce('foo>=1.0.0\nbar'); + + const lockFiles = ['requirements.txt']; + const packageFiles = await extractAllPackageFiles({}, lockFiles); + expect(packageFiles).toBeDefined(); + const packageFile = packageFiles!.pop(); + expect(packageFile!.deps).toHaveLength(2); + expect(logger.warn).toHaveBeenCalledWith( + { depName: 'bar', lockFile: 'requirements.txt' }, + 'pip-compile: dependency not found in lock file', + ); + }); }); diff --git a/lib/modules/manager/pip-compile/extract.ts b/lib/modules/manager/pip-compile/extract.ts index c66750ac66461da86e242d7b5b691898f83e14ef..72dc9c40c1b24e96e094497bba8871c975e69e46 100644 --- a/lib/modules/manager/pip-compile/extract.ts +++ b/lib/modules/manager/pip-compile/extract.ts @@ -1,5 +1,6 @@ import { logger } from '../../../logger'; import { readLocalFile } from '../../../util/fs'; +import { normalizeDepName } from '../../datasource/pypi/common'; import { extractPackageFile as extractRequirementsFile } from '../pip_requirements/extract'; import { extractPackageFile as extractSetupPyFile } from '../pip_setup'; import type { ExtractConfig, PackageFile, PackageFileContent } from '../types'; @@ -87,8 +88,15 @@ export async function extractAllPackageFiles( type: 'constraint', }); } - // TODO(not7cd): handle locked deps - // const lockedDeps = extractRequirementsFile(content); + const lockedDeps = extractRequirementsFile(fileContent)?.deps; + if (!lockedDeps) { + logger.debug( + { fileMatch }, + 'pip-compile: Failed to extract dependencies from lock file', + ); + continue; + } + for (const packageFile of compileArgs.sourceFiles) { depsBetweenFiles.push({ sourceFile: packageFile, @@ -122,6 +130,21 @@ export async function extractAllPackageFiles( config, ); if (packageFileContent) { + for (const dep of packageFileContent.deps) { + const lockedVersion = lockedDeps?.find( + (lockedDep) => + normalizeDepName(lockedDep.depName!) === + normalizeDepName(dep.depName!), + )?.currentVersion; + if (lockedVersion) { + dep.lockedVersion = lockedVersion; + } else { + logger.warn( + { depName: dep.depName, lockFile: fileMatch }, + 'pip-compile: dependency not found in lock file', + ); + } + } packageFiles.set(packageFile, { ...packageFileContent, lockFiles: [fileMatch],