diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts index 44acf798b116db445b030873264b19fda6da1050..6757b39e5943918430120f067c25ebbeeb85d644 100644 --- a/lib/modules/datasource/pypi/index.ts +++ b/lib/modules/datasource/pypi/index.ts @@ -21,9 +21,9 @@ export class PypiDatasource extends Datasource { override readonly customRegistrySupport = true; - override readonly defaultRegistryUrls = [ - process.env.PIP_INDEX_URL ?? 'https://pypi.org/pypi/', - ]; + static readonly defaultURL = + process.env.PIP_INDEX_URL ?? 'https://pypi.org/pypi/'; + override readonly defaultRegistryUrls = [PypiDatasource.defaultURL]; override readonly defaultVersioning = pep440.id; diff --git a/lib/modules/manager/api.ts b/lib/modules/manager/api.ts index 41dc108d9d1d8c550a265bb49f9c6756d97772c2..fcd8aca94e6e2a55bd25889ff7850d2866b81bcb 100644 --- a/lib/modules/manager/api.ts +++ b/lib/modules/manager/api.ts @@ -58,6 +58,7 @@ import * as npm from './npm'; import * as nuget from './nuget'; import * as nvm from './nvm'; import * as osgi from './osgi'; +import * as pep621 from './pep621'; import * as pipCompile from './pip-compile'; import * as pip_requirements from './pip_requirements'; import * as pip_setup from './pip_setup'; @@ -146,6 +147,7 @@ api.set('npm', npm); api.set('nuget', nuget); api.set('nvm', nvm); api.set('osgi', osgi); +api.set('pep621', pep621); api.set('pip-compile', pipCompile); api.set('pip_requirements', pip_requirements); api.set('pip_setup', pip_setup); diff --git a/lib/modules/manager/pep621/__fixtures__/pyproject_pdm_sources.toml b/lib/modules/manager/pep621/__fixtures__/pyproject_pdm_sources.toml new file mode 100644 index 0000000000000000000000000000000000000000..a09c836f3922782e23736cdbc7de2f62a9f18c13 --- /dev/null +++ b/lib/modules/manager/pep621/__fixtures__/pyproject_pdm_sources.toml @@ -0,0 +1,33 @@ +[project] +name = "pdm" +dynamic = ["version"] +requires-python = ">=3.7" +license = {text = "MIT"} +dependencies = [ + "blinker", + "packaging>=20.9,!=22.0", +] +readme = "README.md" + +[project.optional-dependencies] +pytest = [ + "pytest>12", +] + +[tool.pdm.dev-dependencies] +test = [ + "pytest-rerunfailures>=10.2", +] +tox = [ + "tox-pdm>=0.5", +] + +[[tool.pdm.source]] +url = "https://private-site.org/pypi/simple" +verify_ssl = true +name = "internal" + +[[tool.pdm.source]] +url = "https://private.pypi.org/simple" +verify_ssl = true +name = "pypi" diff --git a/lib/modules/manager/pep621/__fixtures__/pyproject_with_pdm.toml b/lib/modules/manager/pep621/__fixtures__/pyproject_with_pdm.toml new file mode 100644 index 0000000000000000000000000000000000000000..915cf6063ca561ebb6c79248368711d2bc577a82 --- /dev/null +++ b/lib/modules/manager/pep621/__fixtures__/pyproject_with_pdm.toml @@ -0,0 +1,37 @@ +[project] +name = "pdm" +dynamic = ["version"] +requires-python = ">=3.7" +license = {text = "MIT"} +dependencies = [ + "blinker", + "packaging>=20.9,!=22.0", + "rich>=12.3.0", + "virtualenv==20.0.0", + "pyproject-hooks", + "unearth>=0.9.0", + "tomlkit>=0.11.1,<1", + "installer<0.8,>=0.7", + "cachecontrol[filecache]>=0.12.11", + "tomli>=1.1.0; python_version < \"3.11\"", + "typing-extensions; python_version < \"3.8\"", + "importlib-metadata>=3.6; python_version < \"3.10\"", +] +readme = "README.md" + +[project.optional-dependencies] +pytest = [ + "pytest>12", + "pytest-mock", +] + +[tool.pdm.dev-dependencies] +test = [ + "pdm[pytest]", + "pytest-rerunfailures>=10.2", +] +tox = [ + "tox", + "tox-pdm>=0.5", + "", # fail to parse +] diff --git a/lib/modules/manager/pep621/extract.spec.ts b/lib/modules/manager/pep621/extract.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..75e1ad6e2745bcb00563cfa1bba96818133f8b6f --- /dev/null +++ b/lib/modules/manager/pep621/extract.spec.ts @@ -0,0 +1,267 @@ +import { codeBlock } from 'common-tags'; +import { Fixtures } from '../../../../test/fixtures'; +import { extractPackageFile } from './extract'; + +const pdmPyProject = Fixtures.get('pyproject_with_pdm.toml'); +const pdmSourcesPyProject = Fixtures.get('pyproject_pdm_sources.toml'); + +describe('modules/manager/pep621/extract', () => { + describe('extractPackageFile()', () => { + it('should return null for empty content', function () { + const result = extractPackageFile('', 'pyproject.toml'); + expect(result).toBeNull(); + }); + + it('should return null for invalid toml', function () { + const result = extractPackageFile( + codeBlock` + [project] + name = + `, + 'pyproject.toml' + ); + expect(result).toBeNull(); + }); + + it('should return dependencies for valid content', function () { + const result = extractPackageFile(pdmPyProject, 'pyproject.toml'); + + const dependencies = result?.deps.filter( + (dep) => dep.depType === 'project.dependencies' + ); + expect(dependencies).toEqual([ + { + packageName: 'blinker', + depName: 'blinker', + datasource: 'pypi', + depType: 'project.dependencies', + skipReason: 'any-version', + }, + { + packageName: 'packaging', + depName: 'packaging', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=20.9,!=22.0', + }, + { + packageName: 'rich', + depName: 'rich', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=12.3.0', + }, + { + packageName: 'virtualenv', + depName: 'virtualenv', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '==20.0.0', + }, + { + packageName: 'pyproject-hooks', + depName: 'pyproject-hooks', + datasource: 'pypi', + depType: 'project.dependencies', + skipReason: 'any-version', + }, + { + packageName: 'unearth', + depName: 'unearth', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=0.9.0', + }, + { + packageName: 'tomlkit', + depName: 'tomlkit', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=0.11.1,<1', + }, + { + packageName: 'installer', + depName: 'installer', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '<0.8,>=0.7', + }, + { + packageName: 'cachecontrol', + depName: 'cachecontrol', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=0.12.11', + }, + { + packageName: 'tomli', + depName: 'tomli', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=1.1.0', + }, + { + packageName: 'typing-extensions', + depName: 'typing-extensions', + datasource: 'pypi', + depType: 'project.dependencies', + skipReason: 'any-version', + }, + { + packageName: 'importlib-metadata', + depName: 'importlib-metadata', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=3.6', + }, + ]); + + const optionalDependencies = result?.deps.filter( + (dep) => dep.depType === 'project.optional-dependencies' + ); + expect(optionalDependencies).toEqual([ + { + packageName: 'pytest', + datasource: 'pypi', + depType: 'project.optional-dependencies', + currentValue: '>12', + depName: 'pytest/pytest', + }, + { + packageName: 'pytest-mock', + datasource: 'pypi', + depType: 'project.optional-dependencies', + skipReason: 'any-version', + depName: 'pytest/pytest-mock', + }, + ]); + + const pdmDevDependencies = result?.deps.filter( + (dep) => dep.depType === 'tool.pdm.dev-dependencies' + ); + expect(pdmDevDependencies).toEqual([ + { + packageName: 'pdm', + datasource: 'pypi', + depType: 'tool.pdm.dev-dependencies', + skipReason: 'any-version', + depName: 'test/pdm', + }, + { + packageName: 'pytest-rerunfailures', + datasource: 'pypi', + depType: 'tool.pdm.dev-dependencies', + currentValue: '>=10.2', + depName: 'test/pytest-rerunfailures', + }, + { + packageName: 'tox', + datasource: 'pypi', + depType: 'tool.pdm.dev-dependencies', + skipReason: 'any-version', + depName: 'tox/tox', + }, + { + packageName: 'tox-pdm', + datasource: 'pypi', + depType: 'tool.pdm.dev-dependencies', + currentValue: '>=0.5', + depName: 'tox/tox-pdm', + }, + ]); + }); + + it('should return dependencies with overwritten pypi registryUrl', function () { + const result = extractPackageFile(pdmSourcesPyProject, 'pyproject.toml'); + + expect(result?.deps).toEqual([ + { + packageName: 'blinker', + depName: 'blinker', + datasource: 'pypi', + depType: 'project.dependencies', + skipReason: 'any-version', + registryUrls: [ + 'https://private-site.org/pypi/simple', + 'https://private.pypi.org/simple', + ], + }, + { + packageName: 'packaging', + depName: 'packaging', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=20.9,!=22.0', + registryUrls: [ + 'https://private-site.org/pypi/simple', + 'https://private.pypi.org/simple', + ], + }, + { + packageName: 'pytest', + datasource: 'pypi', + depType: 'project.optional-dependencies', + currentValue: '>12', + depName: 'pytest/pytest', + registryUrls: [ + 'https://private-site.org/pypi/simple', + 'https://private.pypi.org/simple', + ], + }, + { + packageName: 'pytest-rerunfailures', + datasource: 'pypi', + depType: 'tool.pdm.dev-dependencies', + currentValue: '>=10.2', + depName: 'test/pytest-rerunfailures', + registryUrls: [ + 'https://private-site.org/pypi/simple', + 'https://private.pypi.org/simple', + ], + }, + { + packageName: 'tox-pdm', + datasource: 'pypi', + depType: 'tool.pdm.dev-dependencies', + currentValue: '>=0.5', + depName: 'tox/tox-pdm', + registryUrls: [ + 'https://private-site.org/pypi/simple', + 'https://private.pypi.org/simple', + ], + }, + ]); + }); + + it('should return dependencies with original pypi registryUrl', function () { + const result = extractPackageFile( + codeBlock` + [project] + dependencies = [ + "packaging>=20.9,!=22.0", + ] + + [[tool.pdm.source]] + url = "https://private-site.org/pypi/simple" + verify_ssl = true + name = "internal" + `, + 'pyproject.toml' + ); + + expect(result?.deps).toEqual([ + { + packageName: 'packaging', + depName: 'packaging', + datasource: 'pypi', + depType: 'project.dependencies', + currentValue: '>=20.9,!=22.0', + registryUrls: [ + 'https://pypi.org/pypi/', + 'https://private-site.org/pypi/simple', + ], + }, + ]); + }); + }); +}); diff --git a/lib/modules/manager/pep621/extract.ts b/lib/modules/manager/pep621/extract.ts new file mode 100644 index 0000000000000000000000000000000000000000..09064595188470f63378c171800bc6c8f1742bb7 --- /dev/null +++ b/lib/modules/manager/pep621/extract.ts @@ -0,0 +1,51 @@ +import toml from '@iarna/toml'; +import { logger } from '../../../logger'; +import type { + ExtractConfig, + PackageDependency, + PackageFileContent, +} from '../types'; +import { processors } from './processors'; +import { PyProject, PyProjectSchema } from './schema'; +import { parseDependencyGroupRecord, parseDependencyList } from './utils'; + +export function extractPackageFile( + content: string, + fileName: string, + config?: ExtractConfig +): PackageFileContent | null { + logger.trace({ fileName }, 'pep621.extractPackageFile'); + + const deps: PackageDependency[] = []; + + let def: PyProject; + try { + const jsonMap = toml.parse(content); + def = PyProjectSchema.parse(jsonMap); + } catch (err) { + logger.warn( + { fileName, err }, + `Failed to parse and validate pyproject file` + ); + return null; + } + + // pyProject standard definitions + deps.push( + ...parseDependencyList('project.dependencies', def.project?.dependencies) + ); + deps.push( + ...parseDependencyGroupRecord( + 'project.optional-dependencies', + def.project?.['optional-dependencies'] + ) + ); + + // process specific tool sets + let processedDeps = deps; + for (const processor of processors) { + processedDeps = processor.process(def, processedDeps); + } + + return processedDeps.length ? { deps: processedDeps } : null; +} diff --git a/lib/modules/manager/pep621/index.ts b/lib/modules/manager/pep621/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..9b1adb54893e40469eb6813e9cc4bb90cd5bb34a --- /dev/null +++ b/lib/modules/manager/pep621/index.ts @@ -0,0 +1,8 @@ +import { PypiDatasource } from '../../datasource/pypi'; +export { extractPackageFile } from './extract'; + +export const supportedDatasources = [PypiDatasource.id]; + +export const defaultConfig = { + fileMatch: ['(^|/)pyproject\\.toml$'], +}; diff --git a/lib/modules/manager/pep621/processors/index.ts b/lib/modules/manager/pep621/processors/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..ba80de12ccbe20843f90dc51af4d00e53677c2f6 --- /dev/null +++ b/lib/modules/manager/pep621/processors/index.ts @@ -0,0 +1,3 @@ +import { PdmProcessor } from './pdm'; + +export const processors = [new PdmProcessor()]; diff --git a/lib/modules/manager/pep621/processors/pdm.ts b/lib/modules/manager/pep621/processors/pdm.ts new file mode 100644 index 0000000000000000000000000000000000000000..e362694b55a7455e3780e7ca2051a9a7004ea919 --- /dev/null +++ b/lib/modules/manager/pep621/processors/pdm.ts @@ -0,0 +1,42 @@ +import is from '@sindresorhus/is'; +import { PypiDatasource } from '../../../datasource/pypi'; +import type { PackageDependency } from '../../types'; +import type { PyProject } from '../schema'; +import { parseDependencyGroupRecord } from '../utils'; +import type { PyProjectProcessor } from './types'; + +export class PdmProcessor implements PyProjectProcessor { + process(project: PyProject, deps: PackageDependency[]): PackageDependency[] { + const pdm = project.tool?.pdm; + if (is.nullOrUndefined(pdm)) { + return deps; + } + + deps.push( + ...parseDependencyGroupRecord( + 'tool.pdm.dev-dependencies', + pdm['dev-dependencies'] + ) + ); + + const pdmSource = pdm.source; + if (is.nullOrUndefined(pdmSource)) { + return deps; + } + + // add pypi default url, if there is no source declared with the name `pypi`. https://daobook.github.io/pdm/pyproject/tool-pdm/#specify-other-sources-for-finding-packages + const containsPyPiUrl = pdmSource.some((value) => value.name === 'pypi'); + const registryUrls: string[] = []; + if (!containsPyPiUrl) { + registryUrls.push(PypiDatasource.defaultURL); + } + for (const source of pdmSource) { + registryUrls.push(source.url); + } + for (const dep of deps) { + dep.registryUrls = registryUrls; + } + + return deps; + } +} diff --git a/lib/modules/manager/pep621/processors/types.ts b/lib/modules/manager/pep621/processors/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..dbe645396e76c2006d26e6509270fd8f01f160f8 --- /dev/null +++ b/lib/modules/manager/pep621/processors/types.ts @@ -0,0 +1,12 @@ +import type { PackageDependency } from '../../types'; +import type { PyProject } from '../schema'; + +export interface PyProjectProcessor { + /** + * Extracts additional dependencies and/or modifies existing ones based on the tool configuration. + * If no relevant section for the processor exists, then it should return the received dependencies unmodified. + * @param project PyProject object + * @param deps List of already extracted/processed dependencies + */ + process(project: PyProject, deps: PackageDependency[]): PackageDependency[]; +} diff --git a/lib/modules/manager/pep621/readme.md b/lib/modules/manager/pep621/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..1b52481f7d6b97c384889c3f7c409d3fc46d7211 --- /dev/null +++ b/lib/modules/manager/pep621/readme.md @@ -0,0 +1,11 @@ +This manager supports updating dependencies inside of `pyproject.toml` files. + +Outside standard dependencies, following toolsets are supported: + +- `pdm` + +Available `depType`s: + +- `project.dependencies` +- `project.optional-dependencies` +- `tool.pdm.dev-dependencies` diff --git a/lib/modules/manager/pep621/schema.ts b/lib/modules/manager/pep621/schema.ts new file mode 100644 index 0000000000000000000000000000000000000000..04dcc6609ce98c672b6153266bf1fe138ecd2c6b --- /dev/null +++ b/lib/modules/manager/pep621/schema.ts @@ -0,0 +1,35 @@ +import { z } from 'zod'; + +export type PyProject = z.infer<typeof PyProjectSchema>; + +const DependencyListSchema = z.array(z.string()).optional(); +const DependencyRecordSchema = z + .record(z.string(), z.array(z.string())) + .optional(); + +export const PyProjectSchema = z.object({ + project: z + .object({ + dependencies: DependencyListSchema, + 'optional-dependencies': DependencyRecordSchema, + }) + .optional(), + tool: z + .object({ + pdm: z + .object({ + 'dev-dependencies': DependencyRecordSchema, + source: z + .array( + z.object({ + url: z.string(), + name: z.string(), + verify_ssl: z.boolean().optional(), + }) + ) + .optional(), + }) + .optional(), + }) + .optional(), +}); diff --git a/lib/modules/manager/pep621/types.ts b/lib/modules/manager/pep621/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e68aa19818a842c9f5a7883367896465e486b1b --- /dev/null +++ b/lib/modules/manager/pep621/types.ts @@ -0,0 +1,6 @@ +export interface Pep508ParseResult { + packageName: string; + currentValue?: string; + extras?: string[]; + marker?: string; +} diff --git a/lib/modules/manager/pep621/utils.spec.ts b/lib/modules/manager/pep621/utils.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..6029a7b8ec8a51efba5a4c6ee8dfc4f9c65ec6cb --- /dev/null +++ b/lib/modules/manager/pep621/utils.spec.ts @@ -0,0 +1,39 @@ +import is from '@sindresorhus/is'; +import { parsePEP508 } from './utils'; + +describe('modules/manager/pep621/utils', () => { + describe('parsePEP508()', () => { + it.each` + value | success | packageName | currentValue | extras | marker + ${''} | ${false} | ${undefined} | ${undefined} | ${undefined} | ${undefined} + ${undefined} | ${false} | ${undefined} | ${undefined} | ${undefined} | ${undefined} + ${null} | ${false} | ${undefined} | ${undefined} | ${undefined} | ${undefined} + ${'blinker'} | ${true} | ${'blinker'} | ${undefined} | ${undefined} | ${undefined} + ${'packaging==20.0.0'} | ${true} | ${'packaging'} | ${'==20.0.0'} | ${undefined} | ${undefined} + ${'packaging>=20.9,!=22.0'} | ${true} | ${'packaging'} | ${'>=20.9,!=22.0'} | ${undefined} | ${undefined} + ${'cachecontrol[filecache]>=0.12.11'} | ${true} | ${'cachecontrol'} | ${'>=0.12.11'} | ${['filecache']} | ${undefined} + ${'tomli>=1.1.0; python_version < "3.11"'} | ${true} | ${'tomli'} | ${'>=1.1.0'} | ${undefined} | ${'python_version < "3.11"'} + ${'typing-extensions; python_version < "3.8"'} | ${true} | ${'typing-extensions'} | ${undefined} | ${undefined} | ${'python_version < "3.8"'} + ${'typing-extensions[test-feature]; python_version < "3.8"'} | ${true} | ${'typing-extensions'} | ${undefined} | ${['test-feature']} | ${'python_version < "3.8"'} + `( + '(parse $value"', + ({ value, success, packageName, currentValue, extras, marker }) => { + const result = parsePEP508(value); + + const expected = is.truthy(success) + ? clear({ packageName, currentValue, extras, marker }) + : null; + expect(result).toEqual(expected); + } + ); + }); +}); + +function clear(a: any) { + Object.keys(a).forEach((key) => { + if (a[key] === undefined) { + delete a[key]; + } + }); + return a; +} diff --git a/lib/modules/manager/pep621/utils.ts b/lib/modules/manager/pep621/utils.ts new file mode 100644 index 0000000000000000000000000000000000000000..8741c28dd46a210f6d23081b0c95ddcd69823328 --- /dev/null +++ b/lib/modules/manager/pep621/utils.ts @@ -0,0 +1,101 @@ +import is from '@sindresorhus/is'; +import { logger } from '../../../logger'; +import { regEx } from '../../../util/regex'; +import { PypiDatasource } from '../../datasource/pypi'; +import type { PackageDependency } from '../types'; +import type { Pep508ParseResult } from './types'; + +const pep508Regex = regEx( + /^(?<packageName>[A-Z0-9._-]+)\s*(\[(?<extras>[A-Z0-9,._-]+)\])?\s*(?<currentValue>[^;]+)?(;\s*(?<marker>.*))?/i +); + +export function parsePEP508( + value: string | null | undefined +): Pep508ParseResult | null { + if (is.nullOrUndefined(value)) { + return null; + } + + const regExpExec = pep508Regex.exec(value); + if ( + is.nullOrUndefined(regExpExec) || + is.nullOrUndefined(regExpExec?.groups) + ) { + logger.trace(`Pep508 could not be extracted`); + return null; + } + + const result: Pep508ParseResult = { + packageName: regExpExec.groups.packageName, + }; + if (is.nonEmptyString(regExpExec.groups.currentValue)) { + result.currentValue = regExpExec.groups.currentValue; + } + if (is.nonEmptyString(regExpExec.groups.marker)) { + result.marker = regExpExec.groups.marker; + } + if (is.nonEmptyString(regExpExec.groups.extras)) { + result.extras = regExpExec.groups.extras.split(','); + } + + return result; +} + +export function pep508ToPackageDependency( + depType: string, + value: string +): PackageDependency | null { + const parsed = parsePEP508(value); + if (is.nullOrUndefined(parsed)) { + return null; + } + + const dep: PackageDependency = { + packageName: parsed.packageName, + depName: parsed.packageName, + datasource: PypiDatasource.id, + depType, + }; + + if (is.nullOrUndefined(parsed.currentValue)) { + dep.skipReason = 'any-version'; + } else { + dep.currentValue = parsed.currentValue; + } + return dep; +} + +export function parseDependencyGroupRecord( + depType: string, + records: Record<string, string[]> | null | undefined +): PackageDependency[] { + if (is.nullOrUndefined(records)) { + return []; + } + + const deps: PackageDependency[] = []; + for (const [groupName, pep508Strings] of Object.entries(records)) { + for (const dep of parseDependencyList(depType, pep508Strings)) { + deps.push({ ...dep, depName: `${groupName}/${dep.packageName!}` }); + } + } + return deps; +} + +export function parseDependencyList( + depType: string, + list: string[] | null | undefined +): PackageDependency[] { + if (is.nullOrUndefined(list)) { + return []; + } + + const deps: PackageDependency[] = []; + for (const element of list) { + const dep = pep508ToPackageDependency(depType, element); + if (is.truthy(dep)) { + deps.push(dep); + } + } + return deps; +}