From 78e3ea6a5020dfa042b626e6f21e43b112283015 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Wed, 8 May 2024 12:23:12 +0200
Subject: [PATCH] feat(pipenv): better python constraints checking (#28878)
---
.../manager/pipenv/__fixtures__/Pipfile1 | 1 +
lib/modules/manager/pipenv/artifacts.spec.ts | 133 ++++++++++++++++++
lib/modules/manager/pipenv/artifacts.ts | 79 +++++++++--
3 files changed, 204 insertions(+), 9 deletions(-)
diff --git a/lib/modules/manager/pipenv/__fixtures__/Pipfile1 b/lib/modules/manager/pipenv/__fixtures__/Pipfile1
index 3303c31c5a..45643ed668 100644
--- a/lib/modules/manager/pipenv/__fixtures__/Pipfile1
+++ b/lib/modules/manager/pipenv/__fixtures__/Pipfile1
@@ -28,3 +28,4 @@ dev-package = "==0.1.0"
[requires]
python_version = "3.6"
+python_full_version = "3.6.2"
diff --git a/lib/modules/manager/pipenv/artifacts.spec.ts b/lib/modules/manager/pipenv/artifacts.spec.ts
index 6d6d6bf8f3..a3975ef6c3 100644
--- a/lib/modules/manager/pipenv/artifacts.spec.ts
+++ b/lib/modules/manager/pipenv/artifacts.spec.ts
@@ -78,6 +78,7 @@ describe('modules/manager/pipenv/artifacts', () => {
// python
datasource.getPkgReleases.mockResolvedValueOnce({
releases: [
+ { version: '3.6.2' },
{ version: '3.6.5' },
{ version: '3.7.6' },
{ version: '3.8.5' },
@@ -137,6 +138,138 @@ describe('modules/manager/pipenv/artifacts', () => {
]);
});
+ it('gets python full version from Pipfile', async () => {
+ GlobalConfig.set({ ...adminConfig, binarySource: 'install' });
+ pipFileLock._meta!.requires!.python_full_version = '3.7.6';
+ fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
+ fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir);
+ fs.readLocalFile.mockResolvedValueOnce(JSON.stringify(pipFileLock));
+ const execSnapshots = mockExecAll();
+ fs.readLocalFile.mockResolvedValueOnce(JSON.stringify(pipFileLock));
+
+ expect(
+ await updateArtifacts({
+ packageFileName: 'Pipfile',
+ updatedDeps: [],
+ newPackageFileContent: Fixtures.get('Pipfile1'),
+ config,
+ }),
+ ).toBeNull();
+
+ expect(execSnapshots).toMatchObject([
+ { cmd: 'install-tool python 3.6.2' },
+ {},
+ {
+ cmd: 'pipenv lock',
+ options: {
+ cwd: '/tmp/github/some/repo',
+ env: {
+ PIPENV_CACHE_DIR: pipenvCacheDir,
+ },
+ },
+ },
+ ]);
+ });
+
+ it('gets python version from Pipfile', async () => {
+ GlobalConfig.set({ ...adminConfig, binarySource: 'install' });
+ pipFileLock._meta!.requires!.python_full_version = '3.7.6';
+ fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
+ fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir);
+ fs.readLocalFile.mockResolvedValueOnce(JSON.stringify(pipFileLock));
+ const execSnapshots = mockExecAll();
+ fs.readLocalFile.mockResolvedValueOnce(JSON.stringify(pipFileLock));
+
+ expect(
+ await updateArtifacts({
+ packageFileName: 'Pipfile',
+ updatedDeps: [],
+ newPackageFileContent: Fixtures.get('Pipfile2'),
+ config,
+ }),
+ ).toBeNull();
+
+ expect(execSnapshots).toMatchObject([
+ { cmd: 'install-tool python 3.6.5' },
+ {},
+ {
+ cmd: 'pipenv lock',
+ options: {
+ cwd: '/tmp/github/some/repo',
+ env: {
+ PIPENV_CACHE_DIR: pipenvCacheDir,
+ },
+ },
+ },
+ ]);
+ });
+
+ it('gets full python version from .python-version', async () => {
+ GlobalConfig.set({ ...adminConfig, binarySource: 'install' });
+ fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
+ fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir);
+ fs.readLocalFile.mockResolvedValueOnce(JSON.stringify(pipFileLock));
+ const execSnapshots = mockExecAll();
+ fs.getSiblingFileName.mockResolvedValueOnce('.python-version' as never);
+ fs.readLocalFile.mockResolvedValueOnce('3.7.6');
+
+ expect(
+ await updateArtifacts({
+ packageFileName: 'Pipfile',
+ updatedDeps: [],
+ newPackageFileContent: 'some toml',
+ config,
+ }),
+ ).toBeNull();
+
+ expect(execSnapshots).toMatchObject([
+ { cmd: 'install-tool python 3.7.6' },
+ {},
+ {
+ cmd: 'pipenv lock',
+ options: {
+ cwd: '/tmp/github/some/repo',
+ env: {
+ PIPENV_CACHE_DIR: pipenvCacheDir,
+ },
+ },
+ },
+ ]);
+ });
+
+ it('gets python stream, from .python-version', async () => {
+ GlobalConfig.set({ ...adminConfig, binarySource: 'install' });
+ fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
+ fs.ensureCacheDir.mockResolvedValueOnce(virtualenvsCacheDir);
+ fs.readLocalFile.mockResolvedValueOnce(JSON.stringify(pipFileLock));
+ const execSnapshots = mockExecAll();
+ fs.getSiblingFileName.mockResolvedValueOnce('.python-version' as never);
+ fs.readLocalFile.mockResolvedValueOnce('3.8');
+
+ expect(
+ await updateArtifacts({
+ packageFileName: 'Pipfile',
+ updatedDeps: [],
+ newPackageFileContent: 'some toml',
+ config,
+ }),
+ ).toBeNull();
+
+ expect(execSnapshots).toMatchObject([
+ { cmd: 'install-tool python 3.8.5' },
+ {},
+ {
+ cmd: 'pipenv lock',
+ options: {
+ cwd: '/tmp/github/some/repo',
+ env: {
+ PIPENV_CACHE_DIR: pipenvCacheDir,
+ },
+ },
+ },
+ ]);
+ });
+
it('handles no constraint', async () => {
fs.ensureCacheDir.mockResolvedValueOnce(pipenvCacheDir);
fs.ensureCacheDir.mockResolvedValueOnce(pipCacheDir);
diff --git a/lib/modules/manager/pipenv/artifacts.ts b/lib/modules/manager/pipenv/artifacts.ts
index 5c0e98f555..6404c2b9c0 100644
--- a/lib/modules/manager/pipenv/artifacts.ts
+++ b/lib/modules/manager/pipenv/artifacts.ts
@@ -8,14 +8,17 @@ import type { ExecOptions, ExtraEnv, Opt } from '../../../util/exec/types';
import {
deleteLocalFile,
ensureCacheDir,
+ getSiblingFileName,
readLocalFile,
writeLocalFile,
} from '../../../util/fs';
import { getRepoStatus } from '../../../util/git';
import { find } from '../../../util/host-rules';
import { regEx } from '../../../util/regex';
+import { parse as parseToml } from '../../../util/toml';
import { parseUrl } from '../../../util/url';
import { PypiDatasource } from '../../datasource/pypi';
+import pep440 from '../../versioning/pep440';
import type {
UpdateArtifact,
UpdateArtifactsConfig,
@@ -24,38 +27,91 @@ import type {
import { extractPackageFile } from './extract';
import { PipfileLockSchema } from './schema';
-export function getPythonConstraint(
+export async function getPythonConstraint(
+ pipfileName: string,
+ pipfileContent: string,
existingLockFileContent: string,
config: UpdateArtifactsConfig,
-): string | undefined {
+): Promise<string | undefined> {
const { constraints = {} } = config;
const { python } = constraints;
if (python) {
- logger.debug('Using python constraint from config');
+ logger.debug(`Using python constraint ${python} from config`);
return python;
}
+
+ // Try Pipfile first because it may have had its Python version updated
+ try {
+ const pipfile = parseToml(pipfileContent) as any;
+ const pythonFullVersion = pipfile.requires.python_full_version;
+ if (pythonFullVersion) {
+ logger.debug(
+ `Using python full version ${pythonFullVersion} from Pipfile`,
+ );
+ return `== ${pythonFullVersion}`;
+ }
+ const pythonVersion = pipfile.requires.python_version;
+ if (pythonVersion) {
+ logger.debug(`Using python version ${pythonVersion} from Pipfile`);
+ return `== ${pythonVersion}.*`;
+ }
+ } catch (err) {
+ logger.warn({ err }, 'Error parsing Pipfile');
+ }
+
+ // Try Pipfile.lock next
try {
const result = PipfileLockSchema.safeParse(existingLockFileContent);
// istanbul ignore if: not easily testable
if (!result.success) {
- logger.warn({ error: result.error }, 'Invalid Pipfile.lock');
+ logger.warn({ err: result.error }, 'Invalid Pipfile.lock');
return undefined;
}
// Exact python version has been included since 2022.10.9. It is more specific than the major.minor version
// https://github.com/pypa/pipenv/blob/main/CHANGELOG.md#2022109-2022-10-09
- if (result.data._meta?.requires?.python_full_version) {
- const pythonFullVersion = result.data._meta.requires.python_full_version;
+ const pythonFullVersion = result.data._meta?.requires?.python_full_version;
+ if (pythonFullVersion) {
+ logger.debug(
+ `Using python full version ${pythonFullVersion} from Pipfile.lock`,
+ );
return `== ${pythonFullVersion}`;
}
// Before 2022.10.9, only the major.minor version was included
- if (result.data._meta?.requires?.python_version) {
- const pythonVersion = result.data._meta.requires.python_version;
+ const pythonVersion = result.data._meta?.requires?.python_version;
+ if (pythonVersion) {
+ logger.debug(`Using python version ${pythonVersion} from Pipfile.lock`);
return `== ${pythonVersion}.*`;
}
} catch (err) {
// Do nothing
}
+
+ // Try looking for the contents of .python-version
+ const pythonVersionFileName = getSiblingFileName(
+ pipfileName,
+ '.python-version',
+ );
+ try {
+ const pythonVersion = await readLocalFile(pythonVersionFileName, 'utf8');
+ let pythonVersionConstraint;
+ if (pythonVersion && pep440.isVersion(pythonVersion)) {
+ if (pythonVersion.split('.').length >= 3) {
+ pythonVersionConstraint = `== ${pythonVersion}`;
+ } else {
+ pythonVersionConstraint = `== ${pythonVersion}.*`;
+ }
+ }
+ if (pythonVersionConstraint) {
+ logger.debug(
+ `Using python version ${pythonVersionConstraint} from ${pythonVersionFileName}`,
+ );
+ return pythonVersionConstraint;
+ }
+ } catch (err) {
+ // Do nothing
+ }
+
return undefined;
}
@@ -233,7 +289,12 @@ export async function updateArtifacts({
await deleteLocalFile(lockFileName);
}
const cmd = 'pipenv lock';
- const tagConstraint = getPythonConstraint(existingLockFileContent, config);
+ const tagConstraint = await getPythonConstraint(
+ pipfileName,
+ newPipfileContent,
+ existingLockFileContent,
+ config,
+ );
const pipenvConstraint = getPipenvConstraint(
existingLockFileContent,
config,
--
GitLab