From 150d01a4bbda47d45db0e1e04c1143f661b2fe05 Mon Sep 17 00:00:00 2001
From: rwxd <rwxd@pm.me>
Date: Tue, 18 Jan 2022 22:18:48 +0100
Subject: [PATCH] feat(pip_requirements): added support for packages from a git
repository (#13414)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
.../requirements-git-packages.txt | 4 ++
lib/manager/pip_requirements/extract.spec.ts | 41 +++++++++++++++++++
lib/manager/pip_requirements/extract.ts | 31 ++++++++++++--
3 files changed, 73 insertions(+), 3 deletions(-)
create mode 100644 lib/manager/pip_requirements/__fixtures__/requirements-git-packages.txt
diff --git a/lib/manager/pip_requirements/__fixtures__/requirements-git-packages.txt b/lib/manager/pip_requirements/__fixtures__/requirements-git-packages.txt
new file mode 100644
index 0000000000..644b5b47d6
--- /dev/null
+++ b/lib/manager/pip_requirements/__fixtures__/requirements-git-packages.txt
@@ -0,0 +1,4 @@
+git+ssh://git@github.com/rwxd/python-pip-setup-test.git@v1.1.0
+git+ssh://git@github.com/rwxd/test_package@1.0.0
+git+ssh://git@gitlab.company.com/rwxd/python-package.git@abcde
+git+https://peter@github.com/rwxd/python-pip-setup-test.git@v0.9.0
diff --git a/lib/manager/pip_requirements/extract.spec.ts b/lib/manager/pip_requirements/extract.spec.ts
index fce32b69a0..53c23ebc7f 100644
--- a/lib/manager/pip_requirements/extract.spec.ts
+++ b/lib/manager/pip_requirements/extract.spec.ts
@@ -11,6 +11,7 @@ const requirements6 = loadFixture('requirements6.txt');
const requirements7 = loadFixture('requirements7.txt');
const requirements8 = loadFixture('requirements8.txt');
const requirementsWithEnvMarkers = loadFixture('requirements-env-markers.txt');
+const requirementsGitPackages = loadFixture('requirements-git-packages.txt');
describe('manager/pip_requirements/extract', () => {
beforeEach(() => {
@@ -159,5 +160,45 @@ describe('manager/pip_requirements/extract', () => {
],
});
});
+ it('should handle git packages', () => {
+ const res = extractPackageFile(
+ requirementsGitPackages,
+ 'unused_file_name',
+ {}
+ );
+ expect(res.deps).toHaveLength(4);
+ expect(res).toEqual({
+ deps: [
+ {
+ depName: 'python-pip-setup-test',
+ currentValue: 'v1.1.0',
+ currentVersion: 'v1.1.0',
+ lookupName: 'git@github.com:rwxd/python-pip-setup-test.git',
+ datasource: 'git-tags',
+ },
+ {
+ depName: 'test_package',
+ currentValue: '1.0.0',
+ currentVersion: '1.0.0',
+ lookupName: 'git@github.com:rwxd/test_package',
+ datasource: 'git-tags',
+ },
+ {
+ depName: 'python-package',
+ currentValue: 'abcde',
+ currentVersion: 'abcde',
+ lookupName: 'git@gitlab.company.com:rwxd/python-package.git',
+ datasource: 'git-tags',
+ },
+ {
+ depName: 'python-pip-setup-test',
+ currentValue: 'v0.9.0',
+ currentVersion: 'v0.9.0',
+ lookupName: 'peter@github.com:rwxd/python-pip-setup-test.git',
+ datasource: 'git-tags',
+ },
+ ],
+ });
+ });
});
});
diff --git a/lib/manager/pip_requirements/extract.ts b/lib/manager/pip_requirements/extract.ts
index 7806601cdb..14b5d425be 100644
--- a/lib/manager/pip_requirements/extract.ts
+++ b/lib/manager/pip_requirements/extract.ts
@@ -1,6 +1,7 @@
// based on https://www.python.org/dev/peps/pep-0508/#names
import { RANGE_PATTERN } from '@renovatebot/pep440';
import { GlobalConfig } from '../../config/global';
+import { GitTagsDatasource } from '../../datasource/git-tags';
import { PypiDatasource } from '../../datasource/pypi';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
@@ -11,6 +12,9 @@ import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
export const packagePattern =
'[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9._-]*[a-zA-Z0-9]';
const extrasPattern = '(?:\\s*\\[[^\\]]+\\])?';
+const packageGitRegex = regEx(
+ /(?<source>(?:git\+)(git|ssh|https):\/\/(?<gitUrl>(?<user>.*)@(?<hostname>[\w.-]+)(?<delimiter>\/)(?<scmPath>.*\/(?<depName>[\w/-]+))(\.git)?(?:@(?<version>.*))))/
+);
const rangePattern: string = RANGE_PATTERN;
const specifierPartPattern = `\\s*${rangePattern.replace(
@@ -65,12 +69,33 @@ export function extractPackageFile(
}
const [lineNoEnvMarkers] = line.split(';').map((part) => part.trim());
const lineNoHashes = lineNoEnvMarkers.split(' \\')[0];
- const matches =
+ const packageMatches =
pkgValRegex.exec(lineNoHashes) || pkgRegex.exec(lineNoHashes);
- if (!matches) {
+ const gitPackageMatches = packageGitRegex.exec(lineNoHashes);
+ if (!packageMatches && !gitPackageMatches) {
return null;
}
- const [, depName, , currVal] = matches;
+ if (gitPackageMatches) {
+ const currentVersion = gitPackageMatches.groups.version;
+ const depName = gitPackageMatches.groups.depName;
+
+ // we need to replace the / with a :
+ const scmPath = gitPackageMatches.groups.scmPath;
+ const delimiter = gitPackageMatches.groups.delimiter;
+ const lookupName = gitPackageMatches.groups.gitUrl
+ .replace(`${delimiter}${scmPath}`, `:${scmPath}`)
+ .replace(`@${currentVersion}`, '');
+ dep = {
+ ...dep,
+ depName,
+ currentValue: currentVersion,
+ currentVersion: currentVersion,
+ lookupName: lookupName,
+ datasource: GitTagsDatasource.id,
+ };
+ return dep;
+ }
+ const [, depName, , currVal] = packageMatches;
const currentValue = currVal?.trim();
dep = {
...dep,
--
GitLab