feat(pip_requirements): added support for packages from a git repository (#13414)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/manager/pip_requirements/__fixtures__/requirements-git-packages.txt

+git+ssh://git@github.com/rwxd/python-pip-setup-test.git@v1.1.0
+git+ssh://git@github.com/rwxd/test_package@1.0.0
+git+ssh://git@gitlab.company.com/rwxd/python-package.git@abcde
+git+https://peter@github.com/rwxd/python-pip-setup-test.git@v0.9.0

lib/manager/pip_requirements/extract.spec.ts

@@ -11,6 +11,7 @@ const requirements6 = loadFixture('requirements6.txt');
 const requirements7 = loadFixture('requirements7.txt');
 const requirements8 = loadFixture('requirements8.txt');
 const requirementsWithEnvMarkers = loadFixture('requirements-env-markers.txt');
+const requirementsGitPackages = loadFixture('requirements-git-packages.txt');
 
 describe('manager/pip_requirements/extract', () => {
   beforeEach(() => {
@@ -159,5 +160,45 @@ describe('manager/pip_requirements/extract', () => {
         ],
       });
     });
+    it('should handle git packages', () => {
+      const res = extractPackageFile(
+        requirementsGitPackages,
+        'unused_file_name',
+        {}
+      );
+      expect(res.deps).toHaveLength(4);
+      expect(res).toEqual({
+        deps: [
+          {
+            depName: 'python-pip-setup-test',
+            currentValue: 'v1.1.0',
+            currentVersion: 'v1.1.0',
+            lookupName: 'git@github.com:rwxd/python-pip-setup-test.git',
+            datasource: 'git-tags',
+          },
+          {
+            depName: 'test_package',
+            currentValue: '1.0.0',
+            currentVersion: '1.0.0',
+            lookupName: 'git@github.com:rwxd/test_package',
+            datasource: 'git-tags',
+          },
+          {
+            depName: 'python-package',
+            currentValue: 'abcde',
+            currentVersion: 'abcde',
+            lookupName: 'git@gitlab.company.com:rwxd/python-package.git',
+            datasource: 'git-tags',
+          },
+          {
+            depName: 'python-pip-setup-test',
+            currentValue: 'v0.9.0',
+            currentVersion: 'v0.9.0',
+            lookupName: 'peter@github.com:rwxd/python-pip-setup-test.git',
+            datasource: 'git-tags',
+          },
+        ],
+      });
+    });
   });
 });

lib/manager/pip_requirements/extract.ts

 // based on https://www.python.org/dev/peps/pep-0508/#names
 import { RANGE_PATTERN } from '@renovatebot/pep440';
 import { GlobalConfig } from '../../config/global';
+import { GitTagsDatasource } from '../../datasource/git-tags';
 import { PypiDatasource } from '../../datasource/pypi';
 import { logger } from '../../logger';
 import { SkipReason } from '../../types';
@@ -11,6 +12,9 @@ import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
 export const packagePattern =
   '[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9._-]*[a-zA-Z0-9]';
 const extrasPattern = '(?:\\s*\\[[^\\]]+\\])?';
+const packageGitRegex = regEx(
+  /(?<source>(?:git\+)(git|ssh|https):\/\/(?<gitUrl>(?<user>.*)@(?<hostname>[\w.-]+)(?<delimiter>\/)(?<scmPath>.*\/(?<depName>[\w/-]+))(\.git)?(?:@(?<version>.*))))/
+);
 
 const rangePattern: string = RANGE_PATTERN;
 const specifierPartPattern = `\\s*${rangePattern.replace(
@@ -65,12 +69,33 @@ export function extractPackageFile(
       }
       const [lineNoEnvMarkers] = line.split(';').map((part) => part.trim());
       const lineNoHashes = lineNoEnvMarkers.split(' \\')[0];
-      const matches =
+      const packageMatches =
         pkgValRegex.exec(lineNoHashes) || pkgRegex.exec(lineNoHashes);
-      if (!matches) {
+      const gitPackageMatches = packageGitRegex.exec(lineNoHashes);
+      if (!packageMatches && !gitPackageMatches) {
         return null;
       }
-      const [, depName, , currVal] = matches;
+      if (gitPackageMatches) {
+        const currentVersion = gitPackageMatches.groups.version;
+        const depName = gitPackageMatches.groups.depName;
+
+        // we need to replace the / with a :
+        const scmPath = gitPackageMatches.groups.scmPath;
+        const delimiter = gitPackageMatches.groups.delimiter;
+        const lookupName = gitPackageMatches.groups.gitUrl
+          .replace(`${delimiter}${scmPath}`, `:${scmPath}`)
+          .replace(`@${currentVersion}`, '');
+        dep = {
+          ...dep,
+          depName,
+          currentValue: currentVersion,
+          currentVersion: currentVersion,
+          lookupName: lookupName,
+          datasource: GitTagsDatasource.id,
+        };
+        return dep;
+      }
+      const [, depName, , currVal] = packageMatches;
       const currentValue = currVal?.trim();
       dep = {
         ...dep,