From d5db1c68f849a6b4327c79f0ec1cd3010a02ac72 Mon Sep 17 00:00:00 2001
From: Florian Greinacher <florian.greinacher@siemens.com>
Date: Mon, 11 Sep 2023 13:03:16 +0200
Subject: [PATCH] feat: autodiscover repositories by namespace (#24321)
---
docs/usage/self-hosted-configuration.md | 14 +++++++
lib/config/options/index.ts | 11 ++++++
lib/config/types.ts | 1 +
lib/modules/platform/gitlab/index.spec.ts | 47 +++++++++++++++++++++++
lib/modules/platform/gitlab/index.ts | 38 +++++++++++++++---
lib/modules/platform/types.ts | 1 +
lib/workers/global/autodiscover.ts | 1 +
7 files changed, 107 insertions(+), 6 deletions(-)
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index e1d9930563..d7d5608aa0 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -125,6 +125,20 @@ If using negations, all repositories except those who match the regex are added
}
```
+## autodiscoverNamespaces
+
+You can use this option to autodiscover projects in specific namespaces (a.k.a. groups/organizations/workspaces).
+In contrast to `autodiscoverFilter` the filtering is done by the platform and therefore more efficient.
+
+For example:
+
+```json
+{
+ "platform": "gitlab",
+ "autodiscoverNamespaces": ["a-group", "another-group/some-subgroup"]
+}
+```
+
## autodiscoverTopics
Some platforms allow you to add tags, or topics, to repositories and retrieve repository lists by specifying those
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index a1c4eb3c8e..ae28361677 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -797,6 +797,17 @@ const options: RenovateOptions[] = [
default: null,
globalOnly: true,
},
+ {
+ name: 'autodiscoverNamespaces',
+ description:
+ 'Filter the list of autodiscovered repositories by namespaces.',
+ stage: 'global',
+ type: 'array',
+ subType: 'string',
+ default: null,
+ globalOnly: true,
+ supportedPlatforms: ['gitlab'],
+ },
{
name: 'autodiscoverTopics',
description: 'Filter the list of autodiscovered repositories by topics.',
diff --git a/lib/config/types.ts b/lib/config/types.ts
index 1f2ac991c4..e849702b44 100644
--- a/lib/config/types.ts
+++ b/lib/config/types.ts
@@ -95,6 +95,7 @@ export interface RenovateSharedConfig {
export interface GlobalOnlyConfig {
autodiscover?: boolean;
autodiscoverFilter?: string[] | string;
+ autodiscoverNamespaces?: string[];
autodiscoverTopics?: string[];
baseDir?: string;
cacheDir?: string;
diff --git a/lib/modules/platform/gitlab/index.spec.ts b/lib/modules/platform/gitlab/index.spec.ts
index f6f9e32203..cfa104946e 100644
--- a/lib/modules/platform/gitlab/index.spec.ts
+++ b/lib/modules/platform/gitlab/index.spec.ts
@@ -209,6 +209,53 @@ describe('modules/platform/gitlab/index', () => {
const repos = await gitlab.getRepos({ topics: ['one', 'two'] });
expect(repos).toEqual(['a/b', 'c/d']);
});
+
+ it('should query the groups endpoint for each namespace', async () => {
+ httpMock
+ .scope(gitlabApiHost)
+ .get(
+ '/api/v4/groups/a/projects?membership=true&per_page=100&with_merge_requests_enabled=true&min_access_level=30&archived=false&include_subgroups=true&with_shared=false'
+ )
+ .reply(200, [
+ {
+ path_with_namespace: 'a/b',
+ },
+ ])
+ .get(
+ '/api/v4/groups/c%2Fd/projects?membership=true&per_page=100&with_merge_requests_enabled=true&min_access_level=30&archived=false&include_subgroups=true&with_shared=false'
+ )
+ .reply(200, [
+ {
+ path_with_namespace: 'c/d/e',
+ },
+ {
+ path_with_namespace: 'c/d/f',
+ },
+ ]);
+ const repos = await gitlab.getRepos({ namespaces: ['a', 'c/d'] });
+ expect(repos).toEqual(['a/b', 'c/d/e', 'c/d/f']);
+ });
+
+ it('should consider topics when querying the groups endpoint', async () => {
+ httpMock
+ .scope(gitlabApiHost)
+ .get(
+ '/api/v4/groups/a/projects?membership=true&per_page=100&with_merge_requests_enabled=true&min_access_level=30&archived=false&include_subgroups=true&with_shared=false&topic=one%2Ctwo'
+ )
+ .reply(200, [
+ {
+ path_with_namespace: 'a/b',
+ },
+ {
+ path_with_namespace: 'a/c',
+ },
+ ]);
+ const repos = await gitlab.getRepos({
+ namespaces: ['a'],
+ topics: ['one', 'two'],
+ });
+ expect(repos).toEqual(['a/b', 'a/c']);
+ });
});
async function initRepo(
diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts
index d8981e5c72..539fdb4dfa 100644
--- a/lib/modules/platform/gitlab/index.ts
+++ b/lib/modules/platform/gitlab/index.ts
@@ -2,6 +2,7 @@ import URL from 'node:url';
import { setTimeout } from 'timers/promises';
import is from '@sindresorhus/is';
import JSON5 from 'json5';
+import pMap from 'p-map';
import semver from 'semver';
import {
CONFIG_GIT_URL_UNAVAILABLE,
@@ -162,14 +163,38 @@ export async function getRepos(config?: AutodiscoverConfig): Promise<string[]> {
queryParams['topic'] = config.topics.join(',');
}
- const url = 'projects?' + getQueryString(queryParams);
+ const urls = [];
+ if (config?.namespaces?.length) {
+ queryParams['with_shared'] = false;
+ queryParams['include_subgroups'] = true;
+ urls.push(
+ ...config.namespaces.map(
+ (namespace) =>
+ `groups/${urlEscape(namespace)}/projects?${getQueryString(
+ queryParams
+ )}`
+ )
+ );
+ } else {
+ urls.push('projects?' + getQueryString(queryParams));
+ }
try {
- const res = await gitlabApi.getJson<RepoResponse[]>(url, {
- paginate: true,
- });
- logger.debug(`Discovered ${res.body.length} project(s)`);
- return res.body
+ const repos = (
+ await pMap(
+ urls,
+ (url) =>
+ gitlabApi.getJson<RepoResponse[]>(url, {
+ paginate: true,
+ }),
+ {
+ concurrency: 2,
+ }
+ )
+ ).flatMap((response) => response.body);
+
+ logger.debug(`Discovered ${repos.length} project(s)`);
+ return repos
.filter((repo) => !repo.mirror || config?.includeMirrors)
.map((repo) => repo.path_with_namespace);
} catch (err) {
@@ -177,6 +202,7 @@ export async function getRepos(config?: AutodiscoverConfig): Promise<string[]> {
throw err;
}
}
+
function urlEscape(str: string): string;
function urlEscape(str: string | undefined): string | undefined;
function urlEscape(str: string | undefined): string | undefined {
diff --git a/lib/modules/platform/types.ts b/lib/modules/platform/types.ts
index 684005d9c9..1e87dec950 100644
--- a/lib/modules/platform/types.ts
+++ b/lib/modules/platform/types.ts
@@ -169,6 +169,7 @@ export type EnsureIssueResult = 'updated' | 'created';
export interface AutodiscoverConfig {
topics?: string[];
includeMirrors?: boolean;
+ namespaces?: string[];
}
export interface Platform {
diff --git a/lib/workers/global/autodiscover.ts b/lib/workers/global/autodiscover.ts
index 5d1963ffaa..2af10cf07f 100644
--- a/lib/workers/global/autodiscover.ts
+++ b/lib/workers/global/autodiscover.ts
@@ -38,6 +38,7 @@ export async function autodiscoverRepositories(
let discovered = await platform.getRepos({
topics: config.autodiscoverTopics,
includeMirrors: config.includeMirrors,
+ namespaces: config.autodiscoverNamespaces,
});
if (!discovered?.length) {
// Soft fail (no error thrown) if no accessible repositories
--
GitLab