From e27fe66980c5aa4b54ad2a4d30e425df3d318ada Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Wed, 18 Dec 2024 14:01:14 +0100
Subject: [PATCH] feat(git): cloneSubmodulesFilter (#33115)
---
docs/usage/configuration-options.md | 10 +++++++++-
lib/config/options/index.ts | 8 ++++++++
lib/config/types.ts | 1 +
lib/modules/platform/azure/index.ts | 2 ++
lib/modules/platform/bitbucket-server/index.ts | 2 ++
lib/modules/platform/bitbucket/index.ts | 2 ++
lib/modules/platform/gitea/index.ts | 3 +++
lib/modules/platform/github/index.ts | 2 ++
.../gitlab/__snapshots__/index.spec.ts.snap | 2 ++
lib/modules/platform/gitlab/index.ts | 3 +++
lib/modules/platform/types.ts | 1 +
lib/util/git/index.spec.ts | 1 +
lib/util/git/index.ts | 15 +++++++++++++--
lib/util/git/types.ts | 1 +
lib/workers/repository/init/index.ts | 2 +-
15 files changed, 51 insertions(+), 4 deletions(-)
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index c6205f400e..d96d93cf9e 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -456,11 +456,19 @@ For `sbt` note that Renovate will update the version string only for packages th
## cloneSubmodules
-Enabling this option will mean that any detected Git submodules will be cloned at time of repository clone.
+Enabling this option will mean that detected Git submodules will be cloned at time of repository clone.
+By default all will be cloned, but this can be customized by configuring `cloneSubmodulesFilter` too.
Submodules are always cloned recursively.
Important: private submodules aren't supported by Renovate, unless the underlying `ssh` layer already has the correct permissions.
+## cloneSubmodulesFilter
+
+Use this option together with `cloneSubmodules` if you wish to clone only a subset of submodules.
+
+This config option supports regex and glob filters, including negative matches.
+For more details on this syntax see Renovate's [string pattern matching documentation](./string-pattern-matching.md).
+
## commitBody
Configure this if you wish Renovate to add a commit body, otherwise Renovate uses a regular single-line commit.
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index 065238c6f2..263cc69f12 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -2856,6 +2856,14 @@ const options: RenovateOptions[] = [
type: 'boolean',
default: false,
},
+ {
+ name: 'cloneSubmodulesFilter',
+ description:
+ 'List of submodules names or patterns to clone when cloneSubmodules=true.',
+ type: 'array',
+ subType: 'string',
+ default: ['*'],
+ },
{
name: 'ignorePrAuthor',
description:
diff --git a/lib/config/types.ts b/lib/config/types.ts
index 4b7169d168..20d97dad3c 100644
--- a/lib/config/types.ts
+++ b/lib/config/types.ts
@@ -239,6 +239,7 @@ export interface RenovateConfig
baseBranch?: string;
defaultBranch?: string;
branchList?: string[];
+ cloneSubmodulesFilter?: string[];
description?: string | string[];
force?: RenovateConfig;
errors?: ValidationMessage[];
diff --git a/lib/modules/platform/azure/index.ts b/lib/modules/platform/azure/index.ts
index 21bd44eaa3..37deb70b83 100644
--- a/lib/modules/platform/azure/index.ts
+++ b/lib/modules/platform/azure/index.ts
@@ -193,6 +193,7 @@ export async function getJsonFile(
export async function initRepo({
repository,
cloneSubmodules,
+ cloneSubmodulesFilter,
}: RepoParams): Promise<RepoResult> {
logger.debug(`initRepo("${repository}")`);
config = { repository } as Config;
@@ -240,6 +241,7 @@ export async function initRepo({
url,
extraCloneOpts: getStorageExtraCloneOpts(opts),
cloneSubmodules,
+ cloneSubmodulesFilter,
});
const repoConfig: RepoResult = {
defaultBranch,
diff --git a/lib/modules/platform/bitbucket-server/index.ts b/lib/modules/platform/bitbucket-server/index.ts
index d475ae8f5b..5978a3682a 100644
--- a/lib/modules/platform/bitbucket-server/index.ts
+++ b/lib/modules/platform/bitbucket-server/index.ts
@@ -223,6 +223,7 @@ export async function getJsonFile(
export async function initRepo({
repository,
cloneSubmodules,
+ cloneSubmodulesFilter,
ignorePrAuthor,
gitUrl,
}: RepoParams): Promise<RepoResult> {
@@ -274,6 +275,7 @@ export async function initRepo({
url,
extraCloneOpts: getExtraCloneOpts(opts),
cloneSubmodules,
+ cloneSubmodulesFilter,
fullClone: semver.lte(defaults.version, '8.0.0'),
});
diff --git a/lib/modules/platform/bitbucket/index.ts b/lib/modules/platform/bitbucket/index.ts
index 11594cd7bc..f58265dad7 100644
--- a/lib/modules/platform/bitbucket/index.ts
+++ b/lib/modules/platform/bitbucket/index.ts
@@ -185,6 +185,7 @@ export async function getJsonFile(
export async function initRepo({
repository,
cloneSubmodules,
+ cloneSubmodulesFilter,
ignorePrAuthor,
bbUseDevelopmentBranch,
}: RepoParams): Promise<RepoResult> {
@@ -262,6 +263,7 @@ export async function initRepo({
...config,
url,
cloneSubmodules,
+ cloneSubmodulesFilter,
});
const repoConfig: RepoResult = {
defaultBranch: mainBranch,
diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts
index 697be51f84..31450e6c44 100644
--- a/lib/modules/platform/gitea/index.ts
+++ b/lib/modules/platform/gitea/index.ts
@@ -70,6 +70,7 @@ interface GiteaRepoConfig {
labelList: Promise<Label[]> | null;
defaultBranch: string;
cloneSubmodules: boolean;
+ cloneSubmodulesFilter: string[] | undefined;
hasIssuesEnabled: boolean;
}
@@ -255,6 +256,7 @@ const platform: Platform = {
async initRepo({
repository,
cloneSubmodules,
+ cloneSubmodulesFilter,
gitUrl,
}: RepoParams): Promise<RepoResult> {
let repo: Repo;
@@ -262,6 +264,7 @@ const platform: Platform = {
config = {} as any;
config.repository = repository;
config.cloneSubmodules = !!cloneSubmodules;
+ config.cloneSubmodulesFilter = cloneSubmodulesFilter;
// Try to fetch information about repository
try {
diff --git a/lib/modules/platform/github/index.ts b/lib/modules/platform/github/index.ts
index dfe02ab40a..1298d0f908 100644
--- a/lib/modules/platform/github/index.ts
+++ b/lib/modules/platform/github/index.ts
@@ -447,6 +447,7 @@ export async function initRepo({
forkToken,
renovateUsername,
cloneSubmodules,
+ cloneSubmodulesFilter,
ignorePrAuthor,
}: RepoParams): Promise<RepoResult> {
logger.debug(`initRepo("${repository}")`);
@@ -454,6 +455,7 @@ export async function initRepo({
config = {
repository,
cloneSubmodules,
+ cloneSubmodulesFilter,
ignorePrAuthor,
} as any;
// istanbul ignore if
diff --git a/lib/modules/platform/gitlab/__snapshots__/index.spec.ts.snap b/lib/modules/platform/gitlab/__snapshots__/index.spec.ts.snap
index b5e2347602..2ab3cf9572 100644
--- a/lib/modules/platform/gitlab/__snapshots__/index.spec.ts.snap
+++ b/lib/modules/platform/gitlab/__snapshots__/index.spec.ts.snap
@@ -135,6 +135,7 @@ exports[`modules/platform/gitlab/index initRepo should fall back respecting when
[
{
"cloneSubmodules": undefined,
+ "cloneSubmodulesFilter": undefined,
"defaultBranch": "master",
"ignorePrAuthor": undefined,
"mergeMethod": "merge",
@@ -150,6 +151,7 @@ exports[`modules/platform/gitlab/index initRepo should use ssh_url_to_repo if gi
[
{
"cloneSubmodules": undefined,
+ "cloneSubmodulesFilter": undefined,
"defaultBranch": "master",
"ignorePrAuthor": undefined,
"mergeMethod": "merge",
diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts
index 30965b9630..2460b206d9 100644
--- a/lib/modules/platform/gitlab/index.ts
+++ b/lib/modules/platform/gitlab/index.ts
@@ -81,6 +81,7 @@ let config: {
mergeMethod: MergeMethod;
defaultBranch: string;
cloneSubmodules: boolean | undefined;
+ cloneSubmodulesFilter: string[] | undefined;
ignorePrAuthor: boolean | undefined;
squash: boolean;
} = {} as any;
@@ -299,6 +300,7 @@ function getRepoUrl(
export async function initRepo({
repository,
cloneSubmodules,
+ cloneSubmodulesFilter,
ignorePrAuthor,
gitUrl,
endpoint,
@@ -307,6 +309,7 @@ export async function initRepo({
config = {} as any;
config.repository = urlEscape(repository);
config.cloneSubmodules = cloneSubmodules;
+ config.cloneSubmodulesFilter = cloneSubmodulesFilter;
config.ignorePrAuthor = ignorePrAuthor;
let res: HttpResponse<RepoResponse>;
diff --git a/lib/modules/platform/types.ts b/lib/modules/platform/types.ts
index cc770c1d8e..81a8a84994 100644
--- a/lib/modules/platform/types.ts
+++ b/lib/modules/platform/types.ts
@@ -47,6 +47,7 @@ export interface RepoParams {
forkProcessing?: 'enabled' | 'disabled';
renovateUsername?: string;
cloneSubmodules?: boolean;
+ cloneSubmodulesFilter?: string[];
ignorePrAuthor?: boolean;
bbUseDevelopmentBranch?: boolean;
includeMirrors?: boolean;
diff --git a/lib/util/git/index.spec.ts b/lib/util/git/index.spec.ts
index 9b600e4200..b7607533fb 100644
--- a/lib/util/git/index.spec.ts
+++ b/lib/util/git/index.spec.ts
@@ -271,6 +271,7 @@ describe('util/git/index', () => {
await repo.commit('Add submodules');
await git.initRepo({
cloneSubmodules: true,
+ cloneSubmodulesFilter: ['file'],
url: base.path,
});
await git.syncGit();
diff --git a/lib/util/git/index.ts b/lib/util/git/index.ts
index aa80763d30..475fb958b9 100644
--- a/lib/util/git/index.ts
+++ b/lib/util/git/index.ts
@@ -24,6 +24,7 @@ import type { GitProtocol } from '../../types/git';
import { incLimitedValue } from '../../workers/global/limits';
import { getCache } from '../cache/repository';
import { newlineRegex, regEx } from '../regex';
+import { matchRegexOrGlobList } from '../string-match';
import { parseGitAuthor } from './author';
import {
getCachedBehindBaseResult,
@@ -344,7 +345,10 @@ export async function getSubmodules(): Promise<string[]> {
}
}
-export async function cloneSubmodules(shouldClone: boolean): Promise<void> {
+export async function cloneSubmodules(
+ shouldClone: boolean,
+ cloneSubmodulesFilter: string[] | undefined,
+): Promise<void> {
if (!shouldClone || submodulesInitizialized) {
return;
}
@@ -352,6 +356,13 @@ export async function cloneSubmodules(shouldClone: boolean): Promise<void> {
await syncGit();
const submodules = await getSubmodules();
for (const submodule of submodules) {
+ if (!matchRegexOrGlobList(submodule, cloneSubmodulesFilter ?? ['*'])) {
+ logger.debug(
+ { cloneSubmodulesFilter },
+ `Skipping submodule ${submodule}`,
+ );
+ continue;
+ }
try {
logger.debug(`Cloning git submodule at ${submodule}`);
await gitRetry(() =>
@@ -458,7 +469,7 @@ export async function syncGit(): Promise<void> {
throw err;
}
// This will only happen now if set in global config
- await cloneSubmodules(!!config.cloneSubmodules);
+ await cloneSubmodules(!!config.cloneSubmodules, config.cloneSubmodulesFilter);
try {
const latestCommit = (await git.log({ n: 1 })).latest;
logger.debug({ latestCommit }, 'latest repository commit');
diff --git a/lib/util/git/types.ts b/lib/util/git/types.ts
index 6354483805..91cf358ecc 100644
--- a/lib/util/git/types.ts
+++ b/lib/util/git/types.ts
@@ -21,6 +21,7 @@ export interface StorageConfig {
url: string;
extraCloneOpts?: GitOptions;
cloneSubmodules?: boolean;
+ cloneSubmodulesFilter?: string[];
fullClone?: boolean;
}
diff --git a/lib/workers/repository/init/index.ts b/lib/workers/repository/init/index.ts
index 5e284c6671..e8b36f4433 100644
--- a/lib/workers/repository/init/index.ts
+++ b/lib/workers/repository/init/index.ts
@@ -72,6 +72,6 @@ export async function initRepo(
'Full resolved config and hostRules including presets',
);
}
- await cloneSubmodules(!!config.cloneSubmodules);
+ await cloneSubmodules(!!config.cloneSubmodules, config.cloneSubmodulesFilter);
return config;
}
--
GitLab