From 0a6d97f23a734a8bccbce78543dca2f483db0999 Mon Sep 17 00:00:00 2001
From: Sebastian Poxhofer <secustor@users.noreply.github.com>
Date: Wed, 5 Oct 2022 09:50:33 +0200
Subject: [PATCH] feat(config)!: move autodiscovery filter from string to array
(#16525)
Change type of autodiscoverFilter from string to array.
Closes #8763
BREAKING CHANGE: autodiscover filters can no longer include commas
---
docs/usage/self-hosted-configuration.md | 23 ++++++++++++---
lib/config/options/index.ts | 4 ++-
lib/config/types.ts | 2 +-
lib/workers/global/autodiscover.spec.ts | 28 ++++++++++++++----
lib/workers/global/autodiscover.ts | 38 ++++++++++++++++---------
5 files changed, 71 insertions(+), 24 deletions(-)
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index 61fbb6e6d2..32f562c246 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -85,11 +85,24 @@ You can limit which repositories Renovate can access by using the `autodiscoverF
You can use this option to filter the list of repositories that the Renovate bot account can access through `autodiscover`.
It takes a [minimatch](https://www.npmjs.com/package/minimatch) glob-style or regex pattern.
+If you set multiple filters, then the matches of each filter are added to the overall result.
+
+If you use an environment variable or the CLI to set the value for `autodiscoverFilter`, then commas `,` within filters are not supported.
+Commas will be used as delimiter for a new filter.
+
+```
+# DO NOT use commas inside the filter if your are using env or cli variables to configure it.
+RENOVATE_AUTODISCOVER_FILTER="/myapp/{readme.md,src/**}"
+
+# in this example you can use regex instead
+RENOVATE_AUTODISCOVER_FILTER="/myapp/(readme\.md|src/.*)/"
+```
+
**Minimatch**:
```json
{
- "autodiscoverFilter": "project/*"
+ "autodiscoverFilter": ["project/*"]
}
```
@@ -99,15 +112,17 @@ All text inside the start and end `/` will be treated as a regular expression.
```json
{
- "autodiscoverFilter": "/project/.*/"
+ "autodiscoverFilter": ["/project/.*/"]
}
```
-You can negate the regex by putting a `!` in front:
+You can negate the regex by putting a `!` in front.
+Only use a single negation and don't mix with other filters because all filters are combined with `or`.
+If using negations, all repositories except those who match the regex are added to the result:
```json
{
- "autodiscoverFilter": "!/project/.*/"
+ "autodiscoverFilter": ["!/project/.*/"]
}
```
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index 4056537b36..0a707078fc 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -723,7 +723,9 @@ const options: RenovateOptions[] = [
name: 'autodiscoverFilter',
description: 'Filter the list of autodiscovered repositories.',
stage: 'global',
- type: 'string',
+ type: 'array',
+ subType: 'string',
+ allowString: true,
default: null,
globalOnly: true,
},
diff --git a/lib/config/types.ts b/lib/config/types.ts
index 21f6f50559..d7815f3690 100644
--- a/lib/config/types.ts
+++ b/lib/config/types.ts
@@ -84,7 +84,7 @@ export interface RenovateSharedConfig {
// The below should contain config options where stage=global
export interface GlobalOnlyConfig {
autodiscover?: boolean;
- autodiscoverFilter?: string;
+ autodiscoverFilter?: string[];
baseDir?: string;
cacheDir?: string;
containerbaseDir?: string;
diff --git a/lib/workers/global/autodiscover.spec.ts b/lib/workers/global/autodiscover.spec.ts
index 2f083c7e7a..c2409d9408 100644
--- a/lib/workers/global/autodiscover.spec.ts
+++ b/lib/workers/global/autodiscover.spec.ts
@@ -53,7 +53,7 @@ describe('workers/global/autodiscover', () => {
it('filters autodiscovered github repos', async () => {
config.autodiscover = true;
- config.autodiscoverFilter = 'project/re*';
+ config.autodiscoverFilter = ['project/re*'];
config.platform = PlatformId.Github;
hostRules.find = jest.fn(() => ({
token: 'abc',
@@ -67,7 +67,7 @@ describe('workers/global/autodiscover', () => {
it('filters autodiscovered github repos but nothing matches', async () => {
config.autodiscover = true;
- config.autodiscoverFilter = 'project/re*';
+ config.autodiscoverFilter = ['project/re*'];
config.platform = 'github';
hostRules.find = jest.fn(() => ({
token: 'abc',
@@ -81,7 +81,7 @@ describe('workers/global/autodiscover', () => {
it('filters autodiscovered github repos with regex', async () => {
config.autodiscover = true;
- config.autodiscoverFilter = '/project/re*./';
+ config.autodiscoverFilter = ['/project/re*./'];
config.platform = PlatformId.Github;
hostRules.find = jest.fn(() => ({
token: 'abc',
@@ -95,7 +95,7 @@ describe('workers/global/autodiscover', () => {
it('filters autodiscovered github repos with regex negation', async () => {
config.autodiscover = true;
- config.autodiscoverFilter = '!/project/re*./';
+ config.autodiscoverFilter = ['!/project/re*./'];
config.platform = PlatformId.Github;
hostRules.find = jest.fn(() => ({
token: 'abc',
@@ -109,7 +109,7 @@ describe('workers/global/autodiscover', () => {
it('fail if regex pattern is not valid', async () => {
config.autodiscover = true;
- config.autodiscoverFilter = '/project/re**./';
+ config.autodiscoverFilter = ['/project/re**./'];
config.platform = PlatformId.Github;
hostRules.find = jest.fn(() => ({
token: 'abc',
@@ -119,4 +119,22 @@ describe('workers/global/autodiscover', () => {
);
await expect(autodiscoverRepositories(config)).rejects.toThrow();
});
+
+ it('filters autodiscovered github repos with multiple values', async () => {
+ config.autodiscover = true;
+ config.autodiscoverFilter = ['another-project/re*', 'department/dev/*'];
+ config.platform = 'github';
+ hostRules.find = jest.fn(() => ({
+ token: 'abc',
+ }));
+ const expectedRepositories = [
+ 'another-project/repo',
+ 'department/dev/aProject',
+ ];
+ ghApi.getRepos = jest.fn(() =>
+ Promise.resolve(['another-project/another-repo', ...expectedRepositories])
+ );
+ const res = await autodiscoverRepositories(config);
+ expect(res.repositories).toEqual(expectedRepositories);
+ });
});
diff --git a/lib/workers/global/autodiscover.ts b/lib/workers/global/autodiscover.ts
index 4dba5d7cf1..daad41efe2 100644
--- a/lib/workers/global/autodiscover.ts
+++ b/lib/workers/global/autodiscover.ts
@@ -30,20 +30,10 @@ export async function autodiscoverRepositories(
);
return config;
}
+
if (config.autodiscoverFilter) {
- if (isConfigRegex(config.autodiscoverFilter)) {
- const autodiscoveryPred = configRegexPredicate(config.autodiscoverFilter);
- if (!autodiscoveryPred) {
- throw new Error(
- `Failed to parse regex pattern "${config.autodiscoverFilter}"`
- );
- }
- discovered = discovered.filter(autodiscoveryPred);
- } else {
- discovered = discovered.filter(
- minimatch.filter(config.autodiscoverFilter)
- );
- }
+ discovered = applyFilters(discovered, config.autodiscoverFilter);
+
if (!discovered.length) {
// Soft fail (no error thrown) if no accessible repositories match the filter
logger.debug('None of the discovered repositories matched the filter');
@@ -54,6 +44,7 @@ export async function autodiscoverRepositories(
{ length: discovered.length, repositories: discovered },
`Autodiscovered repositories`
);
+
// istanbul ignore if
if (config.repositories?.length) {
logger.debug(
@@ -80,3 +71,24 @@ export async function autodiscoverRepositories(
}
return { ...config, repositories: discovered };
}
+
+export function applyFilters(repos: string[], filters: string[]): string[] {
+ const matched = new Set<string>();
+
+ for (const filter of filters) {
+ let res: string[];
+ if (isConfigRegex(filter)) {
+ const autodiscoveryPred = configRegexPredicate(filter);
+ if (!autodiscoveryPred) {
+ throw new Error(`Failed to parse regex pattern "${filter}"`);
+ }
+ res = repos.filter(autodiscoveryPred);
+ } else {
+ res = repos.filter(minimatch.filter(filter));
+ }
+ for (const repository of res) {
+ matched.add(repository);
+ }
+ }
+ return [...matched];
+}
--
GitLab