From 1b84c5282a1e02528f3f2cc464fb41a65e5ecf4b Mon Sep 17 00:00:00 2001
From: Sebastian Poxhofer <secustor@users.noreply.github.com>
Date: Thu, 4 Nov 2021 09:45:14 +0100
Subject: [PATCH] feat(manager/regex)!: allow arbitrary regex groups for
templates (#12296)
Allow the usage of arbitrary capture groups inside of regex manager templates and further adds refactor which this allows.
BREAKING_CHANGE
Only regex managers using the combination matchStringStrategy are affected of this change!
Currently capture groups which are empty but still match the regex are ignored.
This is no longer the case!
Subsequent matchGroups will now overwrite previous ones, even if the later one is empty.
---
lib/manager/regex/__fixtures__/ansible.yml | 4 +-
.../regex/__snapshots__/index.spec.ts.snap | 22 +++++
lib/manager/regex/index.spec.ts | 23 +++++
lib/manager/regex/index.ts | 87 +++++++------------
lib/manager/regex/types.ts | 4 +
5 files changed, 84 insertions(+), 56 deletions(-)
create mode 100644 lib/manager/regex/types.ts
diff --git a/lib/manager/regex/__fixtures__/ansible.yml b/lib/manager/regex/__fixtures__/ansible.yml
index b3647cd99f..f3cf9c112c 100644
--- a/lib/manager/regex/__fixtures__/ansible.yml
+++ b/lib/manager/regex/__fixtures__/ansible.yml
@@ -1,5 +1,7 @@
prometheus_image: "prom/prometheus" // depName gets initially set
+
+prometheus_registry: "docker.io" // depName gets initially set
+prometheus_repository: "prom/prometheus" // depName gets initially set
prometheus_version: "v2.21.0" // currentValue get set
-someother_image: "" // will not be set as group value is null/empty string
someother_version: "0.12.0" // overwrites currentValue as later values take precedence.
diff --git a/lib/manager/regex/__snapshots__/index.spec.ts.snap b/lib/manager/regex/__snapshots__/index.spec.ts.snap
index c5f0f8e34f..e41338ed9f 100644
--- a/lib/manager/regex/__snapshots__/index.spec.ts.snap
+++ b/lib/manager/regex/__snapshots__/index.spec.ts.snap
@@ -257,6 +257,28 @@ Object {
}
`;
+exports[`manager/regex/index extracts with combination strategy and non standard capture groups 1`] = `
+Object {
+ "datasourceTemplate": "docker",
+ "depNameTemplate": "{{{ registry }}}/{{{ repository }}}",
+ "deps": Array [
+ Object {
+ "currentValue": "v2.21.0",
+ "datasource": "docker",
+ "depName": "docker.io/prom/prometheus",
+ "replaceString": "prometheus_version: \\"v2.21.0\\" //",
+ },
+ ],
+ "matchStrings": Array [
+ "prometheus_registry:\\\\s*\\"(?<registry>.*)\\"\\\\s*\\\\/\\\\/",
+ "prometheus_repository:\\\\s*\\"(?<repository>.*)\\"\\\\s*\\\\/\\\\/",
+ "prometheus_tag:\\\\s*\\"(?<tag>.*)\\"\\\\s*\\\\/\\\\/",
+ "prometheus_version:\\\\s*\\"(?<currentValue>.*)\\"\\\\s*\\\\/\\\\/",
+ ],
+ "matchStringsStrategy": "combination",
+}
+`;
+
exports[`manager/regex/index extracts with combination strategy and registry url 1`] = `
Object {
"datasourceTemplate": "helm",
diff --git a/lib/manager/regex/index.spec.ts b/lib/manager/regex/index.spec.ts
index cec3ca01d3..62e3fdc024 100644
--- a/lib/manager/regex/index.spec.ts
+++ b/lib/manager/regex/index.spec.ts
@@ -206,6 +206,29 @@ describe('manager/regex/index', () => {
expect(res).toMatchSnapshot();
expect(res.deps).toHaveLength(1);
});
+
+ it('extracts with combination strategy and non standard capture groups', async () => {
+ const config: CustomExtractConfig = {
+ matchStrings: [
+ 'prometheus_registry:\\s*"(?<registry>.*)"\\s*\\/\\/',
+ 'prometheus_repository:\\s*"(?<repository>.*)"\\s*\\/\\/',
+ 'prometheus_tag:\\s*"(?<tag>.*)"\\s*\\/\\/',
+ 'prometheus_version:\\s*"(?<currentValue>.*)"\\s*\\/\\/',
+ ],
+ matchStringsStrategy: 'combination',
+ datasourceTemplate: 'docker',
+ depNameTemplate: '{{{ registry }}}/{{{ repository }}}',
+ };
+ const res = await extractPackageFile(
+ ansibleYamlContent,
+ 'ansible.yml',
+ config
+ );
+ expect(res.deps).toHaveLength(1);
+ expect(res.deps[0].depName).toEqual('docker.io/prom/prometheus');
+ expect(res).toMatchSnapshot();
+ });
+
it('extracts with combination strategy and multiple matches', async () => {
const config: CustomExtractConfig = {
matchStrings: [
diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts
index 4558010c2c..0facc64a35 100644
--- a/lib/manager/regex/index.ts
+++ b/lib/manager/regex/index.ts
@@ -8,6 +8,7 @@ import type {
PackageFile,
Result,
} from '../types';
+import type { ExtractionTemplate } from './types';
export const defaultConfig = {
pinDigests: false,
@@ -25,8 +26,6 @@ const validMatchFields = [
'depType',
];
-const mergeFields = ['registryUrls', ...validMatchFields];
-
function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
const matches: RegExpMatchArray[] = [];
let matchResult;
@@ -40,13 +39,12 @@ function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
}
function createDependency(
- matchResult: RegExpMatchArray,
- combinedGroups: Record<string, string>,
+ extractionTemplate: ExtractionTemplate,
config: CustomExtractConfig,
dep?: PackageDependency
): PackageDependency {
const dependency = dep || {};
- const { groups } = matchResult;
+ const { groups, replaceString } = extractionTemplate;
function updateDependency(field: string, value: string): void {
switch (field) {
@@ -69,11 +67,7 @@ function createDependency(
const fieldTemplate = `${field}Template`;
if (config[fieldTemplate]) {
try {
- const compiled = template.compile(
- config[fieldTemplate],
- combinedGroups ?? groups,
- false
- );
+ const compiled = template.compile(config[fieldTemplate], groups, false);
updateDependency(field, compiled);
} catch (err) {
logger.warn(
@@ -86,26 +80,10 @@ function createDependency(
updateDependency(field, groups[field]);
}
}
- dependency.replaceString = String(matchResult[0]);
+ dependency.replaceString = replaceString;
return dependency;
}
-function mergeDependency(deps: PackageDependency[]): PackageDependency {
- const result: PackageDependency = {};
- deps.forEach((dep) => {
- mergeFields.forEach((field) => {
- if (dep[field]) {
- result[field] = dep[field];
- // save the line replaceString of the section which contains the current Value for a speed up lookup during the replace phase
- if (field === 'currentValue') {
- result.replaceString = dep.replaceString;
- }
- }
- });
- });
- return result;
-}
-
function handleAny(
content: string,
packageFile: string,
@@ -114,25 +92,29 @@ function handleAny(
return config.matchStrings
.map((matchString) => regEx(matchString, 'g'))
.flatMap((regex) => regexMatchAll(regex, content)) // match all regex to content, get all matches, reduce to single array
- .map((matchResult) => createDependency(matchResult, null, config));
+ .map((matchResult) =>
+ createDependency(
+ { groups: matchResult.groups, replaceString: matchResult[0] },
+ config
+ )
+ );
}
function mergeGroups(
mergedGroup: Record<string, string>,
secondGroup: Record<string, string>
): Record<string, string> {
- const resultGroup = Object.create(null); // prevent prototype pollution
+ return { ...mergedGroup, ...secondGroup };
+}
- Object.keys(mergedGroup).forEach(
- // eslint-disable-next-line no-return-assign
- (key) => (resultGroup[key] = mergedGroup[key])
- );
- Object.keys(secondGroup).forEach((key) => {
- if (secondGroup[key] && secondGroup[key] !== '') {
- resultGroup[key] = secondGroup[key];
- }
- });
- return resultGroup;
+export function mergeExtractionTemplate(
+ base: ExtractionTemplate,
+ addition: ExtractionTemplate
+): ExtractionTemplate {
+ return {
+ groups: mergeGroups(base.groups, addition.groups),
+ replaceString: addition.replaceString ?? base.replaceString,
+ };
}
function handleCombination(
@@ -148,20 +130,13 @@ function handleCombination(
return [];
}
- const combinedGroup = matches
- .map((match) => match.groups)
- .reduce((mergedGroup, currentGroup) =>
- mergeGroups(mergedGroup, currentGroup)
- );
-
- // TODO: this seems to be buggy behavior, needs to be checked #11387
- const dep = matches
- .map((match) => createDependency(match, combinedGroup, config))
- .reduce(
- (mergedDep, currentDep) => mergeDependency([mergedDep, currentDep]),
- {}
- ); // merge fields of dependencies
- return [dep];
+ const extraction = matches
+ .map((match) => ({
+ groups: match.groups,
+ replaceString: match?.groups?.currentValue ? match[0] : undefined,
+ }))
+ .reduce((base, addition) => mergeExtractionTemplate(base, addition));
+ return [createDependency(extraction, config)];
}
function handleRecursive(
@@ -182,8 +157,10 @@ function handleRecursive(
// if we have a depName and a currentValue with have the minimal viable definition
if (match?.groups?.depName && match?.groups?.currentValue) {
return createDependency(
- match,
- mergeGroups(combinedGroups, match.groups),
+ {
+ groups: mergeGroups(combinedGroups, match.groups),
+ replaceString: match[0],
+ },
config
);
}
diff --git a/lib/manager/regex/types.ts b/lib/manager/regex/types.ts
new file mode 100644
index 0000000000..f75929b912
--- /dev/null
+++ b/lib/manager/regex/types.ts
@@ -0,0 +1,4 @@
+export interface ExtractionTemplate {
+ groups: Record<string, string>;
+ replaceString: string;
+}
--
GitLab