From 384a1e7384b39d5417df341e5fc3e23e7138fff4 Mon Sep 17 00:00:00 2001
From: Matt Palmer <9059517+56KBs@users.noreply.github.com>
Date: Wed, 24 Feb 2021 11:58:15 +0000
Subject: [PATCH] feat(regex): support registryUrlTemplate (#8611)
---
docs/usage/configuration-options.md | 15 ++++---
lib/config/definitions.ts | 19 ++++++---
.../regex/__snapshots__/index.spec.ts.snap | 41 +++++++++++++++++++
lib/manager/regex/index.spec.ts | 38 +++++++++++++++++
lib/manager/regex/index.ts | 39 ++++++++++--------
lib/manager/regex/readme.md | 5 ++-
6 files changed, 128 insertions(+), 29 deletions(-)
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index 11f26bfe06..856d7663a3 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -1675,7 +1675,7 @@ It's not recommended to do both, due to the potential for confusion.
It is recommended to also include `versioning` however if it is missing then it will default to `semver`.
For more details and examples, see the documentation page the for the regex manager [here](/modules/manager/regex/).
-For template fields, use the triple brace `{{{ }}}` notation to avoid `handlebars` escaping any special characters.
+For template fields, use the triple brace `{{{ }}}` notation to avoid Handlebars escaping any special characters.
### matchStrings
@@ -1846,23 +1846,28 @@ In the above example, each regex manager will match a single dependency each.
### depNameTemplate
If `depName` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field.
-It will be compiled using `handlebars` and the regex `groups` result.
+It will be compiled using Handlebars and the regex `groups` result.
### lookupNameTemplate
`lookupName` is used for looking up dependency versions.
-It will be compiled using `handlebars` and the regex `groups` result.
+It will be compiled using Handlebars and the regex `groups` result.
It will default to the value of `depName` if left unconfigured/undefined.
### datasourceTemplate
If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field.
-It will be compiled using `handlebars` and the regex `groups` result.
+It will be compiled using Handlebars and the regex `groups` result.
### versioningTemplate
If the `versioning` for a dependency is not captured with a named group then it can be defined in config using this field.
-It will be compiled using `handlebars` and the regex `groups` result.
+It will be compiled using Handlebars and the regex `groups` result.
+
+### registryUrlTemplate
+
+If the `registryUrls` for a dependency is not captured with a named group then it can be defined in config using this field.
+It will be compiled using Handlebars and the regex `groups` result.
## registryUrls
diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts
index 90ed7311fb..4af168253f 100644
--- a/lib/config/definitions.ts
+++ b/lib/config/definitions.ts
@@ -1856,7 +1856,7 @@ const options: RenovateOptions[] = [
{
name: 'matchStrings',
description:
- 'Regex capture rule to use. Valid only within `regexManagers` object.',
+ 'Regex capture rule to use. Valid only within a `regexManagers` object.',
type: 'array',
subType: 'string',
format: 'regex',
@@ -1876,7 +1876,7 @@ const options: RenovateOptions[] = [
{
name: 'depNameTemplate',
description:
- 'Optional depName for extracted dependencies. Valid only within `regexManagers` object.',
+ 'Optional depName for extracted dependencies. Valid only within a `regexManagers` object.',
type: 'string',
parent: 'regexManagers',
cli: false,
@@ -1885,7 +1885,7 @@ const options: RenovateOptions[] = [
{
name: 'lookupNameTemplate',
description:
- 'Optional lookupName for extracted dependencies, else defaults to depName value. Valid only within `regexManagers` object.',
+ 'Optional lookupName for extracted dependencies, else defaults to depName value. Valid only within a `regexManagers` object.',
type: 'string',
parent: 'regexManagers',
cli: false,
@@ -1894,7 +1894,7 @@ const options: RenovateOptions[] = [
{
name: 'datasourceTemplate',
description:
- 'Optional datasource for extracted dependencies. Valid only within `regexManagers` object.',
+ 'Optional datasource for extracted dependencies. Valid only within a `regexManagers` object.',
type: 'string',
parent: 'regexManagers',
cli: false,
@@ -1903,7 +1903,16 @@ const options: RenovateOptions[] = [
{
name: 'versioningTemplate',
description:
- 'Optional versioning for extracted dependencies. Valid only within `regexManagers` object.',
+ 'Optional versioning for extracted dependencies. Valid only within a `regexManagers` object.',
+ type: 'string',
+ parent: 'regexManagers',
+ cli: false,
+ env: false,
+ },
+ {
+ name: 'registryUrlTemplate',
+ description:
+ 'Optional registry URL for extracted dependencies. Valid only within a `regexManagers` object.',
type: 'string',
parent: 'regexManagers',
cli: false,
diff --git a/lib/manager/regex/__snapshots__/index.spec.ts.snap b/lib/manager/regex/__snapshots__/index.spec.ts.snap
index 38786955f9..cc1d94995c 100644
--- a/lib/manager/regex/__snapshots__/index.spec.ts.snap
+++ b/lib/manager/regex/__snapshots__/index.spec.ts.snap
@@ -1,5 +1,46 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
+exports[`manager/regex/index extracts and applies a registryUrlTemplate 1`] = `
+Object {
+ "deps": Array [
+ Object {
+ "currentValue": "6.2",
+ "datasource": "gradle-version",
+ "depName": "gradle",
+ "registryUrls": Array [
+ "http://registry.gradle.com/",
+ ],
+ "replaceString": "ENV GRADLE_VERSION=6.2 # gradle-version/gradle&versioning=maven
+",
+ "versioning": "maven",
+ },
+ ],
+ "matchStrings": Array [
+ "ENV GRADLE_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\\\&versioning=(?<versioning>.*?))?\\\\s",
+ ],
+ "registryUrlTemplate": "http://registry.{{depName}}.com/",
+}
+`;
+
+exports[`manager/regex/index extracts and does not apply a registryUrlTemplate if the result is an invalid url 1`] = `
+Object {
+ "deps": Array [
+ Object {
+ "currentValue": "6.2",
+ "datasource": "gradle-version",
+ "depName": "gradle",
+ "replaceString": "ENV GRADLE_VERSION=6.2 # gradle-version/gradle&versioning=maven
+",
+ "versioning": "maven",
+ },
+ ],
+ "matchStrings": Array [
+ "ENV GRADLE_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\\\&versioning=(?<versioning>.*?))?\\\\s",
+ ],
+ "registryUrlTemplate": "this-is-not-a-valid-url-{{depName}}",
+}
+`;
+
exports[`manager/regex/index extracts extractVersion 1`] = `
Object {
"deps": Array [
diff --git a/lib/manager/regex/index.spec.ts b/lib/manager/regex/index.spec.ts
index a6c47a4d03..4fee27efcd 100644
--- a/lib/manager/regex/index.spec.ts
+++ b/lib/manager/regex/index.spec.ts
@@ -1,6 +1,7 @@
import { readFileSync } from 'fs';
import { resolve } from 'upath';
import { getName } from '../../../test/util';
+import { logger } from '../../logger';
import { CustomExtractConfig } from '../common';
import { defaultConfig, extractPackageFile } from '.';
@@ -115,6 +116,43 @@ describe(getName(__filename), () => {
);
expect(res).toMatchSnapshot();
});
+ it('extracts and applies a registryUrlTemplate', async () => {
+ const config = {
+ matchStrings: [
+ 'ENV GRADLE_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\&versioning=(?<versioning>.*?))?\\s',
+ ],
+ registryUrlTemplate: 'http://registry.{{depName}}.com/',
+ };
+ const res = await extractPackageFile(
+ dockerfileContent,
+ 'Dockerfile',
+ config
+ );
+ expect(res).toMatchSnapshot();
+ expect(res.deps).toHaveLength(1);
+ expect(
+ res.deps.find((dep) => dep.depName === 'gradle').registryUrls
+ ).toEqual(['http://registry.gradle.com/']);
+ });
+ it('extracts and does not apply a registryUrlTemplate if the result is an invalid url', async () => {
+ jest.mock('../../logger');
+ const config = {
+ matchStrings: [
+ 'ENV GRADLE_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\&versioning=(?<versioning>.*?))?\\s',
+ ],
+ registryUrlTemplate: 'this-is-not-a-valid-url-{{depName}}',
+ };
+ const res = await extractPackageFile(
+ dockerfileContent,
+ 'Dockerfile',
+ config
+ );
+ expect(res).toMatchSnapshot();
+ expect(logger.warn).toHaveBeenCalledWith(
+ { value: 'this-is-not-a-valid-url-gradle' },
+ 'Invalid regex manager registryUrl'
+ );
+ });
it('extracts multiple dependencies with multiple matchStrings', async () => {
const config = {
matchStrings: [
diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts
index 6dba5bf84e..f0085b58a9 100644
--- a/lib/manager/regex/index.ts
+++ b/lib/manager/regex/index.ts
@@ -1,4 +1,4 @@
-import url from 'url';
+import { URL } from 'url';
import { logger } from '../../logger';
import { regEx } from '../../util/regex';
import * as template from '../../util/template';
@@ -43,15 +43,30 @@ function createDependency(
): PackageDependency {
const dependency = dep || {};
const { groups } = matchResult;
+
+ function updateDependency(field: string, value: string): void {
+ switch (field) {
+ case 'registryUrl':
+ // check if URL is valid and pack inside an array
+ try {
+ const url = new URL(value).toString();
+ dependency.registryUrls = [url];
+ } catch (err) {
+ logger.warn({ value }, 'Invalid regex manager registryUrl');
+ }
+ break;
+ default:
+ dependency[field] = value;
+ break;
+ }
+ }
+
for (const field of validMatchFields) {
const fieldTemplate = `${field}Template`;
if (config[fieldTemplate]) {
try {
- dependency[field] = template.compile(
- config[fieldTemplate],
- groups,
- false
- );
+ const compiled = template.compile(config[fieldTemplate], groups, false);
+ updateDependency(field, compiled);
} catch (err) {
logger.warn(
{ template: config[fieldTemplate] },
@@ -60,17 +75,7 @@ function createDependency(
return null;
}
} else if (groups[field]) {
- switch (field) {
- case 'registryUrl':
- // check if URL is valid and pack inside an array
- if (url.parse(groups[field])) {
- dependency.registryUrls = [groups[field]];
- }
- break;
- default:
- dependency[field] = groups[field];
- break;
- }
+ updateDependency(field, groups[field]);
}
}
dependency.replaceString = String(matchResult[0]);
diff --git a/lib/manager/regex/readme.md b/lib/manager/regex/readme.md
index 681ccb8916..c7289ff64a 100644
--- a/lib/manager/regex/readme.md
+++ b/lib/manager/regex/readme.md
@@ -24,7 +24,8 @@ Configuration-wise, it works like this:
- You can optionally have a `versioning` capture group or a `versioningTemplate` config field. If neither are present, `semver` will be used as the default
- You can optionally have an `extractVersion` capture group or an `extractVersionTemplate` config field
- You can optionally have a `currentDigest` capture group.
-- You can optionally have a `registryUrl` capture group. If it's a valid URL, it will be converted to the `registryUrls` field as a single-length array.
+- You can optionally have a `registryUrl` capture group or a `registryUrlTemplate` config field
+ - If it's a valid URL, it will be converted to the `registryUrls` field as a single-length array.
### Regular Expression Capture Groups
@@ -99,6 +100,6 @@ The above (obviously not a complete `Dockerfile`, but abbreviated for this examp
}
```
-In the above the `versioningTemplate` is not actually necessary because Renovate already defaults to `semver` versioning, but it has been included to help illustrate why we call these fields _templates_. They are named this way because they are compiled using `handlebars` and so can be composed from values you collect in named capture groups. You will usually want to use the tripe brace `{{{ }}}` template (e.v. `{{{versioning}}}` to be safe because `handlebars` escapes special characters by default with double braces.
+In the above the `versioningTemplate` is not actually necessary because Renovate already defaults to `semver` versioning, but it has been included to help illustrate why we call these fields _templates_. They are named this way because they are compiled using Handlebars and so can be composed from values you collect in named capture groups. You will usually want to use the tripe brace `{{{ }}}` template (e.v. `{{{versioning}}}` to be safe because Handlebars escapes special characters by default with double braces.
By adding the comments to the `Dockerfile`, you can see that instead of four separate `regexManagers` being required, there is now only one - and the `Dockerfile` itself is now somewhat better documented too. The syntax we used there is completely arbitrary and you may choose your own instead if you prefer - just be sure to update your `matchStrings` regex.
--
GitLab