diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index ba8174893dfc8fa1ed385d31429a73ba89a6f9e8..1565fc70d33710d8b2de89a26a6d667d0ea0f96d 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -1172,6 +1172,49 @@ By default, Renovate will detect if it has proposed an update to a project befor
Typically you shouldn't need to modify this setting.
+## regexManagers
+
+`regexManagers` entries are used to configure the `regex` Manager in Renovate.
+
+Users can define custom managers for cases such as:
+
+- Proprietary file formats or conventions
+- Popular file formats not yet supported as a manager by Renovate
+
+The custom manager concept is based on using Regular Expression named capture groups. For the fields `datasource`, `depName` and `currentValue`, it's mandatory to have either a named capture group matching them (e.g. `(?<depName>.*)`) or to configure it's corresponding template (e.g. `depNameTemplate`). It's not recommended to do both, due to the potential for confusion. It is recommended to also include `versioning` however if it is missing then it will default to `semver`.
+
+For more details and examples, see the documentation page the for the regex manager [here](/modules/managers/regex/).
+
+### matchStrings
+
+`matchStrings` should each be a valid regular expression, optionally with named capture groups. Currently only a length of one `matchString` is supported.
+
+Example:
+
+```json
+{
+ "matchStrings": [
+ "ENV .*?_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)\\s"
+ ]
+}
+```
+
+### depNameTemplate
+
+If `depName` cannot be captured with a named capture group in `matchString` then it can be defined manually using this field. It will be compiled using `handlebars` and the regex `groups` result.
+
+### lookupNameTemplate
+
+`lookupName` is used for looking up dependency versions. It will be compiled using `handlebars` and the regex `groups` result. It will default to the value of `depName` if left unconfigured/undefined.
+
+### datasourceTemplate
+
+If the `datasource` for a dependency is not captured with a named group then it can be defined in config using this field. It will be compiled using `handlebars` and the regex `groups` result.
+
+### versioningTemplate
+
+If the `versioning` for a dependency is not captured with a named group then it can be defined in config using this field. It will be compiled using `handlebars` and the regex `groups` result.
+
## registryUrls
This is only necessary in case you need to manually configure a registry URL to use for datasource lookups. Applies to PyPI (pip) only for now. Supports only one URL for now but is defined as a list for forward compatibility.
diff --git a/lib/config/__snapshots__/validation.spec.ts.snap b/lib/config/__snapshots__/validation.spec.ts.snap
index 1cefb6bf69df24f6e69254e243fd1833347507ca..83336f9b5cb4fa5e3ae6296495f16f666cc57ee5 100644
--- a/lib/config/__snapshots__/validation.spec.ts.snap
+++ b/lib/config/__snapshots__/validation.spec.ts.snap
@@ -75,6 +75,15 @@ Array [
]
`;
+exports[`config/validation validateConfig(config) errors if regexManager fields are missing 1`] = `
+Array [
+ Object {
+ "depName": "Configuration Error",
+ "message": "Regex Managers must contain currentValueTemplate configuration or regex group named currentValue",
+ },
+]
+`;
+
exports[`config/validation validateConfig(config) ignore packageRule nesting validation for presets 1`] = `Array []`;
exports[`config/validation validateConfig(config) included managers of the wrong type 1`] = `
diff --git a/lib/config/common.ts b/lib/config/common.ts
index 993c140d8d0ac8c5a60c0dcfde40189d8b61f658..59e162a31f8e25f2f4fdb2d7b7abcafdc635aaef 100644
--- a/lib/config/common.ts
+++ b/lib/config/common.ts
@@ -102,6 +102,14 @@ export type RenovateRepository =
repository: string;
};
+export interface CustomManager {
+ matchStrings: string[];
+ depNameTemplate?: string;
+ datasourceTemplate?: string;
+ lookupNameTemplate?: string;
+ versioningTemplate?: string;
+}
+
// TODO: Proper typings
export interface RenovateConfig
extends RenovateAdminConfig,
@@ -140,6 +148,7 @@ export interface RenovateConfig
warnings?: ValidationMessage[];
vulnerabilityAlerts?: RenovateSharedConfig;
+ regexManagers?: CustomManager[];
}
export type UpdateType =
diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts
index 52630e0bb7096fafddeb6346ae30efe1d0723626..b77ae0962bf39857d4d42a1bc21554fb70b77fb4 100644
--- a/lib/config/definitions.ts
+++ b/lib/config/definitions.ts
@@ -28,7 +28,7 @@ export interface RenovateOptionBase {
name: string;
- parent?: 'hostRules' | 'packageRules' | 'postUpgradeTasks';
+ parent?: 'hostRules' | 'packageRules' | 'postUpgradeTasks' | 'regexManagers';
// used by tests
relatedOptions?: string[];
@@ -1675,6 +1675,63 @@ const options: RenovateOptions[] = [
type: 'boolean',
default: false,
},
+ {
+ name: 'regexManagers',
+ description: 'Custom managers using regex matching.',
+ type: 'array',
+ subType: 'object',
+ default: [],
+ stage: 'package',
+ cli: true,
+ mergeable: true,
+ },
+ {
+ name: 'matchStrings',
+ description:
+ 'Regex capture rule to use. Valid only within `regexManagers` object.',
+ type: 'array',
+ subType: 'string',
+ format: 'regex',
+ parent: 'regexManagers',
+ cli: false,
+ env: false,
+ },
+ {
+ name: 'depNameTemplate',
+ description:
+ 'Optional depName for extracted dependencies. Valid only within `regexManagers` object.',
+ type: 'string',
+ parent: 'regexManagers',
+ cli: false,
+ env: false,
+ },
+ {
+ name: 'lookupNameTemplate',
+ description:
+ 'Optional lookupName for extracted dependencies, else defaults to depName value. Valid only within `regexManagers` object.',
+ type: 'string',
+ parent: 'regexManagers',
+ cli: false,
+ env: false,
+ },
+ {
+ name: 'datasourceTemplate',
+ description:
+ 'Optional datasource for extracted dependencies. Valid only within `regexManagers` object.',
+ type: 'string',
+ parent: 'regexManagers',
+ cli: false,
+ env: false,
+ },
+ {
+ name: 'versioningTemplate',
+ description:
+ 'Optional versioning for extracted dependencies. Valid only within `regexManagers` object.',
+ type: 'string',
+ parent: 'regexManagers',
+ cli: false,
+ env: false,
+ },
];
export function getOptions(): RenovateOptions[] {
diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts
index 3e20b4db233674825fabf6f2eabc2792a75267fc..494b9f292a70b0832daaf3d90d77affb9aa84675 100644
--- a/lib/config/validation.spec.ts
+++ b/lib/config/validation.spec.ts
@@ -175,5 +175,88 @@ describe('config/validation', () => {
expect(warnings).toHaveLength(0);
expect(errors).toHaveLength(1);
});
+ it('errors if no regexManager matchStrings', async () => {
+ const config = {
+ regexManagers: [
+ {
+ matchStrings: [],
+ },
+ ],
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ config,
+ true
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(1);
+ });
+ it('validates regEx for each matchStrings', async () => {
+ const config = {
+ regexManagers: [
+ {
+ matchStrings: ['***$}{]]['],
+ },
+ ],
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ config,
+ true
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(1);
+ });
+ it('passes if regexManager fields are present', async () => {
+ const config = {
+ regexManagers: [
+ {
+ matchStrings: ['ENV (?<currentValue>.*?)\\s'],
+ depNameTemplate: 'foo',
+ datasourceTemplate: 'bar',
+ },
+ ],
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ config,
+ true
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(0);
+ });
+ it('errors if extra regexManager fields are present', async () => {
+ const config = {
+ regexManagers: [
+ {
+ matchStrings: ['ENV (?<currentValue>.*?)\\s'],
+ depNameTemplate: 'foo',
+ datasourceTemplate: 'bar',
+ automerge: true,
+ },
+ ],
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ config,
+ true
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toHaveLength(1);
+ });
+ it('errors if regexManager fields are missing', async () => {
+ const config = {
+ regexManagers: [
+ {
+ matchStrings: ['ENV (.*?)\\s'],
+ depNameTemplate: 'foo',
+ datasourceTemplate: 'bar',
+ },
+ ],
+ };
+ const { warnings, errors } = await configValidation.validateConfig(
+ config,
+ true
+ );
+ expect(warnings).toHaveLength(0);
+ expect(errors).toMatchSnapshot();
+ expect(errors).toHaveLength(1);
+ });
});
});
diff --git a/lib/config/validation.ts b/lib/config/validation.ts
index 694ec8f9e53125fd64ae309411a8be0ca9e3b194..4e6047fc6cc660e1ca018e54d40d2f800a1d7344 100644
--- a/lib/config/validation.ts
+++ b/lib/config/validation.ts
@@ -75,7 +75,7 @@ export async function validateConfig(
'prTitle',
'semanticCommitScope',
];
- if (templateKeys.includes(key) && val) {
+ if ((key.endsWith('Template') || templateKeys.includes(key)) && val) {
try {
let res = handlebars.compile(val)(config);
res = handlebars.compile(res)(config);
@@ -201,6 +201,72 @@ export async function validateConfig(
}
}
}
+ if (key === 'regexManagers') {
+ const allowedKeys = [
+ 'fileMatch',
+ 'matchStrings',
+ 'depNameTemplate',
+ 'lookupNameTemplate',
+ 'datasourceTemplate',
+ 'versioningTemplate',
+ ];
+ for (const regexManager of val) {
+ if (
+ Object.keys(regexManager).some(k => !allowedKeys.includes(k))
+ ) {
+ const disallowedKeys = Object.keys(regexManager).filter(
+ k => !allowedKeys.includes(k)
+ );
+ errors.push({
+ depName: 'Configuration Error',
+ message: `Regex Manager contains disallowed fields: ${disallowedKeys.join(
+ ', '
+ )}`,
+ });
+ } else if (
+ !regexManager.matchStrings ||
+ regexManager.matchStrings.length !== 1
+ ) {
+ errors.push({
+ depName: 'Configuration Error',
+ message: `Regex Manager ${currentPath} must contain a matchStrings array of length one`,
+ });
+ } else {
+ let validRegex = false;
+ for (const matchString of regexManager.matchStrings) {
+ try {
+ regEx(matchString);
+ validRegex = true;
+ } catch (e) {
+ errors.push({
+ depName: 'Configuration Error',
+ message: `Invalid regExp for ${currentPath}: \`${matchString}\``,
+ });
+ }
+ }
+ if (validRegex) {
+ const mandatoryFields = [
+ 'depName',
+ 'currentValue',
+ 'datasource',
+ ];
+ for (const field of mandatoryFields) {
+ if (
+ !regexManager[`${field}Template`] &&
+ !regexManager.matchStrings.some(matchString =>
+ matchString.includes(`(?<${field}>`)
+ )
+ ) {
+ errors.push({
+ depName: 'Configuration Error',
+ message: `Regex Managers must contain ${field}Template configuration or regex group named ${field}`,
+ });
+ }
+ }
+ }
+ }
+ }
+ }
if (key === 'packagePatterns' || key === 'excludePackagePatterns') {
for (const pattern of val) {
if (pattern !== '*') {
diff --git a/lib/manager/common.ts b/lib/manager/common.ts
index 1ffb55469b99e9496ad86c5f6e2c1bc7cc28c3d9..bcbab989b6a6b94060bf100dc5b550bd04871b94 100644
--- a/lib/manager/common.ts
+++ b/lib/manager/common.ts
@@ -26,6 +26,14 @@ export interface ExtractConfig extends ManagerConfig {
versioning?: string;
}
+export interface CustomExtractConfig extends ExtractConfig {
+ matchStrings: string[];
+ depNameTemplate?: string;
+ lookupNameTemplate?: string;
+ datasourceTemplate?: string;
+ versioningTemplate?: string;
+}
+
export interface UpdateArtifactsConfig extends ManagerConfig {
isLockFileMaintenance?: boolean;
compatibility?: Record<string, string>;
@@ -91,6 +99,7 @@ export interface PackageFile<T = Record<string, any>>
skipInstalls?: boolean;
yarnrc?: string;
yarnWorkspacesPackages?: string[] | string;
+ matchStrings?: string[];
}
export interface Package<T> extends ManagerData<T> {
@@ -216,13 +225,13 @@ export interface ManagerApi {
config: PackageUpdateConfig
): Result<PackageUpdateResult[]>;
- getRangeStrategy(config: RangeConfig): RangeStrategy;
+ getRangeStrategy?(config: RangeConfig): RangeStrategy;
updateArtifacts?(
updateArtifact: UpdateArtifact
): Result<UpdateArtifactsResult[] | null>;
- updateDependency(
+ updateDependency?(
updateDependencyConfig: UpdateDependencyConfig
): Result<string | null>;
}
diff --git a/lib/manager/index.ts b/lib/manager/index.ts
index 1cbb374fd2fac2245aa91aeebca7914e05af0eeb..4d419aac9f1fbc9d3d82429ac0ebe05fc10a945c 100644
--- a/lib/manager/index.ts
+++ b/lib/manager/index.ts
@@ -43,7 +43,6 @@ function validateManager(manager): boolean {
}
const managers = loadModules<ManagerApi>(__dirname, validateManager);
-
const managerList = Object.keys(managers);
const languageList = [
diff --git a/lib/manager/regex/__fixtures__/Dockerfile b/lib/manager/regex/__fixtures__/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..f75da5532d18c0a8d22c2db476ba419d8ddb1b6a
--- /dev/null
+++ b/lib/manager/regex/__fixtures__/Dockerfile
@@ -0,0 +1,251 @@
+FROM amd64/node:10.19.0@sha256:a9d108f82e34c84e6e2a9901fda2048b9f5a40f614c3ea1348cbf276a7c2031c AS tsbuild
+
+COPY package.json .
+COPY yarn.lock .
+COPY tools tools
+RUN yarn install --frozen-lockfile
+
+COPY lib lib
+COPY tsconfig.json tsconfig.json
+COPY tsconfig.app.json tsconfig.app.json
+
+RUN yarn build:docker
+
+
+FROM amd64/ubuntu:18.04@sha256:0925d086715714114c1988f7c947db94064fd385e171a63c07730f1fa014e6f9
+
+LABEL maintainer="Rhys Arkins <rhys@arkins.net>"
+LABEL name="renovate"
+LABEL org.opencontainers.image.source="https://github.com/renovatebot/renovate"
+
+ENV APP_ROOT=/usr/src/app
+WORKDIR ${APP_ROOT}
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN apt-get update && \
+ apt-get install -y gpg curl wget unzip xz-utils openssh-client bsdtar build-essential openjdk-11-jre-headless dirmngr && \
+ rm -rf /var/lib/apt/lists/*
+
+# The git version of ubuntu 18.04 is too old to sort ref tags properly (see #5477), so update it to the latest stable version
+RUN echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu bionic main\ndeb-src http://ppa.launchpad.net/git-core/ppa/ubuntu bionic main" > /etc/apt/sources.list.d/git.list && \
+ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E1DD270288B4E6030699E45FA1715D88E1DF1F24 && \
+ apt-get update && \
+ apt-get -y install git && \
+ rm -rf /var/lib/apt/lists/*
+
+## Gradle (needs java-jre, installed above)
+ENV GRADLE_VERSION=6.2 # gradle-version/gradle&versioning=maven
+
+RUN wget --no-verbose https://services.gradle.org/distributions/gradle-$GRADLE_VERSION-bin.zip && \
+ unzip -q -d /opt/ gradle-$GRADLE_VERSION-bin.zip && \
+ rm -f gradle-$GRADLE_VERSION-bin.zip && \
+ mv /opt/gradle-$GRADLE_VERSION /opt/gradle && \
+ ln -s /opt/gradle/bin/gradle /usr/local/bin/gradle
+
+## Node.js
+
+# START copy Node.js from https://github.com/nodejs/docker-node/blob/master/10/jessie/Dockerfile
+
+ENV NODE_VERSION=10.19.0 # github-tags/nodejs/node&versioning=node
+
+RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
+ && case "${dpkgArch##*-}" in \
+ amd64) ARCH='x64';; \
+ ppc64el) ARCH='ppc64le';; \
+ s390x) ARCH='s390x';; \
+ arm64) ARCH='arm64';; \
+ armhf) ARCH='armv7l';; \
+ i386) ARCH='x86';; \
+ *) echo "unsupported architecture"; exit 1 ;; \
+ esac \
+ # gpg keys listed at https://github.com/nodejs/node#release-keys
+ && set -ex \
+ && for key in \
+ 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
+ FD3A5288F042B6850C66B31F09FE44734EB7990E \
+ 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
+ DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
+ C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
+ B9AE9905FFD7803F25714661B63B535A4C206CA9 \
+ 77984A986EBC2AA786BC0F66B01FBB92821C587A \
+ 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
+ 4ED778F539E3634C779C87C6D7062848A1AB005C \
+ A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
+ B9E2F5981AA6E0CD28160D9FF13993A75599653C \
+ ; do \
+ gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \
+ gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
+ gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \
+ done \
+ && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
+ && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
+ && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
+ && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
+ && bsdtar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
+ && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
+ && ln -s /usr/local/bin/node /usr/local/bin/nodejs
+
+## END copy Node.js
+
+# Erlang
+
+RUN cd /tmp && \
+ curl https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb -o erlang-solutions_1.0_all.deb && \
+ dpkg -i erlang-solutions_1.0_all.deb && \
+ rm -f erlang-solutions_1.0_all.deb
+
+ENV ERLANG_VERSION=22.0.2-1
+
+RUN apt-get update && \
+ apt-cache policy esl-erlang && \
+ apt-get install -y esl-erlang=1:$ERLANG_VERSION && \
+ rm -rf /var/lib/apt/lists/*
+
+# Elixir
+
+ENV ELIXIR_VERSION=1.8.2
+
+RUN curl -L https://github.com/elixir-lang/elixir/releases/download/v${ELIXIR_VERSION}/Precompiled.zip -o Precompiled.zip && \
+ mkdir -p /opt/elixir-${ELIXIR_VERSION}/ && \
+ unzip Precompiled.zip -d /opt/elixir-${ELIXIR_VERSION}/ && \
+ rm Precompiled.zip
+
+ENV PATH=$PATH:/opt/elixir-${ELIXIR_VERSION}/bin
+
+# PHP Composer
+
+RUN apt-get update && apt-get install -y php-cli php-mbstring && \
+ rm -rf /var/lib/apt/lists/*
+
+ENV COMPOSER_VERSION=1.9.3 # github-releases/composer/composer
+
+RUN php -r "copy('https://github.com/composer/composer/releases/download/$COMPOSER_VERSION/composer.phar', '/usr/local/bin/composer');"
+
+RUN chmod +x /usr/local/bin/composer
+
+# Go Modules
+
+RUN apt-get update && apt-get install -y bzr mercurial && \
+ rm -rf /var/lib/apt/lists/*
+
+ENV GOLANG_VERSION=1.13.4
+
+# Disable GOPROXY and GOSUMDB until we offer a solid solution to configure
+# private repositories.
+ENV GOPROXY=direct GOSUMDB=off
+
+RUN wget -q -O go.tgz "https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz" && \
+ tar -C /usr/local -xzf go.tgz && \
+ rm go.tgz && \
+ export PATH="/usr/local/go/bin:$PATH"
+
+ENV GOPATH=/go
+ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
+
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+ENV CGO_ENABLED=0
+
+# Python
+
+RUN apt-get update && apt-get install -y python3.8-dev python3.8-venv python3-distutils && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN rm -fr /usr/bin/python3 && ln /usr/bin/python3.8 /usr/bin/python3
+RUN rm -rf /usr/bin/python && ln /usr/bin/python3.8 /usr/bin/python
+
+# Pip
+
+RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python
+
+# CocoaPods
+RUN apt-get update && apt-get install -y ruby ruby2.5-dev && rm -rf /var/lib/apt/lists/*
+RUN ruby --version
+ENV COCOAPODS_VERSION=1.9.0 # rubygems/cocoapods&versioning=ruby
+RUN gem install --no-rdoc --no-ri cocoapods -v ${COCOAPODS_VERSION}
+
+# Set up ubuntu user and home directory with access to users in the root group (0)
+
+ENV HOME=/home/ubuntu
+RUN groupadd --gid 1000 ubuntu && \
+ useradd --uid 1000 --gid ubuntu --groups 0 --shell /bin/bash --home-dir ${HOME} --create-home ubuntu
+
+
+RUN chown -R ubuntu:0 ${APP_ROOT} ${HOME} && \
+ chmod -R g=u ${APP_ROOT} ${HOME}
+
+# Docker client and group
+
+RUN groupadd -g 999 docker
+RUN usermod -aG docker ubuntu
+
+ENV DOCKER_VERSION=19.03.1 # github-releases/docker/docker-ce&versioning=docker
+
+RUN curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz \
+ && tar xzvf docker-${DOCKER_VERSION}.tgz --strip 1 \
+ -C /usr/local/bin docker/docker \
+ && rm docker-${DOCKER_VERSION}.tgz
+
+USER ubuntu
+
+# Cargo
+
+ENV RUST_BACKTRACE=1 \
+ PATH=${HOME}/.cargo/bin:$PATH
+
+ENV RUST_VERSION=1.36.0
+
+RUN set -ex ;\
+ curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --profile minimal --default-toolchain ${RUST_VERSION} -y
+
+# Mix and Rebar
+
+RUN mix local.hex --force
+RUN mix local.rebar --force
+
+# Pipenv
+
+ENV PATH="${HOME}/.local/bin:$PATH"
+
+RUN pip install --user pipenv
+
+# Poetry
+
+ENV POETRY_VERSION=1.0.0 # github-releases/python-poetry/poetry
+
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - --version ${POETRY_VERSION}
+
+ENV PATH="${HOME}/.poetry/bin:$PATH"
+RUN poetry config virtualenvs.in-project false
+
+# npm
+
+ENV NPM_VERSION=6.10.2 # npm/npm
+
+RUN npm install -g npm@$NPM_VERSION
+
+# Yarn
+
+ENV YARN_VERSION=1.19.1 # npm/yarn
+
+RUN curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version ${YARN_VERSION}
+
+ENV PATH="${HOME}/.yarn/bin:${HOME}/.config/yarn/global/node_modules/.bin:$PATH"
+
+COPY package.json .
+COPY yarn.lock .
+RUN yarn install --production --frozen-lockfile && yarn cache clean
+RUN rm -f yarn.lock
+COPY --from=tsbuild dist dist
+COPY bin bin
+COPY data data
+
+
+# Numeric user ID for the ubuntu user. Used to indicate a non-root user to OpenShift
+USER 1000
+
+ENTRYPOINT ["node", "/usr/src/app/dist/renovate.js"]
+CMD []
diff --git a/lib/manager/regex/__snapshots__/index.spec.ts.snap b/lib/manager/regex/__snapshots__/index.spec.ts.snap
new file mode 100644
index 0000000000000000000000000000000000000000..3cb01713fcf099de06d7c331232b80d9260a011d
--- /dev/null
+++ b/lib/manager/regex/__snapshots__/index.spec.ts.snap
@@ -0,0 +1,107 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`manager/custom/extract extracts multiple dependencies 1`] = `
+Object {
+ "deps": Array [
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 0,
+ "replaceString": "ENV GRADLE_VERSION=6.2 # gradle-version/gradle&versioning=maven
+",
+ },
+ "currentValue": "6.2",
+ "datasource": "gradle-version",
+ "depName": "gradle",
+ "lookupName": undefined,
+ "versioning": "maven",
+ },
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 1,
+ "replaceString": "ENV NODE_VERSION=10.19.0 # github-tags/nodejs/node&versioning=node
+",
+ },
+ "currentValue": "10.19.0",
+ "datasource": "github-tags",
+ "depName": "nodejs/node",
+ "lookupName": undefined,
+ "versioning": "node",
+ },
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 2,
+ "replaceString": "ENV COMPOSER_VERSION=1.9.3 # github-releases/composer/composer
+",
+ },
+ "currentValue": "1.9.3",
+ "datasource": "github-releases",
+ "depName": "composer/composer",
+ "lookupName": undefined,
+ "versioning": "semver",
+ },
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 3,
+ "replaceString": "ENV COCOAPODS_VERSION=1.9.0 # rubygems/cocoapods&versioning=ruby
+",
+ },
+ "currentValue": "1.9.0",
+ "datasource": "rubygems",
+ "depName": "cocoapods",
+ "lookupName": undefined,
+ "versioning": "ruby",
+ },
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 4,
+ "replaceString": "ENV DOCKER_VERSION=19.03.1 # github-releases/docker/docker-ce&versioning=docker
+",
+ },
+ "currentValue": "19.03.1",
+ "datasource": "github-releases",
+ "depName": "docker/docker-ce",
+ "lookupName": undefined,
+ "versioning": "docker",
+ },
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 5,
+ "replaceString": "ENV POETRY_VERSION=1.0.0 # github-releases/python-poetry/poetry
+",
+ },
+ "currentValue": "1.0.0",
+ "datasource": "github-releases",
+ "depName": "python-poetry/poetry",
+ "lookupName": undefined,
+ "versioning": "semver",
+ },
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 6,
+ "replaceString": "ENV NPM_VERSION=6.10.2 # npm/npm
+",
+ },
+ "currentValue": "6.10.2",
+ "datasource": "npm",
+ "depName": "npm",
+ "lookupName": undefined,
+ "versioning": "semver",
+ },
+ Object {
+ "autoReplaceData": Object {
+ "depIndex": 7,
+ "replaceString": "ENV YARN_VERSION=1.19.1 # npm/yarn
+",
+ },
+ "currentValue": "1.19.1",
+ "datasource": "npm",
+ "depName": "yarn",
+ "lookupName": undefined,
+ "versioning": "semver",
+ },
+ ],
+ "matchStrings": Array [
+ "ENV .*?_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\\\&versioning=(?<versioning>.*?))?\\\\s",
+ ],
+}
+`;
diff --git a/lib/manager/regex/index.spec.ts b/lib/manager/regex/index.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..71cc814ae8cfebf1f823547c38af045f75e08149
--- /dev/null
+++ b/lib/manager/regex/index.spec.ts
@@ -0,0 +1,57 @@
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+import { extractPackageFile } from '.';
+
+const dockerfileContent = readFileSync(
+ resolve(__dirname, `./__fixtures__/Dockerfile`),
+ 'utf8'
+);
+describe('manager/custom/extract', () => {
+ it('extracts multiple dependencies', async () => {
+ const config = {
+ matchStrings: [
+ 'ENV .*?_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\&versioning=(?<versioning>.*?))?\\s',
+ ],
+ versioningTemplate:
+ '{{#if versioning}}{{versioning}}{{else}}semver{{/if}}',
+ };
+ const res = await extractPackageFile(
+ dockerfileContent,
+ 'Dockerfile',
+ config
+ );
+ expect(res).toMatchSnapshot();
+ expect(res.deps).toHaveLength(8);
+ expect(res.deps.find(dep => dep.depName === 'yarn').versioning).toEqual(
+ 'semver'
+ );
+ expect(res.deps.find(dep => dep.depName === 'gradle').versioning).toEqual(
+ 'maven'
+ );
+ });
+ it('returns null if no dependencies found', async () => {
+ const config = {
+ matchStrings: [
+ 'ENV .*?_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\&versioning=(?<versioning>.*?))?\\s',
+ ],
+ versioningTemplate:
+ '{{#if versioning}}{{versioning}}{{else}}semver{{/if}}',
+ };
+ const res = await extractPackageFile('', 'Dockerfile', config);
+ expect(res).toBeNull();
+ });
+ it('returns null if invalid handlebars template', async () => {
+ const config = {
+ matchStrings: [
+ 'ENV .*?_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\&versioning=(?<versioning>.*?))?\\s',
+ ],
+ versioningTemplate: '{{#if versioning}}{{versioning}}{{else}}semver',
+ };
+ const res = await extractPackageFile(
+ dockerfileContent,
+ 'Dockerfile',
+ config
+ );
+ expect(res).toBeNull();
+ });
+});
diff --git a/lib/manager/regex/index.ts b/lib/manager/regex/index.ts
new file mode 100644
index 0000000000000000000000000000000000000000..866c0ed6ceeda713ac349c514ec1781373a0dbb4
--- /dev/null
+++ b/lib/manager/regex/index.ts
@@ -0,0 +1,57 @@
+import * as handlebars from 'handlebars';
+import { CustomExtractConfig, PackageFile, Result } from '../common';
+import { regEx } from '../../util/regex';
+import { logger } from '../../logger';
+
+export const autoReplace = true;
+
+export const defaultConfig = {};
+
+export function extractPackageFile(
+ content: string,
+ packageFile: string,
+ config: CustomExtractConfig
+): Result<PackageFile | null> {
+ const regexMatch = regEx(config.matchStrings[0], 'g');
+ const deps = [];
+ let matchResult;
+ let depIndex = 0;
+ do {
+ matchResult = regexMatch.exec(content);
+ if (matchResult) {
+ const dep: any = {};
+ const { groups } = matchResult;
+ const fields = [
+ 'depName',
+ 'lookupName',
+ 'currentValue',
+ 'datasource',
+ 'versioning',
+ ];
+ for (const field of fields) {
+ const fieldTemplate = `${field}Template`;
+ if (config[fieldTemplate]) {
+ try {
+ dep[field] = handlebars.compile(config[fieldTemplate])(groups);
+ } catch (err) {
+ logger.warn(
+ { template: config[fieldTemplate] },
+ 'Error compiling handlebars template for custom manager'
+ );
+ return null;
+ }
+ } else {
+ dep[field] = groups[field];
+ }
+ }
+ dep.autoReplaceData = {
+ depIndex,
+ replaceString: `${matchResult[0]}`,
+ };
+ deps.push(dep);
+ }
+ depIndex += 1;
+ } while (matchResult);
+ if (deps.length) return { deps, matchStrings: config.matchStrings };
+ return null;
+}
diff --git a/lib/manager/regex/readme.md b/lib/manager/regex/readme.md
new file mode 100644
index 0000000000000000000000000000000000000000..f13653051cd1ae823fb8ebf2d9eaa12b2cba6433
--- /dev/null
+++ b/lib/manager/regex/readme.md
@@ -0,0 +1,88 @@
+The `regex` manager is designed to allow users to manually configure Renovate for how to find dependencies that aren't detected by the built-in package managers.
+
+This manager is unique in Renovate in that:
+
+- It is configurable via regex named capture groups
+- Through the use of the `regexManagers` config, multiple "regex managers" can be created for the same repository.
+
+### Required Fields
+
+The first two required fields are `fileMatch` and `matchStrings`. `fileMatch` works the same as any manager, while `matchStrings` is a `regexManagers` concept and is used for configuring a regular expression with named capture groups.
+
+In order for Renovate to look up a dependency and decide about updates, it then needs the following information about each dependency:
+
+- The dependency's name
+- Which `datasource` to look up (e.g. npm, Docker, GitHub tags, etc)
+- Which version scheme to apply (defaults to `semver`, but also may be other values like `pep440`)
+
+Configuration-wise, it works like this:
+
+- You must capture the `currentValue` of the dependency in a named capture group
+- You must have either a `depName` capture group or a `depNameTemplate` config field
+- You can optionally have a `lookupName` capture group or a `lookupNameTemplate` if it differs from `depName`
+- You must have either a `datasource` capture group or a `datasourceTemplate` config field
+- You can optionally have a `versioning` capture group or a `versioningTemplate` config field. If neither are present, `semver` will be used as the default
+
+### Regular Expression Capture Groups
+
+To be fully effective with the regex manager, you will need to understand regular expressions and named capture groups, although sometimes enough examples can compensate for lack of experience.
+
+Consider this `Dockerfile`:
+
+```
+FROM node:12
+ENV YARN_VERSION=1.19.1
+RUN curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version ${YARN_VERSION}
+```
+
+You would need to capture the `currentValue` using a named capture group, like so: `ENV YARN_VERSION=(?<currentValue>.*?)\n`.
+
+If you're looking for an online regex testing tool that supports capture groups, try [https://regex101.com/](https://regex101.com/).
+
+### Configuration templates
+
+In many cases, named capture groups alone won't be enough and you'll need to configure Renovate with additional information about how to look up a dependency. Continuing the above example with Yarn, here is the full config:
+
+```json
+{
+ "regexManagers": [
+ {
+ "fileMatch": ["^Dockerfile$"],
+ "matchStrings": ["ENV YARN_VERSION=(?<currentValue>.*?)\n"],
+ "depNameTemplate": "yarn",
+ "datasourceTemplate": "npm"
+ }
+ ]
+}
+```
+
+### Advanced Capture
+
+Let's say that your `Dockerfile` has many `ENV` variables you want to keep updated and you prefer not to write one `regexManagers` rule per variable. Instead you could enhance your `Dockerfile` like the following:
+
+```
+ENV NODE_VERSION=10.19.0 # github-tags/nodejs/node&versioning=node
+ENV COMPOSER_VERSION=1.9.3 # github-releases/composer/composer
+ENV DOCKER_VERSION=19.03.1 # github-releases/docker/docker-ce&versioning=docker
+ENV YARN_VERSION=1.19.1 # npm/yarn
+```
+
+The above (obviously not a complete `Dockerfile`, but abbreviated for this example), could then be supported accordingly:
+
+```json
+{
+ "regexManagers": [
+ {
+ "fileMatch": ["^Dockerfile$"],
+ "matchStrings": [
+ "ENV .*?_VERSION=(?<currentValue>.*) # (?<datasource>.*?)/(?<depName>.*?)(\\&versioning=(?<versioning>.*?))?\\s"
+ ],
+ "versioningTemplate": "{{#if versioning}}{{versioning}}{{else}}semver{{/if}}"
+ }
+ ]
+}
+```
+
+In the above the `versioningTemplate` is not actually necessary because Renovate already defaults to `semver` versioning, but it has been included to help illustrate why we call these fields _templates_. They are named this way because they are compiled using `handlebars` and so can be composed from values you collect in named capture groups.
+
+By adding the comments to the `Dockerfile`, you can see that instead of four separate `regexManagers` being required, there is now only one - and the `Dockerfile` itself is now somewhat better documented too. The syntax we used there is completely arbitrary and you may choose your own instead if you prefer - just be sure to update your `matchStrings` regex.
diff --git a/lib/workers/repository/extract/__snapshots__/index.spec.ts.snap b/lib/workers/repository/extract/__snapshots__/index.spec.ts.snap
index c3293e31cf1b3203791acc88238fd03d60c3a67f..310997713ce6dfb22c37062debc2221b51681243 100644
--- a/lib/workers/repository/extract/__snapshots__/index.spec.ts.snap
+++ b/lib/workers/repository/extract/__snapshots__/index.spec.ts.snap
@@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
-exports[`workers/repository/extract/index extractAllDependencies() skips non-enabled maangers 1`] = `
+exports[`workers/repository/extract/index extractAllDependencies() skips non-enabled managers 1`] = `
Object {
"npm": Array [
Object {},
diff --git a/lib/workers/repository/extract/index.spec.ts b/lib/workers/repository/extract/index.spec.ts
index 7014d9ae4b6a9ce48bbfa71a996bcaa917cdb9b4..4703e6f1c1f18d94b84ef564951fe527fa61d9f9 100644
--- a/lib/workers/repository/extract/index.spec.ts
+++ b/lib/workers/repository/extract/index.spec.ts
@@ -19,11 +19,17 @@ describe('workers/repository/extract/index', () => {
const res = await extractAllDependencies(config);
expect(Object.keys(res).includes('ansible')).toBe(true);
});
- it('skips non-enabled maangers', async () => {
+ it('skips non-enabled managers', async () => {
config.enabledManagers = ['npm'];
managerFiles.getManagerPackageFiles.mockResolvedValue([{} as never]);
const res = await extractAllDependencies(config);
expect(res).toMatchSnapshot();
});
+ it('checks custom managers', async () => {
+ managerFiles.getManagerPackageFiles.mockResolvedValue([{} as never]);
+ config.regexManagers = [{ matchStrings: [''] }];
+ const res = await extractAllDependencies(config);
+ expect(Object.keys(res).includes('regex')).toBe(true);
+ });
});
});
diff --git a/lib/workers/repository/extract/index.ts b/lib/workers/repository/extract/index.ts
index 5baf4643b156fab819b83ca617d5318113a3d6b6..639e3144bea5b10ac5e2af471ebae207a6d745b0 100644
--- a/lib/workers/repository/extract/index.ts
+++ b/lib/workers/repository/extract/index.ts
@@ -1,6 +1,10 @@
import { logger } from '../../../logger';
import { getManagerList } from '../../../manager';
-import { getManagerConfig, RenovateConfig } from '../../../config';
+import {
+ getManagerConfig,
+ mergeChildConfig,
+ RenovateConfig,
+} from '../../../config';
import { getManagerPackageFiles } from './manager-files';
import { PackageFile } from '../../../manager/common';
@@ -18,8 +22,24 @@ export async function extractAllDependencies(
continue; // eslint-disable-line
}
const managerConfig = getManagerConfig(config, manager);
+ let packageFiles = [];
+ if (manager === 'regex') {
+ for (const regexManager of config.regexManagers) {
+ const regexManagerConfig = mergeChildConfig(
+ managerConfig,
+ regexManager
+ );
+ const customPackageFiles = await getManagerPackageFiles(
+ regexManagerConfig
+ );
+ if (customPackageFiles) {
+ packageFiles = packageFiles.concat(customPackageFiles);
+ }
+ }
+ } else {
+ packageFiles = await getManagerPackageFiles(managerConfig);
+ }
managerConfig.manager = manager;
- const packageFiles = await getManagerPackageFiles(managerConfig);
if (packageFiles && packageFiles.length) {
fileCount += packageFiles.length;
logger.debug(`Found ${manager} package files`);