From 5b0c431dceeee00f23790d44eeae1b6d0966efb4 Mon Sep 17 00:00:00 2001
From: Carlin St Pierre <cstpierre@atlassian.com>
Date: Tue, 4 Feb 2020 16:59:13 +1100
Subject: [PATCH] feat: post-upgrade tasks (#5202)
---
docs/usage/configuration-options.md | 27 ++++++++
docs/usage/self-hosted-configuration.md | 12 ++++
lib/config/common.ts | 7 +++
lib/config/definitions.ts | 41 +++++++++++-
lib/workers/branch/index.ts | 83 +++++++++++++++++++++++++
renovate-schema.json | 42 +++++++++++++
test/workers/branch/index.spec.ts | 49 +++++++++++++++
7 files changed, 260 insertions(+), 1 deletion(-)
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index a4fd039f83..de9f0a9459 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -1100,6 +1100,33 @@ Warning: `pipenv` support is currently in beta, so it is not enabled by default.
- `yarnDedupeFewer`: Run `yarn-deduplicate --strategy fewer` after `yarn.lock` updates
- `yarnDedupeHighest`: Run `yarn-deduplicate --strategy highest` after `yarn.lock` updates
+## postUpgradeTasks
+
+Post-upgrade tasks are commands that are executed by Renovate after a dependency has been updated but before the commit is created. The intention is to run any additional command line tools that would modify existing files or generate new files when a dependency changes.
+
+This is only available on Renovate instances that have a `trustLevel` of 'high'. Each command must match at least one of the patterns defined in `allowedPostUpgradeTasks` in order to be executed. If the list of allowed tasks is empty then no tasks will be executed.
+
+e.g.
+
+```json
+{
+ "postUpgradeTasks": {
+ "commands": ["tslint --fix"],
+ "fileFilters": ["yarn.lock", "**/*.js"]
+ }
+}
+```
+
+The `postUpdateTasks` configuration consists of two fields:
+
+### commands
+
+A list of commands that are executed after Renovate has updated a dependency but before the commit it made
+
+### fileFilters
+
+A list of glob-style matchers that determine which files will be included in the final commit made by Renovate
+
## prBodyColumns
Use this array to provide a list of column names you wish to include in the PR tables.
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
index 1df50e1478..9c86f9510f 100644
--- a/docs/usage/self-hosted-configuration.md
+++ b/docs/usage/self-hosted-configuration.md
@@ -7,6 +7,18 @@ description: Self-Hosted Configuration usable in renovate.json or package.json
The below configuration options are applicable only if you are running your own instance ("bot") of Renovate.
+## allowedPostUpgradeCommands
+
+A list of regular expressions that determine which commands in `postUpgradeTasks` are allowed to be executed. If this list is empty then no tasks will be executed.
+
+e.g.
+
+```json
+{
+ "allowedPostUpgradeCommands": ["^tslint --fix$", "^tslint --[a-z]+$"]
+}
+```
+
## autodiscover
Be cautious when using this option - it will run Renovate over _every_ repository that the bot account has access to. To filter this list, use `autodiscoverFilter`.
diff --git a/lib/config/common.ts b/lib/config/common.ts
index 1b91283eb7..8fc2d2baf6 100644
--- a/lib/config/common.ts
+++ b/lib/config/common.ts
@@ -29,8 +29,15 @@ export interface RenovateSharedConfig {
statusCheckVerify?: boolean;
suppressNotifications?: string[];
timezone?: string;
+ allowedPostUpgradeCommands?: string[];
+ postUpgradeTasks?: PostUpgradeTasks;
}
+export type PostUpgradeTasks = {
+ commands?: string[];
+ fileFilters?: string[];
+};
+
type UpdateConfig<
T extends RenovateSharedConfig = RenovateSharedConfig
> = Partial<Record<UpdateType, T>>;
diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts
index 9bf118b65f..b46650e7fe 100644
--- a/lib/config/definitions.ts
+++ b/lib/config/definitions.ts
@@ -45,7 +45,7 @@ export interface RenovateOptionBase {
name: string;
- parent?: 'hostRules' | 'packageRules';
+ parent?: 'hostRules' | 'packageRules' | 'postUpgradeTasks';
// used by tests
relatedOptions?: string[];
@@ -102,6 +102,45 @@ export type RenovateOptions =
| RenovateObjectOption;
const options: RenovateOptions[] = [
+ {
+ name: 'allowedPostUpgradeCommands',
+ description:
+ 'A list of regular expressions that determine which post-upgrade tasks are allowed. A task has to match at least one of the patterns to be allowed to run',
+ type: 'array',
+ subType: 'string',
+ default: [],
+ admin: true,
+ },
+ {
+ name: 'postUpgradeTasks',
+ description:
+ 'Post-upgrade tasks that are executed before a commit is made by Renovate',
+ type: 'object',
+ default: {
+ commands: [],
+ fileFilters: [],
+ },
+ },
+ {
+ name: 'commands',
+ description:
+ 'A list of post-upgrade commands that are executed before a commit is made by Renovate',
+ type: 'array',
+ subType: 'string',
+ parent: 'postUpgradeTasks',
+ default: [],
+ cli: false,
+ },
+ {
+ name: 'fileFilters',
+ description:
+ 'Files that match these glob patterns will be committed if they are present after running a post-upgrade task',
+ type: 'array',
+ subType: 'string',
+ parent: 'postUpgradeTasks',
+ default: [],
+ cli: false,
+ },
{
name: 'onboardingBranch',
description:
diff --git a/lib/workers/branch/index.ts b/lib/workers/branch/index.ts
index 472fe374b0..e5bbc2c180 100644
--- a/lib/workers/branch/index.ts
+++ b/lib/workers/branch/index.ts
@@ -1,5 +1,10 @@
import { DateTime } from 'luxon';
+import _ from 'lodash';
+import { readFile } from 'fs-extra';
+import is from '@sindresorhus/is';
+import minimatch from 'minimatch';
+import { join } from 'upath';
import { logger } from '../../logger';
import { isScheduledNow } from './schedule';
import { getUpdatedPackageFiles } from './get-updated';
@@ -30,6 +35,7 @@ import {
PLATFORM_FAILURE,
} from '../../constants/error-messages';
import { BRANCH_STATUS_FAILURE } from '../../constants/branch-constants';
+import { exec } from '../../util/exec';
export type ProcessBranchResult =
| 'already-existed'
@@ -320,6 +326,83 @@ export async function processBranch(
} else {
logger.debug('No updated lock files in branch');
}
+
+ if (
+ global.trustLevel === 'high' &&
+ is.nonEmptyArray(config.allowedPostUpgradeCommands)
+ ) {
+ logger.debug(
+ {
+ tasks: config.postUpgradeTasks,
+ allowedCommands: config.allowedPostUpgradeCommands,
+ },
+ 'Checking for post-upgrade tasks'
+ );
+ const commands = config.postUpgradeTasks.commands || [];
+ const fileFilters = config.postUpgradeTasks.fileFilters || [];
+
+ if (is.nonEmptyArray(commands)) {
+ for (const cmd of commands) {
+ if (
+ !_.some(config.allowedPostUpgradeCommands, (pattern: string) =>
+ cmd.match(pattern)
+ )
+ ) {
+ logger.warn(
+ {
+ cmd,
+ allowedPostUpgradeCommands: config.allowedPostUpgradeCommands,
+ },
+ 'Post-upgrade task did not match any on allowed list'
+ );
+ } else {
+ logger.debug({ cmd }, 'Executing post-upgrade task');
+
+ const execResult = await exec(cmd, {
+ cwd: config.localDir,
+ });
+
+ logger.debug({ cmd, ...execResult }, 'Executed post-upgrade task');
+ }
+ }
+
+ const status = await platform.getRepoStatus();
+
+ for (const relativePath of status.modified.concat(status.not_added)) {
+ for (const pattern of fileFilters) {
+ if (minimatch(relativePath, pattern)) {
+ logger.debug(
+ { file: relativePath, pattern },
+ 'Post-upgrade file saved'
+ );
+ const existingContent = await readFile(
+ join(config.localDir, relativePath)
+ );
+ config.updatedArtifacts.push({
+ name: relativePath,
+ contents: existingContent.toString(),
+ });
+ }
+ }
+ }
+
+ for (const relativePath of status.deleted || []) {
+ for (const pattern of fileFilters) {
+ if (minimatch(relativePath, pattern)) {
+ logger.debug(
+ { file: relativePath, pattern },
+ 'Post-upgrade file removed'
+ );
+ config.updatedArtifacts.push({
+ name: '|delete|',
+ contents: relativePath,
+ });
+ }
+ }
+ }
+ }
+ }
+
if (config.artifactErrors && config.artifactErrors.length) {
if (config.releaseTimestamp) {
logger.debug(`Branch timestamp: ` + config.releaseTimestamp);
diff --git a/renovate-schema.json b/renovate-schema.json
index 4a952be5df..1bb82903a1 100644
--- a/renovate-schema.json
+++ b/renovate-schema.json
@@ -3,6 +3,48 @@
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
+ "allowedPostUpgradeCommands": {
+ "description": "A list of regular expressions that determine which post-upgrade tasks are allowed. A task has to match at least one of the patterns to be allowed to run",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "default": []
+ },
+ "postUpgradeTasks": {
+ "description": "Post-upgrade tasks that are executed before a commit is made by Renovate",
+ "type": "object",
+ "default": {
+ "commands": [],
+ "fileFilters": []
+ },
+ "$ref": "#",
+ "items": {
+ "allOf": [
+ {
+ "type": "object",
+ "properties": {
+ "commands": {
+ "description": "A list of post-upgrade commands that are executed before a commit is made by Renovate",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "default": []
+ },
+ "fileFilters": {
+ "description": "Files that match these glob patterns will be committed if they are present after running a post-upgrade task",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "default": []
+ }
+ }
+ }
+ ]
+ }
+ },
"onboardingBranch": {
"description": "Change this value in order to override the default onboarding branch name.",
"type": "string",
diff --git a/test/workers/branch/index.spec.ts b/test/workers/branch/index.spec.ts
index 0af6abf55e..5e670ed6d8 100644
--- a/test/workers/branch/index.spec.ts
+++ b/test/workers/branch/index.spec.ts
@@ -1,3 +1,4 @@
+import * as _fs from 'fs-extra';
import * as branchWorker from '../../../lib/workers/branch';
import * as _schedule from '../../../lib/workers/branch/schedule';
import * as _checkExisting from '../../../lib/workers/branch/check-existing';
@@ -8,6 +9,7 @@ import * as _statusChecks from '../../../lib/workers/branch/status-checks';
import * as _automerge from '../../../lib/workers/branch/automerge';
import * as _prWorker from '../../../lib/workers/pr';
import * as _getUpdated from '../../../lib/workers/branch/get-updated';
+import * as _exec from '../../../lib/util/exec';
import { defaultConfig, platform, mocked } from '../../util';
import { BranchConfig } from '../../../lib/workers/common';
import {
@@ -15,6 +17,7 @@ import {
REPOSITORY_CHANGED,
} from '../../../lib/constants/error-messages';
import { BRANCH_STATUS_PENDING } from '../../../lib/constants/branch-constants';
+import { StatusResult } from '../../../lib/platform/git/storage';
jest.mock('../../../lib/workers/branch/get-updated');
jest.mock('../../../lib/workers/branch/schedule');
@@ -25,6 +28,8 @@ jest.mock('../../../lib/workers/branch/status-checks');
jest.mock('../../../lib/workers/branch/automerge');
jest.mock('../../../lib/workers/branch/commit');
jest.mock('../../../lib/workers/pr');
+jest.mock('../../../lib/util/exec');
+jest.mock('fs-extra');
const getUpdated = mocked(_getUpdated);
const schedule = mocked(_schedule);
@@ -35,6 +40,8 @@ const statusChecks = mocked(_statusChecks);
const automerge = mocked(_automerge);
const commit = mocked(_commit);
const prWorker = mocked(_prWorker);
+const exec = mocked(_exec);
+const fs = mocked(_fs);
describe('workers/branch', () => {
describe('processBranch', () => {
@@ -524,5 +531,47 @@ describe('workers/branch', () => {
})
).toEqual('done');
});
+
+ it('executes post-upgrade tasks if trust is high', async () => {
+ getUpdated.getUpdatedPackageFiles.mockResolvedValueOnce({
+ updatedPackageFiles: [{}],
+ artifactErrors: [],
+ } as never);
+ npmPostExtract.getAdditionalFiles.mockResolvedValueOnce({
+ artifactErrors: [],
+ updatedArtifacts: [{}],
+ } as never);
+ platform.branchExists.mockResolvedValueOnce(true);
+ platform.getBranchPr.mockResolvedValueOnce({
+ title: 'rebase!',
+ state: 'open',
+ body: `- [x] <!-- rebase-check -->`,
+ isModified: true,
+ } as never);
+ platform.getRepoStatus.mockResolvedValueOnce({
+ modified: ['modified_file'],
+ not_added: [],
+ deleted: ['deleted_file'],
+ } as StatusResult);
+ global.trustLevel = 'high';
+
+ fs.readFile.mockResolvedValueOnce(Buffer.from('modified file content'));
+
+ schedule.isScheduledNow.mockReturnValueOnce(false);
+ commit.commitFilesToBranch.mockResolvedValueOnce(false);
+
+ const result = await branchWorker.processBranch({
+ ...config,
+ postUpgradeTasks: {
+ commands: ['echo 1', 'disallowed task'],
+ fileFilters: ['modified_file', 'deleted_file'],
+ },
+ localDir: '/localDir',
+ allowedPostUpgradeCommands: ['^echo 1$'],
+ });
+
+ expect(result).toEqual('done');
+ expect(exec.exec).toHaveBeenCalledWith('echo 1', { cwd: '/localDir' });
+ });
});
});
--
GitLab