From 4349a21484b6c6eb38e902131f6fa975e9f0d912 Mon Sep 17 00:00:00 2001 From: Rhys Arkins <rhys@arkins.net> Date: Fri, 27 Apr 2018 19:54:44 +0200 Subject: [PATCH] feat: buildkite plugin renovation (#1874) This PR adds initial support for buildkite plugin renovation. It supports `plugin-name` or `my/plugin-name` plugins, and fully specified semver versions only (e.g. `v1.3.2`). Currently it will always propose an upgrade to the latest version available, e.g. if current version is v1.3.1 and both v1.3.2 and v2.0.0 exist then v2.0.0 will be proposed. Looks for any yml file in the `.buildkite/` directory. Closes #1869 --- lib/config/definitions.js | 14 ++++ lib/config/templates/buildkite/pr-body.hbs | 72 +++++++++++++++++++ lib/manager/buildkite/extract.js | 39 ++++++++++ lib/manager/buildkite/index.js | 12 ++++ lib/manager/buildkite/package.js | 56 +++++++++++++++ lib/manager/buildkite/update.js | 26 +++++++ lib/manager/index.js | 1 + test/_fixtures/buildkite/pipeline1.yml | 3 + test/_fixtures/buildkite/pipeline2.yml | 17 +++++ test/config/__snapshots__/index.spec.js.snap | 7 ++ .../__snapshots__/extract.spec.js.snap | 29 ++++++++ .../__snapshots__/package.spec.js.snap | 15 ++++ .../__snapshots__/update.spec.js.snap | 22 ++++++ test/manager/buildkite/extract.spec.js | 32 +++++++++ test/manager/buildkite/package.spec.js | 68 ++++++++++++++++++ test/manager/buildkite/update.spec.js | 61 ++++++++++++++++ .../2017-10-05-configuration-options.md | 9 +++ 17 files changed, 483 insertions(+) create mode 100644 lib/config/templates/buildkite/pr-body.hbs create mode 100644 lib/manager/buildkite/extract.js create mode 100644 lib/manager/buildkite/index.js create mode 100644 lib/manager/buildkite/package.js create mode 100644 lib/manager/buildkite/update.js create mode 100644 test/_fixtures/buildkite/pipeline1.yml create mode 100644 test/_fixtures/buildkite/pipeline2.yml create mode 100644 test/manager/buildkite/__snapshots__/extract.spec.js.snap create mode 100644 test/manager/buildkite/__snapshots__/package.spec.js.snap create mode 100644 test/manager/buildkite/__snapshots__/update.spec.js.snap create mode 100644 test/manager/buildkite/extract.spec.js create mode 100644 test/manager/buildkite/package.spec.js create mode 100644 test/manager/buildkite/update.spec.js diff --git a/lib/config/definitions.js b/lib/config/definitions.js index 27c8195373..5e3331619c 100644 --- a/lib/config/definitions.js +++ b/lib/config/definitions.js @@ -866,6 +866,20 @@ const options = [ default: {}, mergeable: true, }, + { + name: 'buildkite', + description: 'Configuration object for buildkite pipeline renovation', + stage: 'repository', + type: 'json', + default: { + enabled: false, + commitMessageTopic: 'buildkite plugin {{depName}}', + commitMessageExtra: 'to {{newVersion}}', + managerBranchPrefix: 'buildkite-', + prBody: template('prBody', 'buildkite'), + }, + mergeable: true, + }, { name: 'supportPolicy', description: diff --git a/lib/config/templates/buildkite/pr-body.hbs b/lib/config/templates/buildkite/pr-body.hbs new file mode 100644 index 0000000000..966a86d44a --- /dev/null +++ b/lib/config/templates/buildkite/pr-body.hbs @@ -0,0 +1,72 @@ +This Pull Request updates buildkite plugin {{#if repositoryUrl}}[{{{depName}}}]({{{repositoryUrl}}}){{else}}`{{{depName}}}`{{/if}} from `{{{currentVersion}}}` to `{{{newVersion}}}`. + +{{#if releases.length}} + +{{#if schedule}} +**Note**: This PR was created on a configured schedule ("{{{schedule}}}"{{#if timezone}} in timezone `{{{timezone}}}`{{/if}}) and will not receive updates outside those times. +{{/if}} + +{{#if isPin}} +**Important**: Renovate will wait until you have merged this Pin request before creating PRs for any *upgrades*. If you do not wish to pin anything, please update your config accordingly instead of leaving this PR open. +{{/if}} +{{#if hasReleaseNotes}} + +<details> +<summary>Release Notes</summary> + +{{#each releases as |release|}} +{{#if release.releaseNotes}} +### [`v{{{release.version}}}`]({{{release.releaseNotes.url}}}) + +{{{release.releaseNotes.body}}} + +--- + +{{/if}} +{{/each}} +</details> +{{/if}} + +{{#if hasCommits}} + +<details> +<summary>Commits</summary> + +{{#each releases as |release|}} +{{#if release.hasCommits}} +#### v{{{release.version}}} +{{#each release.commits as |commit|}} +- [`{{commit.shortSha}}`]({{commit.url}}) {{commit.message}} +{{/each}} +{{/if}} +{{/each}} + +</details> +{{/if}} +{{/if}} + +{{#if hasErrors}} + +--- + +# Errors + +Renovate encountered some errors when processing your repository, so you are being notified here even if they do not directly apply to this PR. + +{{#each errors as |error|}} +- `{{error.depName}}`: {{error.message}} +{{/each}} +{{/if}} + +{{#if hasWarnings}} + +--- + +# Warnings + +Please make sure the following warnings are safe to ignore: + +{{#each warnings as |warning|}} +- `{{warning.depName}}`: {{warning.message}} +{{/each}} +{{/if}} diff --git a/lib/manager/buildkite/extract.js b/lib/manager/buildkite/extract.js new file mode 100644 index 0000000000..6445eb5719 --- /dev/null +++ b/lib/manager/buildkite/extract.js @@ -0,0 +1,39 @@ +module.exports = { + extractDependencies, +}; + +function extractDependencies(content) { + logger.debug('buildkite.extractDependencies()'); + logger.trace({ content }); + const deps = []; + try { + const lines = content.split('\n'); + for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { + const line = lines[lineNumber]; + const plugins = line.match(/^\s*-?\s*plugins:\s*$/); + if (plugins) { + logger.trace(`Matched plugins on line ${lineNumber}`); + const depLine = lines[lineNumber + 1]; + logger.debug(`serviceImageLine: "${depLine}"`); + const depLineMatch = depLine.match(/^\s+([^#]+)#([^:]+):/); + if (depLineMatch) { + logger.trace('depLineMatch'); + lineNumber += 1; + const [, depName, currentVersion] = depLineMatch; + deps.push({ + depType: 'plugins', + lineNumber, + depName, + currentVersion, + }); + } + } + } + } catch (err) /* istanbul ignore next */ { + logger.error( + { err, message: err.message }, + 'Error extracting buildkite plugins' + ); + } + return deps; +} diff --git a/lib/manager/buildkite/index.js b/lib/manager/buildkite/index.js new file mode 100644 index 0000000000..f5a02ded7a --- /dev/null +++ b/lib/manager/buildkite/index.js @@ -0,0 +1,12 @@ +const { extractDependencies } = require('./extract'); +const { getPackageUpdates } = require('./package'); +const { updateDependency } = require('./update'); + +const filePattern = new RegExp('\\.buildkite/.+\\.yml$'); + +module.exports = { + extractDependencies, + filePattern, + getPackageUpdates, + updateDependency, +}; diff --git a/lib/manager/buildkite/package.js b/lib/manager/buildkite/package.js new file mode 100644 index 0000000000..e92318c7da --- /dev/null +++ b/lib/manager/buildkite/package.js @@ -0,0 +1,56 @@ +const { + isGreaterThan, + isPinnedVersion, + semverSort, + getMajor, + getMinor, +} = require('../../util/semver'); +const { getRepoTags } = require('../../datasource/github'); + +module.exports = { + getPackageUpdates, +}; + +async function getPackageUpdates(config) { + const { depName, currentVersion } = config; + logger.debug({ depName }, 'buildkite.getPackageUpdates()'); + if (depName.startsWith('https://') || depName.startsWith('git@')) { + logger.debug({ depName }, 'Skipping git plugin'); + return []; + } + if (!isPinnedVersion(currentVersion)) { + logger.debug({ currentVersion }, 'Skipping non-pinned current version'); + return []; + } + let sourceRepo = ''; + const splitName = depName.split('/'); + if (splitName.length === 1) { + sourceRepo = `buildkite-plugins/${depName}-buildkite-plugin`; + } else if (splitName.length === 2) { + sourceRepo = `${depName}-buildkite-plugin`; + } else { + logger.warn({ depName }, 'Something is wrong with buildkite plugin name'); + return []; + } + const repoTags = await getRepoTags(sourceRepo); + const newerVersions = repoTags + .filter(tag => isGreaterThan(tag, currentVersion)) + .sort(semverSort); + if (newerVersions.length) { + logger.debug({ newerVersions }, 'Found newer versions'); + } else { + return []; + } + const newVersion = newerVersions.pop(); + return [ + { + type: getMajor(newVersion) > getMajor(currentVersion) ? 'major' : 'minor', + newVersion, + newVersionMajor: getMajor(newVersion), + newVersionMinor: getMinor(newVersion), + changeLogFromVersion: currentVersion, + changeLogToVersion: newVersion, + repositoryUrl: `https://github.com/${sourceRepo}`, + }, + ]; +} diff --git a/lib/manager/buildkite/update.js b/lib/manager/buildkite/update.js new file mode 100644 index 0000000000..4ced055465 --- /dev/null +++ b/lib/manager/buildkite/update.js @@ -0,0 +1,26 @@ +module.exports = { + updateDependency, +}; + +function updateDependency(currentFileContent, upgrade) { + try { + logger.debug(`buildkite.updateDependency: ${upgrade.newVersion}`); + const lines = currentFileContent.split('\n'); + const lineToChange = lines[upgrade.lineNumber]; + const depLine = new RegExp(/^(\s+[^#]+#)[^:]+(:.*)$/); + if (!lineToChange.match(depLine)) { + logger.debug('No image line found'); + return null; + } + const newLine = lineToChange.replace(depLine, `$1${upgrade.newVersion}$2`); + if (newLine === lineToChange) { + logger.debug('No changes necessary'); + return currentFileContent; + } + lines[upgrade.lineNumber] = newLine; + return lines.join('\n'); + } catch (err) { + logger.info({ err }, 'Error setting new buildkite version'); + return null; + } +} diff --git a/lib/manager/index.js b/lib/manager/index.js index 166244f8e9..df80bf566d 100644 --- a/lib/manager/index.js +++ b/lib/manager/index.js @@ -5,6 +5,7 @@ const { checkMonorepos } = require('../manager/npm/monorepos'); const managers = {}; const managerList = [ 'bazel', + 'buildkite', 'circleci', 'docker', 'docker-compose', diff --git a/test/_fixtures/buildkite/pipeline1.yml b/test/_fixtures/buildkite/pipeline1.yml new file mode 100644 index 0000000000..b115d18add --- /dev/null +++ b/test/_fixtures/buildkite/pipeline1.yml @@ -0,0 +1,3 @@ +steps: + - plugins: + detect-clowns#v2.0.0: ~ diff --git a/test/_fixtures/buildkite/pipeline2.yml b/test/_fixtures/buildkite/pipeline2.yml new file mode 100644 index 0000000000..0db18c7a43 --- /dev/null +++ b/test/_fixtures/buildkite/pipeline2.yml @@ -0,0 +1,17 @@ +steps: + # Prebuild the app image, upload it to a registry for later steps + - name: "Docker Build" + plugins: + docker-compose#v1.3.2: + build: app + image-repository: index.docker.io/org/repo + + - wait + + # Use the app image built above to run concurrent tests + - name: "Docker Test %n" + command: test.sh + parallelism: 25 + plugins: + docker-compose#v1.3.2: + run: app diff --git a/test/config/__snapshots__/index.spec.js.snap b/test/config/__snapshots__/index.spec.js.snap index 15bad95a23..49b3da4405 100644 --- a/test/config/__snapshots__/index.spec.js.snap +++ b/test/config/__snapshots__/index.spec.js.snap @@ -19,6 +19,13 @@ Object { "branchName": "{{{branchPrefix}}}{{{managerBranchPrefix}}}{{{branchTopic}}}", "branchPrefix": "renovate/", "branchTopic": "{{{depNameSanitized}}}-{{{newVersionMajor}}}.x", + "buildkite": Object { + "commitMessageExtra": "to {{newVersion}}", + "commitMessageTopic": "buildkite plugin {{depName}}", + "enabled": false, + "managerBranchPrefix": "buildkite-", + "prBody": "This Pull Request updates buildkite plugin {{#if repositoryUrl}}[{{{depName}}}]({{{repositoryUrl}}}){{else}}\`{{{depName}}}\`{{/if}} from \`{{{currentVersion}}}\` to \`{{{newVersion}}}\`.\\n\\n{{#if releases.length}}\\n\\n{{#if schedule}}\\n**Note**: This PR was created on a configured schedule (\\"{{{schedule}}}\\"{{#if timezone}} in timezone \`{{{timezone}}}\`{{/if}}) and will not receive updates outside those times.\\n{{/if}}\\n\\n{{#if isPin}}\\n**Important**: Renovate will wait until you have merged this Pin request before creating PRs for any *upgrades*. If you do not wish to pin anything, please update your config accordingly instead of leaving this PR open.\\n{{/if}}\\n{{#if hasReleaseNotes}}\\n\\n<details>\\n<summary>Release Notes</summary>\\n\\n{{#each releases as |release|}}\\n{{#if release.releaseNotes}}\\n### [\`v{{{release.version}}}\`]({{{release.releaseNotes.url}}})\\n\\n{{{release.releaseNotes.body}}}\\n\\n---\\n\\n{{/if}}\\n{{/each}}\\n</details>\\n{{/if}}\\n\\n{{#if hasCommits}}\\n\\n<details>\\n<summary>Commits</summary>\\n\\n{{#each releases as |release|}}\\n{{#if release.hasCommits}}\\n#### v{{{release.version}}}\\n{{#each release.commits as |commit|}}\\n- [\`{{commit.shortSha}}\`]({{commit.url}}) {{commit.message}}\\n{{/each}}\\n{{/if}}\\n{{/each}}\\n\\n</details>\\n{{/if}}\\n{{/if}}\\n\\n{{#if hasErrors}}\\n\\n---\\n\\n# Errors\\n\\nRenovate encountered some errors when processing your repository, so you are being notified here even if they do not directly apply to this PR.\\n\\n{{#each errors as |error|}}\\n- \`{{error.depName}}\`: {{error.message}}\\n{{/each}}\\n{{/if}}\\n\\n{{#if hasWarnings}}\\n\\n---\\n\\n# Warnings\\n\\nPlease make sure the following warnings are safe to ignore:\\n\\n{{#each warnings as |warning|}}\\n- \`{{warning.depName}}\`: {{warning.message}}\\n{{/each}}\\n{{/if}}", + }, "bumpVersion": null, "circleci": Object {}, "commitBody": null, diff --git a/test/manager/buildkite/__snapshots__/extract.spec.js.snap b/test/manager/buildkite/__snapshots__/extract.spec.js.snap new file mode 100644 index 0000000000..030fecf4c1 --- /dev/null +++ b/test/manager/buildkite/__snapshots__/extract.spec.js.snap @@ -0,0 +1,29 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`lib/manager/buildkite/extract extractDependencies() extracts multiple plugins in same file 1`] = ` +Array [ + Object { + "currentVersion": "v1.3.2", + "depName": "docker-compose", + "depType": "plugins", + "lineNumber": 4, + }, + Object { + "currentVersion": "v1.3.2", + "depName": "docker-compose", + "depType": "plugins", + "lineNumber": 15, + }, +] +`; + +exports[`lib/manager/buildkite/extract extractDependencies() extracts simple single plugin 1`] = ` +Array [ + Object { + "currentVersion": "v2.0.0", + "depName": "detect-clowns", + "depType": "plugins", + "lineNumber": 2, + }, +] +`; diff --git a/test/manager/buildkite/__snapshots__/package.spec.js.snap b/test/manager/buildkite/__snapshots__/package.spec.js.snap new file mode 100644 index 0000000000..41de9ae4af --- /dev/null +++ b/test/manager/buildkite/__snapshots__/package.spec.js.snap @@ -0,0 +1,15 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`lib/manager/buildkite/package getPackageUpdates() returns one upgrade 1`] = ` +Array [ + Object { + "changeLogFromVersion": "v1.0.0", + "changeLogToVersion": "v1.2.0", + "newVersion": "v1.2.0", + "newVersionMajor": 1, + "newVersionMinor": 2, + "repositoryUrl": "https://github.com/some/plugin-buildkite-plugin", + "type": "minor", + }, +] +`; diff --git a/test/manager/buildkite/__snapshots__/update.spec.js.snap b/test/manager/buildkite/__snapshots__/update.spec.js.snap new file mode 100644 index 0000000000..14775ceb70 --- /dev/null +++ b/test/manager/buildkite/__snapshots__/update.spec.js.snap @@ -0,0 +1,22 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`manager/buildkite/update updateDependency replaces two values in one file 1`] = ` +"steps: + # Prebuild the app image, upload it to a registry for later steps + - name: \\"Docker Build\\" + plugins: + docker-compose#v1.5.0: + build: app + image-repository: index.docker.io/org/repo + + - wait + + # Use the app image built above to run concurrent tests + - name: \\"Docker Test %n\\" + command: test.sh + parallelism: 25 + plugins: + docker-compose#v1.5.0: + run: app +" +`; diff --git a/test/manager/buildkite/extract.spec.js b/test/manager/buildkite/extract.spec.js new file mode 100644 index 0000000000..6afd9f27de --- /dev/null +++ b/test/manager/buildkite/extract.spec.js @@ -0,0 +1,32 @@ +const fs = require('fs'); +const { + extractDependencies, +} = require('../../../lib/manager/buildkite/extract'); + +const pipeline1 = fs.readFileSync( + 'test/_fixtures/buildkite/pipeline1.yml', + 'utf8' +); +const pipeline2 = fs.readFileSync( + 'test/_fixtures/buildkite/pipeline2.yml', + 'utf8' +); + +describe('lib/manager/buildkite/extract', () => { + describe('extractDependencies()', () => { + let config; + beforeEach(() => { + config = {}; + }); + it('extracts simple single plugin', () => { + const res = extractDependencies(pipeline1, config); + expect(res).toMatchSnapshot(); + expect(res).toHaveLength(1); + }); + it('extracts multiple plugins in same file', () => { + const res = extractDependencies(pipeline2, config); + expect(res).toMatchSnapshot(); + expect(res).toHaveLength(2); + }); + }); +}); diff --git a/test/manager/buildkite/package.spec.js b/test/manager/buildkite/package.spec.js new file mode 100644 index 0000000000..64600263b3 --- /dev/null +++ b/test/manager/buildkite/package.spec.js @@ -0,0 +1,68 @@ +const { getPackageUpdates } = require('../../../lib/manager/buildkite/package'); + +const defaultConfig = require('../../../lib/config/defaults').getConfig(); +const ghGot = require('../../../lib/platform/github/gh-got-wrapper'); + +jest.mock('../../../lib/platform/github/gh-got-wrapper'); + +describe('lib/manager/buildkite/package', () => { + describe('getPackageUpdates()', () => { + let config; + beforeEach(() => { + config = { + ...defaultConfig, + }; + }); + it('returns empty if remote is https', async () => { + config.depName = 'https://github.com/a/b'; + expect(await getPackageUpdates(config)).toEqual([]); + }); + it('returns empty if remote is git', async () => { + config.depName = 'git@github.com/a/b.git'; + expect(await getPackageUpdates(config)).toEqual([]); + }); + it('returns empty if current version is not semver', async () => { + config.depName = 'some-plugin'; + config.currentVersion = 'abcdefg'; + expect(await getPackageUpdates(config)).toEqual([]); + }); + it('returns empty if depName has more than one slash', async () => { + config.depName = 'some/weird/plugin'; + config.currentVersion = 'v1.0.0'; + expect(await getPackageUpdates(config)).toEqual([]); + }); + it('returns empty if no newer versions', async () => { + config.depName = 'some-plugin'; + config.currentVersion = 'v1.0.0'; + ghGot.mockReturnValueOnce({ + body: [ + { + ref: 'refs/tags/v0.9.0', + }, + { + ref: 'refs/tags/v1.0.0', + }, + ], + }); + expect(await getPackageUpdates(config)).toEqual([]); + }); + it('returns one upgrade', async () => { + config.depName = 'some/plugin'; + config.currentVersion = 'v1.0.0'; + ghGot.mockReturnValueOnce({ + body: [ + { + ref: 'refs/tags/v1.0.0', + }, + { + ref: 'refs/tags/v1.1.0', + }, + { + ref: 'refs/tags/v1.2.0', + }, + ], + }); + expect(await getPackageUpdates(config)).toMatchSnapshot(); + }); + }); +}); diff --git a/test/manager/buildkite/update.spec.js b/test/manager/buildkite/update.spec.js new file mode 100644 index 0000000000..45733ccebe --- /dev/null +++ b/test/manager/buildkite/update.spec.js @@ -0,0 +1,61 @@ +const fs = require('fs'); +const bkUpdate = require('../../../lib/manager/buildkite/update'); + +const pipeline1 = fs.readFileSync( + 'test/_fixtures/buildkite/pipeline1.yml', + 'utf8' +); +const pipeline2 = fs.readFileSync( + 'test/_fixtures/buildkite/pipeline2.yml', + 'utf8' +); + +describe('manager/buildkite/update', () => { + describe('updateDependency', () => { + it('replaces existing value', () => { + const upgrade = { + lineNumber: 2, + newVersion: 'v2.2.0', + }; + const res = bkUpdate.updateDependency(pipeline1, upgrade); + expect(res).not.toEqual(pipeline1); + expect(res.includes(upgrade.newVersion)).toBe(true); + }); + it('replaces two values in one file', () => { + const upgrade1 = { + lineNumber: 4, + newVersion: 'v1.5.0', + }; + const res1 = bkUpdate.updateDependency(pipeline2, upgrade1); + expect(res1).not.toEqual(pipeline2); + expect(res1.includes(upgrade1.newVersion)).toBe(true); + const upgrade2 = { + lineNumber: 15, + newVersion: 'v1.5.0', + }; + const res2 = bkUpdate.updateDependency(res1, upgrade2); + expect(res2).not.toEqual(res1); + expect(res2).toMatchSnapshot(); + }); + it('returns same', () => { + const upgrade = { + lineNumber: 2, + newVersion: 'v2.0.0', + }; + const res = bkUpdate.updateDependency(pipeline1, upgrade); + expect(res).toEqual(pipeline1); + }); + it('returns null if mismatch', () => { + const upgrade = { + lineNumber: 3, + newVersion: 'v2.2.0', + }; + const res = bkUpdate.updateDependency(pipeline1, upgrade); + expect(res).toBe(null); + }); + it('returns null if error', () => { + const res = bkUpdate.updateDependency(null, null); + expect(res).toBe(null); + }); + }); +}); diff --git a/website/docs/_posts/2017-10-05-configuration-options.md b/website/docs/_posts/2017-10-05-configuration-options.md index fe1c5afc2e..7d455bedd3 100644 --- a/website/docs/_posts/2017-10-05-configuration-options.md +++ b/website/docs/_posts/2017-10-05-configuration-options.md @@ -149,6 +149,15 @@ The main name/text that Renovate should use when creating a branch on your repos This field is combined with `branchPrefix` and `managerBranchPrefix` to form the full `branchName`. `branchName` uniqueness is important for dependency update grouping or non-grouping so be cautious about ever editing this field manually. +## buildkite + +Configuration specific for buildkite plugins updates. + +| name | value | +| ------- | ----------------------------------------------------- | +| type | object | +| default | { enabled: false, managerBranchPrefix: 'buildkite-' } | + ## bumpVersion Bump the version in the package.json being updated -- GitLab