From 1af60f515854abf774b67e2e0771a9d6ddcc710e Mon Sep 17 00:00:00 2001 From: Rhys Arkins <rhys@keylocation.sg> Date: Mon, 6 Nov 2017 11:36:06 +0100 Subject: [PATCH] feat: Support 'AS' names in Dockerfile from line (#1110) This PR adds support for 'AS' names in Dockerfiles. e.g. `FROM node:8 AS base`. It also adds logic to detect and ignore - for now - any image sources from custom registries. --- lib/manager/docker/extract.js | 32 +++-- lib/manager/docker/package.js | 82 +++++------ lib/manager/docker/registry.js | 5 +- lib/manager/docker/resolve.js | 2 +- lib/manager/docker/update.js | 22 ++- lib/manager/index.js | 4 +- .../docker/__snapshots__/extract.spec.js.snap | 127 ++++++++++++++++++ .../docker/__snapshots__/package.spec.js.snap | 9 +- .../docker/__snapshots__/update.spec.js.snap | 7 + test/manager/docker/extract.spec.js | 57 ++++++++ test/manager/docker/package.spec.js | 15 +++ test/manager/docker/update.spec.js | 49 ++++--- 12 files changed, 320 insertions(+), 91 deletions(-) create mode 100644 test/manager/docker/__snapshots__/extract.spec.js.snap create mode 100644 test/manager/docker/extract.spec.js diff --git a/lib/manager/docker/extract.js b/lib/manager/docker/extract.js index e43a4d889b..277e72a80f 100644 --- a/lib/manager/docker/extract.js +++ b/lib/manager/docker/extract.js @@ -4,23 +4,37 @@ module.exports = { function extractDependencies(content, config) { const { logger } = config; - const strippedComment = content.replace(/^(#.*?\n)+/, ''); - const fromMatch = strippedComment.match(/^FROM (.*)\n/); + const fromMatch = content.match(/(\n|^)([Ff][Rr][Oo][Mm] .*)\n/); if (!fromMatch) { - logger.warn({ content, strippedComment }, 'No FROM found'); + logger.warn({ content }, 'No FROM found'); return []; } - const [, currentFrom] = fromMatch; - const [imagetag, currentDigest] = currentFrom.split('@'); - const [depName, currentTag] = imagetag.split(':'); - logger.info({ depName, currentTag, currentDigest }, 'Dockerfile'); + const [, , fromLine] = fromMatch; + const [fromPrefix, currentFrom, ...fromRest] = fromLine.split(' '); + const fromSuffix = fromRest.join(' '); + let dockerRegistry; + let currentDepTagDigest; + if (currentFrom.includes('/')) { + [dockerRegistry, currentDepTagDigest] = currentFrom.split('/'); + } else { + currentDepTagDigest = currentFrom; + } + const [currentDepTag, currentDigest] = currentDepTagDigest.split('@'); + const [depName, currentTag] = currentDepTag.split(':'); + logger.info({ depName, currentTag, currentDigest }, 'Dockerfile FROM'); return [ { depType: 'Dockerfile', - depName, + fromLine, + fromPrefix, currentFrom, - currentTag: currentTag || 'latest', + fromSuffix, + currentDepTagDigest, + dockerRegistry, + currentDepTag, currentDigest, + depName, + currentTag, }, ]; } diff --git a/lib/manager/docker/package.js b/lib/manager/docker/package.js index 6ef6f287bf..d33bc7e3e9 100644 --- a/lib/manager/docker/package.js +++ b/lib/manager/docker/package.js @@ -8,27 +8,30 @@ module.exports = { }; async function getPackageUpdates(config) { - const { currentFrom, currentTag, logger } = config; + const { + dockerRegistry, + currentFrom, + depName, + currentDepTag, + currentTag, + currentDigest, + logger, + } = config; + if (dockerRegistry) { + logger.info({ currentFrom }, 'Skipping Dockerfile image with custom host'); + return []; + } const upgrades = []; - if (config.pinDigests) { + if (currentDigest || config.pinDigests) { logger.debug('Checking docker pinDigests'); - const newDigest = await dockerApi.getDigest( - config.depName, - currentTag, - config.logger - ); + const newDigest = await dockerApi.getDigest(depName, currentTag, logger); if (newDigest && config.currentDigest !== newDigest) { const upgrade = {}; - upgrade.newTag = currentTag; + upgrade.newTag = currentTag || 'latest'; upgrade.newDigest = newDigest; upgrade.newDigestShort = newDigest.slice(7, 13); - upgrade.newVersion = newDigest; - upgrade.newFrom = config.depName; - if (upgrade.newTag) { - upgrade.newFrom += `:${upgrade.newTag}`; - } - upgrade.newFrom += `@${upgrade.newDigest}`; - if (config.currentDigest) { + upgrade.newFrom = `${depName}:${upgrade.newTag}@${newDigest}`; + if (currentDigest) { upgrade.type = 'digest'; upgrade.isDigest = true; } else { @@ -39,24 +42,26 @@ async function getPackageUpdates(config) { } } if (currentTag) { - const currentVersion = getVersion(currentTag); - const currentSuffix = getSuffix(currentTag); - if (!versions.isValidVersion(currentVersion)) { - logger.info({ currentFrom }, 'Docker tag is not valid semver - skipping'); + const tagVersion = getVersion(currentTag); + const tagSuffix = getSuffix(currentTag); + if (!versions.isValidVersion(tagVersion)) { + logger.info( + { currentDepTag }, + 'Docker tag is not valid semver - skipping' + ); return upgrades; } let versionList = []; const allTags = await dockerApi.getTags(config.depName, config.logger); if (allTags) { versionList = allTags - .filter(tag => getSuffix(tag) === currentSuffix) + .filter(tag => getSuffix(tag) === tagSuffix) .map(getVersion) .filter(versions.isValidVersion) .filter( - prefix => - prefix.split('.').length === currentVersion.split('.').length + prefix => prefix.split('.').length === tagVersion.split('.').length ) - .filter(prefix => compareVersions(prefix, currentVersion) > 0); + .filter(prefix => compareVersions(prefix, tagVersion) > 0); } logger.debug({ versionList }, 'upgrades versionList'); const versionUpgrades = {}; @@ -71,37 +76,38 @@ async function getPackageUpdates(config) { } } logger.debug({ versionUpgrades }, 'Docker versionUpgrades'); - const currentMajor = semver.major(padRange(currentVersion)); + const currentMajor = semver.major(padRange(tagVersion)); for (const newVersionMajor of Object.keys(versionUpgrades)) { let newTag = versionUpgrades[newVersionMajor]; - if (currentSuffix) { - newTag += `-${currentSuffix}`; + if (tagSuffix) { + newTag += `-${tagSuffix}`; } const upgrade = { newTag, newVersionMajor, }; - upgrade.currentVersion = config.currentTag; - upgrade.newVersion = upgrade.newTag; - upgrade.newFrom = `${config.depName}:${upgrade.newTag}`; - if (newVersionMajor > currentMajor) { - upgrade.type = 'major'; - upgrade.isMajor = true; - } else { - upgrade.type = 'minor'; - upgrade.isMinor = true; - } + upgrade.newVersion = newTag; + upgrade.newDepTag = `${config.depName}:${upgrade.newTag}`; + let newFrom = upgrade.newDepTag; if (config.currentDigest || config.pinDigests) { upgrade.newDigest = await dockerApi.getDigest( config.depName, upgrade.newTag, config.logger ); - upgrade.newFrom += `@${upgrade.newDigest}`; + newFrom = `${newFrom}@${upgrade.newDigest}`; + } + upgrade.newFrom = newFrom; + if (newVersionMajor > currentMajor) { + upgrade.type = 'major'; + upgrade.isMajor = true; + } else { + upgrade.type = 'minor'; + upgrade.isMinor = true; } upgrades.push(upgrade); logger.info( - { currentFrom, newFrom: upgrade.newFrom }, + { currentDepTag, newDepTag: upgrade.newDepTag }, 'Docker tag version upgrade found' ); } diff --git a/lib/manager/docker/registry.js b/lib/manager/docker/registry.js index 52d10cbc69..8408f7ad1a 100644 --- a/lib/manager/docker/registry.js +++ b/lib/manager/docker/registry.js @@ -5,7 +5,7 @@ module.exports = { getTags, }; -async function getDigest(name, tag, logger) { +async function getDigest(name, tag = 'latest', logger) { const repository = name.includes('/') ? name : `library/${name}`; try { const authUrl = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repository}:pull`; @@ -16,8 +16,7 @@ async function getDigest(name, tag, logger) { return null; } logger.debug('Got docker registry token'); - const url = `https://index.docker.io/v2/${repository}/manifests/${tag || - 'latest'}`; + const url = `https://index.docker.io/v2/${repository}/manifests/${tag}`; const headers = { Authorization: `Bearer ${token}`, Accept: 'application/vnd.docker.distribution.manifest.v2+json', diff --git a/lib/manager/docker/resolve.js b/lib/manager/docker/resolve.js index cedb0a4060..019b09bc0c 100644 --- a/lib/manager/docker/resolve.js +++ b/lib/manager/docker/resolve.js @@ -18,7 +18,7 @@ async function resolvePackageFile(config, inputFile) { return null; } const strippedComment = packageFile.content.replace(/^(#.*?\n)+/, ''); - const fromMatch = strippedComment.match(/^FROM (.*)\n/); + const fromMatch = strippedComment.match(/^[Ff][Rr][Oo][Mm] (.*)\n/); if (!fromMatch) { logger.debug( { content: packageFile.content, strippedComment }, diff --git a/lib/manager/docker/update.js b/lib/manager/docker/update.js index 4ced9834a0..e0a9982a36 100644 --- a/lib/manager/docker/update.js +++ b/lib/manager/docker/update.js @@ -2,20 +2,18 @@ module.exports = { setNewValue, }; -function setNewValue( - currentFileContent, - depName, - currentVersion, - newVersion, - logger -) { +function setNewValue(currentFileContent, upgrade, logger) { try { - logger.debug(`setNewValue: ${depName} = ${newVersion}`); - const regexReplace = new RegExp(`(^|\n)FROM ${depName}.*?\n`); - const newFileContent = currentFileContent.replace( - regexReplace, - `$1FROM ${newVersion}\n` + logger.debug(`setNewValue: ${upgrade.newFrom}`); + const oldLine = new RegExp( + `(^|\n)${upgrade.fromPrefix} ${upgrade.depName}.*? ?${upgrade.fromSuffix}\n` ); + let newLine = `$1${upgrade.fromPrefix} ${upgrade.newFrom}`; + if (upgrade.fromSuffix.length) { + newLine += ` ${upgrade.fromSuffix}`; + } + newLine += '\n'; + const newFileContent = currentFileContent.replace(oldLine, newLine); return newFileContent; } catch (err) { logger.info({ err }, 'Error setting new Dockerfile value'); diff --git a/lib/manager/index.js b/lib/manager/index.js index 3073b5876d..b3411b0687 100644 --- a/lib/manager/index.js +++ b/lib/manager/index.js @@ -90,9 +90,7 @@ async function getUpdatedPackageFiles(config) { } else if (upgrade.packageFile.endsWith('Dockerfile')) { newContent = dockerfileHelper.setNewValue( existingContent, - upgrade.depName, - upgrade.currentFrom, - upgrade.newFrom, + upgrade, config.logger ); } diff --git a/test/manager/docker/__snapshots__/extract.spec.js.snap b/test/manager/docker/__snapshots__/extract.spec.js.snap new file mode 100644 index 0000000000..b43bc2a492 --- /dev/null +++ b/test/manager/docker/__snapshots__/extract.spec.js.snap @@ -0,0 +1,127 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`lib/manager/docker/extract extractDependencies() handles comments 1`] = ` +Array [ + Object { + "currentDepTag": "node", + "currentDepTagDigest": "node", + "currentDigest": undefined, + "currentFrom": "node", + "currentTag": undefined, + "depName": "node", + "depType": "Dockerfile", + "dockerRegistry": undefined, + "fromLine": "FROM node", + "fromPrefix": "FROM", + "fromSuffix": "", + }, +] +`; + +exports[`lib/manager/docker/extract extractDependencies() handles custom hosts 1`] = ` +Array [ + Object { + "currentDepTag": "node:8", + "currentDepTagDigest": "node:8", + "currentDigest": undefined, + "currentFrom": "registry2.something.info:5005/node:8", + "currentTag": "8", + "depName": "node", + "depType": "Dockerfile", + "dockerRegistry": "registry2.something.info:5005", + "fromLine": "FROM registry2.something.info:5005/node:8", + "fromPrefix": "FROM", + "fromSuffix": "", + }, +] +`; + +exports[`lib/manager/docker/extract extractDependencies() handles digest 1`] = ` +Array [ + Object { + "currentDepTag": "node", + "currentDepTagDigest": "node@sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "currentDigest": "sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "currentFrom": "node@sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "currentTag": undefined, + "depName": "node", + "depType": "Dockerfile", + "dockerRegistry": undefined, + "fromLine": "FROM node@sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "fromPrefix": "FROM", + "fromSuffix": "", + }, +] +`; + +exports[`lib/manager/docker/extract extractDependencies() handles from as 1`] = ` +Array [ + Object { + "currentDepTag": "node:8.9.0-alpine", + "currentDepTagDigest": "node:8.9.0-alpine", + "currentDigest": undefined, + "currentFrom": "node:8.9.0-alpine", + "currentTag": "8.9.0-alpine", + "depName": "node", + "depType": "Dockerfile", + "dockerRegistry": undefined, + "fromLine": "FROM node:8.9.0-alpine as base", + "fromPrefix": "FROM", + "fromSuffix": "as base", + }, +] +`; + +exports[`lib/manager/docker/extract extractDependencies() handles naked dep 1`] = ` +Array [ + Object { + "currentDepTag": "node", + "currentDepTagDigest": "node", + "currentDigest": undefined, + "currentFrom": "node", + "currentTag": undefined, + "depName": "node", + "depType": "Dockerfile", + "dockerRegistry": undefined, + "fromLine": "FROM node", + "fromPrefix": "FROM", + "fromSuffix": "", + }, +] +`; + +exports[`lib/manager/docker/extract extractDependencies() handles tag 1`] = ` +Array [ + Object { + "currentDepTag": "node:8.9.0-alpine", + "currentDepTagDigest": "node:8.9.0-alpine", + "currentDigest": undefined, + "currentFrom": "node:8.9.0-alpine", + "currentTag": "8.9.0-alpine", + "depName": "node", + "depType": "Dockerfile", + "dockerRegistry": undefined, + "fromLine": "FROM node:8.9.0-alpine", + "fromPrefix": "FROM", + "fromSuffix": "", + }, +] +`; + +exports[`lib/manager/docker/extract extractDependencies() handles tag and digest 1`] = ` +Array [ + Object { + "currentDepTag": "node:8.9.0", + "currentDepTagDigest": "node:8.9.0@sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "currentDigest": "sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "currentFrom": "node:8.9.0@sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "currentTag": "8.9.0", + "depName": "node", + "depType": "Dockerfile", + "dockerRegistry": undefined, + "fromLine": "FROM node:8.9.0@sha256:aaaaaaaabbbbbbbbccccccccddddddd", + "fromPrefix": "FROM", + "fromSuffix": "", + }, +] +`; diff --git a/test/manager/docker/__snapshots__/package.spec.js.snap b/test/manager/docker/__snapshots__/package.spec.js.snap index c63a6345f5..55e0b1559d 100644 --- a/test/manager/docker/__snapshots__/package.spec.js.snap +++ b/test/manager/docker/__snapshots__/package.spec.js.snap @@ -8,12 +8,11 @@ Array [ "newDigestShort": "one", "newFrom": "some-dep:1.0.0-something@sha256:one", "newTag": "1.0.0-something", - "newVersion": "sha256:one", "type": "pin", }, Object { - "currentVersion": "1.0.0-something", "isMinor": true, + "newDepTag": "some-dep:1.1.0-something", "newDigest": undefined, "newFrom": "some-dep:1.1.0-something@undefined", "newTag": "1.1.0-something", @@ -27,8 +26,8 @@ Array [ exports[`lib/workers/package/docker getPackageUpdates returns major and minor upgrades 1`] = ` Array [ Object { - "currentVersion": "1.0.0", "isMinor": true, + "newDepTag": "some-dep:1.2.0", "newDigest": "sha256:one", "newFrom": "some-dep:1.2.0@sha256:one", "newTag": "1.2.0", @@ -37,8 +36,8 @@ Array [ "type": "minor", }, Object { - "currentVersion": "1.0.0", "isMajor": true, + "newDepTag": "some-dep:2.0.0", "newDigest": "sha256:two", "newFrom": "some-dep:2.0.0@sha256:two", "newTag": "2.0.0", @@ -47,8 +46,8 @@ Array [ "type": "major", }, Object { - "currentVersion": "1.0.0", "isMajor": true, + "newDepTag": "some-dep:3.0.0", "newDigest": "sha256:three", "newFrom": "some-dep:3.0.0@sha256:three", "newTag": "3.0.0", diff --git a/test/manager/docker/__snapshots__/update.spec.js.snap b/test/manager/docker/__snapshots__/update.spec.js.snap index f69760bb1b..5ccf0fa460 100644 --- a/test/manager/docker/__snapshots__/update.spec.js.snap +++ b/test/manager/docker/__snapshots__/update.spec.js.snap @@ -6,3 +6,10 @@ FROM node:8@sha256:abcdefghijklmnop RUN something " `; + +exports[`workers/branch/dockerfile setNewValue replaces existing value with suffix 1`] = ` +"# comment FROM node:8 +FROM node:8@sha256:abcdefghijklmnop as base +RUN something +" +`; diff --git a/test/manager/docker/extract.spec.js b/test/manager/docker/extract.spec.js new file mode 100644 index 0000000000..7702835657 --- /dev/null +++ b/test/manager/docker/extract.spec.js @@ -0,0 +1,57 @@ +const { extractDependencies } = require('../../../lib/manager/docker/extract'); +const logger = require('../../_fixtures/logger'); + +describe('lib/manager/docker/extract', () => { + describe('extractDependencies()', () => { + let config; + beforeEach(() => { + config = { + logger, + }; + }); + it('handles naked dep', () => { + const res = extractDependencies('FROM node\n', config); + expect(res).toMatchSnapshot(); + }); + it('handles tag', () => { + const res = extractDependencies('FROM node:8.9.0-alpine\n', config); + expect(res).toMatchSnapshot(); + }); + it('handles digest', () => { + const res = extractDependencies( + 'FROM node@sha256:aaaaaaaabbbbbbbbccccccccddddddd\n', + config + ); + expect(res).toMatchSnapshot(); + }); + it('handles tag and digest', () => { + const res = extractDependencies( + 'FROM node:8.9.0@sha256:aaaaaaaabbbbbbbbccccccccddddddd\n', + config + ); + expect(res).toMatchSnapshot(); + }); + it('handles from as', () => { + const res = extractDependencies( + 'FROM node:8.9.0-alpine as base\n', + config + ); + expect(res).toMatchSnapshot(); + // expect(res.currentTag.includes(' ')).toBe(false); + }); + it('handles comments', () => { + const res = extractDependencies( + '# some comment\n# another\n\nFROM node\n', + config + ); + expect(res).toMatchSnapshot(); + }); + it('handles custom hosts', () => { + const res = extractDependencies( + 'FROM registry2.something.info:5005/node:8\n', + config + ); + expect(res).toMatchSnapshot(); + }); + }); +}); diff --git a/test/manager/docker/package.spec.js b/test/manager/docker/package.spec.js index 6b963261c2..eb1e5a0a0e 100644 --- a/test/manager/docker/package.spec.js +++ b/test/manager/docker/package.spec.js @@ -15,6 +15,8 @@ describe('lib/workers/package/docker', () => { ...defaultConfig, logger, depName: 'some-dep', + currentFrom: 'some-dep:1.0.0@sha256:abcdefghijklmnop', + currentDepTag: 'some-dep:1.0.0', currentTag: '1.0.0', currentDigest: 'sha256:abcdefghijklmnop', }; @@ -32,6 +34,13 @@ describe('lib/workers/package/docker', () => { expect(res).toHaveLength(1); expect(res[0].type).toEqual('digest'); }); + it('adds latest tag', async () => { + delete config.currentTag; + dockerApi.getDigest.mockReturnValueOnce('sha256:1234567890'); + const res = await docker.getPackageUpdates(config); + expect(res).toHaveLength(1); + expect(res[0].type).toEqual('digest'); + }); it('returns a pin', async () => { delete config.currentDigest; dockerApi.getDigest.mockReturnValueOnce('sha256:1234567890'); @@ -77,5 +86,11 @@ describe('lib/workers/package/docker', () => { expect(res[1].type).toEqual('minor'); expect(res[1].newVersion).toEqual('1.1.0-something'); }); + it('ignores deps with custom registry', async () => { + delete config.currentDigest; + config.dockerRegistry = 'registry.something.info:5005'; + const res = await docker.getPackageUpdates(config); + expect(res).toHaveLength(0); + }); }); }); diff --git a/test/manager/docker/update.spec.js b/test/manager/docker/update.spec.js index 4ba3b2f7bf..8058305a9f 100644 --- a/test/manager/docker/update.spec.js +++ b/test/manager/docker/update.spec.js @@ -6,30 +6,39 @@ describe('workers/branch/dockerfile', () => { it('replaces existing value', () => { const currentFileContent = '# comment FROM node:8\nFROM node:8\nRUN something\n'; - const depName = 'node'; - const currentVersion = 'node:8'; - const newVersion = 'node:8@sha256:abcdefghijklmnop'; - const res = dockerfile.setNewValue( - currentFileContent, - depName, - currentVersion, - newVersion, - logger - ); + const upgrade = { + depName: 'node', + currentVersion: 'node:8', + fromPrefix: 'FROM', + fromSuffix: '', + newFrom: 'node:8@sha256:abcdefghijklmnop', + }; + const res = dockerfile.setNewValue(currentFileContent, upgrade, logger); + expect(res).toMatchSnapshot(); + }); + it('replaces existing value with suffix', () => { + const currentFileContent = + '# comment FROM node:8\nFROM node:8 as base\nRUN something\n'; + const upgrade = { + depName: 'node', + currentVersion: 'node:8', + fromPrefix: 'FROM', + fromSuffix: 'as base', + newFrom: 'node:8@sha256:abcdefghijklmnop', + }; + const res = dockerfile.setNewValue(currentFileContent, upgrade, logger); expect(res).toMatchSnapshot(); }); it('returns null on error', () => { const currentFileContent = null; - const depName = 'node'; - const currentVersion = 'node:8'; - const newVersion = 'node:8@sha256:abcdefghijklmnop'; - const res = dockerfile.setNewValue( - currentFileContent, - depName, - currentVersion, - newVersion, - logger - ); + const upgrade = { + depName: 'node', + currentVersion: 'node:8', + fromPrefix: 'FROM', + fromSuffix: '', + newFrom: 'node:8@sha256:abcdefghijklmnop', + }; + const res = dockerfile.setNewValue(currentFileContent, upgrade, logger); expect(res).toBe(null); }); }); -- GitLab