From 37b1c8f0de59a781a499fa249351e075e3231db8 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Sun, 22 Jul 2018 06:33:11 +0200
Subject: [PATCH] feat: gitlabci.yml support (#1744)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Adds support for .gitlabci.yml files. Part of the logic is same as Docker Compose files, however the “services†list is new/different so requires additional logic.
Closes #1598
---
lib/config/definitions.js | 12 +++
lib/manager/gitlabci/extract.js | 56 +++++++++++
lib/manager/gitlabci/index.js | 10 ++
lib/manager/gitlabci/update.js | 42 ++++++++
lib/manager/index.js | 1 +
test/_fixtures/gitlabci/gitlab-ci.yaml | 97 +++++++++++++++++++
.../__snapshots__/extract.spec.js.snap | 81 ++++++++++++++++
test/manager/gitlabci/extract.spec.js | 26 +++++
test/manager/gitlabci/update.spec.js | 82 ++++++++++++++++
.../extract/__snapshots__/index.spec.js.snap | 3 +
website/docs/configuration-options.md | 4 +
11 files changed, 414 insertions(+)
create mode 100644 lib/manager/gitlabci/extract.js
create mode 100644 lib/manager/gitlabci/index.js
create mode 100644 lib/manager/gitlabci/update.js
create mode 100644 test/_fixtures/gitlabci/gitlab-ci.yaml
create mode 100644 test/manager/gitlabci/__snapshots__/extract.spec.js.snap
create mode 100644 test/manager/gitlabci/extract.spec.js
create mode 100644 test/manager/gitlabci/update.spec.js
diff --git a/lib/config/definitions.js b/lib/config/definitions.js
index e0ab85a21d..b0ad426459 100644
--- a/lib/config/definitions.js
+++ b/lib/config/definitions.js
@@ -1066,6 +1066,18 @@ const options = [
mergeable: true,
cli: false,
},
+ {
+ name: 'gitlabci',
+ description:
+ 'Configuration object for GitLab CI yml renovation. Also inherits settings from `docker` object.',
+ stage: 'repository',
+ type: 'json',
+ default: {
+ fileMatch: ['^.gitlab-ci.yml$'],
+ },
+ mergeable: true,
+ cli: false,
+ },
{
name: 'nuget',
description: 'Configuration object for C#/Nuget',
diff --git a/lib/manager/gitlabci/extract.js b/lib/manager/gitlabci/extract.js
new file mode 100644
index 0000000000..f8017e699e
--- /dev/null
+++ b/lib/manager/gitlabci/extract.js
@@ -0,0 +1,56 @@
+const { getDep } = require('../dockerfile/extract');
+
+module.exports = {
+ extractDependencies,
+};
+
+function extractDependencies(content) {
+ const deps = [];
+ try {
+ const lines = content.split('\n');
+ for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
+ const line = lines[lineNumber];
+ const imageMatch = line.match(/^\s*image:\s*'?"?([^\s]+)'?"?\s*$/);
+ if (imageMatch) {
+ logger.trace(`Matched image on line ${lineNumber}`);
+ const currentFrom = imageMatch[1];
+ const dep = getDep(currentFrom);
+ dep.lineNumber = lineNumber;
+ dep.depType = 'image';
+ deps.push(dep);
+ }
+ const services = line.match(/^\s*services:\s*$/);
+ if (services) {
+ logger.trace(`Matched services on line ${lineNumber}`);
+ let foundImage;
+ do {
+ foundImage = false;
+ const serviceImageLine = lines[lineNumber + 1];
+ logger.trace(`serviceImageLine: "${serviceImageLine}"`);
+ const serviceImageMatch = serviceImageLine.match(
+ /^\s*-\s*'?"?([^\s'"]+)'?"?\s*$/
+ );
+ if (serviceImageMatch) {
+ logger.trace('serviceImageMatch');
+ foundImage = true;
+ const currentFrom = serviceImageMatch[1];
+ lineNumber += 1;
+ const dep = getDep(currentFrom);
+ dep.lineNumber = lineNumber;
+ dep.depType = 'service-image';
+ deps.push(dep);
+ }
+ } while (foundImage);
+ }
+ }
+ } catch (err) /* istanbul ignore next */ {
+ logger.error(
+ { err, message: err.message },
+ 'Error extracting GitLab CI dependencies'
+ );
+ }
+ if (!deps.length) {
+ return null;
+ }
+ return { deps };
+}
diff --git a/lib/manager/gitlabci/index.js b/lib/manager/gitlabci/index.js
new file mode 100644
index 0000000000..5549e42b60
--- /dev/null
+++ b/lib/manager/gitlabci/index.js
@@ -0,0 +1,10 @@
+const { extractDependencies } = require('./extract');
+const { updateDependency } = require('./update');
+
+const language = 'docker';
+
+module.exports = {
+ extractDependencies,
+ language,
+ updateDependency,
+};
diff --git a/lib/manager/gitlabci/update.js b/lib/manager/gitlabci/update.js
new file mode 100644
index 0000000000..462dd4e078
--- /dev/null
+++ b/lib/manager/gitlabci/update.js
@@ -0,0 +1,42 @@
+const { getNewFrom } = require('../dockerfile/update');
+
+module.exports = {
+ updateDependency,
+};
+
+function updateDependency(currentFileContent, upgrade) {
+ try {
+ const newFrom = getNewFrom(upgrade);
+ const lines = currentFileContent.split('\n');
+ const lineToChange = lines[upgrade.lineNumber];
+ if (upgrade.depType === 'image') {
+ const imageLine = new RegExp(/^(\s*image:\s*'?"?)[^\s'"]+('?"?\s*)$/);
+ if (!lineToChange.match(imageLine)) {
+ logger.debug('No image line found');
+ return null;
+ }
+ const newLine = lineToChange.replace(imageLine, `$1${newFrom}$2`);
+ if (newLine === lineToChange) {
+ logger.debug('No changes necessary');
+ return currentFileContent;
+ }
+ lines[upgrade.lineNumber] = newLine;
+ return lines.join('\n');
+ }
+ const serviceLine = new RegExp(/^(\s*-\s*'?"?)[^\s'"]+('?"?\s*)$/);
+ if (!lineToChange.match(serviceLine)) {
+ logger.debug('No image line found');
+ return null;
+ }
+ const newLine = lineToChange.replace(serviceLine, `$1${newFrom}$2`);
+ if (newLine === lineToChange) {
+ logger.debug('No changes necessary');
+ return currentFileContent;
+ }
+ lines[upgrade.lineNumber] = newLine;
+ return lines.join('\n');
+ } catch (err) {
+ logger.info({ err }, 'Error setting new Dockerfile value');
+ return null;
+ }
+}
diff --git a/lib/manager/index.js b/lib/manager/index.js
index 4d3c3f686c..d648305761 100644
--- a/lib/manager/index.js
+++ b/lib/manager/index.js
@@ -5,6 +5,7 @@ const managerList = [
'composer',
'docker-compose',
'dockerfile',
+ 'gitlabci',
'meteor',
'npm',
'nvm',
diff --git a/test/_fixtures/gitlabci/gitlab-ci.yaml b/test/_fixtures/gitlabci/gitlab-ci.yaml
new file mode 100644
index 0000000000..37736f369d
--- /dev/null
+++ b/test/_fixtures/gitlabci/gitlab-ci.yaml
@@ -0,0 +1,97 @@
+.executor-docker: &executor-docker
+ tags:
+ - docker
+
+.executor-docker-in-docker: &executor-docker-privileged
+ tags:
+ - docker-privileged
+
+.executor-docker-in-docker: &executor-docker-in-docker
+ tags:
+ - docker-in-docker
+
+.docker-login: &docker-login
+ before_script:
+ - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
+
+.docker-logout: &docker-logout
+ after_script:
+ - docker logout $CI_REGISTRY
+
+.build-image: &build-image
+ export BUILD_IMAGE=$CI_REGISTRY_IMAGE:${CI_COMMIT_TAG:-$CI_COMMIT_REF_SLUG}
+
+stages:
+ - compliance-tests
+ - build
+ - sast
+ - unit-tests
+ - release
+
+variables:
+ LATEST_IMAGE: $CI_REGISTRY_IMAGE:latest
+
+hadolint:
+ stage: compliance-tests
+ <<: *executor-docker
+ image: hadolint/hadolint:latest
+ script:
+ - hadolint Dockerfile
+
+build:
+ stage: build
+ <<: *executor-docker-privileged
+ <<: *docker-login
+ script:
+ - *build-image
+ - docker build --label "org.label-schema.build-date=$(date +%Y-%m-%dT%T%z)" --label "org.label-schema.version=$CI_COMMIT_REF_NAME" --tag $BUILD_IMAGE .
+ - docker push $BUILD_IMAGE
+ <<: *docker-logout
+
+sast:container:
+ stage: sast
+ <<: *executor-docker-in-docker
+ image: docker:latest
+ services:
+ - docker:dind
+ <<: *docker-login
+ script:
+ - *build-image
+ - docker run -d --name db arminc/clair-db:latest
+ - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1
+ - apk add -U wget ca-certificates
+ - docker pull $BUILD_IMAGE
+ - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
+ - mv clair-scanner_linux_amd64 clair-scanner
+ - chmod +x clair-scanner
+ - touch clair-whitelist.yml
+ - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log -w clair-whitelist.yml $BUILD_IMAGE || true
+ <<: *docker-logout
+ artifacts:
+ paths: [gl-sast-container-report.json]
+
+constainer-structure-test:
+ stage: unit-tests
+ <<: *executor-docker-in-docker
+ image: docker:latest
+ services:
+ - docker:dind
+ <<: *docker-login
+ script:
+ - *build-image
+ - docker pull $BUILD_IMAGE
+ - echo "container-structure-test test --verbose --image $BUILD_IMAGE --config /tests/container/*.json" | docker run --rm -i --volume /var/run/docker.sock:/var/run/docker.sock --volume $CI_PROJECT_DIR/tests/container:/tests/container:ro $LATEST_IMAGE /bin/sh; exit $?
+ <<: *docker-logout
+
+release:
+ stage: release
+ <<: *executor-docker-privileged
+ <<: *docker-login
+ script:
+ - *build-image
+ - docker pull $BUILD_IMAGE
+ - docker tag $BUILD_IMAGE $LATEST_IMAGE
+ - docker push $LATEST_IMAGE
+ <<: *docker-logout
+ only:
+ - tags
diff --git a/test/manager/gitlabci/__snapshots__/extract.spec.js.snap b/test/manager/gitlabci/__snapshots__/extract.spec.js.snap
new file mode 100644
index 0000000000..70e590613e
--- /dev/null
+++ b/test/manager/gitlabci/__snapshots__/extract.spec.js.snap
@@ -0,0 +1,81 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`lib/manager/gitlabci/extract extractDependencies() extracts multiple image lines 1`] = `
+Array [
+ Object {
+ "currentDepTag": "hadolint/hadolint:latest",
+ "currentDepTagDigest": "hadolint/hadolint:latest",
+ "currentDigest": undefined,
+ "currentFrom": "hadolint/hadolint:latest",
+ "currentTag": "latest",
+ "currentValue": "latest",
+ "depName": "hadolint/hadolint",
+ "depType": "image",
+ "dockerRegistry": undefined,
+ "lineNumber": 36,
+ "purl": "pkg:docker/hadolint/hadolint",
+ "tagSuffix": undefined,
+ "versionScheme": "docker",
+ },
+ Object {
+ "currentDepTag": "docker:latest",
+ "currentDepTagDigest": "docker:latest",
+ "currentDigest": undefined,
+ "currentFrom": "docker:latest",
+ "currentTag": "latest",
+ "currentValue": "latest",
+ "depName": "docker",
+ "depType": "image",
+ "dockerRegistry": undefined,
+ "lineNumber": 53,
+ "purl": "pkg:docker/docker",
+ "tagSuffix": undefined,
+ "versionScheme": "docker",
+ },
+ Object {
+ "currentDepTag": "docker:dind",
+ "currentDepTagDigest": "docker:dind",
+ "currentDigest": undefined,
+ "currentFrom": "docker:dind",
+ "currentTag": "dind",
+ "currentValue": "dind",
+ "depName": "docker",
+ "depType": "service-image",
+ "dockerRegistry": undefined,
+ "lineNumber": 55,
+ "purl": "pkg:docker/docker",
+ "tagSuffix": undefined,
+ "versionScheme": "docker",
+ },
+ Object {
+ "currentDepTag": "docker:latest",
+ "currentDepTagDigest": "docker:latest",
+ "currentDigest": undefined,
+ "currentFrom": "docker:latest",
+ "currentTag": "latest",
+ "currentValue": "latest",
+ "depName": "docker",
+ "depType": "image",
+ "dockerRegistry": undefined,
+ "lineNumber": 75,
+ "purl": "pkg:docker/docker",
+ "tagSuffix": undefined,
+ "versionScheme": "docker",
+ },
+ Object {
+ "currentDepTag": "docker:dind",
+ "currentDepTagDigest": "docker:dind",
+ "currentDigest": undefined,
+ "currentFrom": "docker:dind",
+ "currentTag": "dind",
+ "currentValue": "dind",
+ "depName": "docker",
+ "depType": "service-image",
+ "dockerRegistry": undefined,
+ "lineNumber": 77,
+ "purl": "pkg:docker/docker",
+ "tagSuffix": undefined,
+ "versionScheme": "docker",
+ },
+]
+`;
diff --git a/test/manager/gitlabci/extract.spec.js b/test/manager/gitlabci/extract.spec.js
new file mode 100644
index 0000000000..78741eefc0
--- /dev/null
+++ b/test/manager/gitlabci/extract.spec.js
@@ -0,0 +1,26 @@
+const fs = require('fs');
+const {
+ extractDependencies,
+} = require('../../../lib/manager/gitlabci/extract');
+
+const yamlFile = fs.readFileSync(
+ 'test/_fixtures/gitlabci/gitlab-ci.yaml',
+ 'utf8'
+);
+
+describe('lib/manager/gitlabci/extract', () => {
+ describe('extractDependencies()', () => {
+ let config;
+ beforeEach(() => {
+ config = {};
+ });
+ it('returns null for empty', () => {
+ expect(extractDependencies('nothing here', config)).toBe(null);
+ });
+ it('extracts multiple image lines', () => {
+ const res = extractDependencies(yamlFile, config);
+ expect(res.deps).toMatchSnapshot();
+ expect(res.deps).toHaveLength(5);
+ });
+ });
+});
diff --git a/test/manager/gitlabci/update.spec.js b/test/manager/gitlabci/update.spec.js
new file mode 100644
index 0000000000..ac6f77e84a
--- /dev/null
+++ b/test/manager/gitlabci/update.spec.js
@@ -0,0 +1,82 @@
+const fs = require('fs');
+const dcUpdate = require('../../../lib/manager/gitlabci/update');
+
+const yamlFile = fs.readFileSync(
+ 'test/_fixtures/gitlabci/gitlab-ci.yaml',
+ 'utf8'
+);
+
+describe('manager/gitlabci/update', () => {
+ describe('updateDependency', () => {
+ it('replaces existing value', () => {
+ const upgrade = {
+ lineNumber: 36,
+ depType: 'image',
+ depName: 'hadolint/hadolint',
+ newValue: '7.0.0',
+ newDigest: 'sha256:abcdefghijklmnop',
+ };
+ const res = dcUpdate.updateDependency(yamlFile, upgrade);
+ expect(res).not.toEqual(yamlFile);
+ expect(res.includes(upgrade.newDigest)).toBe(true);
+ });
+ it('returns same', () => {
+ const upgrade = {
+ depType: 'image',
+ lineNumber: 36,
+ depName: 'hadolint/hadolint',
+ newValue: 'latest',
+ };
+ const res = dcUpdate.updateDependency(yamlFile, upgrade);
+ expect(res).toEqual(yamlFile);
+ });
+ it('returns null if mismatch', () => {
+ const upgrade = {
+ lineNumber: 17,
+ depType: 'image',
+ depName: 'postgres',
+ newValue: '9.6.8',
+ newDigest: 'sha256:abcdefghijklmnop',
+ };
+ const res = dcUpdate.updateDependency(yamlFile, upgrade);
+ expect(res).toBe(null);
+ });
+ it('replaces service-image update', () => {
+ const upgrade = {
+ lineNumber: 55,
+ depType: 'service-image',
+ depName: 'hadolint/hadolint',
+ newValue: '7.0.0',
+ newDigest: 'sha256:abcdefghijklmnop',
+ };
+ const res = dcUpdate.updateDependency(yamlFile, upgrade);
+ expect(res).not.toEqual(yamlFile);
+ expect(res.includes(upgrade.newDigest)).toBe(true);
+ });
+ it('returns null if service-image mismatch', () => {
+ const upgrade = {
+ lineNumber: 17,
+ depType: 'service-image',
+ depName: 'postgres',
+ newValue: '9.6.8',
+ newDigest: 'sha256:abcdefghijklmnop',
+ };
+ const res = dcUpdate.updateDependency(yamlFile, upgrade);
+ expect(res).toBe(null);
+ });
+ it('returns service-image same', () => {
+ const upgrade = {
+ depType: 'serviceimage',
+ lineNumber: 55,
+ depName: 'docker',
+ newValue: 'dind',
+ };
+ const res = dcUpdate.updateDependency(yamlFile, upgrade);
+ expect(res).toEqual(yamlFile);
+ });
+ it('returns null if error', () => {
+ const res = dcUpdate.updateDependency(null, null);
+ expect(res).toBe(null);
+ });
+ });
+});
diff --git a/test/workers/repository/extract/__snapshots__/index.spec.js.snap b/test/workers/repository/extract/__snapshots__/index.spec.js.snap
index 707924ce03..c281ecd58c 100644
--- a/test/workers/repository/extract/__snapshots__/index.spec.js.snap
+++ b/test/workers/repository/extract/__snapshots__/index.spec.js.snap
@@ -20,6 +20,9 @@ Object {
"dockerfile": Array [
Object {},
],
+ "gitlabci": Array [
+ Object {},
+ ],
"meteor": Array [
Object {},
],
diff --git a/website/docs/configuration-options.md b/website/docs/configuration-options.md
index 12a309dfcf..e0b6f3b62d 100644
--- a/website/docs/configuration-options.md
+++ b/website/docs/configuration-options.md
@@ -199,6 +199,10 @@ See https://renovatebot.com/docs/configuration-reference/config-presets for deta
## fileMatch
+## gitlabci
+
+Add to this configuration setting if you need to override any of the GitLab CI default settings. Use the `docker` config object instead if you wish for configuration to apply across all Docker-related package managers.
+
## group
The default configuration for groups are essentially internal to Renovate and you normally shouldn't need to modify them. However, you may choose to _add_ settings to any group by defining your own `group` configuration object.
--
GitLab