From e47712f3679509550dcd4321f9836c5effb3b5ca Mon Sep 17 00:00:00 2001
From: Hasan Awad <90554456+hasanwhitesource@users.noreply.github.com>
Date: Thu, 7 Apr 2022 14:32:54 +0300
Subject: [PATCH] feat(gitlabci): used yaml parsing (#14879)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
.../gitlabci/__fixtures__/gitlab-ci.6.yaml | 8 +
.../__snapshots__/extract.spec.ts.snap | 54 +++++++
lib/modules/manager/gitlabci/extract.spec.ts | 79 +++++++++-
lib/modules/manager/gitlabci/extract.ts | 149 +++++++++---------
lib/modules/manager/gitlabci/types.ts | 15 ++
5 files changed, 226 insertions(+), 79 deletions(-)
diff --git a/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.6.yaml b/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.6.yaml
index 2fe3c85a81..6639df8271 100644
--- a/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.6.yaml
+++ b/lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.6.yaml
@@ -17,3 +17,11 @@ job1:
- -something2
- -something3
- -something4
+job2:
+ services:
+ - "mariadb:10.4.11"
+ - postgres:11.7
+ - redis:latest
+ - name: "registry.example.com/myimage:latest"
+ - myimage@sha256:0ecb2ad60
+ - tomcat:7-jre8
diff --git a/lib/modules/manager/gitlabci/__snapshots__/extract.spec.ts.snap b/lib/modules/manager/gitlabci/__snapshots__/extract.spec.ts.snap
index be3028decd..d522679e36 100644
--- a/lib/modules/manager/gitlabci/__snapshots__/extract.spec.ts.snap
+++ b/lib/modules/manager/gitlabci/__snapshots__/extract.spec.ts.snap
@@ -224,6 +224,60 @@ Array [
"depType": "service-image",
"replaceString": "mooseagency/postgresql:12.3-1@sha256:a5a65569456f221ee1f8a0b3b4e2d440eb5830772d9440c9b30b1dbfd454c778",
},
+ Object {
+ "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+ "currentDigest": undefined,
+ "currentValue": "10.4.11",
+ "datasource": "docker",
+ "depName": "mariadb",
+ "depType": "service-image",
+ "replaceString": "mariadb:10.4.11",
+ },
+ Object {
+ "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+ "currentDigest": undefined,
+ "currentValue": "11.7",
+ "datasource": "docker",
+ "depName": "postgres",
+ "depType": "service-image",
+ "replaceString": "postgres:11.7",
+ },
+ Object {
+ "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+ "currentDigest": undefined,
+ "currentValue": "latest",
+ "datasource": "docker",
+ "depName": "redis",
+ "depType": "service-image",
+ "replaceString": "redis:latest",
+ },
+ Object {
+ "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+ "currentDigest": undefined,
+ "currentValue": "latest",
+ "datasource": "docker",
+ "depName": "registry.example.com/myimage",
+ "depType": "service-image",
+ "replaceString": "registry.example.com/myimage:latest",
+ },
+ Object {
+ "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+ "currentDigest": "sha256:0ecb2ad60",
+ "currentValue": undefined,
+ "datasource": "docker",
+ "depName": "myimage",
+ "depType": "service-image",
+ "replaceString": "myimage@sha256:0ecb2ad60",
+ },
+ Object {
+ "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+ "currentDigest": undefined,
+ "currentValue": "7-jre8",
+ "datasource": "docker",
+ "depName": "tomcat",
+ "depType": "service-image",
+ "replaceString": "tomcat:7-jre8",
+ },
],
"packageFile": "lib/modules/manager/gitlabci/__fixtures__/gitlab-ci.6.yaml",
},
diff --git a/lib/modules/manager/gitlabci/extract.spec.ts b/lib/modules/manager/gitlabci/extract.spec.ts
index 8fe02f84c1..2eedb77994 100644
--- a/lib/modules/manager/gitlabci/extract.spec.ts
+++ b/lib/modules/manager/gitlabci/extract.spec.ts
@@ -2,7 +2,12 @@ import { logger } from '../../../../test/util';
import { GlobalConfig } from '../../../config/global';
import type { RepoGlobalConfig } from '../../../config/types';
import type { ExtractConfig, PackageDependency } from '../types';
-import { extractAllPackageFiles } from './extract';
+import {
+ extractAllPackageFiles,
+ extractFromImage,
+ extractFromJob,
+ extractFromServices,
+} from './extract';
const config: ExtractConfig = {};
@@ -57,7 +62,7 @@ describe('modules/manager/gitlabci/extract', () => {
]);
expect(res).toMatchSnapshot();
expect(res).toHaveLength(1);
- expect(res[0].deps).toHaveLength(4);
+ expect(res[0].deps).toHaveLength(10);
});
it('extracts multiple image lines', async () => {
@@ -155,5 +160,75 @@ describe('modules/manager/gitlabci/extract', () => {
},
]);
});
+ it('extracts from image', () => {
+ let expectedRes = {
+ autoReplaceStringTemplate:
+ '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
+ currentDigest: undefined,
+ currentValue: 'test',
+ datasource: 'docker',
+ depName: 'image',
+ depType: 'image',
+ replaceString: 'image:test',
+ };
+
+ expect(extractFromImage('image:test')).toEqual(expectedRes);
+
+ expectedRes = { ...expectedRes, depType: 'image-name' };
+ expect(
+ extractFromImage({
+ name: 'image:test',
+ })
+ ).toEqual(expectedRes);
+
+ expect(extractFromImage(undefined)).toBeNull();
+ });
+
+ it('extracts from services', () => {
+ const expectedRes = [
+ {
+ autoReplaceStringTemplate:
+ '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
+ currentDigest: undefined,
+ currentValue: 'test',
+ datasource: 'docker',
+ depName: 'image',
+ depType: 'service-image',
+ replaceString: 'image:test',
+ },
+ {
+ autoReplaceStringTemplate:
+ '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
+ currentDigest: undefined,
+ currentValue: 'test2',
+ datasource: 'docker',
+ depName: 'image2',
+ depType: 'service-image',
+ replaceString: 'image2:test2',
+ },
+ ];
+ const services = ['image:test', 'image2:test2'];
+ expect(extractFromServices(undefined)).toBeEmptyArray();
+ expect(extractFromServices(services)).toEqual(expectedRes);
+ expect(
+ extractFromServices([{ name: 'image:test' }, { name: 'image2:test2' }])
+ ).toEqual(expectedRes);
+ });
+ it('extracts from job object', () => {
+ const expectedRes = [
+ {
+ autoReplaceStringTemplate:
+ '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}',
+ currentDigest: undefined,
+ currentValue: 'test',
+ datasource: 'docker',
+ depName: 'image',
+ depType: 'image',
+ replaceString: 'image:test',
+ },
+ ];
+ expect(extractFromJob(undefined)).toBeEmptyArray();
+ expect(extractFromJob({ image: 'image:test' })).toEqual(expectedRes);
+ });
});
});
diff --git a/lib/modules/manager/gitlabci/extract.ts b/lib/modules/manager/gitlabci/extract.ts
index 0319139103..f7b9b79127 100644
--- a/lib/modules/manager/gitlabci/extract.ts
+++ b/lib/modules/manager/gitlabci/extract.ts
@@ -2,95 +2,90 @@ import is from '@sindresorhus/is';
import { load } from 'js-yaml';
import { logger } from '../../../logger';
import { readLocalFile } from '../../../util/fs';
-import { newlineRegex, regEx } from '../../../util/regex';
+import { regEx } from '../../../util/regex';
import { getDep } from '../dockerfile/extract';
import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
-import type { GitlabPipeline } from './types';
+import type { GitlabPipeline, Image, Job, Services } from './types';
import { replaceReferenceTags } from './utils';
-const commentsRe = regEx(/^\s*#/);
-const aliasesRe = regEx(`^\\s*-?\\s*alias:`);
-const whitespaceRe = regEx(`^(?<whitespace>\\s*)`);
-const imageRe = regEx(
- `^(?<whitespace>\\s*)image:(?:\\s+['"]?(?<image>[^\\s'"]+)['"]?)?\\s*$`
-);
-const nameRe = regEx(`^\\s*name:\\s+['"]?(?<depName>[^\\s'"]+)['"]?\\s*$`);
-const serviceRe = regEx(
- `^\\s*-?\\s*(?:name:\\s+)?['"]?(?<depName>[^\\s'"]+[^:]$)['"]?\\s*$`
-);
-function skipCommentAndAliasLines(
- lines: string[],
- lineNumber: number
-): { lineNumber: number; line: string } {
- let ln = lineNumber;
- while (
- ln < lines.length - 1 &&
- (commentsRe.test(lines[ln]) || aliasesRe.test(lines[ln]))
- ) {
- ln += 1;
+export function extractFromImage(
+ image: Image | undefined
+): PackageDependency | null {
+ if (is.undefined(image)) {
+ return null;
+ }
+ let dep: PackageDependency = null;
+ if (is.string(image)) {
+ dep = getDep(image);
+ dep.depType = 'image';
+ } else if (is.string(image?.name)) {
+ dep = getDep(image.name);
+ dep.depType = 'image-name';
}
- return { line: lines[ln], lineNumber: ln };
+ return dep;
}
-export function extractPackageFile(content: string): PackageFile | null {
+export function extractFromServices(
+ services: Services | undefined
+): PackageDependency[] {
+ if (is.undefined(services)) {
+ return [];
+ }
+ const deps: PackageDependency[] = [];
+ for (const service of services) {
+ if (is.string(service)) {
+ const dep = getDep(service);
+ dep.depType = 'service-image';
+ deps.push(dep);
+ } else if (is.string(service?.name)) {
+ const dep = getDep(service.name);
+ dep.depType = 'service-image';
+ deps.push(dep);
+ }
+ }
+ return deps;
+}
+
+export function extractFromJob(job: Job | undefined): PackageDependency[] {
+ if (is.undefined(job)) {
+ return [];
+ }
const deps: PackageDependency[] = [];
+ if (is.object(job)) {
+ const { image, services } = { ...job };
+ if (is.object(image) || is.string(image)) {
+ deps.push(extractFromImage(image));
+ }
+ if (is.array(services)) {
+ deps.push(...extractFromServices(services));
+ }
+ }
+ return deps;
+}
+
+export function extractPackageFile(content: string): PackageFile | null {
+ let deps: PackageDependency[] = [];
try {
- const lines = content.split(newlineRegex);
- for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
- const line = lines[lineNumber];
- const imageMatch = imageRe.exec(line);
- if (imageMatch) {
- switch (imageMatch.groups.image) {
- case undefined:
- case '': {
- let blockLine;
- do {
- lineNumber += 1;
- blockLine = lines[lineNumber];
- const imageNameMatch = nameRe.exec(blockLine);
- if (imageNameMatch) {
- logger.trace(`Matched image name on line ${lineNumber}`);
- const dep = getDep(imageNameMatch.groups.depName);
- dep.depType = 'image-name';
- deps.push(dep);
- break;
- }
- } while (
- whitespaceRe.exec(blockLine)?.groups.whitespace.length >
- imageMatch.groups.whitespace.length
- );
+ const doc = load(replaceReferenceTags(content), {
+ json: true,
+ }) as Record<string, Image | Services | Job>;
+ if (is.object(doc)) {
+ for (const [property, value] of Object.entries(doc)) {
+ switch (property) {
+ case 'image':
+ deps.push(extractFromImage(value as Image));
+ break;
+
+ case 'services':
+ deps.push(...extractFromServices(value as Services));
+ break;
+
+ default:
+ deps.push(...extractFromJob(value as Job));
break;
- }
- default: {
- logger.trace(`Matched image on line ${lineNumber}`);
- const dep = getDep(imageMatch.groups.image);
- dep.depType = 'image';
- deps.push(dep);
- }
}
}
- const services = regEx(/^\s*services:\s*$/).test(line);
- if (services) {
- logger.trace(`Matched services on line ${lineNumber}`);
- let foundImage: boolean;
- do {
- foundImage = false;
- const serviceImageLine = skipCommentAndAliasLines(
- lines,
- lineNumber + 1
- );
- logger.trace(`serviceImageLine: "${serviceImageLine.line}"`);
- const serviceImageMatch = serviceRe.exec(serviceImageLine.line);
- if (serviceImageMatch) {
- logger.trace('serviceImageMatch');
- foundImage = true;
- lineNumber = serviceImageLine.lineNumber;
- const dep = getDep(serviceImageMatch.groups.depName);
- dep.depType = 'service-image';
- deps.push(dep);
- }
- } while (foundImage);
- }
+ deps = deps.filter(is.truthy);
}
} catch (err) /* istanbul ignore next */ {
logger.warn({ err }, 'Error extracting GitLab CI dependencies');
diff --git a/lib/modules/manager/gitlabci/types.ts b/lib/modules/manager/gitlabci/types.ts
index ace7be0584..18aabed7d9 100644
--- a/lib/modules/manager/gitlabci/types.ts
+++ b/lib/modules/manager/gitlabci/types.ts
@@ -5,3 +5,18 @@ export interface GitlabInclude {
export interface GitlabPipeline {
include?: GitlabInclude[] | string;
}
+
+export interface ImageObject {
+ name: string;
+ entrypoint?: string[];
+}
+export interface ServicesObject extends ImageObject {
+ command?: string[];
+ alias?: string;
+}
+export interface Job {
+ image?: Image;
+ services?: Services;
+}
+export type Image = ImageObject | string;
+export type Services = (string | ServicesObject)[];
--
GitLab