From 21ff27d5ab1003616b6a16c26ee1a59b91e8b908 Mon Sep 17 00:00:00 2001
From: fredrondina <fredrondina96@gmail.com>
Date: Tue, 16 Aug 2022 10:36:05 -0500
Subject: [PATCH] feat(gitlab-ci): ref add logic for updating non top level
`include`s (#16819)
Co-authored-by: Fred Rondina <fred.rondina@daveramsey.com>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
.../__fixtures__/gitlab-ci.4.yaml | 13 ++++
.../manager/gitlabci-include/common.spec.ts | 60 ++++++++++++++++
.../manager/gitlabci-include/common.ts | 34 +++++++++
.../manager/gitlabci-include/extract.spec.ts | 27 +++++++
.../manager/gitlabci-include/extract.ts | 70 +++++++++++++------
lib/modules/manager/gitlabci/common.spec.ts | 16 +++++
lib/modules/manager/gitlabci/common.ts | 8 +++
lib/modules/manager/gitlabci/extract.ts | 13 ++--
lib/modules/manager/gitlabci/types.ts | 26 ++++++-
9 files changed, 237 insertions(+), 30 deletions(-)
create mode 100644 lib/modules/manager/gitlabci-include/__fixtures__/gitlab-ci.4.yaml
create mode 100644 lib/modules/manager/gitlabci-include/common.spec.ts
create mode 100644 lib/modules/manager/gitlabci-include/common.ts
create mode 100644 lib/modules/manager/gitlabci/common.spec.ts
create mode 100644 lib/modules/manager/gitlabci/common.ts
diff --git a/lib/modules/manager/gitlabci-include/__fixtures__/gitlab-ci.4.yaml b/lib/modules/manager/gitlabci-include/__fixtures__/gitlab-ci.4.yaml
new file mode 100644
index 0000000000..ec7d4848a0
--- /dev/null
+++ b/lib/modules/manager/gitlabci-include/__fixtures__/gitlab-ci.4.yaml
@@ -0,0 +1,13 @@
+---
+include:
+- project: mikebryant/include-source-example
+ file: /template.yaml
+ ref: 1.0.0
+
+trigger-my-job:
+ extends: .extend-trigger-job
+ trigger:
+ include:
+ - project: mikebryant/include-source-example
+ file: /template.yaml
+ ref: master
diff --git a/lib/modules/manager/gitlabci-include/common.spec.ts b/lib/modules/manager/gitlabci-include/common.spec.ts
new file mode 100644
index 0000000000..a1c2dcd024
--- /dev/null
+++ b/lib/modules/manager/gitlabci-include/common.spec.ts
@@ -0,0 +1,60 @@
+import { load } from 'js-yaml';
+import { Fixtures } from '../../../../test/fixtures';
+import type { GitlabPipeline } from '../gitlabci/types';
+import { replaceReferenceTags } from '../gitlabci/utils';
+import {
+ filterIncludeFromGitlabPipeline,
+ isGitlabIncludeLocal,
+ isGitlabIncludeProject,
+ isNonEmptyObject,
+} from './common';
+
+const yamlFileMultiConfig = Fixtures.get('gitlab-ci.1.yaml');
+const pipeline = load(
+ replaceReferenceTags(yamlFileMultiConfig)
+) as GitlabPipeline;
+const includeLocal = { local: 'something' };
+const includeProject = { project: 'something' };
+
+describe('modules/manager/gitlabci-include/common', () => {
+ describe('filterIncludeFromGitlabPipeline()', () => {
+ it('returns GitlabPipeline without top level include key', () => {
+ expect(pipeline).toHaveProperty('include');
+ const filtered_pipeline = filterIncludeFromGitlabPipeline(pipeline);
+ expect(filtered_pipeline).not.toHaveProperty('include');
+ expect(filtered_pipeline).toEqual({
+ script: [null, null],
+ });
+ });
+ });
+
+ describe('isGitlabIncludeLocal()', () => {
+ it('returns true if GitlabInclude is GitlabIncludeLocal', () => {
+ expect(isGitlabIncludeLocal(includeLocal)).toBe(true);
+ });
+
+ it('returns false if GitlabInclude is not GitlabIncludeLocal', () => {
+ expect(isGitlabIncludeLocal(includeProject)).toBe(false);
+ });
+ });
+
+ describe('isGitlabIncludeProject()', () => {
+ it('returns true if GitlabInclude is GitlabIncludeProject', () => {
+ expect(isGitlabIncludeProject(includeProject)).toBe(true);
+ });
+
+ it('returns false if GitlabInclude is not GitlabIncludeProject', () => {
+ expect(isGitlabIncludeProject(includeLocal)).toBe(false);
+ });
+ });
+
+ describe('isNonEmptyObject()', () => {
+ it('returns true if not empty', () => {
+ expect(isNonEmptyObject({ attribute1: 1 })).toBe(true);
+ });
+
+ it('returns false if empty', () => {
+ expect(isNonEmptyObject({})).toBe(false);
+ });
+ });
+});
diff --git a/lib/modules/manager/gitlabci-include/common.ts b/lib/modules/manager/gitlabci-include/common.ts
new file mode 100644
index 0000000000..da163cb0ea
--- /dev/null
+++ b/lib/modules/manager/gitlabci-include/common.ts
@@ -0,0 +1,34 @@
+import is from '@sindresorhus/is';
+import type {
+ GitlabInclude,
+ GitlabIncludeLocal,
+ GitlabIncludeProject,
+ GitlabPipeline,
+} from '../gitlabci/types';
+
+export function isNonEmptyObject(obj: any): boolean {
+ return is.object(obj) && Object.keys(obj).length !== 0;
+}
+
+export function filterIncludeFromGitlabPipeline(
+ pipeline: GitlabPipeline
+): GitlabPipeline {
+ const pipeline_without_include = {} as GitlabPipeline;
+ for (const key of Object.keys(pipeline).filter((key) => key !== 'include')) {
+ const pipeline_key = key as keyof typeof pipeline;
+ pipeline_without_include[pipeline_key] = pipeline[pipeline_key];
+ }
+ return pipeline_without_include;
+}
+
+export function isGitlabIncludeProject(
+ include: GitlabInclude
+): include is GitlabIncludeProject {
+ return !is.undefined((include as GitlabIncludeProject).project);
+}
+
+export function isGitlabIncludeLocal(
+ include: GitlabInclude
+): include is GitlabIncludeLocal {
+ return !is.undefined((include as GitlabIncludeLocal).local);
+}
diff --git a/lib/modules/manager/gitlabci-include/extract.spec.ts b/lib/modules/manager/gitlabci-include/extract.spec.ts
index 1cf0dd2fda..9ebd9b8c57 100644
--- a/lib/modules/manager/gitlabci-include/extract.spec.ts
+++ b/lib/modules/manager/gitlabci-include/extract.spec.ts
@@ -5,6 +5,7 @@ import { extractPackageFile } from '.';
const yamlFileMultiConfig = Fixtures.get('gitlab-ci.1.yaml');
const yamlFileSingleConfig = Fixtures.get('gitlab-ci.2.yaml');
const yamlWithEmptyIncludeConfig = Fixtures.get('gitlab-ci.3.yaml');
+const yamlWithTriggerRef = Fixtures.get('gitlab-ci.4.yaml');
describe('modules/manager/gitlabci-include/extract', () => {
describe('extractPackageFile()', () => {
@@ -29,6 +30,32 @@ describe('modules/manager/gitlabci-include/extract', () => {
expect(res?.deps).toHaveLength(3);
});
+ it('extracts multiple embedded include blocks', () => {
+ const res = extractPackageFile(yamlWithTriggerRef);
+ expect(res?.deps).toHaveLength(2);
+ expect(res?.deps).toMatchObject([
+ {
+ currentValue: 'master',
+ datasource: 'gitlab-tags',
+ depName: 'mikebryant/include-source-example',
+ },
+ {
+ currentValue: '1.0.0',
+ datasource: 'gitlab-tags',
+ depName: 'mikebryant/include-source-example',
+ },
+ ]);
+ });
+
+ it('ignores includes without project and file keys', () => {
+ const includeWithoutProjectRef = `include:
+ - 'https://gitlab.com/mikebryant/include-source-example.yml'
+ - remote: 'https://gitlab.com/mikebryant/include-source-example.yml'
+ - local: mikebryant/include-source-example`;
+ const res = extractPackageFile(includeWithoutProjectRef);
+ expect(res).toBeNull();
+ });
+
it('normalizes configured endpoints', () => {
const endpoints = [
'http://gitlab.test/api/v4',
diff --git a/lib/modules/manager/gitlabci-include/extract.ts b/lib/modules/manager/gitlabci-include/extract.ts
index ea6be88efc..1fa8c9d6e7 100644
--- a/lib/modules/manager/gitlabci-include/extract.ts
+++ b/lib/modules/manager/gitlabci-include/extract.ts
@@ -4,14 +4,22 @@ import { GlobalConfig } from '../../../config/global';
import { logger } from '../../../logger';
import { regEx } from '../../../util/regex';
import { GitlabTagsDatasource } from '../../datasource/gitlab-tags';
+import type {
+ GitlabInclude,
+ GitlabIncludeProject,
+ GitlabPipeline,
+} from '../gitlabci/types';
import { replaceReferenceTags } from '../gitlabci/utils';
import type { PackageDependency, PackageFile } from '../types';
+import {
+ filterIncludeFromGitlabPipeline,
+ isGitlabIncludeProject,
+ isNonEmptyObject,
+} from './common';
-function extractDepFromIncludeFile(includeObj: {
- file: any;
- project: string;
- ref: string;
-}): PackageDependency {
+function extractDepFromIncludeFile(
+ includeObj: GitlabIncludeProject
+): PackageDependency {
const dep: PackageDependency = {
datasource: GitlabTagsDatasource.id,
depName: includeObj.project,
@@ -25,28 +33,50 @@ function extractDepFromIncludeFile(includeObj: {
return dep;
}
+function getIncludeProjectsFromInclude(
+ includeValue: GitlabInclude[] | GitlabInclude
+): GitlabIncludeProject[] {
+ const includes = is.array(includeValue) ? includeValue : [includeValue];
+
+ // Filter out includes that dont have a file & project.
+ return includes.filter(isGitlabIncludeProject);
+}
+
+function getAllIncludeProjects(data: GitlabPipeline): GitlabIncludeProject[] {
+ // If Array, search each element.
+ if (is.array(data)) {
+ return (data as GitlabPipeline[])
+ .filter(isNonEmptyObject)
+ .map(getAllIncludeProjects)
+ .flat();
+ }
+
+ const childrenData = Object.values(filterIncludeFromGitlabPipeline(data))
+ .filter(isNonEmptyObject)
+ .map(getAllIncludeProjects)
+ .flat();
+
+ // Process include key.
+ if (data.include) {
+ childrenData.push(...getIncludeProjectsFromInclude(data.include));
+ }
+ return childrenData;
+}
+
export function extractPackageFile(content: string): PackageFile | null {
const deps: PackageDependency[] = [];
const { platform, endpoint } = GlobalConfig.get();
try {
- // TODO: fix me (#9610)
- const doc: any = load(replaceReferenceTags(content), {
+ const doc = load(replaceReferenceTags(content), {
json: true,
- });
- let includes;
- if (doc?.include && is.array(doc.include)) {
- includes = doc.include;
- } else {
- includes = [doc.include];
- }
+ }) as GitlabPipeline;
+ const includes = getAllIncludeProjects(doc);
for (const includeObj of includes) {
- if (includeObj?.file && includeObj.project) {
- const dep = extractDepFromIncludeFile(includeObj);
- if (platform === 'gitlab' && endpoint) {
- dep.registryUrls = [endpoint.replace(regEx(/\/api\/v4\/?/), '')];
- }
- deps.push(dep);
+ const dep = extractDepFromIncludeFile(includeObj);
+ if (platform === 'gitlab' && endpoint) {
+ dep.registryUrls = [endpoint.replace(regEx(/\/api\/v4\/?/), '')];
}
+ deps.push(dep);
}
} catch (err) /* istanbul ignore next */ {
if (err.stack?.startsWith('YAMLException:')) {
diff --git a/lib/modules/manager/gitlabci/common.spec.ts b/lib/modules/manager/gitlabci/common.spec.ts
new file mode 100644
index 0000000000..d40f202d26
--- /dev/null
+++ b/lib/modules/manager/gitlabci/common.spec.ts
@@ -0,0 +1,16 @@
+import { isGitlabIncludeLocal } from './common';
+
+const includeLocal = { local: 'something' };
+const includeProject = { project: 'something' };
+
+describe('modules/manager/gitlabci/common', () => {
+ describe('isGitlabIncludeLocal()', () => {
+ it('returns true if GitlabInclude is GitlabIncludeLocal', () => {
+ expect(isGitlabIncludeLocal(includeLocal)).toBe(true);
+ });
+
+ it('returns false if GitlabInclude is not GitlabIncludeLocal', () => {
+ expect(isGitlabIncludeLocal(includeProject)).toBe(false);
+ });
+ });
+});
diff --git a/lib/modules/manager/gitlabci/common.ts b/lib/modules/manager/gitlabci/common.ts
new file mode 100644
index 0000000000..baa89d32dd
--- /dev/null
+++ b/lib/modules/manager/gitlabci/common.ts
@@ -0,0 +1,8 @@
+import is from '@sindresorhus/is';
+import type { GitlabInclude, GitlabIncludeLocal } from '../gitlabci/types';
+
+export function isGitlabIncludeLocal(
+ include: GitlabInclude
+): include is GitlabIncludeLocal {
+ return !is.undefined((include as GitlabIncludeLocal).local);
+}
diff --git a/lib/modules/manager/gitlabci/extract.ts b/lib/modules/manager/gitlabci/extract.ts
index ff674a2bff..32abce4ff0 100644
--- a/lib/modules/manager/gitlabci/extract.ts
+++ b/lib/modules/manager/gitlabci/extract.ts
@@ -4,6 +4,7 @@ import { logger } from '../../../logger';
import { readLocalFile } from '../../../util/fs';
import { trimLeadingSlash } from '../../../util/url';
import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
+import { isGitlabIncludeLocal } from './common';
import type { GitlabPipeline, Image, Job, Services } from './types';
import { getGitlabDep, replaceReferenceTags } from './utils';
@@ -144,13 +145,11 @@ export async function extractAllPackageFiles(
}
if (is.array(doc?.include)) {
- for (const includeObj of doc.include) {
- if (is.string(includeObj.local)) {
- const fileObj = trimLeadingSlash(includeObj.local);
- if (!seen.has(fileObj)) {
- seen.add(fileObj);
- filesToExamine.push(fileObj);
- }
+ for (const includeObj of doc.include.filter(isGitlabIncludeLocal)) {
+ const fileObj = trimLeadingSlash(includeObj.local);
+ if (!seen.has(fileObj)) {
+ seen.add(fileObj);
+ filesToExamine.push(fileObj);
}
}
} else if (is.string(doc?.include)) {
diff --git a/lib/modules/manager/gitlabci/types.ts b/lib/modules/manager/gitlabci/types.ts
index 18aabed7d9..9f8d7726b2 100644
--- a/lib/modules/manager/gitlabci/types.ts
+++ b/lib/modules/manager/gitlabci/types.ts
@@ -1,9 +1,23 @@
-export interface GitlabInclude {
- local?: string;
+export interface GitlabIncludeLocal {
+ local: string;
+}
+
+export interface GitlabIncludeProject {
+ project: string;
+ file?: string;
+ ref?: string;
+}
+
+export interface GitlabIncludeRemote {
+ remote: string;
+}
+
+export interface GitlabIncludeTemplate {
+ template: string;
}
export interface GitlabPipeline {
- include?: GitlabInclude[] | string;
+ include?: GitlabInclude[] | GitlabInclude;
}
export interface ImageObject {
@@ -20,3 +34,9 @@ export interface Job {
}
export type Image = ImageObject | string;
export type Services = (string | ServicesObject)[];
+export type GitlabInclude =
+ | GitlabIncludeLocal
+ | GitlabIncludeProject
+ | GitlabIncludeRemote
+ | GitlabIncludeTemplate
+ | string;
--
GitLab