diff --git a/lib/manager/gitlabci/__fixtures__/gitlab-ci.3.yaml b/lib/manager/gitlabci/__fixtures__/gitlab-ci.3.yaml
index b977704c90999d8b8b5377b59f66924d7846785b..afd94b368ebad08d8e7c4092b87553127b2c7acb 100644
--- a/lib/manager/gitlabci/__fixtures__/gitlab-ci.3.yaml
+++ b/lib/manager/gitlabci/__fixtures__/gitlab-ci.3.yaml
@@ -10,4 +10,11 @@ services:
include:
- local: 'lib/manager/gitlabci/__fixtures__/include.yml'
+ - local: 'lib/manager/gitlabci/__fixtures__/include.yml' # Loop detection
- local: 'lib/manager/gitlabci/__fixtures__/include.1.yml'
+ - project: 'my-group/my-project'
+ ref: master
+ file: '/templates/.gitlab-ci-template.yml'
+
+script:
+ - !reference [.setup, script]
diff --git a/lib/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml b/lib/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..72ee9af06eced322ac0e5f8ec1862eccf243559e
--- /dev/null
+++ b/lib/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml
@@ -0,0 +1,7 @@
+include:
+ - local: 'lib/manager/gitlabci/__fixtures__/include.yml'
+
+test:
+ script:
+ - !abc [.setup, script]
+ - echo running my own command
diff --git a/lib/manager/gitlabci/__fixtures__/include.1.yml b/lib/manager/gitlabci/__fixtures__/include.1.yml
index 76e4de4fa0e75499989852977a65a90ebca3c6eb..c7fc6fbba010baf97379e53c8fd6002c39ad7acc 100644
--- a/lib/manager/gitlabci/__fixtures__/include.1.yml
+++ b/lib/manager/gitlabci/__fixtures__/include.1.yml
@@ -1,3 +1,6 @@
+# not existing
+include: 'lib/manager/gitlabci/__fixtures__/include.2.yml'
+
test:
stage: test
image: node:12
diff --git a/lib/manager/gitlabci/__fixtures__/include.yml b/lib/manager/gitlabci/__fixtures__/include.yml
index a4c0bb2233106f9a81ebd2c6d5a432dec4d5c184..e2403bd2afc1c9f3778ad6319bb8639e6ea323a7 100644
--- a/lib/manager/gitlabci/__fixtures__/include.yml
+++ b/lib/manager/gitlabci/__fixtures__/include.yml
@@ -1,5 +1,10 @@
+
+# Loop detection
+include: 'lib/manager/gitlabci/__fixtures__/include.yml'
+
test:
stage: test
image: alpine:3.11
script:
- echo test
+ - !reference [.setup, script]
diff --git a/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap b/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap
index 2e5af6d09286a3efe127424c3c550e0799820a59..0cb54034e6210a2d3daa7bb30a8c287a2a479bf6 100644
--- a/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap
+++ b/lib/manager/gitlabci/__snapshots__/extract.spec.ts.snap
@@ -148,30 +148,30 @@ Array [
"deps": Array [
Object {
"autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+ "commitMessageTopic": "Node.js",
"currentDigest": undefined,
- "currentValue": "3.11",
+ "currentValue": "12",
"datasource": "docker",
- "depName": "alpine",
+ "depName": "node",
"depType": "image",
- "replaceString": "alpine:3.11",
+ "replaceString": "node:12",
},
],
- "packageFile": "lib/manager/gitlabci/__fixtures__/include.yml",
+ "packageFile": "lib/manager/gitlabci/__fixtures__/include.1.yml",
},
Object {
"deps": Array [
Object {
"autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
- "commitMessageTopic": "Node.js",
"currentDigest": undefined,
- "currentValue": "12",
+ "currentValue": "3.11",
"datasource": "docker",
- "depName": "node",
+ "depName": "alpine",
"depType": "image",
- "replaceString": "node:12",
+ "replaceString": "alpine:3.11",
},
],
- "packageFile": "lib/manager/gitlabci/__fixtures__/include.1.yml",
+ "packageFile": "lib/manager/gitlabci/__fixtures__/include.yml",
},
]
`;
diff --git a/lib/manager/gitlabci/extract.spec.ts b/lib/manager/gitlabci/extract.spec.ts
index de512c94ef9d37580f1974ce7c18ee6523ea2561..3ba33e008184246129a134fbcd5e271d602a4561 100644
--- a/lib/manager/gitlabci/extract.spec.ts
+++ b/lib/manager/gitlabci/extract.spec.ts
@@ -1,4 +1,4 @@
-import { getName } from '../../../test/util';
+import { getName, logger } from '../../../test/util';
import type { PackageDependency } from '../types';
import { extractAllPackageFiles } from './extract';
@@ -11,6 +11,7 @@ describe(getName(__filename), () => {
])
).toBeNull();
});
+
it('extracts multiple included image lines', async () => {
const res = await extractAllPackageFiles({}, [
'lib/manager/gitlabci/__fixtures__/gitlab-ci.3.yaml',
@@ -26,6 +27,7 @@ describe(getName(__filename), () => {
});
expect(deps).toHaveLength(5);
});
+
it('extracts multiple image lines', async () => {
const res = await extractAllPackageFiles({}, [
'lib/manager/gitlabci/__fixtures__/gitlab-ci.yaml',
@@ -43,6 +45,7 @@ describe(getName(__filename), () => {
expect(deps.some((dep) => dep.currentValue.includes("'"))).toBe(false);
});
+
it('extracts multiple image lines with comments', async () => {
const res = await extractAllPackageFiles({}, [
'lib/manager/gitlabci/__fixtures__/gitlab-ci.1.yaml',
@@ -58,5 +61,13 @@ describe(getName(__filename), () => {
});
expect(deps).toHaveLength(3);
});
+
+ it('catches errors', async () => {
+ const res = await extractAllPackageFiles({}, [
+ 'lib/manager/gitlabci/__fixtures__/gitlab-ci.4.yaml',
+ ]);
+ expect(res).toBeNull();
+ expect(logger.logger.warn).toHaveBeenCalled();
+ });
});
});
diff --git a/lib/manager/gitlabci/extract.ts b/lib/manager/gitlabci/extract.ts
index 72a42e9e5e488dcecb93592bbeb732ecdd981085..356d94176eeccab685ccd3f8c2fd584c6b8ec96f 100644
--- a/lib/manager/gitlabci/extract.ts
+++ b/lib/manager/gitlabci/extract.ts
@@ -4,6 +4,8 @@ import { logger } from '../../logger';
import { readLocalFile } from '../../util/fs';
import { getDep } from '../dockerfile/extract';
import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
+import type { GitlabPipeline } from './types';
+import { replaceReferenceTags } from './utils';
function skipCommentLines(
lines: string[],
@@ -84,28 +86,43 @@ export function extractPackageFile(content: string): PackageFile | null {
}
export async function extractAllPackageFiles(
- config: ExtractConfig,
+ _config: ExtractConfig,
packageFiles: string[]
): Promise<PackageFile[] | null> {
- const filesToExamine = new Set<string>(packageFiles);
+ const filesToExamine = [...packageFiles];
+ const seen = new Set<string>(packageFiles);
const results: PackageFile[] = [];
// extract all includes from the files
- while (filesToExamine.size > 0) {
- const file = filesToExamine.values().next().value;
- filesToExamine.delete(file);
+ while (filesToExamine.length > 0) {
+ const file = filesToExamine.pop();
const content = await readLocalFile(file, 'utf8');
- const doc = yaml.safeLoad(content, { json: true }) as any;
- if (doc?.include && is.array(doc.include)) {
+ let doc: GitlabPipeline;
+ try {
+ doc = yaml.safeLoad(replaceReferenceTags(content), {
+ json: true,
+ }) as GitlabPipeline;
+ } catch (err) {
+ logger.warn({ err, file }, 'Error extracting GitLab CI dependencies');
+ }
+
+ if (is.array(doc?.include)) {
for (const includeObj of doc.include) {
- if (includeObj.local) {
- const fileObj = (includeObj.local as string).replace(/^\//, '');
- if (!filesToExamine.has(fileObj)) {
- filesToExamine.add(fileObj);
+ if (is.string(includeObj.local)) {
+ const fileObj = includeObj.local.replace(/^\//, '');
+ if (!seen.has(fileObj)) {
+ seen.add(fileObj);
+ filesToExamine.push(fileObj);
}
}
}
+ } else if (is.string(doc?.include)) {
+ const fileObj = doc.include.replace(/^\//, '');
+ if (!seen.has(fileObj)) {
+ seen.add(fileObj);
+ filesToExamine.push(fileObj);
+ }
}
const result = extractPackageFile(content);
diff --git a/lib/manager/gitlabci/types.ts b/lib/manager/gitlabci/types.ts
new file mode 100644
index 0000000000000000000000000000000000000000..ace7be0584cd9352ddb3c88b65761506015e6e05
--- /dev/null
+++ b/lib/manager/gitlabci/types.ts
@@ -0,0 +1,7 @@
+export interface GitlabInclude {
+ local?: string;
+}
+
+export interface GitlabPipeline {
+ include?: GitlabInclude[] | string;
+}
diff --git a/lib/manager/gitlabci/utils.ts b/lib/manager/gitlabci/utils.ts
new file mode 100644
index 0000000000000000000000000000000000000000..1cb036baefd756e2dec9dde7e92378e81a38a683
--- /dev/null
+++ b/lib/manager/gitlabci/utils.ts
@@ -0,0 +1,12 @@
+const re = /!reference \[\.\w+?(?:, \w+?)\]/g;
+
+/**
+ * Replaces GitLab reference tags before parsing, because our yaml parser cannot process them anyway.
+ * @param content pipeline yaml
+ * @returns replaced pipeline content
+ * https://docs.gitlab.com/ee/ci/yaml/#reference-tags
+ */
+export function replaceReferenceTags(content: string): string {
+ const res = content.replace(re, '');
+ return res;
+}