From 34b5401812986226a236f67ecfd7b52ce63597d9 Mon Sep 17 00:00:00 2001
From: Michael Kriese <michael.kriese@visualon.de>
Date: Wed, 29 Mar 2023 12:41:05 +0200
Subject: [PATCH] feat(manager/bitbucket-pipelines): support docker image
object (#21102)
---
lib/modules/datasource/types.ts | 3 -
.../__fixtures__/bitbucket-pipelines.yaml | 44 +++++++++
.../bitbucket-pipelines/extract.spec.ts | 92 ++++++++++++-------
.../manager/bitbucket-pipelines/extract.ts | 72 ++++++++-------
.../manager/bitbucket-pipelines/index.ts | 4 +
.../manager/bitbucket-pipelines/util.ts | 66 +++++++++++++
lib/types/base.ts | 3 +
tools/docs/manager.ts | 12 ++-
8 files changed, 227 insertions(+), 69 deletions(-)
create mode 100644 lib/modules/manager/bitbucket-pipelines/util.ts
diff --git a/lib/modules/datasource/types.ts b/lib/modules/datasource/types.ts
index 095d9f2ef1..f8023d3d92 100644
--- a/lib/modules/datasource/types.ts
+++ b/lib/modules/datasource/types.ts
@@ -103,7 +103,4 @@ export interface DatasourceApi extends ModuleApi {
* false: caching is not performed, or performed within the datasource implementation
*/
caching?: boolean | undefined;
-
- /** optional URLs to add to docs as references */
- urls?: string[];
}
diff --git a/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml b/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml
index c56064f849..d50138be94 100644
--- a/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml
+++ b/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml
@@ -1,11 +1,55 @@
image: node:10.15.1
+definitions:
+ steps:
+ - step: &build-test
+ name: Build and test
+ image:
+ # comment
+ name: node:18.15.0
+ script:
+ - mvn package
+ artifacts:
+ - target/**
+
+ - step: &build-test1
+ image:
+ username: xxxx
+ name: node:18.15.1
+
+ - step: &build-test2
+ image:
+ username: xxx
+ password: xxx
+
+ name: node:18.15.2
+
+ - step:
+ image:
+ test:
+ name: malformed
+
+ - step:
+ image:
+ username: xxx
+ test:
+ name: malformed
+
+ - step:
+ image:
+ username: xxx
+ password: xxx
+ test:
+ name: malformed
+
+
pipelines:
default:
- step:
name: Build and Test
image: node:10.15.2
script:
+ - step: *build-test
- pipe: docker://jfrogecosystem/jfrog-setup-cli:2.0.2
- npm install
- npm test
diff --git a/lib/modules/manager/bitbucket-pipelines/extract.spec.ts b/lib/modules/manager/bitbucket-pipelines/extract.spec.ts
index cec1fdd230..e215779525 100644
--- a/lib/modules/manager/bitbucket-pipelines/extract.spec.ts
+++ b/lib/modules/manager/bitbucket-pipelines/extract.spec.ts
@@ -4,49 +4,77 @@ import { extractPackageFile } from '.';
describe('modules/manager/bitbucket-pipelines/extract', () => {
describe('extractPackageFile()', () => {
it('returns null for empty', () => {
- expect(extractPackageFile('nothing here')).toBeNull();
+ expect(
+ extractPackageFile('nothing here', 'bitbucket-pipelines.yaml')
+ ).toBeNull();
+ });
+
+ it('returns null for malformed', () => {
+ expect(
+ extractPackageFile(
+ 'image:\n username: ccc',
+ 'bitbucket-pipelines.yaml'
+ )
+ ).toBeNull();
});
it('extracts dependencies', () => {
- const res = extractPackageFile(Fixtures.get('bitbucket-pipelines.yaml'));
- expect(res?.deps).toMatchInlineSnapshot(`
- [
+ const res = extractPackageFile(
+ Fixtures.get('bitbucket-pipelines.yaml'),
+ 'bitbucket-pipelines.yaml'
+ );
+ expect(res).toMatchObject({
+ deps: [
+ {
+ currentDigest: undefined,
+ currentValue: '10.15.1',
+ datasource: 'docker',
+ depName: 'node',
+ depType: 'docker',
+ },
+ {
+ currentDigest: undefined,
+ currentValue: '18.15.0',
+ datasource: 'docker',
+ depName: 'node',
+ depType: 'docker',
+ },
+ {
+ currentDigest: undefined,
+ currentValue: '18.15.1',
+ datasource: 'docker',
+ depName: 'node',
+ depType: 'docker',
+ },
{
- "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
- "currentDigest": undefined,
- "currentValue": "10.15.1",
- "datasource": "docker",
- "depName": "node",
- "depType": "docker",
- "replaceString": "node:10.15.1",
+ currentDigest: undefined,
+ currentValue: '18.15.2',
+ datasource: 'docker',
+ depName: 'node',
+ depType: 'docker',
},
{
- "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
- "currentDigest": undefined,
- "currentValue": "10.15.2",
- "datasource": "docker",
- "depName": "node",
- "depType": "docker",
- "replaceString": "node:10.15.2",
+ currentDigest: undefined,
+ currentValue: '10.15.2',
+ datasource: 'docker',
+ depName: 'node',
+ depType: 'docker',
},
{
- "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
- "currentDigest": undefined,
- "currentValue": "2.0.2",
- "datasource": "docker",
- "depName": "jfrogecosystem/jfrog-setup-cli",
- "depType": "docker",
- "replaceString": "jfrogecosystem/jfrog-setup-cli:2.0.2",
+ currentDigest: undefined,
+ currentValue: '2.0.2',
+ datasource: 'docker',
+ depName: 'jfrogecosystem/jfrog-setup-cli',
+ depType: 'docker',
},
{
- "currentValue": "0.2.1",
- "datasource": "bitbucket-tags",
- "depName": "atlassian/aws-s3-deploy",
- "depType": "bitbucket-tags",
+ currentValue: '0.2.1',
+ datasource: 'bitbucket-tags',
+ depName: 'atlassian/aws-s3-deploy',
+ depType: 'bitbucket-tags',
},
- ]
- `);
- expect(res?.deps).toHaveLength(4);
+ ],
+ });
});
});
});
diff --git a/lib/modules/manager/bitbucket-pipelines/extract.ts b/lib/modules/manager/bitbucket-pipelines/extract.ts
index f033e89c2a..4348eee72a 100644
--- a/lib/modules/manager/bitbucket-pipelines/extract.ts
+++ b/lib/modules/manager/bitbucket-pipelines/extract.ts
@@ -1,18 +1,44 @@
import { logger } from '../../../logger';
-import { newlineRegex, regEx } from '../../../util/regex';
-import { BitbucketTagsDatasource } from '../../datasource/bitbucket-tags';
-import { getDep } from '../dockerfile/extract';
+import { newlineRegex } from '../../../util/regex';
import type { PackageDependency, PackageFileContent } from '../types';
+import {
+ addDepAsBitbucketTag,
+ addDepAsDockerImage,
+ addDepFromObject,
+ dockerImageObjectRegex,
+ dockerImageRegex,
+ pipeRegex,
+} from './util';
-const pipeRegex = regEx(`^\\s*-\\s?pipe:\\s*'?"?([^\\s'"]+)'?"?\\s*$`);
-const dockerImageRegex = regEx(`^\\s*-?\\s?image:\\s*'?"?([^\\s'"]+)'?"?\\s*$`);
-
-export function extractPackageFile(content: string): PackageFileContent | null {
+export function extractPackageFile(
+ content: string,
+ filename: string
+): PackageFileContent | null {
const deps: PackageDependency[] = [];
try {
- const lines = content.split(newlineRegex);
- for (const line of lines) {
+ const lines = content
+ .replaceAll(/^\s*\r?\n/gm, '') // replace empty lines
+ .replaceAll(/^\s*#.*\r?\n/gm, '') // replace comment lines
+ .split(newlineRegex);
+ const len = lines.length;
+ for (let lineIdx = 0; lineIdx < len; lineIdx++) {
+ const line = lines[lineIdx];
+
+ const dockerImageObjectGroups = dockerImageObjectRegex.exec(line)?.groups;
+ if (dockerImageObjectGroups) {
+ // image object
+ // https://support.atlassian.com/bitbucket-cloud/docs/docker-image-options/
+ lineIdx = addDepFromObject(
+ deps,
+ lines,
+ lineIdx,
+ len,
+ dockerImageObjectGroups.spaces
+ );
+ continue;
+ }
+
const pipeMatch = pipeRegex.exec(line);
if (pipeMatch) {
const pipe = pipeMatch[1];
@@ -23,6 +49,7 @@ export function extractPackageFile(content: string): PackageFileContent | null {
} else {
addDepAsBitbucketTag(deps, pipe);
}
+ continue;
}
const dockerImageMatch = dockerImageRegex.exec(line);
@@ -32,32 +59,13 @@ export function extractPackageFile(content: string): PackageFileContent | null {
}
}
} catch (err) /* istanbul ignore next */ {
- logger.warn({ err }, 'Error extracting Bitbucket Pipes dependencies');
+ logger.debug(
+ { err, filename },
+ 'Error extracting Bitbucket Pipes dependencies'
+ );
}
if (!deps.length) {
return null;
}
return { deps };
}
-function addDepAsBitbucketTag(
- deps: PackageDependency<Record<string, any>>[],
- pipe: string
-): void {
- const [depName, currentValue] = pipe.split(':');
- const dep: PackageDependency = {
- depName,
- currentValue,
- datasource: BitbucketTagsDatasource.id,
- };
- dep.depType = 'bitbucket-tags';
- deps.push(dep);
-}
-
-function addDepAsDockerImage(
- deps: PackageDependency<Record<string, any>>[],
- currentDockerImage: string
-): void {
- const dep = getDep(currentDockerImage);
- dep.depType = 'docker';
- deps.push(dep);
-}
diff --git a/lib/modules/manager/bitbucket-pipelines/index.ts b/lib/modules/manager/bitbucket-pipelines/index.ts
index 930c4a868f..86d39fd3e9 100644
--- a/lib/modules/manager/bitbucket-pipelines/index.ts
+++ b/lib/modules/manager/bitbucket-pipelines/index.ts
@@ -8,3 +8,7 @@ export const defaultConfig = {
};
export const supportedDatasources = [DockerDatasource.id];
+
+export const urls = [
+ 'https://support.atlassian.com/bitbucket-cloud/docs/bitbucket-pipelines-configuration-reference/',
+];
diff --git a/lib/modules/manager/bitbucket-pipelines/util.ts b/lib/modules/manager/bitbucket-pipelines/util.ts
new file mode 100644
index 0000000000..0d04cf9c2d
--- /dev/null
+++ b/lib/modules/manager/bitbucket-pipelines/util.ts
@@ -0,0 +1,66 @@
+import { regEx } from '../../../util/regex';
+import { BitbucketTagsDatasource } from '../../datasource/bitbucket-tags';
+import { getDep } from '../dockerfile/extract';
+import type { PackageDependency } from '../types';
+
+export const pipeRegex = regEx(`^\\s*-\\s?pipe:\\s*'?"?([^\\s'"]+)'?"?\\s*$`);
+export const dockerImageRegex = regEx(
+ `^\\s*-?\\s?image:\\s*'?"?([^\\s'"]+)'?"?\\s*$`
+);
+export const dockerImageObjectRegex = regEx('^(?<spaces>\\s*)image:\\s*$');
+
+export function addDepAsBitbucketTag(
+ deps: PackageDependency[],
+ pipe: string
+): void {
+ const [depName, currentValue] = pipe.split(':');
+ const dep: PackageDependency = {
+ depName,
+ currentValue,
+ datasource: BitbucketTagsDatasource.id,
+ };
+ dep.depType = 'bitbucket-tags';
+ deps.push(dep);
+}
+
+export function addDepAsDockerImage(
+ deps: PackageDependency[],
+ currentDockerImage: string
+): void {
+ const dep = getDep(currentDockerImage);
+ dep.depType = 'docker';
+ deps.push(dep);
+}
+
+export function addDepFromObject(
+ deps: PackageDependency[],
+ lines: string[],
+ start: number,
+ len: number,
+ spaces: string
+): number {
+ const nameRegex = regEx(
+ `^${spaces}\\s+name:\\s*['"]?(?<image>[^\\s'"]+)['"]?\\s*$`
+ );
+ const indentRegex = regEx(`^${spaces}\\s+`);
+
+ for (let idx = start + 1; idx < len; idx++) {
+ const line = lines[idx];
+
+ if (!indentRegex.test(line)) {
+ // malformed
+ return idx;
+ }
+
+ const groups = nameRegex.exec(line)?.groups;
+ if (groups) {
+ const dep = getDep(groups.image);
+ dep.depType = 'docker';
+ deps.push(dep);
+ return idx;
+ }
+ }
+
+ // malformed
+ return start;
+}
diff --git a/lib/types/base.ts b/lib/types/base.ts
index daa6d1f03c..934faec60c 100644
--- a/lib/types/base.ts
+++ b/lib/types/base.ts
@@ -3,6 +3,9 @@ import type { PackageJson } from 'type-fest';
export interface ModuleApi {
displayName?: string;
url?: string;
+
+ /** optional URLs to add to docs as references */
+ urls?: string[];
}
export type RenovatePackageJson = PackageJson & {
diff --git a/tools/docs/manager.ts b/tools/docs/manager.ts
index dddd4511f9..a3c0641526 100644
--- a/tools/docs/manager.ts
+++ b/tools/docs/manager.ts
@@ -2,7 +2,12 @@ import type { RenovateConfig } from '../../lib/config/types';
import { getManagers } from '../../lib/modules/manager';
import { readFile, updateFile } from '../utils';
import { OpenItems, generateFeatureAndBugMarkdown } from './github-query-items';
-import { getDisplayName, getNameWithUrl, replaceContent } from './utils';
+import {
+ formatUrls,
+ getDisplayName,
+ getNameWithUrl,
+ replaceContent,
+} from './utils';
function getTitle(manager: string, displayName: string): string {
if (manager === 'regex') {
@@ -26,7 +31,7 @@ export async function generateManagers(
const language = definition.language ?? 'other';
allLanguages[language] = allLanguages[language] || [];
allLanguages[language].push(manager);
- const { defaultConfig, supportedDatasources } = definition;
+ const { defaultConfig, supportedDatasources, urls } = definition;
const { fileMatch } = defaultConfig as RenovateConfig;
const displayName = getDisplayName(manager, definition);
let md = `---
@@ -70,6 +75,9 @@ sidebar_label: ${displayName}
.join(', ');
md += `This manager supports extracting the following datasources: ${escapedDatasources}.\n\n`;
+ md += '## References';
+ md += formatUrls(urls).replace('**References**:', '');
+
md += '## Default config\n\n';
md += '```json\n';
md += JSON.stringify(definition.defaultConfig, null, 2) + '\n';
--
GitLab