From 4c15038aaadaee1307da1aa27dc880ef79280ca5 Mon Sep 17 00:00:00 2001
From: Sebastian Poxhofer <secustor@users.noreply.github.com>
Date: Thu, 25 Jul 2024 19:51:00 +0200
Subject: [PATCH] feat(datasources): add bitrise datasource (#30138)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
lib/modules/datasource/api.ts | 2 +
lib/modules/datasource/bitrise/index.spec.ts | 235 +++++++++++++++++++
lib/modules/datasource/bitrise/index.ts | 138 +++++++++++
lib/modules/datasource/bitrise/readme.md | 26 ++
lib/modules/datasource/bitrise/schema.ts | 6 +
lib/util/cache/package/types.ts | 1 +
6 files changed, 408 insertions(+)
create mode 100644 lib/modules/datasource/bitrise/index.spec.ts
create mode 100644 lib/modules/datasource/bitrise/index.ts
create mode 100644 lib/modules/datasource/bitrise/readme.md
create mode 100644 lib/modules/datasource/bitrise/schema.ts
diff --git a/lib/modules/datasource/api.ts b/lib/modules/datasource/api.ts
index eff7da102e..e336c3eacb 100644
--- a/lib/modules/datasource/api.ts
+++ b/lib/modules/datasource/api.ts
@@ -5,6 +5,7 @@ import { AzureBicepResourceDatasource } from './azure-bicep-resource';
import { AzurePipelinesTasksDatasource } from './azure-pipelines-tasks';
import { BazelDatasource } from './bazel';
import { BitbucketTagsDatasource } from './bitbucket-tags';
+import { BitriseDatasource } from './bitrise';
import { CdnJsDatasource } from './cdnjs';
import { ClojureDatasource } from './clojure';
import { ConanDatasource } from './conan';
@@ -73,6 +74,7 @@ api.set(AzureBicepResourceDatasource.id, new AzureBicepResourceDatasource());
api.set(AzurePipelinesTasksDatasource.id, new AzurePipelinesTasksDatasource());
api.set(BazelDatasource.id, new BazelDatasource());
api.set(BitbucketTagsDatasource.id, new BitbucketTagsDatasource());
+api.set(BitriseDatasource.id, new BitriseDatasource());
api.set(CdnJsDatasource.id, new CdnJsDatasource());
api.set(ClojureDatasource.id, new ClojureDatasource());
api.set(ConanDatasource.id, new ConanDatasource());
diff --git a/lib/modules/datasource/bitrise/index.spec.ts b/lib/modules/datasource/bitrise/index.spec.ts
new file mode 100644
index 0000000000..54cef5d3ce
--- /dev/null
+++ b/lib/modules/datasource/bitrise/index.spec.ts
@@ -0,0 +1,235 @@
+import { codeBlock } from 'common-tags';
+import * as httpMock from '../../../../test/http-mock';
+import { toBase64 } from '../../../util/string';
+import { getPkgReleases } from '../index';
+import { BitriseDatasource } from './index';
+
+describe('modules/datasource/bitrise/index', () => {
+ describe('getReleases()', () => {
+ it('returns null for unsupported registryUrl', async () => {
+ await expect(
+ getPkgReleases({
+ datasource: BitriseDatasource.id,
+ packageName: 'script',
+ registryUrls: ['https://gitlab.com/bitrise-io/bitrise-steplib'],
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it('support GitHub Enterprise API URL', async () => {
+ httpMock
+ .scope(
+ 'https://github.mycompany.com/api/v3/repos/foo/bar/contents/steps',
+ )
+ .get('/script')
+ .reply(200, [
+ {
+ type: 'dir',
+ name: '1.0.0',
+ path: 'steps/script/1.0.0',
+ },
+ ])
+ .get('/script/1.0.0/step.yml')
+ .reply(200, {
+ type: 'file',
+ name: 'step.yml',
+ path: 'steps/script/1.0.0/step.yml',
+ encoding: 'base64',
+ content: toBase64(codeBlock`
+ published_at: 2024-03-19T13:54:48.081077+01:00
+ source_code_url: https://github.com/bitrise-steplib/bitrise-step-script
+ website: https://github.com/bitrise-steplib/bitrise-step-script
+ `),
+ });
+ await expect(
+ getPkgReleases({
+ datasource: BitriseDatasource.id,
+ packageName: 'script',
+ registryUrls: ['https://github.mycompany.com/foo/bar'],
+ }),
+ ).resolves.toEqual({
+ homepage: 'https://bitrise.io/integrations/steps/script',
+ registryUrl: 'https://github.mycompany.com/foo/bar',
+ releases: [
+ {
+ releaseTimestamp: '2024-03-19T12:54:48.081Z',
+ sourceUrl: 'https://github.com/bitrise-steplib/bitrise-step-script',
+ version: '1.0.0',
+ },
+ ],
+ });
+ });
+
+ it('returns version and filters out the asset folder', async () => {
+ httpMock
+ .scope(
+ 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps',
+ )
+ .get('/activate-build-cache-for-bazel')
+ .reply(200, [
+ {
+ type: 'dir',
+ name: '1.0.0',
+ path: 'steps/activate-build-cache-for-bazel/1.0.0',
+ },
+ {
+ type: 'dir',
+ name: '1.0.1',
+ path: 'steps/activate-build-cache-for-bazel/1.0.1',
+ },
+ {
+ type: 'dir',
+ name: 'assets',
+ path: 'steps/activate-build-cache-for-bazel/assets',
+ },
+ ])
+ .get('/activate-build-cache-for-bazel/1.0.0/step.yml')
+ .reply(200, {
+ type: 'file',
+ name: 'step.yml',
+ path: 'steps/activate-build-cache-for-bazel/1.0.0/step.yml',
+ encoding: 'base64',
+ content: toBase64(codeBlock`
+ published_at: 2024-03-19T13:54:48.081077+01:00
+ source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel
+ website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel
+ `),
+ })
+ .get('/activate-build-cache-for-bazel/1.0.1/step.yml')
+ .reply(200, {
+ type: 'file',
+ name: 'step.yml',
+ path: 'steps/activate-build-cache-for-bazel/1.0.1/step.yml',
+ encoding: 'base64',
+ content: toBase64(codeBlock`
+ published_at: "2024-07-03T08:53:25.668504731Z"
+ source_code_url: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel
+ website: https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel
+ `),
+ });
+
+ await expect(
+ getPkgReleases({
+ datasource: BitriseDatasource.id,
+ packageName: 'activate-build-cache-for-bazel',
+ }),
+ ).resolves.toEqual({
+ homepage:
+ 'https://bitrise.io/integrations/steps/activate-build-cache-for-bazel',
+ registryUrl: 'https://github.com/bitrise-io/bitrise-steplib.git',
+ releases: [
+ {
+ releaseTimestamp: '2024-03-19T12:54:48.081Z',
+ sourceUrl:
+ 'https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel',
+ version: '1.0.0',
+ },
+ {
+ releaseTimestamp: '2024-07-03T08:53:25.668Z',
+ sourceUrl:
+ 'https://github.com/bitrise-steplib/bitrise-step-activate-build-cache-for-bazel',
+ version: '1.0.1',
+ },
+ ],
+ });
+ });
+
+ it('returns null if there are no releases', async () => {
+ httpMock
+ .scope(
+ 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps',
+ )
+ .get('/activate-build-cache-for-bazel')
+ .reply(200, [
+ {
+ type: 'dir',
+ name: 'assets',
+ path: 'steps/activate-build-cache-for-bazel/assets',
+ },
+ ]);
+
+ await expect(
+ getPkgReleases({
+ datasource: BitriseDatasource.id,
+ packageName: 'activate-build-cache-for-bazel',
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it('returns null if the package has an unexpected format', async () => {
+ httpMock
+ .scope(
+ 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps',
+ )
+ .get('/activate-build-cache-for-bazel')
+ .reply(200, {
+ type: 'file',
+ name: 'assets',
+ path: 'steps/activate-build-cache-for-bazel/assets',
+ });
+
+ await expect(
+ getPkgReleases({
+ datasource: BitriseDatasource.id,
+ packageName: 'activate-build-cache-for-bazel',
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it('returns null if the file object has no content', async () => {
+ httpMock
+ .scope(
+ 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps',
+ )
+ .get('/script')
+ .reply(200, [
+ {
+ type: 'dir',
+ name: '1.0.0',
+ path: 'steps/script/1.0.0',
+ },
+ ])
+ .get('/script/1.0.0/step.yml')
+ .reply(200, {
+ type: 'file',
+ name: 'step.yml',
+ path: 'steps/script/1.0.0/step.yml',
+ });
+ await expect(
+ getPkgReleases({
+ datasource: BitriseDatasource.id,
+ packageName: 'script',
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it('returns null if the file object has an unexpected encoding', async () => {
+ httpMock
+ .scope(
+ 'https://api.github.com/repos/bitrise-io/bitrise-steplib/contents/steps',
+ )
+ .get('/script')
+ .reply(200, [
+ {
+ type: 'dir',
+ name: '1.0.0',
+ path: 'steps/script/1.0.0',
+ },
+ ])
+ .get('/script/1.0.0/step.yml')
+ .reply(200, {
+ type: 'file',
+ name: 'step.yml',
+ path: 'steps/script/1.0.0/step.yml',
+ encoding: 'none',
+ content: '',
+ });
+ await expect(
+ getPkgReleases({
+ datasource: BitriseDatasource.id,
+ packageName: 'script',
+ }),
+ ).resolves.toBeNull();
+ });
+ });
+});
diff --git a/lib/modules/datasource/bitrise/index.ts b/lib/modules/datasource/bitrise/index.ts
new file mode 100644
index 0000000000..dfdc2a1f2a
--- /dev/null
+++ b/lib/modules/datasource/bitrise/index.ts
@@ -0,0 +1,138 @@
+import is from '@sindresorhus/is';
+import { logger } from '../../../logger';
+import { cache } from '../../../util/cache/package/decorator';
+import { detectPlatform } from '../../../util/common';
+import { parseGitUrl } from '../../../util/git/url';
+import { GithubHttp } from '../../../util/http/github';
+import { fromBase64 } from '../../../util/string';
+import { joinUrlParts } from '../../../util/url';
+import { parseSingleYaml } from '../../../util/yaml';
+import { GithubContentResponse } from '../../platform/github/schema';
+import semver from '../../versioning/semver';
+import { Datasource } from '../datasource';
+import type { GetReleasesConfig, ReleaseResult } from '../types';
+import { BitriseStepFile } from './schema';
+
+export class BitriseDatasource extends Datasource {
+ static readonly id = 'bitrise';
+
+ override readonly http: GithubHttp;
+
+ constructor() {
+ super(BitriseDatasource.id);
+
+ this.http = new GithubHttp(this.id);
+ }
+
+ override readonly customRegistrySupport = true;
+
+ override readonly defaultRegistryUrls = [
+ 'https://github.com/bitrise-io/bitrise-steplib.git',
+ ];
+
+ override readonly releaseTimestampSupport = true;
+ override readonly releaseTimestampNote =
+ 'The release timestamp is determined from the `published_at` field in the results.';
+ override readonly sourceUrlSupport = 'release';
+ override readonly sourceUrlNote =
+ 'The source URL is determined from the `source_code_url` field of the release object in the results.';
+
+ @cache({
+ namespace: `datasource-${BitriseDatasource.id}`,
+ key: ({ packageName, registryUrl }: GetReleasesConfig) =>
+ `${registryUrl}/${packageName}`,
+ })
+ async getReleases({
+ packageName,
+ registryUrl,
+ }: GetReleasesConfig): Promise<ReleaseResult | null> {
+ // istanbul ignore if
+ if (!registryUrl) {
+ return null;
+ }
+
+ const parsedUrl = parseGitUrl(registryUrl);
+ if (detectPlatform(registryUrl) !== 'github') {
+ logger.once.warn(
+ `${parsedUrl.source} is not a supported Git hoster for this datasource`,
+ );
+ return null;
+ }
+
+ const result: ReleaseResult = {
+ releases: [],
+ };
+
+ const massagedPackageName = encodeURIComponent(packageName);
+ const baseApiURL =
+ parsedUrl.resource === 'github.com'
+ ? 'https://api.github.com'
+ : `https://${parsedUrl.resource}/api/v3`;
+ const packageUrl = joinUrlParts(
+ baseApiURL,
+ 'repos',
+ parsedUrl.full_name,
+ 'contents/steps',
+ massagedPackageName,
+ );
+
+ const { body: packageRaw } = await this.http.getJson(
+ packageUrl,
+ GithubContentResponse,
+ );
+
+ if (!is.array(packageRaw)) {
+ logger.warn(
+ { data: packageRaw, url: packageUrl },
+ 'Got unexpected response for Bitrise package location',
+ );
+ return null;
+ }
+
+ for (const versionDir of packageRaw.filter((element) =>
+ semver.isValid(element.name),
+ )) {
+ const stepUrl = joinUrlParts(packageUrl, versionDir.name, 'step.yml');
+ // TODO use getRawFile when ready #30155
+ const { body } = await this.http.getJson(stepUrl, GithubContentResponse);
+ if (!('content' in body)) {
+ logger.warn(
+ { data: body, url: stepUrl },
+ 'Got unexpected response for Bitrise step location',
+ );
+ return null;
+ }
+ if (body.encoding !== 'base64') {
+ logger.warn(
+ { data: body, url: stepUrl },
+ `Got unexpected encoding for Bitrise step location '${body.encoding}'`,
+ );
+ return null;
+ }
+
+ const content = fromBase64(body.content);
+ const { published_at, source_code_url } = parseSingleYaml(content, {
+ customSchema: BitriseStepFile,
+ });
+
+ const releaseTimestamp = is.string(published_at)
+ ? published_at
+ : published_at.toISOString();
+ result.releases.push({
+ version: versionDir.name,
+ releaseTimestamp,
+ sourceUrl: source_code_url,
+ });
+ }
+
+ // if we have no releases return null
+ if (!result.releases.length) {
+ return null;
+ }
+
+ return {
+ ...result,
+ homepage: `https://bitrise.io/integrations/steps/${packageName}`,
+ };
+ }
+}
diff --git a/lib/modules/datasource/bitrise/readme.md b/lib/modules/datasource/bitrise/readme.md
new file mode 100644
index 0000000000..e80b3fabaa
--- /dev/null
+++ b/lib/modules/datasource/bitrise/readme.md
@@ -0,0 +1,26 @@
+Renovate uses this datasource to fetch Bitrise steps from GitHub repositories.
+
+| Renovate field | What value to use? |
+| -------------- | --------------------------------------- |
+| `packageName` | Name of the Bitrise step |
+| `registryUrl` | GitHub HTTP Git URL, as used by Bitrise |
+
+For example, in the YAML snippet below:
+
+- `packageName` is `script`
+- `registryUrl` is `https://github.com/bitrise-io/bitrise-steplib.git`
+
+```yaml
+format_version: 11
+default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git
+project_type: android
+app:
+ envs:
+ - MY_NAME: My Name
+workflows:
+ test:
+ steps:
+ - script@1.1.5:
+ inputs:
+ - content: echo "Hello ${MY_NAME}!"
+```
diff --git a/lib/modules/datasource/bitrise/schema.ts b/lib/modules/datasource/bitrise/schema.ts
new file mode 100644
index 0000000000..8b04b3cbe7
--- /dev/null
+++ b/lib/modules/datasource/bitrise/schema.ts
@@ -0,0 +1,6 @@
+import { z } from 'zod';
+
+export const BitriseStepFile = z.object({
+ published_at: z.date().or(z.string()),
+ source_code_url: z.string().optional(),
+});
diff --git a/lib/util/cache/package/types.ts b/lib/util/cache/package/types.ts
index e8a479bd1d..0187737243 100644
--- a/lib/util/cache/package/types.ts
+++ b/lib/util/cache/package/types.ts
@@ -33,6 +33,7 @@ export type PackageCacheNamespace =
| 'datasource-azure-pipelines-tasks'
| 'datasource-bazel'
| 'datasource-bitbucket-tags'
+ | 'datasource-bitrise'
| 'datasource-cdnjs-digest'
| 'datasource-cdnjs'
| 'datasource-conan-revisions'
--
GitLab