From 177ffedb85f41791a87dda95a39a0f3311f94f98 Mon Sep 17 00:00:00 2001
From: Paul Gschwendtner <paulgschwendtner@gmail.com>
Date: Wed, 22 Feb 2023 12:24:02 +0100
Subject: [PATCH] feat(datasource/github-releases)!: digest computation use git
tag, not file digest (#20178)
The github-releases datasource has been copied into a new datasource called github-release-attachments.
The github-releases general datasource is updated to use the underlying Git tag of a GitHub release entry for digest computation.
Fixes #20160, Fixes #19552
BREAKING CHANGE: Regex Manager configurations relying on the github-release data-source with digests will have different digest semantics. The digest
will now always correspond to the underlying Git SHA of the release/version. The old behavior can be preserved by switching to the
github-release-attachments datasource.
---
lib/modules/datasource/api.ts | 5 +
.../digest.spec.ts | 30 +--
.../github-release-attachments/index.spec.ts | 154 +++++++++++
.../github-release-attachments/index.ts | 250 ++++++++++++++++++
.../test/index.ts | 2 +-
.../datasource/github-releases/index.spec.ts | 70 ++---
.../datasource/github-releases/index.ts | 194 +-------------
lib/modules/datasource/github-tags/index.ts | 39 +--
lib/util/github/tags.spec.ts | 78 ++++++
lib/util/github/tags.ts | 39 +++
10 files changed, 593 insertions(+), 268 deletions(-)
rename lib/modules/datasource/{github-releases => github-release-attachments}/digest.spec.ts (80%)
create mode 100644 lib/modules/datasource/github-release-attachments/index.spec.ts
create mode 100644 lib/modules/datasource/github-release-attachments/index.ts
rename lib/modules/datasource/{github-releases => github-release-attachments}/test/index.ts (97%)
create mode 100644 lib/util/github/tags.spec.ts
create mode 100644 lib/util/github/tags.ts
diff --git a/lib/modules/datasource/api.ts b/lib/modules/datasource/api.ts
index c61d696bfa..8bb4596025 100644
--- a/lib/modules/datasource/api.ts
+++ b/lib/modules/datasource/api.ts
@@ -19,6 +19,7 @@ import { GalaxyDatasource } from './galaxy';
import { GalaxyCollectionDatasource } from './galaxy-collection';
import { GitRefsDatasource } from './git-refs';
import { GitTagsDatasource } from './git-tags';
+import { GithubReleaseAttachmentsDatasource } from './github-release-attachments';
import { GithubReleasesDatasource } from './github-releases';
import { GithubTagsDatasource } from './github-tags';
import { GitlabPackagesDatasource } from './gitlab-packages';
@@ -76,6 +77,10 @@ api.set(GalaxyDatasource.id, new GalaxyDatasource());
api.set(GalaxyCollectionDatasource.id, new GalaxyCollectionDatasource());
api.set(GitRefsDatasource.id, new GitRefsDatasource());
api.set(GitTagsDatasource.id, new GitTagsDatasource());
+api.set(
+ GithubReleaseAttachmentsDatasource.id,
+ new GithubReleaseAttachmentsDatasource()
+);
api.set(GithubReleasesDatasource.id, new GithubReleasesDatasource());
api.set(GithubTagsDatasource.id, new GithubTagsDatasource());
api.set(GitlabPackagesDatasource.id, new GitlabPackagesDatasource());
diff --git a/lib/modules/datasource/github-releases/digest.spec.ts b/lib/modules/datasource/github-release-attachments/digest.spec.ts
similarity index 80%
rename from lib/modules/datasource/github-releases/digest.spec.ts
rename to lib/modules/datasource/github-release-attachments/digest.spec.ts
index 35fff7e252..19264bc096 100644
--- a/lib/modules/datasource/github-releases/digest.spec.ts
+++ b/lib/modules/datasource/github-release-attachments/digest.spec.ts
@@ -1,17 +1,17 @@
import hasha from 'hasha';
import * as httpMock from '../../../../test/http-mock';
import type { GithubDigestFile } from '../../../util/github/types';
-import { GitHubReleaseMocker } from './test';
+import { GitHubReleaseAttachmentMocker } from './test';
-import { GithubReleasesDatasource } from '.';
+import { GithubReleaseAttachmentsDatasource } from '.';
-describe('modules/datasource/github-releases/digest', () => {
+describe('modules/datasource/github-release-attachments/digest', () => {
const packageName = 'some/dep';
- const releaseMock = new GitHubReleaseMocker(
+ const releaseMock = new GitHubReleaseAttachmentMocker(
'https://api.github.com',
packageName
);
- const githubReleases = new GithubReleasesDatasource();
+ const githubReleaseAttachments = new GithubReleaseAttachmentsDatasource();
describe('findDigestAsset', () => {
it('finds SHASUMS.txt file containing digest', async () => {
@@ -21,7 +21,7 @@ describe('modules/datasource/github-releases/digest', () => {
'another-digest linux-arm64.tar.gz'
);
- const digestAsset = await githubReleases.findDigestAsset(
+ const digestAsset = await githubReleaseAttachments.findDigestAsset(
release,
'test-digest'
);
@@ -40,7 +40,7 @@ describe('modules/datasource/github-releases/digest', () => {
.get(`/repos/${packageName}/releases/download/v1.0.0/SHASUMS.txt`)
.reply(200, '');
- const digestAsset = await githubReleases.findDigestAsset(
+ const digestAsset = await githubReleaseAttachments.findDigestAsset(
release,
'test-digest'
);
@@ -57,7 +57,7 @@ describe('modules/datasource/github-releases/digest', () => {
});
const contentDigest = await hasha.async(content, { algorithm: 'sha256' });
- const digestAsset = await githubReleases.findDigestAsset(
+ const digestAsset = await githubReleaseAttachments.findDigestAsset(
release,
contentDigest
);
@@ -67,7 +67,7 @@ describe('modules/datasource/github-releases/digest', () => {
it('returns null when no assets available', async () => {
const release = releaseMock.release('v1.0.0');
- const digestAsset = await githubReleases.findDigestAsset(
+ const digestAsset = await githubReleaseAttachments.findDigestAsset(
release,
'test-digest'
);
@@ -89,7 +89,7 @@ describe('modules/datasource/github-releases/digest', () => {
'v1.0.1',
'updated-digest asset.zip'
);
- const digest = await githubReleases.mapDigestAssetToRelease(
+ const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
digestAsset,
release
);
@@ -106,7 +106,7 @@ describe('modules/datasource/github-releases/digest', () => {
'v1.0.1',
'updated-digest asset-1.0.1.zip'
);
- const digest = await githubReleases.mapDigestAssetToRelease(
+ const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
digestAssetWithVersion,
release
);
@@ -118,7 +118,7 @@ describe('modules/datasource/github-releases/digest', () => {
'v1.0.1',
'moot-digest asset.tar.gz'
);
- const digest = await githubReleases.mapDigestAssetToRelease(
+ const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
digestAsset,
release
);
@@ -127,7 +127,7 @@ describe('modules/datasource/github-releases/digest', () => {
it('returns null when digest file not found', async () => {
const release = releaseMock.release('v1.0.1');
- const digest = await githubReleases.mapDigestAssetToRelease(
+ const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
digestAsset,
release
);
@@ -151,7 +151,7 @@ describe('modules/datasource/github-releases/digest', () => {
algorithm: 'sha256',
});
- const digest = await githubReleases.mapDigestAssetToRelease(
+ const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
digestAsset,
release
);
@@ -160,7 +160,7 @@ describe('modules/datasource/github-releases/digest', () => {
it('returns null when not found', async () => {
const release = releaseMock.release('v1.0.1');
- const digest = await githubReleases.mapDigestAssetToRelease(
+ const digest = await githubReleaseAttachments.mapDigestAssetToRelease(
digestAsset,
release
);
diff --git a/lib/modules/datasource/github-release-attachments/index.spec.ts b/lib/modules/datasource/github-release-attachments/index.spec.ts
new file mode 100644
index 0000000000..576bf7a004
--- /dev/null
+++ b/lib/modules/datasource/github-release-attachments/index.spec.ts
@@ -0,0 +1,154 @@
+import { getDigest, getPkgReleases } from '..';
+import { mocked } from '../../../../test/util';
+import * as githubGraphql from '../../../util/github/graphql';
+import * as _hostRules from '../../../util/host-rules';
+import { GitHubReleaseAttachmentMocker } from './test';
+import { GithubReleaseAttachmentsDatasource } from '.';
+
+jest.mock('../../../util/host-rules');
+const hostRules = mocked(_hostRules);
+
+const githubApiHost = 'https://api.github.com';
+
+describe('modules/datasource/github-release-attachments/index', () => {
+ beforeEach(() => {
+ hostRules.hosts.mockReturnValue([]);
+ hostRules.find.mockReturnValue({
+ token: 'some-token',
+ });
+ });
+
+ describe('getReleases', () => {
+ it('returns releases', async () => {
+ jest.spyOn(githubGraphql, 'queryReleases').mockResolvedValueOnce([
+ {
+ id: 1,
+ url: 'https://example.com',
+ name: 'some/dep2',
+ description: 'some description',
+ version: 'a',
+ releaseTimestamp: '2020-03-09T13:00:00Z',
+ },
+ {
+ id: 2,
+ url: 'https://example.com',
+ name: 'some/dep2',
+ description: 'some description',
+ version: 'v',
+ releaseTimestamp: '2020-03-09T12:00:00Z',
+ },
+ {
+ id: 3,
+ url: 'https://example.com',
+ name: 'some/dep2',
+ description: 'some description',
+ version: '1.0.0',
+ releaseTimestamp: '2020-03-09T11:00:00Z',
+ },
+ {
+ id: 4,
+ url: 'https://example.com',
+ name: 'some/dep2',
+ description: 'some description',
+ version: 'v1.1.0',
+ releaseTimestamp: '2020-03-09T10:00:00Z',
+ },
+ {
+ id: 5,
+ url: 'https://example.com',
+ name: 'some/dep2',
+ description: 'some description',
+ version: '2.0.0',
+ releaseTimestamp: '2020-04-09T10:00:00Z',
+ isStable: false,
+ },
+ ]);
+
+ const res = await getPkgReleases({
+ datasource: GithubReleaseAttachmentsDatasource.id,
+ packageName: 'some/dep',
+ });
+
+ expect(res).toMatchObject({
+ registryUrl: 'https://github.com',
+ releases: [
+ { releaseTimestamp: '2020-03-09T11:00:00.000Z', version: '1.0.0' },
+ { version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00.000Z' },
+ {
+ version: '2.0.0',
+ releaseTimestamp: '2020-04-09T10:00:00.000Z',
+ isStable: false,
+ },
+ ],
+ sourceUrl: 'https://github.com/some/dep',
+ });
+ });
+ });
+
+ describe('getDigest', () => {
+ const packageName = 'some/dep';
+ const currentValue = 'v1.0.0';
+ const currentDigest = 'v1.0.0-digest';
+
+ const releaseMock = new GitHubReleaseAttachmentMocker(
+ githubApiHost,
+ packageName
+ );
+
+ it('requires currentDigest', async () => {
+ const digest = await getDigest(
+ { datasource: GithubReleaseAttachmentsDatasource.id, packageName },
+ currentValue
+ );
+ expect(digest).toBeNull();
+ });
+
+ it('defaults to currentDigest when currentVersion is missing', async () => {
+ const digest = await getDigest(
+ {
+ datasource: GithubReleaseAttachmentsDatasource.id,
+ packageName,
+ currentDigest,
+ },
+ currentValue
+ );
+ expect(digest).toEqual(currentDigest);
+ });
+
+ it('returns updated digest in new release', async () => {
+ releaseMock.withDigestFileAsset(
+ currentValue,
+ `${currentDigest} asset.zip`
+ );
+ const nextValue = 'v1.0.1';
+ const nextDigest = 'updated-digest';
+ releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`);
+ const digest = await getDigest(
+ {
+ datasource: GithubReleaseAttachmentsDatasource.id,
+ packageName,
+ currentValue,
+ currentDigest,
+ },
+ nextValue
+ );
+ expect(digest).toEqual(nextDigest);
+ });
+
+ // This is awkward, but I found returning `null` in this case to not produce an update
+ // I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision.
+ it('ignores failures verifying currentDigest', async () => {
+ releaseMock.release(currentValue);
+ const digest = await getDigest(
+ {
+ datasource: GithubReleaseAttachmentsDatasource.id,
+ packageName,
+ currentValue,
+ currentDigest,
+ },
+ currentValue
+ );
+ expect(digest).toEqual(currentDigest);
+ });
+ });
+});
diff --git a/lib/modules/datasource/github-release-attachments/index.ts b/lib/modules/datasource/github-release-attachments/index.ts
new file mode 100644
index 0000000000..02516713ee
--- /dev/null
+++ b/lib/modules/datasource/github-release-attachments/index.ts
@@ -0,0 +1,250 @@
+import is from '@sindresorhus/is';
+import hasha from 'hasha';
+import { logger } from '../../../logger';
+import { cache } from '../../../util/cache/package/decorator';
+import { queryReleases } from '../../../util/github/graphql';
+import type {
+ GithubDigestFile,
+ GithubRestAsset,
+ GithubRestRelease,
+} from '../../../util/github/types';
+import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
+import { GithubHttp } from '../../../util/http/github';
+import { newlineRegex, regEx } from '../../../util/regex';
+import { Datasource } from '../datasource';
+import type {
+ DigestConfig,
+ GetReleasesConfig,
+ Release,
+ ReleaseResult,
+} from '../types';
+
+export const cacheNamespace = 'datasource-github-releases';
+
+function inferHashAlg(digest: string): string {
+ switch (digest.length) {
+ case 64:
+ return 'sha256';
+ default:
+ case 96:
+ return 'sha512';
+ }
+}
+
+export class GithubReleaseAttachmentsDatasource extends Datasource {
+ static readonly id = 'github-release-attachments';
+
+ override readonly defaultRegistryUrls = ['https://github.com'];
+
+ override http: GithubHttp;
+
+ constructor() {
+ super(GithubReleaseAttachmentsDatasource.id);
+ this.http = new GithubHttp(GithubReleaseAttachmentsDatasource.id);
+ }
+
+ @cache({
+ ttlMinutes: 1440,
+ namespace: 'datasource-github-releases',
+ key: (release: GithubRestRelease, digest: string) =>
+ `${release.html_url}:${digest}`,
+ })
+ async findDigestFile(
+ release: GithubRestRelease,
+ digest: string
+ ): Promise<GithubDigestFile | null> {
+ const smallAssets = release.assets.filter(
+ (a: GithubRestAsset) => a.size < 5 * 1024
+ );
+ for (const asset of smallAssets) {
+ const res = await this.http.get(asset.browser_download_url);
+ for (const line of res.body.split(newlineRegex)) {
+ const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2);
+ if (lineDigest === digest) {
+ return {
+ assetName: asset.name,
+ digestedFileName: lineFilename,
+ currentVersion: release.tag_name,
+ currentDigest: lineDigest,
+ };
+ }
+ }
+ }
+ return null;
+ }
+
+ @cache({
+ ttlMinutes: 1440,
+ namespace: 'datasource-github-releases',
+ key: (asset: GithubRestAsset, algorithm: string) =>
+ `${asset.browser_download_url}:${algorithm}:assetDigest`,
+ })
+ async downloadAndDigest(
+ asset: GithubRestAsset,
+ algorithm: string
+ ): Promise<string> {
+ const res = this.http.stream(asset.browser_download_url);
+ const digest = await hasha.fromStream(res, { algorithm });
+ return digest;
+ }
+
+ async findAssetWithDigest(
+ release: GithubRestRelease,
+ digest: string
+ ): Promise<GithubDigestFile | null> {
+ const algorithm = inferHashAlg(digest);
+ const assetsBySize = release.assets.sort(
+ (a: GithubRestAsset, b: GithubRestAsset) => {
+ if (a.size < b.size) {
+ return -1;
+ }
+ if (a.size > b.size) {
+ return 1;
+ }
+ return 0;
+ }
+ );
+
+ for (const asset of assetsBySize) {
+ const assetDigest = await this.downloadAndDigest(asset, algorithm);
+ if (assetDigest === digest) {
+ return {
+ assetName: asset.name,
+ currentVersion: release.tag_name,
+ currentDigest: assetDigest,
+ };
+ }
+ }
+ return null;
+ }
+
+ /** Identify the asset associated with a known digest. */
+ async findDigestAsset(
+ release: GithubRestRelease,
+ digest: string
+ ): Promise<GithubDigestFile | null> {
+ const digestFile = await this.findDigestFile(release, digest);
+ if (digestFile) {
+ return digestFile;
+ }
+
+ const asset = await this.findAssetWithDigest(release, digest);
+ return asset;
+ }
+
+ /** Given a digest asset, find the equivalent digest in a different release. */
+ async mapDigestAssetToRelease(
+ digestAsset: GithubDigestFile,
+ release: GithubRestRelease
+ ): Promise<string | null> {
+ const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
+ const next = release.tag_name.replace(regEx(/^v/), '');
+ const releaseChecksumAssetName = digestAsset.assetName.replace(
+ current,
+ next
+ );
+ const releaseAsset = release.assets.find(
+ (a: GithubRestAsset) => a.name === releaseChecksumAssetName
+ );
+ if (!releaseAsset) {
+ return null;
+ }
+ if (digestAsset.digestedFileName) {
+ const releaseFilename = digestAsset.digestedFileName.replace(
+ current,
+ next
+ );
+ const res = await this.http.get(releaseAsset.browser_download_url);
+ for (const line of res.body.split(newlineRegex)) {
+ const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
+ if (lineFn === releaseFilename) {
+ return lineDigest;
+ }
+ }
+ } else {
+ const algorithm = inferHashAlg(digestAsset.currentDigest);
+ const newDigest = await this.downloadAndDigest(releaseAsset, algorithm);
+ return newDigest;
+ }
+ return null;
+ }
+
+ /**
+ * Attempts to resolve the digest for the specified package.
+ *
+ * The `newValue` supplied here should be a valid tag for the GitHub release.
+ * Requires `currentValue` and `currentDigest`.
+ *
+ * There may be many assets attached to the release. This function will:
+ * - Identify the asset pinned by `currentDigest` in the `currentValue` release
+ * - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`).
+ * - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512.
+ * - Map the hashed asset to `newValue` and return the updated digest as a string
+ */
+ override async getDigest(
+ {
+ packageName: repo,
+ currentValue,
+ currentDigest,
+ registryUrl,
+ }: DigestConfig,
+ newValue: string
+ ): Promise<string | null> {
+ logger.debug(
+ { repo, currentValue, currentDigest, registryUrl, newValue },
+ 'getDigest'
+ );
+ if (!currentDigest) {
+ return null;
+ }
+ if (!currentValue) {
+ return currentDigest;
+ }
+
+ const apiBaseUrl = getApiBaseUrl(registryUrl);
+ const { body: currentRelease } = await this.http.getJson<GithubRestRelease>(
+ `${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}`
+ );
+ const digestAsset = await this.findDigestAsset(
+ currentRelease,
+ currentDigest
+ );
+ let newDigest: string | null;
+ if (!digestAsset || newValue === currentValue) {
+ newDigest = currentDigest;
+ } else {
+ const { body: newRelease } = await this.http.getJson<GithubRestRelease>(
+ `${apiBaseUrl}repos/${repo}/releases/tags/${newValue}`
+ );
+ newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease);
+ }
+ return newDigest;
+ }
+
+ /**
+ * This function can be used to fetch releases with a customisable versioning
+ * (e.g. semver) and with releases.
+ *
+ * This function will:
+ * - Fetch all releases
+ * - Sanitize the versions if desired (e.g. strip out leading 'v')
+ * - Return a dependency object containing sourceUrl string and releases array
+ */
+ async getReleases(config: GetReleasesConfig): Promise<ReleaseResult> {
+ const releasesResult = await queryReleases(config, this.http);
+ const releases = releasesResult.map((item) => {
+ const { version, releaseTimestamp, isStable } = item;
+ const result: Release = {
+ version,
+ gitRef: version,
+ releaseTimestamp,
+ };
+ if (is.boolean(isStable)) {
+ result.isStable = isStable;
+ }
+ return result;
+ });
+ const sourceUrl = getSourceUrl(config.packageName, config.registryUrl);
+ return { sourceUrl, releases };
+ }
+}
diff --git a/lib/modules/datasource/github-releases/test/index.ts b/lib/modules/datasource/github-release-attachments/test/index.ts
similarity index 97%
rename from lib/modules/datasource/github-releases/test/index.ts
rename to lib/modules/datasource/github-release-attachments/test/index.ts
index e7dfcc82c9..84f6f3086c 100644
--- a/lib/modules/datasource/github-releases/test/index.ts
+++ b/lib/modules/datasource/github-release-attachments/test/index.ts
@@ -2,7 +2,7 @@ import * as httpMock from '../../../../../test/http-mock';
import { partial } from '../../../../../test/util';
import type { GithubRestRelease } from '../../../../util/github/types';
-export class GitHubReleaseMocker {
+export class GitHubReleaseAttachmentMocker {
constructor(
private readonly githubApiHost: string,
private readonly packageName: string
diff --git a/lib/modules/datasource/github-releases/index.spec.ts b/lib/modules/datasource/github-releases/index.spec.ts
index f90efc018f..42f485fa58 100644
--- a/lib/modules/datasource/github-releases/index.spec.ts
+++ b/lib/modules/datasource/github-releases/index.spec.ts
@@ -1,17 +1,14 @@
import { getDigest, getPkgReleases } from '..';
+import { mocked } from '../../../../test/util';
import * as githubGraphql from '../../../util/github/graphql';
import * as _hostRules from '../../../util/host-rules';
-import { GitHubReleaseMocker } from './test';
import { GithubReleasesDatasource } from '.';
jest.mock('../../../util/host-rules');
-const hostRules: any = _hostRules;
-
-const githubApiHost = 'https://api.github.com';
+const hostRules = mocked(_hostRules);
describe('modules/datasource/github-releases/index', () => {
beforeEach(() => {
- jest.resetAllMocks();
hostRules.hosts.mockReturnValue([]);
hostRules.find.mockReturnValue({
token: 'some-token',
@@ -88,38 +85,48 @@ describe('modules/datasource/github-releases/index', () => {
describe('getDigest', () => {
const packageName = 'some/dep';
const currentValue = 'v1.0.0';
- const currentDigest = 'v1.0.0-digest';
-
- const releaseMock = new GitHubReleaseMocker(githubApiHost, packageName);
+ const currentDigest = 'sha-of-v1';
+ const newValue = 'v15.0.0';
+ const newDigest = 'sha-of-v15';
- it('requires currentDigest', async () => {
- const digest = await getDigest(
- { datasource: GithubReleasesDatasource.id, packageName },
- currentValue
- );
- expect(digest).toBeNull();
+ beforeEach(() => {
+ jest.spyOn(githubGraphql, 'queryTags').mockResolvedValueOnce([
+ {
+ version: 'v1.0.0',
+ gitRef: 'v1.0.0',
+ releaseTimestamp: '2021-01-01',
+ hash: 'sha-of-v1',
+ },
+ {
+ version: 'v15.0.0',
+ gitRef: 'v15.0.0',
+ releaseTimestamp: '2022-10-01',
+ hash: 'sha-of-v15',
+ },
+ ]);
});
- it('defaults to currentDigest when currentVersion is missing', async () => {
+ it('should be independent of the current digest', async () => {
const digest = await getDigest(
{
datasource: GithubReleasesDatasource.id,
packageName,
- currentDigest,
+ currentValue,
},
- currentValue
+ newValue
);
- expect(digest).toEqual(currentDigest);
+ expect(digest).toBe(newDigest);
});
- it('returns updated digest in new release', async () => {
- releaseMock.withDigestFileAsset(
- currentValue,
- `${currentDigest} asset.zip`
+ it('should be independent of the current value', async () => {
+ const digest = await getDigest(
+ { datasource: GithubReleasesDatasource.id, packageName },
+ newValue
);
- const nextValue = 'v1.0.1';
- const nextDigest = 'updated-digest';
- releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`);
+ expect(digest).toBe(newDigest);
+ });
+
+ it('returns updated digest in new release', async () => {
const digest = await getDigest(
{
datasource: GithubReleasesDatasource.id,
@@ -127,15 +134,12 @@ describe('modules/datasource/github-releases/index', () => {
currentValue,
currentDigest,
},
- nextValue
+ newValue
);
- expect(digest).toEqual(nextDigest);
+ expect(digest).toEqual(newDigest);
});
- // This is awkward, but I found returning `null` in this case to not produce an update
- // I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision.
- it('ignores failures verifying currentDigest', async () => {
- releaseMock.release(currentValue);
+ it('returns null if the new value/tag does not exist', async () => {
const digest = await getDigest(
{
datasource: GithubReleasesDatasource.id,
@@ -143,9 +147,9 @@ describe('modules/datasource/github-releases/index', () => {
currentValue,
currentDigest,
},
- currentValue
+ 'unknown-tag'
);
- expect(digest).toEqual(currentDigest);
+ expect(digest).toBeNull();
});
});
});
diff --git a/lib/modules/datasource/github-releases/index.ts b/lib/modules/datasource/github-releases/index.ts
index 346fe27e0a..11714a8593 100644
--- a/lib/modules/datasource/github-releases/index.ts
+++ b/lib/modules/datasource/github-releases/index.ts
@@ -1,17 +1,9 @@
-// TODO: types (#7154)
import is from '@sindresorhus/is';
-import hasha from 'hasha';
import { logger } from '../../../logger';
-import { cache } from '../../../util/cache/package/decorator';
import { queryReleases } from '../../../util/github/graphql';
-import type {
- GithubDigestFile,
- GithubRestAsset,
- GithubRestRelease,
-} from '../../../util/github/types';
-import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
+import { findCommitOfTag } from '../../../util/github/tags';
+import { getSourceUrl } from '../../../util/github/url';
import { GithubHttp } from '../../../util/http/github';
-import { newlineRegex, regEx } from '../../../util/regex';
import { Datasource } from '../datasource';
import type {
DigestConfig,
@@ -22,16 +14,6 @@ import type {
export const cacheNamespace = 'datasource-github-releases';
-function inferHashAlg(digest: string): string {
- switch (digest.length) {
- case 64:
- return 'sha256';
- default:
- case 96:
- return 'sha512';
- }
-}
-
export class GithubReleasesDatasource extends Datasource {
static readonly id = 'github-releases';
@@ -44,145 +26,17 @@ export class GithubReleasesDatasource extends Datasource {
this.http = new GithubHttp(GithubReleasesDatasource.id);
}
- @cache({
- ttlMinutes: 1440,
- namespace: 'datasource-github-releases',
- key: (release: GithubRestRelease, digest: string) =>
- `${release.html_url}:${digest}`,
- })
- async findDigestFile(
- release: GithubRestRelease,
- digest: string
- ): Promise<GithubDigestFile | null> {
- const smallAssets = release.assets.filter(
- (a: GithubRestAsset) => a.size < 5 * 1024
- );
- for (const asset of smallAssets) {
- const res = await this.http.get(asset.browser_download_url);
- for (const line of res.body.split(newlineRegex)) {
- const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2);
- if (lineDigest === digest) {
- return {
- assetName: asset.name,
- digestedFileName: lineFilename,
- currentVersion: release.tag_name,
- currentDigest: lineDigest,
- };
- }
- }
- }
- return null;
- }
-
- @cache({
- ttlMinutes: 1440,
- namespace: 'datasource-github-releases',
- key: (asset: GithubRestAsset, algorithm: string) =>
- `${asset.browser_download_url}:${algorithm}:assetDigest`,
- })
- async downloadAndDigest(
- asset: GithubRestAsset,
- algorithm: string
- ): Promise<string> {
- const res = this.http.stream(asset.browser_download_url);
- const digest = await hasha.fromStream(res, { algorithm });
- return digest;
- }
-
- async findAssetWithDigest(
- release: GithubRestRelease,
- digest: string
- ): Promise<GithubDigestFile | null> {
- const algorithm = inferHashAlg(digest);
- const assetsBySize = release.assets.sort(
- (a: GithubRestAsset, b: GithubRestAsset) => {
- if (a.size < b.size) {
- return -1;
- }
- if (a.size > b.size) {
- return 1;
- }
- return 0;
- }
- );
-
- for (const asset of assetsBySize) {
- const assetDigest = await this.downloadAndDigest(asset, algorithm);
- if (assetDigest === digest) {
- return {
- assetName: asset.name,
- currentVersion: release.tag_name,
- currentDigest: assetDigest,
- };
- }
- }
- return null;
- }
-
- /** Identify the asset associated with a known digest. */
- async findDigestAsset(
- release: GithubRestRelease,
- digest: string
- ): Promise<GithubDigestFile | null> {
- const digestFile = await this.findDigestFile(release, digest);
- if (digestFile) {
- return digestFile;
- }
-
- const asset = await this.findAssetWithDigest(release, digest);
- return asset;
- }
-
- /** Given a digest asset, find the equivalent digest in a different release. */
- async mapDigestAssetToRelease(
- digestAsset: GithubDigestFile,
- release: GithubRestRelease
- ): Promise<string | null> {
- const current = digestAsset.currentVersion.replace(regEx(/^v/), '');
- const next = release.tag_name.replace(regEx(/^v/), '');
- const releaseChecksumAssetName = digestAsset.assetName.replace(
- current,
- next
- );
- const releaseAsset = release.assets.find(
- (a: GithubRestAsset) => a.name === releaseChecksumAssetName
- );
- if (!releaseAsset) {
- return null;
- }
- if (digestAsset.digestedFileName) {
- const releaseFilename = digestAsset.digestedFileName.replace(
- current,
- next
- );
- const res = await this.http.get(releaseAsset.browser_download_url);
- for (const line of res.body.split(newlineRegex)) {
- const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
- if (lineFn === releaseFilename) {
- return lineDigest;
- }
- }
- } else {
- const algorithm = inferHashAlg(digestAsset.currentDigest);
- const newDigest = await this.downloadAndDigest(releaseAsset, algorithm);
- return newDigest;
- }
- return null;
- }
-
/**
- * github.getDigest
+ * Attempts to resolve the digest for the specified package.
*
- * The `newValue` supplied here should be a valid tag for the GitHub release.
- * Requires `currentValue` and `currentDigest`.
+ * The `newValue` supplied here should be a valid tag for the GitHub release. The digest
+ * of a GitHub release will be the underlying SHA of the release tag.
*
- * There may be many assets attached to the release. This function will:
- * - Identify the asset pinned by `currentDigest` in the `currentValue` release
- * - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`).
- * - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512.
- * - Map the hashed asset to `newValue` and return the updated digest as a string
+ * Some managers like Bazel will deal with individual artifacts from releases and handle
+ * the artifact checksum computation separately. This data-source does not know about
+ * specific artifacts being used, as that could vary per manager
*/
- override async getDigest(
+ override getDigest(
{
packageName: repo,
currentValue,
@@ -195,37 +49,13 @@ export class GithubReleasesDatasource extends Datasource {
{ repo, currentValue, currentDigest, registryUrl, newValue },
'getDigest'
);
- if (!currentDigest) {
- return null;
- }
- if (!currentValue) {
- return currentDigest;
- }
- const apiBaseUrl = getApiBaseUrl(registryUrl);
- const { body: currentRelease } = await this.http.getJson<GithubRestRelease>(
- `${apiBaseUrl}repos/${repo}/releases/tags/${currentValue}`
- );
- const digestAsset = await this.findDigestAsset(
- currentRelease,
- currentDigest
- );
- let newDigest: string | null;
- if (!digestAsset || newValue === currentValue) {
- newDigest = currentDigest;
- } else {
- const { body: newRelease } = await this.http.getJson<GithubRestRelease>(
- `${apiBaseUrl}repos/${repo}/releases/tags/${newValue}`
- );
- newDigest = await this.mapDigestAssetToRelease(digestAsset, newRelease);
- }
- return newDigest;
+ return findCommitOfTag(registryUrl, repo, newValue, this.http);
}
/**
- * github.getReleases
- *
- * This function can be used to fetch releases with a customisable versioning (e.g. semver) and with releases.
+ * This function can be used to fetch releases with a customizable versioning
+ * (e.g. semver) and with releases.
*
* This function will:
* - Fetch all releases
diff --git a/lib/modules/datasource/github-tags/index.ts b/lib/modules/datasource/github-tags/index.ts
index 09d7281301..f5e32f4f95 100644
--- a/lib/modules/datasource/github-tags/index.ts
+++ b/lib/modules/datasource/github-tags/index.ts
@@ -2,6 +2,7 @@ import is from '@sindresorhus/is';
import { logger } from '../../../logger';
import { queryReleases, queryTags } from '../../../util/github/graphql';
import type { GithubReleaseItem } from '../../../util/github/graphql/types';
+import { findCommitOfTag } from '../../../util/github/tags';
import { getApiBaseUrl, getSourceUrl } from '../../../util/github/url';
import { GithubHttp } from '../../../util/http/github';
import { Datasource } from '../datasource';
@@ -24,42 +25,6 @@ export class GithubTagsDatasource extends Datasource {
this.http = new GithubHttp(GithubTagsDatasource.id);
}
- async getTagCommit(
- registryUrl: string | undefined,
- packageName: string,
- tag: string
- ): Promise<string | null> {
- logger.trace(`github-tags.getTagCommit(${packageName}, ${tag})`);
- try {
- const tags = await queryTags({ packageName, registryUrl }, this.http);
- // istanbul ignore if
- if (!tags.length) {
- logger.debug(
- `github-tags.getTagCommit(): No tags found for ${packageName}`
- );
- }
- const tagItem = tags.find(({ version }) => version === tag);
- if (tagItem) {
- if (tagItem.hash) {
- return tagItem.hash;
- }
- logger.debug(
- `github-tags.getTagCommit(): Tag ${tag} has no hash for ${packageName}`
- );
- } else {
- logger.debug(
- `github-tags.getTagCommit(): Tag ${tag} not found for ${packageName}`
- );
- }
- } catch (err) {
- logger.debug(
- { githubRepo: packageName, err },
- 'Error getting tag commit from GitHub repo'
- );
- }
- return null;
- }
-
async getCommit(
registryUrl: string | undefined,
githubRepo: string
@@ -91,7 +56,7 @@ export class GithubTagsDatasource extends Datasource {
newValue?: string
): Promise<string | null> {
return newValue
- ? this.getTagCommit(registryUrl, repo!, newValue)
+ ? findCommitOfTag(registryUrl, repo!, newValue, this.http)
: this.getCommit(registryUrl, repo!);
}
diff --git a/lib/util/github/tags.spec.ts b/lib/util/github/tags.spec.ts
new file mode 100644
index 0000000000..9747b8acf1
--- /dev/null
+++ b/lib/util/github/tags.spec.ts
@@ -0,0 +1,78 @@
+import { GithubHttp } from '../http/github';
+import * as githubGraphql from './graphql';
+import { findCommitOfTag } from './tags';
+
+describe('util/github/tags', () => {
+ describe('findCommitOfTag', () => {
+ const http = new GithubHttp();
+ const queryTagsSpy = jest.spyOn(githubGraphql, 'queryTags');
+
+ it('should be able to find the hash of a Git tag', async () => {
+ queryTagsSpy.mockResolvedValueOnce([
+ {
+ version: 'v1.0.0',
+ gitRef: 'v1.0.0',
+ releaseTimestamp: '2021-01-01',
+ hash: '123',
+ },
+ {
+ version: 'v2.0.0',
+ gitRef: 'v2.0.0',
+ releaseTimestamp: '2022-01-01',
+ hash: 'abc',
+ },
+ ]);
+
+ const commit = await findCommitOfTag(
+ undefined,
+ 'some-org/repo',
+ 'v2.0.0',
+ http
+ );
+ expect(commit).toBe('abc');
+ });
+
+ it('should support passing a custom registry URL', async () => {
+ queryTagsSpy.mockResolvedValueOnce([]);
+
+ const commit = await findCommitOfTag(
+ 'https://my-enterprise-github.dev',
+ 'some-org/repo',
+ 'v2.0.0',
+ http
+ );
+ expect(commit).toBeNull();
+ expect(githubGraphql.queryTags).toHaveBeenCalledWith(
+ {
+ packageName: 'some-org/repo',
+ registryUrl: 'https://my-enterprise-github.dev',
+ },
+ http
+ );
+ });
+
+ it('should return `null` if the tag does not exist', async () => {
+ queryTagsSpy.mockResolvedValueOnce([]);
+
+ const commit = await findCommitOfTag(
+ undefined,
+ 'some-org/repo',
+ 'v2.0.0',
+ http
+ );
+ expect(commit).toBeNull();
+ });
+
+ it('should gracefully return `null` if tags cannot be queried', async () => {
+ queryTagsSpy.mockRejectedValue(new Error('some error'));
+
+ const commit = await findCommitOfTag(
+ undefined,
+ 'some-org/repo',
+ 'v2.0.0',
+ http
+ );
+ expect(commit).toBeNull();
+ });
+ });
+});
diff --git a/lib/util/github/tags.ts b/lib/util/github/tags.ts
new file mode 100644
index 0000000000..51101958af
--- /dev/null
+++ b/lib/util/github/tags.ts
@@ -0,0 +1,39 @@
+import { logger } from '../../logger';
+import type { GithubHttp } from '../http/github';
+import { queryTags } from './graphql';
+
+export async function findCommitOfTag(
+ registryUrl: string | undefined,
+ packageName: string,
+ tag: string,
+ http: GithubHttp
+): Promise<string | null> {
+ logger.trace(`github/tags.findCommitOfTag(${packageName}, ${tag})`);
+ try {
+ const tags = await queryTags({ packageName, registryUrl }, http);
+ if (!tags.length) {
+ logger.debug(
+ `github/tags.findCommitOfTag(): No tags found for ${packageName}`
+ );
+ }
+ const tagItem = tags.find(({ version }) => version === tag);
+ if (tagItem) {
+ if (tagItem.hash) {
+ return tagItem.hash;
+ }
+ logger.debug(
+ `github/tags.findCommitOfTag: Tag ${tag} has no hash for ${packageName}`
+ );
+ } else {
+ logger.debug(
+ `github/tags.findCommitOfTag: Tag ${tag} not found for ${packageName}`
+ );
+ }
+ } catch (err) {
+ logger.debug(
+ { githubRepo: packageName, err },
+ 'Error getting tag commit from GitHub repo'
+ );
+ }
+ return null;
+}
--
GitLab