From 91e2743306dab834b88b98822f4e7159377a4b7a Mon Sep 17 00:00:00 2001 From: Pete Wagner <1559510+thepwagner@users.noreply.github.com> Date: Thu, 5 Aug 2021 11:05:22 -0400 Subject: [PATCH] feat(github-releases): getDigest() (#10947) --- jest.config.ts | 2 +- lib/datasource/github-releases/common.spec.ts | 42 +++++ lib/datasource/github-releases/common.ts | 29 ++++ lib/datasource/github-releases/digest.spec.ts | 143 ++++++++++++++++++ lib/datasource/github-releases/digest.ts | 141 +++++++++++++++++ lib/datasource/github-releases/index.spec.ts | 64 +++++++- lib/datasource/github-releases/index.ts | 101 ++++++++++--- lib/datasource/github-releases/test/index.ts | 49 ++++++ lib/datasource/github-releases/types.ts | 14 ++ lib/datasource/index.ts | 11 +- lib/datasource/types.ts | 2 + tsconfig.app.json | 1 + 12 files changed, 569 insertions(+), 30 deletions(-) create mode 100644 lib/datasource/github-releases/common.spec.ts create mode 100644 lib/datasource/github-releases/common.ts create mode 100644 lib/datasource/github-releases/digest.spec.ts create mode 100644 lib/datasource/github-releases/digest.ts create mode 100644 lib/datasource/github-releases/test/index.ts diff --git a/jest.config.ts b/jest.config.ts index bf8c08d5a1..b36ecdf882 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -10,7 +10,7 @@ const config: InitialOptionsTsJest = { collectCoverageFrom: [ 'lib/**/*.{js,ts}', '!lib/**/*.{d,spec}.ts', - '!lib/**/{__fixtures__,__mocks__,__testutil__}/**/*.{js,ts}', + '!lib/**/{__fixtures__,__mocks__,__testutil__,test}/**/*.{js,ts}', '!lib/**/types.ts', ], coverageReporters: ci diff --git a/lib/datasource/github-releases/common.spec.ts b/lib/datasource/github-releases/common.spec.ts new file mode 100644 index 0000000000..56081bfa64 --- /dev/null +++ b/lib/datasource/github-releases/common.spec.ts @@ -0,0 +1,42 @@ +import { getName } from '../../../test/util'; +import { getApiBaseUrl, getGithubRelease, getSourceUrlBase } from './common'; +import { GitHubReleaseMocker } from './test'; + +describe(getName(), () => { + describe('getSourceUrlBase', () => { + it('ensures trailing slash', () => { + const sourceUrl = getSourceUrlBase('https://gh.my-company.com'); + expect(sourceUrl).toBe('https://gh.my-company.com/'); + }); + it('defaults to github.com', () => { + const sourceUrl = getSourceUrlBase(null); + expect(sourceUrl).toBe('https://github.com/'); + }); + }); + + describe('getApiBaseUrl', () => { + it('maps to api.github.com', () => { + const apiUrl = getApiBaseUrl('https://github.com/'); + expect(apiUrl).toBe('https://api.github.com/'); + }); + + it('supports local github installations', () => { + const apiUrl = getApiBaseUrl('https://gh.my-company.com/'); + expect(apiUrl).toBe('https://gh.my-company.com/api/v3/'); + }); + }); + + describe('getGithubRelease', () => { + const apiUrl = 'https://github.com/'; + const lookupName = 'someDep'; + const releaseMock = new GitHubReleaseMocker(apiUrl, lookupName); + + it('returns release', async () => { + const version = 'v1.0.0'; + releaseMock.release(version); + + const release = await getGithubRelease(apiUrl, lookupName, version); + expect(release.tag_name).toBe(version); + }); + }); +}); diff --git a/lib/datasource/github-releases/common.ts b/lib/datasource/github-releases/common.ts new file mode 100644 index 0000000000..58bb4a56cd --- /dev/null +++ b/lib/datasource/github-releases/common.ts @@ -0,0 +1,29 @@ +import { GithubHttp } from '../../util/http/github'; +import { ensureTrailingSlash } from '../../util/url'; +import type { GithubRelease } from './types'; + +const defaultSourceUrlBase = 'https://github.com/'; + +export const cacheNamespace = 'datasource-github-releases'; +export const http = new GithubHttp(); + +export function getSourceUrlBase(registryUrl: string): string { + // default to GitHub.com if no GHE host is specified. + return ensureTrailingSlash(registryUrl ?? defaultSourceUrlBase); +} + +export function getApiBaseUrl(sourceUrlBase: string): string { + return sourceUrlBase === defaultSourceUrlBase + ? `https://api.github.com/` + : `${sourceUrlBase}api/v3/`; +} + +export async function getGithubRelease( + apiBaseUrl: string, + repo: string, + version: string +): Promise<GithubRelease> { + const url = `${apiBaseUrl}repos/${repo}/releases/tags/${version}`; + const res = await http.getJson<GithubRelease>(url); + return res.body; +} diff --git a/lib/datasource/github-releases/digest.spec.ts b/lib/datasource/github-releases/digest.spec.ts new file mode 100644 index 0000000000..af82b8ff10 --- /dev/null +++ b/lib/datasource/github-releases/digest.spec.ts @@ -0,0 +1,143 @@ +import hasha from 'hasha'; +import * as httpMock from '../../../test/http-mock'; +import { getName } from '../../../test/util'; +import { findDigestAsset, mapDigestAssetToRelease } from './digest'; +import { GitHubReleaseMocker } from './test'; +import { DigestAsset } from './types'; + +describe(getName(), () => { + const lookupName = 'some/dep'; + const releaseMock = new GitHubReleaseMocker( + 'https://api.github.com', + lookupName + ); + + describe('findDigestAsset', () => { + it('finds SHASUMS.txt file containing digest', async () => { + const release = releaseMock.withDigestFileAsset( + 'v1.0.0', + 'test-digest linux-amd64.tar.gz', + 'another-digest linux-arm64.tar.gz' + ); + + const digestAsset = await findDigestAsset(release, 'test-digest'); + expect(digestAsset.assetName).toBe('SHASUMS.txt'); + expect(digestAsset.digestedFileName).toBe('linux-amd64.tar.gz'); + }); + + it('returns null when not found in digest file asset', async () => { + const release = releaseMock.withDigestFileAsset( + 'v1.0.0', + 'another-digest linux-arm64.tar.gz' + ); + // Small assets like this digest file may be downloaded twice + httpMock + .scope('https://api.github.com') + .get(`/repos/${lookupName}/releases/download/v1.0.0/SHASUMS.txt`) + .reply(200, ''); + + const digestAsset = await findDigestAsset(release, 'test-digest'); + expect(digestAsset).toBeNull(); + }); + + it('finds asset by digest', async () => { + const content = '1'.repeat(10 * 1024); + const release = releaseMock.withAssets('v1.0.0', { + 'smaller.zip': '1'.repeat(9 * 1024), + 'same-size.zip': '2'.repeat(10 * 1024), + 'asset.zip': content, + 'smallest.zip': '1'.repeat(8 * 1024), + }); + const contentDigest = await hasha.async(content, { algorithm: 'sha256' }); + + const digestAsset = await findDigestAsset(release, contentDigest); + expect(digestAsset.assetName).toBe('asset.zip'); + expect(digestAsset.digestedFileName).toBeUndefined(); + }); + + it('returns null when no assets available', async () => { + const release = releaseMock.release('v1.0.0'); + const digestAsset = await findDigestAsset(release, 'test-digest'); + expect(digestAsset).toBeNull(); + }); + }); + + describe('mapDigestAssetToRelease', () => { + describe('with digest file', () => { + const digestAsset: DigestAsset = { + assetName: 'SHASUMS.txt', + currentVersion: 'v1.0.0', + currentDigest: 'old-digest', + digestedFileName: 'asset.zip', + }; + + it('downloads updated digest file', async () => { + const release = releaseMock.withDigestFileAsset( + 'v1.0.1', + 'updated-digest asset.zip' + ); + const digest = await mapDigestAssetToRelease(digestAsset, release); + expect(digest).toBe('updated-digest'); + }); + + it('maps digested file name to new version', async () => { + const digestAssetWithVersion = { + ...digestAsset, + digestedFileName: 'asset-1.0.0.zip', + }; + + const release = releaseMock.withDigestFileAsset( + 'v1.0.1', + 'updated-digest asset-1.0.1.zip' + ); + const digest = await mapDigestAssetToRelease( + digestAssetWithVersion, + release + ); + expect(digest).toBe('updated-digest'); + }); + + it('returns null when not found in digest file', async () => { + const release = releaseMock.withDigestFileAsset( + 'v1.0.1', + 'moot-digest asset.tar.gz' + ); + const digest = await mapDigestAssetToRelease(digestAsset, release); + expect(digest).toBeNull(); + }); + + it('returns null when digest file not found', async () => { + const release = releaseMock.release('v1.0.1'); + const digest = await mapDigestAssetToRelease(digestAsset, release); + expect(digest).toBeNull(); + }); + }); + + describe('with digested file', () => { + const digestAsset: DigestAsset = { + assetName: 'asset.zip', + currentVersion: 'v1.0.0', + currentDigest: '0'.repeat(64), + }; + + it('digests updated file', async () => { + const updatedContent = 'new content'; + const release = releaseMock.withAssets('v1.0.1', { + 'asset.zip': updatedContent, + }); + const contentDigest = await hasha.async(updatedContent, { + algorithm: 'sha256', + }); + + const digest = await mapDigestAssetToRelease(digestAsset, release); + expect(digest).toEqual(contentDigest); + }); + + it('returns null when not found', async () => { + const release = releaseMock.release('v1.0.1'); + const digest = await mapDigestAssetToRelease(digestAsset, release); + expect(digest).toBeNull(); + }); + }); + }); +}); diff --git a/lib/datasource/github-releases/digest.ts b/lib/datasource/github-releases/digest.ts new file mode 100644 index 0000000000..9afdb7e2cd --- /dev/null +++ b/lib/datasource/github-releases/digest.ts @@ -0,0 +1,141 @@ +import hasha from 'hasha'; +import * as packageCache from '../../util/cache/package'; +import { cacheNamespace, http } from './common'; +import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types'; + +async function findDigestFile( + release: GithubRelease, + digest: string +): Promise<DigestAsset | null> { + const smallAssets = release.assets.filter( + (a: GithubReleaseAsset) => a.size < 5 * 1024 + ); + for (const asset of smallAssets) { + const res = await http.get(asset.browser_download_url); + for (const line of res.body.split('\n')) { + const [lineDigest, lineFn] = line.split(/\s+/, 2); + if (lineDigest === digest) { + return { + assetName: asset.name, + digestedFileName: lineFn, + currentVersion: release.tag_name, + currentDigest: lineDigest, + }; + } + } + } + return null; +} + +function inferHashAlg(digest: string): string { + switch (digest.length) { + case 64: + return 'sha256'; + default: + case 96: + return 'sha512'; + } +} + +function getAssetDigestCacheKey( + downloadUrl: string, + algorithm: string +): string { + const type = 'assetDigest'; + return `${downloadUrl}:${algorithm}:${type}`; +} + +async function downloadAndDigest( + asset: GithubReleaseAsset, + algorithm: string +): Promise<string> { + const downloadUrl = asset.browser_download_url; + const cacheKey = getAssetDigestCacheKey(downloadUrl, algorithm); + const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + + const res = http.stream(downloadUrl); + const digest = await hasha.fromStream(res, { algorithm }); + + const cacheMinutes = 1440; + await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); + return digest; +} + +async function findAssetWithDigest( + release: GithubRelease, + digest: string +): Promise<DigestAsset | null> { + const algorithm = inferHashAlg(digest); + const assetsBySize = release.assets.sort( + (a: GithubReleaseAsset, b: GithubReleaseAsset) => { + if (a.size < b.size) { + return -1; + } + if (a.size > b.size) { + return 1; + } + return 0; + } + ); + + for (const asset of assetsBySize) { + const assetDigest = await downloadAndDigest(asset, algorithm); + if (assetDigest === digest) { + return { + assetName: asset.name, + currentVersion: release.tag_name, + currentDigest: assetDigest, + }; + } + } + return null; +} + +/** Identify the asset associated with a known digest. */ +export async function findDigestAsset( + release: GithubRelease, + digest: string +): Promise<DigestAsset> { + const digestFile = await findDigestFile(release, digest); + if (digestFile) { + return digestFile; + } + + const asset = await findAssetWithDigest(release, digest); + return asset; +} + +/** Given a digest asset, find the equivalent digest in a different release. */ +export async function mapDigestAssetToRelease( + digestAsset: DigestAsset, + release: GithubRelease +): Promise<string | null> { + const current = digestAsset.currentVersion.replace(/^v/, ''); + const next = release.tag_name.replace(/^v/, ''); + const releaseChecksumAssetName = digestAsset.assetName.replace(current, next); + const releaseAsset = release.assets.find( + (a: GithubReleaseAsset) => a.name === releaseChecksumAssetName + ); + if (!releaseAsset) { + return null; + } + if (digestAsset.digestedFileName) { + const releaseFilename = digestAsset.digestedFileName.replace(current, next); + const res = await http.get(releaseAsset.browser_download_url); + for (const line of res.body.split('\n')) { + const [lineDigest, lineFn] = line.split(/\s+/, 2); + if (lineFn === releaseFilename) { + return lineDigest; + } + } + } else { + const algorithm = inferHashAlg(digestAsset.currentDigest); + const newDigest = await downloadAndDigest(releaseAsset, algorithm); + return newDigest; + } + return null; +} diff --git a/lib/datasource/github-releases/index.spec.ts b/lib/datasource/github-releases/index.spec.ts index 35ff90fbc9..72f866134b 100644 --- a/lib/datasource/github-releases/index.spec.ts +++ b/lib/datasource/github-releases/index.spec.ts @@ -1,7 +1,8 @@ -import { getPkgReleases } from '..'; +import { getDigest, getPkgReleases } from '..'; import * as httpMock from '../../../test/http-mock'; import { getName } from '../../../test/util'; import * as _hostRules from '../../util/host-rules'; +import { GitHubReleaseMocker } from './test'; import { id as datasource } from '.'; import * as github from '.'; @@ -67,4 +68,65 @@ describe(getName(), () => { expect(httpMock.getTrace()).toMatchSnapshot(); }); }); + + describe('getDigest', () => { + const lookupName = 'some/dep'; + const currentValue = 'v1.0.0'; + const currentDigest = 'v1.0.0-digest'; + + const releaseMock = new GitHubReleaseMocker(githubApiHost, lookupName); + + it('requires currentDigest', async () => { + const digest = await getDigest({ datasource, lookupName }, currentValue); + expect(digest).toBeNull(); + }); + + it('defaults to currentDigest when currentVersion is missing', async () => { + const digest = await getDigest( + { + datasource, + lookupName, + currentDigest, + }, + currentValue + ); + expect(digest).toEqual(currentDigest); + }); + + it('returns updated digest in new release', async () => { + releaseMock.withDigestFileAsset( + currentValue, + `${currentDigest} asset.zip` + ); + const nextValue = 'v1.0.1'; + const nextDigest = 'updated-digest'; + releaseMock.withDigestFileAsset(nextValue, `${nextDigest} asset.zip`); + const digest = await getDigest( + { + datasource, + lookupName, + currentValue, + currentDigest, + }, + nextValue + ); + expect(digest).toEqual(nextDigest); + }); + + // This is awkward, but I found returning `null` in this case to not produce an update + // I'd prefer a PR with the old digest (that I can manually patch) to no PR, so I made this decision. + it('ignores failures verifying currentDigest', async () => { + releaseMock.release(currentValue); + const digest = await getDigest( + { + datasource, + lookupName, + currentValue, + currentDigest, + }, + currentValue + ); + expect(digest).toEqual(currentDigest); + }); + }); }); diff --git a/lib/datasource/github-releases/index.ts b/lib/datasource/github-releases/index.ts index 4fe6e5469e..bcc91241a3 100644 --- a/lib/datasource/github-releases/index.ts +++ b/lib/datasource/github-releases/index.ts @@ -1,7 +1,14 @@ +import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; -import { GithubHttp } from '../../util/http/github'; -import { ensureTrailingSlash } from '../../util/url'; -import type { GetReleasesConfig, ReleaseResult } from '../types'; +import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; +import { + cacheNamespace, + getApiBaseUrl, + getGithubRelease, + getSourceUrlBase, + http, +} from './common'; +import { findDigestAsset, mapDigestAssetToRelease } from './digest'; import type { GithubRelease } from './types'; export const id = 'github-releases'; @@ -9,13 +16,9 @@ export const customRegistrySupport = true; export const defaultRegistryUrls = ['https://github.com']; export const registryStrategy = 'first'; -const cacheNamespace = 'datasource-github-releases'; - -const http = new GithubHttp(); - -function getCacheKey(depHost: string, repo: string): string { +function getReleasesCacheKey(registryUrl: string, repo: string): string { const type = 'tags'; - return `${depHost}:${repo}:${type}`; + return `${registryUrl}:${repo}:${type}`; } /** @@ -32,22 +35,17 @@ export async function getReleases({ lookupName: repo, registryUrl, }: GetReleasesConfig): Promise<ReleaseResult | null> { + const cacheKey = getReleasesCacheKey(registryUrl, repo); const cachedResult = await packageCache.get<ReleaseResult>( cacheNamespace, - getCacheKey(registryUrl, repo) + cacheKey ); // istanbul ignore if if (cachedResult) { return cachedResult; } - // default to GitHub.com if no GHE host is specified. - const sourceUrlBase = ensureTrailingSlash( - registryUrl ?? 'https://github.com/' - ); - const apiBaseUrl = - sourceUrlBase === 'https://github.com/' - ? `https://api.github.com/` - : `${sourceUrlBase}api/v3/`; + const sourceUrlBase = getSourceUrlBase(registryUrl); + const apiBaseUrl = getApiBaseUrl(sourceUrlBase); const url = `${apiBaseUrl}repos/${repo}/releases?per_page=100`; const res = await http.getJson<GithubRelease[]>(url, { paginate: true, @@ -66,11 +64,66 @@ export async function getReleases({ }) ); const cacheMinutes = 10; - await packageCache.set( - cacheNamespace, - getCacheKey(registryUrl, repo), - dependency, - cacheMinutes - ); + await packageCache.set(cacheNamespace, cacheKey, dependency, cacheMinutes); return dependency; } + +function getDigestCacheKey( + { lookupName: repo, currentValue, currentDigest, registryUrl }: DigestConfig, + newValue: string +): string { + const type = 'digest'; + return `${registryUrl}:${repo}:${currentValue}:${currentDigest}:${newValue}:${type}`; +} + +/** + * github.getDigest + * + * The `newValue` supplied here should be a valid tag for the GitHub release. + * Requires `currentValue` and `currentDigest`. + * + * There may be many assets attached to the release. This function will: + * - Identify the asset pinned by `currentDigest` in the `currentValue` release + * - Download small release assets, parse as checksum manifests (e.g. `SHASUMS.txt`). + * - Download individual assets until `currentDigest` is encountered. This is limited to sha256 and sha512. + * - Map the hashed asset to `newValue` and return the updated digest as a string + */ +export async function getDigest( + { lookupName: repo, currentValue, currentDigest, registryUrl }: DigestConfig, + newValue?: string +): Promise<string | null> { + logger.debug( + { repo, currentValue, currentDigest, registryUrl, newValue }, + 'getDigest' + ); + if (!currentDigest) { + return null; + } + if (!currentValue) { + return currentDigest; + } + const cacheKey = getDigestCacheKey( + { lookupName: repo, currentValue, currentDigest, registryUrl }, + newValue + ); + const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + + const apiBaseUrl = getApiBaseUrl(getSourceUrlBase(registryUrl)); + const currentRelease = await getGithubRelease(apiBaseUrl, repo, currentValue); + const digestAsset = await findDigestAsset(currentRelease, currentDigest); + let newDigest: string; + if (!digestAsset || newValue === currentValue) { + newDigest = currentDigest; + } else { + const newRelease = await getGithubRelease(apiBaseUrl, repo, newValue); + newDigest = await mapDigestAssetToRelease(digestAsset, newRelease); + } + + const cacheMinutes = 1440; + await packageCache.set(cacheNamespace, cacheKey, newDigest, cacheMinutes); + return newDigest; +} diff --git a/lib/datasource/github-releases/test/index.ts b/lib/datasource/github-releases/test/index.ts new file mode 100644 index 0000000000..7c40f0962f --- /dev/null +++ b/lib/datasource/github-releases/test/index.ts @@ -0,0 +1,49 @@ +import * as httpMock from '../../../../test/http-mock'; +import { GithubRelease } from '../types'; + +export class GitHubReleaseMocker { + constructor( + private readonly githubApiHost: string, + private readonly lookupName: string + ) {} + + release(version: string): GithubRelease { + return this.withAssets(version, {}); + } + + withAssets( + version: string, + assets: { [key: string]: string } + ): GithubRelease { + const releaseData = { + tag_name: version, + published_at: '2020-03-09T11:00:00Z', + prerelease: false, + assets: [], + }; + for (const assetFn of Object.keys(assets)) { + const assetPath = `/repos/${this.lookupName}/releases/download/${version}/${assetFn}`; + const assetData = assets[assetFn]; + releaseData.assets.push({ + name: assetFn, + size: assetData.length, + browser_download_url: `${this.githubApiHost}${assetPath}`, + }); + httpMock + .scope(this.githubApiHost) + .get(assetPath) + .once() + .reply(200, assetData); + } + httpMock + .scope(this.githubApiHost) + .get(`/repos/${this.lookupName}/releases/tags/${version}`) + .optionally() + .reply(200, releaseData); + return releaseData; + } + + withDigestFileAsset(version: string, ...digests: string[]): GithubRelease { + return this.withAssets(version, { 'SHASUMS.txt': digests.join('\n') }); + } +} diff --git a/lib/datasource/github-releases/types.ts b/lib/datasource/github-releases/types.ts index 48ca80461e..78dd7af200 100644 --- a/lib/datasource/github-releases/types.ts +++ b/lib/datasource/github-releases/types.ts @@ -2,4 +2,18 @@ export type GithubRelease = { tag_name: string; published_at: string; prerelease: boolean; + assets: GithubReleaseAsset[]; }; + +export interface GithubReleaseAsset { + name: string; + browser_download_url: string; + size: number; +} + +export interface DigestAsset { + assetName: string; + currentVersion: string; + currentDigest: string; + digestedFileName?: string; +} diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 6714289005..1b12c8b5b8 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -365,10 +365,13 @@ export function getDigest( const datasource = getDatasourceFor(config.datasource); const lookupName = config.lookupName || config.depName; const registryUrls = resolveRegistryUrls(datasource, config.registryUrls); - return datasource.getDigest( - { lookupName, registryUrl: registryUrls[0] }, - value - ); + const digestConfig: DigestConfig = { + registryUrl: registryUrls[0], + currentValue: config.currentValue, + currentDigest: config.currentDigest, + lookupName, + }; + return datasource.getDigest(digestConfig, value); } export function getDefaultConfig( diff --git a/lib/datasource/types.ts b/lib/datasource/types.ts index 54dc17ebe7..098a172742 100644 --- a/lib/datasource/types.ts +++ b/lib/datasource/types.ts @@ -7,6 +7,8 @@ export interface Config { export interface DigestConfig extends Config { registryUrl?: string; + currentValue?: string; + currentDigest?: string; } export interface ReleasesConfigBase { diff --git a/tsconfig.app.json b/tsconfig.app.json index 23d3d7e2f0..c486d28662 100644 --- a/tsconfig.app.json +++ b/tsconfig.app.json @@ -16,6 +16,7 @@ "**/__mocks__/**", "**/__fixtures__/**", "**/__testutil__/**", + "**/test/**", "**/*.spec.ts", "jest.config.ts", "./test", -- GitLab