diff --git a/lib/datasource/github-releases/common.spec.ts b/lib/datasource/github-releases/common.spec.ts index ff01c64f1524a48abb95ffb494a437881e43ee89..352dc80df7a07862c6579cdd17390dd6b5af733f 100644 --- a/lib/datasource/github-releases/common.spec.ts +++ b/lib/datasource/github-releases/common.spec.ts @@ -1,5 +1,5 @@ -import { getApiBaseUrl, getGithubRelease, getSourceUrlBase } from './common'; import { GitHubReleaseMocker } from './test'; +import { getApiBaseUrl, getGithubRelease, getSourceUrlBase } from '.'; describe('datasource/github-releases/common', () => { describe('getSourceUrlBase', () => { diff --git a/lib/datasource/github-releases/common.ts b/lib/datasource/github-releases/common.ts deleted file mode 100644 index 7d23c8cd7e38a18236af9b6b40efff0f634b32a3..0000000000000000000000000000000000000000 --- a/lib/datasource/github-releases/common.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { GithubHttp } from '../../util/http/github'; -import { ensureTrailingSlash } from '../../util/url'; -import type { GithubRelease } from './types'; - -const defaultSourceUrlBase = 'https://github.com/'; -export const id = 'github-releases'; - -export const cacheNamespace = 'datasource-github-releases'; -export const http = new GithubHttp(id); - -export function getSourceUrlBase(registryUrl: string): string { - // default to GitHub.com if no GHE host is specified. - return ensureTrailingSlash(registryUrl ?? defaultSourceUrlBase); -} - -export function getApiBaseUrl(registryUrl: string): string { - const sourceUrlBase = getSourceUrlBase(registryUrl); - return sourceUrlBase === defaultSourceUrlBase - ? `https://api.github.com/` - : `${sourceUrlBase}api/v3/`; -} - -export function getSourceUrl(lookupName: string, registryUrl?: string): string { - const sourceUrlBase = getSourceUrlBase(registryUrl); - return `${sourceUrlBase}${lookupName}`; -} - -export async function getGithubRelease( - apiBaseUrl: string, - repo: string, - version: string -): Promise<GithubRelease> { - const url = `${apiBaseUrl}repos/${repo}/releases/tags/${version}`; - const res = await http.getJson<GithubRelease>(url); - return res.body; -} diff --git a/lib/datasource/github-releases/digest.spec.ts b/lib/datasource/github-releases/digest.spec.ts index 6b0db02d09d0ebbf6212bf1aa079dd5c3e277a55..56f4f27ebbd8b9ca678e756a3942feeca0f65cb3 100644 --- a/lib/datasource/github-releases/digest.spec.ts +++ b/lib/datasource/github-releases/digest.spec.ts @@ -1,8 +1,8 @@ import hasha from 'hasha'; import * as httpMock from '../../../test/http-mock'; -import { findDigestAsset, mapDigestAssetToRelease } from './digest'; import { GitHubReleaseMocker } from './test'; import type { DigestAsset } from './types'; +import { findDigestAsset, mapDigestAssetToRelease } from '.'; describe('datasource/github-releases/digest', () => { const lookupName = 'some/dep'; diff --git a/lib/datasource/github-releases/digest.ts b/lib/datasource/github-releases/digest.ts deleted file mode 100644 index 56861fe0b7fe231324732c464457438b0d6dab0a..0000000000000000000000000000000000000000 --- a/lib/datasource/github-releases/digest.ts +++ /dev/null @@ -1,142 +0,0 @@ -import hasha from 'hasha'; -import * as packageCache from '../../util/cache/package'; -import { newlineRegex, regEx } from '../../util/regex'; -import { cacheNamespace, http } from './common'; -import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types'; - -async function findDigestFile( - release: GithubRelease, - digest: string -): Promise<DigestAsset | null> { - const smallAssets = release.assets.filter( - (a: GithubReleaseAsset) => a.size < 5 * 1024 - ); - for (const asset of smallAssets) { - const res = await http.get(asset.browser_download_url); - for (const line of res.body.split(newlineRegex)) { - const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2); - if (lineDigest === digest) { - return { - assetName: asset.name, - digestedFileName: lineFn, - currentVersion: release.tag_name, - currentDigest: lineDigest, - }; - } - } - } - return null; -} - -function inferHashAlg(digest: string): string { - switch (digest.length) { - case 64: - return 'sha256'; - default: - case 96: - return 'sha512'; - } -} - -function getAssetDigestCacheKey( - downloadUrl: string, - algorithm: string -): string { - const type = 'assetDigest'; - return `${downloadUrl}:${algorithm}:${type}`; -} - -async function downloadAndDigest( - asset: GithubReleaseAsset, - algorithm: string -): Promise<string> { - const downloadUrl = asset.browser_download_url; - const cacheKey = getAssetDigestCacheKey(downloadUrl, algorithm); - const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey); - // istanbul ignore if - if (cachedResult) { - return cachedResult; - } - - const res = http.stream(downloadUrl); - const digest = await hasha.fromStream(res, { algorithm }); - - const cacheMinutes = 1440; - await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); - return digest; -} - -async function findAssetWithDigest( - release: GithubRelease, - digest: string -): Promise<DigestAsset | null> { - const algorithm = inferHashAlg(digest); - const assetsBySize = release.assets.sort( - (a: GithubReleaseAsset, b: GithubReleaseAsset) => { - if (a.size < b.size) { - return -1; - } - if (a.size > b.size) { - return 1; - } - return 0; - } - ); - - for (const asset of assetsBySize) { - const assetDigest = await downloadAndDigest(asset, algorithm); - if (assetDigest === digest) { - return { - assetName: asset.name, - currentVersion: release.tag_name, - currentDigest: assetDigest, - }; - } - } - return null; -} - -/** Identify the asset associated with a known digest. */ -export async function findDigestAsset( - release: GithubRelease, - digest: string -): Promise<DigestAsset> { - const digestFile = await findDigestFile(release, digest); - if (digestFile) { - return digestFile; - } - - const asset = await findAssetWithDigest(release, digest); - return asset; -} - -/** Given a digest asset, find the equivalent digest in a different release. */ -export async function mapDigestAssetToRelease( - digestAsset: DigestAsset, - release: GithubRelease -): Promise<string | null> { - const current = digestAsset.currentVersion.replace(regEx(/^v/), ''); - const next = release.tag_name.replace(regEx(/^v/), ''); - const releaseChecksumAssetName = digestAsset.assetName.replace(current, next); - const releaseAsset = release.assets.find( - (a: GithubReleaseAsset) => a.name === releaseChecksumAssetName - ); - if (!releaseAsset) { - return null; - } - if (digestAsset.digestedFileName) { - const releaseFilename = digestAsset.digestedFileName.replace(current, next); - const res = await http.get(releaseAsset.browser_download_url); - for (const line of res.body.split(newlineRegex)) { - const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2); - if (lineFn === releaseFilename) { - return lineDigest; - } - } - } else { - const algorithm = inferHashAlg(digestAsset.currentDigest); - const newDigest = await downloadAndDigest(releaseAsset, algorithm); - return newDigest; - } - return null; -} diff --git a/lib/datasource/github-releases/index.ts b/lib/datasource/github-releases/index.ts index 1c58489a9f51d45c2e518e8e6334bf05f8644da6..4b7f95c2bdc2c08bae082144a9890e22483504cb 100644 --- a/lib/datasource/github-releases/index.ts +++ b/lib/datasource/github-releases/index.ts @@ -1,22 +1,186 @@ +import hasha from 'hasha'; import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; +import { GithubHttp } from '../../util/http/github'; +import { newlineRegex, regEx } from '../../util/regex'; +import { ensureTrailingSlash } from '../../util/url'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; -import { - cacheNamespace, - getApiBaseUrl, - getGithubRelease, - getSourceUrl, - http, - id, -} from './common'; -import { findDigestAsset, mapDigestAssetToRelease } from './digest'; -import type { GithubRelease } from './types'; - -export { id }; +import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types'; + export const customRegistrySupport = true; export const defaultRegistryUrls = ['https://github.com']; export const registryStrategy = 'first'; +const defaultSourceUrlBase = 'https://github.com/'; +export const id = 'github-releases'; + +export const cacheNamespace = 'datasource-github-releases'; +export const http = new GithubHttp(id); + +async function findDigestFile( + release: GithubRelease, + digest: string +): Promise<DigestAsset | null> { + const smallAssets = release.assets.filter( + (a: GithubReleaseAsset) => a.size < 5 * 1024 + ); + for (const asset of smallAssets) { + const res = await http.get(asset.browser_download_url); + for (const line of res.body.split(newlineRegex)) { + const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2); + if (lineDigest === digest) { + return { + assetName: asset.name, + digestedFileName: lineFn, + currentVersion: release.tag_name, + currentDigest: lineDigest, + }; + } + } + } + return null; +} + +function inferHashAlg(digest: string): string { + switch (digest.length) { + case 64: + return 'sha256'; + default: + case 96: + return 'sha512'; + } +} + +function getAssetDigestCacheKey( + downloadUrl: string, + algorithm: string +): string { + const type = 'assetDigest'; + return `${downloadUrl}:${algorithm}:${type}`; +} + +async function downloadAndDigest( + asset: GithubReleaseAsset, + algorithm: string +): Promise<string> { + const downloadUrl = asset.browser_download_url; + const cacheKey = getAssetDigestCacheKey(downloadUrl, algorithm); + const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + + const res = http.stream(downloadUrl); + const digest = await hasha.fromStream(res, { algorithm }); + + const cacheMinutes = 1440; + await packageCache.set(cacheNamespace, cacheKey, digest, cacheMinutes); + return digest; +} + +async function findAssetWithDigest( + release: GithubRelease, + digest: string +): Promise<DigestAsset | null> { + const algorithm = inferHashAlg(digest); + const assetsBySize = release.assets.sort( + (a: GithubReleaseAsset, b: GithubReleaseAsset) => { + if (a.size < b.size) { + return -1; + } + if (a.size > b.size) { + return 1; + } + return 0; + } + ); + + for (const asset of assetsBySize) { + const assetDigest = await downloadAndDigest(asset, algorithm); + if (assetDigest === digest) { + return { + assetName: asset.name, + currentVersion: release.tag_name, + currentDigest: assetDigest, + }; + } + } + return null; +} + +/** Identify the asset associated with a known digest. */ +export async function findDigestAsset( + release: GithubRelease, + digest: string +): Promise<DigestAsset> { + const digestFile = await findDigestFile(release, digest); + if (digestFile) { + return digestFile; + } + + const asset = await findAssetWithDigest(release, digest); + return asset; +} + +/** Given a digest asset, find the equivalent digest in a different release. */ +export async function mapDigestAssetToRelease( + digestAsset: DigestAsset, + release: GithubRelease +): Promise<string | null> { + const current = digestAsset.currentVersion.replace(regEx(/^v/), ''); + const next = release.tag_name.replace(regEx(/^v/), ''); + const releaseChecksumAssetName = digestAsset.assetName.replace(current, next); + const releaseAsset = release.assets.find( + (a: GithubReleaseAsset) => a.name === releaseChecksumAssetName + ); + if (!releaseAsset) { + return null; + } + if (digestAsset.digestedFileName) { + const releaseFilename = digestAsset.digestedFileName.replace(current, next); + const res = await http.get(releaseAsset.browser_download_url); + for (const line of res.body.split(newlineRegex)) { + const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2); + if (lineFn === releaseFilename) { + return lineDigest; + } + } + } else { + const algorithm = inferHashAlg(digestAsset.currentDigest); + const newDigest = await downloadAndDigest(releaseAsset, algorithm); + return newDigest; + } + return null; +} + +export function getSourceUrlBase(registryUrl: string): string { + // default to GitHub.com if no GHE host is specified. + return ensureTrailingSlash(registryUrl ?? defaultSourceUrlBase); +} + +export function getApiBaseUrl(registryUrl: string): string { + const sourceUrlBase = getSourceUrlBase(registryUrl); + return sourceUrlBase === defaultSourceUrlBase + ? `https://api.github.com/` + : `${sourceUrlBase}api/v3/`; +} + +export function getSourceUrl(lookupName: string, registryUrl?: string): string { + const sourceUrlBase = getSourceUrlBase(registryUrl); + return `${sourceUrlBase}${lookupName}`; +} + +export async function getGithubRelease( + apiBaseUrl: string, + repo: string, + version: string +): Promise<GithubRelease> { + const url = `${apiBaseUrl}repos/${repo}/releases/tags/${version}`; + const res = await http.getJson<GithubRelease>(url); + return res.body; +} + function getReleasesCacheKey(registryUrl: string, repo: string): string { const type = 'tags'; return `${registryUrl}:${repo}:${type}`; diff --git a/lib/datasource/github-tags/index.ts b/lib/datasource/github-tags/index.ts index c9abc36762a342cad786265334fd42aaa1aa8970..26719d6096021427bb932354f6ed8ae1779263c2 100644 --- a/lib/datasource/github-tags/index.ts +++ b/lib/datasource/github-tags/index.ts @@ -1,8 +1,11 @@ import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { GithubHttp } from '../../util/http/github'; -import * as githubReleases from '../github-releases'; -import { getApiBaseUrl, getSourceUrl } from '../github-releases/common'; +import { + getApiBaseUrl, + getSourceUrl, + getReleases as githubGetReleases, +} from '../github-releases'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; import type { GitHubTag, TagResponse } from './types'; @@ -158,7 +161,7 @@ export async function getReleases( try { // Fetch additional data from releases endpoint when possible - const releasesResult = await githubReleases.getReleases(config); + const releasesResult = await githubGetReleases(config); const releaseByVersion = {}; releasesResult?.releases?.forEach((release) => { const key = release.version; diff --git a/lib/datasource/go/common.ts b/lib/datasource/go/common.ts index e209babc5c40c2b2e1355d80b188f324cdc02476..fed5620f6c4e178f61f8d06bbe610fe188a8a4e2 100644 --- a/lib/datasource/go/common.ts +++ b/lib/datasource/go/common.ts @@ -1,5 +1,5 @@ import { BitBucketTagsDatasource } from '../bitbucket-tags'; -import { getSourceUrl as githubSourceUrl } from '../github-releases/common'; +import { getSourceUrl as githubSourceUrl } from '../github-releases'; import { id as githubDatasource } from '../github-tags'; import { id as gitlabDatasource } from '../gitlab-tags'; import { getSourceUrl as gitlabSourceUrl } from '../gitlab-tags/util';