From 150092a2a59176ed933d05af1f36c027ef5cd04a Mon Sep 17 00:00:00 2001 From: Taras <9948629+Trane9991@users.noreply.github.com> Date: Sat, 6 Feb 2021 09:05:19 +0200 Subject: [PATCH] feat(go): add support for bitbucket in go datasource (#7892) --- .../__snapshots__/index.spec.ts.snap | 108 ++++++++++++++ lib/datasource/bitbucket-tags/index.spec.ts | 133 ++++++++++++++++++ lib/datasource/bitbucket-tags/index.ts | 132 +++++++++++++++++ lib/datasource/bitbucket-tags/types.ts | 12 ++ .../go/__snapshots__/index.spec.ts.snap | 88 ++++++++++++ lib/datasource/go/index.spec.ts | 72 ++++++++++ lib/datasource/go/index.ts | 60 ++++++-- lib/platform/bitbucket/index.ts | 2 +- lib/util/http/bitbucket.ts | 1 + 9 files changed, 595 insertions(+), 13 deletions(-) create mode 100644 lib/datasource/bitbucket-tags/__snapshots__/index.spec.ts.snap create mode 100644 lib/datasource/bitbucket-tags/index.spec.ts create mode 100644 lib/datasource/bitbucket-tags/index.ts create mode 100644 lib/datasource/bitbucket-tags/types.ts diff --git a/lib/datasource/bitbucket-tags/__snapshots__/index.spec.ts.snap b/lib/datasource/bitbucket-tags/__snapshots__/index.spec.ts.snap new file mode 100644 index 0000000000..271db57a9f --- /dev/null +++ b/lib/datasource/bitbucket-tags/__snapshots__/index.spec.ts.snap @@ -0,0 +1,108 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`datasource/bitbucket-tags/index getDigest returns commits from bitbucket cloud 1`] = `"123"`; + +exports[`datasource/bitbucket-tags/index getDigest returns commits from bitbucket cloud 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/some/dep2", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/some/dep2/commits/master", + }, +] +`; + +exports[`datasource/bitbucket-tags/index getDigest with no commits returns commits from bitbucket cloud 1`] = `null`; + +exports[`datasource/bitbucket-tags/index getDigest with no commits returns commits from bitbucket cloud 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/some/dep2", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/some/dep2/commits/master", + }, +] +`; + +exports[`datasource/bitbucket-tags/index getReleases returns tags from bitbucket cloud 1`] = ` +Object { + "releases": Array [ + Object { + "gitRef": "v1.0.0", + "releaseTimestamp": "2020-11-19T09:05:35+00:00", + "version": "v1.0.0", + }, + Object { + "gitRef": "v1.1.0", + "version": "v1.1.0", + }, + Object { + "gitRef": "v1.1.1", + "version": "v1.1.1", + }, + ], + "sourceUrl": "https://bitbucket.org/some/dep2", +} +`; + +exports[`datasource/bitbucket-tags/index getReleases returns tags from bitbucket cloud 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/some/dep2/refs/tags", + }, +] +`; + +exports[`datasource/bitbucket-tags/index getTagCommit returns tags commit hash from bitbucket cloud 1`] = `"123"`; + +exports[`datasource/bitbucket-tags/index getTagCommit returns tags commit hash from bitbucket cloud 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/some/dep2/refs/tags/v1.0.0", + }, +] +`; diff --git a/lib/datasource/bitbucket-tags/index.spec.ts b/lib/datasource/bitbucket-tags/index.spec.ts new file mode 100644 index 0000000000..33ffa48a69 --- /dev/null +++ b/lib/datasource/bitbucket-tags/index.spec.ts @@ -0,0 +1,133 @@ +import { getDigest, getPkgReleases } from '..'; +import * as httpMock from '../../../test/http-mock'; +import { getName } from '../../../test/util'; +import { id as datasource } from '.'; + +describe(getName(__filename), () => { + beforeEach(() => { + httpMock.reset(); + httpMock.setup(); + }); + describe('getReleases', () => { + it('returns tags from bitbucket cloud', async () => { + const body = { + pagelen: 3, + values: [ + { + name: 'v1.0.0', + target: { + date: '2020-11-19T09:05:35+00:00', + }, + }, + { + name: 'v1.1.0', + target: {}, + }, + { + name: 'v1.1.1', + }, + ], + page: 1, + }; + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/some/dep2/refs/tags') + .reply(200, body); + const res = await getPkgReleases({ + datasource, + depName: 'some/dep2', + }); + expect(res).toMatchSnapshot(); + expect(res.releases).toHaveLength(3); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + }); + describe('getDigest', () => { + it('returns commits from bitbucket cloud', async () => { + const body = { + pagelen: 3, + values: [ + { + hash: '123', + date: '2020-11-19T09:05:35+00:00', + }, + { + hash: '133', + date: '2020-11-19T09:05:36+00:00', + }, + { + hash: '333', + date: '2020-11-19T09:05:37+00:00', + }, + ], + page: 1, + }; + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/some/dep2') + .reply(200, { mainbranch: { name: 'master' } }); + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/some/dep2/commits/master') + .reply(200, body); + const res = await getDigest({ + datasource, + depName: 'some/dep2', + }); + expect(res).toMatchSnapshot(); + expect(res).toBeString(); + expect(res).toEqual('123'); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + }); + describe('getDigest with no commits', () => { + it('returns commits from bitbucket cloud', async () => { + const body = { + pagelen: 0, + values: [], + page: 1, + }; + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/some/dep2') + .reply(200, { mainbranch: { name: 'master' } }); + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/some/dep2/commits/master') + .reply(200, body); + const res = await getDigest({ + datasource, + depName: 'some/dep2', + }); + expect(res).toMatchSnapshot(); + expect(res).toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + }); + describe('getTagCommit', () => { + it('returns tags commit hash from bitbucket cloud', async () => { + const body = { + name: 'v1.0.0', + target: { + date: '2020-11-19T09:05:35+00:00', + hash: '123', + }, + }; + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/some/dep2/refs/tags/v1.0.0') + .reply(200, body); + const res = await getDigest( + { + datasource, + depName: 'some/dep2', + }, + 'v1.0.0' + ); + expect(res).toMatchSnapshot(); + expect(res).toBeString(); + expect(res).toEqual('123'); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + }); +}); diff --git a/lib/datasource/bitbucket-tags/index.ts b/lib/datasource/bitbucket-tags/index.ts new file mode 100644 index 0000000000..b9a79dc813 --- /dev/null +++ b/lib/datasource/bitbucket-tags/index.ts @@ -0,0 +1,132 @@ +import * as utils from '../../platform/bitbucket/utils'; +import * as packageCache from '../../util/cache/package'; +import { BitbucketHttp } from '../../util/http/bitbucket'; +import { ensureTrailingSlash } from '../../util/url'; +import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; +import { BitbucketCommit, BitbucketTag } from './types'; + +const bitbucketHttp = new BitbucketHttp(); + +export const id = 'bitbucket-tags'; +export const registryStrategy = 'first'; +export const defaultRegistryUrls = ['https://bitbucket.org']; + +function getRegistryURL(registryUrl: string): string { + // fallback to default API endpoint if custom not provided + return registryUrl ?? defaultRegistryUrls[0]; +} + +const cacheNamespace = 'datasource-bitbucket'; + +function getCacheKey(registryUrl: string, repo: string, type: string): string { + return `${getRegistryURL(registryUrl)}:${repo}:${type}`; +} + +// getReleases fetches list of tags for the repository +export async function getReleases({ + registryUrl, + lookupName: repo, +}: GetReleasesConfig): Promise<ReleaseResult | null> { + const cacheKey = getCacheKey(registryUrl, repo, 'tags'); + const cachedResult = await packageCache.get<ReleaseResult>( + cacheNamespace, + cacheKey + ); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + + const url = `/2.0/repositories/${repo}/refs/tags`; + + const bitbucketTags = ( + await bitbucketHttp.getJson<utils.PagedResult<BitbucketTag>>(url) + ).body; + + const dependency: ReleaseResult = { + sourceUrl: `${ensureTrailingSlash(getRegistryURL(registryUrl))}${repo}`, + releases: null, + }; + dependency.releases = bitbucketTags.values.map(({ name, target }) => ({ + version: name, + gitRef: name, + releaseTimestamp: target?.date, + })); + + const cacheMinutes = 10; + await packageCache.set(cacheNamespace, cacheKey, dependency, cacheMinutes); + return dependency; +} + +// getTagCommit fetched the commit has for specified tag +async function getTagCommit( + registryUrl: string, + repo: string, + tag: string +): Promise<string | null> { + const cacheKey = getCacheKey(registryUrl, repo, `tag-${tag}`); + const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + + const url = `/2.0/repositories/${repo}/refs/tags/${tag}`; + + const bitbucketTag = (await bitbucketHttp.getJson<BitbucketTag>(url)).body; + + const hash = bitbucketTag.target.hash; + + const cacheMinutes = 10; + await packageCache.set(cacheNamespace, cacheKey, hash, cacheMinutes); + + return hash; +} + +// getDigest fetched the latest commit for repository main branch +// however, if newValue is provided, then getTagCommit is called +export async function getDigest( + { lookupName: repo, registryUrl }: DigestConfig, + newValue?: string +): Promise<string | null> { + if (newValue?.length) { + return getTagCommit(registryUrl, repo, newValue); + } + + const cacheKey = getCacheKey(registryUrl, repo, 'digest'); + const cachedResult = await packageCache.get<string>(cacheNamespace, cacheKey); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + + const branchCacheKey = getCacheKey(registryUrl, repo, 'mainbranch'); + let mainBranch = await packageCache.get<string>( + cacheNamespace, + branchCacheKey + ); + if (!mainBranch) { + mainBranch = ( + await bitbucketHttp.getJson<utils.RepoInfoBody>( + `/2.0/repositories/${repo}` + ) + ).body.mainbranch.name; + await packageCache.set(cacheNamespace, branchCacheKey, mainBranch, 60); + } + + const url = `/2.0/repositories/${repo}/commits/${mainBranch}`; + const bitbucketCommits = ( + await bitbucketHttp.getJson<utils.PagedResult<BitbucketCommit>>(url) + ).body; + + if (bitbucketCommits.values.length === 0) { + return null; + } + + const latestCommit = bitbucketCommits.values[0].hash; + + const cacheMinutes = 10; + await packageCache.set(cacheNamespace, cacheKey, latestCommit, cacheMinutes); + + return latestCommit; +} diff --git a/lib/datasource/bitbucket-tags/types.ts b/lib/datasource/bitbucket-tags/types.ts new file mode 100644 index 0000000000..d301617146 --- /dev/null +++ b/lib/datasource/bitbucket-tags/types.ts @@ -0,0 +1,12 @@ +export type BitbucketTag = { + name: string; + target?: { + date?: string; + hash: string; + }; +}; + +export type BitbucketCommit = { + hash: string; + date?: string; +}; diff --git a/lib/datasource/go/__snapshots__/index.spec.ts.snap b/lib/datasource/go/__snapshots__/index.spec.ts.snap index 585f8e427f..7a5a6c3853 100644 --- a/lib/datasource/go/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/go/__snapshots__/index.spec.ts.snap @@ -1,5 +1,19 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`datasource/go getDigest gitlab digest is not supported at the moment 1`] = ` +Array [ + Object { + "headers": Object { + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/golang/text?go-get=1", + }, +] +`; + exports[`datasource/go getDigest returns digest 1`] = ` Array [ Object { @@ -52,6 +66,33 @@ Array [ ] `; +exports[`datasource/go getDigest support bitbucket digest 1`] = `"123"`; + +exports[`datasource/go getDigest support bitbucket digest 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/golang/text", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/golang/text/commits/master", + }, +] +`; + exports[`datasource/go getReleases falls back to old behaviour 1`] = ` Array [ Object { @@ -236,6 +277,37 @@ Array [ ] `; +exports[`datasource/go getReleases support bitbucket tags 1`] = ` +Object { + "releases": Array [ + Object { + "gitRef": "v1.0.0", + "version": "v1.0.0", + }, + Object { + "gitRef": "v2.0.0", + "version": "v2.0.0", + }, + ], + "sourceUrl": "https://bitbucket.org/golang/text", +} +`; + +exports[`datasource/go getReleases support bitbucket tags 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "api.bitbucket.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://api.bitbucket.org/2.0/repositories/golang/text/refs/tags", + }, +] +`; + exports[`datasource/go getReleases support ghe 1`] = ` Object { "releases": Array [ @@ -326,6 +398,22 @@ Array [ ] `; +exports[`datasource/go getReleases unknown datasource returns null 1`] = `null`; + +exports[`datasource/go getReleases unknown datasource returns null 2`] = ` +Array [ + Object { + "headers": Object { + "accept-encoding": "gzip, deflate", + "host": "some.unknown.website", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://some.unknown.website/example/module?go-get=1", + }, +] +`; + exports[`datasource/go getReleases works for known servers 1`] = ` Array [ Object { diff --git a/lib/datasource/go/index.spec.ts b/lib/datasource/go/index.spec.ts index 7bc5fbe512..76492d4ba9 100644 --- a/lib/datasource/go/index.spec.ts +++ b/lib/datasource/go/index.spec.ts @@ -57,6 +57,18 @@ describe('datasource/go', () => { expect(res).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('gitlab digest is not supported at the moment', async () => { + httpMock + .scope('https://gitlab.com/') + .get('/golang/text?go-get=1') + .reply(200, ''); + const res = await getDigest( + { lookupName: 'gitlab.com/golang/text' }, + null + ); + expect(res).toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); it('returns digest', async () => { httpMock .scope('https://golang.org/') @@ -70,6 +82,35 @@ describe('datasource/go', () => { expect(res).toBe('abcdefabcdefabcdefabcdef'); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('support bitbucket digest', async () => { + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/golang/text') + .reply(200, { mainbranch: { name: 'master' } }); + httpMock + .scope('https://api.bitbucket.org') + .get('/2.0/repositories/golang/text/commits/master') + .reply(200, { + pagelen: 1, + values: [ + { + hash: '123', + date: '2020-11-19T09:05:35+00:00', + }, + ], + page: 1, + }); + const res = await getDigest( + { + lookupName: 'bitbucket.org/golang/text', + }, + null + ); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(res).toBeDefined(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); }); describe('getReleases', () => { it('returns null for empty result', async () => { @@ -152,6 +193,37 @@ describe('datasource/go', () => { expect(res).toBeDefined(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('support bitbucket tags', async () => { + httpMock + .scope('https://api.bitbucket.org/') + .get('/2.0/repositories/golang/text/refs/tags') + .reply(200, { + pagelen: 2, + page: 1, + values: [{ name: 'v1.0.0' }, { name: 'v2.0.0' }], + }); + const res = await getPkgReleases({ + datasource, + depName: 'bitbucket.org/golang/text', + }); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(res).toBeDefined(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + it('unknown datasource returns null', async () => { + httpMock + .scope('https://some.unknown.website/') + .get('/example/module?go-get=1') + .reply(404); + const res = await getPkgReleases({ + datasource, + depName: 'some.unknown.website/example/module', + }); + expect(res).toMatchSnapshot(); + expect(res).toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); it('support ghe', async () => { httpMock .scope('https://git.enterprise.com/') diff --git a/lib/datasource/go/index.ts b/lib/datasource/go/index.ts index e249819088..88bf12a1b7 100644 --- a/lib/datasource/go/index.ts +++ b/lib/datasource/go/index.ts @@ -2,6 +2,7 @@ import URL from 'url'; import { logger } from '../../logger'; import { Http } from '../../util/http'; import { regEx } from '../../util/regex'; +import * as bitbucket from '../bitbucket-tags'; import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common'; import * as github from '../github-tags'; import * as gitlab from '../gitlab-tags'; @@ -37,6 +38,15 @@ async function getDatasource(goModule: string): Promise<DataSource | null> { }; } + if (goModule.startsWith('bitbucket.org/')) { + const split = goModule.split('/'); + const lookupName = split[1] + '/' + split[2]; + return { + datasource: bitbucket.id, + lookupName, + }; + } + const pkgUrl = `https://${goModule}?go-get=1`; const res = (await http.get(pkgUrl)).body; const sourceMatch = regEx( @@ -113,13 +123,27 @@ export async function getReleases({ }: GetReleasesConfig): Promise<ReleaseResult | null> { logger.trace(`go.getReleases(${lookupName})`); const source = await getDatasource(lookupName); - if (source?.datasource !== github.id && source?.datasource !== gitlab.id) { - return null; + let res = null; + + switch (source.datasource) { + case github.id: { + res = await github.getReleases(source); + break; + } + case gitlab.id: { + res = await gitlab.getReleases(source); + break; + } + case bitbucket.id: { + res = await bitbucket.getReleases(source); + break; + } + /* istanbul ignore next: can never happen, makes lint happy */ + default: { + return null; + } } - const res = - source.datasource === github.id - ? await github.getReleases(source) - : await gitlab.getReleases(source); + // istanbul ignore if if (!res) { return res; @@ -172,11 +196,23 @@ export async function getDigest( value?: string ): Promise<string | null> { const source = await getDatasource(lookupName); - if (source && source.datasource === github.id) { - // ignore v0.0.0- pseudo versions that are used Go Modules - look up default branch instead - const tag = value && !value.startsWith('v0.0.0-2') ? value : undefined; - const digest = await github.getDigest(source, tag); - return digest; + if (!source) { + return null; + } + + // ignore v0.0.0- pseudo versions that are used Go Modules - look up default branch instead + const tag = value && !value.startsWith('v0.0.0-2') ? value : undefined; + + switch (source.datasource) { + case github.id: { + return github.getDigest(source, tag); + } + case bitbucket.id: { + return bitbucket.getDigest(source, tag); + } + /* istanbul ignore next: can never happen, makes lint happy */ + default: { + return null; + } } - return null; } diff --git a/lib/platform/bitbucket/index.ts b/lib/platform/bitbucket/index.ts index bf09e2ca7b..f0edfd32c8 100644 --- a/lib/platform/bitbucket/index.ts +++ b/lib/platform/bitbucket/index.ts @@ -186,7 +186,7 @@ export async function initRepo({ // Returns true if repository has rule enforcing PRs are up-to-date with base branch before merging export function getRepoForceRebase(): Promise<boolean> { - // BB doesnt have an option to flag staled branches + // BB doesn't have an option to flag staled branches return Promise.resolve(false); } diff --git a/lib/util/http/bitbucket.ts b/lib/util/http/bitbucket.ts index f64653e84a..db74f462b0 100644 --- a/lib/util/http/bitbucket.ts +++ b/lib/util/http/bitbucket.ts @@ -2,6 +2,7 @@ import { PLATFORM_TYPE_BITBUCKET } from '../../constants/platforms'; import { Http, HttpOptions, HttpResponse, InternalHttpOptions } from '.'; let baseUrl = 'https://api.bitbucket.org/'; + export const setBaseUrl = (url: string): void => { baseUrl = url; }; -- GitLab