diff --git a/lib/config/presets/gitlab/index.ts b/lib/config/presets/gitlab/index.ts index ee3d30a98663f835b946a45470402ec6d79902d3..a3a742978fde085b4668092368bb8639b08c2eab 100644 --- a/lib/config/presets/gitlab/index.ts +++ b/lib/config/presets/gitlab/index.ts @@ -1,9 +1,9 @@ import { logger } from '../../../logger'; -import { api } from '../../../platform/gitlab/gl-got-wrapper'; +import { GitlabHttp } from '../../../util/http/gitlab'; import { ensureTrailingSlash } from '../../../util/url'; import { Preset, PresetConfig } from '../common'; -const { get: glGot } = api; +const gitlabApi = new GitlabHttp(); async function getDefaultBranchName( urlEncodedPkgName: string, @@ -15,7 +15,7 @@ async function getDefaultBranchName( name: string; }[]; - const res = await glGot<GlBranch>(branchesUrl); + const res = await gitlabApi.getJson<GlBranch>(branchesUrl); const branches = res.body; let defautlBranchName = 'master'; for (const branch of branches) { @@ -52,7 +52,7 @@ export async function getPresetFromEndpoint( const presetUrl = `${endpoint}projects/${urlEncodedPkgName}/repository/files/renovate.json?ref=${defautlBranchName}`; res = Buffer.from( - (await glGot(presetUrl)).body.content, + (await gitlabApi.getJson<{ content: string }>(presetUrl)).body.content, 'base64' ).toString(); } catch (err) { diff --git a/lib/datasource/gitlab-tags/index.ts b/lib/datasource/gitlab-tags/index.ts index 46d086cf1676b1b107cecf01154eabbc37cfd389..15411d83cf8c0c013ded9d40472beee3576ff6bb 100644 --- a/lib/datasource/gitlab-tags/index.ts +++ b/lib/datasource/gitlab-tags/index.ts @@ -1,10 +1,10 @@ import is from '@sindresorhus/is'; import { logger } from '../../logger'; -import { api } from '../../platform/gitlab/gl-got-wrapper'; import * as globalCache from '../../util/cache/global'; +import { GitlabHttp } from '../../util/http/gitlab'; import { GetReleasesConfig, ReleaseResult } from '../common'; -const { get: glGot } = api; +const gitlabApi = new GitlabHttp(); export const id = 'gitlab-tags'; @@ -46,7 +46,7 @@ export async function getReleases({ const url = `${depHost}/api/v4/projects/${urlEncodedRepo}/repository/tags?per_page=100`; gitlabTags = ( - await glGot<GitlabTag[]>(url, { + await gitlabApi.getJson<GitlabTag[]>(url, { paginate: true, }) ).body; diff --git a/lib/platform/gitlab/gl-got-wrapper.ts b/lib/platform/gitlab/gl-got-wrapper.ts deleted file mode 100644 index 60e69d479bb401e4479bf90ba0ee40864adc0177..0000000000000000000000000000000000000000 --- a/lib/platform/gitlab/gl-got-wrapper.ts +++ /dev/null @@ -1,86 +0,0 @@ -import parseLinkHeader from 'parse-link-header'; - -import { PLATFORM_FAILURE } from '../../constants/error-messages'; -import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; -import { logger } from '../../logger'; -import got from '../../util/got'; -import { GotApi, GotResponse } from '../common'; - -const hostType = PLATFORM_TYPE_GITLAB; -let baseUrl = 'https://gitlab.com/api/v4/'; - -async function get(path: string, options: any): Promise<GotResponse> { - const opts = { - hostType, - baseUrl, - json: true, - ...options, - }; - try { - const res = await got(path, opts); - if (opts.paginate) { - // Check if result is paginated - try { - const linkHeader = parseLinkHeader(res.headers.link as string); - if (linkHeader && linkHeader.next) { - res.body = res.body.concat( - (await get(linkHeader.next.url, opts)).body - ); - } - } catch (err) /* istanbul ignore next */ { - logger.warn({ err }, 'Pagination error'); - } - } - return res; - } catch (err) /* istanbul ignore next */ { - if (err.statusCode === 404) { - logger.trace({ err }, 'GitLab 404'); - logger.debug({ url: err.url }, 'GitLab API 404'); - throw err; - } - logger.debug({ err }, 'Gitlab API error'); - if ( - err.statusCode === 429 || - (err.statusCode >= 500 && err.statusCode < 600) - ) { - throw new Error(PLATFORM_FAILURE); - } - const platformFailureCodes = [ - 'EAI_AGAIN', - 'ECONNRESET', - 'ETIMEDOUT', - 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', - ]; - if (platformFailureCodes.includes(err.code)) { - throw new Error(PLATFORM_FAILURE); - } - if (err.name === 'ParseError') { - throw new Error(PLATFORM_FAILURE); - } - throw err; - } -} - -const helpers = ['get', 'post', 'put', 'patch', 'head', 'delete']; - -interface GlGotApi - extends GotApi<{ - paginate?: boolean; - token?: string; - }> { - setBaseUrl(url: string): void; -} - -export const api: GlGotApi = {} as any; - -for (const x of helpers) { - (api as any)[x] = (url: string, opts: any): Promise<GotResponse> => - get(url, { ...opts, method: x.toUpperCase() }); -} - -// eslint-disable-next-line @typescript-eslint/unbound-method -api.setBaseUrl = (e: string): void => { - baseUrl = e; -}; - -export default api; diff --git a/lib/platform/gitlab/index.ts b/lib/platform/gitlab/index.ts index 071932b999005ccd6ca42b1f9e0fd27aa447d2b5..486dc278d64917db3808d74eaa132d886a326138 100644 --- a/lib/platform/gitlab/index.ts +++ b/lib/platform/gitlab/index.ts @@ -19,6 +19,8 @@ import { PR_STATE_ALL, PR_STATE_OPEN } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; import * as hostRules from '../../util/host-rules'; +import { HttpResponse } from '../../util/http'; +import { GitlabHttp, setBaseUrl } from '../../util/http/gitlab'; import { sanitize } from '../../util/sanitize'; import { ensureTrailingSlash } from '../../util/url'; import { @@ -29,7 +31,6 @@ import { EnsureCommentRemovalConfig, EnsureIssueConfig, FindPRConfig, - GotResponse, Issue, PlatformConfig, Pr, @@ -39,7 +40,8 @@ import { } from '../common'; import GitStorage, { StatusResult } from '../git/storage'; import { smartTruncate } from '../utils/pr-body'; -import { api } from './gl-got-wrapper'; + +const gitlabApi = new GitlabHttp(); type MergeMethod = 'merge' | 'rebase_merge' | 'ff'; const defaultConfigFile = configFileNames[0]; @@ -76,13 +78,18 @@ export async function initPlatform({ } if (endpoint) { defaults.endpoint = ensureTrailingSlash(endpoint); - api.setBaseUrl(defaults.endpoint); + setBaseUrl(defaults.endpoint); } else { logger.debug('Using default GitLab endpoint: ' + defaults.endpoint); } let gitAuthor: string; try { - const user = (await api.get(`user`, { token })).body; + const user = ( + await gitlabApi.getJson<{ email: string; name: string; id: number }>( + `user`, + { token } + ) + ).body; gitAuthor = `${user.name} <${user.email}>`; authorId = user.id; } catch (err) { @@ -104,11 +111,12 @@ export async function getRepos(): Promise<string[]> { logger.debug('Autodiscovering GitLab repositories'); try { const url = `projects?membership=true&per_page=100&with_merge_requests_enabled=true&min_access_level=30`; - const res = await api.get(url, { paginate: true }); - logger.debug(`Discovered ${res.body.length} project(s)`); - return res.body.map( - (repo: { path_with_namespace: string }) => repo.path_with_namespace + const res = await gitlabApi.getJson<{ path_with_namespace: string }[]>( + url, + { paginate: true } ); + logger.debug(`Discovered ${res.body.length} project(s)`); + return res.body.map((repo) => repo.path_with_namespace); } catch (err) { logger.error({ err }, `GitLab getRepos error`); throw err; @@ -141,7 +149,7 @@ export async function initRepo({ config.gitPrivateKey = gitPrivateKey; config.localDir = localDir; - let res: GotResponse<{ + type RepoResponse = { archived: boolean; mirror: boolean; default_branch: string; @@ -151,9 +159,12 @@ export async function initRepo({ repository_access_level: 'disabled' | 'private' | 'enabled'; merge_requests_access_level: 'disabled' | 'private' | 'enabled'; merge_method: MergeMethod; - }>; + }; + let res: HttpResponse<RepoResponse>; try { - res = await api.get(`projects/${config.repository}`); + res = await gitlabApi.getJson<RepoResponse>( + `projects/${config.repository}` + ); if (res.body.archived) { logger.debug( 'Repository is archived - throwing error to abort renovation' @@ -187,7 +198,7 @@ export async function initRepo({ renovateConfig = JSON.parse( Buffer.from( ( - await api.get( + await gitlabApi.getJson<{ content: string }>( `projects/${config.repository}/repository/files/${defaultConfigFile}?ref=${res.body.default_branch}` ) ).body.content, @@ -206,7 +217,9 @@ export async function initRepo({ config.mergeMethod = res.body.merge_method || 'merge'; logger.debug(`${repository} default branch = ${config.baseBranch}`); // Discover our user email - config.email = (await api.get(`user`)).body.email; + config.email = ( + await gitlabApi.getJson<{ email: string }>(`user`) + ).body.email; logger.debug('Bot email=' + config.email); delete config.prList; logger.debug('Enabling Git FS'); @@ -311,7 +324,12 @@ async function getStatus( const branchSha = await config.storage.getBranchCommit(branchName); const url = `projects/${config.repository}/repository/commits/${branchSha}/statuses`; - return (await api.get(url, { paginate: true, useCache })).body; + return ( + await gitlabApi.getJson<GitlabBranchStatus[]>(url, { + paginate: true, + useCache, + }) + ).body; } const gitlabToRenovateStatusMapping: Record<string, BranchStatus> = { @@ -390,16 +408,19 @@ export async function createPr({ ? config.defaultBranch : config.baseBranch; logger.debug(`Creating Merge Request: ${title}`); - const res = await api.post(`projects/${config.repository}/merge_requests`, { - body: { - source_branch: branchName, - target_branch: targetBranch, - remove_source_branch: true, - title, - description, - labels: is.array(labels) ? labels.join(',') : null, - }, - }); + const res = await gitlabApi.postJson<Pr & { iid: number }>( + `projects/${config.repository}/merge_requests`, + { + body: { + source_branch: branchName, + target_branch: targetBranch, + remove_source_branch: true, + title, + description, + labels: is.array(labels) ? labels.join(',') : null, + }, + } + ); const pr = res.body; pr.number = pr.iid; pr.branchName = branchName; @@ -416,9 +437,10 @@ export async function createPr({ // Check for correct merge request status before setting `merge_when_pipeline_succeeds` to `true`. for (let attempt = 1; attempt <= retryTimes; attempt += 1) { - const { body } = await api.get( - `projects/${config.repository}/merge_requests/${pr.iid}` - ); + const { body } = await gitlabApi.getJson<{ + merge_status: string; + pipeline: string; + }>(`projects/${config.repository}/merge_requests/${pr.iid}`); // Only continue if the merge request can be merged and has a pipeline. if (body.merge_status === desiredStatus && body.pipeline !== null) { break; @@ -426,7 +448,7 @@ export async function createPr({ await delay(500 * attempt); } - await api.put( + await gitlabApi.putJson( `projects/${config.repository}/merge_requests/${pr.iid}/merge`, { body: { @@ -446,7 +468,18 @@ export async function createPr({ export async function getPr(iid: number): Promise<Pr> { logger.debug(`getPr(${iid})`); const url = `projects/${config.repository}/merge_requests/${iid}?include_diverged_commits_count=1`; - const pr = (await api.get(url)).body; + const pr = ( + await gitlabApi.getJson< + Pr & { + iid: number; + source_branch: string; + target_branch: string; + description: string; + diverged_commits_count: number; + merge_status: string; + } + >(url) + ).body; // Harmonize fields with GitHub pr.branchName = pr.source_branch; pr.targetBranch = pr.target_branch; @@ -472,7 +505,9 @@ export async function getPr(iid: number): Promise<Pr> { config.repository }/repository/branches/${urlEscape(pr.source_branch)}`; try { - const branch = (await api.get(branchUrl)).body; + const branch = ( + await gitlabApi.getJson<{ commit: { author_email: string } }>(branchUrl) + ).body; const branchCommitEmail = branch && branch.commit ? branch.commit.author_email : null; // istanbul ignore if @@ -497,11 +532,14 @@ export async function getPr(iid: number): Promise<Pr> { // istanbul ignore next async function closePr(iid: number): Promise<void> { - await api.put(`projects/${config.repository}/merge_requests/${iid}`, { - body: { - state_event: 'close', - }, - }); + await gitlabApi.putJson( + `projects/${config.repository}/merge_requests/${iid}`, + { + body: { + state_event: 'close', + }, + } + ); } export async function updatePr( @@ -509,21 +547,27 @@ export async function updatePr( title: string, description: string ): Promise<void> { - await api.put(`projects/${config.repository}/merge_requests/${iid}`, { - body: { - title, - description: sanitize(description), - }, - }); + await gitlabApi.putJson( + `projects/${config.repository}/merge_requests/${iid}`, + { + body: { + title, + description: sanitize(description), + }, + } + ); } export async function mergePr(iid: number): Promise<boolean> { try { - await api.put(`projects/${config.repository}/merge_requests/${iid}/merge`, { - body: { - should_remove_source_branch: true, - }, - }); + await gitlabApi.putJson( + `projects/${config.repository}/merge_requests/${iid}/merge`, + { + body: { + should_remove_source_branch: true, + }, + } + ); return true; } catch (err) /* istanbul ignore next */ { if (err.statusCode === 401) { @@ -565,10 +609,12 @@ export async function getBranchPr(branchName: string): Promise<Pr> { source_branch: branchName, }).toString(); const urlString = `projects/${config.repository}/merge_requests?${query}`; - const res = await api.get(urlString, { paginate: true }); + const res = await gitlabApi.getJson<{ source_branch: string }[]>(urlString, { + paginate: true, + }); logger.debug(`Got res with ${res.body.length} results`); let pr: any = null; - res.body.forEach((result: { source_branch: string }) => { + res.body.forEach((result) => { if (result.source_branch === branchName) { pr = result; } @@ -677,7 +723,7 @@ export async function setBranchStatus({ options.target_url = targetUrl; } try { - await api.post(url, { body: options }); + await gitlabApi.postJson(url, { body: options }); // update status cache await getStatus(branchName, false); @@ -702,7 +748,7 @@ export async function setBranchStatus({ export async function getIssueList(): Promise<any[]> { if (!config.issueList) { - const res = await api.get( + const res = await gitlabApi.getJson<{ iid: number; title: string }[]>( `projects/${config.repository}/issues?state=opened`, { useCache: false, @@ -713,7 +759,7 @@ export async function getIssueList(): Promise<any[]> { logger.warn({ responseBody: res.body }, 'Could not retrieve issue list'); return []; } - config.issueList = res.body.map((i: { iid: number; title: string }) => ({ + config.issueList = res.body.map((i) => ({ iid: i.iid, title: i.title, })); @@ -730,7 +776,9 @@ export async function findIssue(title: string): Promise<Issue | null> { return null; } const issueBody = ( - await api.get(`projects/${config.repository}/issues/${issue.iid}`) + await gitlabApi.getJson<{ description: string }>( + `projects/${config.repository}/issues/${issue.iid}` + ) ).body.description; return { number: issue.iid, @@ -753,17 +801,22 @@ export async function ensureIssue({ const issue = issueList.find((i: { title: string }) => i.title === title); if (issue) { const existingDescription = ( - await api.get(`projects/${config.repository}/issues/${issue.iid}`) + await gitlabApi.getJson<{ description: string }>( + `projects/${config.repository}/issues/${issue.iid}` + ) ).body.description; if (existingDescription !== description) { logger.debug('Updating issue body'); - await api.put(`projects/${config.repository}/issues/${issue.iid}`, { - body: { description }, - }); + await gitlabApi.putJson( + `projects/${config.repository}/issues/${issue.iid}`, + { + body: { description }, + } + ); return 'updated'; } } else { - await api.post(`projects/${config.repository}/issues`, { + await gitlabApi.postJson(`projects/${config.repository}/issues`, { body: { title, description, @@ -790,9 +843,12 @@ export async function ensureIssueClosing(title: string): Promise<void> { for (const issue of issueList) { if (issue.title === title) { logger.debug({ issue }, 'Closing issue'); - await api.put(`projects/${config.repository}/issues/${issue.iid}`, { - body: { state_event: 'close' }, - }); + await gitlabApi.putJson( + `projects/${config.repository}/issues/${issue.iid}`, + { + body: { state_event: 'close' }, + } + ); } } } @@ -803,19 +859,25 @@ export async function addAssignees( ): Promise<void> { logger.debug(`Adding assignees ${assignees} to #${iid}`); try { - let assigneeId = (await api.get(`users?username=${assignees[0]}`)).body[0] - .id; + let assigneeId = ( + await gitlabApi.getJson<{ id: number }[]>( + `users?username=${assignees[0]}` + ) + ).body[0].id; let url = `projects/${config.repository}/merge_requests/${iid}?assignee_id=${assigneeId}`; - await api.put(url); + await gitlabApi.putJson(url); try { if (assignees.length > 1) { url = `projects/${config.repository}/merge_requests/${iid}?assignee_ids[]=${assigneeId}`; for (let i = 1; i < assignees.length; i += 1) { - assigneeId = (await api.get(`users?username=${assignees[i]}`)).body[0] - .id; + assigneeId = ( + await gitlabApi.getJson<{ id: number }[]>( + `users?username=${assignees[i]}` + ) + ).body[0].id; url += `&assignee_ids[]=${assigneeId}`; } - await api.put(url); + await gitlabApi.putJson(url); } } catch (error) { logger.error({ iid, assignees }, 'Failed to add multiple assignees'); @@ -840,9 +902,12 @@ export async function deleteLabel( try { const pr = await getPr(issueNo); const labels = (pr.labels || []).filter((l: string) => l !== label).join(); - await api.put(`projects/${config.repository}/merge_requests/${issueNo}`, { - body: { labels }, - }); + await gitlabApi.putJson( + `projects/${config.repository}/merge_requests/${issueNo}`, + { + body: { labels }, + } + ); } catch (err) /* istanbul ignore next */ { logger.warn({ err, issueNo, label }, 'Failed to delete label'); } @@ -852,14 +917,16 @@ async function getComments(issueNo: number): Promise<GitlabComment[]> { // GET projects/:owner/:repo/merge_requests/:number/notes logger.debug(`Getting comments for #${issueNo}`); const url = `projects/${config.repository}/merge_requests/${issueNo}/notes`; - const comments = (await api.get(url, { paginate: true })).body; + const comments = ( + await gitlabApi.getJson<GitlabComment[]>(url, { paginate: true }) + ).body; logger.debug(`Found ${comments.length} comments`); return comments; } async function addComment(issueNo: number, body: string): Promise<void> { // POST projects/:owner/:repo/merge_requests/:number/notes - await api.post( + await gitlabApi.postJson( `projects/${config.repository}/merge_requests/${issueNo}/notes`, { body: { body }, @@ -873,7 +940,7 @@ async function editComment( body: string ): Promise<void> { // PUT projects/:owner/:repo/merge_requests/:number/notes/:id - await api.put( + await gitlabApi.putJson( `projects/${config.repository}/merge_requests/${issueNo}/notes/${commentId}`, { body: { body }, @@ -886,7 +953,7 @@ async function deleteComment( commentId: number ): Promise<void> { // DELETE projects/:owner/:repo/merge_requests/:number/notes/:id - await api.delete( + await gitlabApi.deleteJson( `projects/${config.repository}/merge_requests/${issueNo}/notes/${commentId}` ); } @@ -975,20 +1042,6 @@ export async function ensureCommentRemoval({ } } -const mapPullRequests = (pr: { - iid: number; - source_branch: string; - title: string; - state: string; - created_at: string; -}): Pr => ({ - number: pr.iid, - branchName: pr.source_branch, - title: pr.title, - state: pr.state === 'opened' ? PR_STATE_OPEN : pr.state, - createdAt: pr.created_at, -}); - async function fetchPrList(): Promise<Pr[]> { const query = new URLSearchParams({ per_page: '100', @@ -996,8 +1049,22 @@ async function fetchPrList(): Promise<Pr[]> { }).toString(); const urlString = `projects/${config.repository}/merge_requests?${query}`; try { - const res = await api.get(urlString, { paginate: true }); - return res.body.map(mapPullRequests); + const res = await gitlabApi.getJson< + { + iid: number; + source_branch: string; + title: string; + state: string; + created_at: string; + }[] + >(urlString, { paginate: true }); + return res.body.map((pr) => ({ + number: pr.iid, + branchName: pr.source_branch, + title: pr.title, + state: pr.state === 'opened' ? PR_STATE_OPEN : pr.state, + createdAt: pr.created_at, + })); } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'Error fetching PR list'); if (err.statusCode === 403) { diff --git a/lib/platform/gitlab/__snapshots__/gl-got-wrapper.spec.ts.snap b/lib/util/http/__snapshots__/gitlab.spec.ts.snap similarity index 90% rename from lib/platform/gitlab/__snapshots__/gl-got-wrapper.spec.ts.snap rename to lib/util/http/__snapshots__/gitlab.spec.ts.snap index 728ddbe5b3eab1958ca0c089d15b797d49185592..da14991173d2f0dfdfd3ce114403da5f666947af 100644 --- a/lib/platform/gitlab/__snapshots__/gl-got-wrapper.spec.ts.snap +++ b/lib/util/http/__snapshots__/gitlab.spec.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`platform/gitlab/gl-got-wrapper attempts to paginate 1`] = ` +exports[`util/http/gitlab attempts to paginate 1`] = ` Array [ Object { "headers": Object { @@ -16,7 +16,7 @@ Array [ ] `; -exports[`platform/gitlab/gl-got-wrapper paginates 1`] = ` +exports[`util/http/gitlab paginates 1`] = ` Array [ Object { "headers": Object { @@ -54,7 +54,7 @@ Array [ ] `; -exports[`platform/gitlab/gl-got-wrapper posts 1`] = ` +exports[`util/http/gitlab posts 1`] = ` Array [ Object { "headers": Object { diff --git a/lib/platform/gitlab/gl-got-wrapper.spec.ts b/lib/util/http/gitlab.spec.ts similarity index 74% rename from lib/platform/gitlab/gl-got-wrapper.spec.ts rename to lib/util/http/gitlab.spec.ts index 9d61f03f4ae091a770b95d3cc77b958098bfe377..04fe86fc3f52c69588cc89f98f0c12fe2f466d1d 100644 --- a/lib/platform/gitlab/gl-got-wrapper.spec.ts +++ b/lib/util/http/gitlab.spec.ts @@ -1,7 +1,8 @@ import * as httpMock from '../../../test/httpMock'; +import { getName } from '../../../test/util'; import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; -import * as hostRules from '../../util/host-rules'; -import { api } from './gl-got-wrapper'; +import * as hostRules from '../host-rules'; +import { GitlabHttp, setBaseUrl } from './gitlab'; hostRules.add({ hostType: PLATFORM_TYPE_GITLAB, @@ -10,16 +11,20 @@ hostRules.add({ const gitlabApiHost = 'https://gitlab.com'; -describe('platform/gitlab/gl-got-wrapper', () => { - const body = ['a', 'b']; +describe(getName(__filename), () => { + let gitlabApi: GitlabHttp; + beforeEach(() => { - // (delay as any).mockImplementation(() => Promise.resolve()); + gitlabApi = new GitlabHttp(); + setBaseUrl(`${gitlabApiHost}/api/v4/`); httpMock.setup(); }); + afterEach(() => { jest.resetAllMocks(); httpMock.reset(); }); + it('paginates', async () => { httpMock .scope(gitlabApiHost) @@ -35,7 +40,7 @@ describe('platform/gitlab/gl-got-wrapper', () => { }) .get('/api/v4/some-url&page=3') .reply(200, ['d']); - const res = await api.get('/some-url', { paginate: true }); + const res = await gitlabApi.getJson('some-url', { paginate: true }); expect(res.body).toHaveLength(4); const trace = httpMock.getTrace(); @@ -46,7 +51,7 @@ describe('platform/gitlab/gl-got-wrapper', () => { httpMock.scope(gitlabApiHost).get('/api/v4/some-url').reply(200, ['a'], { link: '<https://gitlab.com/api/v4/some-url&page=3>; rel="last"', }); - const res = await api.get('/some-url', { paginate: true }); + const res = await gitlabApi.getJson('some-url', { paginate: true }); expect(res.body).toHaveLength(1); const trace = httpMock.getTrace(); @@ -54,14 +59,15 @@ describe('platform/gitlab/gl-got-wrapper', () => { expect(trace).toMatchSnapshot(); }); it('posts', async () => { + const body = ['a', 'b']; httpMock.scope(gitlabApiHost).post('/api/v4/some-url').reply(200, body); - const res = await api.post('/some-url'); + const res = await gitlabApi.postJson('some-url'); expect(res.body).toEqual(body); expect(httpMock.getTrace()).toMatchSnapshot(); }); it('sets baseUrl', () => { expect(() => - api.setBaseUrl('https://gitlab.renovatebot.com/api/v4/') + setBaseUrl('https://gitlab.renovatebot.com/api/v4/') ).not.toThrow(); }); }); diff --git a/lib/util/http/gitlab.ts b/lib/util/http/gitlab.ts new file mode 100644 index 0000000000000000000000000000000000000000..9a5d0f160509da53281bd33b21395a60fdfc493d --- /dev/null +++ b/lib/util/http/gitlab.ts @@ -0,0 +1,83 @@ +import { URL } from 'url'; +import parseLinkHeader from 'parse-link-header'; +import { PLATFORM_FAILURE } from '../../constants/error-messages'; +import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; +import { logger } from '../../logger'; +import { Http, HttpResponse, InternalHttpOptions } from '.'; + +let baseUrl = 'https://gitlab.com/api/v4/'; +export const setBaseUrl = (url: string): void => { + baseUrl = url; +}; + +interface GitlabInternalOptions extends InternalHttpOptions { + body?: string; +} + +export interface GitlabHttpOptions extends InternalHttpOptions { + paginate?: boolean; + token?: string; +} + +export class GitlabHttp extends Http<GitlabHttpOptions, GitlabHttpOptions> { + constructor(options?: GitlabHttpOptions) { + super(PLATFORM_TYPE_GITLAB, options); + } + + protected async request<T>( + url: string | URL, + options?: GitlabInternalOptions & GitlabHttpOptions + ): Promise<HttpResponse<T> | null> { + let result = null; + + const opts = { + baseUrl, + ...options, + throwHttpErrors: true, + }; + + try { + result = await super.request<T>(url, opts); + if (opts.paginate) { + // Check if result is paginated + try { + const linkHeader = parseLinkHeader(result.headers.link as string); + if (linkHeader && linkHeader.next) { + result.body = result.body.concat( + (await this.request<T>(linkHeader.next.url, opts)).body + ); + } + } catch (err) /* istanbul ignore next */ { + logger.warn({ err }, 'Pagination error'); + } + } + return result; + } catch (err) /* istanbul ignore next */ { + if (err.statusCode === 404) { + logger.trace({ err }, 'GitLab 404'); + logger.debug({ url: err.url }, 'GitLab API 404'); + throw err; + } + logger.debug({ err }, 'Gitlab API error'); + if ( + err.statusCode === 429 || + (err.statusCode >= 500 && err.statusCode < 600) + ) { + throw new Error(PLATFORM_FAILURE); + } + const platformFailureCodes = [ + 'EAI_AGAIN', + 'ECONNRESET', + 'ETIMEDOUT', + 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + ]; + if (platformFailureCodes.includes(err.code)) { + throw new Error(PLATFORM_FAILURE); + } + if (err.name === 'ParseError') { + throw new Error(PLATFORM_FAILURE); + } + throw err; + } + } +} diff --git a/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap b/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap index ac1b2505c7c6fec320e35867bc953b54a228c938..5f394fc746a9b6143d26e9c5a591cbb03102cf50 100644 --- a/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap +++ b/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap @@ -1,5 +1,152 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`workers/pr/changelog getChangeLogJSON handles empty GitLab tags response 1`] = ` +Object { + "hasReleaseNotes": false, + "project": Object { + "apiBaseUrl": "https://gitlab.com/api/v4/", + "baseUrl": "https://gitlab.com/", + "depName": "renovate", + "gitlab": "meno/dropzone", + "repository": "https://gitlab.com/meno/dropzone/", + }, + "versions": Array [ + Object { + "changes": Array [], + "compare": Object {}, + "date": undefined, + "releaseNotes": null, + "version": "5.6.1", + }, + Object { + "changes": Array [], + "compare": Object {}, + "date": "2020-02-13T15:37:00.000Z", + "releaseNotes": null, + "version": "5.6.0", + }, + Object { + "changes": Array [], + "compare": Object {}, + "date": undefined, + "releaseNotes": null, + "version": "5.5.0", + }, + Object { + "changes": Array [], + "compare": Object {}, + "date": "2018-08-24T14:23:00.000Z", + "releaseNotes": null, + "version": "5.4.0", + }, + ], +} +`; + +exports[`workers/pr/changelog getChangeLogJSON handles empty GitLab tags response 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2Fdropzone/repository/tags", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, +] +`; + exports[`workers/pr/changelog getChangeLogJSON supports gitlab enterprise and gitlab enterprise changelog 1`] = ` Object { "hasReleaseNotes": false, @@ -102,6 +249,110 @@ Object { } `; +exports[`workers/pr/changelog getChangeLogJSON uses GitLab tags 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2Fdropzone/repository/tags", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, +] +`; + exports[`workers/pr/changelog getChangeLogJSON uses GitLab tags with error 1`] = ` Object { "hasReleaseNotes": false, @@ -145,6 +396,110 @@ Object { } `; +exports[`workers/pr/changelog getChangeLogJSON uses GitLab tags with error 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2Fdropzone/repository/tags", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno/dropzone/repository/tree/", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "gitlab.com", + "private-token": "abc", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/projects/meno%2fdropzone/releases?per_page=100", + }, +] +`; + exports[`workers/pr/changelog getChangeLogJSON works without GitLab 1`] = ` Object { "hasReleaseNotes": false, diff --git a/lib/workers/pr/changelog/gitlab.spec.ts b/lib/workers/pr/changelog/gitlab.spec.ts index bd507911dce4a5a553aeb6715a14747551daabad..f069c020a560affc8d985cbb9011e44936b26232 100644 --- a/lib/workers/pr/changelog/gitlab.spec.ts +++ b/lib/workers/pr/changelog/gitlab.spec.ts @@ -1,16 +1,12 @@ -import { mocked } from '../../../../test/util'; +import * as httpMock from '../../../../test/httpMock'; import { PLATFORM_TYPE_GITLAB } from '../../../constants/platforms'; -import { api } from '../../../platform/gitlab/gl-got-wrapper'; import * as hostRules from '../../../util/host-rules'; import * as semverVersioning from '../../../versioning/semver'; import { BranchUpgradeConfig } from '../../common'; import { getChangeLogJSON } from '.'; -jest.mock('../../../../lib/platform/gitlab/gl-got-wrapper'); jest.mock('../../../../lib/datasource/npm'); -const glGot = mocked(api).get; - const upgrade: BranchUpgradeConfig = { branchName: undefined, endpoint: 'https://gitlab.com/api/v4/ ', @@ -32,27 +28,34 @@ const upgrade: BranchUpgradeConfig = { ], }; +const baseUrl = 'https://gitlab.com/'; + describe('workers/pr/changelog', () => { describe('getChangeLogJSON', () => { beforeEach(() => { - glGot.mockClear(); + httpMock.setup(); hostRules.clear(); hostRules.add({ hostType: PLATFORM_TYPE_GITLAB, - baseUrl: 'https://gitlab.com/', + baseUrl, token: 'abc', }); }); + afterEach(() => { + httpMock.reset(); + }); it('returns null if @types', async () => { + httpMock.scope(baseUrl); expect( await getChangeLogJSON({ ...upgrade, fromVersion: null, }) ).toBeNull(); - expect(glGot).toHaveBeenCalledTimes(0); + expect(httpMock.getTrace()).toBeEmpty(); }); it('returns null if fromVersion equals toVersion', async () => { + httpMock.scope(baseUrl); expect( await getChangeLogJSON({ ...upgrade, @@ -60,7 +63,7 @@ describe('workers/pr/changelog', () => { toVersion: '1.0.0', }) ).toBeNull(); - expect(glGot).toHaveBeenCalledTimes(0); + expect(httpMock.getTrace()).toBeEmpty(); }); it('skips invalid repos', async () => { expect( @@ -78,31 +81,65 @@ describe('workers/pr/changelog', () => { ).toMatchSnapshot(); }); it('uses GitLab tags', async () => { - glGot.mockResolvedValueOnce({ - body: [ + httpMock + .scope(baseUrl) + .get('/api/v4/projects/meno%2Fdropzone/repository/tags') + .reply(200, [ { name: 'v5.2.0' }, { name: 'v5.4.0' }, { name: 'v5.5.0' }, { name: 'v5.6.0' }, { name: 'v5.6.1' }, { name: 'v5.7.0' }, - ], - } as never); + ]) + .persist() + .get('/api/v4/projects/meno/dropzone/repository/tree/') + .reply(200, []) + .persist() + .get('/api/v4/projects/meno%2fdropzone/releases?per_page=100') + .reply(200, []); + expect( + await getChangeLogJSON({ + ...upgrade, + }) + ).toMatchSnapshot(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + it('handles empty GitLab tags response', async () => { + httpMock + .scope(baseUrl) + .get('/api/v4/projects/meno%2Fdropzone/repository/tags') + .reply(200, []) + .persist() + .get('/api/v4/projects/meno/dropzone/repository/tree/') + .reply(200, []) + .persist() + .get('/api/v4/projects/meno%2fdropzone/releases?per_page=100') + .reply(200, []); expect( await getChangeLogJSON({ ...upgrade, }) ).toMatchSnapshot(); + expect(httpMock.getTrace()).toMatchSnapshot(); }); it('uses GitLab tags with error', async () => { - glGot.mockImplementation(() => { - throw new Error('Unknown GitLab Repo'); - }); + httpMock + .scope(baseUrl) + .get('/api/v4/projects/meno%2Fdropzone/repository/tags') + .replyWithError('Unknown GitLab Repo') + .persist() + .get('/api/v4/projects/meno/dropzone/repository/tree/') + .reply(200, []) + .persist() + .get('/api/v4/projects/meno%2fdropzone/releases?per_page=100') + .reply(200, []); expect( await getChangeLogJSON({ ...upgrade, }) ).toMatchSnapshot(); + expect(httpMock.getTrace()).toMatchSnapshot(); }); it('handles no sourceUrl', async () => { expect( diff --git a/lib/workers/pr/changelog/release-notes.ts b/lib/workers/pr/changelog/release-notes.ts index 90026c665d648a444f440623ce08546010f74dbb..2ac8bae972b6177270958690f5a1144f89567957 100644 --- a/lib/workers/pr/changelog/release-notes.ts +++ b/lib/workers/pr/changelog/release-notes.ts @@ -4,18 +4,17 @@ import { linkify } from 'linkify-markdown'; import MarkdownIt from 'markdown-it'; import { logger } from '../../../logger'; -import { api as api_gitlab } from '../../../platform/gitlab/gl-got-wrapper'; import * as globalCache from '../../../util/cache/global'; import * as runCache from '../../../util/cache/run'; import { GithubHttp } from '../../../util/http/github'; +import { GitlabHttp } from '../../../util/http/gitlab'; import { ChangeLogNotes, ChangeLogResult } from './common'; -const { get: glGot } = api_gitlab; - const markdown = new MarkdownIt('zero'); markdown.enable(['heading', 'lheading']); -const http = new GithubHttp(); +const githubHttp = new GithubHttp(); +const gitlabHttp = new GitlabHttp(); export async function getReleaseList( apiBaseUrl: string, @@ -34,7 +33,7 @@ export async function getReleaseList( /\//g, '%2f' )}/releases?per_page=100`; - const res = await glGot< + const res = await gitlabHttp.getJson< { name: string; release: string; @@ -53,7 +52,7 @@ export async function getReleaseList( })); } url += `repos/${repository}/releases?per_page=100`; - const res = await http.getJson< + const res = await githubHttp.getJson< { html_url: string; id: number; @@ -206,11 +205,11 @@ export async function getReleaseNotesMdFileInner( if (apiBaseUrl.includes('gitlab')) { apiTree = apiPrefix + `projects/${repository}/repository/tree/`; apiFiles = apiPrefix + `projects/${repository}/repository/files/`; - filesRes = await glGot<{ name: string }[]>(apiTree); + filesRes = await gitlabHttp.getJson<{ name: string }[]>(apiTree); } else { apiTree = apiPrefix + `repos/${repository}/contents/`; apiFiles = apiTree; - filesRes = await http.getJson<{ name: string }[]>(apiTree); + filesRes = await githubHttp.getJson<{ name: string }[]>(apiTree); } const files = filesRes.body .map((f) => f.name) @@ -228,11 +227,11 @@ export async function getReleaseNotesMdFileInner( } let fileRes: { body: { content: string } }; if (apiBaseUrl.includes('gitlab')) { - fileRes = await glGot<{ content: string }>( + fileRes = await gitlabHttp.getJson<{ content: string }>( `${apiFiles}${changelogFile}?ref=master` ); } else { - fileRes = await http.getJson<{ content: string }>( + fileRes = await githubHttp.getJson<{ content: string }>( `${apiFiles}${changelogFile}` ); } diff --git a/lib/workers/pr/changelog/source-gitlab.ts b/lib/workers/pr/changelog/source-gitlab.ts index dd6bdd09b72d97cb7c5ca42f700d3c293f93c5ba..4826fe4dd612cb695d85f143ea1e8436e8f7d170 100644 --- a/lib/workers/pr/changelog/source-gitlab.ts +++ b/lib/workers/pr/changelog/source-gitlab.ts @@ -1,16 +1,16 @@ import URL from 'url'; import { Release } from '../../../datasource'; import { logger } from '../../../logger'; -import { api } from '../../../platform/gitlab/gl-got-wrapper'; import * as globalCache from '../../../util/cache/global'; import * as runCache from '../../../util/cache/run'; +import { GitlabHttp } from '../../../util/http/gitlab'; import { regEx } from '../../../util/regex'; import * as allVersioning from '../../../versioning'; import { BranchUpgradeConfig } from '../../common'; import { ChangeLogRelease, ChangeLogResult } from './common'; import { addReleaseNotes } from './release-notes'; -const { get: glGot } = api; +const gitlabHttp = new GitlabHttp(); const cacheNamespace = 'changelog-gitlab-release'; @@ -24,7 +24,7 @@ async function getTagsInner( const repoid = repository.replace(/\//g, '%2F'); url += `projects/${repoid}/repository/tags`; try { - const res = await glGot<{ name: string }[]>(url, { + const res = await gitlabHttp.getJson<{ name: string }[]>(url, { paginate: true, });