diff --git a/lib/workers/pr/changelog/github/index.ts b/lib/workers/pr/changelog/github/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..b24bcc61b6c4607e520e72a04c8d72f8c79df5a1 --- /dev/null +++ b/lib/workers/pr/changelog/github/index.ts @@ -0,0 +1,96 @@ +import changelogFilenameRegex from 'changelog-filename-regex'; +import { logger } from '../../../../logger'; +import { GithubGitBlob } from '../../../../types/platform/github'; +import { GithubHttp } from '../../../../util/http/github'; +import { ensureTrailingSlash } from '../../../../util/url'; +import { ChangeLogFile, ChangeLogNotes } from '../common'; + +const http = new GithubHttp(); + +export async function getTags( + endpoint: string, + repository: string +): Promise<string[]> { + logger.trace('github.getTags()'); + const url = `${endpoint}repos/${repository}/tags?per_page=100`; + try { + const res = await http.getJson<{ name: string }[]>(url, { + paginate: true, + }); + + const tags = res.body; + + if (!tags.length) { + logger.debug({ repository }, 'repository has no Github tags'); + } + + return tags.map((tag) => tag.name).filter(Boolean); + } catch (err) { + logger.debug({ sourceRepo: repository }, 'Failed to fetch Github tags'); + logger.debug({ err }); + // istanbul ignore if + if (err.message && err.message.includes('Bad credentials')) { + logger.warn('Bad credentials triggering tag fail lookup in changelog'); + throw err; + } + return []; + } +} + +export async function getReleaseNotesMd( + repository: string, + apiBaseUrl: string +): Promise<ChangeLogFile> | null { + logger.trace('github.getReleaseNotesMd()'); + const apiPrefix = `${ensureTrailingSlash(apiBaseUrl)}repos/${repository}`; + + const res = await http.getJson<{ name: string }[]>(`${apiPrefix}/contents/`); + + const files = res.body.filter((f) => changelogFilenameRegex.test(f.name)); + + if (!files.length) { + logger.trace('no changelog file found'); + return null; + } + const { name: changelogFile } = files.shift(); + /* istanbul ignore if */ + if (files.length > 1) { + logger.debug( + `Multiple candidates for changelog file, using ${changelogFile}` + ); + } + + const fileRes = await http.getJson<GithubGitBlob>( + `${apiPrefix}/contents/${changelogFile}` + ); + + const changelogMd = + Buffer.from(fileRes.body.content, 'base64').toString() + '\n#\n##'; + return { changelogFile, changelogMd }; +} + +export async function getReleaseList( + apiBaseUrl: string, + repository: string +): Promise<ChangeLogNotes[]> { + logger.trace('github.getReleaseList()'); + const url = `${ensureTrailingSlash( + apiBaseUrl + )}repos/${repository}/releases?per_page=100`; + const res = await http.getJson< + { + html_url: string; + id: number; + tag_name: string; + name: string; + body: string; + }[] + >(url); + return res.body.map((release) => ({ + url: release.html_url, + id: release.id, + tag: release.tag_name, + name: release.name, + body: release.body, + })); +} diff --git a/lib/workers/pr/changelog/gitlab/index.ts b/lib/workers/pr/changelog/gitlab/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..ddf9a644bfe0b940fa372f7845a6586c2fc4f6bc --- /dev/null +++ b/lib/workers/pr/changelog/gitlab/index.ts @@ -0,0 +1,102 @@ +import changelogFilenameRegex from 'changelog-filename-regex'; +import { logger } from '../../../../logger'; +import { GitlabTreeNode } from '../../../../types/platform/gitlab'; +import { GitlabHttp } from '../../../../util/http/gitlab'; +import { ensureTrailingSlash } from '../../../../util/url'; +import { ChangeLogFile, ChangeLogNotes } from '../common'; + +const http = new GitlabHttp(); + +function getRepoId(repository: string): string { + return repository.replace(/\//g, '%2f'); +} + +export async function getTags( + endpoint: string, + repository: string +): Promise<string[]> { + logger.trace('gitlab.getTags()'); + const url = `${ensureTrailingSlash(endpoint)}projects/${getRepoId( + repository + )}/repository/tags`; + try { + const res = await http.getJson<{ name: string }[]>(url); + + const tags = res.body; + + if (!tags.length) { + logger.debug({ sourceRepo: repository }, 'repository has no Gitlab tags'); + } + + return tags.map((tag) => tag.name).filter(Boolean); + } catch (err) { + logger.info({ sourceRepo: repository }, 'Failed to fetch Gitlab tags'); + // istanbul ignore if + if (err.message && err.message.includes('Bad credentials')) { + logger.warn('Bad credentials triggering tag fail lookup in changelog'); + throw err; + } + return []; + } +} + +export async function getReleaseNotesMd( + repository: string, + apiBaseUrl: string +): Promise<ChangeLogFile> | null { + logger.trace('gitlab.getReleaseNotesMd()'); + const apiPrefix = `${ensureTrailingSlash( + apiBaseUrl + )}projects/${repository}/repository/`; + + // https://docs.gitlab.com/13.2/ee/api/repositories.html#list-repository-tree + let files = (await http.getJson<GitlabTreeNode[]>(`${apiPrefix}tree/`)).body; + + files = files.filter((f) => changelogFilenameRegex.test(f.name)); + if (!files.length) { + logger.trace('no changelog file found'); + return null; + } + const { name: changelogFile } = files.shift(); + /* istanbul ignore if */ + if (files.length > 1) { + logger.debug( + `Multiple candidates for changelog file, using ${changelogFile}` + ); + } + + const fileRes = await http.getJson<{ content: string }>( + `${apiPrefix}files/${changelogFile}?ref=master` + ); + const changelogMd = + Buffer.from(fileRes.body.content, 'base64').toString() + '\n#\n##'; + return { changelogFile, changelogMd }; +} + +export async function getReleaseList( + apiBaseUrl: string, + repository: string +): Promise<ChangeLogNotes[]> { + logger.trace('gitlab.getReleaseNotesMd()'); + + const repoId = getRepoId(repository); + const apiUrl = `${ensureTrailingSlash( + apiBaseUrl + )}projects/${repoId}/releases`; + const res = await http.getJson< + { + name: string; + release: string; + description: string; + tag_name: string; + }[] + >(`${apiUrl}?per_page=100`, { + paginate: true, + }); + return res.body.map((release) => ({ + url: `${apiUrl}/${release.tag_name}`, + name: release.name, + body: release.description, + tag: release.tag_name, + })); +} diff --git a/lib/workers/pr/changelog/release-notes.ts b/lib/workers/pr/changelog/release-notes.ts index 09859f2fdffdfeb5a8366aa35640ea1c4fcdf976..861fc9ae6227f227f2f55dd01be20204b1525c81 100644 --- a/lib/workers/pr/changelog/release-notes.ts +++ b/lib/workers/pr/changelog/release-notes.ts @@ -1,21 +1,17 @@ import * as URL from 'url'; -import changelogFilenameRegex from 'changelog-filename-regex'; import { linkify } from 'linkify-markdown'; import MarkdownIt from 'markdown-it'; import { logger } from '../../../logger'; import * as memCache from '../../../util/cache/memory'; import * as packageCache from '../../../util/cache/package'; -import { GithubHttp } from '../../../util/http/github'; -import { GitlabHttp } from '../../../util/http/gitlab'; import { ChangeLogFile, ChangeLogNotes, ChangeLogResult } from './common'; +import * as github from './github'; +import * as gitlab from './gitlab'; const markdown = new MarkdownIt('zero'); markdown.enable(['heading', 'lheading']); -const githubHttp = new GithubHttp(); -const gitlabHttp = new GitlabHttp(); - export async function getReleaseList( apiBaseUrl: string, repository: string @@ -27,47 +23,10 @@ export async function getReleaseList( return []; } try { - let url = apiBaseUrl.replace(/\/?$/, '/'); if (apiBaseUrl.includes('gitlab')) { - url += `projects/${repository.replace( - /\//g, - '%2f' - )}/releases?per_page=100`; - const res = await gitlabHttp.getJson< - { - name: string; - release: string; - description: string; - tag_name: string; - }[] - >(url); - return res.body.map((release) => ({ - url: `${apiBaseUrl}projects/${repository.replace( - /\//g, - '%2f' - )}/releases/${release.tag_name}`, - name: release.name, - body: release.description, - tag: release.tag_name, - })); + return await gitlab.getReleaseList(apiBaseUrl, repository); } - url += `repos/${repository}/releases?per_page=100`; - const res = await githubHttp.getJson< - { - html_url: string; - id: number; - tag_name: string; - name: string; - body: string; - }[] - >(url); - return res.body.map((release) => ({ - url: release.html_url, - id: release.id, - tag: release.tag_name, - name: release.name, - body: release.body, - })); + return await github.getReleaseList(apiBaseUrl, repository); } catch (err) /* istanbul ignore next */ { if (err.statusCode === 404) { logger.debug({ repository }, 'getReleaseList 404'); @@ -196,54 +155,16 @@ export async function getReleaseNotesMdFileInner( repository: string, apiBaseUrl: string ): Promise<ChangeLogFile> | null { - let changelogFile: string; - let apiTree: string; - let apiFiles: string; - let filesRes: { body: { name: string }[] }; try { - const apiPrefix = apiBaseUrl.replace(/\/?$/, '/'); if (apiBaseUrl.includes('gitlab')) { - apiTree = apiPrefix + `projects/${repository}/repository/tree/`; - apiFiles = apiPrefix + `projects/${repository}/repository/files/`; - filesRes = await gitlabHttp.getJson<{ name: string }[]>(apiTree); - } else { - apiTree = apiPrefix + `repos/${repository}/contents/`; - apiFiles = apiTree; - filesRes = await githubHttp.getJson<{ name: string }[]>(apiTree); - } - const files = filesRes.body - .map((f) => f.name) - .filter((f) => changelogFilenameRegex.test(f)); - if (!files.length) { - logger.trace('no changelog file found'); - return null; + return await gitlab.getReleaseNotesMd(repository, apiBaseUrl); } - [changelogFile] = files; - /* istanbul ignore if */ - if (files.length > 1) { - logger.debug( - `Multiple candidates for changelog file, using ${changelogFile}` - ); - } - let fileRes: { body: { content: string } }; - if (apiBaseUrl.includes('gitlab')) { - fileRes = await gitlabHttp.getJson<{ content: string }>( - `${apiFiles}${changelogFile}?ref=master` - ); - } else { - fileRes = await githubHttp.getJson<{ content: string }>( - `${apiFiles}${changelogFile}` - ); - } - - const changelogMd = - Buffer.from(fileRes.body.content, 'base64').toString() + '\n#\n##'; - return { changelogFile, changelogMd }; + return await github.getReleaseNotesMd(repository, apiBaseUrl); } catch (err) /* istanbul ignore next */ { if (err.statusCode === 404) { logger.debug('Error 404 getting changelog md'); } else { - logger.debug({ err }, 'Error getting changelog md'); + logger.debug({ err, repository }, 'Error getting changelog md'); } return null; } @@ -301,6 +222,7 @@ export async function getReleaseNotesMd( for (const word of title) { if (word.includes(version) && !isUrl(word)) { logger.trace({ body }, 'Found release notes for v' + version); + // TODO: fix url let url = `${baseUrl}${repository}/blob/master/${changelogFile}#`; url += title.join('-').replace(/[^A-Za-z0-9-]/g, ''); body = massageBody(body, baseUrl); diff --git a/lib/workers/pr/changelog/source-github.ts b/lib/workers/pr/changelog/source-github.ts index 6f74f932fef3bb5956327cb8b58a7b26adbbd1d0..8c8fcfca12a10d0e222a80374cd326dca168ae4e 100644 --- a/lib/workers/pr/changelog/source-github.ts +++ b/lib/workers/pr/changelog/source-github.ts @@ -5,51 +5,23 @@ import { logger } from '../../../logger'; import * as memCache from '../../../util/cache/memory'; import * as packageCache from '../../../util/cache/package'; import * as hostRules from '../../../util/host-rules'; -import { GithubHttp } from '../../../util/http/github'; import * as allVersioning from '../../../versioning'; import { BranchUpgradeConfig } from '../../common'; import { ChangeLogError, ChangeLogRelease, ChangeLogResult } from './common'; +import { getTags } from './github'; import { addReleaseNotes } from './release-notes'; -const http = new GithubHttp(); - -async function getTagsInner( +function getCachedTags( endpoint: string, repository: string ): Promise<string[]> { - const url = `${endpoint}repos/${repository}/tags?per_page=100`; - try { - const res = await http.getJson<{ name: string }[]>(url, { - paginate: true, - }); - - const tags = res?.body || []; - - if (!tags.length) { - logger.debug({ repository }, 'repository has no Github tags'); - } - - return tags.map((tag) => tag.name).filter(Boolean); - } catch (err) { - logger.debug({ sourceRepo: repository }, 'Failed to fetch Github tags'); - logger.debug({ err }); - // istanbul ignore if - if (err.message && err.message.includes('Bad credentials')) { - logger.warn('Bad credentials triggering tag fail lookup in changelog'); - throw err; - } - return []; - } -} - -function getTags(endpoint: string, repository: string): Promise<string[]> { const cacheKey = `getTags-${endpoint}-${repository}`; const cachedResult = memCache.get(cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; } - const promisedRes = getTagsInner(endpoint, repository); + const promisedRes = getTags(endpoint, repository); memCache.set(cacheKey, promisedRes); return promisedRes; } @@ -118,7 +90,7 @@ export async function getChangeLogJSON({ async function getRef(release: Release): Promise<string | null> { if (!tags) { - tags = await getTags(apiBaseUrl, repository); + tags = await getCachedTags(apiBaseUrl, repository); } const regex = new RegExp(`(?:${depName}|release)[@-]`); const tagName = tags diff --git a/lib/workers/pr/changelog/source-gitlab.ts b/lib/workers/pr/changelog/source-gitlab.ts index 8b1566e210ad5137425bcd9aca966e10405d5272..5303312256430413bdf05d09b6c0da47d190033c 100644 --- a/lib/workers/pr/changelog/source-gitlab.ts +++ b/lib/workers/pr/changelog/source-gitlab.ts @@ -3,50 +3,16 @@ import { Release } from '../../../datasource'; import { logger } from '../../../logger'; import * as memCache from '../../../util/cache/memory'; import * as packageCache from '../../../util/cache/package'; -import { GitlabHttp } from '../../../util/http/gitlab'; import { regEx } from '../../../util/regex'; import * as allVersioning from '../../../versioning'; import { BranchUpgradeConfig } from '../../common'; import { ChangeLogRelease, ChangeLogResult } from './common'; +import { getTags } from './gitlab'; import { addReleaseNotes } from './release-notes'; -const gitlabHttp = new GitlabHttp(); - const cacheNamespace = 'changelog-gitlab-release'; -async function getTagsInner( - endpoint: string, - versionScheme: string, - repository: string -): Promise<string[]> { - logger.trace('getTags() from gitlab'); - let url = endpoint; - const repoid = repository.replace(/\//g, '%2f'); - url += `projects/${repoid}/repository/tags`; - try { - const res = await gitlabHttp.getJson<{ name: string }[]>(url, { - paginate: true, - }); - - const tags = res?.body || []; - - if (!tags.length) { - logger.debug({ sourceRepo: repository }, 'repository has no Gitlab tags'); - } - - return tags.map((tag) => tag.name).filter(Boolean); - } catch (err) { - logger.info({ sourceRepo: repository }, 'Failed to fetch Gitlab tags'); - // istanbul ignore if - if (err.message && err.message.includes('Bad credentials')) { - logger.warn('Bad credentials triggering tag fail lookup in changelog'); - throw err; - } - return []; - } -} - -function getTags( +function getCachedTags( endpoint: string, versionScheme: string, repository: string @@ -57,7 +23,7 @@ function getTags( if (cachedResult !== undefined) { return cachedResult; } - const promisedRes = getTagsInner(endpoint, versionScheme, repository); + const promisedRes = getTags(endpoint, repository); memCache.set(cacheKey, promisedRes); return promisedRes; } @@ -103,7 +69,7 @@ export async function getChangeLogJSON({ async function getRef(release: Release): Promise<string | null> { if (!tags) { - tags = await getTags(apiBaseUrl, versioning, repository); + tags = await getCachedTags(apiBaseUrl, versioning, repository); } const regex = regEx(`(?:${depName}|release)[@-]`); const tagName = tags