diff --git a/lib/workers/pr/changelog/index.js b/lib/workers/pr/changelog/index.js index b6171edb927bdee98b2b0ac63541fa6812041a5a..f5f9887203327b5efb03ad7dbcd6c542e577f443 100644 --- a/lib/workers/pr/changelog/index.js +++ b/lib/workers/pr/changelog/index.js @@ -1,25 +1,28 @@ const url = require('url'); +const versioning = require('../../../versioning'); const { addReleaseNotes } = require('../release-notes'); const sourceCache = require('./source-cache'); const sourceGithub = require('./source-github'); -const managerNpm = require('./manager-npm'); -const managerPip = require('./manager-pip'); - module.exports = { getChangeLogJSON, }; async function getChangeLogJSON(args) { - const { manager, fromVersion, newValue } = args; - logger.debug({ args }, `getChangeLogJSON(args)`); - if (!fromVersion || fromVersion === newValue) { + const { repositoryUrl, versionScheme, fromVersion, toVersion } = args; + if (!repositoryUrl) { + return null; + } + // releases is too noisy in the logs + const { releases, ...param } = args; + logger.debug({ param }, `getChangeLogJSON(args)`); + const { equals } = versioning(versionScheme); + if (!fromVersion || equals(fromVersion, toVersion)) { return null; } - let isGHE = false; let token; let endpoint; let gheBaseURL; @@ -28,79 +31,40 @@ async function getChangeLogJSON(args) { }; if (process.env.GITHUB_ENDPOINT) { - // By default we consider the dependency we are retrieving the changelog for, hosted on github.com - // Because of this we unset GHE GITHUB_ENPOINT and we use GITHUB_COM_TOKEN as the token used to authenticate with github.com - isGHE = true; - token = process.env.GITHUB_TOKEN; - endpoint = process.env.GITHUB_ENDPOINT; - const parsedEndpoint = url.parse(endpoint); + const parsedEndpoint = url.parse(process.env.GITHUB_ENDPOINT); gheBaseURL = `${parsedEndpoint.protocol}//${parsedEndpoint.hostname}/`; - delete process.env.GITHUB_ENDPOINT; - process.env.GITHUB_TOKEN = process.env.GITHUB_COM_TOKEN; + if (repositoryUrl.startsWith(gheBaseURL)) { + opts.githubBaseURL = gheBaseURL; + } else { + // Switch tokens + token = process.env.GITHUB_TOKEN; + endpoint = process.env.GITHUB_ENDPOINT; + delete process.env.GITHUB_ENDPOINT; + process.env.GITHUB_TOKEN = process.env.GITHUB_COM_TOKEN; + } } - try { - // Return from cache if present let res = await sourceCache.getChangeLogJSON(args); if (!res) { - let pkg = null; - if (['npm', 'meteor'].includes(manager)) { - pkg = await managerNpm.getPackage(args); - } - - if (manager === 'pip_requirements') { - pkg = await managerPip.getPackage(args); - } - - if ( - isGHE && - pkg && - pkg.repositoryUrl && - pkg.repositoryUrl.startsWith(gheBaseURL) - ) { - // IF we are using GithubEnterprise and the dependency is hosted on it (instead of github.com) then we use - // the GHE token and endpoint. The hosting condition if verified by comparin github enterprise and dependency hostnames - logger.debug( - 'Found package hosted on internal GHE. Restoring GHE token' - ); - opts.githubBaseURL = gheBaseURL; - process.env.GITHUB_TOKEN = token; - process.env.GITHUB_ENDPOINT = endpoint; - } - res = await sourceGithub.getChangeLogJSON({ ...args, - ...pkg, ...opts, }); - await sourceCache.setChangeLogJSON(args, res); } - - if ( - isGHE && - res && - res.project && - res.project.githubBaseURL === gheBaseURL && - process.env.GITHUB_ENDPOINT !== endpoint - ) { - // If we are using GithubEnterprise and the dependency is hosted on it (instead of github.com) then we use - // the GHE token and endpoint, if we hadn't done it already (when res is coming from cache) - logger.debug('Found package hosted on internal GHE. Restoring GHE token'); - process.env.GITHUB_TOKEN = token; - process.env.GITHUB_ENDPOINT = endpoint; - } - const output = await addReleaseNotes(res); return output; } catch (err) /* istanbul ignore next */ { - logger.error({ err, message: err.message }, 'getChangeLogJSON error'); + logger.error( + { err, message: err.message, stack: err.stack }, + 'getChangeLogJSON error' + ); return null; } finally { // wrap everything in a try/finally to ensure process.env.GITHUB_TOKEN is restored no matter if // getChangeLogJSON and addReleaseNotes succed or fails - if (isGHE) { + if (token) { logger.debug('Restoring GHE token and endpoint'); process.env.GITHUB_TOKEN = token; process.env.GITHUB_ENDPOINT = endpoint; diff --git a/lib/workers/pr/changelog/manager-npm.js b/lib/workers/pr/changelog/manager-npm.js deleted file mode 100644 index fa4e152f756388f4ff6daf99da686e10bbaa5b45..0000000000000000000000000000000000000000 --- a/lib/workers/pr/changelog/manager-npm.js +++ /dev/null @@ -1,28 +0,0 @@ -const npmRegistry = require('../../../datasource/npm'); -const versioning = require('../../../versioning'); - -module.exports = { - getPackage, -}; - -async function getPackage({ versionScheme, depName, depType }) { - if (depType === 'engines') { - return null; - } - const { sortVersions } = versioning(versionScheme); - const dep = await npmRegistry.getDependency(depName); - if (!dep) { - return null; - } - const releases = Object.keys(dep.versions); - releases.sort(sortVersions); - const versions = releases.map(release => ({ - version: release, - date: dep.versions[release].time, - gitHead: dep.versions[release].gitHead, - })); - return { - repositoryUrl: dep.repositoryUrl, - versions, - }; -} diff --git a/lib/workers/pr/changelog/manager-pip.js b/lib/workers/pr/changelog/manager-pip.js deleted file mode 100644 index 66e390fcfb25ef0432c94bfe3752408ca78abbf9..0000000000000000000000000000000000000000 --- a/lib/workers/pr/changelog/manager-pip.js +++ /dev/null @@ -1,37 +0,0 @@ -const got = require('got'); -const versioning = require('../../../versioning'); - -module.exports = { - getPackage, -}; - -async function getPackage({ versionScheme, depName }) { - try { - const { sortVersions, isVersion } = versioning(versionScheme); - logger.debug({ depName }, 'fetching pip package versions'); - const rep = await got(`https://pypi.org/pypi/${depName}/json`, { - json: true, - }); - - const dep = rep && rep.body; - if (!dep) { - logger.debug({ depName }, 'pip package not found'); - return null; - } - const releases = Object.keys(dep.releases).filter(isVersion); - releases.sort(sortVersions); - const versions = releases.map(release => ({ - version: release, - date: (dep.releases[release][0] || {}).upload_time, - })); - const res = { - repositoryUrl: dep.info.home_page, - versions, - }; - logger.debug({ res }, 'found pip package versions'); - return res; - } catch (err) { - logger.debug({ err }, 'failed to fetch pip package versions'); - return null; - } -} diff --git a/lib/workers/pr/changelog/source-cache.js b/lib/workers/pr/changelog/source-cache.js index 45b1fe3266be5c4c0d62160722b32d8d6ce1153c..7ea96c517d1606083c85b1c0e923a52c76142bf9 100644 --- a/lib/workers/pr/changelog/source-cache.js +++ b/lib/workers/pr/changelog/source-cache.js @@ -7,10 +7,10 @@ module.exports = { rmAllCache, }; -function getCache({ depName, fromVersion, newValue }) { +function getCache({ depName, fromVersion, toVersion }) { const tmpdir = process.env.RENOVATE_TMPDIR || os.tmpdir(); const cachePath = tmpdir + '/renovate-cache-changelog'; - const cacheKey = `${depName}-${fromVersion}-${newValue}`; + const cacheKey = `${depName}-${fromVersion}-${toVersion}`; return [cachePath, cacheKey]; } diff --git a/lib/workers/pr/changelog/source-github.js b/lib/workers/pr/changelog/source-github.js index f04a0c8b6a5a7a3c4bd58866d06d6e731876c7b1..107534f43b3cd176046d6cb1637335f3785889e2 100644 --- a/lib/workers/pr/changelog/source-github.js +++ b/lib/workers/pr/changelog/source-github.js @@ -8,8 +8,6 @@ module.exports = { async function getTags(versionScheme, repository) { const { isVersion } = versioning(versionScheme); try { - const versions = {}; - const res = await ghGot(`repos/${repository}/tags?per_page=100`, { paginate: true, }); @@ -20,13 +18,7 @@ async function getTags(versionScheme, repository) { logger.debug({ repository }, 'repository has no Github tags'); } - tags.forEach(tag => { - const version = isVersion(tag.name); - if (version) { - versions[version] = { gitHead: tag.name }; - } - }); - return versions; + return tags.filter(tag => isVersion(tag.name)).map(tag => tag.name); } catch (err) { logger.info({ sourceRepo: repository }, 'Failed to fetch Github tags'); logger.debug({ @@ -34,20 +26,17 @@ async function getTags(versionScheme, repository) { message: err.message, body: err.response ? err.response.body : undefined, }); - return {}; + return []; } } -async function getRepositoryHead(repository, version) { - if (version.gitHead) { - return version.gitHead; - } - if (!version.date) { +async function getDateRef(repository, timestamp) { + if (!timestamp) { return null; } - logger.trace({ repository, version }, 'Looking for commit SHA by date'); + logger.trace({ repository, timestamp }, 'Looking for commit SHA by date'); try { - const res = await ghGot(`repos/${repository}/commits/@{${version.date}}`); + const res = await ghGot(`repos/${repository}/commits/@{${timestamp}}`); const commit = res && res.body; return commit && commit.sha; } catch (err) { @@ -59,59 +48,73 @@ async function getRepositoryHead(repository, version) { async function getChangeLogJSON({ versionScheme, githubBaseURL, - repositoryUrl, fromVersion, - newValue, - versions, + toVersion, + repositoryUrl, + releases, }) { - const { equals, isGreaterThan } = versioning(versionScheme); - logger.debug('Checking for github source URL manually'); - const include = version => - isGreaterThan(version, fromVersion) && !isGreaterThan(version, newValue); - + const { isVersion, equals, isGreaterThan, sortVersions } = versioning( + versionScheme + ); if (!(repositoryUrl && repositoryUrl.startsWith(githubBaseURL))) { - logger.debug('No repo found manually'); + logger.debug('Repository URL does not match base URL'); return null; } - logger.debug({ url: repositoryUrl }, 'Found github URL manually'); const repository = repositoryUrl .replace(githubBaseURL, '') .replace(/#.*/, ''); if (repository.split('/').length !== 2) { - logger.debug('Invalid github URL found'); + logger.info('Invalid github URL found'); + return null; + } + if (!(releases && releases.length)) { + logger.debug('No releases'); + return null; + } + // This extra filter/sort should not be necessary, but better safe than sorry + const validReleases = [...releases] + .filter(release => isVersion(release.version)) + .sort((a, b) => sortVersions(a.version, b.version)); + + if (validReleases.length < 2) { + logger.debug('Not enough valid releases'); return null; } const tags = await getTags(versionScheme, repository); - function getHead(version) { - const tagName = Object.keys(tags).find(key => equals(key, version.version)); - return getRepositoryHead(repository, { - ...version, - ...tags[tagName], - }); + function getRef(release) { + const tagName = tags.find(tag => equals(tag, release.version)); + if (tagName) { + return tagName; + } + if (release.gitRef) { + return release.gitRef; + } + return getDateRef(repository, release.date || release.time); } - const releases = []; - + const changelogReleases = []; // compare versions - for (let i = 1; i < versions.length; i += 1) { - const prev = versions[i - 1]; - const next = versions[i]; + const include = version => + isGreaterThan(version, fromVersion) && !isGreaterThan(version, toVersion); + for (let i = 1; i < validReleases.length; i += 1) { + const prev = validReleases[i - 1]; + const next = validReleases[i]; if (include(next.version)) { const release = { version: next.version, - date: next.date, + date: next.date || next.time, // put empty changes so that existing templates won't break changes: [], compare: {}, }; - const prevHead = await getHead(prev); - const nextHead = await getHead(next); + const prevHead = await getRef(prev); + const nextHead = await getRef(next); if (prevHead && nextHead) { release.compare.url = `${githubBaseURL}${repository}/compare/${prevHead}...${nextHead}`; } - releases.unshift(release); + changelogReleases.unshift(release); } } @@ -121,7 +124,7 @@ async function getChangeLogJSON({ github: repository, repository: repositoryUrl, }, - versions: releases, + versions: changelogReleases, }; logger.debug({ res }, 'Manual res'); diff --git a/lib/workers/pr/index.js b/lib/workers/pr/index.js index 661da99d154cb4369f73ef615b106fac74385119..80ccf8bef163de0881709e4319df1140523e65ac 100644 --- a/lib/workers/pr/index.js +++ b/lib/workers/pr/index.js @@ -114,12 +114,13 @@ async function ensurePr(prConfig) { processedUpgrades.push(upgradeKey); const logJSON = await changelogHelper.getChangeLogJSON({ - manager: upgrade.manager, versionScheme: upgrade.versionScheme, depType: upgrade.depType, depName: upgrade.depName, fromVersion: upgrade.fromVersion, - newValue: upgrade.toVersion, + toVersion: upgrade.toVersion, + repositoryUrl: config.repositoryUrl, + releases: config.releases, }); if (logJSON) { diff --git a/test/workers/pr/__snapshots__/changelog.spec.js.snap b/test/workers/pr/__snapshots__/changelog.spec.js.snap index cdded869d1163a3c027e7a434b000432fd6059fd..67581a19c878f369e10037190059c3d25f425bd6 100644 --- a/test/workers/pr/__snapshots__/changelog.spec.js.snap +++ b/test/workers/pr/__snapshots__/changelog.spec.js.snap @@ -296,9 +296,9 @@ Object { } `; -exports[`workers/pr/changelog getChangeLogJSON supports pip 1`] = ` +exports[`workers/pr/changelog getChangeLogJSON supports node engines 1`] = ` Object { - "hasReleaseNotes": false, + "hasReleaseNotes": true, "project": Object { "github": "chalk/chalk", "githubBaseURL": "https://github.com/", @@ -321,16 +321,24 @@ Object { }, Object { "changes": Array [], - "compare": Object {}, + "compare": Object { + "url": "https://github.com/chalk/chalk/compare/npm_2.2.2...npm_2.3.0", + }, "date": "2017-10-24T03:20:46.238Z", - "releaseNotes": undefined, + "releaseNotes": Object { + "url": "https://github.com/chalk/chalk/compare/npm_2.2.2...npm_2.3.0", + }, "version": "2.3.0", }, Object { "changes": Array [], - "compare": Object {}, + "compare": Object { + "url": "https://github.com/chalk/chalk/compare/npm_1.0.0...npm_2.2.2", + }, "date": undefined, - "releaseNotes": undefined, + "releaseNotes": Object { + "url": "https://github.com/chalk/chalk/compare/npm_1.0.0...npm_2.2.2", + }, "version": "2.2.2", }, ], diff --git a/test/workers/pr/changelog.spec.js b/test/workers/pr/changelog.spec.js index 11ecf482b9f6b18df9e2299f8ec5006d2b5bec9e..b601abeab6bef0c470d98395ecf3a7580e76a849 100644 --- a/test/workers/pr/changelog.spec.js +++ b/test/workers/pr/changelog.spec.js @@ -3,8 +3,6 @@ jest.mock('../../../lib/datasource/npm'); jest.mock('got'); const ghGot = require('../../../lib/platform/github/gh-got-wrapper'); -const npmRegistry = require('../../../lib/datasource/npm'); -const got = require('got'); const { getChangeLogJSON } = require('../../../lib/workers/pr/changelog'); const { @@ -12,52 +10,30 @@ const { } = require('../../../lib/workers/pr/changelog/source-cache'); const upgrade = { - manager: 'npm', depName: 'renovate', + versionScheme: 'semver', fromVersion: '1.0.0', - newValue: '3.0.0', -}; - -function npmResponse() { - return { - repositoryUrl: 'https://github.com/chalk/chalk', - versions: { - '0.9.0': {}, - '1.0.0': { gitHead: 'npm_1.0.0' }, - '2.3.0': { gitHead: 'npm_2.3.0', time: '2017-10-24T03:20:46.238Z' }, - '2.2.2': { gitHead: 'npm_2.2.2' }, - '2.4.2': { time: '2017-12-24T03:20:46.238Z' }, - '2.5.2': {}, - }, - }; -} - -function pipResponse() { - return { - info: { - home_page: 'https://github.com/chalk/chalk', - }, - releases: { - '0.9.0': [], - '1.0.0': [], - '2.3.0': [{ upload_time: '2017-10-24T03:20:46.238Z' }], - '2.2.2': [{}], - '2.4.2': [{ upload_time: '2017-12-24T03:20:46.238Z' }], - '2.5.2': [], + toVersion: '3.0.0', + repositoryUrl: 'https://github.com/chalk/chalk', + releases: [ + { version: '0.9.0' }, + { version: '1.0.0', gitRef: 'npm_1.0.0' }, + { + version: '2.3.0', + gitRef: 'npm_2.3.0', + time: '2017-10-24T03:20:46.238Z', }, - }; -} + { version: '2.2.2', gitRef: 'npm_2.2.2' }, + { version: '2.4.2', time: '2017-12-24T03:20:46.238Z' }, + { version: '2.5.2' }, + ], +}; describe('workers/pr/changelog', () => { describe('getChangeLogJSON', () => { beforeEach(async () => { - npmRegistry.getDependency.mockClear(); ghGot.mockClear(); - npmRegistry.getDependency.mockReturnValueOnce( - Promise.resolve(npmResponse()) - ); - await rmAllCache(); }); it('returns null if no fromVersion', async () => { @@ -67,34 +43,32 @@ describe('workers/pr/changelog', () => { fromVersion: null, }) ).toBe(null); - expect(npmRegistry.getDependency.mock.calls).toHaveLength(0); expect(ghGot.mock.calls).toHaveLength(0); }); - it('returns null if fromVersion equals newValue', async () => { + it('returns null if fromVersion equals toVersion', async () => { expect( await getChangeLogJSON({ ...upgrade, fromVersion: '1.0.0', - newValue: '1.0.0', + toVersion: '1.0.0', }) ).toBe(null); expect(ghGot.mock.calls).toHaveLength(0); }); - it('logs when no JSON', async () => { - // clear the mock - npmRegistry.getDependency.mockReset(); - expect(await getChangeLogJSON({ ...upgrade })).toBe(null); - }); it('skips invalid repos', async () => { - // clear the mock - npmRegistry.getDependency.mockReset(); - const res = npmResponse(); - res.repositoryUrl = 'https://github.com/about'; - npmRegistry.getDependency.mockReturnValueOnce(Promise.resolve(res)); - expect(await getChangeLogJSON({ ...upgrade })).toBe(null); + expect( + await getChangeLogJSON({ + ...upgrade, + repositoryUrl: 'https://github.com/about', + }) + ).toBe(null); }); it('works without Github', async () => { - expect(await getChangeLogJSON({ ...upgrade })).toMatchSnapshot(); + expect( + await getChangeLogJSON({ + ...upgrade, + }) + ).toMatchSnapshot(); }); it('uses GitHub tags', async () => { ghGot.mockReturnValueOnce( @@ -109,7 +83,11 @@ describe('workers/pr/changelog', () => { ], }) ); - expect(await getChangeLogJSON({ ...upgrade })).toMatchSnapshot(); + expect( + await getChangeLogJSON({ + ...upgrade, + }) + ).toMatchSnapshot(); }); it('falls back to commit from release time', async () => { // mock tags response @@ -129,10 +107,12 @@ describe('workers/pr/changelog', () => { }); it('returns cached JSON', async () => { const first = await getChangeLogJSON({ ...upgrade }); - npmRegistry.getDependency.mockClear(); + const firstCalls = [...ghGot.mock.calls]; + ghGot.mockClear(); const second = await getChangeLogJSON({ ...upgrade }); + const secondCalls = [...ghGot.mock.calls]; expect(first).toEqual(second); - expect(npmRegistry.getDependency.mock.calls).toHaveLength(0); + expect(firstCalls.length).toBeGreaterThan(secondCalls.length); }); it('filters unnecessary warns', async () => { ghGot.mockImplementation(() => { @@ -145,58 +125,69 @@ describe('workers/pr/changelog', () => { }) ).toMatchSnapshot(); }); - it('skips node engines', async () => { - expect(await getChangeLogJSON({ ...upgrade, depType: 'engines' })).toBe( - null - ); + it('supports node engines', async () => { + expect( + await getChangeLogJSON({ + ...upgrade, + depType: 'engines', + }) + ).toMatchSnapshot(); }); - it('supports pip', async () => { - got.mockReturnValueOnce( - Promise.resolve({ - body: pipResponse(), + it('handles no repositoryUrl', async () => { + expect( + await getChangeLogJSON({ + ...upgrade, + repositoryUrl: undefined, }) - ); + ).toBe(null); + }); + it('handles invalid repositoryUrl', async () => { expect( - await getChangeLogJSON({ ...upgrade, manager: 'pip_requirements' }) - ).toMatchSnapshot(); + await getChangeLogJSON({ + ...upgrade, + repositoryUrl: 'http://example.com', + }) + ).toBe(null); }); - it('works without pip', async () => { + it('handles no releases', async () => { expect( - await getChangeLogJSON({ ...upgrade, manager: 'pip_requirements' }) + await getChangeLogJSON({ + ...upgrade, + releases: [], + }) ).toBe(null); }); - it('handles pip errors', async () => { - got.mockImplementation(() => { - throw new Error('Unknown Pip Repo'); - }); + it('handles not enough releases', async () => { expect( - await getChangeLogJSON({ ...upgrade, manager: 'pip_requirements' }) + await getChangeLogJSON({ + ...upgrade, + releases: [{ version: '0.9.0' }], + }) ).toBe(null); }); it('supports github enterprise and github.com changelog', async () => { - // clear the mock - npmRegistry.getDependency.mockReset(); - const res = npmResponse(); - npmRegistry.getDependency.mockReturnValueOnce(Promise.resolve(res)); - + const token = process.env.GITHUB_TOKEN; const endpoint = process.env.GITHUB_ENDPOINT; + process.env.GITHUB_TOKEN = 'super_secret'; process.env.GITHUB_ENDPOINT = 'https://github-enterprise.example.com/'; - expect(await getChangeLogJSON({ ...upgrade })).toMatchSnapshot(); - + const oldenv = { ...process.env }; + expect( + await getChangeLogJSON({ + ...upgrade, + }) + ).toMatchSnapshot(); + // check that process env was restored + expect(process.env).toEqual(oldenv); + process.env.GITHUB_TOKEN = token; process.env.GITHUB_ENDPOINT = endpoint; }); it('supports github enterprise and github enterprise changelog', async () => { - // clear the mock - npmRegistry.getDependency.mockReset(); - const res = npmResponse(); - res.repositoryUrl = 'https://github-enterprise.example.com/chalk/chalk'; - npmRegistry.getDependency.mockReturnValueOnce(Promise.resolve(res)); - const endpoint = process.env.GITHUB_ENDPOINT; process.env.GITHUB_ENDPOINT = 'https://github-enterprise.example.com/'; expect( await getChangeLogJSON({ ...upgrade, + repositoryUrl: 'https://github-enterprise.example.com/chalk/chalk', }) ).toMatchSnapshot(); @@ -204,17 +195,21 @@ describe('workers/pr/changelog', () => { }); it('supports github enterprise alwo when retrieving data from cache', async () => { - // clear the mock - npmRegistry.getDependency.mockReset(); - const res = npmResponse(); - res.repositoryUrl = 'https://github-enterprise.example.com/chalk/chalk'; - npmRegistry.getDependency.mockReturnValueOnce(Promise.resolve(res)); - const endpoint = process.env.GITHUB_ENDPOINT; process.env.GITHUB_ENDPOINT = 'https://github-enterprise.example.com/'; - expect(await getChangeLogJSON({ ...upgrade })).toMatchSnapshot(); + expect( + await getChangeLogJSON({ + ...upgrade, + repositoryUrl: 'https://github-enterprise.example.com/chalk/chalk', + }) + ).toMatchSnapshot(); - expect(await getChangeLogJSON({ ...upgrade })).toMatchSnapshot(); + expect( + await getChangeLogJSON({ + ...upgrade, + repositoryUrl: 'https://github-enterprise.example.com/chalk/chalk', + }) + ).toMatchSnapshot(); process.env.GITHUB_ENDPOINT = endpoint; }); });