diff --git a/lib/manager/npm/registry.js b/lib/manager/npm/registry.js index 2f69fa90ffaf2feae72605970142a28ac7398221..ffdb7142ff2e16758e1498c25e476a048ee5ca2d 100644 --- a/lib/manager/npm/registry.js +++ b/lib/manager/npm/registry.js @@ -1,23 +1,27 @@ // Much of this borrowed from https://github.com/sindresorhus/package-json/blob/master/index.js +const got = require('got'); const url = require('url'); const ini = require('ini'); +const Keyv = require('keyv'); const getRegistryUrl = require('registry-auth-token/registry-url'); const registryAuthToken = require('registry-auth-token'); const parse = require('github-url-from-git'); -const os = require('os'); -const fetch = require('make-fetch-happen').defaults({ - cacheManager: - (process.env.RENOVATE_TMPDIR || os.tmpdir()) + '/renovate-npm-cache', -}); + +const cache = new Keyv({ namespace: 'npm' }); module.exports = { setNpmrc, getDependency, + resetCache, }; let npmrc = null; +function resetCache() { + cache.clear(); +} + function setNpmrc(input) { if (input) { npmrc = ini.parse(input); @@ -46,11 +50,22 @@ async function getDependency(name) { headers.authorization = `Bearer ${process.env.NPM_TOKEN}`; } + // Cache based on combinatino of package URL and headers + const cacheKey = pkgUrl + JSON.stringify(headers); + + // Return from cache if present + const cacheVal = await cache.get(cacheKey); + if (cacheVal) { + logger.trace(`Returning cached version of ${name}`); + return cacheVal; + } + // Retrieve from API if not cached try { - const res = await fetch(pkgUrl, { + const res = (await got(pkgUrl, { + json: true, headers, - }).then(r => r.json()); + })).body; // Determine repository URL let repositoryUrl; if (res.repository) { @@ -76,6 +91,8 @@ async function getDependency(name) { time: res.time ? res.time[version] : '2017-01-01T12:00:00.000Z', }; }); + const expiryMs = 5 * 60 * 1000; + await cache.set(cacheKey, dep, expiryMs); logger.trace({ dependency: dep }, 'dependency'); return dep; } catch (err) { diff --git a/package.json b/package.json index ccad215dc57f8f9c94b8499014e42d5f2914495c..e15e1e6571cd14a5dcbdfe5d4c6c88efe32df2f7 100644 --- a/package.json +++ b/package.json @@ -61,9 +61,9 @@ "json-dup-key-validator": "1.0.2", "json-stringify-pretty-compact": "1.0.4", "jsonwebtoken": "8.1.0", + "keyv": "3.0.0", "later": "1.2.0", "lodash": "4.17.4", - "make-fetch-happen": "2.6.0", "minimatch": "3.0.4", "moment": "2.19.2", "moment-timezone": "0.5.14", @@ -93,7 +93,6 @@ "jest": "21.2.1", "mkdirp": "0.5.1", "mockdate": "2.0.2", - "nock": "9.1.0", "prettier": "1.8.2", "rimraf": "2.6.2", "semantic-release": "8.2.0" diff --git a/test/manager/npm/__snapshots__/registry.spec.js.snap b/test/manager/npm/__snapshots__/registry.spec.js.snap index 2220264b22e7844f6496bfd13912609dd4514a79..b86e4548788b260a1e10062f4f868a959a741a50 100644 --- a/test/manager/npm/__snapshots__/registry.spec.js.snap +++ b/test/manager/npm/__snapshots__/registry.spec.js.snap @@ -5,10 +5,10 @@ Object { "dist-tags": Object { "latest": "0.0.1", }, - "homepage": undefined, + "homepage": "https://google.com", "name": undefined, "renovate-config": undefined, - "repositoryUrl": undefined, + "repositoryUrl": "https://google.com", "versions": Object { "0.0.1": Object { "time": "", @@ -17,6 +17,16 @@ Object { } `; +exports[`api/npm should fetch package info from custom registry 2`] = ` +Array [ + "https://npm.mycustomregistry.com/foobar", + Object { + "headers": Object {}, + "json": true, + }, +] +`; + exports[`api/npm should fetch package info from npm 1`] = ` Object { "dist-tags": Object { @@ -34,15 +44,25 @@ Object { } `; +exports[`api/npm should fetch package info from npm 2`] = ` +Array [ + "https://registry.npmjs.org/foobar", + Object { + "headers": Object {}, + "json": true, + }, +] +`; + exports[`api/npm should send an authorization header if provided 1`] = ` Object { "dist-tags": Object { "latest": "0.0.1", }, - "homepage": undefined, + "homepage": "https://google.com", "name": undefined, "renovate-config": undefined, - "repositoryUrl": undefined, + "repositoryUrl": "https://google.com", "versions": Object { "0.0.1": Object { "time": "", @@ -51,15 +71,27 @@ Object { } `; +exports[`api/npm should send an authorization header if provided 2`] = ` +Array [ + "https://registry.npmjs.org/foobar", + Object { + "headers": Object { + "authorization": "Basic 1234", + }, + "json": true, + }, +] +`; + exports[`api/npm should use NPM_TOKEN if provided 1`] = ` Object { "dist-tags": Object { "latest": "0.0.1", }, - "homepage": undefined, + "homepage": "https://google.com", "name": undefined, "renovate-config": undefined, - "repositoryUrl": undefined, + "repositoryUrl": "https://google.com", "versions": Object { "0.0.1": Object { "time": "", @@ -68,15 +100,27 @@ Object { } `; +exports[`api/npm should use NPM_TOKEN if provided 2`] = ` +Array [ + "https://registry.npmjs.org/foobar", + Object { + "headers": Object { + "authorization": "Bearer some-token", + }, + "json": true, + }, +] +`; + exports[`api/npm should use default registry if missing from npmrc 1`] = ` Object { "dist-tags": Object { "latest": "0.0.1", }, - "homepage": undefined, + "homepage": "https://google.com", "name": undefined, "renovate-config": undefined, - "repositoryUrl": undefined, + "repositoryUrl": "https://google.com", "versions": Object { "0.0.1": Object { "time": "", @@ -85,15 +129,25 @@ Object { } `; +exports[`api/npm should use default registry if missing from npmrc 2`] = ` +Array [ + "https://registry.npmjs.org/foobar", + Object { + "headers": Object {}, + "json": true, + }, +] +`; + exports[`api/npm should use dummy time if missing 1`] = ` Object { "dist-tags": Object { "latest": "0.0.1", }, - "homepage": undefined, + "homepage": "https://google.com", "name": undefined, "renovate-config": undefined, - "repositoryUrl": undefined, + "repositoryUrl": "https://google.com", "versions": Object { "0.0.1": Object { "time": "2017-01-01T12:00:00.000Z", @@ -102,6 +156,16 @@ Object { } `; +exports[`api/npm should use dummy time if missing 2`] = ` +Array [ + "https://registry.npmjs.org/foobar", + Object { + "headers": Object {}, + "json": true, + }, +] +`; + exports[`api/npm should use homepage 1`] = ` Object { "dist-tags": Object { @@ -118,3 +182,13 @@ Object { }, } `; + +exports[`api/npm should use homepage 2`] = ` +Array [ + "https://registry.npmjs.org/foobarhome", + Object { + "headers": Object {}, + "json": true, + }, +] +`; diff --git a/test/manager/npm/registry.spec.js b/test/manager/npm/registry.spec.js index 9e0a4dd978f6f952e7555f4f4dc7cba468f05d43..3d4ba3216c78c4ec69f8e81a5d71dd834dff42e7 100644 --- a/test/manager/npm/registry.spec.js +++ b/test/manager/npm/registry.spec.js @@ -1,24 +1,27 @@ const npm = require('../../../lib/manager/npm/registry'); +const got = require('got'); const registryAuthToken = require('registry-auth-token'); -const nock = require('nock'); jest.mock('registry-auth-token'); +jest.mock('got'); const npmResponse = { - versions: { - '0.0.1': { - foo: 1, + body: { + versions: { + '0.0.1': { + foo: 1, + }, + }, + repository: { + type: 'git', + url: 'git://github.com/renovateapp/dummy.git', + }, + 'dist-tags': { + latest: '0.0.1', + }, + time: { + '0.0.1': '', }, - }, - repository: { - type: 'git', - url: 'git://github.com/renovateapp/dummy.git', - }, - 'dist-tags': { - latest: '0.0.1', - }, - time: { - '0.0.1': '', }, }; @@ -26,36 +29,36 @@ describe('api/npm', () => { delete process.env.NPM_TOKEN; beforeEach(() => { jest.resetAllMocks(); + npm.resetCache(); }); it('should fetch package info from npm', async () => { - nock('https://registry.npmjs.org') - .get('/foobar') - .reply(200, npmResponse); + got.mockImplementation(() => Promise.resolve(npmResponse)); const res = await npm.getDependency('foobar'); expect(res).toMatchSnapshot(); + const call = got.mock.calls[0]; + expect(call).toMatchSnapshot(); }); it('should use homepage', async () => { const npmResponseHomepage = { ...npmResponse }; - npmResponseHomepage.repository.url = ''; - npmResponseHomepage.homepage = 'https://google.com'; - nock('https://registry.npmjs.org') - .get('/foobarhome') - .reply(200, npmResponseHomepage); + npmResponseHomepage.body.repository.url = ''; + npmResponseHomepage.body.homepage = 'https://google.com'; + got.mockImplementationOnce(() => Promise.resolve(npmResponseHomepage)); const res = await npm.getDependency('foobarhome'); expect(res).toMatchSnapshot(); + const call = got.mock.calls[0]; + expect(call).toMatchSnapshot(); }); it('should cache package info from npm', async () => { - nock('https://registry.npmjs.org') - .get('/foobar') - .reply(200, npmResponse); + got.mockImplementation(() => Promise.resolve(npmResponse)); const res1 = await npm.getDependency('foobar'); const res2 = await npm.getDependency('foobar'); expect(res1).toEqual(res2); + expect(got.mock.calls.length).toEqual(1); }); it('should return null if lookup fails', async () => { - nock('https://registry.npmjs.org') - .get('/foobar') - .reply(404); + got.mockImplementation(() => { + throw new Error('not found'); + }); const res = await npm.getDependency('foobar'); expect(res).toBeNull(); }); @@ -64,47 +67,45 @@ describe('api/npm', () => { type: 'Basic', token: '1234', })); - nock('https://registry.npmjs.org') - .matchHeader('authorization', 'Basic 1234') - .get('/foobar') - .reply(200, npmResponse); + got.mockImplementation(() => Promise.resolve(npmResponse)); const res = await npm.getDependency('foobar'); expect(res).toMatchSnapshot(); + const call = got.mock.calls[0]; + expect(call).toMatchSnapshot(); }); it('should use NPM_TOKEN if provided', async () => { - nock('https://registry.npmjs.org') - .matchHeader('authorization', 'Bearer some-token') - .get('/foobar') - .reply(200, npmResponse); + got.mockImplementation(() => Promise.resolve(npmResponse)); const oldToken = process.env.NPM_TOKEN; process.env.NPM_TOKEN = 'some-token'; const res = await npm.getDependency('foobar'); process.env.NPM_TOKEN = oldToken; expect(res).toMatchSnapshot(); + const call = got.mock.calls[0]; + expect(call).toMatchSnapshot(); }); it('should fetch package info from custom registry', async () => { - nock('https://npm.mycustomregistry.com') - .get('/foobar') - .reply(200, npmResponse); + got.mockImplementation(() => Promise.resolve(npmResponse)); npm.setNpmrc('registry=https://npm.mycustomregistry.com/'); const res = await npm.getDependency('foobar'); expect(res).toMatchSnapshot(); + const call = got.mock.calls[0]; + expect(call).toMatchSnapshot(); }); it('should use default registry if missing from npmrc', async () => { - nock('https://registry.npmjs.org') - .get('/foobar') - .reply(200, npmResponse); + got.mockImplementation(() => Promise.resolve(npmResponse)); npm.setNpmrc('foo=bar'); const res = await npm.getDependency('foobar'); expect(res).toMatchSnapshot(); + const call = got.mock.calls[0]; + expect(call).toMatchSnapshot(); }); it('should use dummy time if missing', async () => { const noTimeResponse = { ...npmResponse }; - delete noTimeResponse.time; - nock('https://registry.npmjs.org') - .get('/foobar') - .reply(200, noTimeResponse); + delete noTimeResponse.body.time; + got.mockImplementation(() => Promise.resolve(noTimeResponse)); const res = await npm.getDependency('foobar'); expect(res).toMatchSnapshot(); + const call = got.mock.calls[0]; + expect(call).toMatchSnapshot(); }); }); diff --git a/yarn.lock b/yarn.lock index bd4274372faac281273e4fefd2cd5a23802f3d77..1d876f7036795b105e7b87fae89c88e18dcf066c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -587,24 +587,6 @@ cacache@10.0.1: unique-filename "^1.1.0" y18n "^3.2.1" -cacache@^10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/cacache/-/cacache-10.0.0.tgz#3bba88bf62b0773fd9a691605f60c9d3c595e853" - dependencies: - bluebird "^3.5.0" - chownr "^1.0.1" - glob "^7.1.2" - graceful-fs "^4.1.11" - lru-cache "^4.1.1" - mississippi "^1.3.0" - mkdirp "^0.5.1" - move-concurrently "^1.0.1" - promise-inflight "^1.0.1" - rimraf "^2.6.1" - ssri "^5.0.0" - unique-filename "^1.1.0" - y18n "^3.2.1" - cacache@^9.2.9: version "9.3.0" resolved "https://registry.yarnpkg.com/cacache/-/cacache-9.3.0.tgz#9cd58f2dd0b8c8cacf685b7067b416d6d3cf9db1" @@ -708,14 +690,6 @@ chai@4.1.2: pathval "^1.0.0" type-detect "^4.0.0" -"chai@>=1.9.2 <4.0.0": - version "3.5.0" - resolved "https://registry.yarnpkg.com/chai/-/chai-3.5.0.tgz#4d02637b067fe958bdbfdd3a40ec56fef7373247" - dependencies: - assertion-error "^1.0.1" - deep-eql "^0.1.3" - type-detect "^1.0.0" - chalk@1.1.3, chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" @@ -1155,22 +1129,12 @@ decompress-response@^3.2.0: dependencies: mimic-response "^1.0.0" -deep-eql@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-0.1.3.tgz#ef558acab8de25206cd713906d74e56930eb69f2" - dependencies: - type-detect "0.1.1" - deep-eql@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-3.0.1.tgz#dfc9404400ad1c8fe023e7da1df1c147c4b444df" dependencies: type-detect "^4.0.0" -deep-equal@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" - deep-extend@~0.4.0: version "0.4.2" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.2.tgz#48b699c27e334bf89f10892be432f6e4c7d34a7f" @@ -2206,7 +2170,7 @@ html-encoding-sniffer@^1.0.1: dependencies: whatwg-encoding "^1.0.1" -http-cache-semantics@^3.7.3, http-cache-semantics@^3.8.0: +http-cache-semantics@^3.7.3: version "3.8.0" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.0.tgz#1e3ce248730e189ac692a6697b9e3fdea2ff8da3" @@ -2903,6 +2867,10 @@ jsesc@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b" +json-buffer@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" + json-dup-key-validator@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/json-dup-key-validator/-/json-dup-key-validator-1.0.2.tgz#4e5e1c32c93ede16a8424d28162e5e183b5d8ee5" @@ -3002,6 +2970,12 @@ jws@^3.1.4: jwa "^1.1.4" safe-buffer "^5.0.1" +keyv@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.0.0.tgz#44923ba39e68b12a7cec7df6c3268c031f2ef373" + dependencies: + json-buffer "3.0.0" + kind-of@^3.0.2: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -3266,7 +3240,7 @@ lodash.without@~4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/lodash.without/-/lodash.without-4.4.0.tgz#3cd4574a00b67bae373a94b748772640507b7aac" -lodash@4.17.4, lodash@^4.0.0, lodash@^4.14.0, lodash@^4.17.4, lodash@^4.2.1, lodash@^4.3.0, lodash@~4.17.2: +lodash@4.17.4, lodash@^4.0.0, lodash@^4.14.0, lodash@^4.17.4, lodash@^4.2.1, lodash@^4.3.0: version "4.17.4" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" @@ -3312,22 +3286,6 @@ make-dir@^1.0.0: dependencies: pify "^3.0.0" -make-fetch-happen@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-2.6.0.tgz#8474aa52198f6b1ae4f3094c04e8370d35ea8a38" - dependencies: - agentkeepalive "^3.3.0" - cacache "^10.0.0" - http-cache-semantics "^3.8.0" - http-proxy-agent "^2.0.0" - https-proxy-agent "^2.1.0" - lru-cache "^4.1.1" - mississippi "^1.2.0" - node-fetch-npm "^2.0.2" - promise-retry "^1.1.1" - socks-proxy-agent "^3.0.1" - ssri "^5.0.0" - make-fetch-happen@^2.4.13, make-fetch-happen@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-2.5.0.tgz#08c22d499f4f30111addba79fe87c98cf01b6bc8" @@ -3555,21 +3513,7 @@ netrc@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/netrc/-/netrc-0.1.4.tgz#6be94fcaca8d77ade0a9670dc460914c94472444" -nock@9.1.0: - version "9.1.0" - resolved "https://registry.yarnpkg.com/nock/-/nock-9.1.0.tgz#b6fd783abc1e774cb028058ea81207369a735747" - dependencies: - chai ">=1.9.2 <4.0.0" - debug "^2.2.0" - deep-equal "^1.0.0" - json-stringify-safe "^5.0.1" - lodash "~4.17.2" - mkdirp "^0.5.0" - propagate "0.4.0" - qs "^6.0.2" - semver "^5.3.0" - -node-fetch-npm@^2.0.1, node-fetch-npm@^2.0.2: +node-fetch-npm@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/node-fetch-npm/-/node-fetch-npm-2.0.2.tgz#7258c9046182dca345b4208eda918daf33697ff7" dependencies: @@ -4212,10 +4156,6 @@ promzard@^0.3.0: dependencies: read "1" -propagate@0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/propagate/-/propagate-0.4.0.tgz#f3fcca0a6fe06736a7ba572966069617c130b481" - proto-list@~1.2.1: version "1.2.4" resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" @@ -4265,10 +4205,6 @@ qrcode-terminal@~0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/qrcode-terminal/-/qrcode-terminal-0.11.0.tgz#ffc6c28a2fc0bfb47052b47e23f4f446a5fbdb9e" -qs@^6.0.2, qs@~6.5.1: - version "6.5.1" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" - qs@~6.2.0: version "6.2.3" resolved "https://registry.yarnpkg.com/qs/-/qs-6.2.3.tgz#1cfcb25c10a9b2b483053ff39f5dfc9233908cfe" @@ -4281,6 +4217,10 @@ qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" +qs@~6.5.1: + version "6.5.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" + query-string@~5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/query-string/-/query-string-5.0.1.tgz#6e2b86fe0e08aef682ecbe86e85834765402bd88" @@ -4837,7 +4777,7 @@ sntp@2.x.x: dependencies: hoek "4.x.x" -socks-proxy-agent@^3.0.0, socks-proxy-agent@^3.0.1: +socks-proxy-agent@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-3.0.1.tgz#2eae7cf8e2a82d34565761539a7f9718c5617659" dependencies: @@ -5242,14 +5182,6 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" -type-detect@0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-0.1.1.tgz#0ba5ec2a885640e470ea4e8505971900dac58822" - -type-detect@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-1.0.0.tgz#762217cc06db258ec48908a1298e8b95121e8ea2" - type-detect@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.3.tgz#0e3f2670b44099b0b46c284d136a7ef49c74c2ea"