diff --git a/lib/datasource/github.js b/lib/datasource/github.js
index 3b39c6513ef4b58e3267ad6d33e482defa562f7d..2ce70a436822d49076a629288f1c2225224ff8bf 100644
--- a/lib/datasource/github.js
+++ b/lib/datasource/github.js
@@ -4,7 +4,8 @@ module.exports = {
getDependency,
};
-async function getDependency(repo, options = {}) {
+async function getDependency(purl) {
+ const { fullname: repo, qualifiers: options } = purl;
let versions;
let endpoint;
let token;
diff --git a/lib/datasource/index.js b/lib/datasource/index.js
new file mode 100644
index 0000000000000000000000000000000000000000..00de7df572e93f27f569fad85fde1a25dffe0959
--- /dev/null
+++ b/lib/datasource/index.js
@@ -0,0 +1,29 @@
+const { parse } = require('../util/purl');
+
+const github = require('./github');
+const npm = require('./npm');
+const packagist = require('./packagist');
+const pypi = require('./pypi');
+
+const datasources = {
+ github,
+ npm,
+ packagist,
+ pypi,
+};
+
+function getDependency(purlStr, config) {
+ const purl = parse(purlStr);
+ if (!purl) {
+ return null;
+ }
+ if (!datasources[purl.type]) {
+ logger.warn('Unknown purl type: ' + purl.type);
+ return null;
+ }
+ return datasources[purl.type].getDependency(purl, config);
+}
+
+module.exports = {
+ getDependency,
+};
diff --git a/lib/datasource/npm.js b/lib/datasource/npm.js
index 9ed502554101bc668541b31b23a053a1d37998cc..2c287699aa427974df4d5b960a51f4fff1b23d1e 100644
--- a/lib/datasource/npm.js
+++ b/lib/datasource/npm.js
@@ -88,7 +88,21 @@ function envReplace(value, env = process.env) {
});
}
-async function getDependency(name, retries = 5) {
+function getDependency(input, config) {
+ const retries = config ? config.retries : undefined;
+ if (is.string(input)) {
+ const depName = input;
+ return getDependencyInner(depName, retries);
+ }
+ if (config) {
+ const exposeEnv = config.global ? config.global.exposeEnv : false;
+ setNpmrc(config.npmrc, exposeEnv);
+ }
+ const purl = input;
+ return getDependencyInner(purl.fullname, retries);
+}
+
+async function getDependencyInner(name, retries = 5) {
logger.trace(`getDependency(${name})`);
if (memcache[name]) {
logger.trace('Returning cached result');
@@ -135,7 +149,7 @@ async function getDependency(name, retries = 5) {
}
logger.info('No versions returned, retrying');
await delay(5000 / retries);
- return getDependency(name, 0);
+ return getDependencyInner(name, 0);
}
const latestVersion = res.versions[res['dist-tags'].latest];
@@ -217,7 +231,7 @@ async function getDependency(name, retries = 5) {
}
logger.info({ err }, 'npm registry failure: ParseError, retrying');
await delay(5000 / retries);
- return getDependency(name, retries - 1);
+ return getDependencyInner(name, retries - 1);
}
if (err.statusCode === 429) {
if (retries <= 0) {
@@ -229,7 +243,7 @@ async function getDependency(name, retries = 5) {
`npm too many requests. retrying after ${retryAfter} seconds`
);
await delay(1000 * (retryAfter + 1));
- return getDependency(name, retries - 1);
+ return getDependencyInner(name, retries - 1);
}
if (err.statusCode === 408) {
if (retries <= 0) {
@@ -238,7 +252,7 @@ async function getDependency(name, retries = 5) {
}
logger.info({ err }, 'npm registry failure: timeout, retrying');
await delay(5000 / retries);
- return getDependency(name, retries - 1);
+ return getDependencyInner(name, retries - 1);
}
if (err.statusCode >= 500 && err.statusCode < 600) {
if (retries <= 0) {
@@ -247,7 +261,7 @@ async function getDependency(name, retries = 5) {
}
logger.info({ err }, 'npm registry failure: internal error, retrying');
await delay(5000 / retries);
- return getDependency(name, retries - 1);
+ return getDependencyInner(name, retries - 1);
}
logger.warn({ err, name }, 'npm registry failure: Unknown error');
throw new Error('registry-failure');
diff --git a/lib/datasource/packagist.js b/lib/datasource/packagist.js
index 3f52a2406284c47aff8fccc485e0962311cfee14..79f49b2e704634f05f9d14ebfd56b7b83d6c75fc 100644
--- a/lib/datasource/packagist.js
+++ b/lib/datasource/packagist.js
@@ -7,7 +7,8 @@ module.exports = {
getDependency,
};
-async function getDependency(name) {
+async function getDependency(purl) {
+ const { fullname: name } = purl;
logger.trace(`getDependency(${name})`);
const regUrl = 'https://packagist.org';
diff --git a/lib/datasource/pypi.js b/lib/datasource/pypi.js
index 998e1bf4a285e0826d03e424cb0cca979f0e2bbd..b9a2f1c270e85e03b76878e45172be3af18f38b0 100644
--- a/lib/datasource/pypi.js
+++ b/lib/datasource/pypi.js
@@ -4,7 +4,8 @@ module.exports = {
getDependency,
};
-async function getDependency(depName) {
+async function getDependency(purl) {
+ const { fullname: depName } = purl;
try {
const dependency = {};
const rep = await got(`https://pypi.org/pypi/${depName}/json`, {
diff --git a/lib/workers/repository/process/lookup/index.js b/lib/workers/repository/process/lookup/index.js
index 0c905b0dc32db1bedebe3438bbf55071a1c52bcb..3609e99e4495aa78f2886dda59a25f4c0d8abb15 100644
--- a/lib/workers/repository/process/lookup/index.js
+++ b/lib/workers/repository/process/lookup/index.js
@@ -2,11 +2,7 @@ const versioning = require('../../../../versioning');
const { getRollbackUpdate } = require('./rollback');
const { getRangeStrategy } = require('../../../../manager');
const { filterVersions } = require('./filter');
-const npmApi = require('../../../../datasource/npm');
-const github = require('../../../../datasource/github');
-const packagist = require('../../../../datasource/packagist');
-const pypi = require('../../../../datasource/pypi');
-const { parse } = require('../../../../../lib/util/purl');
+const { getDependency } = require('../../../../datasource');
module.exports = {
lookupUpdates,
@@ -24,29 +20,7 @@ async function lookupUpdates(config) {
matches,
getNewValue,
} = versioning(config.versionScheme);
- const purl = parse(config.purl);
- if (!purl) {
- logger.error('Missing purl');
- return [];
- }
- let dependency;
- if (purl.type === 'npm') {
- // TODO: move this into datasource
- npmApi.setNpmrc(
- config.npmrc,
- config.global ? config.global.exposeEnv : false
- );
- dependency = await npmApi.getDependency(purl.fullname);
- } else if (purl.type === 'github') {
- dependency = await github.getDependency(purl.fullname, purl.qualifiers);
- } else if (purl.type === 'pypi') {
- dependency = await pypi.getDependency(purl.fullname);
- } else if (purl.type === 'packagist') {
- dependency = await packagist.getDependency(purl.fullname);
- } else {
- logger.warn({ config }, 'Unknown purl');
- return [];
- }
+ const dependency = await getDependency(config.purl, config);
if (!dependency) {
// If dependency lookup fails then warn and return
const result = {
diff --git a/test/datasource/__snapshots__/npm.spec.js.snap b/test/datasource/__snapshots__/npm.spec.js.snap
index 5de6c2a098b0f4e1163869fd2a5963b5b37d3739..abdf0438f802418de3a05b7e0ba0d1bcc6a3529e 100644
--- a/test/datasource/__snapshots__/npm.spec.js.snap
+++ b/test/datasource/__snapshots__/npm.spec.js.snap
@@ -66,6 +66,29 @@ Object {
}
`;
+exports[`api/npm should handle purl 1`] = `
+Object {
+ "dist-tags": Object {
+ "latest": "0.0.1",
+ },
+ "homepage": undefined,
+ "latestVersion": "0.0.1",
+ "name": undefined,
+ "renovate-config": undefined,
+ "repositoryUrl": "https://github.com/renovateapp/dummy",
+ "versions": Object {
+ "0.0.1": Object {
+ "canBeUnpublished": false,
+ "time": "2018-05-06T07:21:53+02:00",
+ },
+ "0.0.2": Object {
+ "canBeUnpublished": false,
+ "time": "2018-05-07T07:21:53+02:00",
+ },
+ },
+}
+`;
+
exports[`api/npm should replace any environment variable in npmrc 1`] = `
Object {
"dist-tags": Object {
diff --git a/test/datasource/github.spec.js b/test/datasource/github.spec.js
index 09b248082bbf07a2ed848aec75899ae926a66c8c..7f80095c7362b31d155e307714d733d03557ab87 100644
--- a/test/datasource/github.spec.js
+++ b/test/datasource/github.spec.js
@@ -1,4 +1,4 @@
-const github = require('../../lib/datasource/github');
+const datasource = require('../../lib/datasource');
const ghGot = require('../../lib/platform/github/gh-got-wrapper');
jest.mock('../../lib/platform/github/gh-got-wrapper');
@@ -14,7 +14,9 @@ describe('datasource/github', () => {
{ ref: 'refs/tags/v1.1.0' },
];
ghGot.mockReturnValueOnce({ headers: {}, body });
- const res = await github.getDependency('some/dep', { clean: 'true' });
+ const res = await datasource.getDependency(
+ 'pkg:github/some/dep?clean=true'
+ );
expect(res).toMatchSnapshot();
expect(Object.keys(res.versions)).toHaveLength(4);
expect(res.versions['1.1.0']).toBeDefined();
@@ -27,14 +29,16 @@ describe('datasource/github', () => {
{ tag_name: 'v1.1.0' },
];
ghGot.mockReturnValueOnce({ headers: {}, body });
- const res = await github.getDependency('some/dep', { ref: 'release' });
+ const res = await datasource.getDependency(
+ 'pkg:github/some/dep?ref=release'
+ );
expect(res).toMatchSnapshot();
expect(Object.keys(res.versions)).toHaveLength(4);
expect(res.versions['v1.1.0']).toBeDefined();
});
it('returns null for invalid ref', async () => {
expect(
- await github.getDependency('some/dep', { ref: 'invalid' })
+ await datasource.getDependency('pkg:github/some/dep?ref=invalid')
).toBeNull();
});
});
diff --git a/test/datasource/index.spec.js b/test/datasource/index.spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..86c46f7874bb2eb28fb012b8ef14a3082075e21a
--- /dev/null
+++ b/test/datasource/index.spec.js
@@ -0,0 +1,7 @@
+const datasource = require('../../lib/datasource');
+
+describe('datasource/index', () => {
+ it('returns null for invalid purl', async () => {
+ expect(await datasource.getDependency('pkggithub/some/dep')).toBeNull();
+ });
+});
diff --git a/test/datasource/npm.spec.js b/test/datasource/npm.spec.js
index 70b1e52b2f517b992c3fcad6a91c21a393e3e3df..a1d288445905cfff284b3bb3a6cd06b8915c8655 100644
--- a/test/datasource/npm.spec.js
+++ b/test/datasource/npm.spec.js
@@ -44,7 +44,7 @@ describe('api/npm', () => {
nock('https://registry.npmjs.org')
.get('/foobar')
.reply(200, missingVersions);
- const res = await npm.getDependency('foobar', 1);
+ const res = await npm.getDependency('foobar', { retries: 1 });
expect(res).toBe(null);
});
it('should fetch package info from npm', async () => {
@@ -56,6 +56,13 @@ describe('api/npm', () => {
expect(res.versions['0.0.1'].canBeUnpublished).toBe(false);
expect(res.versions['0.0.2'].canBeUnpublished).toBe(false);
});
+ it('should handle purl', async () => {
+ nock('https://registry.npmjs.org')
+ .get('/foobar')
+ .reply(200, npmResponse);
+ const res = await npm.getDependency({ fullname: 'foobar' });
+ expect(res).toMatchSnapshot();
+ });
it('should handle no time', async () => {
delete npmResponse.time['0.0.2'];
nock('https://registry.npmjs.org')
@@ -110,7 +117,7 @@ describe('api/npm', () => {
.reply(200, 'oops');
let e;
try {
- await npm.getDependency('foobar', 1);
+ await npm.getDependency('foobar', { retries: 1 });
} catch (err) {
e = err;
}
@@ -125,7 +132,7 @@ describe('api/npm', () => {
.reply(429);
let e;
try {
- await npm.getDependency('foobar', 1);
+ await npm.getDependency('foobar', { retries: 1 });
} catch (err) {
e = err;
}
@@ -137,7 +144,7 @@ describe('api/npm', () => {
.reply(503);
let e;
try {
- await npm.getDependency('foobar', 0);
+ await npm.getDependency('foobar', { retries: 0 });
} catch (err) {
e = err;
}
@@ -149,7 +156,7 @@ describe('api/npm', () => {
.reply(408);
let e;
try {
- await npm.getDependency('foobar', 0);
+ await npm.getDependency('foobar', { retries: 0 });
} catch (err) {
e = err;
}
@@ -165,7 +172,7 @@ describe('api/npm', () => {
nock('https://registry.npmjs.org')
.get('/foobar')
.reply(200);
- const res = await npm.getDependency('foobar', 2);
+ const res = await npm.getDependency('foobar', { retries: 2 });
expect(res).toMatchSnapshot();
});
it('should throw error for others', async () => {
diff --git a/test/datasource/packagist.spec.js b/test/datasource/packagist.spec.js
index 80f96ddd6b05ac2c3ed49ec6a600f8cae44bd1bf..991c8975b37e0827da951b7b46545dc583973a22 100644
--- a/test/datasource/packagist.spec.js
+++ b/test/datasource/packagist.spec.js
@@ -1,5 +1,5 @@
const fs = require('fs');
-const packagist = require('../../lib/datasource/packagist');
+const datasource = require('../../lib/datasource');
const got = require('got');
jest.mock('got');
@@ -10,7 +10,9 @@ describe('datasource/packagist', () => {
describe('getDependency', () => {
it('returns null for empty result', async () => {
got.mockReturnValueOnce({});
- expect(await packagist.getDependency('something')).toBeNull();
+ expect(
+ await datasource.getDependency('pkg:packagist/something')
+ ).toBeNull();
});
it('returns null for 404', async () => {
got.mockImplementationOnce(() =>
@@ -18,20 +20,24 @@ describe('datasource/packagist', () => {
statusCode: 404,
})
);
- expect(await packagist.getDependency('something')).toBeNull();
+ expect(
+ await datasource.getDependency('pkg:packagist/something')
+ ).toBeNull();
});
it('returns null for unknown error', async () => {
got.mockImplementationOnce(() => {
throw new Error();
});
- expect(await packagist.getDependency('something')).toBeNull();
+ expect(
+ await datasource.getDependency('pkg:packagist/something')
+ ).toBeNull();
});
it('processes real data', async () => {
got.mockReturnValueOnce({
body: JSON.parse(res1),
});
expect(
- await packagist.getDependency('cristianvuolo/uploader')
+ await datasource.getDependency('pkg:packagist/cristianvuolo/uploader')
).toMatchSnapshot();
});
});
diff --git a/test/datasource/pypi.spec.js b/test/datasource/pypi.spec.js
index fc3b232296e7b93c7b2d6d8820bff43a9470d090..8d2a85c1373d17ae23f313550004f8d2dd0a6ea7 100644
--- a/test/datasource/pypi.spec.js
+++ b/test/datasource/pypi.spec.js
@@ -1,5 +1,5 @@
const fs = require('fs');
-const pypi = require('../../lib/datasource/pypi');
+const datasource = require('../../lib/datasource');
const got = require('got');
jest.mock('got');
@@ -10,19 +10,21 @@ describe('datasource/pypi', () => {
describe('getDependency', () => {
it('returns null for empty result', async () => {
got.mockReturnValueOnce({});
- expect(await pypi.getDependency('something')).toBeNull();
+ expect(await datasource.getDependency('pkg:pypi/something')).toBeNull();
});
it('returns null for 404', async () => {
got.mockImplementationOnce(() => {
throw new Error();
});
- expect(await pypi.getDependency('something')).toBeNull();
+ expect(await datasource.getDependency('pkg:pypi/something')).toBeNull();
});
it('processes real data', async () => {
got.mockReturnValueOnce({
body: JSON.parse(res1),
});
- expect(await pypi.getDependency('azure-cli-monitor')).toMatchSnapshot();
+ expect(
+ await datasource.getDependency('pkg:pypi/azure-cli-monitor')
+ ).toMatchSnapshot();
});
it('returns non-github home_page', async () => {
got.mockReturnValueOnce({
@@ -32,7 +34,9 @@ describe('datasource/pypi', () => {
},
},
});
- expect(await pypi.getDependency('something')).toMatchSnapshot();
+ expect(
+ await datasource.getDependency('pkg:pypi/something')
+ ).toMatchSnapshot();
});
});
});
diff --git a/test/workers/repository/process/lookup/__snapshots__/index.spec.js.snap b/test/workers/repository/process/lookup/__snapshots__/index.spec.js.snap
index f5c2e4c8c4ea21676c9868ba226e6c38d9036ec3..bce2c9f576edd9ea3dc9a1b69ffb8a4e40bc0d9f 100644
--- a/test/workers/repository/process/lookup/__snapshots__/index.spec.js.snap
+++ b/test/workers/repository/process/lookup/__snapshots__/index.spec.js.snap
@@ -114,7 +114,14 @@ Array [
]
`;
-exports[`manager/npm/lookup .lookupUpdates() handles unknown purl 1`] = `Array []`;
+exports[`manager/npm/lookup .lookupUpdates() handles unknown purl 1`] = `
+Array [
+ Object {
+ "message": "Failed to look up dependency foo",
+ "type": "warning",
+ },
+]
+`;
exports[`manager/npm/lookup .lookupUpdates() ignores pinning for ranges when other upgrade exists 1`] = `
Array [
diff --git a/test/workers/repository/process/lookup/index.spec.js b/test/workers/repository/process/lookup/index.spec.js
index 1b2dc4e01f841976c8cd42d7d14e4916bd1c8d40..96a8c4b131af49d74125a5db61c9d46ad3d097b4 100644
--- a/test/workers/repository/process/lookup/index.spec.js
+++ b/test/workers/repository/process/lookup/index.spec.js
@@ -705,6 +705,7 @@ describe('manager/npm/lookup', () => {
config.currentValue = '^4.4.0-canary.3';
config.rangeStrategy = 'replace';
config.depName = 'next';
+ config.purl = 'pkg:npm/next';
nock('https://registry.npmjs.org')
.get('/next')
.reply(200, nextJson);