From 3414421f6f8fe834dd838b173791bcf9ebee3801 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Fri, 8 Mar 2019 08:20:12 +0100
Subject: [PATCH] feat: got host rules (#3327)
---
lib/datasource/cargo/index.js | 1 +
lib/datasource/github/index.js | 6 +--
lib/datasource/go/index.js | 2 +-
lib/datasource/hex/index.js | 16 +++-----
lib/datasource/maven/index.js | 2 +-
lib/datasource/nuget/get.js | 21 ----------
lib/datasource/nuget/v2.js | 4 +-
lib/datasource/nuget/v3.js | 13 +++---
lib/datasource/pypi/index.js | 2 +
lib/datasource/rubygems/get.js | 10 +----
lib/datasource/terraform/index.js | 2 +-
lib/util/got/host-rules.js | 41 +++++++++++++++++++
lib/util/got/index.js | 3 +-
.../__snapshots__/pypi.spec.js.snap | 4 ++
test/datasource/nuget.spec.js | 25 -----------
15 files changed, 68 insertions(+), 84 deletions(-)
delete mode 100644 lib/datasource/nuget/get.js
create mode 100644 lib/util/got/host-rules.js
diff --git a/lib/datasource/cargo/index.js b/lib/datasource/cargo/index.js
index 88c2fb8f4b..d94685c12a 100644
--- a/lib/datasource/cargo/index.js
+++ b/lib/datasource/cargo/index.js
@@ -9,6 +9,7 @@ async function getPkgReleases({ lookupName }) {
try {
const res = (await got(crateUrl, {
json: true,
+ platform: 'cargo',
})).body;
if (!(res && res.crate && res.crate.name && res.versions)) {
logger.warn({ dependency: lookupName }, `Received invalid crate data`);
diff --git a/lib/datasource/github/index.js b/lib/datasource/github/index.js
index c5d1b97608..36873cca37 100644
--- a/lib/datasource/github/index.js
+++ b/lib/datasource/github/index.js
@@ -1,6 +1,5 @@
const ghGot = require('../../platform/github/gh-got-wrapper');
const got = require('../../util/got');
-const hostRules = require('../../util/host-rules');
module.exports = {
getPreset,
@@ -13,11 +12,8 @@ async function fetchJSONFile(repo, fileName) {
const opts = {
headers: { accept: 'application/vnd.github.v3+json' },
json: true,
+ platform: 'github',
};
- const rules = hostRules.find({ platform: 'github', host: 'api.github.com' });
- if (rules && rules.token) {
- opts.headers.Authorization = `Bearer ${rules.token}`;
- }
let res;
try {
res = await got(url, opts);
diff --git a/lib/datasource/go/index.js b/lib/datasource/go/index.js
index 83843c1197..f76ad4bac5 100644
--- a/lib/datasource/go/index.js
+++ b/lib/datasource/go/index.js
@@ -25,7 +25,7 @@ async function getDatasource(name) {
const pkgUrl = `https://${name}?go-get=1`;
try {
const res = (await got(pkgUrl, {
- retry: 5,
+ platform: 'go',
})).body;
const sourceMatch = res.match(
new RegExp(`<meta name="go-source" content="${name}\\s+([^\\s]+)`)
diff --git a/lib/datasource/hex/index.js b/lib/datasource/hex/index.js
index 4b14939f58..fc78696fe8 100644
--- a/lib/datasource/hex/index.js
+++ b/lib/datasource/hex/index.js
@@ -1,20 +1,14 @@
-const URL = require('url');
-
const got = require('../../util/got');
-const hostRules = require('../../util/host-rules');
module.exports = {
getPkgReleases,
};
-function getHostOpts(url) {
- const { host } = URL.parse(url);
- const opts = hostRules.find({ platform: 'hex', host }, { json: true });
- if (opts && opts.token) {
- opts.hearders = { Authorization: opts.token };
- delete opts.token;
- }
- return opts;
+function getHostOpts() {
+ return {
+ json: true,
+ platform: 'hex',
+ };
}
async function getPkgReleases({ lookupName }) {
diff --git a/lib/datasource/maven/index.js b/lib/datasource/maven/index.js
index ff6e23ea23..ca7246f3c1 100644
--- a/lib/datasource/maven/index.js
+++ b/lib/datasource/maven/index.js
@@ -133,7 +133,7 @@ async function downloadFileProtocol(pkgUrl) {
async function downloadHttpProtocol(pkgUrl) {
let raw;
try {
- raw = await got(pkgUrl);
+ raw = await got(pkgUrl, { platform: 'maven' });
} catch (err) {
if (isNotFoundError(err)) {
logger.debug(`Url not found ${pkgUrl}`);
diff --git a/lib/datasource/nuget/get.js b/lib/datasource/nuget/get.js
deleted file mode 100644
index cdb2e735f5..0000000000
--- a/lib/datasource/nuget/get.js
+++ /dev/null
@@ -1,21 +0,0 @@
-const got = require('../../util/got');
-const hostRules = require('../../util/host-rules');
-
-module.exports = get;
-
-function get(url, options) {
- const finalOptions = options || {};
- const hostRule = hostRules.find({ platform: 'nuget', endpoint: url });
- if (hostRule && hostRule.username && hostRule.password) {
- const auth = Buffer.from(
- `${hostRule.username}:${hostRule.password}`
- ).toString('base64');
- finalOptions.headers = finalOptions.headers || {};
- finalOptions.headers.Authorization = `Basic ${auth}`;
- logger.debug(
- { url },
- `Setting basic auth header as configured via host rule`
- );
- }
- return got(url, finalOptions);
-}
diff --git a/lib/datasource/nuget/v2.js b/lib/datasource/nuget/v2.js
index 4c51e93800..5580462410 100644
--- a/lib/datasource/nuget/v2.js
+++ b/lib/datasource/nuget/v2.js
@@ -1,6 +1,6 @@
const parse = require('github-url-from-git');
const { XmlDocument } = require('xmldoc');
-const get = require('./get');
+const got = require('../../util/got');
module.exports = {
getPkgReleases,
@@ -12,7 +12,7 @@ async function getPkgReleases(feedUrl, pkgName) {
pkgName,
};
try {
- const pkgVersionsListRaw = await get(pkgUrlList, { retry: 5 });
+ const pkgVersionsListRaw = await got(pkgUrlList, { platform: 'nuget' });
if (pkgVersionsListRaw.statusCode !== 200) {
logger.debug(
{ dependency: pkgName, pkgVersionsListRaw },
diff --git a/lib/datasource/nuget/v3.js b/lib/datasource/nuget/v3.js
index 5c8528e3c7..dee96f9cfa 100644
--- a/lib/datasource/nuget/v3.js
+++ b/lib/datasource/nuget/v3.js
@@ -1,6 +1,6 @@
const parse = require('github-url-from-git');
const { XmlDocument } = require('xmldoc');
-const get = require('./get');
+const got = require('../../util/got');
module.exports = {
getQueryUrl,
@@ -18,10 +18,7 @@ function getDefaultFeed() {
async function getQueryUrl(url) {
// https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource
try {
- const servicesIndexRaw = await get(url, {
- retry: 5,
- json: true,
- });
+ const servicesIndexRaw = await got(url, { json: true, platform: 'nuget' });
if (servicesIndexRaw.statusCode !== 200) {
logger.debug(
{ dependency: url, servicesIndexRaw },
@@ -48,9 +45,9 @@ async function getPkgReleases(registryUrl, feedUrl, pkgName) {
pkgName,
};
try {
- const pkgUrlListRaw = await get(queryUrl, {
- retry: 5,
+ const pkgUrlListRaw = await got(queryUrl, {
json: true,
+ platform: 'nuget',
});
if (pkgUrlListRaw.statusCode !== 200) {
logger.debug(
@@ -75,7 +72,7 @@ async function getPkgReleases(registryUrl, feedUrl, pkgName) {
const nugetOrgApi = `https://api.nuget.org/v3-flatcontainer/${pkgName.toLowerCase()}/${
[...dep.releases].pop().version
}/${pkgName.toLowerCase()}.nuspec`;
- const result = await get(nugetOrgApi);
+ const result = await got(nugetOrgApi, { platform: 'nuget' });
const nuspec = new XmlDocument(result.body);
if (nuspec) {
const sourceUrl = parse(
diff --git a/lib/datasource/pypi/index.js b/lib/datasource/pypi/index.js
index 124eaefe86..8520ce1239 100644
--- a/lib/datasource/pypi/index.js
+++ b/lib/datasource/pypi/index.js
@@ -56,6 +56,7 @@ async function getDependency(depName, hostUrl, compatibility) {
const dependency = {};
const rep = await got(url.parse(lookupUrl), {
json: true,
+ platform: 'pypi',
});
const dep = rep && rep.body;
if (!dep) {
@@ -104,6 +105,7 @@ async function getSimpleDependency(depName, hostUrl) {
const dependency = {};
const response = await got(url.parse(lookupUrl), {
json: false,
+ platform: 'pypi',
});
const dep = response && response.body;
if (!dep) {
diff --git a/lib/datasource/rubygems/get.js b/lib/datasource/rubygems/get.js
index 3c7fd5699b..ca25bc202b 100644
--- a/lib/datasource/rubygems/get.js
+++ b/lib/datasource/rubygems/get.js
@@ -1,8 +1,5 @@
-const URL = require('url');
-
const got = require('../../util/got');
const { maskToken } = require('../../util/mask');
-const hostRules = require('../../util/host-rules');
const retriable = require('./retriable');
const { UNAUTHORIZED, FORBIDDEN, NOT_FOUND } = require('./errors');
@@ -33,11 +30,8 @@ const processError = ({ err, ...rest }) => {
}
};
-const getHeaders = url => {
- const { host } = URL.parse(url.toString());
- const { token } = hostRules.find({ host, platform: 'rubygems' }) || {};
-
- return token ? { Authorization: token } : {};
+const getHeaders = () => {
+ return { platform: 'rubygems' };
};
const fetch = async ({ dependency, registry, path }) => {
diff --git a/lib/datasource/terraform/index.js b/lib/datasource/terraform/index.js
index aaa1410a74..1603303fff 100644
--- a/lib/datasource/terraform/index.js
+++ b/lib/datasource/terraform/index.js
@@ -52,7 +52,7 @@ async function getPkgReleases({ lookupName, registryUrls }) {
try {
const res = (await got(pkgUrl, {
json: true,
- retry: 5,
+ platform: 'terraform',
})).body;
const returnedName = res.namespace + '/' + res.name + '/' + res.provider;
if (returnedName !== repository) {
diff --git a/lib/util/got/host-rules.js b/lib/util/got/host-rules.js
new file mode 100644
index 0000000000..39862230da
--- /dev/null
+++ b/lib/util/got/host-rules.js
@@ -0,0 +1,41 @@
+const got = require('got');
+const hostRules = require('../host-rules');
+
+// Apply host rules to requests
+
+// istanbul ignore next
+module.exports = got.create({
+ options: {},
+ handler: (options, next) => {
+ const { platform, ...opts } = options;
+ if (!options.hostname) {
+ return next(opts);
+ }
+ const hostRule = hostRules.find({
+ host: options.hostname,
+ platform,
+ });
+ if (!hostRule) {
+ return next(opts);
+ }
+ if (!options.headers.authorization) {
+ if (hostRule.username && hostRule.password) {
+ logger.debug(
+ 'Applying Basic authentication for host ' + options.hostname
+ );
+ const auth = Buffer.from(
+ `${hostRule.username}:${hostRule.password}`
+ ).toString('base64');
+ opts.headers.authorization = `Basic ${auth}`;
+ } else if (hostRule.token) {
+ logger.debug(
+ 'Applying Bearer authentication for host ' + options.hostname
+ );
+ opts.headers.authorization = `Bearer ${hostRule.token}`;
+ }
+ }
+ // TODO: apply other options/headers
+ // istanbul ignore next
+ return next(opts);
+ },
+});
diff --git a/lib/util/got/index.js b/lib/util/got/index.js
index 70130c1ff8..38d24e7fb6 100644
--- a/lib/util/got/index.js
+++ b/lib/util/got/index.js
@@ -1,6 +1,7 @@
const got = require('got');
const cacheGet = require('./cache-get');
const renovateAgent = require('./renovate-agent');
+const hostRules = require('./host-rules');
/*
* This is the default got instance for Renovate.
@@ -10,4 +11,4 @@ const renovateAgent = require('./renovate-agent');
* Important: always put the renovateAgent one last, to make sure the correct user agent is used
*/
-module.exports = got.mergeInstances(cacheGet, renovateAgent);
+module.exports = got.mergeInstances(cacheGet, renovateAgent, hostRules);
diff --git a/test/datasource/__snapshots__/pypi.spec.js.snap b/test/datasource/__snapshots__/pypi.spec.js.snap
index 27ee1735f2..607b5894e6 100644
--- a/test/datasource/__snapshots__/pypi.spec.js.snap
+++ b/test/datasource/__snapshots__/pypi.spec.js.snap
@@ -181,6 +181,7 @@ Array [
},
Object {
"json": true,
+ "platform": "pypi",
},
],
]
@@ -205,6 +206,7 @@ Array [
},
Object {
"json": true,
+ "platform": "pypi",
},
],
]
@@ -229,6 +231,7 @@ Array [
},
Object {
"json": true,
+ "platform": "pypi",
},
],
Array [
@@ -248,6 +251,7 @@ Array [
},
Object {
"json": true,
+ "platform": "pypi",
},
],
]
diff --git a/test/datasource/nuget.spec.js b/test/datasource/nuget.spec.js
index 4e39ae9069..7fc833df36 100644
--- a/test/datasource/nuget.spec.js
+++ b/test/datasource/nuget.spec.js
@@ -1,7 +1,6 @@
const fs = require('fs');
const got = require('../../lib/util/got');
const datasource = require('../../lib/datasource');
-const hostRules = require('../../lib/util/host-rules');
jest.mock('../../lib/util/got');
jest.mock('../../lib/util/host-rules');
@@ -80,30 +79,6 @@ describe('datasource/nuget', () => {
).toBeNull();
});
- it('supports basic authentication', async () => {
- got.mockReturnValueOnce({
- body: JSON.parse(nugetIndexV3),
- statusCode: 200,
- });
- got.mockReturnValueOnce({
- body: JSON.parse('{"totalHits": 0}'),
- statusCode: 200,
- });
-
- hostRules.find.mockReturnValue({
- username: 'some-username',
- password: 'some-password',
- });
-
- await datasource.getPkgReleases({
- ...configV3,
- });
-
- expect(got.mock.calls[0][1].headers.Authorization).toBe(
- 'Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk'
- );
- });
-
it('queries the default nuget feed if no registries are supplied', async () => {
await datasource.getPkgReleases({
...configNoRegistryUrls,
--
GitLab