From 4e4bfe925606829ee2ab87390cad22890cdad22f Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Fri, 3 Apr 2020 13:45:55 +0200
Subject: [PATCH] feat(internal): http util wrapper (#5841)
Co-Authored-By: Michael Kriese <michael.kriese@visualon.de>
---
lib/config/presets/github.ts | 11 +-
lib/datasource/cdnjs/index.ts | 8 +-
lib/datasource/crate/index.ts | 8 +-
lib/datasource/dart/index.ts | 13 +-
.../docker/__snapshots__/index.spec.ts.snap | 189 +++++++++++++++++-
lib/datasource/docker/index.ts | 52 ++---
lib/datasource/galaxy/index.ts | 8 +-
lib/datasource/go/index.ts | 16 +-
lib/datasource/gradle-version/index.spec.ts | 4 +-
lib/datasource/gradle-version/index.ts | 25 +--
lib/datasource/helm/index.ts | 6 +-
lib/datasource/hex/index.ts | 9 +-
lib/datasource/maven/util.ts | 22 +-
lib/datasource/npm/get.ts | 13 +-
lib/datasource/nuget/v2.ts | 16 +-
lib/datasource/nuget/v3.ts | 33 +--
lib/datasource/orb/index.spec.ts | 8 +-
lib/datasource/orb/index.ts | 9 +-
lib/datasource/packagist/index.ts | 33 +--
.../pypi/__snapshots__/index.spec.ts.snap | 88 +++-----
lib/datasource/pypi/index.ts | 14 +-
lib/datasource/ruby-version/index.ts | 6 +-
lib/datasource/rubygems/get-rubygems-org.ts | 7 +-
lib/datasource/rubygems/get.ts | 10 +-
lib/datasource/rubygems/retriable.spec.ts | 31 ---
lib/datasource/rubygems/retriable.ts | 63 ------
lib/datasource/terraform-module/index.ts | 11 +-
lib/datasource/terraform-provider/index.ts | 11 +-
lib/manager/bazel/update.ts | 6 +-
lib/manager/gradle-wrapper/update.ts | 8 +-
lib/manager/homebrew/update.ts | 8 +-
lib/util/got/common.ts | 1 +
lib/util/http/index.ts | 104 ++++++++++
tools/jest-gh-reporter.ts | 5 +-
34 files changed, 479 insertions(+), 377 deletions(-)
delete mode 100644 lib/datasource/rubygems/retriable.spec.ts
delete mode 100644 lib/datasource/rubygems/retriable.ts
create mode 100644 lib/util/http/index.ts
diff --git a/lib/config/presets/github.ts b/lib/config/presets/github.ts
index 175c7dbb82..addef24b7a 100644
--- a/lib/config/presets/github.ts
+++ b/lib/config/presets/github.ts
@@ -1,22 +1,23 @@
import { logger } from '../../logger';
import { Preset } from './common';
-import got, { GotJSONOptions } from '../../util/got';
+import { Http, HttpOptions } from '../../util/http';
import { PLATFORM_FAILURE } from '../../constants/error-messages';
+const id = 'github';
+const http = new Http(id);
+
async function fetchJSONFile(repo: string, fileName: string): Promise<Preset> {
const url = `https://api.github.com/repos/${repo}/contents/${fileName}`;
- const opts: GotJSONOptions = {
+ const opts: HttpOptions = {
headers: {
accept: global.appMode
? 'application/vnd.github.machine-man-preview+json'
: 'application/vnd.github.v3+json',
},
- json: true,
- hostType: 'github',
};
let res: { body: { content: string } };
try {
- res = await got(url, opts);
+ res = await http.getJson(url, opts);
} catch (err) {
if (err.message === PLATFORM_FAILURE) {
throw err;
diff --git a/lib/datasource/cdnjs/index.ts b/lib/datasource/cdnjs/index.ts
index fea307efd2..eb37fc211a 100644
--- a/lib/datasource/cdnjs/index.ts
+++ b/lib/datasource/cdnjs/index.ts
@@ -1,5 +1,5 @@
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { DatasourceError, ReleaseResult, GetReleasesConfig } from '../common';
export interface CdnjsAsset {
@@ -10,6 +10,8 @@ export interface CdnjsAsset {
export const id = 'cdnjs';
+const http = new Http(id);
+
const cacheNamespace = `datasource-${id}`;
const cacheMinutes = 60;
@@ -36,7 +38,7 @@ export async function getDigest(
const assetName = lookupName.replace(`${library}/`, '');
let res = null;
try {
- res = await got(url, { hostType: id, json: true });
+ res = await http.getJson(url);
} catch (e) /* istanbul ignore next */ {
return null;
}
@@ -68,7 +70,7 @@ export async function getPkgReleases({
const url = depUrl(library);
try {
- const res = await got(url, { hostType: id, json: true });
+ const res = await http.getJson(url);
const cdnjsResp: CdnjsResponse = res.body;
diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts
index 0640c13f1a..519daf8b43 100644
--- a/lib/datasource/crate/index.ts
+++ b/lib/datasource/crate/index.ts
@@ -1,5 +1,5 @@
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import {
DatasourceError,
GetReleasesConfig,
@@ -9,6 +9,8 @@ import {
export const id = 'crate';
+const http = new Http(id);
+
export async function getPkgReleases({
lookupName,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
@@ -41,9 +43,7 @@ export async function getPkgReleases({
'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/';
const crateUrl = baseUrl + path;
try {
- let res: any = await got(crateUrl, {
- hostType: id,
- });
+ let res: any = await http.get(crateUrl);
if (!res || !res.body) {
logger.warn(
{ dependency: lookupName },
diff --git a/lib/datasource/dart/index.ts b/lib/datasource/dart/index.ts
index 7c571066ad..edbe714f0b 100644
--- a/lib/datasource/dart/index.ts
+++ b/lib/datasource/dart/index.ts
@@ -1,9 +1,11 @@
-import got from '../../util/got';
+import { Http, HttpResponse } from '../../util/http';
import { logger } from '../../logger';
import { DatasourceError, ReleaseResult, GetReleasesConfig } from '../common';
export const id = 'dart';
+const http = new Http(id);
+
export async function getPkgReleases({
lookupName,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
@@ -18,14 +20,9 @@ export async function getPkgReleases({
};
}
- let raw: {
- body: DartResult;
- } = null;
+ let raw: HttpResponse<DartResult> = null;
try {
- raw = await got(pkgUrl, {
- hostType: id,
- json: true,
- });
+ raw = await http.getJson<DartResult>(pkgUrl);
} catch (err) {
if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
logger.debug({ lookupName }, `Dependency lookup failure: not found`);
diff --git a/lib/datasource/docker/__snapshots__/index.spec.ts.snap b/lib/datasource/docker/__snapshots__/index.spec.ts.snap
index 88b83ee46b..da12801f3f 100644
--- a/lib/datasource/docker/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/docker/__snapshots__/index.spec.ts.snap
@@ -6,6 +6,13 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (explicit
Array [
"https://index.docker.io/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -15,7 +22,14 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (explicit
"headers": Object {
"authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk",
},
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
"json": true,
+ "method": "get",
},
],
Array [
@@ -24,12 +38,25 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (explicit
"headers": Object {
"authorization": "Bearer some-token ",
},
- "json": true,
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
Array [
"https://index.docker.io/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -39,6 +66,13 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (explicit
"headers": Object {
"accept": "application/vnd.docker.distribution.manifest.v2+json",
},
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
],
@@ -94,6 +128,13 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (implicit
Array [
"https://index.docker.io/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -103,7 +144,14 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (implicit
"headers": Object {
"authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk",
},
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
"json": true,
+ "method": "get",
},
],
Array [
@@ -112,12 +160,25 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (implicit
"headers": Object {
"authorization": "Bearer some-token ",
},
- "json": true,
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
Array [
"https://index.docker.io/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -127,6 +188,13 @@ exports[`api/docker getPkgReleases adds library/ prefix for Docker Hub (implicit
"headers": Object {
"accept": "application/vnd.docker.distribution.manifest.v2+json",
},
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
],
@@ -182,6 +250,13 @@ exports[`api/docker getPkgReleases adds no library/ prefix for other registries
Array [
"https://k8s.gcr.io/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -191,7 +266,14 @@ exports[`api/docker getPkgReleases adds no library/ prefix for other registries
"headers": Object {
"authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk",
},
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
"json": true,
+ "method": "get",
},
],
Array [
@@ -200,12 +282,25 @@ exports[`api/docker getPkgReleases adds no library/ prefix for other registries
"headers": Object {
"authorization": "Bearer some-token ",
},
- "json": true,
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
Array [
"https://k8s.gcr.io/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -215,6 +310,13 @@ exports[`api/docker getPkgReleases adds no library/ prefix for other registries
"headers": Object {
"accept": "application/vnd.docker.distribution.manifest.v2+json",
},
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
],
@@ -270,6 +372,13 @@ exports[`api/docker getPkgReleases uses custom registry in depName 1`] = `
Array [
"https://registry.company.com/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -277,12 +386,25 @@ exports[`api/docker getPkgReleases uses custom registry in depName 1`] = `
"https://registry.company.com/v2/node/tags/list?n=10000",
Object {
"headers": Object {},
- "json": true,
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
Array [
"https://registry.company.com/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -318,6 +440,13 @@ Array [
Array [
"https://registry.company.com/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -325,19 +454,38 @@ Array [
"https://registry.company.com/v2/node/tags/list?n=10000",
Object {
"headers": Object {},
- "json": true,
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
Array [
"https://api.github.com/user/9287/repos?page=3&per_page=100",
Object {
"headers": Object {},
- "json": true,
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
Array [
"https://registry.company.com/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -347,6 +495,13 @@ Array [
"headers": Object {
"accept": "application/vnd.docker.distribution.manifest.v2+json",
},
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
]
@@ -358,6 +513,13 @@ exports[`api/docker getPkgReleases uses lower tag limit for ECR deps 1`] = `
Array [
"https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
@@ -365,12 +527,25 @@ exports[`api/docker getPkgReleases uses lower tag limit for ECR deps 1`] = `
"https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/node/tags/list?n=1000",
Object {
"headers": Object {},
- "json": true,
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
},
],
Array [
"https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
+ "hostType": "docker",
+ "method": "get",
"throwHttpErrors": false,
},
],
diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts
index b0f80b414e..05a7b84b7b 100644
--- a/lib/datasource/docker/index.ts
+++ b/lib/datasource/docker/index.ts
@@ -6,10 +6,9 @@ import wwwAuthenticate from 'www-authenticate';
import { OutgoingHttpHeaders } from 'http';
import AWS from 'aws-sdk';
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http, HttpResponse } from '../../util/http';
import * as hostRules from '../../util/host-rules';
import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
-import { GotResponse } from '../../platform';
import { HostRule } from '../../types';
// TODO: add got typings when available
@@ -17,6 +16,8 @@ import { HostRule } from '../../types';
export const id = 'docker';
+const http = new Http(id);
+
const ecrRegex = /\d+\.dkr\.ecr\.([-a-z0-9]+)\.amazonaws\.com/;
export interface RegistryRepository {
@@ -98,7 +99,9 @@ async function getAuthHeaders(
): Promise<OutgoingHttpHeaders | null> {
try {
const apiCheckUrl = `${registry}/v2/`;
- const apiCheckResponse = await got(apiCheckUrl, { throwHttpErrors: false });
+ const apiCheckResponse = await http.get(apiCheckUrl, {
+ throwHttpErrors: false,
+ });
if (apiCheckResponse.headers['www-authenticate'] === undefined) {
return {};
}
@@ -127,7 +130,7 @@ async function getAuthHeaders(
if (authenticateHeader.scheme.toUpperCase() === 'BASIC') {
logger.debug(`Using Basic auth for docker registry ${repository}`);
- await got(apiCheckUrl, opts);
+ await http.get(apiCheckUrl, opts);
return opts.headers;
}
@@ -136,7 +139,12 @@ async function getAuthHeaders(
logger.trace(
`Obtaining docker registry token for ${repository} using url ${authUrl}`
);
- const authResponse = (await got(authUrl, opts)).body;
+ const authResponse = (
+ await http.getJson<{ token?: string; access_token?: string }>(
+ authUrl,
+ opts
+ )
+ ).body;
const token = authResponse.token || authResponse.access_token;
// istanbul ignore if
@@ -187,7 +195,7 @@ function digestFromManifestStr(str: hasha.HashaInput): string {
return 'sha256:' + hasha(str, { algorithm: 'sha256' });
}
-function extractDigestFromResponse(manifestResponse: GotResponse): string {
+function extractDigestFromResponse(manifestResponse: HttpResponse): string {
if (manifestResponse.headers['docker-content-digest'] === undefined) {
return digestFromManifestStr(manifestResponse.body);
}
@@ -198,7 +206,7 @@ async function getManifestResponse(
registry: string,
repository: string,
tag: string
-): Promise<GotResponse> {
+): Promise<HttpResponse> {
logger.debug(`getManifestResponse(${registry}, ${repository}, ${tag})`);
try {
const headers = await getAuthHeaders(registry, repository);
@@ -208,7 +216,7 @@ async function getManifestResponse(
}
headers.accept = 'application/vnd.docker.distribution.manifest.v2+json';
const url = `${registry}/v2/${repository}/manifests/${tag}`;
- const manifestResponse = await got(url, {
+ const manifestResponse = await http.get(url, {
headers,
});
return manifestResponse;
@@ -347,7 +355,7 @@ async function getTags(
}
let page = 1;
do {
- const res = await got<{ tags: string[] }>(url, { json: true, headers });
+ const res = await http.getJson<{ tags: string[] }>(url, { headers });
tags = tags.concat(res.body.tags);
const linkHeader = parseLinkHeader(res.headers.link as string);
url =
@@ -415,31 +423,9 @@ async function getTags(
export function getConfigResponse(
url: string,
headers: OutgoingHttpHeaders
-): Promise<GotResponse> {
- return got(url, {
+): Promise<HttpResponse> {
+ return http.get(url, {
headers,
- hooks: {
- beforeRedirect: [
- (options: any): void => {
- if (
- options.search &&
- options.search.indexOf('X-Amz-Algorithm') !== -1
- ) {
- // if there is no port in the redirect URL string, then delete it from the redirect options.
- // This can be evaluated for removal after upgrading to Got v10
- const portInUrl = options.href.split('/')[2].split(':')[1];
- if (!portInUrl) {
- // eslint-disable-next-line no-param-reassign
- delete options.port; // Redirect will instead use 80 or 443 for HTTP or HTTPS respectively
- }
-
- // docker registry is hosted on amazon, redirect url includes authentication.
- // eslint-disable-next-line no-param-reassign
- delete options.headers.authorization;
- }
- },
- ],
- },
});
}
diff --git a/lib/datasource/galaxy/index.ts b/lib/datasource/galaxy/index.ts
index f51b9378dd..f4a2d53c1a 100644
--- a/lib/datasource/galaxy/index.ts
+++ b/lib/datasource/galaxy/index.ts
@@ -1,5 +1,5 @@
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import {
DatasourceError,
GetReleasesConfig,
@@ -9,6 +9,8 @@ import {
export const id = 'galaxy';
+const http = new Http(id);
+
export async function getPkgReleases({
lookupName,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
@@ -36,9 +38,7 @@ export async function getPkgReleases({
projectName;
const galaxyProjectUrl = baseUrl + userName + '/' + projectName;
try {
- let res: any = await got(galaxyAPIUrl, {
- hostType: id,
- });
+ let res: any = await http.get(galaxyAPIUrl);
if (!res || !res.body) {
logger.warn(
{ dependency: lookupName },
diff --git a/lib/datasource/go/index.ts b/lib/datasource/go/index.ts
index 5e9c85964b..8e31730d71 100644
--- a/lib/datasource/go/index.ts
+++ b/lib/datasource/go/index.ts
@@ -1,11 +1,13 @@
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import * as github from '../github-tags';
import { DigestConfig, GetReleasesConfig, ReleaseResult } from '../common';
import { regEx } from '../../util/regex';
export const id = 'go';
+const http = new Http(id);
+
interface DataSource {
datasource: string;
lookupName: string;
@@ -32,14 +34,10 @@ async function getDatasource(goModule: string): Promise<DataSource | null> {
}
const pkgUrl = `https://${goModule}?go-get=1`;
try {
- const res = (
- await got(pkgUrl, {
- hostType: id,
- })
- ).body;
- const sourceMatch = res.match(
- regEx(`<meta\\s+name="go-source"\\s+content="${goModule}\\s+([^\\s]+)`)
- );
+ const res = (await http.get(pkgUrl)).body;
+ const sourceMatch = regEx(
+ `<meta\\s+name="go-source"\\s+content="${goModule}\\s+([^\\s]+)`
+ ).exec(res);
if (sourceMatch) {
const [, goSourceUrl] = sourceMatch;
logger.debug({ goModule, goSourceUrl }, 'Go lookup source url');
diff --git a/lib/datasource/gradle-version/index.spec.ts b/lib/datasource/gradle-version/index.spec.ts
index e9d24669bc..49fb650f70 100644
--- a/lib/datasource/gradle-version/index.spec.ts
+++ b/lib/datasource/gradle-version/index.spec.ts
@@ -15,7 +15,9 @@ let config: any = {};
describe('datasource/gradle-version', () => {
describe('getPkgReleases', () => {
beforeEach(() => {
- config = {};
+ config = {
+ lookupName: 'abc',
+ };
jest.clearAllMocks();
global.repoCache = {};
return global.renovateCache.rmAll();
diff --git a/lib/datasource/gradle-version/index.ts b/lib/datasource/gradle-version/index.ts
index 6c4ac4945a..9e22603806 100644
--- a/lib/datasource/gradle-version/index.ts
+++ b/lib/datasource/gradle-version/index.ts
@@ -2,7 +2,7 @@ import is from '@sindresorhus/is';
import { coerce } from 'semver';
import { regEx } from '../../util/regex';
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import {
DatasourceError,
GetReleasesConfig,
@@ -12,18 +12,18 @@ import {
export const id = 'gradle-version';
+const http = new Http(id);
+
const GradleVersionsServiceUrl = 'https://services.gradle.org/versions/all';
interface GradleRelease {
- body: {
- snapshot?: boolean;
- nightly?: boolean;
- rcFor?: string;
- version: string;
- downloadUrl?: string;
- checksumUrl?: string;
- buildTime?: string;
- }[];
+ snapshot?: boolean;
+ nightly?: boolean;
+ rcFor?: string;
+ version: string;
+ downloadUrl?: string;
+ checksumUrl?: string;
+ buildTime?: string;
}
const buildTimeRegex = regEx(
@@ -50,10 +50,7 @@ export async function getPkgReleases({
const allReleases: Release[][] = await Promise.all(
versionsUrls.map(async url => {
try {
- const response: GradleRelease = await got(url, {
- hostType: id,
- json: true,
- });
+ const response = await http.getJson<GradleRelease[]>(url);
const releases = response.body
.filter(release => !release.snapshot && !release.nightly)
.filter(
diff --git a/lib/datasource/helm/index.ts b/lib/datasource/helm/index.ts
index c2ba01f962..2b7bbbdbc1 100644
--- a/lib/datasource/helm/index.ts
+++ b/lib/datasource/helm/index.ts
@@ -1,11 +1,13 @@
import yaml from 'js-yaml';
import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { logger } from '../../logger';
export const id = 'helm';
+const http = new Http(id);
+
export async function getRepositoryData(
repository: string
): Promise<ReleaseResult[]> {
@@ -17,7 +19,7 @@ export async function getRepositoryData(
}
let res: any;
try {
- res = await got('index.yaml', { hostType: id, baseUrl: repository });
+ res = await http.get('index.yaml', { baseUrl: repository });
if (!res || !res.body) {
logger.warn(`Received invalid response from ${repository}`);
return null;
diff --git a/lib/datasource/hex/index.ts b/lib/datasource/hex/index.ts
index 3067723be6..79dd1d8f94 100644
--- a/lib/datasource/hex/index.ts
+++ b/lib/datasource/hex/index.ts
@@ -1,9 +1,11 @@
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { DatasourceError, ReleaseResult, GetReleasesConfig } from '../common';
export const id = 'hex';
+const http = new Http(id);
+
interface HexRelease {
html_url: string;
meta?: { links?: Record<string, string> };
@@ -24,10 +26,7 @@ export async function getPkgReleases({
const hexPackageName = lookupName.split(':')[0];
const hexUrl = `https://hex.pm/api/packages/${hexPackageName}`;
try {
- const response = await got(hexUrl, {
- json: true,
- hostType: id,
- });
+ const response = await http.getJson<HexRelease>(hexUrl);
const hexRelease: HexRelease = response.body;
diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts
index 717762391b..aa1814739d 100644
--- a/lib/datasource/maven/util.ts
+++ b/lib/datasource/maven/util.ts
@@ -1,10 +1,12 @@
import url from 'url';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { logger } from '../../logger';
import { DatasourceError } from '../common';
import { id, MAVEN_REPO, MAVEN_REPO_DEPRECATED } from './common';
+const http = new Http(id);
+
const getHost = (x: string): string => new url.URL(x).host;
const defaultHosts = [MAVEN_REPO, MAVEN_REPO_DEPRECATED].map(getHost);
@@ -46,23 +48,7 @@ export async function downloadHttpProtocol(
): Promise<string | null> {
let raw: { body: string };
try {
- raw = await got(pkgUrl, {
- hostType,
- hooks: {
- beforeRedirect: [
- (options: any): void => {
- if (
- options.search &&
- options.search.indexOf('X-Amz-Algorithm') !== -1
- ) {
- // maven repository is hosted on amazon, redirect url includes authentication.
- // eslint-disable-next-line no-param-reassign
- delete options.auth;
- }
- },
- ],
- },
- });
+ raw = await http.get(pkgUrl.toString());
} catch (err) {
const failedUrl = pkgUrl.toString();
if (isNotFoundError(err)) {
diff --git a/lib/datasource/npm/get.ts b/lib/datasource/npm/get.ts
index 6a54de6e95..4788527560 100644
--- a/lib/datasource/npm/get.ts
+++ b/lib/datasource/npm/get.ts
@@ -7,12 +7,14 @@ import { OutgoingHttpHeaders } from 'http';
import is from '@sindresorhus/is';
import { logger } from '../../logger';
import { find } from '../../util/host-rules';
-import got, { GotJSONOptions } from '../../util/got';
+import { Http, HttpOptions } from '../../util/http';
import { maskToken } from '../../util/mask';
import { getNpmrc } from './npmrc';
import { DatasourceError, Release, ReleaseResult } from '../common';
import { id } from './common';
+const http = new Http(id);
+
let memcache = {};
export function resetMemCache(): void {
@@ -127,14 +129,12 @@ export async function getDependency(
try {
const useCache = retries === 3; // Disable cache if we're retrying
- const opts: GotJSONOptions = {
- hostType: id,
- json: true,
- retry: 5,
+ const opts: HttpOptions = {
headers,
useCache,
};
- const raw = await got(pkgUrl, opts);
+ // TODO: fix type
+ const raw = await http.getJson<any>(pkgUrl, opts);
if (retries < 3) {
logger.debug({ pkgUrl, retries }, 'Recovered from npm error');
}
@@ -249,6 +249,7 @@ export async function getDependency(
return null;
}
if (uri.host === 'registry.npmjs.org') {
+ // istanbul ignore if
if (
(err.name === 'ParseError' ||
err.code === 'ECONNRESET' ||
diff --git a/lib/datasource/nuget/v2.ts b/lib/datasource/nuget/v2.ts
index 89c8afbf39..86b765dc93 100644
--- a/lib/datasource/nuget/v2.ts
+++ b/lib/datasource/nuget/v2.ts
@@ -1,10 +1,12 @@
import { XmlDocument, XmlElement } from 'xmldoc';
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { ReleaseResult } from '../common';
import { id } from './common';
+const http = new Http(id);
+
function getPkgProp(pkgInfo: XmlElement, propName: string): string {
return pkgInfo.childNamed('m:properties').childNamed(`d:${propName}`).val;
}
@@ -20,17 +22,7 @@ export async function getPkgReleases(
try {
let pkgUrlList = `${feedUrl}/FindPackagesById()?id=%27${pkgName}%27&$select=Version,IsLatestVersion,ProjectUrl`;
do {
- const pkgVersionsListRaw = await got(pkgUrlList, {
- hostType: id,
- });
- if (pkgVersionsListRaw.statusCode !== 200) {
- logger.debug(
- { dependency: pkgName, pkgVersionsListRaw },
- `nuget registry failure: status code != 200`
- );
- return null;
- }
-
+ const pkgVersionsListRaw = await http.get(pkgUrlList);
const pkgVersionsListDoc = new XmlDocument(pkgVersionsListRaw.body);
const pkgInfoList = pkgVersionsListDoc.childrenNamed('entry');
diff --git a/lib/datasource/nuget/v3.ts b/lib/datasource/nuget/v3.ts
index 9bbb381014..75d57e170a 100644
--- a/lib/datasource/nuget/v3.ts
+++ b/lib/datasource/nuget/v3.ts
@@ -1,11 +1,13 @@
import * as semver from 'semver';
import { XmlDocument } from 'xmldoc';
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { ReleaseResult } from '../common';
import { id } from './common';
+const http = new Http(id);
+
// https://api.nuget.org/v3/index.json is a default official nuget feed
const defaultNugetFeed = 'https://api.nuget.org/v3/index.json';
const cacheNamespace = 'datasource-nuget';
@@ -29,17 +31,8 @@ export async function getQueryUrl(url: string): Promise<string | null> {
}
try {
- const servicesIndexRaw = await got(url, {
- json: true,
- hostType: id,
- });
- if (servicesIndexRaw.statusCode !== 200) {
- logger.debug(
- { dependency: url, servicesIndexRaw },
- `nuget registry failure: status code != 200`
- );
- return null;
- }
+ // TODO: fix types
+ const servicesIndexRaw = await http.getJson<any>(url);
const searchQueryService = servicesIndexRaw.body.resources.find(
resource =>
resource['@type'] && resource['@type'].startsWith(resourceType)
@@ -78,18 +71,8 @@ export async function getPkgReleases(
releases: [],
};
try {
- const pkgUrlListRaw = await got(queryUrl, {
- json: true,
- hostType: id,
- });
- if (pkgUrlListRaw.statusCode !== 200) {
- logger.debug(
- { dependency: pkgName, pkgUrlListRaw },
- `nuget registry failure: status code != 200`
- );
- return null;
- }
-
+ // TODO: fix types
+ const pkgUrlListRaw = await http.getJson<any>(queryUrl);
const match = pkgUrlListRaw.body.data.find(
item => item.id.toLowerCase() === pkgName.toLowerCase()
);
@@ -121,7 +104,7 @@ export async function getPkgReleases(
const nugetOrgApi = `https://api.nuget.org/v3-flatcontainer/${pkgName.toLowerCase()}/${lastVersion}/${pkgName.toLowerCase()}.nuspec`;
let metaresult: { body: string };
try {
- metaresult = await got(nugetOrgApi, { hostType: id });
+ metaresult = await http.get(nugetOrgApi);
} catch (err) /* istanbul ignore next */ {
logger.debug(
`Cannot fetch metadata for ${pkgName} using popped version ${lastVersion}`
diff --git a/lib/datasource/orb/index.spec.ts b/lib/datasource/orb/index.spec.ts
index 5ede9da3e9..b491e0793c 100644
--- a/lib/datasource/orb/index.spec.ts
+++ b/lib/datasource/orb/index.spec.ts
@@ -34,7 +34,7 @@ describe('datasource/orb', () => {
return global.renovateCache.rmAll();
});
it('returns null for empty result', async () => {
- got.post.mockReturnValueOnce({ body: {} });
+ got.mockReturnValueOnce({ body: {} });
expect(
await datasource.getPkgReleases({
lookupName: 'hyper-expanse/library-release-workflows',
@@ -42,7 +42,7 @@ describe('datasource/orb', () => {
).toBeNull();
});
it('returns null for missing orb', async () => {
- got.post.mockReturnValueOnce({ body: { data: {} } });
+ got.mockReturnValueOnce({ body: { data: {} } });
expect(
await datasource.getPkgReleases({
lookupName: 'hyper-expanse/library-release-wonkflows',
@@ -72,7 +72,7 @@ describe('datasource/orb', () => {
).toBeNull();
});
it('processes real data', async () => {
- got.post.mockReturnValueOnce({
+ got.mockReturnValueOnce({
body: orbData,
});
const res = await datasource.getPkgReleases({
@@ -83,7 +83,7 @@ describe('datasource/orb', () => {
});
it('processes homeUrl', async () => {
orbData.data.orb.homeUrl = 'https://google.com';
- got.post.mockReturnValueOnce({
+ got.mockReturnValueOnce({
body: orbData,
});
const res = await datasource.getPkgReleases({
diff --git a/lib/datasource/orb/index.ts b/lib/datasource/orb/index.ts
index 2c6903b3b8..9d7388966b 100644
--- a/lib/datasource/orb/index.ts
+++ b/lib/datasource/orb/index.ts
@@ -1,9 +1,11 @@
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'orb';
+const http = new Http(id);
+
interface OrbRelease {
homeUrl?: string;
versions: {
@@ -38,11 +40,8 @@ export async function getPkgReleases({
};
try {
const res: OrbRelease = (
- await got.post(url, {
+ await http.postJson<{ data: { orb: OrbRelease } }>(url, {
body,
- hostType: id,
- json: true,
- retry: 5,
})
).body.data.orb;
if (!res) {
diff --git a/lib/datasource/packagist/index.ts b/lib/datasource/packagist/index.ts
index 8c989a7ca3..0755d2a4aa 100644
--- a/lib/datasource/packagist/index.ts
+++ b/lib/datasource/packagist/index.ts
@@ -4,17 +4,17 @@ import URL from 'url';
import delay from 'delay';
import pAll from 'p-all';
import { logger } from '../../logger';
-
-import got, { GotJSONOptions } from '../../util/got';
+import { Http, HttpOptions } from '../../util/http';
import * as hostRules from '../../util/host-rules';
import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'packagist';
-function getHostOpts(url: string): GotJSONOptions {
- const opts: GotJSONOptions = {
- json: true,
- };
+const http = new Http(id);
+
+// We calculate auth at this datasource layer so that we can know whether it's safe to cache or not
+function getHostOpts(url: string): HttpOptions {
+ const opts: HttpOptions = {};
const { username, password } = hostRules.find({
hostType: id,
url,
@@ -47,7 +47,7 @@ async function getRegistryMeta(regUrl: string): Promise<RegistryMeta | null> {
try {
const url = URL.resolve(regUrl.replace(/\/?$/, '/'), 'packages.json');
const opts = getHostOpts(url);
- const res: PackageMeta = (await got(url, opts)).body;
+ const res = (await http.getJson<PackageMeta>(url, opts)).body;
const meta: RegistryMeta = {};
meta.packages = res.packages;
if (res.includes) {
@@ -107,7 +107,8 @@ async function getPackagistFile(
const fileName = key.replace('%hash%', sha256);
const opts = getHostOpts(regUrl);
if (opts.auth || (opts.headers && opts.headers.authorization)) {
- return (await got(regUrl + '/' + fileName, opts)).body;
+ return (await http.getJson<PackagistFile>(regUrl + '/' + fileName, opts))
+ .body;
}
const cacheNamespace = 'datasource-packagist-files';
const cacheKey = regUrl + key;
@@ -117,7 +118,8 @@ async function getPackagistFile(
if (cachedResult && cachedResult.sha256 === sha256) {
return cachedResult.res;
}
- const res = (await got(regUrl + '/' + fileName, opts)).body;
+ const res = (await http.getJson<PackagistFile>(regUrl + '/' + fileName, opts))
+ .body;
const cacheMinutes = 1440; // 1 day
await renovateCache.set(
cacheNamespace,
@@ -223,12 +225,8 @@ async function packagistOrgLookup(name: string): Promise<ReleaseResult> {
let dep: ReleaseResult = null;
const regUrl = 'https://packagist.org';
const pkgUrl = URL.resolve(regUrl, `/p/${name}.json`);
- const res = (
- await got(pkgUrl, {
- json: true,
- retry: 5,
- })
- ).body.packages[name];
+ // TODO: fix types
+ const res = (await http.getJson<any>(pkgUrl)).body.packages[name];
if (res) {
dep = extractDepReleases(res);
dep.name = name;
@@ -276,7 +274,10 @@ async function packageLookup(
.replace('%hash%', providerPackages[name])
);
const opts = getHostOpts(regUrl);
- const versions = (await got(pkgUrl, opts)).body.packages[name];
+ // TODO: fix types
+ const versions = (await http.getJson<any>(pkgUrl, opts)).body.packages[
+ name
+ ];
const dep = extractDepReleases(versions);
dep.name = name;
logger.trace({ dep }, 'dep');
diff --git a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
index b1e8f65fac..7e9db5af86 100644
--- a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
@@ -215,23 +215,15 @@ Object {
exports[`datasource/pypi getPkgReleases supports custom datasource url 1`] = `
Array [
Array [
- Url {
- "auth": null,
- "hash": null,
- "host": "custom.pypi.net",
- "hostname": "custom.pypi.net",
- "href": "https://custom.pypi.net/foo/azure-cli-monitor/json",
- "path": "/foo/azure-cli-monitor/json",
- "pathname": "/foo/azure-cli-monitor/json",
- "port": null,
- "protocol": "https:",
- "query": null,
- "search": null,
- "slashes": true,
- },
+ "https://custom.pypi.net/foo/azure-cli-monitor/json",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
"hostType": "pypi",
- "json": true,
+ "method": "get",
},
],
]
@@ -240,23 +232,15 @@ Array [
exports[`datasource/pypi getPkgReleases supports custom datasource url from environmental variable 1`] = `
Array [
Array [
- Url {
- "auth": null,
- "hash": null,
- "host": "my.pypi.python",
- "hostname": "my.pypi.python",
- "href": "https://my.pypi.python/pypi/azure-cli-monitor/json",
- "path": "/pypi/azure-cli-monitor/json",
- "pathname": "/pypi/azure-cli-monitor/json",
- "port": null,
- "protocol": "https:",
- "query": null,
- "search": null,
- "slashes": true,
- },
+ "https://my.pypi.python/pypi/azure-cli-monitor/json",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
"hostType": "pypi",
- "json": true,
+ "method": "get",
},
],
]
@@ -265,43 +249,27 @@ Array [
exports[`datasource/pypi getPkgReleases supports multiple custom datasource urls 1`] = `
Array [
Array [
- Url {
- "auth": null,
- "hash": null,
- "host": "custom.pypi.net",
- "hostname": "custom.pypi.net",
- "href": "https://custom.pypi.net/foo/azure-cli-monitor/json",
- "path": "/foo/azure-cli-monitor/json",
- "pathname": "/foo/azure-cli-monitor/json",
- "port": null,
- "protocol": "https:",
- "query": null,
- "search": null,
- "slashes": true,
- },
+ "https://custom.pypi.net/foo/azure-cli-monitor/json",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
"hostType": "pypi",
- "json": true,
+ "method": "get",
},
],
Array [
- Url {
- "auth": null,
- "hash": null,
- "host": "second-index",
- "hostname": "second-index",
- "href": "https://second-index/foo/azure-cli-monitor/json",
- "path": "/foo/azure-cli-monitor/json",
- "pathname": "/foo/azure-cli-monitor/json",
- "port": null,
- "protocol": "https:",
- "query": null,
- "search": null,
- "slashes": true,
- },
+ "https://second-index/foo/azure-cli-monitor/json",
Object {
+ "hooks": Object {
+ "beforeRedirect": Array [
+ [Function],
+ ],
+ },
"hostType": "pypi",
- "json": true,
+ "method": "get",
},
],
]
diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts
index ba210f0536..bbb7a30884 100644
--- a/lib/datasource/pypi/index.ts
+++ b/lib/datasource/pypi/index.ts
@@ -3,11 +3,13 @@ import url from 'url';
import { parse } from 'node-html-parser';
import { logger } from '../../logger';
import { matches } from '../../versioning/pep440';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'pypi';
+const http = new Http(id);
+
function normalizeName(input: string): string {
return input.toLowerCase().replace(/(-|\.)/g, '_');
}
@@ -39,10 +41,8 @@ async function getDependency(
const lookupUrl = url.resolve(hostUrl, `${packageName}/json`);
const dependency: ReleaseResult = { releases: null };
logger.trace({ lookupUrl }, 'Pypi api got lookup');
- const rep = await got(url.parse(lookupUrl), {
- json: true,
- hostType: id,
- });
+ // TODO: fix type
+ const rep = await http.getJson<any>(lookupUrl);
const dep = rep && rep.body;
if (!dep) {
logger.trace({ dependency: packageName }, 'pip package not found');
@@ -123,9 +123,7 @@ async function getSimpleDependency(
const lookupUrl = url.resolve(hostUrl, `${packageName}`);
try {
const dependency: ReleaseResult = { releases: null };
- const response = await got<string>(url.parse(lookupUrl), {
- hostType: id,
- });
+ const response = await http.get(lookupUrl);
const dep = response && response.body;
if (!dep) {
logger.trace({ dependency: packageName }, 'pip package not found');
diff --git a/lib/datasource/ruby-version/index.ts b/lib/datasource/ruby-version/index.ts
index f19c3270be..d449ae2f2c 100644
--- a/lib/datasource/ruby-version/index.ts
+++ b/lib/datasource/ruby-version/index.ts
@@ -1,11 +1,13 @@
import { parse } from 'node-html-parser';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { isVersion } from '../../versioning/ruby';
import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'ruby-version';
+const http = new Http(id);
+
const rubyVersionsUrl = 'https://www.ruby-lang.org/en/downloads/releases/';
export async function getPkgReleases(
@@ -27,7 +29,7 @@ export async function getPkgReleases(
sourceUrl: 'https://github.com/ruby/ruby',
releases: [],
};
- const response = await got(rubyVersionsUrl);
+ const response = await http.get(rubyVersionsUrl);
const root: any = parse(response.body);
const rows = root.querySelector('.release-list').querySelectorAll('tr');
for (const row of rows) {
diff --git a/lib/datasource/rubygems/get-rubygems-org.ts b/lib/datasource/rubygems/get-rubygems-org.ts
index fa9f2d94e1..71d5916283 100644
--- a/lib/datasource/rubygems/get-rubygems-org.ts
+++ b/lib/datasource/rubygems/get-rubygems-org.ts
@@ -1,7 +1,10 @@
import { hrtime } from 'process';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { logger } from '../../logger';
import { DatasourceError, ReleaseResult } from '../common';
+import { id } from './common';
+
+const http = new Http(id);
let lastSync = new Date('2000-01-01');
let packageReleases: Record<string, string[]> = Object.create(null); // Because we might need a "constructor" key
@@ -22,7 +25,7 @@ async function updateRubyGemsVersions(): Promise<void> {
try {
logger.debug('Rubygems: Fetching rubygems.org versions');
const startTime = hrtime();
- newLines = (await got(url, options)).body;
+ newLines = (await http.get(url, options)).body;
const duration = hrtime(startTime);
const seconds = Math.round(duration[0] + duration[1] / 1e9);
logger.debug({ seconds }, 'Rubygems: Fetched rubygems.org versions');
diff --git a/lib/datasource/rubygems/get.ts b/lib/datasource/rubygems/get.ts
index 9a32f01c65..fec4e2cca3 100644
--- a/lib/datasource/rubygems/get.ts
+++ b/lib/datasource/rubygems/get.ts
@@ -1,12 +1,13 @@
import { OutgoingHttpHeaders } from 'http';
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { maskToken } from '../../util/mask';
-import retriable from './retriable';
import { UNAUTHORIZED, FORBIDDEN, NOT_FOUND } from './errors';
import { ReleaseResult } from '../common';
import { id } from './common';
+const http = new Http(id);
+
const INFO_PATH = '/api/v1/gems';
const VERSIONS_PATH = '/api/v1/versions';
@@ -35,16 +36,13 @@ const getHeaders = (): OutgoingHttpHeaders => {
};
const fetch = async ({ dependency, registry, path }): Promise<any> => {
- const json = true;
-
- const retry = { retries: retriable() };
const headers = getHeaders();
const name = `${path}/${dependency}.json`;
const baseUrl = registry;
logger.trace({ dependency }, `RubyGems lookup request: ${baseUrl} ${name}`);
- const response = (await got(name, { retry, json, baseUrl, headers })) || {
+ const response = (await http.getJson(name, { baseUrl, headers })) || {
body: undefined,
};
diff --git a/lib/datasource/rubygems/retriable.spec.ts b/lib/datasource/rubygems/retriable.spec.ts
deleted file mode 100644
index 19ddba9fe7..0000000000
--- a/lib/datasource/rubygems/retriable.spec.ts
+++ /dev/null
@@ -1,31 +0,0 @@
-import retriable from './retriable';
-
-describe('datasource/rubygems/retriable', () => {
- it('returns 0 when numberOfRetries equals 0', () => {
- expect(retriable(0)(null, null)).toEqual(0);
- });
-
- it('returns retry after header + 1 second if request is banned', () => {
- expect(
- retriable(1)(null, {
- statusCode: 429,
- headers: { 'retry-after': '5' },
- })
- ).toEqual(6000);
-
- expect(
- retriable(1)(null, {
- statusCode: 503,
- headers: { 'retry-after': '9' },
- })
- ).toEqual(10000);
- });
-
- it('returns default delay if request is not banned', () => {
- expect(retriable(1)(null, { statusCode: 500 })).toEqual(2000);
- });
-
- it('uses default numberOfRetries', () => {
- expect(retriable()(null, { statusCode: 500 })).toEqual(1000);
- });
-});
diff --git a/lib/datasource/rubygems/retriable.ts b/lib/datasource/rubygems/retriable.ts
deleted file mode 100644
index 8d85ed2c4e..0000000000
--- a/lib/datasource/rubygems/retriable.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-import got from 'got';
-import { logger } from '../../logger';
-import {
- UNAUTHORIZED,
- FORBIDDEN,
- REQUEST_TIMEOUT,
- TOO_MANY_REQUEST,
- SERVICE_UNAVAILABLE,
-} from './errors';
-
-const DEFAULT_BANNED_RETRY_AFTER = 600;
-const NUMBER_OF_RETRIES = 2;
-
-const getBannedDelay = (retryAfter: string): number =>
- (parseInt(retryAfter, 10) || DEFAULT_BANNED_RETRY_AFTER) + 1;
-const getDefaultDelay = (count: number): number =>
- +(NUMBER_OF_RETRIES / count).toFixed(3);
-
-const getErrorMessage = (status: number): string => {
- // istanbul ignore next
- switch (status) {
- case UNAUTHORIZED:
- case FORBIDDEN:
- return `RubyGems registry: Authentication failed.`;
- case TOO_MANY_REQUEST:
- return `RubyGems registry: Too Many Requests.`;
- case REQUEST_TIMEOUT:
- case SERVICE_UNAVAILABLE:
- return `RubyGems registry: Temporary Unavailable`;
- default:
- return `RubyGems registry: Internal Server Error`;
- }
-};
-
-// TODO: workaround because got does not export HTTPError, should be moved to `lib/util/got`
-export type HTTPError = InstanceType<got.GotInstance['HTTPError']>;
-
-export default (numberOfRetries = NUMBER_OF_RETRIES): got.RetryFunction => (
- _?: number,
- err?: Partial<HTTPError>
-): number => {
- if (numberOfRetries === 0) {
- return 0;
- }
-
- const { headers, statusCode } = err;
-
- // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
- const isBanned = [TOO_MANY_REQUEST, SERVICE_UNAVAILABLE].includes(statusCode);
- const delaySec = isBanned
- ? getBannedDelay(headers['retry-after'])
- : getDefaultDelay(numberOfRetries);
-
- // eslint-disable-next-line
- numberOfRetries--;
-
- const errorMessage = getErrorMessage(statusCode);
- const message = `${errorMessage} Retry in ${delaySec} seconds.`;
-
- logger.debug(message);
-
- return delaySec * 1000;
-};
diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts
index a9578d7753..5e44921767 100644
--- a/lib/datasource/terraform-module/index.ts
+++ b/lib/datasource/terraform-module/index.ts
@@ -1,10 +1,12 @@
import is from '@sindresorhus/is';
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'terraform-module';
+const http = new Http(id);
+
interface RegistryRepository {
registry: string;
repository: string;
@@ -72,12 +74,7 @@ export async function getPkgReleases({
return cachedResult;
}
try {
- const res: TerraformRelease = (
- await got(pkgUrl, {
- json: true,
- hostType: id,
- })
- ).body;
+ const res = (await http.getJson<TerraformRelease>(pkgUrl)).body;
const returnedName = res.namespace + '/' + res.name + '/' + res.provider;
if (returnedName !== repository) {
logger.warn({ pkgUrl }, 'Terraform registry result mismatch');
diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts
index 0c3544d3bd..480e1d5769 100644
--- a/lib/datasource/terraform-provider/index.ts
+++ b/lib/datasource/terraform-provider/index.ts
@@ -1,9 +1,11 @@
import { logger } from '../../logger';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'terraform-provider';
+const http = new Http(id);
+
interface TerraformProvider {
namespace: string;
name: string;
@@ -35,12 +37,7 @@ export async function getPkgReleases({
return cachedResult;
}
try {
- const res: TerraformProvider = (
- await got(pkgUrl, {
- json: true,
- hostType: id,
- })
- ).body;
+ const res = (await http.getJson<TerraformProvider>(pkgUrl)).body;
// Simplify response before caching and returning
const dep: ReleaseResult = {
name: repository,
diff --git a/lib/manager/bazel/update.ts b/lib/manager/bazel/update.ts
index 2c0670b256..11e7efb6ef 100644
--- a/lib/manager/bazel/update.ts
+++ b/lib/manager/bazel/update.ts
@@ -1,9 +1,11 @@
import { fromStream } from 'hasha';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { logger } from '../../logger';
import { UpdateDependencyConfig } from '../common';
import { regEx } from '../../util/regex';
+const http = new Http('bazel');
+
function updateWithNewVersion(
content: string,
currentValue: string,
@@ -52,7 +54,7 @@ async function getHashFromUrl(url: string): Promise<string | null> {
return cachedResult;
}
try {
- const hash = await fromStream(got.stream(url), {
+ const hash = await fromStream(http.stream(url), {
algorithm: 'sha256',
});
const cacheMinutes = 3 * 24 * 60; // 3 days
diff --git a/lib/manager/gradle-wrapper/update.ts b/lib/manager/gradle-wrapper/update.ts
index fe9e77d612..051c66b9cb 100644
--- a/lib/manager/gradle-wrapper/update.ts
+++ b/lib/manager/gradle-wrapper/update.ts
@@ -1,4 +1,4 @@
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { logger } from '../../logger';
import { UpdateDependencyConfig } from '../common';
import {
@@ -7,14 +7,16 @@ import {
VERSION_REGEX,
} from './search';
+const http = new Http('gradle-wrapper');
+
function replaceType(url: string): string {
return url.replace('bin', 'all');
}
async function getChecksum(url: string): Promise<string> {
try {
- const response = await got(url);
- return response.body as string;
+ const response = await http.get(url);
+ return response.body;
} catch (err) {
if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
logger.debug('Gradle checksum lookup failure: not found');
diff --git a/lib/manager/homebrew/update.ts b/lib/manager/homebrew/update.ts
index 39bb54ea97..cdd58b6665 100644
--- a/lib/manager/homebrew/update.ts
+++ b/lib/manager/homebrew/update.ts
@@ -2,10 +2,12 @@ import { fromStream } from 'hasha';
import { coerce } from 'semver';
import { parseUrlPath } from './extract';
import { skip, isSpace, removeComments } from './util';
-import got from '../../util/got';
+import { Http } from '../../util/http';
import { logger } from '../../logger';
import { UpdateDependencyConfig } from '../common';
+const http = new Http('homebrew');
+
function replaceUrl(
idx: number,
content: string,
@@ -160,7 +162,7 @@ export async function updateDependency({
}/releases/download/${upgrade.newValue}/${
upgrade.managerData.repoName
}-${coerce(upgrade.newValue)}.tar.gz`;
- newSha256 = await fromStream(got.stream(newUrl), {
+ newSha256 = await fromStream(http.stream(newUrl), {
algorithm: 'sha256',
});
} catch (errOuter) {
@@ -169,7 +171,7 @@ export async function updateDependency({
);
try {
newUrl = `https://github.com/${upgrade.managerData.ownerName}/${upgrade.managerData.repoName}/archive/${upgrade.newValue}.tar.gz`;
- newSha256 = await fromStream(got.stream(newUrl), {
+ newSha256 = await fromStream(http.stream(newUrl), {
algorithm: 'sha256',
});
} catch (errInner) {
diff --git a/lib/util/got/common.ts b/lib/util/got/common.ts
index b431d7fbe9..23ff6abde7 100644
--- a/lib/util/got/common.ts
+++ b/lib/util/got/common.ts
@@ -2,6 +2,7 @@ import got from 'got';
import { Url } from 'url';
export interface Options {
+ hooks?: any;
hostType?: string;
search?: string;
token?: string;
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
new file mode 100644
index 0000000000..0aa11e1c2c
--- /dev/null
+++ b/lib/util/http/index.ts
@@ -0,0 +1,104 @@
+import is from '@sindresorhus/is/dist';
+import URL from 'url';
+import got from '../got';
+
+interface OutgoingHttpHeaders {
+ [header: string]: number | string | string[] | undefined;
+}
+
+export interface HttpOptions {
+ auth?: string;
+ baseUrl?: string;
+ headers?: OutgoingHttpHeaders;
+ throwHttpErrors?: boolean;
+ useCache?: boolean;
+}
+
+export interface HttpPostOptions extends HttpOptions {
+ body: unknown;
+}
+
+interface InternalHttpOptions extends HttpOptions {
+ json?: boolean;
+ method?: 'get' | 'post';
+}
+
+export interface HttpResponse<T = string> {
+ body: T;
+ headers: any;
+}
+
+export class Http {
+ constructor(private hostType: string, private options?: HttpOptions) {}
+
+ private async request(
+ url: string | URL,
+ options?: InternalHttpOptions
+ ): Promise<HttpResponse | null> {
+ let resolvedUrl = url.toString();
+ if (options?.baseUrl) {
+ resolvedUrl = URL.resolve(options.baseUrl, resolvedUrl);
+ }
+ // TODO: deep merge in order to merge headers
+ const combinedOptions: any = {
+ method: 'get',
+ ...this.options,
+ hostType: this.hostType,
+ ...options,
+ };
+ combinedOptions.hooks = {
+ beforeRedirect: [
+ (opts: any): void => {
+ // Check if request has been redirected to Amazon
+ if (opts.search?.includes('X-Amz-Algorithm')) {
+ // if there is no port in the redirect URL string, then delete it from the redirect options.
+ // This can be evaluated for removal after upgrading to Got v10
+ const portInUrl = opts.href.split('/')[2].split(':')[1];
+ if (!portInUrl) {
+ // eslint-disable-next-line no-param-reassign
+ delete opts.port; // Redirect will instead use 80 or 443 for HTTP or HTTPS respectively
+ }
+
+ // registry is hosted on amazon, redirect url includes authentication.
+ delete opts.headers.authorization; // eslint-disable-line no-param-reassign
+ delete opts.auth; // eslint-disable-line no-param-reassign
+ }
+ },
+ ],
+ };
+ const res = await got(resolvedUrl, combinedOptions);
+ return { body: res.body, headers: res.headers };
+ }
+
+ get(url: string, options: HttpOptions = {}): Promise<HttpResponse> {
+ return this.request(url, options);
+ }
+
+ async getJson<T = unknown>(
+ url: string,
+ options: HttpOptions = {}
+ ): Promise<HttpResponse<T>> {
+ const res = await this.request(url, options);
+ const body = is.string(res.body) ? JSON.parse(res.body) : res.body;
+ return { ...res, body };
+ }
+
+ async postJson<T = unknown>(
+ url: string,
+ options: HttpPostOptions
+ ): Promise<HttpResponse<T>> {
+ const res = await this.request(url, { ...options, method: 'post' });
+ const body = is.string(res.body) ? JSON.parse(res.body) : res.body;
+ return { ...res, body };
+ }
+
+ stream(url: string, options?: HttpOptions): NodeJS.ReadableStream {
+ const combinedOptions: any = {
+ method: 'get',
+ ...this.options,
+ hostType: this.hostType,
+ ...options,
+ };
+ return got.stream(url, combinedOptions);
+ }
+}
diff --git a/tools/jest-gh-reporter.ts b/tools/jest-gh-reporter.ts
index 856de776f5..8b33cc512d 100644
--- a/tools/jest-gh-reporter.ts
+++ b/tools/jest-gh-reporter.ts
@@ -135,8 +135,9 @@ class GitHubReporter extends BaseReporter {
info(`repo: ${owner} / ${repo}`);
info(`sha: ${ref}`);
- const output: Octokit.ChecksUpdateParamsOutput = {
+ const output: Octokit.ChecksCreateParamsOutput = {
summary: 'Jest test results',
+ title: 'Jest',
};
if (annotations.length) {
output.annotations = annotations;
@@ -171,7 +172,7 @@ class GitHubReporter extends BaseReporter {
head_sha: ref,
completed_at: new Date().toISOString(),
conclusion: success ? 'success' : 'failure',
- output: { ...output, title: 'Jest' },
+ output: { ...output },
});
}
}
--
GitLab