From 8785f70b93e876ba5187c7083be055e43e771e65 Mon Sep 17 00:00:00 2001
From: Michael Kriese <michael.kriese@visualon.de>
Date: Mon, 6 Sep 2021 21:07:52 +0200
Subject: [PATCH] fix(core/http): fallback to github /gitlab hosttype (#11605)
* fix(core/http): fallback to github /gitlab hosttype
* fix: remove generic host rule
* Update lib/util/http/index.ts
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
.../platform.spec.ts} | 31 ++++++++---
lib/constants/platforms.ts | 13 +++++
.../github/__snapshots__/index.spec.ts.snap | 27 ----------
lib/platform/github/index.spec.ts | 11 ----
lib/platform/github/index.ts | 8 +--
lib/types/github.ts | 8 ---
lib/types/index.ts | 1 -
lib/types/platform/gitlab/index.spec.ts | 25 ---------
lib/types/platform/gitlab/index.ts | 7 ---
lib/util/http/auth.ts | 12 +++--
lib/util/http/host-rules.spec.ts | 41 +++++++++++++++
lib/util/http/host-rules.ts | 51 ++++++++++++++++---
lib/util/http/index.spec.ts | 7 +++
lib/util/http/index.ts | 16 ++++--
14 files changed, 149 insertions(+), 109 deletions(-)
rename lib/{types/github.spec.ts => constants/platform.spec.ts} (50%)
delete mode 100644 lib/types/github.ts
delete mode 100644 lib/types/platform/gitlab/index.spec.ts
diff --git a/lib/types/github.spec.ts b/lib/constants/platform.spec.ts
similarity index 50%
rename from lib/types/github.spec.ts
rename to lib/constants/platform.spec.ts
index 8e79ef350a..22984c3e06 100644
--- a/lib/types/github.spec.ts
+++ b/lib/constants/platform.spec.ts
@@ -1,13 +1,32 @@
-import {
- PLATFORM_TYPE_GITHUB,
- PLATFORM_TYPE_GITLAB,
-} from '../constants/platforms';
import { id as GH_RELEASES_DS } from '../datasource/github-releases';
import { id as GH_TAGS_DS } from '../datasource/github-tags';
+import { GitlabReleasesDatasource } from '../datasource/gitlab-releases';
+import { id as GL_TAGS_DS } from '../datasource/gitlab-tags';
import { id as POD_DS } from '../datasource/pod';
-import { GITHUB_API_USING_HOST_TYPES } from './github';
+import {
+ GITHUB_API_USING_HOST_TYPES,
+ GITLAB_API_USING_HOST_TYPES,
+ PLATFORM_TYPE_GITHUB,
+ PLATFORM_TYPE_GITLAB,
+} from './platforms';
+
+describe('constants/platform', () => {
+ it('should be part of the GITLAB_API_USING_HOST_TYPES', () => {
+ expect(GITLAB_API_USING_HOST_TYPES.includes(GL_TAGS_DS)).toBeTrue();
+ expect(
+ GITLAB_API_USING_HOST_TYPES.includes(GitlabReleasesDatasource.id)
+ ).toBeTrue();
+ expect(
+ GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITLAB)
+ ).toBeTrue();
+ });
+
+ it('should be not part of the GITLAB_API_USING_HOST_TYPES ', () => {
+ expect(
+ GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITHUB)
+ ).toBeFalse();
+ });
-describe('types/github', () => {
it('should be part of the GITHUB_API_USING_HOST_TYPES ', () => {
expect(GITHUB_API_USING_HOST_TYPES.includes(GH_TAGS_DS)).toBeTrue();
expect(GITHUB_API_USING_HOST_TYPES.includes(GH_RELEASES_DS)).toBeTrue();
diff --git a/lib/constants/platforms.ts b/lib/constants/platforms.ts
index 07b779c55e..94725ae52d 100644
--- a/lib/constants/platforms.ts
+++ b/lib/constants/platforms.ts
@@ -4,3 +4,16 @@ export const PLATFORM_TYPE_BITBUCKET_SERVER = 'bitbucket-server';
export const PLATFORM_TYPE_GITEA = 'gitea';
export const PLATFORM_TYPE_GITHUB = 'github';
export const PLATFORM_TYPE_GITLAB = 'gitlab';
+
+export const GITHUB_API_USING_HOST_TYPES = [
+ PLATFORM_TYPE_GITHUB,
+ 'github-releases',
+ 'github-tags',
+ 'pod',
+];
+
+export const GITLAB_API_USING_HOST_TYPES = [
+ PLATFORM_TYPE_GITLAB,
+ 'gitlab-releases',
+ 'gitlab-tags',
+];
diff --git a/lib/platform/github/__snapshots__/index.spec.ts.snap b/lib/platform/github/__snapshots__/index.spec.ts.snap
index 57149d2ed2..37ff4dedb2 100644
--- a/lib/platform/github/__snapshots__/index.spec.ts.snap
+++ b/lib/platform/github/__snapshots__/index.spec.ts.snap
@@ -6875,33 +6875,6 @@ Array [
]
`;
-exports[`platform/github/index initPlatform() should add initialized platform with predefined generic host rule for github api 1`] = `
-Object {
- "endpoint": "https://api.github.com/",
- "gitAuthor": "renovate@whitesourcesoftware.com",
- "renovateUsername": "renovate-bot",
-}
-`;
-
-exports[`platform/github/index initPlatform() should add initialized platform with predefined generic host rule for github api 2`] = `
-[MockFunction] {
- "calls": Array [
- Array [
- Object {
- "matchHost": "api.github.com",
- "token": "abc123",
- },
- ],
- ],
- "results": Array [
- Object {
- "type": "return",
- "value": undefined,
- },
- ],
-}
-`;
-
exports[`platform/github/index initPlatform() should support custom endpoint 1`] = `
Object {
"endpoint": "https://ghe.renovatebot.com/",
diff --git a/lib/platform/github/index.spec.ts b/lib/platform/github/index.spec.ts
index ff04c1f864..23271d258b 100644
--- a/lib/platform/github/index.spec.ts
+++ b/lib/platform/github/index.spec.ts
@@ -129,17 +129,6 @@ describe('platform/github/index', () => {
).toMatchSnapshot();
expect(httpMock.getTrace()).toMatchSnapshot();
});
-
- it('should add initialized platform with predefined generic host rule for github api', async () => {
- expect(
- await github.initPlatform({
- token: 'abc123',
- username: 'renovate-bot',
- gitAuthor: 'renovate@whitesourcesoftware.com',
- } as any)
- ).toMatchSnapshot();
- expect(hostRules.add).toMatchSnapshot();
- });
});
describe('getRepos', () => {
diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts
index f45144e82a..76c599a40a 100644
--- a/lib/platform/github/index.ts
+++ b/lib/platform/github/index.ts
@@ -23,7 +23,7 @@ import * as git from '../../util/git';
import * as hostRules from '../../util/host-rules';
import * as githubHttp from '../../util/http/github';
import { sanitize } from '../../util/sanitize';
-import { ensureTrailingSlash, parseUrl } from '../../util/url';
+import { ensureTrailingSlash } from '../../util/url';
import type {
AggregatedVulnerabilities,
BranchStatusConfig,
@@ -115,12 +115,6 @@ export async function initPlatform({
renovateUsername,
};
- // Generic github hostRule that per default all datasources using github api are enabled
- const genericGithubHostRule = {
- matchHost: parseUrl(defaults.endpoint).hostname,
- token,
- };
- hostRules.add(genericGithubHostRule);
return platformConfig;
}
diff --git a/lib/types/github.ts b/lib/types/github.ts
deleted file mode 100644
index 9281983377..0000000000
--- a/lib/types/github.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-import { PLATFORM_TYPE_GITHUB } from '../constants/platforms';
-
-export const GITHUB_API_USING_HOST_TYPES = [
- PLATFORM_TYPE_GITHUB,
- 'github-releases',
- 'github-tags',
- 'pod',
-];
diff --git a/lib/types/index.ts b/lib/types/index.ts
index a89dd31798..e1e0fa9d42 100644
--- a/lib/types/index.ts
+++ b/lib/types/index.ts
@@ -4,4 +4,3 @@ export * from './versioning';
export * from './branch-status';
export * from './vulnerability-alert';
export * from './pr-state';
-export * from './github';
diff --git a/lib/types/platform/gitlab/index.spec.ts b/lib/types/platform/gitlab/index.spec.ts
deleted file mode 100644
index 945f2e4476..0000000000
--- a/lib/types/platform/gitlab/index.spec.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import {
- PLATFORM_TYPE_GITHUB,
- PLATFORM_TYPE_GITLAB,
-} from '../../../constants/platforms';
-import { GitlabReleasesDatasource } from '../../../datasource/gitlab-releases';
-import { id as GL_TAGS_DS } from '../../../datasource/gitlab-tags';
-import { GITLAB_API_USING_HOST_TYPES } from './index';
-
-describe('types/platform/gitlab/index', () => {
- it('should be part of the GITLAB_API_USING_HOST_TYPES', () => {
- expect(GITLAB_API_USING_HOST_TYPES.includes(GL_TAGS_DS)).toBeTrue();
- expect(
- GITLAB_API_USING_HOST_TYPES.includes(GitlabReleasesDatasource.id)
- ).toBeTrue();
- expect(
- GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITLAB)
- ).toBeTrue();
- });
-
- it('should be not part of the GITLAB_API_USING_HOST_TYPES ', () => {
- expect(
- GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITHUB)
- ).toBeFalse();
- });
-});
diff --git a/lib/types/platform/gitlab/index.ts b/lib/types/platform/gitlab/index.ts
index 999f9149b1..a67df13e1a 100644
--- a/lib/types/platform/gitlab/index.ts
+++ b/lib/types/platform/gitlab/index.ts
@@ -1,4 +1,3 @@
-import { PLATFORM_TYPE_GITLAB } from '../../../constants/platforms';
import { GitTreeNode } from '../../git';
export type GitLabBranch = {
@@ -13,9 +12,3 @@ export type GitlabTreeNode = {
id: string;
name: string;
} & GitTreeNode;
-
-export const GITLAB_API_USING_HOST_TYPES = [
- PLATFORM_TYPE_GITLAB,
- 'gitlab-releases',
- 'gitlab-tags',
-];
diff --git a/lib/util/http/auth.ts b/lib/util/http/auth.ts
index b3a5383dfc..a8dfc3f42f 100644
--- a/lib/util/http/auth.ts
+++ b/lib/util/http/auth.ts
@@ -1,9 +1,11 @@
import is from '@sindresorhus/is';
-import { NormalizedOptions } from 'got';
-import { PLATFORM_TYPE_GITEA } from '../../constants/platforms';
-import { GITHUB_API_USING_HOST_TYPES } from '../../types';
-import { GITLAB_API_USING_HOST_TYPES } from '../../types/platform/gitlab';
-import { GotOptions } from './types';
+import type { NormalizedOptions } from 'got';
+import {
+ GITHUB_API_USING_HOST_TYPES,
+ GITLAB_API_USING_HOST_TYPES,
+ PLATFORM_TYPE_GITEA,
+} from '../../constants/platforms';
+import type { GotOptions } from './types';
export function applyAuthorization(inOptions: GotOptions): GotOptions {
const options: GotOptions = { ...inOptions };
diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts
index ea59b0461f..a46e6d3230 100644
--- a/lib/util/http/host-rules.spec.ts
+++ b/lib/util/http/host-rules.spec.ts
@@ -36,6 +36,11 @@ describe('util/http/host-rules', () => {
authType: 'Basic',
token: 'XXX',
});
+
+ hostRules.add({
+ hostType: 'gitlab-tags',
+ token: 'abc',
+ });
});
afterEach(() => {
@@ -111,4 +116,40 @@ describe('util/http/host-rules', () => {
}
`);
});
+
+ it('noAuth', () => {
+ expect(applyHostRules(url, { ...options, noAuth: true }))
+ .toMatchInlineSnapshot(`
+ Object {
+ "hostType": "github",
+ "noAuth": true,
+ }
+ `);
+ });
+
+ it('fallback to github', () => {
+ expect(applyHostRules(url, { ...options, hostType: 'github-tags' }))
+ .toMatchInlineSnapshot(`
+ Object {
+ "context": Object {
+ "authType": undefined,
+ },
+ "hostType": "github-tags",
+ "token": "token",
+ }
+ `);
+ });
+
+ it('fallback to gitlab', () => {
+ expect(applyHostRules(url, { ...options, hostType: 'gitlab-tags' }))
+ .toMatchInlineSnapshot(`
+ Object {
+ "context": Object {
+ "authType": undefined,
+ },
+ "hostType": "gitlab-tags",
+ "token": "abc",
+ }
+ `);
+ });
});
diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts
index 975a2ae541..d327f0418e 100644
--- a/lib/util/http/host-rules.ts
+++ b/lib/util/http/host-rules.ts
@@ -1,17 +1,54 @@
+import {
+ GITHUB_API_USING_HOST_TYPES,
+ GITLAB_API_USING_HOST_TYPES,
+ PLATFORM_TYPE_GITHUB,
+ PLATFORM_TYPE_GITLAB,
+} from '../../constants/platforms';
import { logger } from '../../logger';
import { hasProxy } from '../../proxy';
+import type { HostRule } from '../../types';
import * as hostRules from '../host-rules';
-import { GotOptions } from './types';
+import type { GotOptions } from './types';
-// Apply host rules to requests
+function findMatchingRules(options: GotOptions, url: string): HostRule {
+ const { hostType } = options;
+ let res = hostRules.find({ hostType, url });
+
+ // Fallback to `github` hostType
+ if (
+ GITHUB_API_USING_HOST_TYPES.includes(hostType) &&
+ hostType !== PLATFORM_TYPE_GITHUB
+ ) {
+ res = {
+ ...hostRules.find({
+ hostType: PLATFORM_TYPE_GITHUB,
+ url,
+ }),
+ ...res,
+ };
+ }
+
+ // Fallback to `gitlab` hostType
+ if (
+ GITLAB_API_USING_HOST_TYPES.includes(hostType) &&
+ hostType !== PLATFORM_TYPE_GITLAB
+ ) {
+ res = {
+ ...hostRules.find({
+ hostType: PLATFORM_TYPE_GITLAB,
+ url,
+ }),
+ ...res,
+ };
+ }
+ return res;
+}
+
+// Apply host rules to requests
export function applyHostRules(url: string, inOptions: GotOptions): GotOptions {
const options: GotOptions = { ...inOptions };
- const foundRules =
- hostRules.find({
- hostType: options.hostType,
- url,
- }) || /* istanbul ignore next: can only happen in tests */ {};
+ const foundRules = findMatchingRules(options, url);
const { username, password, token, enabled, authType } = foundRules;
if (options.noAuth) {
logger.trace({ url }, `Authorization disabled`);
diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts
index d7c7531a46..c42837687d 100644
--- a/lib/util/http/index.spec.ts
+++ b/lib/util/http/index.spec.ts
@@ -208,4 +208,11 @@ describe('util/http/index', () => {
expect(bar).toBeTrue();
expect(baz).toBeTrue();
});
+
+ it('getBuffer', async () => {
+ httpMock.scope(baseUrl).get('/').reply(200, Buffer.from('test'));
+ const res = await http.getBuffer('http://renovate.com');
+ expect(res.body).toBeInstanceOf(Buffer);
+ expect(res.body.toString('utf-8')).toEqual('test');
+ });
});
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index a3530c9526..d9c727ecab 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -53,12 +53,16 @@ export interface HttpResponse<T = string> {
authorization?: boolean;
}
-function cloneResponse<T>(response: any): HttpResponse<T> {
+function cloneResponse<T extends Buffer | string | any>(
+ response: HttpResponse<T>
+): HttpResponse<T> {
+ const { body, statusCode, headers } = response;
// clone body and headers so that the cached result doesn't get accidentally mutated
+ // Don't use json clone for buffers
return {
- statusCode: response.statusCode,
- body: clone<T>(response.body),
- headers: clone(response.headers),
+ statusCode,
+ body: body instanceof Buffer ? (body.slice() as T) : clone<T>(body),
+ headers: clone(headers),
authorization: !!response.authorization,
};
}
@@ -95,7 +99,8 @@ async function gotRoutine<T>(
// Cheat the TS compiler using `as` to pick a specific overload.
// Otherwise it doesn't typecheck.
const resp = await got<T>(url, options as GotJSONOptions);
- const duration = resp.timings.phases.total || 0;
+ const duration =
+ resp.timings.phases.total || /* istanbul ignore next: can't be tested */ 0;
const httpRequests = memCache.get('http-requests') || [];
httpRequests.push({ ...requestStats, duration });
@@ -166,6 +171,7 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
resPromise = memCache.get(cacheKey);
}
+ // istanbul ignore else: no cache tests
if (!resPromise) {
const startTime = Date.now();
const queueTask = (): Promise<Response<T>> => {
--
GitLab