From 9f791552d32d8efa5fa22e7b49d5a292ba16d72a Mon Sep 17 00:00:00 2001
From: Jesko Steinberg <61695677+jeskosda@users.noreply.github.com>
Date: Wed, 25 Aug 2021 09:07:23 +0200
Subject: [PATCH] feat: enable right filtering when specifing hostRules with
github-api using datasources (#11136)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
lib/datasource/github-releases/common.ts | 3 +-
lib/datasource/github-releases/index.ts | 3 +-
lib/datasource/github-tags/index.ts | 3 +-
lib/datasource/pod/index.ts | 2 +-
.../github/__snapshots__/index.spec.ts.snap | 27 +++++++
lib/platform/github/index.spec.ts | 11 +++
lib/platform/github/index.ts | 9 ++-
lib/types/github.spec.ts | 25 +++++++
lib/types/github.ts | 8 +++
lib/types/index.ts | 1 +
lib/util/host-rules.spec.ts | 71 ++++++++++++++++++-
lib/util/http/auth.spec.ts | 38 ++++++++++
lib/util/http/auth.ts | 4 +-
lib/util/http/github.spec.ts | 18 +++++
lib/util/http/github.ts | 7 +-
15 files changed, 220 insertions(+), 10 deletions(-)
create mode 100644 lib/types/github.spec.ts
create mode 100644 lib/types/github.ts
diff --git a/lib/datasource/github-releases/common.ts b/lib/datasource/github-releases/common.ts
index 58bb4a56cd..8b395e47b5 100644
--- a/lib/datasource/github-releases/common.ts
+++ b/lib/datasource/github-releases/common.ts
@@ -3,9 +3,10 @@ import { ensureTrailingSlash } from '../../util/url';
import type { GithubRelease } from './types';
const defaultSourceUrlBase = 'https://github.com/';
+export const id = 'github-releases';
export const cacheNamespace = 'datasource-github-releases';
-export const http = new GithubHttp();
+export const http = new GithubHttp(id);
export function getSourceUrlBase(registryUrl: string): string {
// default to GitHub.com if no GHE host is specified.
diff --git a/lib/datasource/github-releases/index.ts b/lib/datasource/github-releases/index.ts
index bcc91241a3..05e09e93d5 100644
--- a/lib/datasource/github-releases/index.ts
+++ b/lib/datasource/github-releases/index.ts
@@ -7,11 +7,12 @@ import {
getGithubRelease,
getSourceUrlBase,
http,
+ id,
} from './common';
import { findDigestAsset, mapDigestAssetToRelease } from './digest';
import type { GithubRelease } from './types';
-export const id = 'github-releases';
+export { id };
export const customRegistrySupport = true;
export const defaultRegistryUrls = ['https://github.com'];
export const registryStrategy = 'first';
diff --git a/lib/datasource/github-tags/index.ts b/lib/datasource/github-tags/index.ts
index 1801050c2b..2300a58e29 100644
--- a/lib/datasource/github-tags/index.ts
+++ b/lib/datasource/github-tags/index.ts
@@ -11,9 +11,10 @@ export const customRegistrySupport = true;
export const defaultRegistryUrls = ['https://github.com'];
export const registryStrategy = 'first';
-const http = new GithubHttp();
+const http = new GithubHttp(id);
const cacheNamespace = 'datasource-github-tags';
+
function getCacheKey(registryUrl: string, repo: string, type: string): string {
return `${registryUrl}:${repo}:${type}`;
}
diff --git a/lib/datasource/pod/index.ts b/lib/datasource/pod/index.ts
index 54be19969d..31b53e167f 100644
--- a/lib/datasource/pod/index.ts
+++ b/lib/datasource/pod/index.ts
@@ -17,7 +17,7 @@ export const registryStrategy = 'hunt';
const cacheNamespace = `datasource-${id}`;
const cacheMinutes = 30;
-const githubHttp = new GithubHttp();
+const githubHttp = new GithubHttp(id);
const http = new Http(id);
function shardParts(lookupName: string): string[] {
diff --git a/lib/platform/github/__snapshots__/index.spec.ts.snap b/lib/platform/github/__snapshots__/index.spec.ts.snap
index 37ff4dedb2..57149d2ed2 100644
--- a/lib/platform/github/__snapshots__/index.spec.ts.snap
+++ b/lib/platform/github/__snapshots__/index.spec.ts.snap
@@ -6875,6 +6875,33 @@ Array [
]
`;
+exports[`platform/github/index initPlatform() should add initialized platform with predefined generic host rule for github api 1`] = `
+Object {
+ "endpoint": "https://api.github.com/",
+ "gitAuthor": "renovate@whitesourcesoftware.com",
+ "renovateUsername": "renovate-bot",
+}
+`;
+
+exports[`platform/github/index initPlatform() should add initialized platform with predefined generic host rule for github api 2`] = `
+[MockFunction] {
+ "calls": Array [
+ Array [
+ Object {
+ "matchHost": "api.github.com",
+ "token": "abc123",
+ },
+ ],
+ ],
+ "results": Array [
+ Object {
+ "type": "return",
+ "value": undefined,
+ },
+ ],
+}
+`;
+
exports[`platform/github/index initPlatform() should support custom endpoint 1`] = `
Object {
"endpoint": "https://ghe.renovatebot.com/",
diff --git a/lib/platform/github/index.spec.ts b/lib/platform/github/index.spec.ts
index 99a5463b1a..3d0c716cff 100644
--- a/lib/platform/github/index.spec.ts
+++ b/lib/platform/github/index.spec.ts
@@ -130,6 +130,17 @@ describe('platform/github/index', () => {
).toMatchSnapshot();
expect(httpMock.getTrace()).toMatchSnapshot();
});
+
+ it('should add initialized platform with predefined generic host rule for github api', async () => {
+ expect(
+ await github.initPlatform({
+ token: 'abc123',
+ username: 'renovate-bot',
+ gitAuthor: 'renovate@whitesourcesoftware.com',
+ } as any)
+ ).toMatchSnapshot();
+ expect(hostRules.add).toMatchSnapshot();
+ });
});
describe('getRepos', () => {
diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts
index b9b3ca365d..7b3af04531 100644
--- a/lib/platform/github/index.ts
+++ b/lib/platform/github/index.ts
@@ -23,7 +23,7 @@ import * as git from '../../util/git';
import * as hostRules from '../../util/host-rules';
import * as githubHttp from '../../util/http/github';
import { sanitize } from '../../util/sanitize';
-import { ensureTrailingSlash } from '../../util/url';
+import { ensureTrailingSlash, parseUrl } from '../../util/url';
import type {
AggregatedVulnerabilities,
BranchStatusConfig,
@@ -114,6 +114,13 @@ export async function initPlatform({
gitAuthor: gitAuthor || discoveredGitAuthor,
renovateUsername,
};
+
+ // Generic github hostRule that per default all datasources using github api are enabled
+ const genericGithubHostRule = {
+ matchHost: parseUrl(defaults.endpoint).hostname,
+ token,
+ };
+ hostRules.add(genericGithubHostRule);
return platformConfig;
}
diff --git a/lib/types/github.spec.ts b/lib/types/github.spec.ts
new file mode 100644
index 0000000000..8e79ef350a
--- /dev/null
+++ b/lib/types/github.spec.ts
@@ -0,0 +1,25 @@
+import {
+ PLATFORM_TYPE_GITHUB,
+ PLATFORM_TYPE_GITLAB,
+} from '../constants/platforms';
+import { id as GH_RELEASES_DS } from '../datasource/github-releases';
+import { id as GH_TAGS_DS } from '../datasource/github-tags';
+import { id as POD_DS } from '../datasource/pod';
+import { GITHUB_API_USING_HOST_TYPES } from './github';
+
+describe('types/github', () => {
+ it('should be part of the GITHUB_API_USING_HOST_TYPES ', () => {
+ expect(GITHUB_API_USING_HOST_TYPES.includes(GH_TAGS_DS)).toBeTrue();
+ expect(GITHUB_API_USING_HOST_TYPES.includes(GH_RELEASES_DS)).toBeTrue();
+ expect(GITHUB_API_USING_HOST_TYPES.includes(POD_DS)).toBeTrue();
+ expect(
+ GITHUB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITHUB)
+ ).toBeTrue();
+ });
+
+ it('should be not part of the GITHUB_API_USING_HOST_TYPES ', () => {
+ expect(
+ GITHUB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITLAB)
+ ).toBeFalse();
+ });
+});
diff --git a/lib/types/github.ts b/lib/types/github.ts
new file mode 100644
index 0000000000..9281983377
--- /dev/null
+++ b/lib/types/github.ts
@@ -0,0 +1,8 @@
+import { PLATFORM_TYPE_GITHUB } from '../constants/platforms';
+
+export const GITHUB_API_USING_HOST_TYPES = [
+ PLATFORM_TYPE_GITHUB,
+ 'github-releases',
+ 'github-tags',
+ 'pod',
+];
diff --git a/lib/types/index.ts b/lib/types/index.ts
index e1e0fa9d42..a89dd31798 100644
--- a/lib/types/index.ts
+++ b/lib/types/index.ts
@@ -4,3 +4,4 @@ export * from './versioning';
export * from './branch-status';
export * from './vulnerability-alert';
export * from './pr-state';
+export * from './github';
diff --git a/lib/util/host-rules.spec.ts b/lib/util/host-rules.spec.ts
index 145bc91937..1e01f83c89 100644
--- a/lib/util/host-rules.spec.ts
+++ b/lib/util/host-rules.spec.ts
@@ -1,4 +1,7 @@
-import { PLATFORM_TYPE_AZURE } from '../constants/platforms';
+import {
+ PLATFORM_TYPE_AZURE,
+ PLATFORM_TYPE_GITHUB,
+} from '../constants/platforms';
import * as datasourceNuget from '../datasource/nuget';
import { add, clear, find, findAll, hosts } from './host-rules';
@@ -106,6 +109,72 @@ describe('util/host-rules', () => {
.token
).toBeUndefined();
});
+
+ it('matches on specific path', () => {
+ // Initialized platform holst rule
+ add({
+ hostType: PLATFORM_TYPE_GITHUB,
+ matchHost: 'https://api.github.com',
+ token: 'abc',
+ });
+ // Initialized generic host rule for github platform
+ add({
+ hostType: PLATFORM_TYPE_GITHUB,
+ matchHost: 'https://api.github.com',
+ token: 'abc',
+ });
+ // specific host rule for using other token in different org
+ add({
+ hostType: PLATFORM_TYPE_GITHUB,
+ matchHost: 'https://api.github.com/repos/org-b/',
+ token: 'def',
+ });
+ expect(
+ find({
+ hostType: PLATFORM_TYPE_GITHUB,
+ url: 'https://api.github.com/repos/org-b/someRepo/tags?per_page=100',
+ }).token
+ ).toEqual('def');
+ });
+
+ it('matches for several hostTypes when no hostType rule is configured', () => {
+ add({
+ matchHost: 'https://api.github.com',
+ token: 'abc',
+ });
+ expect(
+ find({
+ hostType: PLATFORM_TYPE_GITHUB,
+ url: 'https://api.github.com/repos/org-b/someRepo/tags?per_page=100',
+ }).token
+ ).toEqual('abc');
+ expect(
+ find({
+ hostType: 'github-releases',
+ url: 'https://api.github.com/repos/org-b/someRepo/tags?per_page=100',
+ }).token
+ ).toEqual('abc');
+ });
+
+ it('matches if hostType is configured and host rule is filtered with datasource', () => {
+ add({
+ hostType: PLATFORM_TYPE_GITHUB,
+ matchHost: 'https://api.github.com',
+ token: 'abc',
+ });
+ add({
+ hostType: 'github-tags',
+ matchHost: 'https://api.github.com/repos/org-b/',
+ token: 'def',
+ });
+ expect(
+ find({
+ hostType: 'github-tags',
+ url: 'https://api.github.com/repos/org-b/someRepo/tags?per_page=100',
+ }).token
+ ).toEqual('def');
+ });
+
it('matches on hostName', () => {
add({
hostName: 'nuget.local',
diff --git a/lib/util/http/auth.spec.ts b/lib/util/http/auth.spec.ts
index 11d5c9d877..a2d4df7175 100644
--- a/lib/util/http/auth.spec.ts
+++ b/lib/util/http/auth.spec.ts
@@ -2,6 +2,7 @@ import { NormalizedOptions } from 'got';
import { partial } from '../../../test/util';
import {
PLATFORM_TYPE_GITEA,
+ PLATFORM_TYPE_GITHUB,
PLATFORM_TYPE_GITLAB,
} from '../../constants/platforms';
import { applyAuthorization, removeAuthorization } from './auth';
@@ -69,6 +70,43 @@ describe('util/http/auth', () => {
`);
});
+ it('github token', () => {
+ const opts: GotOptions = {
+ headers: {},
+ token: 'XXX',
+ hostType: PLATFORM_TYPE_GITHUB,
+ };
+
+ applyAuthorization(opts);
+
+ expect(opts).toEqual({
+ headers: {
+ authorization: 'token XXX',
+ },
+ hostType: 'github',
+ token: 'XXX',
+ });
+ });
+
+ it('github token for datasource using github api', () => {
+ const opts: GotOptions = {
+ headers: {},
+ token: 'ZZZZ',
+ hostType: 'github-releases',
+ };
+ applyAuthorization(opts);
+
+ expect(opts).toMatchInlineSnapshot(`
+ Object {
+ "headers": Object {
+ "authorization": "token ZZZZ",
+ },
+ "hostType": "github-releases",
+ "token": "ZZZZ",
+ }
+ `);
+ });
+
it(`gitlab personal access token`, () => {
const opts: GotOptions = {
headers: {},
diff --git a/lib/util/http/auth.ts b/lib/util/http/auth.ts
index 74e90b0d46..4a7671eeb8 100644
--- a/lib/util/http/auth.ts
+++ b/lib/util/http/auth.ts
@@ -2,9 +2,9 @@ import is from '@sindresorhus/is';
import { NormalizedOptions } from 'got';
import {
PLATFORM_TYPE_GITEA,
- PLATFORM_TYPE_GITHUB,
PLATFORM_TYPE_GITLAB,
} from '../../constants/platforms';
+import { GITHUB_API_USING_HOST_TYPES } from '../../types';
import { GotOptions } from './types';
export function applyAuthorization(inOptions: GotOptions): GotOptions {
@@ -15,7 +15,7 @@ export function applyAuthorization(inOptions: GotOptions): GotOptions {
if (options.token) {
if (options.hostType === PLATFORM_TYPE_GITEA) {
options.headers.authorization = `token ${options.token}`;
- } else if (options.hostType === PLATFORM_TYPE_GITHUB) {
+ } else if (GITHUB_API_USING_HOST_TYPES.includes(options.hostType)) {
options.headers.authorization = `token ${options.token}`;
if (options.token.startsWith('x-access-token:')) {
const appToken = options.token.replace('x-access-token:', '');
diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts
index 53111f6cc2..36e19ebcd8 100644
--- a/lib/util/http/github.spec.ts
+++ b/lib/util/http/github.spec.ts
@@ -6,6 +6,7 @@ import {
PLATFORM_RATE_LIMIT_EXCEEDED,
REPOSITORY_CHANGED,
} from '../../constants/error-messages';
+import { id as GITHUB_RELEASES_ID } from '../../datasource/github-releases';
import * as hostRules from '../host-rules';
import { GithubHttp, setBaseUrl } from './github';
@@ -61,7 +62,23 @@ describe('util/http/github', () => {
expect(req.headers.accept).toBe(
'some-accept, application/vnd.github.machine-man-preview+json'
);
+ expect(req.headers.authorization).toBe('token abc123');
});
+
+ it('supports different datasources', async () => {
+ const githubApiDatasource = new GithubHttp(GITHUB_RELEASES_ID);
+ hostRules.add({ hostType: 'github', token: 'abc' });
+ hostRules.add({
+ hostType: GITHUB_RELEASES_ID,
+ token: 'def',
+ });
+ httpMock.scope(githubApiHost).get('/some-url').reply(200);
+ await githubApiDatasource.get('/some-url');
+ const [req] = httpMock.getTrace();
+ expect(req).toBeDefined();
+ expect(req.headers.authorization).toBe('token def');
+ });
+
it('paginates', async () => {
const url = '/some-url';
httpMock
@@ -148,6 +165,7 @@ describe('util/http/github', () => {
);
await githubApi.getJson(url);
}
+
async function failWithError(error: string | Record<string, unknown>) {
const url = '/some-url';
httpMock.scope(githubApiHost).get(url).replyWithError(error);
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 722ea322a3..478c72f54f 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -158,8 +158,11 @@ function constructAcceptString(input?: any): string {
}
export class GithubHttp extends Http<GithubHttpOptions, GithubHttpOptions> {
- constructor(options?: GithubHttpOptions) {
- super(PLATFORM_TYPE_GITHUB, options);
+ constructor(
+ hostType: string = PLATFORM_TYPE_GITHUB,
+ options?: GithubHttpOptions
+ ) {
+ super(hostType, options);
}
protected override async request<T>(
--
GitLab