From 5c0628bf3b3b09ec91e68d555c51fc71559a2edd Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Tue, 23 Apr 2024 00:26:20 -0300
Subject: [PATCH] feat(host-rules): Support `readOnly` request matching
(#28562)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
docs/usage/configuration-options.md | 21 +++++++++++++++++++
lib/config/options/index.ts | 10 +++++++++
lib/modules/platform/github/index.ts | 4 ++++
lib/types/host-rules.ts | 3 ++-
lib/util/github/graphql/datasource-fetcher.ts | 1 +
lib/util/host-rules.spec.ts | 19 +++++++++++++++++
lib/util/host-rules.ts | 18 +++++++++++++---
lib/util/http/github.ts | 12 ++++++++++-
lib/util/http/host-rules.ts | 9 ++++----
lib/util/http/index.ts | 15 +++++++++++++
lib/util/http/types.ts | 2 ++
.../update/pr/changelog/github/source.ts | 1 +
12 files changed, 106 insertions(+), 9 deletions(-)
diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index c3c79115b3..e52762dabf 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -898,6 +898,27 @@ It will be compiled using Handlebars and the regex `groups` result.
It will be compiled using Handlebars and the regex `groups` result.
It will default to the value of `depName` if left unconfigured/undefined.
+### readOnly
+
+If the `readOnly` field is being set to `true` inside the host rule, it will match only against the requests that are known to be read operations.
+Examples are `GET` requests or `HEAD` requests, but also it could be certain types of GraphQL queries.
+
+This option could be used to avoid rate limits for certain platforms like GitHub or Bitbucket, by offloading the read operations to a different user.
+
+```json
+{
+ "hostRules": [
+ {
+ "matchHost": "api.github.com",
+ "readOnly": true,
+ "token": "********"
+ }
+ ]
+}
+```
+
+If more than one token matches for a read-only request then the `readOnly` token will be given preference.
+
### currentValueTemplate
If the `currentValue` for a dependency is not captured with a named group then it can be defined in config using this field.
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index 8f1151fc6a..1c1285115f 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -2476,6 +2476,16 @@ const options: RenovateOptions[] = [
cli: false,
env: false,
},
+ {
+ name: 'readOnly',
+ description:
+ 'Match against requests that only read data and do not mutate anything.',
+ type: 'boolean',
+ stage: 'repository',
+ parents: ['hostRules'],
+ cli: false,
+ env: false,
+ },
{
name: 'timeout',
description: 'Timeout (in milliseconds) for queries to external endpoints.',
diff --git a/lib/modules/platform/github/index.ts b/lib/modules/platform/github/index.ts
index 9e1016824a..9357a3adc9 100644
--- a/lib/modules/platform/github/index.ts
+++ b/lib/modules/platform/github/index.ts
@@ -461,6 +461,7 @@ export async function initRepo({
const opts = hostRules.find({
hostType: 'github',
url: platformConfig.endpoint,
+ readOnly: true,
});
config.renovateUsername = renovateUsername;
[config.repositoryOwner, config.repositoryName] = repository.split('/');
@@ -499,6 +500,7 @@ export async function initRepo({
name: config.repositoryName,
user: renovateUsername,
},
+ readOnly: true,
});
if (res?.errors) {
@@ -1214,6 +1216,7 @@ async function getIssues(): Promise<Issue[]> {
name: config.repositoryName,
user: config.renovateUsername,
},
+ readOnly: true,
},
);
@@ -1975,6 +1978,7 @@ export async function getVulnerabilityAlerts(): Promise<VulnerabilityAlert[]> {
variables: { owner: config.repositoryOwner, name: config.repositoryName },
paginate: false,
acceptHeader: 'application/vnd.github.vixen-preview+json',
+ readOnly: true,
});
} catch (err) {
logger.debug({ err }, 'Error retrieving vulnerability alerts');
diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts
index f8a61fd9f7..23ac43d7f6 100644
--- a/lib/types/host-rules.ts
+++ b/lib/types/host-rules.ts
@@ -25,9 +25,10 @@ export interface HostRule {
hostType?: string;
matchHost?: string;
resolvedHost?: string;
+ readOnly?: boolean;
}
export type CombinedHostRule = Omit<
HostRule,
- 'encrypted' | 'hostType' | 'matchHost' | 'resolvedHost'
+ 'encrypted' | 'hostType' | 'matchHost' | 'resolvedHost' | 'readOnly'
>;
diff --git a/lib/util/github/graphql/datasource-fetcher.ts b/lib/util/github/graphql/datasource-fetcher.ts
index b32995881c..f8fe2d47a9 100644
--- a/lib/util/github/graphql/datasource-fetcher.ts
+++ b/lib/util/github/graphql/datasource-fetcher.ts
@@ -107,6 +107,7 @@ export class GithubGraphqlDatasourceFetcher<
return {
baseUrl,
repository,
+ readOnly: true,
body: { query, variables },
};
}
diff --git a/lib/util/host-rules.spec.ts b/lib/util/host-rules.spec.ts
index a16a9332e3..f5fa0cbf23 100644
--- a/lib/util/host-rules.spec.ts
+++ b/lib/util/host-rules.spec.ts
@@ -295,6 +295,25 @@ describe('util/host-rules', () => {
}),
).toEqual({ token: 'longest' });
});
+
+ it('matches readOnly requests', () => {
+ add({
+ matchHost: 'https://api.github.com/repos/',
+ token: 'aaa',
+ hostType: 'github',
+ });
+ add({
+ matchHost: 'https://api.github.com',
+ token: 'bbb',
+ readOnly: true,
+ });
+ expect(
+ find({
+ url: 'https://api.github.com/repos/foo/bar/tags',
+ readOnly: true,
+ }),
+ ).toEqual({ token: 'bbb' });
+ });
});
describe('hosts()', () => {
diff --git a/lib/util/host-rules.ts b/lib/util/host-rules.ts
index 3054c6d698..1da59fcd9e 100644
--- a/lib/util/host-rules.ts
+++ b/lib/util/host-rules.ts
@@ -73,6 +73,7 @@ export function add(params: HostRule): void {
export interface HostRuleSearch {
hostType?: string;
url?: string;
+ readOnly?: boolean;
}
function matchesHost(url: string, matchHost: string): boolean {
@@ -107,8 +108,9 @@ function fromShorterToLongerMatchHost(a: HostRule, b: HostRule): number {
return a.matchHost.length - b.matchHost.length;
}
-function hostRuleRank({ hostType, matchHost }: HostRule): number {
- if (hostType && matchHost) {
+function hostRuleRank({ hostType, matchHost, readOnly }: HostRule): number {
+ // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+ if ((hostType || readOnly) && matchHost) {
return 3;
}
@@ -142,6 +144,7 @@ export function find(search: HostRuleSearch): CombinedHostRule {
for (const rule of sortedRules) {
let hostTypeMatch = true;
let hostMatch = true;
+ let readOnlyMatch = true;
if (rule.hostType) {
hostTypeMatch = false;
@@ -157,7 +160,15 @@ export function find(search: HostRuleSearch): CombinedHostRule {
}
}
- if (hostTypeMatch && hostMatch) {
+ if (!is.undefined(rule.readOnly)) {
+ readOnlyMatch = false;
+ if (search.readOnly === rule.readOnly) {
+ readOnlyMatch = true;
+ hostTypeMatch = true; // When we match `readOnly`, we don't care about `hostType`
+ }
+ }
+
+ if (hostTypeMatch && readOnlyMatch && hostMatch) {
matchedRules.push(clone(rule));
}
}
@@ -166,6 +177,7 @@ export function find(search: HostRuleSearch): CombinedHostRule {
delete res.hostType;
delete res.resolvedHost;
delete res.matchHost;
+ delete res.readOnly;
return res;
}
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 67276d3c50..3db9d7b4a7 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -276,7 +276,7 @@ export class GithubHttp extends Http<GithubHttpOptions> {
options?: InternalHttpOptions & GithubHttpOptions,
okToRetry = true,
): Promise<HttpResponse<T>> {
- const opts: GithubHttpOptions = {
+ const opts: InternalHttpOptions & GithubHttpOptions = {
baseUrl,
...options,
throwHttpErrors: true,
@@ -296,8 +296,17 @@ export class GithubHttp extends Http<GithubHttpOptions> {
);
}
+ let readOnly = opts.readOnly;
+ const { method = 'get' } = opts;
+ if (
+ readOnly === undefined &&
+ ['get', 'head'].includes(method.toLowerCase())
+ ) {
+ readOnly = true;
+ }
const { token } = findMatchingRule(authUrl.toString(), {
hostType: this.hostType,
+ readOnly,
});
opts.token = token;
}
@@ -393,6 +402,7 @@ export class GithubHttp extends Http<GithubHttpOptions> {
baseUrl: baseUrl.replace('/v3/', '/'), // GHE uses unversioned graphql path
body,
headers: { accept: options?.acceptHeader },
+ readOnly: options.readOnly,
};
if (options.token) {
opts.token = options.token;
diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts
index 20c8981037..09775fe9f4 100644
--- a/lib/util/http/host-rules.ts
+++ b/lib/util/http/host-rules.ts
@@ -14,10 +14,10 @@ import { matchRegexOrGlobList } from '../string-match';
import { parseUrl } from '../url';
import { dnsLookup } from './dns';
import { keepAliveAgents } from './keep-alive';
-import type { GotOptions } from './types';
+import type { GotOptions, InternalHttpOptions } from './types';
export type HostRulesGotOptions = Pick<
- GotOptions,
+ GotOptions & InternalHttpOptions,
| 'hostType'
| 'url'
| 'noAuth'
@@ -34,14 +34,15 @@ export type HostRulesGotOptions = Pick<
| 'agent'
| 'http2'
| 'https'
+ | 'readOnly'
>;
export function findMatchingRule<GotOptions extends HostRulesGotOptions>(
url: string,
options: GotOptions,
): HostRule {
- const { hostType } = options;
- let res = hostRules.find({ hostType, url });
+ const { hostType, readOnly } = options;
+ let res = hostRules.find({ hostType, url, readOnly });
if (
is.nonEmptyString(res.token) ||
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index 1b084542ba..b93c659497 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -162,6 +162,13 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
applyDefaultHeaders(options);
+ if (
+ is.undefined(options.readOnly) &&
+ ['head', 'get'].includes(options.method)
+ ) {
+ options.readOnly = true;
+ }
+
const hostRule = findMatchingRule(url, options);
options = applyHostRule(url, options, hostRule);
if (options.enabled === false) {
@@ -457,6 +464,14 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
}
applyDefaultHeaders(combinedOptions);
+
+ if (
+ is.undefined(combinedOptions.readOnly) &&
+ ['head', 'get'].includes(combinedOptions.method)
+ ) {
+ combinedOptions.readOnly = true;
+ }
+
const hostRule = findMatchingRule(url, combinedOptions);
combinedOptions = applyHostRule(resolvedUrl, combinedOptions, hostRule);
if (combinedOptions.enabled === false) {
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
index 396ccbfa8c..a767c29c5b 100644
--- a/lib/util/http/types.ts
+++ b/lib/util/http/types.ts
@@ -48,6 +48,7 @@ export interface GraphqlOptions {
cursor?: string | null;
acceptHeader?: string;
token?: string;
+ readOnly?: boolean;
}
export interface HttpOptions {
@@ -67,6 +68,7 @@ export interface HttpOptions {
token?: string;
memCache?: boolean;
cacheProvider?: HttpCacheProvider;
+ readOnly?: boolean;
}
export interface InternalHttpOptions extends HttpOptions {
diff --git a/lib/workers/repository/update/pr/changelog/github/source.ts b/lib/workers/repository/update/pr/changelog/github/source.ts
index e1efda0f73..c8909dad98 100644
--- a/lib/workers/repository/update/pr/changelog/github/source.ts
+++ b/lib/workers/repository/update/pr/changelog/github/source.ts
@@ -53,6 +53,7 @@ export class GitHubChangeLogSource extends ChangeLogSource {
const { token } = hostRules.find({
hostType: 'github',
url,
+ readOnly: true,
});
// istanbul ignore if
if (host && !token) {
--
GitLab