diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md
index 34b8621470a5fb787046ed6e8a9440a55e5d605e..8102ed15bc53b1a5658c4b0ab564039782974072 100644
--- a/docs/usage/configuration-options.md
+++ b/docs/usage/configuration-options.md
@@ -1768,6 +1768,26 @@ Example config:
}
```
+### maxRetryAfter
+
+A remote host may return a `4xx` response with a `Retry-After` header value, which indicates that Renovate has been rate-limited.
+Renovate may try to contact the host again after waiting a certain time, that's set by the host.
+By default, Renovate tries again after the `Retry-After` header value has passed, up to a maximum of 60 seconds.
+If the `Retry-After` value is more than 60 seconds, Renovate will abort the request instead of waiting.
+
+You can configure a different maximum value in seconds using `maxRetryAfter`:
+
+```json
+{
+ "hostRules": [
+ {
+ "matchHost": "api.github.com",
+ "maxRetryAfter": 25
+ }
+ ]
+}
+```
+
### dnsCache
Enable got [dnsCache](https://github.com/sindresorhus/got/blob/v11.5.2/readme.md#dnsCache) support.
diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts
index bab96cece2086c4285d443763f84948d980b71fa..c057b68f9b86661f4c099600f6d649c6c3b9648c 100644
--- a/lib/config/options/index.ts
+++ b/lib/config/options/index.ts
@@ -2741,6 +2741,17 @@ const options: RenovateOptions[] = [
globalOnly: true,
default: [],
},
+ {
+ name: 'maxRetryAfter',
+ description:
+ 'Maximum retry-after header value to wait for before retrying a failed request.',
+ type: 'integer',
+ default: 60,
+ stage: 'package',
+ parent: 'hostRules',
+ cli: false,
+ env: false,
+ },
];
export function getOptions(): RenovateOptions[] {
diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts
index c8b26a1358da5861864e8e7745405d0d68be2007..e29576f4bd655ff121ea640aae8d9044f30352cf 100644
--- a/lib/types/host-rules.ts
+++ b/lib/types/host-rules.ts
@@ -11,6 +11,7 @@ export interface HostRuleSearchResult {
enableHttp2?: boolean;
concurrentRequestLimit?: number;
maxRequestsPerSecond?: number;
+ maxRetryAfter?: number;
dnsCache?: boolean;
keepAlive?: boolean;
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index 7c3164438e979328c0d3aeb3c832ee8e67eaf8f7..c675e378f5692e1d15ce6e8eb4b3cd6fafb95548 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -15,6 +15,7 @@ import { applyAuthorization, removeAuthorization } from './auth';
import { hooks } from './hooks';
import { applyHostRule, findMatchingRule } from './host-rules';
import { getQueue } from './queue';
+import { getRetryAfter, wrapWithRetry } from './retry-after';
import { Throttle, getThrottle } from './throttle';
import type {
GotJSONOptions,
@@ -130,11 +131,14 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
protected hostType: string,
options: HttpOptions = {},
) {
- this.options = merge<GotOptions>(options, { context: { hostType } });
-
- if (process.env.NODE_ENV === 'test') {
- this.options.retry = 0;
- }
+ const retryLimit = process.env.NODE_ENV === 'test' ? 0 : 2;
+ this.options = merge<GotOptions>(options, {
+ context: { hostType },
+ retry: {
+ limit: retryLimit,
+ maxRetryAfter: 0, // Don't rely on `got` retry-after handling, just let it fail and then we'll handle it
+ },
+ });
}
protected getThrottle(url: string): Throttle | null {
@@ -226,7 +230,8 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
? () => queue.add<HttpResponse<T>>(throttledTask)
: throttledTask;
- resPromise = queuedTask();
+ const { maxRetryAfter = 60 } = hostRule;
+ resPromise = wrapWithRetry(queuedTask, url, getRetryAfter, maxRetryAfter);
if (memCacheKey) {
memCache.set(memCacheKey, resPromise);
diff --git a/lib/util/http/retry-after.spec.ts b/lib/util/http/retry-after.spec.ts
new file mode 100644
index 0000000000000000000000000000000000000000..9535cfabb3328d079ebf2935fd8b87ccb719ddac
--- /dev/null
+++ b/lib/util/http/retry-after.spec.ts
@@ -0,0 +1,162 @@
+import { RequestError } from 'got';
+import { getRetryAfter, wrapWithRetry } from './retry-after';
+
+function requestError(
+ response: {
+ statusCode?: number;
+ headers?: Record<string, string | string[]>;
+ } | null = null,
+) {
+ const err = new RequestError('request error', {}, null as never);
+ if (response) {
+ (err as any).response = response;
+ }
+ return err;
+}
+
+describe('util/http/retry-after', () => {
+ beforeEach(() => {
+ jest.useFakeTimers();
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ describe('wrapWithRetry', () => {
+ it('works', async () => {
+ const task = jest.fn(() => Promise.resolve(42));
+ const res = await wrapWithRetry(task, 'foobar', () => null, 60);
+ expect(res).toBe(42);
+ expect(task).toHaveBeenCalledTimes(1);
+ });
+
+ it('throws', async () => {
+ const task = jest.fn(() => Promise.reject(new Error('error')));
+
+ await expect(
+ wrapWithRetry(task, 'http://example.com', () => null, 60),
+ ).rejects.toThrow('error');
+
+ expect(task).toHaveBeenCalledTimes(1);
+ });
+
+ it('retries', async () => {
+ const task = jest
+ .fn()
+ .mockRejectedValueOnce(new Error('error-1'))
+ .mockRejectedValueOnce(new Error('error-2'))
+ .mockResolvedValueOnce(42);
+
+ const p = wrapWithRetry(task, 'http://example.com', () => 1, 60);
+ await jest.advanceTimersByTimeAsync(2000);
+
+ const res = await p;
+ expect(res).toBe(42);
+ expect(task).toHaveBeenCalledTimes(3);
+ });
+
+ it('gives up after max retries', async () => {
+ const task = jest
+ .fn()
+ .mockRejectedValueOnce('error-1')
+ .mockRejectedValueOnce('error-2')
+ .mockRejectedValueOnce('error-3')
+ .mockRejectedValue('error-4');
+
+ const p = wrapWithRetry(task, 'http://example.com', () => 1, 60).catch(
+ (err) => err,
+ );
+ await jest.advanceTimersByTimeAsync(2000);
+
+ await expect(p).resolves.toBe('error-3');
+ expect(task).toHaveBeenCalledTimes(3);
+ });
+
+ it('gives up when delay exceeds maxRetryAfter', async () => {
+ const task = jest.fn().mockRejectedValue('error');
+
+ const p = wrapWithRetry(task, 'http://example.com', () => 61, 60).catch(
+ (err) => err,
+ );
+
+ await expect(p).resolves.toBe('error');
+ expect(task).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ describe('getRetryAfter', () => {
+ it('returns null for non-RequestError', () => {
+ expect(getRetryAfter(new Error())).toBeNull();
+ });
+
+ it('returns null for RequestError without response', () => {
+ expect(getRetryAfter(requestError())).toBeNull();
+ });
+
+ it('returns null for status other than 429', () => {
+ const err = new RequestError('request-error', {}, null as never);
+ (err as any).response = { statusCode: 302 };
+ expect(getRetryAfter(requestError({ statusCode: 302 }))).toBeNull();
+ });
+
+ it('returns null missing "retry-after" header', () => {
+ expect(
+ getRetryAfter(requestError({ statusCode: 429, headers: {} })),
+ ).toBeNull();
+ });
+
+ it('returns null for non-integer "retry-after" header', () => {
+ expect(
+ getRetryAfter(
+ requestError({
+ statusCode: 429,
+ headers: {
+ 'retry-after': 'Wed, 21 Oct 2015 07:28:00 GMT',
+ },
+ }),
+ ),
+ ).toBeNull();
+ });
+
+ it('returns delay in seconds from date', () => {
+ jest.setSystemTime(new Date('2020-01-01T00:00:00Z'));
+ expect(
+ getRetryAfter(
+ requestError({
+ statusCode: 429,
+ headers: {
+ 'retry-after': 'Wed, 01 Jan 2020 00:00:42 GMT',
+ },
+ }),
+ ),
+ ).toBe(42);
+ });
+
+ it('returns delay in seconds from number', () => {
+ expect(
+ getRetryAfter(
+ requestError({
+ statusCode: 429,
+ headers: {
+ 'retry-after': '42',
+ },
+ }),
+ ),
+ ).toBe(42);
+ });
+
+ it('returns null for invalid header value', () => {
+ expect(
+ getRetryAfter(
+ requestError({
+ statusCode: 429,
+ headers: {
+ 'retry-after': 'invalid',
+ },
+ }),
+ ),
+ ).toBeNull();
+ });
+ });
+});
diff --git a/lib/util/http/retry-after.ts b/lib/util/http/retry-after.ts
new file mode 100644
index 0000000000000000000000000000000000000000..6cde4ae33a1d852e6d7e2e8194af35a85dfb492a
--- /dev/null
+++ b/lib/util/http/retry-after.ts
@@ -0,0 +1,115 @@
+import { setTimeout } from 'timers/promises';
+import { RequestError } from 'got';
+import { DateTime } from 'luxon';
+import { logger } from '../../logger';
+import { parseUrl } from '../url';
+import type { Task } from './types';
+
+const hostDelays = new Map<string, Promise<unknown>>();
+
+const maxRetries = 2;
+
+/**
+ * Given a task that returns a promise, retry the task if it fails with a
+ * 429 Too Many Requests or 403 Forbidden response, using the Retry-After
+ * header to determine the delay.
+ *
+ * For response codes other than 429 or 403, or if the Retry-After header
+ * is not present or invalid, the task is not retried, throwing the error.
+ */
+export async function wrapWithRetry<T>(
+ task: Task<T>,
+ url: string,
+ getRetryAfter: (err: unknown) => number | null,
+ maxRetryAfter: number,
+): Promise<T> {
+ const key = parseUrl(url)?.host ?? url;
+
+ let retries = 0;
+ for (;;) {
+ try {
+ await hostDelays.get(key);
+ hostDelays.delete(key);
+
+ return await task();
+ } catch (err) {
+ const delaySeconds = getRetryAfter(err);
+ if (delaySeconds === null) {
+ throw err;
+ }
+
+ if (retries === maxRetries) {
+ logger.debug(
+ `Retry-After: reached maximum retries (${maxRetries}) for ${url}`,
+ );
+ throw err;
+ }
+
+ if (delaySeconds > maxRetryAfter) {
+ logger.debug(
+ `Retry-After: delay ${delaySeconds} seconds exceeds maxRetryAfter ${maxRetryAfter} seconds for ${url}`,
+ );
+ throw err;
+ }
+
+ logger.debug(
+ `Retry-After: will retry ${url} after ${delaySeconds} seconds`,
+ );
+
+ const delay = Promise.all([
+ hostDelays.get(key),
+ setTimeout(1000 * delaySeconds),
+ ]);
+ hostDelays.set(key, delay);
+ retries += 1;
+ }
+ }
+}
+
+export function getRetryAfter(err: unknown): number | null {
+ if (!(err instanceof RequestError)) {
+ return null;
+ }
+
+ if (!err.response) {
+ return null;
+ }
+
+ if (err.response.statusCode < 400 || err.response.statusCode >= 500) {
+ logger.warn(
+ { url: err.response.url },
+ `Retry-After: unexpected status code ${err.response.statusCode}`,
+ );
+ return null;
+ }
+
+ const retryAfter = err.response.headers['retry-after']?.trim();
+ if (!retryAfter) {
+ return null;
+ }
+
+ const date = DateTime.fromHTTP(retryAfter);
+ if (date.isValid) {
+ const seconds = Math.floor(date.diffNow('seconds').seconds);
+ if (seconds < 0) {
+ logger.debug(
+ { url: err.response.url, retryAfter },
+ 'Retry-After: date in the past',
+ );
+ return null;
+ }
+
+ return seconds;
+ }
+
+ const seconds = parseInt(retryAfter, 10);
+ if (!Number.isNaN(seconds) && seconds > 0) {
+ return seconds;
+ }
+
+ logger.debug(
+ { url: err.response.url, retryAfter },
+ 'Retry-After: unsupported format',
+ );
+ return null;
+}
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
index 4f576ef17a17885ceb8dc6e72206fe1bb62db006..86b5366ac53b647343ad5faf178b779c1aab7075 100644
--- a/lib/util/http/types.ts
+++ b/lib/util/http/types.ts
@@ -92,4 +92,5 @@ export interface HttpResponse<T = string> {
authorization?: boolean;
}
-export type GotTask<T> = () => Promise<HttpResponse<T>>;
+export type Task<T> = () => Promise<T>;
+export type GotTask<T> = Task<HttpResponse<T>>;