From 290b78d87dee349ce76a8b4b4b2a4b0cd1e33676 Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Thu, 25 May 2023 15:17:43 +0300
Subject: [PATCH] feat(http): Etag support (#22302)
---
lib/util/http/bitbucket-server.ts | 9 +++-
lib/util/http/bitbucket.ts | 6 +--
lib/util/http/gitea.ts | 9 +++-
lib/util/http/github.ts | 3 +-
lib/util/http/gitlab.ts | 9 +++-
lib/util/http/index.spec.ts | 74 +++++++++++++++++++++++++++
lib/util/http/index.ts | 83 +++++++++++++++++++++++--------
lib/util/http/jira.ts | 9 +++-
lib/util/http/types.ts | 9 ++++
9 files changed, 179 insertions(+), 32 deletions(-)
diff --git a/lib/util/http/bitbucket-server.ts b/lib/util/http/bitbucket-server.ts
index 449754f4ce..f9a7fa3ac7 100644
--- a/lib/util/http/bitbucket-server.ts
+++ b/lib/util/http/bitbucket-server.ts
@@ -1,5 +1,10 @@
import { resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+ HttpOptions,
+ HttpRequestOptions,
+ HttpResponse,
+ InternalHttpOptions,
+} from './types';
import { Http } from '.';
let baseUrl: string;
@@ -14,7 +19,7 @@ export class BitbucketServerHttp extends Http {
protected override request<T>(
path: string,
- options?: InternalHttpOptions
+ options?: InternalHttpOptions & HttpRequestOptions<T>
): Promise<HttpResponse<T>> {
const url = resolveBaseUrl(baseUrl, path);
const opts = {
diff --git a/lib/util/http/bitbucket.ts b/lib/util/http/bitbucket.ts
index 0d4a51c28a..34c5166ca8 100644
--- a/lib/util/http/bitbucket.ts
+++ b/lib/util/http/bitbucket.ts
@@ -2,7 +2,7 @@ import is from '@sindresorhus/is';
import { logger } from '../../logger';
import type { PagedResult } from '../../modules/platform/bitbucket/types';
import { parseUrl, resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse } from './types';
+import type { HttpOptions, HttpRequestOptions, HttpResponse } from './types';
import { Http } from '.';
const MAX_PAGES = 100;
@@ -26,7 +26,7 @@ export class BitbucketHttp extends Http<BitbucketHttpOptions> {
protected override async request<T>(
path: string,
- options?: BitbucketHttpOptions
+ options?: BitbucketHttpOptions & HttpRequestOptions<T>
): Promise<HttpResponse<T>> {
const opts = { baseUrl, ...options };
@@ -53,7 +53,7 @@ export class BitbucketHttp extends Http<BitbucketHttpOptions> {
while (is.nonEmptyString(nextURL) && page <= MAX_PAGES) {
const nextResult = await super.request<PagedResult<T>>(
nextURL,
- options
+ options as BitbucketHttpOptions
);
resultBody.values.push(...nextResult.body.values);
diff --git a/lib/util/http/gitea.ts b/lib/util/http/gitea.ts
index 394f3589e4..32e2ea27d7 100644
--- a/lib/util/http/gitea.ts
+++ b/lib/util/http/gitea.ts
@@ -1,6 +1,11 @@
import is from '@sindresorhus/is';
import { resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+ HttpOptions,
+ HttpRequestOptions,
+ HttpResponse,
+ InternalHttpOptions,
+} from './types';
import { Http } from '.';
let baseUrl: string;
@@ -36,7 +41,7 @@ export class GiteaHttp extends Http<GiteaHttpOptions> {
protected override async request<T>(
path: string,
- options?: InternalHttpOptions & GiteaHttpOptions
+ options?: InternalHttpOptions & GiteaHttpOptions & HttpRequestOptions<T>
): Promise<HttpResponse<T>> {
const resolvedUrl = resolveUrl(path, options?.baseUrl ?? baseUrl);
const opts = {
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 9dfb182d67..440a042d3c 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -19,6 +19,7 @@ import type { GotLegacyError } from './legacy';
import type {
GraphqlOptions,
HttpOptions,
+ HttpRequestOptions,
HttpResponse,
InternalHttpOptions,
} from './types';
@@ -271,7 +272,7 @@ export class GithubHttp extends Http<GithubHttpOptions> {
protected override async request<T>(
url: string | URL,
- options?: InternalHttpOptions & GithubHttpOptions,
+ options?: InternalHttpOptions & GithubHttpOptions & HttpRequestOptions<T>,
okToRetry = true
): Promise<HttpResponse<T>> {
const opts: GithubHttpOptions = {
diff --git a/lib/util/http/gitlab.ts b/lib/util/http/gitlab.ts
index 6a72c31cd8..9078bbeee0 100644
--- a/lib/util/http/gitlab.ts
+++ b/lib/util/http/gitlab.ts
@@ -2,7 +2,12 @@ import is from '@sindresorhus/is';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import { parseLinkHeader, parseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+ HttpOptions,
+ HttpRequestOptions,
+ HttpResponse,
+ InternalHttpOptions,
+} from './types';
import { Http } from '.';
let baseUrl = 'https://gitlab.com/api/v4/';
@@ -21,7 +26,7 @@ export class GitlabHttp extends Http<GitlabHttpOptions> {
protected override async request<T>(
url: string | URL,
- options?: InternalHttpOptions & GitlabHttpOptions
+ options?: InternalHttpOptions & GitlabHttpOptions & HttpRequestOptions<T>
): Promise<HttpResponse<T>> {
const opts = {
baseUrl,
diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts
index 9563f29956..7e8668154c 100644
--- a/lib/util/http/index.spec.ts
+++ b/lib/util/http/index.spec.ts
@@ -425,4 +425,78 @@ describe('util/http/index', () => {
expect(t2 - t1).toBeGreaterThanOrEqual(4000);
});
});
+
+ describe('Etag caching', () => {
+ it('returns cached data for status=304', async () => {
+ type FooBar = { foo: string; bar: string };
+ const data: FooBar = { foo: 'foo', bar: 'bar' };
+ httpMock
+ .scope(baseUrl, { reqheaders: { 'If-None-Match': 'foobar' } })
+ .get('/foo')
+ .reply(304);
+
+ const res = await http.getJson<FooBar>(`/foo`, {
+ baseUrl,
+ etagCache: {
+ etag: 'foobar',
+ data,
+ },
+ });
+
+ expect(res.statusCode).toBe(304);
+ expect(res.body).toEqual(data);
+ expect(res.body).not.toBe(data);
+ });
+
+ it('bypasses schema parsing', async () => {
+ const FooBar = z
+ .object({ foo: z.string(), bar: z.string() })
+ .transform(({ foo, bar }) => ({
+ foobar: `${foo}${bar}`.toUpperCase(),
+ }));
+ const data = FooBar.parse({ foo: 'foo', bar: 'bar' });
+ httpMock
+ .scope(baseUrl, { reqheaders: { 'If-None-Match': 'foobar' } })
+ .get('/foo')
+ .reply(304);
+
+ const res = await http.getJson(
+ `/foo`,
+ {
+ baseUrl,
+ etagCache: {
+ etag: 'foobar',
+ data,
+ },
+ },
+ FooBar
+ );
+
+ expect(res.statusCode).toBe(304);
+ expect(res.body).toEqual(data);
+ expect(res.body).not.toBe(data);
+ });
+
+ it('returns new data for status=200', async () => {
+ type FooBar = { foo: string; bar: string };
+ const oldData: FooBar = { foo: 'foo', bar: 'bar' };
+ const newData: FooBar = { foo: 'FOO', bar: 'BAR' };
+ httpMock
+ .scope(baseUrl, { reqheaders: { 'If-None-Match': 'foobar' } })
+ .get('/foo')
+ .reply(200, newData);
+
+ const res = await http.getJson<FooBar>(`/foo`, {
+ baseUrl,
+ etagCache: {
+ etag: 'foobar',
+ data: oldData,
+ },
+ });
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body).toEqual(newData);
+ expect(res.body).not.toBe(newData);
+ });
+ });
});
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index 20bca08a33..f435c27460 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -18,6 +18,7 @@ import type {
GotJSONOptions,
GotOptions,
HttpOptions,
+ HttpRequestOptions,
HttpResponse,
InternalHttpOptions,
RequestStats,
@@ -28,7 +29,7 @@ import './legacy';
export { RequestError as HttpError };
type JsonArgs<
- Opts extends HttpOptions,
+ Opts extends HttpOptions & HttpRequestOptions<ResT>,
ResT = unknown,
Schema extends ZodType<ResT> = ZodType<ResT>
> = {
@@ -39,18 +40,24 @@ type JsonArgs<
type Task<T> = () => Promise<HttpResponse<T>>;
-function cloneResponse<T extends Buffer | string | any>(
- response: HttpResponse<T>
+// Copying will help to avoid circular structure
+// and mutation of the cached response.
+function copyResponse<T extends Buffer | string | any>(
+ response: HttpResponse<T>,
+ deep: boolean
): HttpResponse<T> {
const { body, statusCode, headers } = response;
- // clone body and headers so that the cached result doesn't get accidentally mutated
- // Don't use json clone for buffers
- return {
- statusCode,
- body: body instanceof Buffer ? (body.slice() as T) : clone<T>(body),
- headers: clone(headers),
- authorization: !!response.authorization,
- };
+ return deep
+ ? {
+ statusCode,
+ body: body instanceof Buffer ? (body.slice() as T) : clone<T>(body),
+ headers: clone(headers),
+ }
+ : {
+ statusCode,
+ body,
+ headers,
+ };
}
function applyDefaultHeaders(options: Options): void {
@@ -120,7 +127,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
protected async request<T>(
requestUrl: string | URL,
- httpOptions: InternalHttpOptions = {}
+ httpOptions: InternalHttpOptions & HttpRequestOptions<T> = {}
): Promise<HttpResponse<T>> {
let url = requestUrl.toString();
if (httpOptions?.baseUrl) {
@@ -136,6 +143,18 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
httpOptions
);
+ const etagCache =
+ httpOptions.etagCache &&
+ (options.method === 'get' || options.method === 'head')
+ ? httpOptions.etagCache
+ : null;
+ if (etagCache) {
+ options.headers = {
+ ...options.headers,
+ 'If-None-Match': etagCache.etag,
+ };
+ }
+
if (process.env.NODE_ENV === 'test') {
options.retry = 0;
}
@@ -204,8 +223,10 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
try {
const res = await resPromise;
- res.authorization = !!options?.headers?.authorization;
- return cloneResponse(res);
+ const deepCopyNeeded = !!memCacheKey && res.statusCode !== 304;
+ const resCopy = copyResponse(res, deepCopyNeeded);
+ resCopy.authorization = !!options?.headers?.authorization;
+ return resCopy;
} catch (err) {
const { abortOnError, abortIgnoreStatusCodes } = options;
if (abortOnError && !abortIgnoreStatusCodes?.includes(err.statusCode)) {
@@ -215,7 +236,10 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
}
}
- get(url: string, options: HttpOptions = {}): Promise<HttpResponse> {
+ get(
+ url: string,
+ options: HttpOptions & HttpRequestOptions<string> = {}
+ ): Promise<HttpResponse> {
return this.request<string>(url, options);
}
@@ -235,7 +259,11 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
private async requestJson<ResT = unknown>(
method: InternalHttpOptions['method'],
- { url, httpOptions: requestOptions, schema }: JsonArgs<Opts, ResT>
+ {
+ url,
+ httpOptions: requestOptions,
+ schema,
+ }: JsonArgs<Opts & HttpRequestOptions<ResT>, ResT>
): Promise<HttpResponse<ResT>> {
const { body, ...httpOptions } = { ...requestOptions };
const opts: InternalHttpOptions = {
@@ -253,11 +281,23 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
}
const res = await this.request<ResT>(url, opts);
+ const etagCacheHit =
+ httpOptions.etagCache && res.statusCode === 304
+ ? clone(httpOptions.etagCache.data)
+ : null;
+
if (!schema) {
+ if (etagCacheHit) {
+ res.body = etagCacheHit;
+ }
return res;
}
- res.body = await schema.parseAsync(res.body);
+ if (etagCacheHit) {
+ res.body = etagCacheHit;
+ } else {
+ res.body = await schema.parseAsync(res.body);
+ }
return res;
}
@@ -281,19 +321,22 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
return res;
}
- getJson<ResT>(url: string, options?: Opts): Promise<HttpResponse<ResT>>;
+ getJson<ResT>(
+ url: string,
+ options?: Opts & HttpRequestOptions<ResT>
+ ): Promise<HttpResponse<ResT>>;
getJson<ResT, Schema extends ZodType<ResT> = ZodType<ResT>>(
url: string,
schema: Schema
): Promise<HttpResponse<Infer<Schema>>>;
getJson<ResT, Schema extends ZodType<ResT> = ZodType<ResT>>(
url: string,
- options: Opts,
+ options: Opts & HttpRequestOptions<Infer<Schema>>,
schema: Schema
): Promise<HttpResponse<Infer<Schema>>>;
getJson<ResT = unknown, Schema extends ZodType<ResT> = ZodType<ResT>>(
arg1: string,
- arg2?: Opts | Schema,
+ arg2?: (Opts & HttpRequestOptions<ResT>) | Schema,
arg3?: Schema
): Promise<HttpResponse<ResT>> {
const args = this.resolveArgs<ResT>(arg1, arg2, arg3);
diff --git a/lib/util/http/jira.ts b/lib/util/http/jira.ts
index df51c152a4..1137cc33cc 100644
--- a/lib/util/http/jira.ts
+++ b/lib/util/http/jira.ts
@@ -1,4 +1,9 @@
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+ HttpOptions,
+ HttpRequestOptions,
+ HttpResponse,
+ InternalHttpOptions,
+} from './types';
import { Http } from '.';
let baseUrl: string;
@@ -14,7 +19,7 @@ export class JiraHttp extends Http {
protected override request<T>(
url: string | URL,
- options?: InternalHttpOptions
+ options?: InternalHttpOptions & HttpRequestOptions<T>
): Promise<HttpResponse<T>> {
const opts = { baseUrl, ...options };
return super.request<T>(url, opts);
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
index 4b23298258..861959a9ac 100644
--- a/lib/util/http/types.ts
+++ b/lib/util/http/types.ts
@@ -65,6 +65,15 @@ export interface HttpOptions {
memCache?: boolean;
}
+export interface EtagCache<T = any> {
+ etag: string;
+ data: T;
+}
+
+export interface HttpRequestOptions<T = any> {
+ etagCache?: EtagCache<T>;
+}
+
export interface InternalHttpOptions extends HttpOptions {
json?: HttpOptions['body'];
responseType?: 'json' | 'buffer';
--
GitLab