diff --git a/lib/config/presets/gitlab/index.ts b/lib/config/presets/gitlab/index.ts
index 2af74c99d094cf0ede9395c33e2098e5ee0fac04..bfb1768c04860928145c265dd52af9ee9379073a 100644
--- a/lib/config/presets/gitlab/index.ts
+++ b/lib/config/presets/gitlab/index.ts
@@ -43,7 +43,7 @@ export async function fetchJSONFile(
}
url += `projects/${urlEncodedRepo}/repository/files/${urlEncodedPkgName}/raw${ref}`;
logger.trace({ url }, `Preset URL`);
- res = await gitlabApi.get(url);
+ res = await gitlabApi.getText(url);
} catch (err) {
if (err instanceof ExternalHostError) {
throw err;
diff --git a/lib/config/presets/http/index.ts b/lib/config/presets/http/index.ts
index 90bd220f14e7384bc16b8543edb2b876fe18d511..e8aa66d516ae74b398327878f8ace97703d1391b 100644
--- a/lib/config/presets/http/index.ts
+++ b/lib/config/presets/http/index.ts
@@ -20,7 +20,7 @@ export async function getPreset({
}
try {
- response = await http.get(url);
+ response = await http.getText(url);
} catch (err) {
if (err instanceof ExternalHostError) {
throw err;
diff --git a/lib/modules/datasource/artifactory/index.ts b/lib/modules/datasource/artifactory/index.ts
index c9389ab3630a10e6b6213dc9859eef30c3e5c1a8..e11b74522c3cb252d021f61cfeb0ddbfef1c03e7 100644
--- a/lib/modules/datasource/artifactory/index.ts
+++ b/lib/modules/datasource/artifactory/index.ts
@@ -50,7 +50,7 @@ export class ArtifactoryDatasource extends Datasource {
releases: [],
};
try {
- const response = await this.http.get(url);
+ const response = await this.http.getText(url);
const body = parse(response.body, {
blockTextElements: {
script: true,
diff --git a/lib/modules/datasource/crate/index.ts b/lib/modules/datasource/crate/index.ts
index d16585261c4e3221a56a4216a64bf998f853c53d..453e0c5f8d8e4ea839a77fd751d5fb8f2f3cc398 100644
--- a/lib/modules/datasource/crate/index.ts
+++ b/lib/modules/datasource/crate/index.ts
@@ -193,7 +193,7 @@ export class CrateDatasource extends Datasource {
);
const crateUrl = joinUrlParts(baseUrl, ...packageSuffix);
try {
- return (await this.http.get(crateUrl)).body;
+ return (await this.http.getText(crateUrl)).body;
} catch (err) {
this.handleGenericErrors(err);
}
diff --git a/lib/modules/datasource/custom/formats/html.ts b/lib/modules/datasource/custom/formats/html.ts
index 6195b1b609f212dce7441d56a2be03b075dfb488..e79d104f83880ca6779e784305da793b8eb54128 100644
--- a/lib/modules/datasource/custom/formats/html.ts
+++ b/lib/modules/datasource/custom/formats/html.ts
@@ -30,7 +30,7 @@ function extractLinks(content: string): ReleaseResult {
export class HtmlFetcher implements CustomDatasourceFetcher {
async fetch(http: Http, registryURL: string): Promise<unknown> {
- const response = await http.get(registryURL, {
+ const response = await http.getText(registryURL, {
headers: {
Accept: 'text/html',
},
diff --git a/lib/modules/datasource/custom/formats/yaml.ts b/lib/modules/datasource/custom/formats/yaml.ts
index 541eb2a9d47bd6a91019da84d38ed73f02715d56..2784b9af699aec3a1bc0de762e6f05790405d6ea 100644
--- a/lib/modules/datasource/custom/formats/yaml.ts
+++ b/lib/modules/datasource/custom/formats/yaml.ts
@@ -5,7 +5,7 @@ import type { CustomDatasourceFetcher } from './types';
export class YamlFetcher implements CustomDatasourceFetcher {
async fetch(http: Http, registryURL: string): Promise<unknown> {
- const response = await http.get(registryURL);
+ const response = await http.getText(registryURL);
return parseSingleYaml(response.body);
}
diff --git a/lib/modules/datasource/deb/index.ts b/lib/modules/datasource/deb/index.ts
index 192bc6fdb2591037bc81229945d90bd7be8c351f..c99dfea0ab34ad021c9bc78aef6e0c02dd56b2ff 100644
--- a/lib/modules/datasource/deb/index.ts
+++ b/lib/modules/datasource/deb/index.ts
@@ -187,7 +187,7 @@ export class DebDatasource extends Datasource {
*/
private async fetchInReleaseFile(baseReleaseUrl: string): Promise<string> {
const inReleaseUrl = joinUrlParts(baseReleaseUrl, 'InRelease');
- const response = await this.http.get(inReleaseUrl);
+ const response = await this.http.getText(inReleaseUrl);
return response.body;
}
diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts
index 5631990a9fa49ec53ff148837de2e90ad5da1c80..c206b764c09846e9ab88791f9401cd820c2bee59 100644
--- a/lib/modules/datasource/docker/index.ts
+++ b/lib/modules/datasource/docker/index.ts
@@ -98,7 +98,7 @@ export class DockerDatasource extends Datasource {
registryHost: string,
dockerRepository: string,
tag: string,
- mode: 'head' | 'get' = 'get',
+ mode: 'head' | 'getText' = 'getText',
): Promise<HttpResponse | null> {
logger.debug(
`getManifestResponse(${registryHost}, ${dockerRepository}, ${tag}, ${mode})`,
diff --git a/lib/modules/datasource/github-release-attachments/index.ts b/lib/modules/datasource/github-release-attachments/index.ts
index 2b95ca53fea59b06099a67676bc6e81c6837e5a6..42b0f6368239a06ccaa55101858138627e99e722 100644
--- a/lib/modules/datasource/github-release-attachments/index.ts
+++ b/lib/modules/datasource/github-release-attachments/index.ts
@@ -65,7 +65,7 @@ export class GithubReleaseAttachmentsDatasource extends Datasource {
(a: GithubRestAsset) => a.size < 5 * 1024,
);
for (const asset of smallAssets) {
- const res = await this.http.get(asset.browser_download_url);
+ const res = await this.http.getText(asset.browser_download_url);
for (const line of res.body.split(newlineRegex)) {
const [lineDigest, lineFilename] = line.split(regEx(/\s+/), 2);
if (lineDigest === digest) {
@@ -162,7 +162,7 @@ export class GithubReleaseAttachmentsDatasource extends Datasource {
current,
next,
);
- const res = await this.http.get(releaseAsset.browser_download_url);
+ const res = await this.http.getText(releaseAsset.browser_download_url);
for (const line of res.body.split(newlineRegex)) {
const [lineDigest, lineFn] = line.split(regEx(/\s+/), 2);
if (lineFn === releaseFilename) {
diff --git a/lib/modules/datasource/go/base.ts b/lib/modules/datasource/go/base.ts
index ba3c74800afc4e5f9bf3c7f469a18dde9af8d224..060f7526d204aa9956b43663206a49938113c6e2 100644
--- a/lib/modules/datasource/go/base.ts
+++ b/lib/modules/datasource/go/base.ts
@@ -135,7 +135,7 @@ export class BaseGoDatasource {
): Promise<DataSource | null> {
const goModuleUrl = goModule.replace(/\.git(\/[a-z0-9/]*)?$/, '');
const pkgUrl = `https://${goModuleUrl}?go-get=1`;
- const { body: html } = await BaseGoDatasource.http.get(pkgUrl);
+ const { body: html } = await BaseGoDatasource.http.getText(pkgUrl);
const goSourceHeader = BaseGoDatasource.goSourceHeader(html, goModule);
if (goSourceHeader) {
diff --git a/lib/modules/datasource/go/releases-goproxy.ts b/lib/modules/datasource/go/releases-goproxy.ts
index e3f87682a3478f280d976b80fce71970b7cbb40f..f94a7cce4c362fd37aa5a8a62b39e8fd307be72c 100644
--- a/lib/modules/datasource/go/releases-goproxy.ts
+++ b/lib/modules/datasource/go/releases-goproxy.ts
@@ -134,7 +134,7 @@ export class GoProxyDatasource extends Datasource {
'@v',
'list',
);
- const { body } = await this.http.get(url);
+ const { body } = await this.http.getText(url);
return filterMap(body.split(newlineRegex), (str) => {
if (!is.nonEmptyStringAndNotWhitespace(str)) {
return null;
diff --git a/lib/modules/datasource/golang-version/index.ts b/lib/modules/datasource/golang-version/index.ts
index 6d355a60aeb4bc94a87e2bc0b4fbfc05136b8aad..65bf1514cf2cc650c0f8df29cbde52eaed491132 100644
--- a/lib/modules/datasource/golang-version/index.ts
+++ b/lib/modules/datasource/golang-version/index.ts
@@ -60,7 +60,7 @@ export class GolangVersionDatasource extends Datasource {
'/HEAD/internal/history/release.go',
);
- const response = await this.http.get(golangVersionsUrl);
+ const response = await this.http.getText(golangVersionsUrl);
const lines = response.body.split(lineTerminationRegex);
diff --git a/lib/modules/datasource/hexpm-bob/index.ts b/lib/modules/datasource/hexpm-bob/index.ts
index 780bbfb91fcb820a848aeb692f039d3119aebd59..89f111efe66637c27618a96cbf0808ba5fc52e54 100644
--- a/lib/modules/datasource/hexpm-bob/index.ts
+++ b/lib/modules/datasource/hexpm-bob/index.ts
@@ -59,7 +59,7 @@ export class HexpmBobDatasource extends Datasource {
...HexpmBobDatasource.getPackageDetails(packageType),
};
try {
- const { body } = await this.http.get(url);
+ const { body } = await this.http.getText(url);
result.releases = body
.split('\n')
.map((line) => line.trim())
diff --git a/lib/modules/datasource/maven/util.spec.ts b/lib/modules/datasource/maven/util.spec.ts
index dd9891abc615fb5275b7c1dee62b0f68b31e29b5..0bb61bd46494a33f2fa4be3795667a0c93ee2ae6 100644
--- a/lib/modules/datasource/maven/util.spec.ts
+++ b/lib/modules/datasource/maven/util.spec.ts
@@ -68,7 +68,7 @@ describe('modules/datasource/maven/util', () => {
describe('downloadHttpProtocol', () => {
it('returns empty for HOST_DISABLED error', async () => {
const http = partial<Http>({
- get: () => Promise.reject(httpError({ message: HOST_DISABLED })),
+ getText: () => Promise.reject(httpError({ message: HOST_DISABLED })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res.unwrap()).toEqual({
@@ -79,7 +79,7 @@ describe('modules/datasource/maven/util', () => {
it('returns empty for host error', async () => {
const http = partial<Http>({
- get: () => Promise.reject(httpError({ code: 'ETIMEDOUT' })),
+ getText: () => Promise.reject(httpError({ code: 'ETIMEDOUT' })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res.unwrap()).toEqual({
@@ -90,7 +90,7 @@ describe('modules/datasource/maven/util', () => {
it('returns empty for temporary error', async () => {
const http = partial<Http>({
- get: () => Promise.reject(httpError({ code: 'ECONNRESET' })),
+ getText: () => Promise.reject(httpError({ code: 'ECONNRESET' })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res.unwrap()).toEqual({
@@ -101,7 +101,7 @@ describe('modules/datasource/maven/util', () => {
it('returns empty for connection error', async () => {
const http = partial<Http>({
- get: () => Promise.reject(httpError({ code: 'ECONNREFUSED' })),
+ getText: () => Promise.reject(httpError({ code: 'ECONNREFUSED' })),
});
const res = await downloadHttpProtocol(http, 'some://');
expect(res.unwrap()).toEqual({
@@ -112,7 +112,7 @@ describe('modules/datasource/maven/util', () => {
it('returns empty for unsupported error', async () => {
const http = partial<Http>({
- get: () =>
+ getText: () =>
Promise.reject(httpError({ name: 'UnsupportedProtocolError' })),
});
const res = await downloadHttpProtocol(http, 'some://');
diff --git a/lib/modules/datasource/maven/util.ts b/lib/modules/datasource/maven/util.ts
index 696836fd41f913cc4ef5b213a711131f7fa90c0b..b4c50a797b85c9a3780331df0ad1b8c86deedde2 100644
--- a/lib/modules/datasource/maven/util.ts
+++ b/lib/modules/datasource/maven/util.ts
@@ -74,7 +74,7 @@ export async function downloadHttpProtocol(
): Promise<MavenFetchResult> {
const url = pkgUrl.toString();
const fetchResult = await Result.wrap<HttpResponse, Error>(
- http.get(url, opts),
+ http.getText(url, opts),
)
.transform((res): MavenFetchSuccess => {
const result: MavenFetchSuccess = { data: res.body };
diff --git a/lib/modules/datasource/nuget/v2.ts b/lib/modules/datasource/nuget/v2.ts
index 888be53280970e751e5a2e5cf346a8dee47be9b5..04c1e8eade94c32fa5bead8dfec73102b20932c8 100644
--- a/lib/modules/datasource/nuget/v2.ts
+++ b/lib/modules/datasource/nuget/v2.ts
@@ -26,7 +26,7 @@ export class NugetV2Api {
)}/FindPackagesById()?id=%27${pkgName}%27&$select=Version,IsLatestVersion,ProjectUrl,Published`;
while (pkgUrlList !== null) {
// typescript issue
- const pkgVersionsListRaw = await http.get(pkgUrlList);
+ const pkgVersionsListRaw = await http.getText(pkgUrlList);
const pkgVersionsListDoc = new XmlDocument(pkgVersionsListRaw.body);
const pkgInfoList = pkgVersionsListDoc.childrenNamed('entry');
diff --git a/lib/modules/datasource/nuget/v3.ts b/lib/modules/datasource/nuget/v3.ts
index 2492c02d70a674de8985f5a554d1ade5a0e8eb46..96a813686b5161cc1db5001357603bd7a4737990 100644
--- a/lib/modules/datasource/nuget/v3.ts
+++ b/lib/modules/datasource/nuget/v3.ts
@@ -209,7 +209,7 @@ export class NugetV3Api {
// TODO: types (#22198)
latestStable
}/${pkgName.toLowerCase()}.nuspec`;
- const metaresult = await http.get(nuspecUrl);
+ const metaresult = await http.getText(nuspecUrl);
const nuspec = new XmlDocument(metaresult.body);
const sourceUrl = nuspec.valueWithPath('metadata.repository@url');
if (sourceUrl) {
diff --git a/lib/modules/datasource/pod/index.ts b/lib/modules/datasource/pod/index.ts
index 5ebd9c6cf06829c723aed2ab0e0c0bd453736017..d7b0a684a9add532ac1647d9a9d84a9c5b2bd5e8 100644
--- a/lib/modules/datasource/pod/index.ts
+++ b/lib/modules/datasource/pod/index.ts
@@ -109,7 +109,7 @@ export class PodDatasource extends Datasource {
packageName: string,
): Promise<string | null> {
try {
- const resp = await this.http.get(url);
+ const resp = await this.http.getText(url);
if (resp?.body) {
return resp.body;
}
diff --git a/lib/modules/datasource/pypi/index.ts b/lib/modules/datasource/pypi/index.ts
index 48fda7604991f10ebe8345d1b153176351108bb7..1a86701e17c9ed58d41d5c63bcbfbe062a11f19a 100644
--- a/lib/modules/datasource/pypi/index.ts
+++ b/lib/modules/datasource/pypi/index.ts
@@ -259,7 +259,7 @@ export class PypiDatasource extends Datasource {
);
const dependency: ReleaseResult = { releases: [] };
const headers = await this.getAuthHeaders(lookupUrl);
- const response = await this.http.get(lookupUrl, { headers });
+ const response = await this.http.getText(lookupUrl, { headers });
const dep = response?.body;
if (!dep) {
logger.trace({ dependency: packageName }, 'pip package not found');
diff --git a/lib/modules/datasource/ruby-version/index.ts b/lib/modules/datasource/ruby-version/index.ts
index edf3f52c46d6d47dbd9517d8123c432cf69b03b7..69d4b77dde9d9993153ccf34df1e747d17ce742c 100644
--- a/lib/modules/datasource/ruby-version/index.ts
+++ b/lib/modules/datasource/ruby-version/index.ts
@@ -40,7 +40,7 @@ export class RubyVersionDatasource extends Datasource {
// TODO: types (#22198)
const rubyVersionsUrl = `${registryUrl}en/downloads/releases/`;
try {
- const response = await this.http.get(rubyVersionsUrl);
+ const response = await this.http.getText(rubyVersionsUrl);
const root = parse(response.body);
const rows =
diff --git a/lib/modules/datasource/rubygems/index.ts b/lib/modules/datasource/rubygems/index.ts
index aa52a008791d4d0a29899739c15a02ea54180a39..9cb920352739812df94e2c8476e91102c5969aed 100644
--- a/lib/modules/datasource/rubygems/index.ts
+++ b/lib/modules/datasource/rubygems/index.ts
@@ -118,7 +118,7 @@ export class RubygemsDatasource extends Datasource {
packageName: string,
): AsyncResult<ReleaseResult, Error | ZodError> {
const url = joinUrlParts(registryUrl, '/info', packageName);
- return Result.wrap(this.http.get(url))
+ return Result.wrap(this.http.getText(url))
.transform(({ body }) => body)
.parse(GemInfo);
}
diff --git a/lib/modules/datasource/rubygems/versions-endpoint-cache.ts b/lib/modules/datasource/rubygems/versions-endpoint-cache.ts
index 5a6ca75ff8297a963d415679ec400516634bbd48..79ecc45f14583637443f74217ccb979bd6d6c61d 100644
--- a/lib/modules/datasource/rubygems/versions-endpoint-cache.ts
+++ b/lib/modules/datasource/rubygems/versions-endpoint-cache.ts
@@ -209,7 +209,7 @@ export class VersionsEndpointCache {
try {
const url = `${registryUrl}/versions`;
const opts: HttpOptions = { headers: { 'Accept-Encoding': 'gzip' } };
- const { body } = await this.http.get(url, opts);
+ const { body } = await this.http.getText(url, opts);
return parseFullBody(body);
} catch (err) {
if (err instanceof HttpError && err.response?.statusCode === 404) {
@@ -233,7 +233,7 @@ export class VersionsEndpointCache {
['Range']: `bytes=${startByte}-`,
},
};
- const { statusCode, body } = await this.http.get(url, opts);
+ const { statusCode, body } = await this.http.getText(url, opts);
/**
* Rubygems will return the full body instead of `416 Range Not Satisfiable`.
diff --git a/lib/modules/datasource/terraform-provider/index.ts b/lib/modules/datasource/terraform-provider/index.ts
index 0f3e72dcacab757796dcc96730d35af9caf303f4..9f37151eff32baf8d8f72aecd437db18582fdd99 100644
--- a/lib/modules/datasource/terraform-provider/index.ts
+++ b/lib/modules/datasource/terraform-provider/index.ts
@@ -304,7 +304,7 @@ export class TerraformProviderDatasource extends TerraformDatasource {
// The hashes are formatted as the result of sha256sum in plain text, each line: <hash>\t<filename>
let rawHashData: string;
try {
- rawHashData = (await this.http.get(zipHashUrl)).body;
+ rawHashData = (await this.http.getText(zipHashUrl)).body;
} catch (err) {
/* istanbul ignore next */
if (err instanceof ExternalHostError) {
diff --git a/lib/modules/manager/batect-wrapper/artifacts.ts b/lib/modules/manager/batect-wrapper/artifacts.ts
index 344782786f8fb04117fe2f56a9a921d2f1010e78..fb1466d3fd17d9712e5a448a3326d4bb9a8c373b 100644
--- a/lib/modules/manager/batect-wrapper/artifacts.ts
+++ b/lib/modules/manager/batect-wrapper/artifacts.ts
@@ -12,7 +12,7 @@ async function updateArtifact(
const url = `https://github.com/batect/batect/releases/download/${version}/${fileName}`;
try {
- const response = await http.get(url);
+ const response = await http.getText(url);
const contents = response.body;
return {
diff --git a/lib/modules/manager/gradle-wrapper/artifacts.ts b/lib/modules/manager/gradle-wrapper/artifacts.ts
index b523ae69860ce0c49e4a8e5166e4b438f34aea17..50146dc908e56bc06270a34039cd326fbf54c5d4 100644
--- a/lib/modules/manager/gradle-wrapper/artifacts.ts
+++ b/lib/modules/manager/gradle-wrapper/artifacts.ts
@@ -63,7 +63,7 @@ function getDistributionUrl(newPackageFileContent: string): string | null {
}
async function getDistributionChecksum(url: string): Promise<string> {
- const { body } = await http.get(`${url}.sha256`);
+ const { body } = await http.getText(`${url}.sha256`);
return body;
}
diff --git a/lib/modules/platform/bitbucket/index.ts b/lib/modules/platform/bitbucket/index.ts
index 628aefa3ef87f4199c14737bf10cfc593ac8a009..8052ca9204f93eda85c533ab6ab2ab23036cae9e 100644
--- a/lib/modules/platform/bitbucket/index.ts
+++ b/lib/modules/platform/bitbucket/index.ts
@@ -169,7 +169,7 @@ export async function getRawFile(
`/2.0/repositories/${repo}/src/` +
(finalBranchOrTag ?? `HEAD`) +
`/${path}`;
- const res = await bitbucketHttp.get(url, {
+ const res = await bitbucketHttp.getText(url, {
cacheProvider: repoCacheProvider,
memCache: true,
});
diff --git a/lib/modules/platform/gerrit/client.ts b/lib/modules/platform/gerrit/client.ts
index e058f0b703f791e4470bac2196fdb777c875e783..c187dbf2b30f0bf92d09b17c66d19733f4bb6597 100644
--- a/lib/modules/platform/gerrit/client.ts
+++ b/lib/modules/platform/gerrit/client.ts
@@ -181,7 +181,7 @@ class GerritClient {
branch: string,
fileName: string,
): Promise<string> {
- const base64Content = await this.gerritHttp.get(
+ const base64Content = await this.gerritHttp.getText(
`a/projects/${encodeURIComponent(
repo,
)}/branches/${encodeURIComponent(branch)}/files/${encodeURIComponent(fileName)}/content`,
diff --git a/lib/util/http/auth.spec.ts b/lib/util/http/auth.spec.ts
index 208a8209b67da11fa1267681c8a24595a37d9332..e5a8e8dd69c773add8927b709dd56200296593f3 100644
--- a/lib/util/http/auth.spec.ts
+++ b/lib/util/http/auth.spec.ts
@@ -221,6 +221,7 @@ describe('util/http/auth', () => {
hostname: 'amazon.com',
href: 'https://amazon.com',
search: 'something X-Amz-Algorithm something',
+ headers: {},
});
removeAuthorization(opts);
@@ -229,6 +230,7 @@ describe('util/http/auth', () => {
hostname: 'amazon.com',
href: 'https://amazon.com',
search: 'something X-Amz-Algorithm something',
+ headers: {},
});
});
diff --git a/lib/util/http/auth.ts b/lib/util/http/auth.ts
index 0beec921d901c98fc6174b98be402ac138a10e34..afbd65f03455dde370c23d0868ace5955e2a59bd 100644
--- a/lib/util/http/auth.ts
+++ b/lib/util/http/auth.ts
@@ -109,7 +109,6 @@ export function removeAuthorization(options: Options): void {
// if there is no port in the redirect URL string, then delete it from the redirect options.
// This can be evaluated for removal after upgrading to Got v10
const portInUrl = options.href?.split?.('/')?.[2]?.split(':')?.[1];
- // istanbul ignore next
if (!portInUrl) {
delete options.port; // Redirect will instead use 80 or 443 for HTTP or HTTPS respectively
}
diff --git a/lib/util/http/bitbucket-server.spec.ts b/lib/util/http/bitbucket-server.spec.ts
index 855c50db94dfd7de38c7e2d4de098a92fb7ef84b..ec528f79e61970c586bbb21d314be13390e2a973 100644
--- a/lib/util/http/bitbucket-server.spec.ts
+++ b/lib/util/http/bitbucket-server.spec.ts
@@ -32,9 +32,7 @@ describe('util/http/bitbucket-server', () => {
it('invalid path', async () => {
setBaseUrl('some-in|valid-host');
const res = api.getJsonUnchecked('/some-url');
- await expect(res).rejects.toThrow(
- 'Bitbucket Server: cannot parse path /some-url',
- );
+ await expect(res).rejects.toThrow('Invalid URL');
});
it('pagination: uses default limit if not configured', async () => {
diff --git a/lib/util/http/bitbucket-server.ts b/lib/util/http/bitbucket-server.ts
index 2665ae1bdc6287cf3e4960ef0c763e1fb5c03729..e458d9a63a26831716307f016be03a3597b6420a 100644
--- a/lib/util/http/bitbucket-server.ts
+++ b/lib/util/http/bitbucket-server.ts
@@ -1,7 +1,6 @@
import is from '@sindresorhus/is';
-import { logger } from '../../logger';
-import { parseUrl, resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type { InternalJsonUnsafeOptions } from './http';
+import type { HttpMethod, HttpOptions, HttpResponse } from './types';
import { Http } from '.';
const MAX_LIMIT = 100;
@@ -22,45 +21,44 @@ interface PagedResult<T = unknown> {
}
export class BitbucketServerHttp extends Http<BitbucketServerHttpOptions> {
+ protected override get baseUrl(): string | undefined {
+ return baseUrl;
+ }
+
constructor(options?: HttpOptions) {
super('bitbucket-server', options);
}
- protected override async request<T>(
- path: string,
- options?: InternalHttpOptions & BitbucketServerHttpOptions,
+ protected override async requestJsonUnsafe<T>(
+ method: HttpMethod,
+ options: InternalJsonUnsafeOptions<BitbucketServerHttpOptions>,
): Promise<HttpResponse<T>> {
- const opts = { baseUrl, ...options };
- opts.headers = {
- ...opts.headers,
- 'X-Atlassian-Token': 'no-check',
- };
-
- const resolvedUrl = parseUrl(resolveBaseUrl(baseUrl, path));
- if (!resolvedUrl) {
- logger.error({ path }, 'Bitbucket Server: cannot parse path');
- throw new Error(`Bitbucket Server: cannot parse path ${path}`);
- }
+ const resolvedUrl = this.resolveUrl(options.url, options.httpOptions);
+ const opts = { ...options, url: resolvedUrl };
+ opts.httpOptions ??= {};
+ opts.httpOptions.headers ??= {};
+ opts.httpOptions.headers['X-Atlassian-Token'] = 'no-check';
- if (opts.paginate) {
- const limit = opts.limit ?? MAX_LIMIT;
+ const paginate = opts.httpOptions.paginate;
+ if (paginate) {
+ const limit = opts.httpOptions.limit ?? MAX_LIMIT;
resolvedUrl.searchParams.set('limit', limit.toString());
}
- const result = await super.request<T | PagedResult<T>>(
- resolvedUrl.toString(),
+ const result = await super.requestJsonUnsafe<T | PagedResult<T>>(
+ method,
opts,
);
- if (opts.paginate && isPagedResult(result.body)) {
+ if (paginate && isPagedResult(result.body)) {
const collectedValues = [...result.body.values];
let nextPageStart = result.body.nextPageStart;
while (nextPageStart) {
resolvedUrl.searchParams.set('start', nextPageStart.toString());
- const nextResult = await super.request<PagedResult<T>>(
- resolvedUrl.toString(),
+ const nextResult = await super.requestJsonUnsafe<PagedResult<T>>(
+ method,
opts,
);
collectedValues.push(...nextResult.body.values);
diff --git a/lib/util/http/bitbucket.ts b/lib/util/http/bitbucket.ts
index 28341b0291def4f87e1b73583b9e18e41c2a742e..bee3ca1c84c2d9d51d3468ac565e52319a74a0d4 100644
--- a/lib/util/http/bitbucket.ts
+++ b/lib/util/http/bitbucket.ts
@@ -1,8 +1,7 @@
import is from '@sindresorhus/is';
-import { logger } from '../../logger';
import type { PagedResult } from '../../modules/platform/bitbucket/types';
-import { parseUrl, resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse } from './types';
+import type { InternalJsonUnsafeOptions } from './http';
+import type { HttpMethod, HttpOptions, HttpResponse } from './types';
import { Http } from '.';
const MAX_PAGES = 100;
@@ -10,9 +9,9 @@ const MAX_PAGELEN = 100;
let baseUrl = 'https://api.bitbucket.org/';
-export const setBaseUrl = (url: string): void => {
+export function setBaseUrl(url: string): void {
baseUrl = url;
-};
+}
export interface BitbucketHttpOptions extends HttpOptions {
paginate?: boolean;
@@ -20,46 +19,46 @@ export interface BitbucketHttpOptions extends HttpOptions {
}
export class BitbucketHttp extends Http<BitbucketHttpOptions> {
+ protected override get baseUrl(): string | undefined {
+ return baseUrl;
+ }
+
constructor(type = 'bitbucket', options?: BitbucketHttpOptions) {
super(type, options);
}
- protected override async request<T>(
- path: string,
- options?: BitbucketHttpOptions,
+ protected override async requestJsonUnsafe<T>(
+ method: HttpMethod,
+ options: InternalJsonUnsafeOptions<BitbucketHttpOptions>,
): Promise<HttpResponse<T>> {
- const opts = { baseUrl, ...options };
+ const resolvedUrl = this.resolveUrl(options.url, options.httpOptions);
+ const opts = { ...options, url: resolvedUrl };
+ const paginate = opts.httpOptions?.paginate;
- const resolvedURL = parseUrl(resolveBaseUrl(baseUrl, path));
-
- // istanbul ignore if: this should never happen
- if (is.nullOrUndefined(resolvedURL)) {
- logger.error({ path }, 'Bitbucket: cannot parse path');
- throw new Error(`Bitbucket: cannot parse path ${path}`);
+ if (paginate && !hasPagelen(resolvedUrl)) {
+ const pagelen = opts.httpOptions!.pagelen ?? MAX_PAGELEN;
+ resolvedUrl.searchParams.set('pagelen', pagelen.toString());
}
- if (opts.paginate && !hasPagelen(resolvedURL)) {
- const pagelen = opts.pagelen ?? MAX_PAGELEN;
- resolvedURL.searchParams.set('pagelen', pagelen.toString());
- }
-
- const result = await super.request<T>(resolvedURL.toString(), opts);
-
- if (opts.paginate && isPagedResult(result.body)) {
- const resultBody = result.body as PagedResult<T>;
- let page = 1;
- let nextURL = resultBody.next;
-
- while (is.nonEmptyString(nextURL) && page <= MAX_PAGES) {
- const nextResult = await super.request<PagedResult<T>>(
- nextURL,
- options!,
+ const result = await super.requestJsonUnsafe<T | PagedResult<T>>(
+ method,
+ opts,
+ );
+
+ if (paginate && isPagedResult(result.body)) {
+ const resultBody = result.body;
+ let nextURL = result.body.next;
+ let page = 2;
+
+ for (; nextURL && page <= MAX_PAGES; page++) {
+ resolvedUrl.searchParams.set('page', page.toString());
+ const nextResult = await super.requestJsonUnsafe<PagedResult<T>>(
+ method,
+ opts,
);
resultBody.values.push(...nextResult.body.values);
-
- nextURL = nextResult.body?.next;
- page += 1;
+ nextURL = nextResult.body.next;
}
// Override other page-related attributes
@@ -67,12 +66,12 @@ export class BitbucketHttp extends Http<BitbucketHttpOptions> {
resultBody.size =
page <= MAX_PAGES
? resultBody.values.length
- : /* istanbul ignore next */ undefined;
+ : /* v8 ignore next */ undefined;
resultBody.next =
- page <= MAX_PAGES ? nextURL : /* istanbul ignore next */ undefined;
+ page <= MAX_PAGES ? nextURL : /* v8 ignore next */ undefined;
}
- return result;
+ return result as HttpResponse<T>;
}
}
@@ -80,6 +79,6 @@ function hasPagelen(url: URL): boolean {
return !is.nullOrUndefined(url.searchParams.get('pagelen'));
}
-function isPagedResult(obj: any): obj is PagedResult {
+function isPagedResult<T>(obj: any): obj is PagedResult<T> {
return is.nonEmptyObject(obj) && Array.isArray(obj.values);
}
diff --git a/lib/util/http/cache/abstract-http-cache-provider.ts b/lib/util/http/cache/abstract-http-cache-provider.ts
index 2d1eb05955890b309608a206289e65222bce982a..08288a0bddc079b8b932776d929398d9367ed19d 100644
--- a/lib/util/http/cache/abstract-http-cache-provider.ts
+++ b/lib/util/http/cache/abstract-http-cache-provider.ts
@@ -66,7 +66,7 @@ export abstract class AbstractHttpCacheProvider implements HttpCacheProvider {
timestamp,
});
- // istanbul ignore if: should never happen
+ /* v8 ignore next 4: should never happen */
if (!newHttpCache) {
logger.debug(`http cache: failed to persist cache for ${url}`);
return resp;
diff --git a/lib/util/http/cache/package-http-cache-provider.spec.ts b/lib/util/http/cache/package-http-cache-provider.spec.ts
index 6db9bf461feb5396a74a63b92621371335044d06..1e0d30a121354a5694459694c792f89d2168f05f 100644
--- a/lib/util/http/cache/package-http-cache-provider.spec.ts
+++ b/lib/util/http/cache/package-http-cache-provider.spec.ts
@@ -52,7 +52,7 @@ describe('util/http/cache/package-http-cache-provider', () => {
});
httpMock.scope(url).get('').reply(200, 'new response');
- const res = await http.get(url, { cacheProvider });
+ const res = await http.getText(url, { cacheProvider });
expect(res.body).toBe('new response');
});
@@ -69,7 +69,7 @@ describe('util/http/cache/package-http-cache-provider', () => {
namespace: '_test-namespace',
});
- const res = await http.get(url, { cacheProvider });
+ const res = await http.getText(url, { cacheProvider });
expect(res.body).toBe('cached response');
expect(packageCache.set).not.toHaveBeenCalled();
@@ -77,7 +77,7 @@ describe('util/http/cache/package-http-cache-provider', () => {
mockTime('2024-06-15T00:15:00.000Z');
httpMock.scope(url).get('').reply(200, 'new response', { etag: 'foobar' });
- const res2 = await http.get(url, { cacheProvider });
+ const res2 = await http.getText(url, { cacheProvider });
expect(res2.body).toBe('new response');
expect(packageCache.set).toHaveBeenCalled();
});
@@ -91,7 +91,7 @@ describe('util/http/cache/package-http-cache-provider', () => {
.get('')
.reply(200, 'fetched response', { etag: 'foobar' });
- const res = await http.get(url, { cacheProvider });
+ const res = await http.getText(url, { cacheProvider });
expect(res.body).toBe('fetched response');
expect(cache).toEqual({
@@ -121,7 +121,7 @@ describe('util/http/cache/package-http-cache-provider', () => {
});
httpMock.scope(url).get('').reply(500);
- const res = await http.get(url, { cacheProvider });
+ const res = await http.getText(url, { cacheProvider });
expect(res.body).toBe('cached response');
expect(packageCache.set).not.toHaveBeenCalled();
diff --git a/lib/util/http/gerrit.spec.ts b/lib/util/http/gerrit.spec.ts
index 7d6563817cf5a8d1b7b3ad19296b4c0ba5ce6141..3e92dd7bba2cc22155a19ffbeb3a54a212a4efc6 100644
--- a/lib/util/http/gerrit.spec.ts
+++ b/lib/util/http/gerrit.spec.ts
@@ -18,7 +18,7 @@ describe('util/http/gerrit', () => {
.get(/some-url\/$/)
.reply(200, body, { 'content-type': 'text/plain;charset=utf-8' });
- const res = await api.get(pathOrUrl);
+ const res = await api.getText(pathOrUrl);
expect(res.body).toEqual(body);
});
diff --git a/lib/util/http/gerrit.ts b/lib/util/http/gerrit.ts
index a690ba951b3147e660566b4670228e475dcaa58d..babd02bf43a99d719a45516071283803217abd70 100644
--- a/lib/util/http/gerrit.ts
+++ b/lib/util/http/gerrit.ts
@@ -1,8 +1,9 @@
import { parseJson } from '../common';
import { regEx } from '../regex';
import { isHttpUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
-import { Http } from './index';
+import type { InternalHttpOptions } from './http';
+import type { HttpOptions } from './types';
+import { Http } from '.';
let baseUrl: string;
export function setBaseUrl(url: string): void {
@@ -16,23 +17,30 @@ export function setBaseUrl(url: string): void {
export class GerritHttp extends Http {
private static magicPrefix = regEx(/^\)]}'\n/g);
+ protected override get baseUrl(): string | undefined {
+ return baseUrl;
+ }
+
constructor(options?: HttpOptions) {
super('gerrit', options);
}
- protected override async request<T>(
- path: string,
- options?: InternalHttpOptions,
- ): Promise<HttpResponse<T>> {
- const url = isHttpUrl(path) ? path : baseUrl + path;
- const opts: InternalHttpOptions = {
- parseJson: (text: string) =>
- parseJson(text.replace(GerritHttp.magicPrefix, ''), path),
- ...options,
- };
- opts.headers = {
- ...opts.headers,
- };
- return await super.request<T>(url, opts);
+ protected override resolveUrl(
+ requestUrl: string | URL,
+ options: HttpOptions | undefined,
+ ): URL {
+ // ensure trailing slash for gerrit
+ return super.resolveUrl(
+ isHttpUrl(requestUrl) ? requestUrl : `${baseUrl}${requestUrl}`,
+ options,
+ );
+ }
+
+ protected override processOptions(
+ url: URL,
+ options: InternalHttpOptions,
+ ): void {
+ options.parseJson = (text: string) =>
+ parseJson(text.replace(GerritHttp.magicPrefix, ''), url.pathname);
}
}
diff --git a/lib/util/http/gitea.ts b/lib/util/http/gitea.ts
index f69470c44ed6f317c72003c638282afa596f62fd..51fb0cba281446e38dd379e6ac189ec6ef9db7f6 100644
--- a/lib/util/http/gitea.ts
+++ b/lib/util/http/gitea.ts
@@ -1,6 +1,6 @@
import is from '@sindresorhus/is';
-import { resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type { InternalJsonUnsafeOptions } from './http';
+import type { HttpMethod, HttpOptions, HttpResponse } from './types';
import { Http } from '.';
let baseUrl: string;
@@ -24,28 +24,28 @@ function getPaginationContainer<T = unknown>(body: unknown): T[] | null {
return null;
}
-function resolveUrl(path: string, base: string): URL {
- const resolvedUrlString = resolveBaseUrl(base, path);
- return new URL(resolvedUrlString);
-}
-
export class GiteaHttp extends Http<GiteaHttpOptions> {
+ protected override get baseUrl(): string | undefined {
+ return baseUrl;
+ }
+
constructor(hostType?: string, options?: HttpOptions) {
super(hostType ?? 'gitea', options);
}
- protected override async request<T>(
- path: string,
- options?: InternalHttpOptions & GiteaHttpOptions,
+ protected override async requestJsonUnsafe<T = unknown>(
+ method: HttpMethod,
+ options: InternalJsonUnsafeOptions<GiteaHttpOptions>,
): Promise<HttpResponse<T>> {
- const resolvedUrl = resolveUrl(path, options?.baseUrl ?? baseUrl);
+ const resolvedUrl = this.resolveUrl(options.url, options.httpOptions);
const opts = {
- baseUrl,
...options,
+ url: resolvedUrl,
};
- const res = await super.request<T>(resolvedUrl, opts);
+ const res = await super.requestJsonUnsafe<T>(method, opts);
const pc = getPaginationContainer<T>(res.body);
- if (opts.paginate && pc) {
+ if (opts.httpOptions?.paginate && pc) {
+ delete opts.httpOptions.paginate;
const total = parseInt(res.headers['x-total-count'] as string, 10);
let nextPage = parseInt(resolvedUrl.searchParams.get('page') ?? '1', 10);
@@ -53,7 +53,7 @@ export class GiteaHttp extends Http<GiteaHttpOptions> {
nextPage += 1;
resolvedUrl.searchParams.set('page', nextPage.toString());
- const nextRes = await super.request<T>(resolvedUrl.toString(), opts);
+ const nextRes = await super.requestJsonUnsafe<T>(method, opts);
const nextPc = getPaginationContainer<T>(nextRes.body);
if (nextPc === null) {
break;
diff --git a/lib/util/http/github.spec.ts b/lib/util/http/github.spec.ts
index 7223400b9ba493fed33e4dc96f0339100d157e32..322cf10d7c318571e706f51e80af684afa02a832 100644
--- a/lib/util/http/github.spec.ts
+++ b/lib/util/http/github.spec.ts
@@ -1,4 +1,3 @@
-import { Buffer } from 'node:buffer';
import { codeBlock } from 'common-tags';
import { DateTime } from 'luxon';
import * as httpMock from '../../../test/http-mock';
@@ -566,7 +565,7 @@ describe('util/http/github', () => {
.scope('https://ghe.mycompany.com')
.post('/api/graphql')
.reply(200, { data: { repository } });
- await githubApi.requestGraphql(graphqlQuery);
+ await githubApi.requestGraphql(graphqlQuery, { token: 'abc' });
const [req] = httpMock.getTrace();
expect(req).toBeDefined();
expect(req.url).toBe('https://ghe.mycompany.com/api/graphql');
@@ -939,21 +938,5 @@ describe('util/http/github', () => {
body: 'test',
});
});
-
- it('throw error if a', async () => {
- httpMock
- .scope(githubApiHost)
- .get('/foo/bar/contents/lore/ipsum.bin')
- .matchHeader(
- 'accept',
- 'application/vnd.github.raw+json, application/vnd.github.v3+json',
- )
- .reply(200, Buffer.from('foo', 'binary'));
- await expect(
- githubApi.getRawTextFile(`foo/bar/contents/lore/ipsum.bin`, {
- responseType: 'buffer',
- }),
- ).rejects.toThrow();
- });
});
});
diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts
index 56db54db38716c77c989888242f97489d607ab88..b1f45271cd0f061a9685c497790f7c4e71eec900 100644
--- a/lib/util/http/github.ts
+++ b/lib/util/http/github.ts
@@ -13,14 +13,15 @@ import { maskToken } from '../mask';
import * as p from '../promises';
import { range } from '../range';
import { regEx } from '../regex';
-import { joinUrlParts, parseLinkHeader, resolveBaseUrl } from '../url';
+import { joinUrlParts, parseLinkHeader } from '../url';
import { findMatchingRule } from './host-rules';
+import type { InternalHttpOptions, InternalJsonUnsafeOptions } from './http';
import type { GotLegacyError } from './legacy';
import type {
GraphqlOptions,
+ HttpMethod,
HttpOptions,
HttpResponse,
- InternalHttpOptions,
} from './types';
import { Http } from '.';
@@ -267,23 +268,20 @@ function replaceUrlBase(url: URL, baseUrl: string): URL {
}
export class GithubHttp extends Http<GithubHttpOptions> {
+ protected override get baseUrl(): string | undefined {
+ return baseUrl;
+ }
+
constructor(hostType = 'github', options?: GithubHttpOptions) {
super(hostType, options);
}
- protected override async request<T>(
- url: string | URL,
- options?: InternalHttpOptions & GithubHttpOptions,
- okToRetry = true,
- ): Promise<HttpResponse<T>> {
- const opts: InternalHttpOptions & GithubHttpOptions = {
- baseUrl,
- ...options,
- throwHttpErrors: true,
- };
-
+ protected override processOptions(
+ url: URL,
+ opts: InternalHttpOptions & GithubHttpOptions,
+ ): void {
if (!opts.token) {
- const authUrl = new URL(resolveBaseUrl(opts.baseUrl!, url));
+ const authUrl = new URL(url);
if (opts.repository) {
// set authUrl to https://api.github.com/repos/org/repo or https://gihub.domain.com/api/v3/repos/org/repo
@@ -317,68 +315,87 @@ export class GithubHttp extends Http<GithubHttpOptions> {
...opts.headers,
accept,
};
+ }
- try {
- const result = await super.request<T>(url, opts);
- if (opts.paginate) {
- // Check if result is paginated
- const pageLimit = opts.pageLimit ?? 10;
- const linkHeader = parseLinkHeader(result?.headers?.link);
- const next = linkHeader?.next;
- if (next?.url && linkHeader?.last?.page) {
- let lastPage = parseInt(linkHeader.last.page, 10);
- // istanbul ignore else: needs a test
- if (!process.env.RENOVATE_PAGINATE_ALL && opts.paginate !== 'all') {
- lastPage = Math.min(pageLimit, lastPage);
- }
- const baseUrl = opts.baseUrl;
- const parsedUrl = new URL(next.url, baseUrl);
- const rebasePagination =
- !!baseUrl &&
- !!process.env.RENOVATE_X_REBASE_PAGINATION_LINKS &&
- // Preserve github.com URLs for use cases like release notes
- parsedUrl.origin !== 'https://api.github.com';
- const firstPageUrl = rebasePagination
- ? replaceUrlBase(parsedUrl, baseUrl)
- : parsedUrl;
- const queue = [...range(2, lastPage)].map(
- (pageNumber) => (): Promise<HttpResponse<T>> => {
- // copy before modifying searchParams
- const nextUrl = new URL(firstPageUrl);
- nextUrl.searchParams.set('page', String(pageNumber));
- return this.request<T>(
- nextUrl,
- { ...opts, paginate: false, cacheProvider: undefined },
- okToRetry,
- );
- },
- );
- const pages = await p.all(queue);
- if (opts.paginationField && is.plainObject(result.body)) {
- const paginatedResult = result.body[opts.paginationField];
- if (is.array<T>(paginatedResult)) {
- for (const nextPage of pages) {
- if (is.plainObject(nextPage.body)) {
- const nextPageResults = nextPage.body[opts.paginationField];
- if (is.array<T>(nextPageResults)) {
- paginatedResult.push(...nextPageResults);
- }
+ protected override handleError(
+ url: string | URL,
+ opts: HttpOptions,
+ err: GotLegacyError,
+ ): never {
+ throw handleGotError(err, url, opts);
+ }
+
+ protected override async requestJsonUnsafe<T>(
+ method: HttpMethod,
+ options: InternalJsonUnsafeOptions<GithubHttpOptions>,
+ ): Promise<HttpResponse<T>> {
+ const httpOptions = options.httpOptions ?? {};
+ const resolvedUrl = this.resolveUrl(options.url, httpOptions);
+ const opts = {
+ ...options,
+ url: resolvedUrl,
+ };
+
+ const result = await super.requestJsonUnsafe<T>(method, opts);
+ if (httpOptions.paginate) {
+ delete httpOptions.cacheProvider;
+ // Check if result is paginated
+ const pageLimit = httpOptions.pageLimit ?? 10;
+ const linkHeader = parseLinkHeader(result?.headers?.link);
+ const next = linkHeader?.next;
+ if (next?.url && linkHeader?.last?.page) {
+ let lastPage = parseInt(linkHeader.last.page, 10);
+ if (
+ !process.env.RENOVATE_PAGINATE_ALL &&
+ httpOptions.paginate !== 'all'
+ ) {
+ lastPage = Math.min(pageLimit, lastPage);
+ }
+ const baseUrl = httpOptions.baseUrl ?? this.baseUrl;
+ const parsedUrl = new URL(next.url, baseUrl);
+ const rebasePagination =
+ !!baseUrl &&
+ !!process.env.RENOVATE_X_REBASE_PAGINATION_LINKS &&
+ // Preserve github.com URLs for use cases like release notes
+ parsedUrl.origin !== 'https://api.github.com';
+ const firstPageUrl = rebasePagination
+ ? replaceUrlBase(parsedUrl, baseUrl)
+ : parsedUrl;
+ const queue = [...range(2, lastPage)].map(
+ (pageNumber) => (): Promise<HttpResponse<T>> => {
+ // copy before modifying searchParams
+ const nextUrl = new URL(firstPageUrl);
+ nextUrl.searchParams.set('page', String(pageNumber));
+ return super.requestJsonUnsafe<T>(method, {
+ ...opts,
+ url: nextUrl,
+ });
+ },
+ );
+ const pages = await p.all(queue);
+ if (httpOptions.paginationField && is.plainObject(result.body)) {
+ const paginatedResult = result.body[httpOptions.paginationField];
+ if (is.array<T>(paginatedResult)) {
+ for (const nextPage of pages) {
+ if (is.plainObject(nextPage.body)) {
+ const nextPageResults =
+ nextPage.body[httpOptions.paginationField];
+ if (is.array<T>(nextPageResults)) {
+ paginatedResult.push(...nextPageResults);
}
}
}
- } else if (is.array<T>(result.body)) {
- for (const nextPage of pages) {
- if (is.array<T>(nextPage.body)) {
- result.body.push(...nextPage.body);
- }
+ }
+ } else if (is.array<T>(result.body)) {
+ for (const nextPage of pages) {
+ if (is.array<T>(nextPage.body)) {
+ result.body.push(...nextPage.body);
}
}
}
}
- return result;
- } catch (err) {
- throw handleGotError(err, url, opts);
}
+ return result;
}
public async requestGraphql<T = unknown>(
@@ -522,12 +539,6 @@ export class GithubHttp extends Http<GithubHttpOptions> {
newURL = joinUrlParts(options.repository, 'contents', url);
}
- const result = await this.get(newURL, newOptions);
- if (!is.string(result.body)) {
- throw new Error(
- `Expected raw text file but received ${typeof result.body}`,
- );
- }
- return result;
+ return await this.getText(newURL, newOptions);
}
}
diff --git a/lib/util/http/gitlab.spec.ts b/lib/util/http/gitlab.spec.ts
index 9b0b9c6d88eb98c067d5e9d16691cdf7b714902c..0838af126451d6295c9b7df7aea2b5ffb123dbdc 100644
--- a/lib/util/http/gitlab.spec.ts
+++ b/lib/util/http/gitlab.spec.ts
@@ -4,6 +4,7 @@ import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
import { GitlabReleasesDatasource } from '../../modules/datasource/gitlab-releases';
import * as hostRules from '../host-rules';
import { GitlabHttp, setBaseUrl } from './gitlab';
+import { logger } from '~test/util';
hostRules.add({
hostType: 'gitlab',
@@ -36,18 +37,26 @@ describe('util/http/gitlab', () => {
.scope(gitlabApiHost)
.get('/api/v4/some-url')
.reply(200, ['a'], {
- link: '<https://gitlab.com/api/v4/some-url&page=2>; rel="next", <https://gitlab.com/api/v4/some-url&page=3>; rel="last"',
+ link: '<https://gitlab.com/api/v4/some-url&page=2>; rel="next", <https://gitlab.com/api/v4/some-url&page=4>; rel="last"',
})
.get('/api/v4/some-url&page=2')
.reply(200, ['b', 'c'], {
- link: '<https://gitlab.com/api/v4/some-url&page=3>; rel="next", <https://gitlab.com/api/v4/some-url&page=3>; rel="last"',
+ link: '<https://gitlab.com/api/v4/some-url&page=3>; rel="next", <https://gitlab.com/api/v4/some-url&page=4>; rel="last"',
})
.get('/api/v4/some-url&page=3')
- .reply(200, ['d']);
+ .reply(200, ['d'], {
+ link: '<https://gitlab.com/api/v4/some-url&page=4>; rel="next", <https://gitlab.com/api/v4/some-url&page=4>; rel="last"',
+ })
+ .get('/api/v4/some-url&page=4')
+ .reply(500);
const res = await gitlabApi.getJsonUnchecked('some-url', {
paginate: true,
});
expect(res.body).toHaveLength(4);
+ expect(logger.logger.warn).toHaveBeenCalledWith(
+ { err: expect.any(Error) },
+ 'Pagination error',
+ );
});
it('paginates with GITLAB_IGNORE_REPO_URL set', async () => {
diff --git a/lib/util/http/gitlab.ts b/lib/util/http/gitlab.ts
index 28beca26261f11dac55063060455ce4ecf4275b4..4bdda9dd7ee21dc4dbf55969726d390ede098a92 100644
--- a/lib/util/http/gitlab.ts
+++ b/lib/util/http/gitlab.ts
@@ -1,9 +1,10 @@
import is from '@sindresorhus/is';
-import type { RetryObject } from 'got';
+import { RequestError, type RetryObject } from 'got';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import { parseLinkHeader, parseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type { InternalJsonUnsafeOptions } from './http';
+import type { HttpMethod, HttpOptions, HttpResponse } from './types';
import { Http } from '.';
let baseUrl = 'https://gitlab.com/api/v4/';
@@ -16,73 +17,92 @@ export interface GitlabHttpOptions extends HttpOptions {
}
export class GitlabHttp extends Http<GitlabHttpOptions> {
+ protected override get baseUrl(): string | undefined {
+ return baseUrl;
+ }
+
constructor(type = 'gitlab', options?: GitlabHttpOptions) {
super(type, options);
}
- protected override async request<T>(
- url: string | URL,
- options?: InternalHttpOptions & GitlabHttpOptions,
+ protected override async requestJsonUnsafe<T = unknown>(
+ method: HttpMethod,
+ options: InternalJsonUnsafeOptions<GitlabHttpOptions>,
): Promise<HttpResponse<T>> {
+ const resolvedUrl = this.resolveUrl(options.url, options.httpOptions);
const opts = {
- baseUrl,
...options,
- throwHttpErrors: true,
+ url: resolvedUrl,
};
+ opts.httpOptions ??= {};
+ opts.httpOptions.throwHttpErrors = true;
+
+ const result = await super.requestJsonUnsafe<T>(method, opts);
+ if (opts.httpOptions.paginate && is.array(result.body)) {
+ // Check if result is paginated
+ try {
+ const linkHeader = parseLinkHeader(result.headers.link);
+ const nextUrl = linkHeader?.next?.url
+ ? parseUrl(linkHeader.next.url)
+ : null;
+ if (nextUrl) {
+ if (process.env.GITLAB_IGNORE_REPO_URL) {
+ const defaultEndpoint = new URL(baseUrl);
+ nextUrl.protocol = defaultEndpoint.protocol;
+ nextUrl.host = defaultEndpoint.host;
+ }
- try {
- const result = await super.request<T>(url, opts);
- if (opts.paginate && is.array(result.body)) {
- // Check if result is paginated
- try {
- const linkHeader = parseLinkHeader(result.headers.link);
- const nextUrl = linkHeader?.next?.url
- ? parseUrl(linkHeader.next.url)
- : null;
- if (nextUrl) {
- if (process.env.GITLAB_IGNORE_REPO_URL) {
- const defaultEndpoint = new URL(baseUrl);
- nextUrl.protocol = defaultEndpoint.protocol;
- nextUrl.host = defaultEndpoint.host;
- }
+ opts.url = nextUrl;
- const nextResult = await this.request<T>(nextUrl, opts);
- if (is.array(nextResult.body)) {
- result.body.push(...nextResult.body);
- }
+ const nextResult = await this.requestJsonUnsafe<T>(method, opts);
+ if (is.array(nextResult.body)) {
+ result.body.push(...nextResult.body);
}
- } catch (err) /* istanbul ignore next */ {
- logger.warn({ err }, 'Pagination error');
}
+ } catch (err) {
+ logger.warn({ err }, 'Pagination error');
}
- return result;
- } catch (err) {
- if (err.statusCode === 404) {
+ }
+ return result;
+ }
+
+ protected override handleError(
+ url: string | URL,
+ _httpOptions: HttpOptions,
+ err: Error,
+ ): never {
+ if (err instanceof RequestError && err.response?.statusCode) {
+ if (err.response.statusCode === 404) {
logger.trace({ err }, 'GitLab 404');
- logger.debug({ url: err.url }, 'GitLab API 404');
+ logger.debug({ url }, 'GitLab API 404');
throw err;
}
logger.debug({ err }, 'Gitlab API error');
if (
- err.statusCode === 429 ||
- (err.statusCode >= 500 && err.statusCode < 600)
+ err.response.statusCode === 429 ||
+ (err.response.statusCode >= 500 && err.response.statusCode < 600)
) {
throw new ExternalHostError(err, 'gitlab');
}
- const platformFailureCodes = [
- 'EAI_AGAIN',
- 'ECONNRESET',
- 'ETIMEDOUT',
- 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
- ];
- if (platformFailureCodes.includes(err.code)) {
- throw new ExternalHostError(err, 'gitlab');
- }
- if (err.name === 'ParseError') {
- throw new ExternalHostError(err, 'gitlab');
- }
- throw err;
}
+ const platformFailureCodes = [
+ 'EAI_AGAIN',
+ 'ECONNRESET',
+ 'ETIMEDOUT',
+ 'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
+ ];
+ // TODO: fix test, should be `RequestError`
+ if (
+ 'code' in err &&
+ is.string(err.code) &&
+ platformFailureCodes.includes(err.code)
+ ) {
+ throw new ExternalHostError(err, 'gitlab');
+ }
+ if (err.name === 'ParseError') {
+ throw new ExternalHostError(err, 'gitlab');
+ }
+ throw err;
}
protected override calculateRetryDelay(retryObject: RetryObject): number {
diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts
index b70b19b20f7292926bfe6a94b809ba268ae21fef..cd946419d0ebf51dcb8df1f3d0f03ddddb10017e 100644
--- a/lib/util/http/host-rules.ts
+++ b/lib/util/http/host-rules.ts
@@ -12,8 +12,9 @@ import type { HostRule } from '../../types';
import * as hostRules from '../host-rules';
import { matchRegexOrGlobList } from '../string-match';
import { parseUrl } from '../url';
+import type { InternalHttpOptions } from './http';
import { keepAliveAgents } from './keep-alive';
-import type { GotOptions, InternalHttpOptions } from './types';
+import type { GotOptions } from './types';
export type HostRulesGotOptions = Pick<
GotOptions & InternalHttpOptions,
diff --git a/lib/util/http/http.ts b/lib/util/http/http.ts
new file mode 100644
index 0000000000000000000000000000000000000000..589bce68527dbedcfbe0a727e8078d371e32408b
--- /dev/null
+++ b/lib/util/http/http.ts
@@ -0,0 +1,29 @@
+import type { Options } from 'got';
+import type { SetRequired } from 'type-fest';
+import type { ZodType } from 'zod';
+import type { GotOptions, HttpMethod, HttpOptions } from './types';
+
+export interface InternalJsonUnsafeOptions<
+ Opts extends HttpOptions = HttpOptions,
+> {
+ url: string | URL;
+ httpOptions?: Opts;
+}
+
+export interface InternalJsonOptions<
+ Opts extends HttpOptions,
+ ResT = unknown,
+ Schema extends ZodType<ResT> = ZodType<ResT>,
+> extends InternalJsonUnsafeOptions<Opts> {
+ schema?: Schema;
+}
+
+export type InternalGotOptions = SetRequired<GotOptions, 'method' | 'context'>;
+
+export interface InternalHttpOptions extends HttpOptions {
+ json?: HttpOptions['body'];
+
+ method?: HttpMethod;
+
+ parseJson?: Options['parseJson'];
+}
diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts
index 8af2746c61ee0311ea1845d582b316237ad1bb19..aa3745b99019cc8b782331cd4f4438268cb845e1 100644
--- a/lib/util/http/index.spec.ts
+++ b/lib/util/http/index.spec.ts
@@ -28,7 +28,7 @@ describe('util/http/index', () => {
it('get', async () => {
httpMock.scope(baseUrl).get('/test').reply(200);
- expect(await http.get('http://renovate.com/test')).toEqual({
+ expect(await http.getText('http://renovate.com/test')).toEqual({
authorization: false,
body: '',
headers: {},
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index 245fdb541988dc4592359ed526a0c220bd827053..6cd54196705b703388f95732f0816c929d7c6e0e 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -1,3 +1,6 @@
+// TODO: refactor code to remove this (#9651)
+import './legacy';
+
import is from '@sindresorhus/is';
import merge from 'deepmerge';
import type { Options, RetryObject } from 'got';
@@ -14,24 +17,28 @@ import * as memCache from '../cache/memory';
import { hash } from '../hash';
import { type AsyncResult, Result } from '../result';
import { type HttpRequestStatsDataPoint, HttpStats } from '../stats';
-import { resolveBaseUrl } from '../url';
+import { isHttpUrl, parseUrl, resolveBaseUrl } from '../url';
import { parseSingleYaml } from '../yaml';
import { applyAuthorization, removeAuthorization } from './auth';
import { hooks } from './hooks';
import { applyHostRule, findMatchingRule } from './host-rules';
+import type {
+ InternalGotOptions,
+ InternalHttpOptions,
+ InternalJsonOptions,
+ InternalJsonUnsafeOptions,
+} from './http';
import { getQueue } from './queue';
import { getRetryAfter, wrapWithRetry } from './retry-after';
import { getThrottle } from './throttle';
import type {
- GotJSONOptions,
+ GotBufferOptions,
GotOptions,
GotTask,
+ HttpMethod,
HttpOptions,
HttpResponse,
- InternalHttpOptions,
} from './types';
-// TODO: refactor code to remove this (#9651)
-import './legacy';
import { copyResponse } from './util';
export { RequestError as HttpError };
@@ -39,16 +46,6 @@ export { RequestError as HttpError };
export class EmptyResultError extends Error {}
export type SafeJsonError = RequestError | ZodError | EmptyResultError;
-interface HttpFnArgs<
- Opts extends HttpOptions,
- ResT = unknown,
- Schema extends ZodType<ResT> = ZodType<ResT>,
-> {
- url: string;
- httpOptions?: Opts;
- schema?: Schema;
-}
-
function applyDefaultHeaders(options: Options): void {
const renovateVersion = pkg.version;
options.headers = {
@@ -61,48 +58,38 @@ function applyDefaultHeaders(options: Options): void {
type QueueStatsData = Pick<HttpRequestStatsDataPoint, 'queueMs'>;
-// Note on types:
-// options.requestType can be either 'json' or 'buffer', but `T` should be
-// `Buffer` in the latter case.
-// We don't declare overload signatures because it's immediately wrapped by
-// `request`.
-async function gotTask<T>(
+async function gotTask(
url: string,
options: SetRequired<GotOptions, 'method'>,
queueStats: QueueStatsData,
-): Promise<HttpResponse<T>> {
+): Promise<HttpResponse<unknown>> {
logger.trace({ url, options }, 'got request');
let duration = 0;
let statusCode = 0;
-
try {
// Cheat the TS compiler using `as` to pick a specific overload.
// Otherwise it doesn't typecheck.
- const resp = await got<T>(url, { ...options, hooks } as GotJSONOptions);
+ const resp = await got(url, { ...options, hooks } as GotBufferOptions);
statusCode = resp.statusCode;
duration =
- resp.timings.phases.total ??
- /* istanbul ignore next: can't be tested */ 0;
+ resp.timings.phases.total ?? /* v8 ignore next: can't be tested */ 0;
return resp;
} catch (error) {
if (error instanceof RequestError) {
- statusCode =
- error.response?.statusCode ??
- /* istanbul ignore next: can't be tested */ -1;
+ statusCode = error.response?.statusCode ?? -1;
duration =
- error.timings?.phases.total ??
- /* istanbul ignore next: can't be tested */ -1;
+ error.timings?.phases.total ?? /* v8 ignore next: can't be tested */ -1;
const method = options.method.toUpperCase();
- const code = error.code ?? /* istanbul ignore next */ 'UNKNOWN';
- const retryCount =
- error.request?.retryCount ?? /* istanbul ignore next */ -1;
+ const code = error.code ?? /* v8 ignore next */ 'UNKNOWN';
+ const retryCount = error.request?.retryCount ?? /* v8 ignore next */ -1;
logger.debug(
`${method} ${url} = (code=${code}, statusCode=${statusCode} retryCount=${retryCount}, duration=${duration})`,
);
}
throw error;
+ /* v8 ignore next: 🛠https://github.com/bcoe/c8/issues/229 */
} finally {
HttpStats.write({
method: options.method,
@@ -115,16 +102,21 @@ async function gotTask<T>(
}
export class Http<Opts extends HttpOptions = HttpOptions> {
- private options?: GotOptions;
+ private readonly options: InternalGotOptions;
+
+ protected get baseUrl(): string | undefined {
+ return undefined;
+ }
constructor(
protected hostType: string,
options: HttpOptions = {},
) {
const retryLimit = process.env.NODE_ENV === 'test' ? 0 : 2;
- this.options = merge<GotOptions>(
+ this.options = merge<InternalGotOptions>(
options,
{
+ method: 'get',
context: { hostType },
retry: {
calculateDelay: (retryObject) =>
@@ -136,19 +128,34 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
{ isMergeableObject: is.plainObject },
);
}
+ private async request(
+ requestUrl: string | URL,
+ httpOptions: InternalHttpOptions,
+ ): Promise<HttpResponse<string>>;
+ private async request(
+ requestUrl: string | URL,
+ httpOptions: InternalHttpOptions & { responseType: 'text' },
+ ): Promise<HttpResponse<string>>;
+ private async request(
+ requestUrl: string | URL,
+ httpOptions: InternalHttpOptions & { responseType: 'buffer' },
+ ): Promise<HttpResponse<Buffer>>;
+ private async request<T = unknown>(
+ requestUrl: string | URL,
+ httpOptions: InternalHttpOptions & { responseType: 'json' },
+ ): Promise<HttpResponse<T>>;
- protected async request<T>(
+ private async request(
requestUrl: string | URL,
httpOptions: InternalHttpOptions,
- ): Promise<HttpResponse<T>> {
- let url = requestUrl.toString();
- if (httpOptions?.baseUrl) {
- url = resolveBaseUrl(httpOptions.baseUrl, url);
- }
+ ): Promise<HttpResponse<unknown>> {
+ const resolvedUrl = this.resolveUrl(requestUrl, httpOptions);
+ const url = resolvedUrl.toString();
- let options = merge<SetRequired<GotOptions, 'method'>, InternalHttpOptions>(
+ this.processOptions(resolvedUrl, httpOptions);
+
+ let options = merge<InternalGotOptions, InternalHttpOptions>(
{
- method: 'get',
...this.options,
hostType: this.hostType,
},
@@ -181,7 +188,9 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
options.timeout ??= 60000;
const { cacheProvider } = options;
- const cachedResponse = await cacheProvider?.bypassServer<T>(url);
+ const cachedResponse = await cacheProvider?.bypassServer<string | Buffer>(
+ url,
+ );
if (cachedResponse) {
return cachedResponse;
}
@@ -198,34 +207,29 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
)
: null;
- let resPromise: Promise<HttpResponse<T>> | null = null;
+ let resPromise: Promise<HttpResponse<unknown>> | null = null;
// Cache GET requests unless memCache=false
if (memCacheKey) {
resPromise = memCache.get(memCacheKey);
}
- // istanbul ignore else: no cache tests
if (!resPromise) {
if (cacheProvider) {
await cacheProvider.setCacheHeaders(url, options);
}
const startTime = Date.now();
- const httpTask: GotTask<T> = () => {
+ const httpTask: GotTask = () => {
const queueMs = Date.now() - startTime;
return gotTask(url, options, { queueMs });
};
const throttle = getThrottle(url);
- const throttledTask: GotTask<T> = throttle
- ? () => throttle.add<HttpResponse<T>>(httpTask)
- : httpTask;
+ const throttledTask = throttle ? () => throttle.add(httpTask) : httpTask;
const queue = getQueue(url);
- const queuedTask: GotTask<T> = queue
- ? () => queue.add<HttpResponse<T>>(throttledTask)
- : throttledTask;
+ const queuedTask = queue ? () => queue.add(throttledTask) : throttledTask;
const { maxRetryAfter = 60 } = hostRule;
resPromise = wrapWithRetry(queuedTask, url, getRetryAfter, maxRetryAfter);
@@ -252,7 +256,10 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
throw new ExternalHostError(err);
}
- const staleResponse = await cacheProvider?.bypassServer<T>(url, true);
+ const staleResponse = await cacheProvider?.bypassServer<string | Buffer>(
+ url,
+ true,
+ );
if (staleResponse) {
logger.debug(
{ err },
@@ -261,57 +268,110 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
return staleResponse;
}
- throw err;
+ this.handleError(requestUrl, httpOptions, err);
}
}
+ protected processOptions(_url: URL, _options: InternalHttpOptions): void {
+ // noop
+ }
+
+ protected handleError(
+ _url: string | URL,
+ _httpOptions: HttpOptions,
+ err: Error,
+ ): never {
+ throw err;
+ }
+
+ protected resolveUrl(
+ requestUrl: string | URL,
+ options: HttpOptions | undefined,
+ ): URL {
+ let url = requestUrl;
+
+ if (url instanceof URL) {
+ // already a aboslute URL
+ return url;
+ }
+
+ const baseUrl = options?.baseUrl ?? this.baseUrl;
+ if (baseUrl) {
+ url = resolveBaseUrl(baseUrl, url);
+ }
+
+ const parsedUrl = parseUrl(url);
+ if (!parsedUrl || !isHttpUrl(parsedUrl)) {
+ logger.error({ url: requestUrl }, 'Request Error: cannot parse url');
+ throw new Error('Invalid URL');
+ }
+ return parsedUrl;
+ }
+
protected calculateRetryDelay({ computedValue }: RetryObject): number {
return computedValue;
}
- get(url: string, options: HttpOptions = {}): Promise<HttpResponse> {
- return this.request<string>(url, options);
+ get(
+ url: string,
+ options: HttpOptions = {},
+ ): Promise<HttpResponse<string | Buffer>> {
+ return this.request(url, options);
+ }
+
+ head(url: string, options: HttpOptions = {}): Promise<HttpResponse<never>> {
+ // to complex to validate
+ return this.request(url, {
+ ...options,
+ responseType: 'text',
+ method: 'head',
+ }) as Promise<HttpResponse<never>>;
}
- head(url: string, options: HttpOptions = {}): Promise<HttpResponse> {
- return this.request<string>(url, { ...options, method: 'head' });
+ getText(
+ url: string | URL,
+ options: HttpOptions = {},
+ ): Promise<HttpResponse<string>> {
+ return this.request(url, { ...options, responseType: 'text' });
}
getBuffer(
- url: string,
+ url: string | URL,
options: HttpOptions = {},
): Promise<HttpResponse<Buffer>> {
- return this.request<Buffer>(url, {
- ...options,
- responseType: 'buffer',
- });
+ return this.request(url, { ...options, responseType: 'buffer' });
}
- private async requestJson<ResT = unknown>(
- method: InternalHttpOptions['method'],
- { url, httpOptions: requestOptions, schema }: HttpFnArgs<Opts, ResT>,
+ protected requestJsonUnsafe<ResT>(
+ method: HttpMethod,
+ { url, httpOptions: requestOptions }: InternalJsonUnsafeOptions<Opts>,
): Promise<HttpResponse<ResT>> {
- const { body, ...httpOptions } = { ...requestOptions };
+ const { body: json, ...httpOptions } = { ...requestOptions };
const opts: InternalHttpOptions = {
...httpOptions,
method,
- responseType: 'json',
};
// signal that we expect a json response
opts.headers = {
accept: 'application/json',
...opts.headers,
};
- if (body) {
- opts.json = body;
+ if (json) {
+ opts.json = json;
}
- const res = await this.request<ResT>(url, opts);
+ return this.request<ResT>(url, { ...opts, responseType: 'json' });
+ }
+
+ private async requestJson<ResT, Schema extends ZodType<ResT> = ZodType<ResT>>(
+ method: HttpMethod,
+ options: InternalJsonOptions<Opts, ResT, Schema>,
+ ): Promise<HttpResponse<ResT>> {
+ const res = await this.requestJsonUnsafe<ResT>(method, options);
- if (!schema) {
- return res;
+ if (options.schema) {
+ res.body = await options.schema.parseAsync(res.body);
}
- res.body = await schema.parseAsync(res.body);
return res;
}
@@ -319,8 +379,8 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
arg1: string,
arg2: Opts | ZodType<ResT> | undefined,
arg3: ZodType<ResT> | undefined,
- ): HttpFnArgs<Opts, ResT> {
- const res: HttpFnArgs<Opts, ResT> = { url: arg1 };
+ ): InternalJsonOptions<Opts, ResT> {
+ const res: InternalJsonOptions<Opts, ResT> = { url: arg1 };
if (arg2 instanceof ZodType) {
res.schema = arg2;
@@ -337,7 +397,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
async getPlain(url: string, options?: Opts): Promise<HttpResponse> {
const opt = options ?? {};
- return await this.get(url, {
+ return await this.getText(url, {
headers: {
Accept: 'text/plain',
},
@@ -352,7 +412,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
url: string,
options?: Opts,
): Promise<HttpResponse<ResT>> {
- const res = await this.get(url, options);
+ const res = await this.getText(url, options);
const body = parseSingleYaml<ResT>(res.body);
return { ...res, body };
}
@@ -386,7 +446,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
method: 'get',
};
- const res = await this.get(url, opts);
+ const res = await this.getText(url, opts);
const body = await schema.parseAsync(parseSingleYaml(res.body));
return { ...res, body };
}
@@ -583,17 +643,12 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
stream(url: string, options?: HttpOptions): NodeJS.ReadableStream {
// TODO: fix types (#22198)
let combinedOptions: any = {
- method: 'get',
...this.options,
hostType: this.hostType,
...options,
};
- let resolvedUrl = url;
- // istanbul ignore else: needs test
- if (options?.baseUrl) {
- resolvedUrl = resolveBaseUrl(options.baseUrl, url);
- }
+ const resolvedUrl = this.resolveUrl(url, options).toString();
applyDefaultHeaders(combinedOptions);
diff --git a/lib/util/http/jira.ts b/lib/util/http/jira.ts
index 706780b284915ea333783a94409509871972be49..adfe8aa3d0322e12db5bc87e587e1023f6796d2f 100644
--- a/lib/util/http/jira.ts
+++ b/lib/util/http/jira.ts
@@ -1,22 +1,18 @@
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type { HttpOptions } from './types';
import { Http } from '.';
let baseUrl: string;
-export const setBaseUrl = (url: string): void => {
+export function setBaseUrl(url: string): void {
baseUrl = url;
-};
+}
export class JiraHttp extends Http {
- constructor(type = 'jira', options?: HttpOptions) {
- super(type, options);
+ protected override get baseUrl(): string | undefined {
+ return baseUrl;
}
- protected override request<T>(
- url: string | URL,
- options?: InternalHttpOptions,
- ): Promise<HttpResponse<T>> {
- const opts = { baseUrl, ...options };
- return super.request<T>(url, opts);
+ constructor(type = 'jira', options?: HttpOptions) {
+ super(type, options);
}
}
diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts
index 978184cf2b16d9afd676a4ea10e666d50a4b0c9e..2b6a31b4e574a95c15e47cc48fb05229a9a3d43c 100644
--- a/lib/util/http/types.ts
+++ b/lib/util/http/types.ts
@@ -2,7 +2,7 @@ import type { IncomingHttpHeaders } from 'node:http';
import type {
OptionsOfBufferResponseBody,
OptionsOfJSONResponseBody,
- ParseJsonFunction,
+ OptionsOfTextResponseBody,
} from 'got';
import type { HttpCacheProvider } from './cache/types';
@@ -11,8 +11,9 @@ export type GotContextOptions = {
} & Record<string, unknown>;
// TODO: Move options to context
-export type GotOptions = GotBufferOptions | GotJSONOptions;
+export type GotOptions = GotBufferOptions | GotTextOptions | GotJSONOptions;
export type GotBufferOptions = OptionsOfBufferResponseBody & GotExtraOptions;
+export type GotTextOptions = OptionsOfTextResponseBody & GotExtraOptions;
export type GotJSONOptions = OptionsOfJSONResponseBody & GotExtraOptions;
export interface GotExtraOptions {
@@ -69,17 +70,12 @@ export interface HttpOptions {
readOnly?: boolean;
}
-export interface InternalHttpOptions extends HttpOptions {
- json?: HttpOptions['body'];
- responseType?: 'json' | 'buffer';
- method?: 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
- parseJson?: ParseJsonFunction;
-}
-
export interface HttpHeaders extends IncomingHttpHeaders {
link?: string | undefined;
}
+export type HttpMethod = 'get' | 'post' | 'put' | 'patch' | 'delete' | 'head';
+
export interface HttpResponse<T = string> {
statusCode: number;
body: T;
@@ -88,7 +84,7 @@ export interface HttpResponse<T = string> {
}
export type Task<T> = () => Promise<T>;
-export type GotTask<T> = Task<HttpResponse<T>>;
+export type GotTask<T = unknown> = Task<HttpResponse<T>>;
export interface ThrottleLimitRule {
matchHost: string;
diff --git a/lib/util/url.spec.ts b/lib/util/url.spec.ts
index e1ed216102e3be2602453dd433e47d1abc0535b8..77d3bdc0f158a31dc7e7ec1355e2fbbc9c3e3bee 100644
--- a/lib/util/url.spec.ts
+++ b/lib/util/url.spec.ts
@@ -106,6 +106,7 @@ describe('util/url', () => {
expect(isHttpUrl('ssh://github.com')).toBeFalse();
expect(isHttpUrl('http://github.com')).toBeTrue();
expect(isHttpUrl('https://github.com')).toBeTrue();
+ expect(isHttpUrl(new URL('https://github.com'))).toBeTrue();
});
it('parses URL', () => {
diff --git a/lib/util/url.ts b/lib/util/url.ts
index d9886939dc1dd3f67ed5a4869a9c7330e867051f..44efb7f5b365486cd38a0c261ccc6d3ada800a79 100644
--- a/lib/util/url.ts
+++ b/lib/util/url.ts
@@ -86,15 +86,11 @@ export function getQueryString(params: Record<string, any>): string {
}
export function isHttpUrl(url: unknown): boolean {
- if (!is.nonEmptyString(url)) {
- return false;
- }
- try {
- const { protocol } = new URL(url);
- return protocol === 'https:' || protocol === 'http:';
- } catch {
+ if (!is.nonEmptyString(url) && !is.urlInstance(url)) {
return false;
}
+ const protocol = parseUrl(url)?.protocol;
+ return protocol === 'https:' || protocol === 'http:';
}
export function parseUrl(url: URL | string | undefined | null): URL | null {
diff --git a/lib/workers/repository/update/pr/changelog/bitbucket/index.ts b/lib/workers/repository/update/pr/changelog/bitbucket/index.ts
index 3f0ca2cce7a7ab239d026cae047b4ffec42bf68f..e9e43bc3a03df8ee609b38b1de487075fed0234f 100644
--- a/lib/workers/repository/update/pr/changelog/bitbucket/index.ts
+++ b/lib/workers/repository/update/pr/changelog/bitbucket/index.ts
@@ -61,7 +61,7 @@ export async function getReleaseNotesMd(
);
}
- const fileRes = await bitbucketHttp.get(
+ const fileRes = await bitbucketHttp.getText(
joinUrlParts(
apiBaseUrl,
'2.0/repositories',
diff --git a/lib/workers/repository/update/pr/changelog/gitlab/index.ts b/lib/workers/repository/update/pr/changelog/gitlab/index.ts
index 4c17d4bac34fa5394281ee1cafffd00e0551f96b..8f838070c34b166a866cf223900c5ae8d3f6d65a 100644
--- a/lib/workers/repository/update/pr/changelog/gitlab/index.ts
+++ b/lib/workers/repository/update/pr/changelog/gitlab/index.ts
@@ -54,7 +54,7 @@ export async function getReleaseNotesMd(
}
// https://docs.gitlab.com/13.2/ee/api/repositories.html#raw-blob-content
- const fileRes = await http.get(`${apiPrefix}blobs/${id}/raw`);
+ const fileRes = await http.getText(`${apiPrefix}blobs/${id}/raw`);
const changelogMd = fileRes.body + '\n#\n##';
return { changelogFile, changelogMd };
}
diff --git a/test/http-mock.ts b/test/http-mock.ts
index ce695dd0aecdb1d1dc6f7d16039fa89eb196b917..d799e59289567bde943b689c776044b402ebbaea 100644
--- a/test/http-mock.ts
+++ b/test/http-mock.ts
@@ -73,7 +73,7 @@ export function clear(check = true): void {
}
if (missing.length) {
- const err = new Error(missingHttpMockMessage(done, missing));
+ const err = new Error(missingHttpMockMessage(done, missing, pending));
massageHttpMockStacktrace(err);
throw err;
}
@@ -158,6 +158,7 @@ function massageHttpMockStacktrace(err: Error): void {
function missingHttpMockMessage(
done: RequestLog[],
missing: MissingRequestLog[],
+ pending: string[],
): string {
const blocks: string[] = [];
@@ -208,6 +209,14 @@ function missingHttpMockMessage(
`);
}
+ if (pending.length) {
+ blocks.push(codeBlock`
+ Pending mocks:
+
+ ${pending.join('\n')}
+ `);
+ }
+
blocks.push(explanation);
return blocks.join('\n\n');
diff --git a/vitest.config.ts b/vitest.config.ts
index 237d98eca691bb86feb35473227ebf89e25ae340..05bacaccd0179176fdc7098c84370693d2b6059a 100644
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -109,6 +109,7 @@ export default defineConfig(() =>
'lib/modules/datasource/hex/v2/package.ts',
'lib/modules/datasource/hex/v2/signed.ts',
'lib/util/cache/package/redis.ts',
+ 'lib/util/http/http.ts', // TODO: remove when code is moved from index
'lib/util/http/legacy.ts',
'lib/workers/repository/cache.ts',
],