diff --git a/lib/modules/datasource/maven/index.ts b/lib/modules/datasource/maven/index.ts
index 62da4ee93750d84a470686763411dfeb93fdc8cd..3738414d501237e4d4d46ad45b8a1d8d95b0f563 100644
--- a/lib/modules/datasource/maven/index.ts
+++ b/lib/modules/datasource/maven/index.ts
@@ -145,19 +145,24 @@ export class MavenDatasource extends Datasource {
try {
const indexUrl = getMavenUrl(dependency, repoUrl, 'index.html');
const res = await downloadHttpProtocol(this.http, indexUrl);
- const { body = '' } = res;
- for (const line of body.split(newlineRegex)) {
- const match = line.trim().match(mavenCentralHtmlVersionRegex);
- if (match) {
- const { version, releaseTimestamp: timestamp } =
- match?.groups ?? /* istanbul ignore next: hard to test */ {};
- if (version && timestamp) {
- const date = DateTime.fromFormat(timestamp, 'yyyy-MM-dd HH:mm', {
- zone: 'UTC',
- });
- if (date.isValid) {
- const releaseTimestamp = date.toISO();
- workingReleaseMap[version] = { version, releaseTimestamp };
+ if (res) {
+ for (const line of res.body.split(newlineRegex)) {
+ const match = line.trim().match(mavenCentralHtmlVersionRegex);
+ if (match) {
+ const { version, releaseTimestamp: timestamp } =
+ match?.groups ?? /* istanbul ignore next: hard to test */ {};
+ if (version && timestamp) {
+ const date = DateTime.fromFormat(
+ timestamp,
+ 'yyyy-MM-dd HH:mm',
+ {
+ zone: 'UTC',
+ },
+ );
+ if (date.isValid) {
+ const releaseTimestamp = date.toISO();
+ workingReleaseMap[version] = { version, releaseTimestamp };
+ }
}
}
}
diff --git a/lib/modules/datasource/maven/util.spec.ts b/lib/modules/datasource/maven/util.spec.ts
index 0263c203762cc0be73e35347563f4357156d3483..fb94fa28ea7caea0cecf2c6d42919c91b72f45ef 100644
--- a/lib/modules/datasource/maven/util.spec.ts
+++ b/lib/modules/datasource/maven/util.spec.ts
@@ -1,6 +1,7 @@
+import type Request from 'got/dist/source/core';
import { partial } from '../../../../test/util';
import { HOST_DISABLED } from '../../../constants/error-messages';
-import type { Http } from '../../../util/http';
+import { type Http, HttpError } from '../../../util/http';
import { parseUrl } from '../../../util/url';
import {
checkResource,
@@ -9,6 +10,38 @@ import {
downloadS3Protocol,
} from './util';
+function httpError({
+ name,
+ message = 'unknown error',
+ code,
+ request = {},
+ response,
+}: {
+ name?: string;
+ message?: string;
+ code?: HttpError['code'];
+ request?: Partial<Request>;
+ response?: Partial<Response>;
+}): HttpError {
+ type Writeable<T> = { -readonly [P in keyof T]: T[P] };
+
+ const err = new HttpError(
+ message,
+ { code },
+ request as never,
+ ) as Writeable<HttpError>;
+
+ if (name) {
+ err.name = name;
+ }
+
+ if (response) {
+ err.response = response as never;
+ }
+
+ return err;
+}
+
describe('modules/datasource/maven/util', () => {
describe('downloadMavenXml', () => {
it('returns empty object for unsupported protocols', async () => {
@@ -39,51 +72,43 @@ describe('modules/datasource/maven/util', () => {
describe('downloadHttpProtocol', () => {
it('returns empty for HOST_DISABLED error', async () => {
const http = partial<Http>({
- get: () =>
- Promise.reject(
- Object.assign(new Error(), { message: HOST_DISABLED }),
- ),
+ get: () => Promise.reject(httpError({ message: HOST_DISABLED })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toStrictEqual({});
+ expect(res).toBeNull();
});
it('returns empty for host error', async () => {
const http = partial<Http>({
- get: () =>
- Promise.reject(Object.assign(new Error(), { code: 'ETIMEDOUT' })),
+ get: () => Promise.reject(httpError({ code: 'ETIMEDOUT' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toStrictEqual({});
+ expect(res).toBeNull();
});
- it('returns empty for temporal error', async () => {
+ it('returns empty for temporary error', async () => {
const http = partial<Http>({
- get: () =>
- Promise.reject(Object.assign(new Error(), { code: 'ECONNRESET' })),
+ get: () => Promise.reject(httpError({ code: 'ECONNRESET' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toStrictEqual({});
+ expect(res).toBeNull();
});
it('returns empty for connection error', async () => {
const http = partial<Http>({
- get: () =>
- Promise.reject(Object.assign(new Error(), { code: 'ECONNREFUSED' })),
+ get: () => Promise.reject(httpError({ code: 'ECONNREFUSED' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toStrictEqual({});
+ expect(res).toBeNull();
});
it('returns empty for unsupported error', async () => {
const http = partial<Http>({
get: () =>
- Promise.reject(
- Object.assign(new Error(), { name: 'UnsupportedProtocolError' }),
- ),
+ Promise.reject(httpError({ name: 'UnsupportedProtocolError' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toStrictEqual({});
+ expect(res).toBeNull();
});
});
diff --git a/lib/modules/datasource/maven/util.ts b/lib/modules/datasource/maven/util.ts
index 1ddc2eaf4abe36a5a22ab8722737e9b6202e5d87..b521b99d53ab1dcdd1ff8e826f56736cc0af4b33 100644
--- a/lib/modules/datasource/maven/util.ts
+++ b/lib/modules/datasource/maven/util.ts
@@ -5,9 +5,10 @@ import { XmlDocument } from 'xmldoc';
import { HOST_DISABLED } from '../../../constants/error-messages';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
-import type { Http } from '../../../util/http';
+import { type Http, HttpError } from '../../../util/http';
import type { HttpOptions, HttpResponse } from '../../../util/http/types';
import { regEx } from '../../../util/regex';
+import { Result } from '../../../util/result';
import type { S3UrlParts } from '../../../util/s3';
import { getS3Client, parseS3Url } from '../../../util/s3';
import { streamToString } from '../../../util/streams';
@@ -26,32 +27,33 @@ function getHost(url: string): string | null {
return parseUrl(url)?.host ?? /* istanbul ignore next: not possible */ null;
}
-function isMavenCentral(pkgUrl: URL | string): boolean {
- const host = typeof pkgUrl === 'string' ? pkgUrl : pkgUrl.host;
- return getHost(MAVEN_REPO) === host;
-}
+function isTemporaryError(err: HttpError): boolean {
+ if (err.code === 'ECONNRESET') {
+ return true;
+ }
-function isTemporalError(err: { code: string; statusCode: number }): boolean {
- return (
- err.code === 'ECONNRESET' ||
- err.statusCode === 429 ||
- (err.statusCode >= 500 && err.statusCode < 600)
- );
+ if (err.response) {
+ const status = err.response.statusCode;
+ return status === 429 || (status >= 500 && status < 600);
+ }
+
+ return false;
}
-function isHostError(err: { code: string }): boolean {
+function isHostError(err: HttpError): boolean {
return err.code === 'ETIMEDOUT';
}
-function isNotFoundError(err: { code: string; statusCode: number }): boolean {
- return err.code === 'ENOTFOUND' || err.statusCode === 404;
+function isNotFoundError(err: HttpError): boolean {
+ return err.code === 'ENOTFOUND' || err.response?.statusCode === 404;
}
-function isPermissionsIssue(err: { statusCode: number }): boolean {
- return err.statusCode === 401 || err.statusCode === 403;
+function isPermissionsIssue(err: HttpError): boolean {
+ const status = err.response?.statusCode;
+ return status === 401 || status === 403;
}
-function isConnectionError(err: { code: string }): boolean {
+function isConnectionError(err: HttpError): boolean {
return (
err.code === 'EAI_AGAIN' ||
err.code === 'ERR_TLS_CERT_ALTNAME_INVALID' ||
@@ -59,7 +61,7 @@ function isConnectionError(err: { code: string }): boolean {
);
}
-function isUnsupportedHostError(err: { name: string }): boolean {
+function isUnsupportedHostError(err: HttpError): boolean {
return err.name === 'UnsupportedProtocolError';
}
@@ -67,37 +69,73 @@ export async function downloadHttpProtocol(
http: Http,
pkgUrl: URL | string,
opts: HttpOptions = {},
-): Promise<Partial<HttpResponse>> {
- let raw: HttpResponse;
- try {
- raw = await http.get(pkgUrl.toString(), opts);
- return raw;
- } catch (err) {
- const failedUrl = pkgUrl.toString();
- if (err.message === HOST_DISABLED) {
- logger.trace({ failedUrl }, 'Host disabled');
- } else if (isNotFoundError(err)) {
- logger.trace({ failedUrl }, `Url not found`);
- } else if (isHostError(err)) {
- logger.debug(`Cannot connect to host ${failedUrl}`);
- } else if (isPermissionsIssue(err)) {
- logger.debug(
- `Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`,
- );
- } else if (isTemporalError(err)) {
- logger.debug({ failedUrl, err }, 'Temporary error');
- if (isMavenCentral(pkgUrl)) {
- throw new ExternalHostError(err);
+): Promise<HttpResponse | null> {
+ const url = pkgUrl.toString();
+ const res = await Result.wrap(http.get(url, opts))
+ .onError((err) => {
+ // istanbul ignore next: never happens, needs for type narrowing
+ if (!(err instanceof HttpError)) {
+ return;
}
- } else if (isConnectionError(err)) {
- logger.debug(`Connection refused to maven registry ${failedUrl}`);
- } else if (isUnsupportedHostError(err)) {
- logger.debug(`Unsupported host ${failedUrl} `);
- } else {
+
+ const failedUrl = url;
+ if (err.message === HOST_DISABLED) {
+ logger.trace({ failedUrl }, 'Host disabled');
+ return;
+ }
+
+ if (isNotFoundError(err)) {
+ logger.trace({ failedUrl }, `Url not found`);
+ return;
+ }
+
+ if (isHostError(err)) {
+ logger.debug(`Cannot connect to host ${failedUrl}`);
+ return;
+ }
+
+ if (isPermissionsIssue(err)) {
+ logger.debug(
+ `Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`,
+ );
+ return;
+ }
+
+ if (isTemporaryError(err)) {
+ logger.debug({ failedUrl, err }, 'Temporary error');
+ return;
+ }
+
+ if (isConnectionError(err)) {
+ logger.debug(`Connection refused to maven registry ${failedUrl}`);
+ return;
+ }
+
+ if (isUnsupportedHostError(err)) {
+ logger.debug(`Unsupported host ${failedUrl}`);
+ return;
+ }
+
logger.info({ failedUrl, err }, 'Unknown HTTP download error');
- }
- return {};
+ })
+ .catch((err): Result<HttpResponse | 'silent-error', ExternalHostError> => {
+ if (
+ err instanceof HttpError &&
+ isTemporaryError(err) &&
+ getHost(url) === getHost(MAVEN_REPO)
+ ) {
+ return Result.err(new ExternalHostError(err));
+ }
+
+ return Result.ok('silent-error');
+ })
+ .unwrapOrThrow();
+
+ if (res === 'silent-error') {
+ return null;
}
+
+ return res;
}
function isS3NotFound(err: Error): boolean {
@@ -145,7 +183,7 @@ export async function downloadS3Protocol(pkgUrl: URL): Promise<string | null> {
export async function downloadArtifactRegistryProtocol(
http: Http,
pkgUrl: URL,
-): Promise<Partial<HttpResponse>> {
+): Promise<HttpResponse | null> {
const opts: HttpOptions = {};
const host = pkgUrl.host;
const path = pkgUrl.pathname;
@@ -275,48 +313,42 @@ export async function downloadMavenXml(
return {};
}
- let isCacheable = false;
-
- let rawContent: string | undefined;
- let authorization: boolean | undefined;
- let statusCode: number | undefined;
- switch (pkgUrl.protocol) {
- case 'http:':
- case 'https:':
- ({
- authorization,
- body: rawContent,
- statusCode,
- } = await downloadHttpProtocol(http, pkgUrl));
- break;
- case 's3:':
- rawContent = (await downloadS3Protocol(pkgUrl)) ?? undefined;
- break;
- case 'artifactregistry:':
- ({
- authorization,
- body: rawContent,
- statusCode,
- } = await downloadArtifactRegistryProtocol(http, pkgUrl));
- break;
- default:
- logger.debug(`Unsupported Maven protocol url:${pkgUrl.toString()}`);
- return {};
+ const protocol = pkgUrl.protocol;
+
+ if (protocol === 'http:' || protocol === 'https:') {
+ const res = await downloadHttpProtocol(http, pkgUrl);
+ const body = res?.body;
+ if (body) {
+ return {
+ xml: new XmlDocument(body),
+ isCacheable: !res.authorization,
+ };
+ }
}
- if (!rawContent) {
- logger.debug(
- { url: pkgUrl.toString(), statusCode },
- `Content is not found for Maven url`,
- );
- return {};
+ if (protocol === 'artifactregistry:') {
+ const res = await downloadArtifactRegistryProtocol(http, pkgUrl);
+ const body = res?.body;
+ if (body) {
+ return {
+ xml: new XmlDocument(body),
+ isCacheable: !res.authorization,
+ };
+ }
}
- if (!authorization) {
- isCacheable = true;
+ if (protocol === 's3:') {
+ const res = await downloadS3Protocol(pkgUrl);
+ if (res) {
+ return { xml: new XmlDocument(res) };
+ }
}
- return { isCacheable, xml: new XmlDocument(rawContent) };
+ logger.debug(
+ { url: pkgUrl.toString() },
+ `Content is not found for Maven url`,
+ );
+ return {};
}
export function getDependencyParts(packageName: string): MavenDependency {
diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts
index afbb629cc5df840c71746b65e72030d3cc95ac3a..c0d5403b63055fdcf591debe30a92f31321b5d91 100644
--- a/lib/modules/datasource/sbt-package/index.ts
+++ b/lib/modules/datasource/sbt-package/index.ts
@@ -39,10 +39,8 @@ export class SbtPackageDatasource extends MavenDatasource {
scalaVersion: string,
): Promise<string[] | null> {
const pkgUrl = ensureTrailingSlash(searchRoot);
- const { body: indexContent } = await downloadHttpProtocol(
- this.http,
- pkgUrl,
- );
+ const res = await downloadHttpProtocol(this.http, pkgUrl);
+ const indexContent = res?.body;
if (indexContent) {
const rootPath = new URL(pkgUrl).pathname;
let artifactSubdirs = extractPageLinks(indexContent, (href) => {
@@ -81,7 +79,8 @@ export class SbtPackageDatasource extends MavenDatasource {
const releases: string[] = [];
for (const searchSubdir of artifactSubdirs) {
const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`);
- const { body: content } = await downloadHttpProtocol(this.http, pkgUrl);
+ const res = await downloadHttpProtocol(this.http, pkgUrl);
+ const content = res?.body;
if (content) {
const rootPath = new URL(pkgUrl).pathname;
const subdirReleases = extractPageLinks(content, (href) => {
@@ -128,8 +127,8 @@ export class SbtPackageDatasource extends MavenDatasource {
for (const pomFileName of pomFileNames) {
const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`;
- const { body: content } = await downloadHttpProtocol(this.http, pomUrl);
-
+ const res = await downloadHttpProtocol(this.http, pomUrl);
+ const content = res?.body;
if (content) {
const pomXml = new XmlDocument(content);
diff --git a/lib/modules/datasource/sbt-plugin/index.ts b/lib/modules/datasource/sbt-plugin/index.ts
index e316d1e22031c34292eefa0c82d946b1a3acd7f6..e053811936ec6858f6a7f2581392f1b77fd8c323 100644
--- a/lib/modules/datasource/sbt-plugin/index.ts
+++ b/lib/modules/datasource/sbt-plugin/index.ts
@@ -50,13 +50,13 @@ export class SbtPluginDatasource extends SbtPackageDatasource {
return href;
};
- const { body: indexContent } = await downloadHttpProtocol(
+ const res = await downloadHttpProtocol(
this.http,
ensureTrailingSlash(searchRoot),
);
- if (indexContent) {
+ if (res) {
const releases: string[] = [];
- const scalaVersionItems = extractPageLinks(indexContent, hrefFilterMap);
+ const scalaVersionItems = extractPageLinks(res.body, hrefFilterMap);
const scalaVersions = scalaVersionItems.map((x) =>
x.replace(regEx(/^scala_/), ''),
);
@@ -65,22 +65,24 @@ export class SbtPluginDatasource extends SbtPackageDatasource {
: scalaVersions;
for (const searchVersion of searchVersions) {
const searchSubRoot = `${searchRoot}/scala_${searchVersion}`;
- const { body: subRootContent } = await downloadHttpProtocol(
+ const subRootRes = await downloadHttpProtocol(
this.http,
ensureTrailingSlash(searchSubRoot),
);
- if (subRootContent) {
+ if (subRootRes) {
+ const { body: subRootContent } = subRootRes;
const sbtVersionItems = extractPageLinks(
subRootContent,
hrefFilterMap,
);
for (const sbtItem of sbtVersionItems) {
const releasesRoot = `${searchSubRoot}/${sbtItem}`;
- const { body: releasesIndexContent } = await downloadHttpProtocol(
+ const releaseIndexRes = await downloadHttpProtocol(
this.http,
ensureTrailingSlash(releasesRoot),
);
- if (releasesIndexContent) {
+ if (releaseIndexRes) {
+ const { body: releasesIndexContent } = releaseIndexRes;
const releasesParsed = extractPageLinks(
releasesIndexContent,
hrefFilterMap,