From 606ab430d1ac897bd4f6eada5cc80ad2a9ddbd90 Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Mon, 9 Dec 2024 15:58:07 -0300
Subject: [PATCH] refactor(maven): Unified result type for http fetch (#32813)
---
lib/modules/datasource/maven/util.spec.ts | 25 ++++-
lib/modules/datasource/maven/util.ts | 103 ++++++++++--------
.../datasource/sbt-package/index.spec.ts | 6 +-
lib/modules/datasource/sbt-package/index.ts | 26 +++--
lib/modules/datasource/sbt-plugin/index.ts | 30 +++--
5 files changed, 110 insertions(+), 80 deletions(-)
diff --git a/lib/modules/datasource/maven/util.spec.ts b/lib/modules/datasource/maven/util.spec.ts
index 0921b7de58..dd9891abc6 100644
--- a/lib/modules/datasource/maven/util.spec.ts
+++ b/lib/modules/datasource/maven/util.spec.ts
@@ -71,7 +71,10 @@ describe('modules/datasource/maven/util', () => {
get: () => Promise.reject(httpError({ message: HOST_DISABLED })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toBeNull();
+ expect(res.unwrap()).toEqual({
+ ok: false,
+ err: { type: 'host-disabled' } satisfies MavenFetchError,
+ });
});
it('returns empty for host error', async () => {
@@ -79,7 +82,10 @@ describe('modules/datasource/maven/util', () => {
get: () => Promise.reject(httpError({ code: 'ETIMEDOUT' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toBeNull();
+ expect(res.unwrap()).toEqual({
+ ok: false,
+ err: { type: 'host-error' } satisfies MavenFetchError,
+ });
});
it('returns empty for temporary error', async () => {
@@ -87,7 +93,10 @@ describe('modules/datasource/maven/util', () => {
get: () => Promise.reject(httpError({ code: 'ECONNRESET' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toBeNull();
+ expect(res.unwrap()).toEqual({
+ ok: false,
+ err: { type: 'temporary-error' } satisfies MavenFetchError,
+ });
});
it('returns empty for connection error', async () => {
@@ -95,7 +104,10 @@ describe('modules/datasource/maven/util', () => {
get: () => Promise.reject(httpError({ code: 'ECONNREFUSED' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toBeNull();
+ expect(res.unwrap()).toEqual({
+ ok: false,
+ err: { type: 'connection-error' } satisfies MavenFetchError,
+ });
});
it('returns empty for unsupported error', async () => {
@@ -104,7 +116,10 @@ describe('modules/datasource/maven/util', () => {
Promise.reject(httpError({ name: 'UnsupportedProtocolError' })),
});
const res = await downloadHttpProtocol(http, 'some://');
- expect(res).toBeNull();
+ expect(res.unwrap()).toEqual({
+ ok: false,
+ err: { type: 'unsupported-host' } satisfies MavenFetchError,
+ });
});
});
diff --git a/lib/modules/datasource/maven/util.ts b/lib/modules/datasource/maven/util.ts
index 3a6d85956b..1edc357068 100644
--- a/lib/modules/datasource/maven/util.ts
+++ b/lib/modules/datasource/maven/util.ts
@@ -71,73 +71,92 @@ export async function downloadHttpProtocol(
http: Http,
pkgUrl: URL | string,
opts: HttpOptions = {},
-): Promise<HttpResponse | null> {
+): Promise<MavenFetchResult> {
const url = pkgUrl.toString();
- const res = await Result.wrap(http.get(url, opts))
- .onError((err) => {
+ const fetchResult = await Result.wrap<HttpResponse, Error>(
+ http.get(url, opts),
+ )
+ .transform((res): MavenFetchSuccess => {
+ const result: MavenFetchSuccess = { data: res.body };
+
+ if (!res.authorization) {
+ result.isCacheable = true;
+ }
+
+ const lastModified = normalizeDate(res?.headers?.['last-modified']);
+ if (lastModified) {
+ result.lastModified = lastModified;
+ }
+
+ return result;
+ })
+ .catch((err): MavenFetchResult => {
// istanbul ignore next: never happens, needs for type narrowing
if (!(err instanceof HttpError)) {
- return;
+ return Result.err({ type: 'unknown', err });
}
const failedUrl = url;
if (err.message === HOST_DISABLED) {
logger.trace({ failedUrl }, 'Host disabled');
- return;
+ return Result.err({ type: 'host-disabled' });
}
if (isNotFoundError(err)) {
logger.trace({ failedUrl }, `Url not found`);
- return;
+ return Result.err({ type: 'not-found' });
}
if (isHostError(err)) {
logger.debug(`Cannot connect to host ${failedUrl}`);
- return;
+ return Result.err({ type: 'host-error' });
}
if (isPermissionsIssue(err)) {
logger.debug(
`Dependency lookup unauthorized. Please add authentication with a hostRule for ${failedUrl}`,
);
- return;
+ return Result.err({ type: 'permission-issue' });
}
if (isTemporaryError(err)) {
logger.debug({ failedUrl, err }, 'Temporary error');
- return;
+ if (getHost(url) === getHost(MAVEN_REPO)) {
+ return Result.err({ type: 'maven-central-temporary-error', err });
+ } else {
+ return Result.err({ type: 'temporary-error' });
+ }
}
if (isConnectionError(err)) {
logger.debug(`Connection refused to maven registry ${failedUrl}`);
- return;
+ return Result.err({ type: 'connection-error' });
}
if (isUnsupportedHostError(err)) {
logger.debug(`Unsupported host ${failedUrl}`);
- return;
+ return Result.err({ type: 'unsupported-host' });
}
logger.info({ failedUrl, err }, 'Unknown HTTP download error');
- })
- .catch((err): Result<HttpResponse | 'silent-error', ExternalHostError> => {
- if (
- err instanceof HttpError &&
- isTemporaryError(err) &&
- getHost(url) === getHost(MAVEN_REPO)
- ) {
- return Result.err(new ExternalHostError(err));
- }
-
- return Result.ok('silent-error');
- })
- .unwrapOrThrow();
+ return Result.err({ type: 'unknown', err });
+ });
- if (res === 'silent-error') {
- return null;
+ const { err } = fetchResult.unwrap();
+ if (err?.type === 'maven-central-temporary-error') {
+ throw new ExternalHostError(err.err);
}
- return res;
+ return fetchResult;
+}
+
+export async function downloadHttpContent(
+ http: Http,
+ pkgUrl: URL | string,
+ opts: HttpOptions = {},
+): Promise<string | null> {
+ const fetchResult = await downloadHttpProtocol(http, pkgUrl, opts);
+ return fetchResult.transform(({ data }) => data).unwrapOrNull();
}
function isS3NotFound(err: Error): boolean {
@@ -228,7 +247,7 @@ export async function downloadS3Protocol(
export async function downloadArtifactRegistryProtocol(
http: Http,
pkgUrl: URL,
-): Promise<HttpResponse | null> {
+): Promise<MavenFetchResult> {
const opts: HttpOptions = {};
const host = pkgUrl.host;
const path = pkgUrl.pathname;
@@ -357,25 +376,21 @@ export async function downloadMavenXml(
const protocol = pkgUrl.protocol;
if (protocol === 'http:' || protocol === 'https:') {
- const res = await downloadHttpProtocol(http, pkgUrl);
- const body = res?.body;
- if (body) {
- return {
- xml: new XmlDocument(body),
- isCacheable: !res.authorization,
- };
- }
+ const rawResult = await downloadHttpProtocol(http, pkgUrl);
+ const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
+ const xml = new XmlDocument(data);
+ return { isCacheable, xml };
+ });
+ return xmlResult.unwrapOr({});
}
if (protocol === 'artifactregistry:') {
- const res = await downloadArtifactRegistryProtocol(http, pkgUrl);
- const body = res?.body;
- if (body) {
- return {
- xml: new XmlDocument(body),
- isCacheable: !res.authorization,
- };
- }
+ const rawResult = await downloadArtifactRegistryProtocol(http, pkgUrl);
+ const xmlResult = rawResult.transform(({ isCacheable, data }): MavenXml => {
+ const xml = new XmlDocument(data);
+ return { isCacheable, xml };
+ });
+ return xmlResult.unwrapOr({});
}
if (protocol === 's3:') {
diff --git a/lib/modules/datasource/sbt-package/index.spec.ts b/lib/modules/datasource/sbt-package/index.spec.ts
index aa86cc3d12..c3011e5631 100644
--- a/lib/modules/datasource/sbt-package/index.spec.ts
+++ b/lib/modules/datasource/sbt-package/index.spec.ts
@@ -149,9 +149,9 @@ describe('modules/datasource/sbt-package/index', () => {
.get('/org/example/example_2.12/')
.reply(200, `<a href='1.2.3/'>1.2.3/</a>`)
.get('/org/example/example_2.12/1.2.3/example-1.2.3.pom')
- .reply(200, ``)
+ .reply(404)
.get('/org/example/example_2.12/1.2.3/example_2.12-1.2.3.pom')
- .reply(200, ``);
+ .reply(404);
const res = await getPkgReleases({
versioning: mavenVersioning.id,
@@ -267,7 +267,7 @@ describe('modules/datasource/sbt-package/index', () => {
`,
)
.get('/org/example/example_2.13/1.2.3/example_2.13-1.2.3.pom')
- .reply(200);
+ .reply(404);
const res = await getPkgReleases({
versioning: mavenVersioning.id,
diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts
index 463661b51c..c956922097 100644
--- a/lib/modules/datasource/sbt-package/index.ts
+++ b/lib/modules/datasource/sbt-package/index.ts
@@ -10,8 +10,7 @@ import * as ivyVersioning from '../../versioning/ivy';
import { compare } from '../../versioning/maven/compare';
import { MavenDatasource } from '../maven';
import { MAVEN_REPO } from '../maven/common';
-import { downloadHttpProtocol } from '../maven/util';
-import { normalizeDate } from '../metadata';
+import { downloadHttpContent, downloadHttpProtocol } from '../maven/util';
import type {
GetReleasesConfig,
PostprocessReleaseConfig,
@@ -88,8 +87,11 @@ export class SbtPackageDatasource extends MavenDatasource {
let dependencyUrl: string | undefined;
let packageUrls: string[] | undefined;
for (const packageRootUrl of packageRootUrls) {
- const res = await downloadHttpProtocol(this.http, packageRootUrl);
- if (!res) {
+ const packageRootContent = await downloadHttpContent(
+ this.http,
+ packageRootUrl,
+ );
+ if (!packageRootContent) {
continue;
}
@@ -103,7 +105,7 @@ export class SbtPackageDatasource extends MavenDatasource {
dependencyUrl = trimTrailingSlash(packageRootUrl);
const rootPath = new URL(packageRootUrl).pathname;
- const artifactSubdirs = extractPageLinks(res.body, (href) => {
+ const artifactSubdirs = extractPageLinks(packageRootContent, (href) => {
const path = href.replace(rootPath, '');
if (
@@ -149,15 +151,15 @@ export class SbtPackageDatasource extends MavenDatasource {
const allVersions = new Set<string>();
for (const pkgUrl of packageUrls) {
- const res = await downloadHttpProtocol(this.http, pkgUrl);
+ const packageContent = await downloadHttpContent(this.http, pkgUrl);
// istanbul ignore if
- if (!res) {
+ if (!packageContent) {
invalidPackageUrls.add(pkgUrl);
continue;
}
const rootPath = new URL(pkgUrl).pathname;
- const versions = extractPageLinks(res.body, (href) => {
+ const versions = extractPageLinks(packageContent, (href) => {
const path = href.replace(rootPath, '');
if (path.startsWith('.')) {
return null;
@@ -275,20 +277,20 @@ export class SbtPackageDatasource extends MavenDatasource {
}
const res = await downloadHttpProtocol(this.http, pomUrl);
- const content = res?.body;
- if (!content) {
+ const { val } = res.unwrap();
+ if (!val) {
invalidPomFiles.add(pomUrl);
continue;
}
const result: PomInfo = {};
- const releaseTimestamp = normalizeDate(res.headers['last-modified']);
+ const releaseTimestamp = val.lastModified;
if (releaseTimestamp) {
result.releaseTimestamp = releaseTimestamp;
}
- const pomXml = new XmlDocument(content);
+ const pomXml = new XmlDocument(val.data);
const homepage = pomXml.valueWithPath('url');
if (homepage) {
diff --git a/lib/modules/datasource/sbt-plugin/index.ts b/lib/modules/datasource/sbt-plugin/index.ts
index fc84d65642..e1e49ec371 100644
--- a/lib/modules/datasource/sbt-plugin/index.ts
+++ b/lib/modules/datasource/sbt-plugin/index.ts
@@ -7,7 +7,7 @@ import * as ivyVersioning from '../../versioning/ivy';
import { compare } from '../../versioning/maven/compare';
import { Datasource } from '../datasource';
import { MAVEN_REPO } from '../maven/common';
-import { downloadHttpProtocol } from '../maven/util';
+import { downloadHttpContent } from '../maven/util';
import { extractPageLinks, getLatestVersion } from '../sbt-package/util';
import type {
GetReleasesConfig,
@@ -43,8 +43,7 @@ export class SbtPluginDatasource extends Datasource {
scalaVersion: string,
): Promise<string[] | null> {
const pkgUrl = ensureTrailingSlash(searchRoot);
- const res = await downloadHttpProtocol(this.http, pkgUrl);
- const indexContent = res?.body;
+ const indexContent = await downloadHttpContent(this.http, pkgUrl);
if (indexContent) {
const rootPath = new URL(pkgUrl).pathname;
let artifactSubdirs = extractPageLinks(indexContent, (href) => {
@@ -84,8 +83,7 @@ export class SbtPluginDatasource extends Datasource {
const releases: string[] = [];
for (const searchSubdir of artifactSubdirs) {
const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`);
- const res = await downloadHttpProtocol(this.http, pkgUrl);
- const content = res?.body;
+ const content = await downloadHttpContent(this.http, pkgUrl);
if (content) {
const rootPath = new URL(pkgUrl).pathname;
const subdirReleases = extractPageLinks(content, (href) => {
@@ -133,8 +131,7 @@ export class SbtPluginDatasource extends Datasource {
for (const pomFileName of pomFileNames) {
const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`;
- const res = await downloadHttpProtocol(this.http, pomUrl);
- const content = res?.body;
+ const content = await downloadHttpContent(this.http, pomUrl);
if (content) {
const pomXml = new XmlDocument(content);
@@ -173,13 +170,16 @@ export class SbtPluginDatasource extends Datasource {
return href;
};
- const res = await downloadHttpProtocol(
+ const searchRootContent = await downloadHttpContent(
this.http,
ensureTrailingSlash(searchRoot),
);
- if (res) {
+ if (searchRootContent) {
const releases: string[] = [];
- const scalaVersionItems = extractPageLinks(res.body, hrefFilterMap);
+ const scalaVersionItems = extractPageLinks(
+ searchRootContent,
+ hrefFilterMap,
+ );
const scalaVersions = scalaVersionItems.map((x) =>
x.replace(regEx(/^scala_/), ''),
);
@@ -188,24 +188,22 @@ export class SbtPluginDatasource extends Datasource {
: scalaVersions;
for (const searchVersion of searchVersions) {
const searchSubRoot = `${searchRoot}/scala_${searchVersion}`;
- const subRootRes = await downloadHttpProtocol(
+ const subRootContent = await downloadHttpContent(
this.http,
ensureTrailingSlash(searchSubRoot),
);
- if (subRootRes) {
- const { body: subRootContent } = subRootRes;
+ if (subRootContent) {
const sbtVersionItems = extractPageLinks(
subRootContent,
hrefFilterMap,
);
for (const sbtItem of sbtVersionItems) {
const releasesRoot = `${searchSubRoot}/${sbtItem}`;
- const releaseIndexRes = await downloadHttpProtocol(
+ const releasesIndexContent = await downloadHttpContent(
this.http,
ensureTrailingSlash(releasesRoot),
);
- if (releaseIndexRes) {
- const { body: releasesIndexContent } = releaseIndexRes;
+ if (releasesIndexContent) {
const releasesParsed = extractPageLinks(
releasesIndexContent,
hrefFilterMap,
--
GitLab