From 81fc75630b0b43fb4b89a0b65c1086d487e65d2e Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Wed, 9 Oct 2024 02:06:46 -0300
Subject: [PATCH] refactor(sbt-package): Flatten fetching code (#31836)
---
.../datasource/sbt-package/index.spec.ts | 2 -
lib/modules/datasource/sbt-package/index.ts | 214 +++++++++---------
2 files changed, 109 insertions(+), 107 deletions(-)
diff --git a/lib/modules/datasource/sbt-package/index.spec.ts b/lib/modules/datasource/sbt-package/index.spec.ts
index 5613cdd4c1..72237f9d2f 100644
--- a/lib/modules/datasource/sbt-package/index.spec.ts
+++ b/lib/modules/datasource/sbt-package/index.spec.ts
@@ -66,8 +66,6 @@ describe('modules/datasource/sbt-package/index', () => {
)
.get('/maven2/com/example/empty/')
.reply(200, '')
- .get('/maven2/com.example/')
- .reply(404)
.get('/maven2/com/example/empty/maven-metadata.xml')
.reply(404)
.get('/maven2/com/example/empty/index.html')
diff --git a/lib/modules/datasource/sbt-package/index.ts b/lib/modules/datasource/sbt-package/index.ts
index 763e8dbe12..2f9848a6b4 100644
--- a/lib/modules/datasource/sbt-package/index.ts
+++ b/lib/modules/datasource/sbt-package/index.ts
@@ -1,8 +1,9 @@
+import * as upath from 'upath';
import { XmlDocument } from 'xmldoc';
import { logger } from '../../../logger';
import { Http } from '../../../util/http';
import { regEx } from '../../../util/regex';
-import { ensureTrailingSlash } from '../../../util/url';
+import { ensureTrailingSlash, trimTrailingSlash } from '../../../util/url';
import * as ivyVersioning from '../../versioning/ivy';
import { compare } from '../../versioning/maven/compare';
import { MavenDatasource } from '../maven';
@@ -18,6 +19,12 @@ import type {
} from '../types';
import { extractPageLinks, getLatestVersion } from './util';
+interface ScalaDepCoordinate {
+ groupId: string;
+ artifactId: string;
+ scalaVersion?: string;
+}
+
export class SbtPackageDatasource extends MavenDatasource {
static override readonly id = 'sbt-package';
@@ -36,100 +43,135 @@ export class SbtPackageDatasource extends MavenDatasource {
this.http = new Http('sbt');
}
- async getArtifactSubdirs(
- searchRoot: string,
- artifact: string,
- scalaVersion: string,
- ): Promise<string[] | null> {
- const pkgUrl = ensureTrailingSlash(searchRoot);
- const res = await downloadHttpProtocol(this.http, pkgUrl);
- const indexContent = res?.body;
- if (indexContent) {
- const rootPath = new URL(pkgUrl).pathname;
- let artifactSubdirs = extractPageLinks(indexContent, (href) => {
+ protected static parseDepCoordinate(packageName: string): ScalaDepCoordinate {
+ const [groupId, javaArtifactId] = packageName.split(':');
+ const [artifactId, scalaVersion] = javaArtifactId.split('_');
+ return { groupId, artifactId, scalaVersion };
+ }
+
+ async getSbtReleases(
+ registryUrl: string,
+ packageName: string,
+ ): Promise<ReleaseResult | null> {
+ const { groupId, artifactId, scalaVersion } =
+ SbtPackageDatasource.parseDepCoordinate(packageName);
+
+ const groupIdSplit = groupId.split('.');
+ const repoRootUrl = ensureTrailingSlash(registryUrl);
+ const packageRootUrlWith = (sep: string): string =>
+ `${repoRootUrl}${groupIdSplit.join(sep)}`;
+ const packageRootUrls: string[] = [];
+ packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('/')));
+ packageRootUrls.push(ensureTrailingSlash(packageRootUrlWith('.')));
+
+ let dependencyUrl: string | undefined;
+ let packageUrls: string[] | undefined;
+ for (const packageRootUrl of packageRootUrls) {
+ const res = await downloadHttpProtocol(this.http, packageRootUrl);
+ if (!res) {
+ continue;
+ }
+
+ dependencyUrl = trimTrailingSlash(packageRootUrl);
+
+ const rootPath = new URL(packageRootUrl).pathname;
+ const artifactSubdirs = extractPageLinks(res.body, (href) => {
const path = href.replace(rootPath, '');
+
if (
- path.startsWith(`${artifact}_native`) ||
- path.startsWith(`${artifact}_sjs`)
+ path.startsWith(`${artifactId}_native`) ||
+ path.startsWith(`${artifactId}_sjs`)
) {
return null;
}
- if (path === artifact || path.startsWith(`${artifact}_`)) {
- return path;
+ if (path === artifactId || path.startsWith(`${artifactId}_`)) {
+ return ensureTrailingSlash(`${packageRootUrl}${path}`);
}
return null;
});
- if (
- scalaVersion &&
- artifactSubdirs.includes(`${artifact}_${scalaVersion}`)
- ) {
- artifactSubdirs = [`${artifact}_${scalaVersion}`];
+ if (scalaVersion) {
+ const scalaSubdir = artifactSubdirs.find((x) =>
+ x.endsWith(`/${artifactId}_${scalaVersion}/`),
+ );
+ if (scalaSubdir) {
+ packageUrls = [scalaSubdir];
+ break;
+ }
}
- return artifactSubdirs;
- }
- return null;
- }
+ packageUrls = artifactSubdirs;
+ break;
+ }
- async getPackageReleases(
- searchRoot: string,
- artifactSubdirs: string[] | null,
- ): Promise<string[] | null> {
- if (artifactSubdirs) {
- const releases: string[] = [];
- for (const searchSubdir of artifactSubdirs) {
- const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`);
- const res = await downloadHttpProtocol(this.http, pkgUrl);
- const content = res?.body;
- if (content) {
- const rootPath = new URL(pkgUrl).pathname;
- const subdirReleases = extractPageLinks(content, (href) => {
- const path = href.replace(rootPath, '');
- if (path.startsWith('.')) {
- return null;
- }
+ if (!packageUrls) {
+ return null;
+ }
- return path;
- });
+ const validPackageUrls: string[] = [];
+ const allVersions = new Set<string>();
+ for (const pkgUrl of packageUrls) {
+ const res = await downloadHttpProtocol(this.http, pkgUrl);
+ // istanbul ignore if
+ if (!res) {
+ continue;
+ }
+ validPackageUrls.push(pkgUrl);
- subdirReleases.forEach((x) => releases.push(x));
+ const rootPath = new URL(pkgUrl).pathname;
+ const versions = extractPageLinks(res.body, (href) => {
+ const path = href.replace(rootPath, '');
+ if (path.startsWith('.')) {
+ return null;
}
- }
- if (releases.length) {
- return [...new Set(releases)].sort(compare);
+
+ return path;
+ });
+
+ for (const version of versions) {
+ allVersions.add(version);
}
}
- return null;
+ const versions = [...allVersions];
+ if (!versions.length) {
+ return null;
+ }
+
+ const latestVersion = getLatestVersion(versions);
+ const pomInfo = await this.getPomInfo(packageUrls, latestVersion);
+
+ const releases: Release[] = [...allVersions]
+ .sort(compare)
+ .map((version) => ({ version }));
+ return { releases, dependencyUrl, ...pomInfo };
}
- async getUrls(
- searchRoot: string,
- artifactDirs: string[] | null,
+ async getPomInfo(
+ packageUrls: string[],
version: string | null,
- ): Promise<Partial<ReleaseResult>> {
- const result: Partial<ReleaseResult> = {};
+ ): Promise<Pick<ReleaseResult, 'homepage' | 'sourceUrl'>> {
+ const result: Pick<ReleaseResult, 'homepage' | 'sourceUrl'> = {};
- if (!artifactDirs?.length) {
+ // istanbul ignore if
+ if (!packageUrls?.length) {
return result;
}
+ // istanbul ignore if
if (!version) {
return result;
}
- for (const artifactDir of artifactDirs) {
+ for (const packageUrl of packageUrls) {
+ const artifactDir = upath.basename(packageUrl);
const [artifact] = artifactDir.split('_');
- const pomFileNames = [
- `${artifactDir}-${version}.pom`,
- `${artifact}-${version}.pom`,
- ];
- for (const pomFileName of pomFileNames) {
- const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`;
+ for (const pomFilePrefix of [artifactDir, artifact]) {
+ const pomFileName = `${pomFilePrefix}-${version}.pom`;
+ const pomUrl = `${packageUrl}${version}/${pomFileName}`;
const res = await downloadHttpProtocol(this.http, pomUrl);
const content = res?.body;
if (content) {
@@ -166,58 +208,20 @@ export class SbtPackageDatasource extends MavenDatasource {
return null;
}
- const [groupId, artifactId] = packageName.split(':');
- const groupIdSplit = groupId.split('.');
- const artifactIdSplit = artifactId.split('_');
- const [artifact, scalaVersion] = artifactIdSplit;
-
- const repoRoot = ensureTrailingSlash(registryUrl);
- const searchRoots: string[] = [];
- // Optimize lookup order
- searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`);
- searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`);
-
- for (let idx = 0; idx < searchRoots.length; idx += 1) {
- const searchRoot = searchRoots[idx];
- const artifactSubdirs = await this.getArtifactSubdirs(
- searchRoot,
- artifact,
- scalaVersion,
- );
- const versions = await this.getPackageReleases(
- searchRoot,
- artifactSubdirs,
- );
- const latestVersion = getLatestVersion(versions);
- const urls = await this.getUrls(
- searchRoot,
- artifactSubdirs,
- latestVersion,
- );
-
- const dependencyUrl = searchRoot;
-
- logger.trace({ dependency: packageName, versions }, `Package versions`);
- if (versions) {
- return {
- ...urls,
- dependencyUrl,
- releases: versions.map((v) => ({ version: v })),
- };
- }
+ const sbtReleases = await this.getSbtReleases(registryUrl, packageName);
+ if (sbtReleases) {
+ return sbtReleases;
}
logger.debug(
- `No versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`,
+ `Sbt: no versions discovered for ${packageName} listing organization root package folder, fallback to maven datasource for version discovery`,
);
const mavenReleaseResult = await super.getReleases(config);
if (mavenReleaseResult) {
return mavenReleaseResult;
}
- logger.debug(
- `No versions found for ${packageName} in ${searchRoots.length} repositories`,
- );
+ logger.debug(`Sbt: no versions found for "${packageName}"`);
return null;
}
--
GitLab