diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index 251e2912dc58bb5a51f3fcc5d2c5e524b6bc8b5d..381b5c80d45b305e80cba44816b586d9f929a2fc 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -23,7 +23,7 @@ import { GradleVersionDatasource } from './gradle-version'; import { HelmDatasource } from './helm'; import { HexDatasource } from './hex'; import { JenkinsPluginsDatasource } from './jenkins-plugins'; -import * as maven from './maven'; +import { MavenDatasource } from './maven'; import { NodeDatasource } from './node'; import * as npm from './npm'; import { NugetDatasource } from './nuget'; @@ -34,8 +34,8 @@ import { PypiDatasource } from './pypi'; import { RepologyDatasource } from './repology'; import { RubyVersionDatasource } from './ruby-version'; import { RubyGemsDatasource } from './rubygems'; -import * as sbtPackage from './sbt-package'; -import * as sbtPlugin from './sbt-plugin'; +import { SbtPackageDatasource } from './sbt-package'; +import { SbtPluginDatasource } from './sbt-plugin'; import { TerraformModuleDatasource } from './terraform-module'; import { TerraformProviderDatasource } from './terraform-provider'; import type { DatasourceApi } from './types'; @@ -68,7 +68,7 @@ api.set(GradleVersionDatasource.id, new GradleVersionDatasource()); api.set(HelmDatasource.id, new HelmDatasource()); api.set(HexDatasource.id, new HexDatasource()); api.set(JenkinsPluginsDatasource.id, new JenkinsPluginsDatasource()); -api.set('maven', maven); +api.set(MavenDatasource.id, new MavenDatasource()); api.set(NodeDatasource.id, new NodeDatasource()); api.set('npm', npm); api.set(NugetDatasource.id, new NugetDatasource()); @@ -79,7 +79,7 @@ api.set(PypiDatasource.id, new PypiDatasource()); api.set(RepologyDatasource.id, new RepologyDatasource()); api.set(RubyVersionDatasource.id, new RubyVersionDatasource()); api.set(RubyGemsDatasource.id, new RubyGemsDatasource()); -api.set('sbt-package', sbtPackage); -api.set('sbt-plugin', sbtPlugin); +api.set(SbtPackageDatasource.id, new SbtPackageDatasource()); +api.set(SbtPluginDatasource.id, new SbtPluginDatasource()); api.set(TerraformModuleDatasource.id, new TerraformModuleDatasource()); api.set(TerraformProviderDatasource.id, new TerraformProviderDatasource()); diff --git a/lib/datasource/clojure/__snapshots__/index.spec.ts.snap b/lib/datasource/clojure/__snapshots__/index.spec.ts.snap index 8be1ab42749a8dbeceba417c8ca37c1ef7091be3..a396d9bd382e2bfc81369e657769642067fbb83a 100644 --- a/lib/datasource/clojure/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/clojure/__snapshots__/index.spec.ts.snap @@ -122,6 +122,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -131,6 +132,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -140,6 +142,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -747,6 +750,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -756,6 +760,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -765,6 +770,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -774,6 +780,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -783,6 +790,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -792,6 +800,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -801,6 +810,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -810,6 +820,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -819,6 +830,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -828,6 +840,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -837,6 +850,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -846,6 +860,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, @@ -855,6 +870,7 @@ Array [ Object { "headers": Object { "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer 123test", "host": "custom.registry.renovatebot.com", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, diff --git a/lib/datasource/clojure/index.ts b/lib/datasource/clojure/index.ts index 513909ab9a8598a56ece92939e3548c5e45ce085..10a1426b4ea2365cab5f40fa5c6b59dfea190986 100644 --- a/lib/datasource/clojure/index.ts +++ b/lib/datasource/clojure/index.ts @@ -1,10 +1,8 @@ -import { Datasource } from '../datasource'; -import { getReleases } from '../maven'; +import { MavenDatasource } from '../maven'; import { MAVEN_REPO } from '../maven/common'; -import type { GetReleasesConfig, ReleaseResult } from '../types'; -export class ClojureDatasource extends Datasource { - static readonly id = 'clojure'; +export class ClojureDatasource extends MavenDatasource { + static override readonly id = 'clojure'; constructor() { super(ClojureDatasource.id); @@ -16,11 +14,4 @@ export class ClojureDatasource extends Datasource { 'https://clojars.org/repo', MAVEN_REPO, ]; - - getReleases({ - lookupName, - registryUrl, - }: GetReleasesConfig): Promise<ReleaseResult | null> { - return getReleases({ lookupName, registryUrl }); - } } diff --git a/lib/datasource/maven/index.spec.ts b/lib/datasource/maven/index.spec.ts index f57fbd52a6b2464eb1dadbbf76a5471cc4b656d5..cc0ce64362cc88ce6aac1330815dee97960449ab 100644 --- a/lib/datasource/maven/index.spec.ts +++ b/lib/datasource/maven/index.spec.ts @@ -4,7 +4,9 @@ import { loadFixture } from '../../../test/util'; import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import * as hostRules from '../../util/host-rules'; import { id as versioning } from '../../versioning/maven'; -import { id as datasource } from '.'; +import { MavenDatasource } from '.'; + +const datasource = MavenDatasource.id; const baseUrl = 'https://repo.maven.apache.org/maven2'; const baseUrlCustom = 'https://custom.registry.renovatebot.com'; diff --git a/lib/datasource/maven/index.ts b/lib/datasource/maven/index.ts index bab89b4513a97dc6016502188757374248b716e6..d42e5d089cc1e675f85147e9770f57090e658039 100644 --- a/lib/datasource/maven/index.ts +++ b/lib/datasource/maven/index.ts @@ -9,6 +9,7 @@ import { ensureTrailingSlash } from '../../util/url'; import mavenVersion from '../../versioning/maven'; import * as mavenVersioning from '../../versioning/maven'; import { compare } from '../../versioning/maven/compare'; +import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; import { MAVEN_REPO } from './common'; import type { MavenDependency, ReleaseMap } from './types'; @@ -21,24 +22,13 @@ import { getMavenUrl, } from './util'; -export { id } from './common'; - -export const customRegistrySupport = true; -export const defaultRegistryUrls = [MAVEN_REPO]; -export const defaultVersioning = mavenVersioning.id; -export const registryStrategy = 'merge'; - -function isStableVersion(x: string): boolean { - return mavenVersion.isStable(x); -} - function getLatestSuitableVersion(releases: Release[]): string | null { // istanbul ignore if if (!releases?.length) { return null; } const allVersions = releases.map(({ version }) => version); - const stableVersions = allVersions.filter(isStableVersion); + const stableVersions = allVersions.filter((x) => mavenVersion.isStable(x)); const versions = stableVersions.length ? stableVersions : allVersions; return versions.reduce((latestVersion, version) => compare(version, latestVersion) === 1 ? version : latestVersion @@ -54,98 +44,11 @@ function extractVersions(metadata: XmlDocument): string[] { return elements.map((el) => el.val); } -async function fetchReleasesFromMetadata( - dependency: MavenDependency, - repoUrl: string -): Promise<ReleaseMap> { - const metadataUrl = getMavenUrl(dependency, repoUrl, 'maven-metadata.xml'); - - const cacheNamespace = 'datasource-maven:metadata-xml'; - const cacheKey = metadataUrl.toString(); - const cachedVersions = await packageCache.get<ReleaseMap>( - cacheNamespace, - cacheKey - ); - /* istanbul ignore if */ - if (cachedVersions) { - return cachedVersions; - } - - const { authorization, xml: mavenMetadata } = await downloadMavenXml( - metadataUrl - ); - if (!mavenMetadata) { - return {}; - } - - const versions = extractVersions(mavenMetadata); - const releaseMap = versions.reduce( - (acc, version) => ({ ...acc, [version]: null }), - {} - ); - if (!authorization) { - await packageCache.set(cacheNamespace, cacheKey, releaseMap, 30); - } - return releaseMap; -} - const mavenCentralHtmlVersionRegex = regEx( '^<a href="(?<version>[^"]+)\\/" title="(?:[^"]+)\\/">(?:[^"]+)\\/<\\/a>\\s+(?<releaseTimestamp>\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d)\\s+-$', 'i' ); -async function addReleasesFromIndexPage( - inputReleaseMap: ReleaseMap, - dependency: MavenDependency, - repoUrl: string -): Promise<ReleaseMap> { - const cacheNs = 'datasource-maven:index-html-releases'; - const cacheKey = `${repoUrl}${dependency.dependencyUrl}`; - let workingReleaseMap = await packageCache.get<ReleaseMap>(cacheNs, cacheKey); - if (!workingReleaseMap) { - workingReleaseMap = {}; - let retryEarlier = false; - try { - if (repoUrl.startsWith(MAVEN_REPO)) { - const indexUrl = getMavenUrl(dependency, repoUrl, 'index.html'); - const res = await downloadHttpProtocol(indexUrl); - const { body = '' } = res; - for (const line of body.split(newlineRegex)) { - const match = line.trim().match(mavenCentralHtmlVersionRegex); - if (match) { - const { version, releaseTimestamp: timestamp } = - match?.groups ?? {}; - if (version && timestamp) { - const date = DateTime.fromFormat(timestamp, 'yyyy-MM-dd HH:mm', { - zone: 'UTC', - }); - if (date.isValid) { - const releaseTimestamp = date.toISO(); - workingReleaseMap[version] = { version, releaseTimestamp }; - } - } - } - } - } - } catch (err) /* istanbul ignore next */ { - retryEarlier = true; - logger.debug( - { dependency, err }, - 'Failed to get releases from index.html' - ); - } - const cacheTTL = retryEarlier ? 60 : 24 * 60; - await packageCache.set(cacheNs, cacheKey, workingReleaseMap, cacheTTL); - } - - const releaseMap = { ...inputReleaseMap }; - for (const version of Object.keys(releaseMap)) { - releaseMap[version] ||= workingReleaseMap[version] ?? null; - } - - return releaseMap; -} - function isSnapshotVersion(version: string): boolean { if (version.endsWith('-SNAPSHOT')) { return true; @@ -177,162 +80,286 @@ function extractSnapshotVersion(metadata: XmlDocument): string | null { return `${version}-${timestamp}-${build}`; } -async function getSnapshotFullVersion( - version: string, - dependency: MavenDependency, - repoUrl: string -): Promise<string | null> { - // To determine what actual files are available for the snapshot, first we have to fetch and parse - // the metadata located at http://<repo>/<group>/<artifact>/<version-SNAPSHOT>/maven-metadata.xml - const metadataUrl = getMavenUrl( - dependency, - repoUrl, - `${version}/maven-metadata.xml` - ); +export const defaultRegistryUrls = [MAVEN_REPO]; - const { xml: mavenMetadata } = await downloadMavenXml(metadataUrl); - if (!mavenMetadata) { - return null; - } +export class MavenDatasource extends Datasource { + static id = 'maven'; - return extractSnapshotVersion(mavenMetadata); -} + override readonly defaultRegistryUrls = defaultRegistryUrls; -async function createUrlForDependencyPom( - version: string, - dependency: MavenDependency, - repoUrl: string -): Promise<string> { - if (isSnapshotVersion(version)) { - // By default, Maven snapshots are deployed to the repository with fixed file names. - // Resolve the full, actual pom file name for the version. - const fullVersion = await getSnapshotFullVersion( - version, - dependency, - repoUrl - ); + override readonly defaultVersioning = mavenVersioning.id; - // If we were able to resolve the version, use that, otherwise fall back to using -SNAPSHOT - if (fullVersion !== null) { - return `${version}/${dependency.name}-${fullVersion}.pom`; - } + override readonly registryStrategy = 'merge'; + + constructor(id = MavenDatasource.id) { + super(id); } - return `${version}/${dependency.name}-${version}.pom`; -} + async fetchReleasesFromMetadata( + dependency: MavenDependency, + repoUrl: string + ): Promise<ReleaseMap> { + const metadataUrl = getMavenUrl(dependency, repoUrl, 'maven-metadata.xml'); + + const cacheNamespace = 'datasource-maven:metadata-xml'; + const cacheKey = metadataUrl.toString(); + const cachedVersions = await packageCache.get<ReleaseMap>( + cacheNamespace, + cacheKey + ); + /* istanbul ignore if */ + if (cachedVersions) { + return cachedVersions; + } -async function addReleasesUsingHeadRequests( - inputReleaseMap: ReleaseMap, - dependency: MavenDependency, - repoUrl: string -): Promise<ReleaseMap> { - const releaseMap = { ...inputReleaseMap }; + const { authorization, xml: mavenMetadata } = await downloadMavenXml( + this.http, + metadataUrl + ); + if (!mavenMetadata) { + return {}; + } - if (process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK) { + const versions = extractVersions(mavenMetadata); + const releaseMap = versions.reduce( + (acc, version) => ({ ...acc, [version]: null }), + {} + ); + if (!authorization) { + await packageCache.set(cacheNamespace, cacheKey, releaseMap, 30); + } return releaseMap; } - const cacheNs = 'datasource-maven:head-requests'; - const cacheKey = `${repoUrl}${dependency.dependencyUrl}`; - const oldReleaseMap: ReleaseMap | undefined = - await packageCache.get<ReleaseMap>(cacheNs, cacheKey); - const newReleaseMap: ReleaseMap = oldReleaseMap ?? {}; - - if (!oldReleaseMap) { - const unknownVersions = Object.entries(releaseMap) - .filter(([version, release]) => { - const isDiscoveredOutside = !!release; - const isDiscoveredInsideAndCached = !is.undefined( - newReleaseMap[version] + async addReleasesFromIndexPage( + inputReleaseMap: ReleaseMap, + dependency: MavenDependency, + repoUrl: string + ): Promise<ReleaseMap> { + const cacheNs = 'datasource-maven:index-html-releases'; + const cacheKey = `${repoUrl}${dependency.dependencyUrl}`; + let workingReleaseMap = await packageCache.get<ReleaseMap>( + cacheNs, + cacheKey + ); + if (!workingReleaseMap) { + workingReleaseMap = {}; + let retryEarlier = false; + try { + if (repoUrl.startsWith(MAVEN_REPO)) { + const indexUrl = getMavenUrl(dependency, repoUrl, 'index.html'); + const res = await downloadHttpProtocol(this.http, indexUrl); + const { body = '' } = res; + for (const line of body.split(newlineRegex)) { + const match = line.trim().match(mavenCentralHtmlVersionRegex); + if (match) { + const { version, releaseTimestamp: timestamp } = + match?.groups ?? {}; + if (version && timestamp) { + const date = DateTime.fromFormat( + timestamp, + 'yyyy-MM-dd HH:mm', + { + zone: 'UTC', + } + ); + if (date.isValid) { + const releaseTimestamp = date.toISO(); + workingReleaseMap[version] = { version, releaseTimestamp }; + } + } + } + } + } + } catch (err) /* istanbul ignore next */ { + retryEarlier = true; + logger.debug( + { dependency, err }, + 'Failed to get releases from index.html' ); - const isDiscovered = isDiscoveredOutside || isDiscoveredInsideAndCached; - return !isDiscovered; - }) - .map(([k]) => k); + } + const cacheTTL = retryEarlier ? 60 : 24 * 60; + await packageCache.set(cacheNs, cacheKey, workingReleaseMap, cacheTTL); + } - if (unknownVersions.length) { - let retryEarlier = false; - const queue = unknownVersions.map( - (version) => async (): Promise<void> => { - const pomUrl = await createUrlForDependencyPom( - version, - dependency, - repoUrl - ); - const artifactUrl = getMavenUrl(dependency, repoUrl, pomUrl); - const release: Release = { version }; + const releaseMap = { ...inputReleaseMap }; + for (const version of Object.keys(releaseMap)) { + releaseMap[version] ||= workingReleaseMap[version] ?? null; + } - const res = await checkHttpResource(artifactUrl); + return releaseMap; + } - if (res === 'error') { - retryEarlier = true; - } + async getSnapshotFullVersion( + version: string, + dependency: MavenDependency, + repoUrl: string + ): Promise<string | null> { + // To determine what actual files are available for the snapshot, first we have to fetch and parse + // the metadata located at http://<repo>/<group>/<artifact>/<version-SNAPSHOT>/maven-metadata.xml + const metadataUrl = getMavenUrl( + dependency, + repoUrl, + `${version}/maven-metadata.xml` + ); - if (is.date(res)) { - release.releaseTimestamp = res.toISOString(); - } + const { xml: mavenMetadata } = await downloadMavenXml( + this.http, + metadataUrl + ); + if (!mavenMetadata) { + return null; + } - if (res !== 'not-found' && res !== 'error') { - newReleaseMap[version] = release; - } - } + return extractSnapshotVersion(mavenMetadata); + } + + async createUrlForDependencyPom( + version: string, + dependency: MavenDependency, + repoUrl: string + ): Promise<string> { + if (isSnapshotVersion(version)) { + // By default, Maven snapshots are deployed to the repository with fixed file names. + // Resolve the full, actual pom file name for the version. + const fullVersion = await this.getSnapshotFullVersion( + version, + dependency, + repoUrl ); - await pAll(queue, { concurrency: 5 }); - const cacheTTL = retryEarlier ? 60 : 24 * 60; - await packageCache.set(cacheNs, cacheKey, newReleaseMap, cacheTTL); + // If we were able to resolve the version, use that, otherwise fall back to using -SNAPSHOT + if (fullVersion !== null) { + return `${version}/${dependency.name}-${fullVersion}.pom`; + } } - } - for (const version of Object.keys(releaseMap)) { - releaseMap[version] ||= newReleaseMap[version] ?? null; + return `${version}/${dependency.name}-${version}.pom`; } - return releaseMap; -} + async addReleasesUsingHeadRequests( + inputReleaseMap: ReleaseMap, + dependency: MavenDependency, + repoUrl: string + ): Promise<ReleaseMap> { + const releaseMap = { ...inputReleaseMap }; + + if (process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK) { + return releaseMap; + } + + const cacheNs = 'datasource-maven:head-requests'; + const cacheKey = `${repoUrl}${dependency.dependencyUrl}`; + const oldReleaseMap: ReleaseMap | undefined = + await packageCache.get<ReleaseMap>(cacheNs, cacheKey); + const newReleaseMap: ReleaseMap = oldReleaseMap ?? {}; + + if (!oldReleaseMap) { + const unknownVersions = Object.entries(releaseMap) + .filter(([version, release]) => { + const isDiscoveredOutside = !!release; + const isDiscoveredInsideAndCached = !is.undefined( + newReleaseMap[version] + ); + const isDiscovered = + isDiscoveredOutside || isDiscoveredInsideAndCached; + return !isDiscovered; + }) + .map(([k]) => k); + + if (unknownVersions.length) { + let retryEarlier = false; + const queue = unknownVersions.map( + (version) => async (): Promise<void> => { + const pomUrl = await this.createUrlForDependencyPom( + version, + dependency, + repoUrl + ); + const artifactUrl = getMavenUrl(dependency, repoUrl, pomUrl); + const release: Release = { version }; + + const res = await checkHttpResource(this.http, artifactUrl); + + if (res === 'error') { + retryEarlier = true; + } + + if (is.date(res)) { + release.releaseTimestamp = res.toISOString(); + } + + if (res !== 'not-found' && res !== 'error') { + newReleaseMap[version] = release; + } + } + ); + + await pAll(queue, { concurrency: 5 }); + const cacheTTL = retryEarlier ? 60 : 24 * 60; + await packageCache.set(cacheNs, cacheKey, newReleaseMap, cacheTTL); + } + } + + for (const version of Object.keys(releaseMap)) { + releaseMap[version] ||= newReleaseMap[version] ?? null; + } -function getReleasesFromMap(releaseMap: ReleaseMap): Release[] { - const releases = Object.values(releaseMap).filter(is.truthy); - if (releases.length) { - return releases; + return releaseMap; } - return Object.keys(releaseMap).map((version) => ({ version })); -} -export async function getReleases({ - lookupName, - registryUrl, -}: GetReleasesConfig): Promise<ReleaseResult | null> { - // istanbul ignore if - if (!registryUrl) { - return null; + getReleasesFromMap(releaseMap: ReleaseMap): Release[] { + const releases = Object.values(releaseMap).filter(is.truthy); + if (releases.length) { + return releases; + } + return Object.keys(releaseMap).map((version) => ({ version })); } - const dependency = getDependencyParts(lookupName); - const repoUrl = ensureTrailingSlash(registryUrl); + async getReleases({ + lookupName, + registryUrl, + }: GetReleasesConfig): Promise<ReleaseResult | null> { + // istanbul ignore if + if (!registryUrl) { + return null; + } - logger.debug(`Looking up ${dependency.display} in repository ${repoUrl}`); + const dependency = getDependencyParts(lookupName); + const repoUrl = ensureTrailingSlash(registryUrl); - let releaseMap = await fetchReleasesFromMetadata(dependency, repoUrl); - releaseMap = await addReleasesFromIndexPage(releaseMap, dependency, repoUrl); - releaseMap = await addReleasesUsingHeadRequests( - releaseMap, - dependency, - repoUrl - ); - const releases = getReleasesFromMap(releaseMap); - if (!releases?.length) { - return null; - } + logger.debug(`Looking up ${dependency.display} in repository ${repoUrl}`); - logger.debug( - `Found ${releases.length} new releases for ${dependency.display} in repository ${repoUrl}` - ); + let releaseMap = await this.fetchReleasesFromMetadata(dependency, repoUrl); + releaseMap = await this.addReleasesFromIndexPage( + releaseMap, + dependency, + repoUrl + ); + releaseMap = await this.addReleasesUsingHeadRequests( + releaseMap, + dependency, + repoUrl + ); + const releases = this.getReleasesFromMap(releaseMap); + if (!releases?.length) { + return null; + } - const latestSuitableVersion = getLatestSuitableVersion(releases); - const dependencyInfo = - latestSuitableVersion && - (await getDependencyInfo(dependency, repoUrl, latestSuitableVersion)); + logger.debug( + `Found ${releases.length} new releases for ${dependency.display} in repository ${repoUrl}` + ); - return { ...dependency, ...dependencyInfo, releases }; + const latestSuitableVersion = getLatestSuitableVersion(releases); + const dependencyInfo = + latestSuitableVersion && + (await getDependencyInfo( + this.http, + dependency, + repoUrl, + latestSuitableVersion + )); + + return { ...dependency, ...dependencyInfo, releases }; + } } diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts index 04d2d245d441024a794b0ed720ccb6a6acab6f83..4c4e5383b772df7a411a51e06749737da440c239 100644 --- a/lib/datasource/maven/util.ts +++ b/lib/datasource/maven/util.ts @@ -4,27 +4,18 @@ import { XmlDocument } from 'xmldoc'; import { HOST_DISABLED } from '../../constants/error-messages'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import { Http, HttpResponse } from '../../util/http'; +import type { Http, HttpResponse } from '../../util/http'; import { regEx } from '../../util/regex'; import { normalizeDate } from '../metadata'; import type { ReleaseResult } from '../types'; -import { MAVEN_REPO, id } from './common'; +import { MAVEN_REPO } from './common'; import type { HttpResourceCheckResult, MavenDependency, MavenXml, } from './types'; -const http: Record<string, Http> = {}; - -function httpByHostType(hostType: string): Http { - if (!http[hostType]) { - http[hostType] = new Http(hostType); - } - return http[hostType]; -} - const getHost = (x: string): string => new url.URL(x).host; function isMavenCentral(pkgUrl: url.URL | string): boolean { @@ -65,13 +56,12 @@ function isUnsupportedHostError(err: { name: string }): boolean { } export async function downloadHttpProtocol( - pkgUrl: url.URL | string, - hostType = id + http: Http, + pkgUrl: url.URL | string ): Promise<Partial<HttpResponse>> { let raw: HttpResponse; try { - const httpClient = httpByHostType(hostType); - raw = await httpClient.get(pkgUrl.toString()); + raw = await http.get(pkgUrl.toString()); return raw; } catch (err) { const failedUrl = pkgUrl.toString(); @@ -82,7 +72,7 @@ export async function downloadHttpProtocol( logger.trace({ failedUrl }, `Url not found`); } else if (isHostError(err)) { // istanbul ignore next - logger.debug({ failedUrl }, `Cannot connect to ${hostType} host`); + logger.debug({ failedUrl }, `Cannot connect to host`); } else if (isPermissionsIssue(err)) { logger.debug( { failedUrl }, @@ -107,12 +97,11 @@ export async function downloadHttpProtocol( } export async function checkHttpResource( - pkgUrl: url.URL | string, - hostType = id + http: Http, + pkgUrl: url.URL | string ): Promise<HttpResourceCheckResult> { try { - const httpClient = httpByHostType(hostType); - const res = await httpClient.head(pkgUrl.toString()); + const res = await http.head(pkgUrl.toString()); const timestamp = res?.headers?.['last-modified']; if (timestamp) { const isoTimestamp = normalizeDate(timestamp); @@ -151,6 +140,7 @@ export function getMavenUrl( } export async function downloadMavenXml( + http: Http, pkgUrl: url.URL | null ): Promise<MavenXml> { /* istanbul ignore if */ @@ -167,7 +157,7 @@ export async function downloadMavenXml( authorization, body: rawContent, statusCode, - } = await downloadHttpProtocol(pkgUrl)); + } = await downloadHttpProtocol(http, pkgUrl)); break; case 's3:': logger.debug('Skipping s3 dependency'); @@ -200,6 +190,7 @@ export function getDependencyParts(lookupName: string): MavenDependency { } export async function getDependencyInfo( + http: Http, dependency: MavenDependency, repoUrl: string, version: string, @@ -209,7 +200,7 @@ export async function getDependencyInfo( const path = `${version}/${dependency.name}-${version}.pom`; const pomUrl = getMavenUrl(dependency, repoUrl, path); - const { xml: pomContent } = await downloadMavenXml(pomUrl); + const { xml: pomContent } = await downloadMavenXml(http, pomUrl); // istanbul ignore if if (!pomContent) { return result; @@ -249,6 +240,7 @@ export async function getDependencyInfo( const parentDisplayId = `${parentGroupId}:${parentArtifactId}`; const parentDependency = getDependencyParts(parentDisplayId); const parentInformation = await getDependencyInfo( + http, parentDependency, repoUrl, parentVersion, diff --git a/lib/datasource/metadata.spec.ts b/lib/datasource/metadata.spec.ts index 1e77729530e3e8a7801c0dad6d5d183347e0c8c8..019be661bdf3f6e9d5c19eb5675aa06db9c99908 100644 --- a/lib/datasource/metadata.spec.ts +++ b/lib/datasource/metadata.spec.ts @@ -1,4 +1,4 @@ -import * as datasourceMaven from './maven'; +import { MavenDatasource } from './maven'; import { addMetaData, massageGithubUrl } from './metadata'; import * as datasourceNpm from './npm'; import { PypiDatasource } from './pypi'; @@ -175,7 +175,7 @@ describe('datasource/metadata', () => { sourceUrl: 'http://www.github.com/mockk/mockk/', releases: [{ version: '1.9.3' }], }; - const datasource = datasourceMaven.id; + const datasource = MavenDatasource.id; const lookupName = 'io.mockk:mockk'; addMetaData(dep, datasource, lookupName); @@ -188,7 +188,7 @@ describe('datasource/metadata', () => { releases: [{ version: '1.9.3' }], sourceUrl: undefined, }; - const datasource = datasourceMaven.id; + const datasource = MavenDatasource.id; const lookupName = 'io.mockk:mockk'; addMetaData(dep, datasource, lookupName); @@ -201,7 +201,7 @@ describe('datasource/metadata', () => { sourceUrl: 'http://gitlab.com/meno/dropzone/', releases: [{ version: '5.7.0' }], }; - const datasource = datasourceMaven.id; + const datasource = MavenDatasource.id; const lookupName = 'dropzone'; addMetaData(dep, datasource, lookupName); @@ -216,7 +216,7 @@ describe('datasource/metadata', () => { { version: '1.0.3', releaseTimestamp: '2000-01-03T14:34:56.000+02:00' }, ], }; - addMetaData(dep, datasourceMaven.id, 'foobar'); + addMetaData(dep, MavenDatasource.id, 'foobar'); expect(dep.releases).toMatchObject([ { releaseTimestamp: '2000-01-01T12:34:56.000Z' }, { releaseTimestamp: '2000-01-02T12:34:56.000Z' }, diff --git a/lib/datasource/sbt-package/index.spec.ts b/lib/datasource/sbt-package/index.spec.ts index ec165e4d09306ddf5a0ac12e716fe9cfc70c19f5..0ec1b96bc05b4d97dd34f32753dcc0ffd65e0c0c 100644 --- a/lib/datasource/sbt-package/index.spec.ts +++ b/lib/datasource/sbt-package/index.spec.ts @@ -3,8 +3,8 @@ import { Fixtures } from '../../../test/fixtures'; import * as httpMock from '../../../test/http-mock'; import * as mavenVersioning from '../../versioning/maven'; import { MAVEN_REPO } from '../maven/common'; -import { parseIndexDir } from '../sbt-plugin/util'; -import * as sbtPackage from '.'; +import { parseIndexDir } from './util'; +import { SbtPackageDatasource } from '.'; describe('datasource/sbt-package/index', () => { it('parses Maven index directory', () => { @@ -201,7 +201,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'org.scalatest:scalatest', registryUrls: ['https://failed_repo/maven'], }) @@ -212,7 +212,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'com.example:empty', registryUrls: [], }) @@ -223,7 +223,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'org.scalatest:scalatest', registryUrls: ['https://failed_repo/maven', MAVEN_REPO], }) @@ -238,7 +238,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'org.scalatest:scalatest_2.12', registryUrls: [], }) @@ -253,7 +253,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'io.confluent:kafka-avro-serializer', registryUrls: ['https://packages.confluent.io/maven'], }) @@ -268,7 +268,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'org.scalatest:scalatest-app_2.12', registryUrls: [], }) @@ -282,7 +282,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'org.scalatest:scalatest-flatspec_2.12', registryUrls: [], }) @@ -295,7 +295,7 @@ describe('datasource/sbt-package/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPackage.id, + datasource: SbtPackageDatasource.id, depName: 'org.scalatest:scalatest-matchers-core_2.12', registryUrls: [], }) diff --git a/lib/datasource/sbt-package/index.ts b/lib/datasource/sbt-package/index.ts index 0514b8c4286078c62c2c8dafa92cd2d2f603edba..b790bd5313b876b52f15a55a29ba9474ad809a8b 100644 --- a/lib/datasource/sbt-package/index.ts +++ b/lib/datasource/sbt-package/index.ts @@ -1,184 +1,200 @@ import { XmlDocument } from 'xmldoc'; import { logger } from '../../logger'; +import { Http } from '../../util/http'; import { regEx } from '../../util/regex'; import { ensureTrailingSlash } from '../../util/url'; import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; +import { Datasource } from '../datasource'; import { MAVEN_REPO } from '../maven/common'; import { downloadHttpProtocol } from '../maven/util'; -import { normalizeRootRelativeUrls, parseIndexDir } from '../sbt-plugin/util'; import type { GetReleasesConfig, ReleaseResult } from '../types'; +import { + getLatestVersion, + normalizeRootRelativeUrls, + parseIndexDir, +} from './util'; -export const id = 'sbt-package'; -export const customRegistrySupport = true; -export const defaultRegistryUrls = [MAVEN_REPO]; -export const defaultVersioning = ivyVersioning.id; -export const registryStrategy = 'hunt'; - -export async function getArtifactSubdirs( - searchRoot: string, - artifact: string, - scalaVersion: string -): Promise<string[] | null> { - const pkgUrl = ensureTrailingSlash(searchRoot); - const { body: indexContent } = await downloadHttpProtocol(pkgUrl, 'sbt'); - if (indexContent) { - const parseSubdirs = (content: string): string[] => - parseIndexDir(content, (x) => { - if (x === artifact) { - return true; - } - if (x.startsWith(`${artifact}_native`)) { - return false; - } - if (x.startsWith(`${artifact}_sjs`)) { - return false; - } - return x.startsWith(`${artifact}_`); - }); - const normalizedContent = normalizeRootRelativeUrls(indexContent, pkgUrl); - let artifactSubdirs = parseSubdirs(normalizedContent); - if ( - scalaVersion && - artifactSubdirs.includes(`${artifact}_${scalaVersion}`) - ) { - artifactSubdirs = [`${artifact}_${scalaVersion}`]; - } - return artifactSubdirs; - } +export class SbtPackageDatasource extends Datasource { + static id = 'sbt-package'; - return null; -} + override readonly defaultRegistryUrls = [MAVEN_REPO]; -export async function getPackageReleases( - searchRoot: string, - artifactSubdirs: string[] | null -): Promise<string[] | null> { - if (artifactSubdirs) { - const releases: string[] = []; - const parseReleases = (content: string): string[] => - parseIndexDir(content, (x) => !regEx(/^\.+$/).test(x)); - for (const searchSubdir of artifactSubdirs) { - const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`); - const { body: content } = await downloadHttpProtocol(pkgUrl, 'sbt'); - if (content) { - const normalizedContent = normalizeRootRelativeUrls(content, pkgUrl); - const subdirReleases = parseReleases(normalizedContent); - subdirReleases.forEach((x) => releases.push(x)); - } - } - if (releases.length) { - return [...new Set(releases)].sort(compare); - } - } + override readonly defaultVersioning = ivyVersioning.id; - return null; -} + override readonly registryStrategy = 'hunt'; -export function getLatestVersion(versions: string[] | null): string | null { - if (versions?.length) { - return versions.reduce((latestVersion, version) => - compare(version, latestVersion) === 1 ? version : latestVersion - ); + constructor(id = SbtPackageDatasource.id) { + super(id); + this.http = new Http('sbt'); } - return null; -} -export async function getUrls( - searchRoot: string, - artifactDirs: string[] | null, - version: string | null -): Promise<Partial<ReleaseResult>> { - const result: Partial<ReleaseResult> = {}; + async getArtifactSubdirs( + searchRoot: string, + artifact: string, + scalaVersion: string + ): Promise<string[] | null> { + const pkgUrl = ensureTrailingSlash(searchRoot); + const { body: indexContent } = await downloadHttpProtocol( + this.http, + pkgUrl + ); + if (indexContent) { + const parseSubdirs = (content: string): string[] => + parseIndexDir(content, (x) => { + if (x === artifact) { + return true; + } + if (x.startsWith(`${artifact}_native`)) { + return false; + } + if (x.startsWith(`${artifact}_sjs`)) { + return false; + } + return x.startsWith(`${artifact}_`); + }); + const normalizedContent = normalizeRootRelativeUrls(indexContent, pkgUrl); + let artifactSubdirs = parseSubdirs(normalizedContent); + if ( + scalaVersion && + artifactSubdirs.includes(`${artifact}_${scalaVersion}`) + ) { + artifactSubdirs = [`${artifact}_${scalaVersion}`]; + } + return artifactSubdirs; + } - if (!artifactDirs?.length) { - return result; + return null; } - if (!version) { - return result; - } + async getPackageReleases( + searchRoot: string, + artifactSubdirs: string[] | null + ): Promise<string[] | null> { + if (artifactSubdirs) { + const releases: string[] = []; + const parseReleases = (content: string): string[] => + parseIndexDir(content, (x) => !regEx(/^\.+$/).test(x)); + for (const searchSubdir of artifactSubdirs) { + const pkgUrl = ensureTrailingSlash(`${searchRoot}/${searchSubdir}`); + const { body: content } = await downloadHttpProtocol(this.http, pkgUrl); + if (content) { + const normalizedContent = normalizeRootRelativeUrls(content, pkgUrl); + const subdirReleases = parseReleases(normalizedContent); + subdirReleases.forEach((x) => releases.push(x)); + } + } + if (releases.length) { + return [...new Set(releases)].sort(compare); + } + } - for (const artifactDir of artifactDirs) { - const [artifact] = artifactDir.split('_'); - const pomFileNames = [ - `${artifactDir}-${version}.pom`, - `${artifact}-${version}.pom`, - ]; + return null; + } - for (const pomFileName of pomFileNames) { - const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`; - const { body: content } = await downloadHttpProtocol(pomUrl, 'sbt'); + async getUrls( + searchRoot: string, + artifactDirs: string[] | null, + version: string | null + ): Promise<Partial<ReleaseResult>> { + const result: Partial<ReleaseResult> = {}; - if (content) { - const pomXml = new XmlDocument(content); + if (!artifactDirs?.length) { + return result; + } - const homepage = pomXml.valueWithPath('url'); - if (homepage) { - result.homepage = homepage; - } + if (!version) { + return result; + } - const sourceUrl = pomXml.valueWithPath('scm.url'); - if (sourceUrl) { - result.sourceUrl = sourceUrl - .replace(regEx(/^scm:/), '') - .replace(regEx(/^git:/), '') - .replace(regEx(/^git@github.com:/), 'https://github.com/') - .replace(regEx(/\.git$/), ''); + for (const artifactDir of artifactDirs) { + const [artifact] = artifactDir.split('_'); + const pomFileNames = [ + `${artifactDir}-${version}.pom`, + `${artifact}-${version}.pom`, + ]; + + for (const pomFileName of pomFileNames) { + const pomUrl = `${searchRoot}/${artifactDir}/${version}/${pomFileName}`; + const { body: content } = await downloadHttpProtocol(this.http, pomUrl); + + if (content) { + const pomXml = new XmlDocument(content); + + const homepage = pomXml.valueWithPath('url'); + if (homepage) { + result.homepage = homepage; + } + + const sourceUrl = pomXml.valueWithPath('scm.url'); + if (sourceUrl) { + result.sourceUrl = sourceUrl + .replace(regEx(/^scm:/), '') + .replace(regEx(/^git:/), '') + .replace(regEx(/^git@github.com:/), 'https://github.com/') + .replace(regEx(/\.git$/), ''); + } + + return result; } - - return result; } } + + return result; } - return result; -} + async getReleases({ + lookupName, + registryUrl, + }: GetReleasesConfig): Promise<ReleaseResult | null> { + // istanbul ignore if + if (!registryUrl) { + return null; + } -export async function getReleases({ - lookupName, - registryUrl, -}: GetReleasesConfig): Promise<ReleaseResult | null> { - // istanbul ignore if - if (!registryUrl) { - return null; - } + const [groupId, artifactId] = lookupName.split(':'); + const groupIdSplit = groupId.split('.'); + const artifactIdSplit = artifactId.split('_'); + const [artifact, scalaVersion] = artifactIdSplit; + + const repoRoot = ensureTrailingSlash(registryUrl); + const searchRoots: string[] = []; + // Optimize lookup order + searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); + searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); + + for (let idx = 0; idx < searchRoots.length; idx += 1) { + const searchRoot = searchRoots[idx]; + const artifactSubdirs = await this.getArtifactSubdirs( + searchRoot, + artifact, + scalaVersion + ); + const versions = await this.getPackageReleases( + searchRoot, + artifactSubdirs + ); + const latestVersion = getLatestVersion(versions); + const urls = await this.getUrls( + searchRoot, + artifactSubdirs, + latestVersion + ); + + const dependencyUrl = searchRoot; + + if (versions) { + return { + ...urls, + dependencyUrl, + releases: versions.map((v) => ({ version: v })), + }; + } + } - const [groupId, artifactId] = lookupName.split(':'); - const groupIdSplit = groupId.split('.'); - const artifactIdSplit = artifactId.split('_'); - const [artifact, scalaVersion] = artifactIdSplit; - - const repoRoot = ensureTrailingSlash(registryUrl); - const searchRoots: string[] = []; - // Optimize lookup order - searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); - searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); - - for (let idx = 0; idx < searchRoots.length; idx += 1) { - const searchRoot = searchRoots[idx]; - const artifactSubdirs = await getArtifactSubdirs( - searchRoot, - artifact, - scalaVersion + logger.debug( + `No versions found for ${lookupName} in ${searchRoots.length} repositories` ); - const versions = await getPackageReleases(searchRoot, artifactSubdirs); - const latestVersion = getLatestVersion(versions); - const urls = await getUrls(searchRoot, artifactSubdirs, latestVersion); - - const dependencyUrl = searchRoot; - - if (versions) { - return { - ...urls, - dependencyUrl, - releases: versions.map((v) => ({ version: v })), - }; - } + return null; } - - logger.debug( - `No versions found for ${lookupName} in ${searchRoots.length} repositories` - ); - return null; } diff --git a/lib/datasource/sbt-plugin/util.ts b/lib/datasource/sbt-package/util.ts similarity index 64% rename from lib/datasource/sbt-plugin/util.ts rename to lib/datasource/sbt-package/util.ts index 7facccfea54de851202af4f1698ca5a0a78c13c9..98655bd04c4032abd5de5ba671170bf0a53a5f9c 100644 --- a/lib/datasource/sbt-plugin/util.ts +++ b/lib/datasource/sbt-package/util.ts @@ -1,10 +1,8 @@ import { regEx } from '../../util/regex'; +import { compare } from '../../versioning/maven/compare'; const linkRegExp = /(?<=href=['"])[^'"]*(?=\/['"])/gi; -export const SBT_PLUGINS_REPO = - 'https://dl.bintray.com/sbt/sbt-plugin-releases'; - export function parseIndexDir( content: string, filterFn = (x: string): boolean => !regEx(/^\.+/).test(x) @@ -22,3 +20,12 @@ export function normalizeRootRelativeUrls( href.replace(rootRelativePath, '') ); } + +export function getLatestVersion(versions: string[] | null): string | null { + if (versions?.length) { + return versions.reduce((latestVersion, version) => + compare(version, latestVersion) === 1 ? version : latestVersion + ); + } + return null; +} diff --git a/lib/datasource/sbt-plugin/index.spec.ts b/lib/datasource/sbt-plugin/index.spec.ts index 805a629bc49346332c3ed49eb06226993d4ce1a8..13bf3fb91ec13cdc52a5e740759a2f826ff7682e 100644 --- a/lib/datasource/sbt-plugin/index.spec.ts +++ b/lib/datasource/sbt-plugin/index.spec.ts @@ -3,8 +3,8 @@ import * as httpMock from '../../../test/http-mock'; import { loadFixture } from '../../../test/util'; import * as mavenVersioning from '../../versioning/maven'; import { MAVEN_REPO } from '../maven/common'; -import { parseIndexDir } from './util'; -import * as sbtPlugin from '.'; +import { parseIndexDir } from '../sbt-package/util'; +import { SbtPluginDatasource } from '.'; const mavenIndexHtml = loadFixture(`maven-index.html`); const sbtPluginIndex = loadFixture(`sbt-plugins-index.html`); @@ -136,7 +136,7 @@ describe('datasource/sbt-plugin/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPlugin.id, + datasource: SbtPluginDatasource.id, depName: 'org.scalatest:scalatest', registryUrls: ['https://failed_repo/maven'], }) @@ -144,7 +144,7 @@ describe('datasource/sbt-plugin/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPlugin.id, + datasource: SbtPluginDatasource.id, depName: 'org.scalatest:scalaz', registryUrls: [], }) @@ -155,7 +155,7 @@ describe('datasource/sbt-plugin/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPlugin.id, + datasource: SbtPluginDatasource.id, depName: 'org.foundweekends:sbt-bintray', registryUrls: [], }) @@ -170,7 +170,7 @@ describe('datasource/sbt-plugin/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPlugin.id, + datasource: SbtPluginDatasource.id, depName: 'org.foundweekends:sbt-bintray_2.12', registryUrls: [], }) @@ -186,7 +186,7 @@ describe('datasource/sbt-plugin/index', () => { expect( await getPkgReleases({ versioning: mavenVersioning.id, - datasource: sbtPlugin.id, + datasource: SbtPluginDatasource.id, depName: 'io.get-coursier:sbt-coursier', registryUrls: [MAVEN_REPO], }) diff --git a/lib/datasource/sbt-plugin/index.ts b/lib/datasource/sbt-plugin/index.ts index 4a975f13a77ea686009779cf2a14b935329c2a17..69b7cf4ffaad5b1fa4147905347366b9e2c89bea 100644 --- a/lib/datasource/sbt-plugin/index.ts +++ b/lib/datasource/sbt-plugin/index.ts @@ -1,126 +1,136 @@ import { logger } from '../../logger'; +import { Http } from '../../util/http'; import { regEx } from '../../util/regex'; import { ensureTrailingSlash } from '../../util/url'; import * as ivyVersioning from '../../versioning/ivy'; import { compare } from '../../versioning/maven/compare'; import { downloadHttpProtocol } from '../maven/util'; -import { - getArtifactSubdirs, - getLatestVersion, - getPackageReleases, - getUrls, -} from '../sbt-package'; +import { SbtPackageDatasource } from '../sbt-package'; +import { getLatestVersion, parseIndexDir } from '../sbt-package/util'; import type { GetReleasesConfig, ReleaseResult } from '../types'; -import { SBT_PLUGINS_REPO, parseIndexDir } from './util'; -export const id = 'sbt-plugin'; -export const customRegistrySupport = true; +export const SBT_PLUGINS_REPO = + 'https://dl.bintray.com/sbt/sbt-plugin-releases'; + export const defaultRegistryUrls = [SBT_PLUGINS_REPO]; -export const defaultVersioning = ivyVersioning.id; -export const registryStrategy = 'hunt'; - -async function resolvePluginReleases( - rootUrl: string, - artifact: string, - scalaVersion: string -): Promise<string[] | null> { - const searchRoot = `${rootUrl}/${artifact}`; - const parse = (content: string): string[] => - parseIndexDir(content, (x) => !regEx(/^\.+$/).test(x)); - const { body: indexContent } = await downloadHttpProtocol( - ensureTrailingSlash(searchRoot), - 'sbt' - ); - if (indexContent) { - const releases: string[] = []; - const scalaVersionItems = parse(indexContent); - const scalaVersions = scalaVersionItems.map((x) => - x.replace(regEx(/^scala_/), '') + +export class SbtPluginDatasource extends SbtPackageDatasource { + static override readonly id = 'sbt-plugin'; + + override readonly defaultRegistryUrls = defaultRegistryUrls; + + override readonly registryStrategy = 'hunt'; + + override readonly defaultVersioning = ivyVersioning.id; + + constructor() { + super(SbtPluginDatasource.id); + this.http = new Http('sbt'); + } + + async resolvePluginReleases( + rootUrl: string, + artifact: string, + scalaVersion: string + ): Promise<string[] | null> { + const searchRoot = `${rootUrl}/${artifact}`; + const parse = (content: string): string[] => + parseIndexDir(content, (x) => !regEx(/^\.+$/).test(x)); + const { body: indexContent } = await downloadHttpProtocol( + this.http, + ensureTrailingSlash(searchRoot) ); - const searchVersions = scalaVersions.includes(scalaVersion) - ? [scalaVersion] - : scalaVersions; - for (const searchVersion of searchVersions) { - const searchSubRoot = `${searchRoot}/scala_${searchVersion}`; - const { body: subRootContent } = await downloadHttpProtocol( - ensureTrailingSlash(searchSubRoot), - 'sbt' + if (indexContent) { + const releases: string[] = []; + const scalaVersionItems = parse(indexContent); + const scalaVersions = scalaVersionItems.map((x) => + x.replace(regEx(/^scala_/), '') ); - if (subRootContent) { - const sbtVersionItems = parse(subRootContent); - for (const sbtItem of sbtVersionItems) { - const releasesRoot = `${searchSubRoot}/${sbtItem}`; - const { body: releasesIndexContent } = await downloadHttpProtocol( - ensureTrailingSlash(releasesRoot), - 'sbt' - ); - if (releasesIndexContent) { - const releasesParsed = parse(releasesIndexContent); - releasesParsed.forEach((x) => releases.push(x)); + const searchVersions = scalaVersions.includes(scalaVersion) + ? [scalaVersion] + : scalaVersions; + for (const searchVersion of searchVersions) { + const searchSubRoot = `${searchRoot}/scala_${searchVersion}`; + const { body: subRootContent } = await downloadHttpProtocol( + this.http, + ensureTrailingSlash(searchSubRoot) + ); + if (subRootContent) { + const sbtVersionItems = parse(subRootContent); + for (const sbtItem of sbtVersionItems) { + const releasesRoot = `${searchSubRoot}/${sbtItem}`; + const { body: releasesIndexContent } = await downloadHttpProtocol( + this.http, + ensureTrailingSlash(releasesRoot) + ); + if (releasesIndexContent) { + const releasesParsed = parse(releasesIndexContent); + releasesParsed.forEach((x) => releases.push(x)); + } } } } + if (releases.length) { + return [...new Set(releases)].sort(compare); + } } - if (releases.length) { - return [...new Set(releases)].sort(compare); - } - } - return null; -} - -export async function getReleases({ - lookupName, - registryUrl, -}: GetReleasesConfig): Promise<ReleaseResult | null> { - // istanbul ignore if - if (!registryUrl) { return null; } - const [groupId, artifactId] = lookupName.split(':'); - const groupIdSplit = groupId.split('.'); - const artifactIdSplit = artifactId.split('_'); - const [artifact, scalaVersion] = artifactIdSplit; - - const repoRoot = ensureTrailingSlash(registryUrl); - const searchRoots: string[] = []; - // Optimize lookup order - searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); - searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); - - for (let idx = 0; idx < searchRoots.length; idx += 1) { - const searchRoot = searchRoots[idx]; - let versions = await resolvePluginReleases( - searchRoot, - artifact, - scalaVersion - ); - let urls = {}; + override async getReleases({ + lookupName, + registryUrl, + }: GetReleasesConfig): Promise<ReleaseResult | null> { + // istanbul ignore if + if (!registryUrl) { + return null; + } + + const [groupId, artifactId] = lookupName.split(':'); + const groupIdSplit = groupId.split('.'); + const artifactIdSplit = artifactId.split('_'); + const [artifact, scalaVersion] = artifactIdSplit; - if (!versions?.length) { - const artifactSubdirs = await getArtifactSubdirs( + const repoRoot = ensureTrailingSlash(registryUrl); + const searchRoots: string[] = []; + // Optimize lookup order + searchRoots.push(`${repoRoot}${groupIdSplit.join('.')}`); + searchRoots.push(`${repoRoot}${groupIdSplit.join('/')}`); + + for (let idx = 0; idx < searchRoots.length; idx += 1) { + const searchRoot = searchRoots[idx]; + let versions = await this.resolvePluginReleases( searchRoot, artifact, scalaVersion ); - versions = await getPackageReleases(searchRoot, artifactSubdirs); - const latestVersion = getLatestVersion(versions); - urls = await getUrls(searchRoot, artifactSubdirs, latestVersion); - } + let urls = {}; + + if (!versions?.length) { + const artifactSubdirs = await this.getArtifactSubdirs( + searchRoot, + artifact, + scalaVersion + ); + versions = await this.getPackageReleases(searchRoot, artifactSubdirs); + const latestVersion = getLatestVersion(versions); + urls = await this.getUrls(searchRoot, artifactSubdirs, latestVersion); + } - const dependencyUrl = `${searchRoot}/${artifact}`; + const dependencyUrl = `${searchRoot}/${artifact}`; - if (versions) { - return { - ...urls, - dependencyUrl, - releases: versions.map((v) => ({ version: v })), - }; + if (versions) { + return { + ...urls, + dependencyUrl, + releases: versions.map((v) => ({ version: v })), + }; + } } - } - logger.debug( - `No versions found for ${lookupName} in ${searchRoots.length} repositories` - ); - return null; + logger.debug( + `No versions found for ${lookupName} in ${searchRoots.length} repositories` + ); + return null; + } } diff --git a/lib/manager/gradle/deep/gradle-updates-report.ts b/lib/manager/gradle/deep/gradle-updates-report.ts index c8567eee451182e0dfa5d4168da3c5116f18042b..42978a8bcd78dde775b472efadeb939e4775b4e7 100644 --- a/lib/manager/gradle/deep/gradle-updates-report.ts +++ b/lib/manager/gradle/deep/gradle-updates-report.ts @@ -1,5 +1,5 @@ import upath from 'upath'; -import * as datasourceSbtPackage from '../../../datasource/sbt-package'; +import { SbtPackageDatasource } from '../../../datasource/sbt-package'; import { logger } from '../../../logger'; import { localPathExists, @@ -146,7 +146,7 @@ export async function extractDependenciesFromUpdatesReport( return { ...dep, depName: depName.replace(regEx(/_%%/), ''), - datasource: datasourceSbtPackage.id, + datasource: SbtPackageDatasource.id, }; } if (regEx(/^%.*%$/).test(currentValue)) { diff --git a/lib/manager/gradle/deep/index.ts b/lib/manager/gradle/deep/index.ts index ce1572ed8b937a0b7d132ef40153f7c7d3df2f63..2e657e371a29faede3ce0593b07035ab25ff9d83 100644 --- a/lib/manager/gradle/deep/index.ts +++ b/lib/manager/gradle/deep/index.ts @@ -2,7 +2,7 @@ import type { Stats } from 'fs'; import upath from 'upath'; import { GlobalConfig } from '../../../config/global'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; -import * as datasourceMaven from '../../../datasource/maven'; +import { MavenDatasource } from '../../../datasource/maven'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import { exec } from '../../../util/exec'; @@ -145,7 +145,7 @@ export async function extractAllPackageFiles( if (content) { gradleFiles.push({ packageFile, - datasource: datasourceMaven.id, + datasource: MavenDatasource.id, deps: dependencies, }); diff --git a/lib/manager/gradle/index.ts b/lib/manager/gradle/index.ts index 2dd67902e1999dd567b81572623303d4dfa689d5..c48f3d623a545c31b3d080c5d4f259b016cc24cf 100644 --- a/lib/manager/gradle/index.ts +++ b/lib/manager/gradle/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceMaven from '../../datasource/maven'; +import { MavenDatasource } from '../../datasource/maven'; import * as gradleVersioning from '../../versioning/gradle'; import type { ExtractConfig, @@ -40,4 +40,4 @@ export const defaultConfig = { versioning: gradleVersioning.id, }; -export const supportedDatasources = [datasourceMaven.id]; +export const supportedDatasources = [MavenDatasource.id]; diff --git a/lib/manager/gradle/shallow/extract.ts b/lib/manager/gradle/shallow/extract.ts index 0e32f19e38b54132048dffc87a1d794b38feb0ac..a1ee6b165c283d3be890c9e83ba87ee2d4500b75 100644 --- a/lib/manager/gradle/shallow/extract.ts +++ b/lib/manager/gradle/shallow/extract.ts @@ -1,6 +1,6 @@ import upath from 'upath'; import { - id as datasource, + MavenDatasource, defaultRegistryUrls, } from '../../../datasource/maven'; import { logger } from '../../../logger'; @@ -23,6 +23,8 @@ import { toAbsolutePath, } from './utils'; +const datasource = MavenDatasource.id; + // Enables reverse sorting in generateBranchConfig() // // Required for grouped dependencies to be upgraded diff --git a/lib/manager/maven/extract.ts b/lib/manager/maven/extract.ts index e56d97c5bb55258a3b47a4b932d69b5320b2fa59..e0613881a64b1c2d84645f098e2df7a663b7efa4 100644 --- a/lib/manager/maven/extract.ts +++ b/lib/manager/maven/extract.ts @@ -1,7 +1,7 @@ import is from '@sindresorhus/is'; import upath from 'upath'; import { XmlDocument, XmlElement } from 'xmldoc'; -import * as datasourceMaven from '../../datasource/maven'; +import { MavenDatasource } from '../../datasource/maven'; import { MAVEN_REPO } from '../../datasource/maven/common'; import { logger } from '../../logger'; import { readLocalFile } from '../../util/fs'; @@ -56,7 +56,7 @@ function depFromNode( const depName = `${groupId}:${artifactId}`; const versionNode = node.descendantWithPath('version'); const fileReplacePosition = versionNode.position; - const datasource = datasourceMaven.id; + const datasource = MavenDatasource.id; const registryUrls = [MAVEN_REPO]; const result: PackageDependency = { datasource, @@ -204,7 +204,7 @@ export function extractPackage( } const result: PackageFile = { - datasource: datasourceMaven.id, + datasource: MavenDatasource.id, packageFile, deps: [], }; diff --git a/lib/manager/maven/index.ts b/lib/manager/maven/index.ts index 7b285561f4e513b552507f2b6f4b621f08777003..53c9cf4cb3c11ab490c9a08a8a6b691534eb065c 100644 --- a/lib/manager/maven/index.ts +++ b/lib/manager/maven/index.ts @@ -1,5 +1,5 @@ import { ProgrammingLanguage } from '../../constants'; -import * as datasourceMaven from '../../datasource/maven'; +import { MavenDatasource } from '../../datasource/maven'; import * as mavenVersioning from '../../versioning/maven'; export { extractAllPackageFiles } from './extract'; @@ -12,4 +12,4 @@ export const defaultConfig = { versioning: mavenVersioning.id, }; -export const supportedDatasources = [datasourceMaven.id]; +export const supportedDatasources = [MavenDatasource.id]; diff --git a/lib/manager/sbt/extract.ts b/lib/manager/sbt/extract.ts index 73352f25b31775e7b0d3c8729bf96234e38841c2..8f17ca5a4bd3d4802c5de7bec84d3c71c2d65bcf 100644 --- a/lib/manager/sbt/extract.ts +++ b/lib/manager/sbt/extract.ts @@ -1,7 +1,10 @@ -import * as datasourceMaven from '../../datasource/maven'; +import { MavenDatasource } from '../../datasource/maven'; import { MAVEN_REPO } from '../../datasource/maven/common'; -import * as datasourceSbtPackage from '../../datasource/sbt-package'; -import * as datasourceSbtPlugin from '../../datasource/sbt-plugin'; +import { SbtPackageDatasource } from '../../datasource/sbt-package'; +import { + SbtPluginDatasource, + defaultRegistryUrls as sbtPluginDefaultRegistries, +} from '../../datasource/sbt-plugin'; import { regEx } from '../../util/regex'; import { get } from '../../versioning'; import * as mavenVersioning from '../../versioning/maven'; @@ -241,7 +244,7 @@ function parseSbtLine( const rawScalaVersion = getScalaVersion(line); scalaVersion = normalizeScalaVersion(rawScalaVersion); dep = { - datasource: datasourceMaven.id, + datasource: MavenDatasource.id, depName: 'scala', lookupName: 'org.scala-lang:scala-library', currentValue: rawScalaVersion, @@ -309,13 +312,10 @@ function parseSbtLine( if (dep) { if (!dep.datasource) { if (dep.depType === 'plugin') { - dep.datasource = datasourceSbtPlugin.id; - dep.registryUrls = [ - ...registryUrls, - ...datasourceSbtPlugin.defaultRegistryUrls, - ]; + dep.datasource = SbtPluginDatasource.id; + dep.registryUrls = [...registryUrls, ...sbtPluginDefaultRegistries]; } else { - dep.datasource = datasourceSbtPackage.id; + dep.datasource = SbtPackageDatasource.id; } } deps.push({ diff --git a/lib/manager/sbt/index.ts b/lib/manager/sbt/index.ts index e281179a86cb21896f8cd29b072f5035492375b3..48e0fa094b3b47063392759e99ab3dda067bddd6 100644 --- a/lib/manager/sbt/index.ts +++ b/lib/manager/sbt/index.ts @@ -1,15 +1,15 @@ -import * as datasourceMaven from '../../datasource/maven'; -import * as datasourceSbtPackage from '../../datasource/sbt-package'; -import * as datasourceSbtPlugin from '../../datasource/sbt-plugin'; +import { MavenDatasource } from '../../datasource/maven'; +import { SbtPackageDatasource } from '../../datasource/sbt-package'; +import { SbtPluginDatasource } from '../../datasource/sbt-plugin'; import * as ivyVersioning from '../../versioning/ivy'; export { extractPackageFile } from './extract'; export { bumpPackageVersion } from './update'; export const supportedDatasources = [ - datasourceMaven.id, - datasourceSbtPackage.id, - datasourceSbtPlugin.id, + MavenDatasource.id, + SbtPackageDatasource.id, + SbtPluginDatasource.id, ]; export const defaultConfig = { diff --git a/lib/workers/repository/init/vulnerability.ts b/lib/workers/repository/init/vulnerability.ts index 53549c420f35c563f85b287c3ce6c6474e247d28..acdeb49b7e18bcd1ca3177836d71ef515438c21c 100644 --- a/lib/workers/repository/init/vulnerability.ts +++ b/lib/workers/repository/init/vulnerability.ts @@ -1,6 +1,6 @@ import type { PackageRule, RenovateConfig } from '../../../config/types'; import { NO_VULNERABILITY_ALERTS } from '../../../constants/error-messages'; -import * as datasourceMaven from '../../../datasource/maven'; +import { MavenDatasource } from '../../../datasource/maven'; import { id as npmId } from '../../../datasource/npm'; import { NugetDatasource } from '../../../datasource/nuget'; import { PypiDatasource } from '../../../datasource/pypi'; @@ -87,7 +87,7 @@ export async function detectVulnerabilityAlerts( continue; } const datasourceMapping: Record<string, string> = { - MAVEN: datasourceMaven.id, + MAVEN: MavenDatasource.id, NPM: npmId, NUGET: NugetDatasource.id, PIP: PypiDatasource.id, @@ -104,7 +104,7 @@ export async function detectVulnerabilityAlerts( let { vulnerableRequirements } = alert; // istanbul ignore if if (!vulnerableRequirements.length) { - if (datasource === datasourceMaven.id) { + if (datasource === MavenDatasource.id) { vulnerableRequirements = `(,${firstPatchedVersion})`; } else { vulnerableRequirements = `< ${firstPatchedVersion}`; diff --git a/lib/workers/repository/process/fetch.spec.ts b/lib/workers/repository/process/fetch.spec.ts index ebabc68a9bc6a82b9eb5bede1153afc9207c1eec..6590a074a9520230beb13df9e7a6b1ee3e2dba78 100644 --- a/lib/workers/repository/process/fetch.spec.ts +++ b/lib/workers/repository/process/fetch.spec.ts @@ -1,5 +1,5 @@ import { RenovateConfig, getConfig, mocked } from '../../../../test/util'; -import * as datasourceMaven from '../../../datasource/maven'; +import { MavenDatasource } from '../../../datasource/maven'; import type { PackageFile } from '../../../manager/types'; import { fetchUpdates } from './fetch'; import * as lookup from './lookup'; @@ -57,7 +57,7 @@ describe('workers/repository/process/fetch', () => { maven: [ { packageFile: 'pom.xml', - deps: [{ datasource: datasourceMaven.id, depName: 'bbb' }], + deps: [{ datasource: MavenDatasource.id, depName: 'bbb' }], }, ], };