diff --git a/lib/datasource/maven/index.spec.ts b/lib/datasource/maven/index.spec.ts
index a15ee49ff762c47cb82f434b9c4b80fca22d2296..0bb37673b382bf957568207a8227d34d170850a8 100644
--- a/lib/datasource/maven/index.spec.ts
+++ b/lib/datasource/maven/index.spec.ts
@@ -7,15 +7,7 @@ import { getPkgReleases } from '..';
import { DATASOURCE_FAILURE } from '../../constants/error-messages';
import * as hostRules from '../../util/host-rules';
-const MYSQL_VERSIONS = [
- '6.0.5',
- '6.0.6',
- '8.0.7',
- '8.0.8',
- '8.0.9',
- '8.0.11',
- '8.0.12',
-];
+const MYSQL_VERSIONS = ['6.0.5', '6.0.6', '8.0.7', '8.0.8', '8.0.9'];
const MYSQL_MAVEN_METADATA = fs.readFileSync(
resolve(
@@ -46,6 +38,9 @@ describe('datasource/maven', () => {
username: 'username',
password: 'password',
});
+ jest.resetAllMocks();
+ global.repoCache = {};
+ nock.cleanAll();
nock.disableNetConnect();
nock('https://repo.maven.apache.org')
.get('/maven2/mysql/mysql-connector-java/maven-metadata.xml')
@@ -88,6 +83,22 @@ describe('datasource/maven', () => {
'/maven2/mysql/mysql-connector-java/8.0.12/mysql-connector-java-8.0.12.pom?X-Amz-Algorithm=AWS4-HMAC-SHA256'
)
.reply(200, MYSQL_MAVEN_MYSQL_POM);
+ Object.entries({
+ '6.0.5': 200,
+ '6.0.6': 200,
+ '8.0.7': 200,
+ '8.0.8': 200,
+ '8.0.9': 200,
+ '8.0.11': 404,
+ '8.0.12': 500,
+ }).forEach(([v, status]) => {
+ const path = `/maven2/mysql/mysql-connector-java/${v}/mysql-connector-java-${v}.jar`;
+ nock('https://repo.maven.apache.org').head(path).reply(status, '', {});
+ nock('http://frontend_for_private_s3_repository')
+ .head(path)
+ .reply(status, '', {});
+ });
+ return global.renovateCache.rmAll();
});
afterEach(() => {
@@ -144,7 +155,7 @@ describe('datasource/maven', () => {
],
});
expect(releases.releases).toEqual(
- generateReleases(['6.0.4', ...MYSQL_VERSIONS])
+ generateReleases(['6.0.4', ...MYSQL_VERSIONS, '8.0.11', '8.0.12'])
);
});
diff --git a/lib/datasource/maven/index.ts b/lib/datasource/maven/index.ts
index e4edde6b69caabc99fb2af541322f4c28d31993a..ff7920bedc33feb6a8ba0892df1162a94a7d6654 100644
--- a/lib/datasource/maven/index.ts
+++ b/lib/datasource/maven/index.ts
@@ -1,10 +1,11 @@
import url from 'url';
import fs from 'fs-extra';
import { XmlDocument } from 'xmldoc';
+import pAll from 'p-all';
import { logger } from '../../logger';
import { compare } from '../../versioning/maven/compare';
import mavenVersion from '../../versioning/maven';
-import { downloadHttpProtocol } from './util';
+import { downloadHttpProtocol, isHttpResourceExists } from './util';
import { GetReleasesConfig, ReleaseResult } from '../common';
import { MAVEN_REPO } from './common';
@@ -43,6 +44,7 @@ function getMavenUrl(
async function downloadMavenXml(
pkgUrl: url.URL | null
): Promise<XmlDocument | null> {
+ /* istanbul ignore if */
if (!pkgUrl) {
return null;
}
@@ -143,6 +145,112 @@ function extractVersions(metadata: XmlDocument): string[] {
return elements.map((el) => el.val);
}
+async function getVersionsFromMetadata(
+ dependency: MavenDependency,
+ repoUrl: string
+): Promise<string[] | null> {
+ const metadataUrl = getMavenUrl(dependency, repoUrl, 'maven-metadata.xml');
+ if (!metadataUrl) {
+ return null;
+ }
+
+ const cacheNamespace = 'datasource-maven-metadata';
+ const cacheKey = metadataUrl.toString();
+ const cachedVersions = await renovateCache.get<string[]>(
+ cacheNamespace,
+ cacheKey
+ );
+ /* istanbul ignore if */
+ if (cachedVersions) {
+ return cachedVersions;
+ }
+
+ const mavenMetadata = await downloadMavenXml(metadataUrl);
+ if (!mavenMetadata) {
+ return null;
+ }
+
+ const versions = extractVersions(mavenMetadata);
+ await renovateCache.set<string[]>(cacheNamespace, cacheKey, versions, 10);
+ return versions;
+}
+
+type ArtifactsInfo = Record<string, boolean | null>;
+
+function isValidArtifactsInfo(
+ info: ArtifactsInfo | null,
+ versions: string[]
+): boolean {
+ if (!info) {
+ return false;
+ }
+ return versions.every((v) => info[v] !== undefined);
+}
+
+type ArtifactInfoResult = [string, boolean | null];
+
+async function getArtifactInfo(
+ version: string,
+ artifactUrl: url.URL
+): Promise<ArtifactInfoResult> {
+ const proto = artifactUrl.protocol;
+ if (proto === 'http:' || proto === 'https:') {
+ const result = await isHttpResourceExists(artifactUrl);
+ return [version, result];
+ }
+ return [version, true];
+}
+
+async function filterMissingArtifacts(
+ dependency: MavenDependency,
+ repoUrl: string,
+ versions: string[]
+): Promise<string[]> {
+ const cacheNamespace = 'datasource-maven-metadata';
+ const cacheKey = dependency.dependencyUrl;
+ let artifactsInfo: ArtifactsInfo | null = await renovateCache.get<
+ ArtifactsInfo
+ >(cacheNamespace, cacheKey);
+
+ if (!isValidArtifactsInfo(artifactsInfo, versions)) {
+ const queue = versions
+ .map((version): [string, url.URL | null] => {
+ const artifactUrl = getMavenUrl(
+ dependency,
+ repoUrl,
+ `${version}/${dependency.name}-${version}.jar`
+ );
+ return [version, artifactUrl];
+ })
+ .filter(([_, artifactUrl]) => Boolean(artifactUrl))
+ .map(([version, artifactUrl]) => (): Promise<ArtifactInfoResult> =>
+ getArtifactInfo(version, artifactUrl)
+ );
+ const results = await pAll(queue, { concurrency: 5 });
+ artifactsInfo = results.reduce(
+ (acc, [key, value]) => ({
+ ...acc,
+ [key]: value,
+ }),
+ {}
+ );
+
+ // Retry earlier for status other than 404
+ const cacheTTL = Object.values(artifactsInfo).some((x) => x === null)
+ ? 60
+ : 24 * 60;
+
+ await renovateCache.set<ArtifactsInfo>(
+ cacheNamespace,
+ cacheKey,
+ artifactsInfo,
+ cacheTTL
+ );
+ }
+
+ return versions.filter((v) => artifactsInfo[v]);
+}
+
export async function getReleases({
lookupName,
registryUrls,
@@ -158,18 +266,24 @@ export async function getReleases({
logger.debug(
`Looking up ${dependency.display} in repository #${i} - ${repoUrl}`
);
- const metadataUrl = getMavenUrl(dependency, repoUrl, 'maven-metadata.xml');
- const mavenMetadata = await downloadMavenXml(metadataUrl);
- if (mavenMetadata) {
- const newVersions = extractVersions(mavenMetadata).filter(
+ const metadataVersions = await getVersionsFromMetadata(dependency, repoUrl);
+ if (metadataVersions) {
+ const availableVersions = await filterMissingArtifacts(
+ dependency,
+ repoUrl,
+ metadataVersions
+ );
+ const filteredVersions = availableVersions.filter(
(version) => !versions.includes(version)
);
- const latestVersion = getLatestStableVersion(newVersions);
+ versions.push(...filteredVersions);
+
+ const latestVersion = getLatestStableVersion(filteredVersions);
if (latestVersion) {
repoForVersions[latestVersion] = repoUrl;
}
- versions.push(...newVersions);
- logger.debug(`Found ${newVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore
+
+ logger.debug(`Found ${availableVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore
}
}
diff --git a/lib/datasource/maven/util.ts b/lib/datasource/maven/util.ts
index 7fa24e30430adde01f1d38343b32d8f18a2fe7f2..9bf3506892c5de1d0d622768ee218ae9b97814b8 100644
--- a/lib/datasource/maven/util.ts
+++ b/lib/datasource/maven/util.ts
@@ -61,6 +61,7 @@ export async function downloadHttpProtocol(
try {
const httpClient = httpByHostType(hostType);
raw = await httpClient.get(pkgUrl.toString());
+ return raw.body;
} catch (err) {
const failedUrl = pkgUrl.toString();
if (isNotFoundError(err)) {
@@ -89,5 +90,23 @@ export async function downloadHttpProtocol(
}
return null;
}
- return raw.body;
+}
+
+export async function isHttpResourceExists(
+ pkgUrl: url.URL | string,
+ hostType = id
+): Promise<boolean | null> {
+ try {
+ const httpClient = httpByHostType(hostType);
+ await httpClient.head(pkgUrl.toString());
+ return true;
+ } catch (err) {
+ if (isNotFoundError(err)) {
+ return false;
+ }
+
+ const failedUrl = pkgUrl.toString();
+ logger.debug({ failedUrl }, `Can't check HTTP resource existence`);
+ return null;
+ }
}
diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts
index 0cd6819b488733dc431fc70abefeb5a60ab364ad..e7eb4b97dec15c05f9271036c51f8544468fe4c6 100644
--- a/lib/util/http/index.ts
+++ b/lib/util/http/index.ts
@@ -74,6 +74,10 @@ export class Http {
return this.request<string>(url, options);
}
+ head(url: string, options: HttpOptions = {}): Promise<HttpResponse> {
+ return this.request<string>(url, { ...options, method: 'head' });
+ }
+
private async requestJson<T = unknown>(
url: string,
options: InternalHttpOptions