diff --git a/docs/usage/self-hosted-experimental.md b/docs/usage/self-hosted-experimental.md
index 0ac6d7d0bb039f2c969731cc7feecf40cb4dccc8..61629fab747cda8282f6590e1e87a62e189441c8 100644
--- a/docs/usage/self-hosted-experimental.md
+++ b/docs/usage/self-hosted-experimental.md
@@ -19,11 +19,6 @@ We will try to keep breakage to a minimum, but make no guarantees that an experi
If set, Renovate will export OpenTelemetry data to the supplied endpoint.
For more information see [the OpenTelemetry docs](opentelemetry.md).
-## `RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK`
-
-If set to any value, Renovate will skip its default artifacts filter check in the Maven datasource.
-Skipping the check will speed things up, but may result in versions being returned which don't properly exist on the server.
-
## `RENOVATE_PAGINATE_ALL`
If set to any value, Renovate will always paginate requests to GitHub fully, instead of stopping after 10 pages.
diff --git a/lib/modules/datasource/clojure/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/clojure/__snapshots__/index.spec.ts.snap
index 19b3e8b369d0335f1e4ff334d839a45fff5555bb..3932a70b27b51c0aeb3c46c3f4bf1a2e805ac913 100644
--- a/lib/modules/datasource/clojure/__snapshots__/index.spec.ts.snap
+++ b/lib/modules/datasource/clojure/__snapshots__/index.spec.ts.snap
@@ -13,15 +13,24 @@ exports[`modules/datasource/clojure/index falls back to next registry url 1`] =
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
@@ -34,15 +43,24 @@ exports[`modules/datasource/clojure/index ignores unsupported protocols 1`] = `
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
]
@@ -62,15 +80,24 @@ exports[`modules/datasource/clojure/index returns releases from custom repositor
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
@@ -90,15 +117,24 @@ exports[`modules/datasource/clojure/index skips registry with invalid XML 1`] =
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
@@ -118,15 +154,24 @@ exports[`modules/datasource/clojure/index skips registry with invalid metadata s
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
diff --git a/lib/modules/datasource/clojure/index.spec.ts b/lib/modules/datasource/clojure/index.spec.ts
index cc3a81a06e3fdc34c363a96783e02f21ffff3248..8b1069e020d10245f28333bcd409259098a3fd5b 100644
--- a/lib/modules/datasource/clojure/index.spec.ts
+++ b/lib/modules/datasource/clojure/index.spec.ts
@@ -12,7 +12,6 @@ const baseUrlCustom = 'https://custom.registry.renovatebot.com';
interface SnapshotOpts {
version: string;
- jarStatus?: number;
meta?: string;
}
@@ -22,7 +21,6 @@ interface MockOpts {
meta?: string | null;
pom?: string | null;
latest?: string;
- jars?: Record<string, number> | null;
snapshots?: SnapshotOpts[] | null;
}
@@ -31,6 +29,7 @@ function mockGenericPackage(opts: MockOpts = {}) {
dep = 'org.example:package',
base = baseUrl,
latest = '2.0.0',
+ snapshots,
} = opts;
const meta =
opts.meta === undefined
@@ -40,39 +39,6 @@ function mockGenericPackage(opts: MockOpts = {}) {
opts.pom === undefined
? Fixtures.get('pom.xml', upath.join('..', 'maven'))
: opts.pom;
- const jars =
- opts.jars === undefined
- ? {
- '0.0.1': 200,
- '1.0.0': 200,
- '1.0.1': 404,
- '1.0.2': 500,
- '2.0.0': 200,
- }
- : opts.jars;
- const snapshots =
- opts.snapshots === undefined
- ? [
- {
- version: '1.0.3-SNAPSHOT',
- meta: Fixtures.get(
- 'metadata-snapshot-version.xml',
- upath.join('..', 'maven'),
- ),
- jarStatus: 200,
- },
- {
- version: '1.0.4-SNAPSHOT',
- meta: Fixtures.get(
- 'metadata-snapshot-version-invalid.xml',
- upath.join('..', 'maven'),
- ),
- },
- {
- version: '1.0.5-SNAPSHOT',
- },
- ]
- : opts.snapshots;
const scope = httpMock.scope(base);
@@ -89,22 +55,6 @@ function mockGenericPackage(opts: MockOpts = {}) {
.reply(200, pom);
}
- if (jars) {
- Object.entries(jars).forEach(([version, status]) => {
- const [major, minor, patch] = version
- .split('.')
- .map((x) => parseInt(x, 10))
- .map((x) => (x < 10 ? `0${x}` : `${x}`));
- const timestamp = `2020-01-01T${major}:${minor}:${patch}.000Z`;
- const headers: httpMock.ReplyHeaders = version.startsWith('0.')
- ? {}
- : { 'Last-Modified': timestamp };
- scope
- .head(`/${packagePath}/${version}/${artifact}-${version}.pom`)
- .reply(status, '', headers);
- });
- }
-
if (snapshots) {
snapshots.forEach((snapshot) => {
if (snapshot.meta) {
@@ -116,31 +66,6 @@ function mockGenericPackage(opts: MockOpts = {}) {
.get(`/${packagePath}/${snapshot.version}/maven-metadata.xml`)
.reply(404, '');
}
-
- if (snapshot.jarStatus) {
- const [major, minor, patch] = snapshot.version
- .replace('-SNAPSHOT', '')
- .split('.')
- .map((x) => parseInt(x, 10))
- .map((x) => (x < 10 ? `0${x}` : `${x}`));
- const timestamp = `2020-01-01T${major}:${minor}:${patch}.000Z`;
- scope
- .head(
- `/${packagePath}/${
- snapshot.version
- }/${artifact}-${snapshot.version.replace(
- '-SNAPSHOT',
- '',
- )}-20200101.${major}${minor}${patch}-${parseInt(patch, 10)}.pom`,
- )
- .reply(snapshot.jarStatus, '', { 'Last-Modified': timestamp });
- } else {
- scope
- .head(
- `/${packagePath}/${snapshot.version}/${artifact}-${snapshot.version}.pom`,
- )
- .reply(404, '');
- }
});
}
}
@@ -163,7 +88,6 @@ describe('modules/datasource/clojure/index', () => {
afterEach(() => {
hostRules.clear();
- delete process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK;
});
it('returns releases from custom repository', async () => {
@@ -180,7 +104,6 @@ describe('modules/datasource/clojure/index', () => {
base: baseUrlCustom,
meta: Fixtures.get('metadata-extra.xml', upath.join('..', 'maven')),
latest: '3.0.0',
- jars: { '3.0.0': 200 },
snapshots: [],
});
@@ -193,7 +116,11 @@ describe('modules/datasource/clojure/index', () => {
expect(releases).toMatchObject([
{ version: '0.0.1' },
{ version: '1.0.0' },
+ { version: '1.0.1' },
+ { version: '1.0.2' },
{ version: '1.0.3-SNAPSHOT' },
+ { version: '1.0.4-SNAPSHOT' },
+ { version: '1.0.5-SNAPSHOT' },
{ version: '2.0.0' },
{ version: '3.0.0' },
]);
diff --git a/lib/modules/datasource/maven/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/maven/__snapshots__/index.spec.ts.snap
index 6e8c05b6e8b186d42ce1ef6730b616defed9ac11..434320c69192148a6465cbb53f75b556bad84218 100644
--- a/lib/modules/datasource/maven/__snapshots__/index.spec.ts.snap
+++ b/lib/modules/datasource/maven/__snapshots__/index.spec.ts.snap
@@ -13,15 +13,24 @@ exports[`modules/datasource/maven/index falls back to next registry url 1`] = `
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
@@ -34,15 +43,24 @@ exports[`modules/datasource/maven/index ignores unsupported protocols 1`] = `
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
]
@@ -99,15 +117,24 @@ exports[`modules/datasource/maven/index returns releases 1`] = `
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
@@ -128,15 +155,24 @@ exports[`modules/datasource/maven/index returns releases from custom repository
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
@@ -156,15 +192,24 @@ exports[`modules/datasource/maven/index skips registry with invalid XML 1`] = `
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
@@ -184,15 +229,24 @@ exports[`modules/datasource/maven/index skips registry with invalid metadata str
"version": "0.0.1",
},
{
- "releaseTimestamp": "2020-01-01T01:00:00.000Z",
"version": "1.0.0",
},
{
- "releaseTimestamp": "2020-01-01T01:00:03.000Z",
+ "version": "1.0.1",
+ },
+ {
+ "version": "1.0.2",
+ },
+ {
"version": "1.0.3-SNAPSHOT",
},
{
- "releaseTimestamp": "2020-01-01T02:00:00.000Z",
+ "version": "1.0.4-SNAPSHOT",
+ },
+ {
+ "version": "1.0.5-SNAPSHOT",
+ },
+ {
"version": "2.0.0",
},
],
diff --git a/lib/modules/datasource/maven/index.spec.ts b/lib/modules/datasource/maven/index.spec.ts
index 402fa00125bc3470ed878e4fd5a29008254acf2a..265910390b2092749e0e3c10b7eecac831f63dc2 100644
--- a/lib/modules/datasource/maven/index.spec.ts
+++ b/lib/modules/datasource/maven/index.spec.ts
@@ -1,5 +1,8 @@
+import { HeadObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { mockClient } from 'aws-sdk-client-mock';
import { GoogleAuth as _googleAuth } from 'google-auth-library';
-import type { ReleaseResult } from '..';
+import { DateTime } from 'luxon';
+import type { Release, ReleaseResult } from '..';
import { getPkgReleases } from '..';
import { Fixtures } from '../../../../test/fixtures';
import * as httpMock from '../../../../test/http-mock';
@@ -7,6 +10,8 @@ import { mocked } from '../../../../test/util';
import { EXTERNAL_HOST_ERROR } from '../../../constants/error-messages';
import * as hostRules from '../../../util/host-rules';
import { id as versioning } from '../../versioning/maven';
+import { postprocessRelease } from '../postprocess-release';
+import { MAVEN_REPO } from './common';
import { MavenDatasource } from '.';
const googleAuth = mocked(_googleAuth);
@@ -23,7 +28,6 @@ const baseUrlARHttps = `https://${arRegistry}`;
interface SnapshotOpts {
version: string;
- jarStatus?: number;
meta?: string;
}
@@ -33,7 +37,6 @@ interface MockOpts {
meta?: string | null;
pom?: string | null;
latest?: string;
- jars?: Record<string, number> | null;
snapshots?: SnapshotOpts[] | null;
html?: string | null;
}
@@ -44,38 +47,11 @@ function mockGenericPackage(opts: MockOpts = {}) {
base = baseUrl,
latest = '2.0.0',
html,
+ snapshots,
} = opts;
const meta =
opts.meta === undefined ? Fixtures.get('metadata.xml') : opts.meta;
const pom = opts.pom === undefined ? Fixtures.get('pom.xml') : opts.pom;
- const jars =
- opts.jars === undefined
- ? {
- '0.0.1': 200,
- '1.0.0': 200,
- '1.0.1': 404,
- '1.0.2': 500,
- '2.0.0': 200,
- }
- : opts.jars;
-
- const snapshots =
- opts.snapshots === undefined
- ? [
- {
- version: '1.0.3-SNAPSHOT',
- meta: Fixtures.get('metadata-snapshot-version.xml'),
- jarStatus: 200,
- },
- {
- version: '1.0.4-SNAPSHOT',
- meta: Fixtures.get('metadata-snapshot-version-invalid.xml'),
- },
- {
- version: '1.0.5-SNAPSHOT',
- },
- ]
- : opts.snapshots;
const scope = httpMock.scope(base);
@@ -114,23 +90,6 @@ function mockGenericPackage(opts: MockOpts = {}) {
}
}
- if (jars) {
- Object.entries(jars).forEach(([version, status]) => {
- const [major, minor, patch] = version
- .replace('-SNAPSHOT', '')
- .split('.')
- .map((x) => parseInt(x, 10))
- .map((x) => (x < 10 ? `0${x}` : `${x}`));
- const timestamp = `2020-01-01T${major}:${minor}:${patch}.000Z`;
- const headers: httpMock.ReplyHeaders = version.startsWith('0.')
- ? {}
- : { 'Last-Modified': timestamp };
- scope
- .head(`/${packagePath}/${version}/${artifact}-${version}.pom`)
- .reply(status, '', headers);
- });
- }
-
if (snapshots) {
snapshots.forEach((snapshot) => {
if (snapshot.meta) {
@@ -142,31 +101,6 @@ function mockGenericPackage(opts: MockOpts = {}) {
.get(`/${packagePath}/${snapshot.version}/maven-metadata.xml`)
.reply(404, '');
}
-
- if (snapshot.jarStatus) {
- const [major, minor, patch] = snapshot.version
- .replace('-SNAPSHOT', '')
- .split('.')
- .map((x) => parseInt(x, 10))
- .map((x) => (x < 10 ? `0${x}` : `${x}`));
- const timestamp = `2020-01-01T${major}:${minor}:${patch}.000Z`;
- scope
- .head(
- `/${packagePath}/${
- snapshot.version
- }/${artifact}-${snapshot.version.replace(
- '-SNAPSHOT',
- '',
- )}-20200101.${major}${minor}${patch}-${parseInt(patch, 10)}.pom`,
- )
- .reply(snapshot.jarStatus, '', { 'Last-Modified': timestamp });
- } else {
- scope
- .head(
- `/${packagePath}/${snapshot.version}/${artifact}-${snapshot.version}.pom`,
- )
- .reply(404, '');
- }
});
}
}
@@ -190,7 +124,6 @@ describe('modules/datasource/maven/index', () => {
afterEach(() => {
hostRules.clear();
- delete process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK;
});
it('returns null when metadata is not found', async () => {
@@ -218,21 +151,15 @@ describe('modules/datasource/maven/index', () => {
const meta = Fixtures.get('metadata-snapshot-version.xml');
mockGenericPackage({
meta: Fixtures.get('metadata-snapshot-only.xml'),
- jars: null,
html: null,
latest: '1.0.3-SNAPSHOT',
snapshots: [
{
version: '1.0.3-SNAPSHOT',
meta,
- jarStatus: 200,
},
],
});
- httpMock
- .scope(baseUrl)
- .get('/org/example/package/1.0.3-SNAPSHOT/maven-metadata.xml')
- .reply(200, meta);
const res = await get();
@@ -243,21 +170,45 @@ describe('modules/datasource/maven/index', () => {
name: 'package',
packageScope: 'org.example',
registryUrl: 'https://repo.maven.apache.org/maven2',
- releases: [
+ releases: [{ version: '1.0.3-SNAPSHOT' }],
+ });
+ });
+
+ it('handles invalid snapshot', async () => {
+ const meta = Fixtures.get('metadata-snapshot-version-invalid.xml');
+ httpMock
+ .scope(MAVEN_REPO)
+ .get('/org/example/package/1.0.3-SNAPSHOT/package-1.0.3-SNAPSHOT.pom')
+ .reply(200, meta);
+
+ mockGenericPackage({
+ meta: Fixtures.get('metadata-snapshot-only.xml'),
+ pom: null,
+ html: null,
+ latest: '1.0.3-SNAPSHOT',
+ snapshots: [
{
- releaseTimestamp: '2020-01-01T01:00:03.000Z',
version: '1.0.3-SNAPSHOT',
+ meta,
},
],
});
+
+ const res = await get();
+
+ expect(res).toEqual({
+ display: 'org.example:package',
+ group: 'org.example',
+ name: 'package',
+ packageScope: 'org.example',
+ registryUrl: 'https://repo.maven.apache.org/maven2',
+ releases: [{ version: '1.0.3-SNAPSHOT' }],
+ });
});
it('returns html-based releases', async () => {
- process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK = 'true';
-
mockGenericPackage({
latest: '2.0.0',
- jars: null,
html: Fixtures.get('index.html'),
meta: Fixtures.get('index.xml'),
snapshots: null,
@@ -403,8 +354,6 @@ describe('modules/datasource/maven/index', () => {
});
it('removes authentication header after redirect', async () => {
- process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK = 'true';
-
const frontendHost = 'frontend_for_private_s3_repository';
const frontendUrl = `https://${frontendHost}/maven2`;
const backendUrl = 'https://private_s3_repository/maven2';
@@ -445,25 +394,17 @@ describe('modules/datasource/maven/index', () => {
});
it('supports artifactregistry urls with auth', async () => {
- const metadataPaths = [
- '/org/example/package/maven-metadata.xml',
- '/org/example/package/1.0.3-SNAPSHOT/maven-metadata.xml',
- '/org/example/package/1.0.4-SNAPSHOT/maven-metadata.xml',
- '/org/example/package/1.0.5-SNAPSHOT/maven-metadata.xml',
- ];
const pomfilePath = '/org/example/package/2.0.0/package-2.0.0.pom';
hostRules.clear();
- for (const path of metadataPaths) {
- httpMock
- .scope(baseUrlARHttps)
- .get(path)
- .matchHeader(
- 'authorization',
- 'Basic b2F1dGgyYWNjZXNzdG9rZW46c29tZS10b2tlbg==',
- )
- .reply(200, Fixtures.get('metadata.xml'));
- }
+ httpMock
+ .scope(baseUrlARHttps)
+ .get('/org/example/package/maven-metadata.xml')
+ .matchHeader(
+ 'authorization',
+ 'Basic b2F1dGgyYWNjZXNzdG9rZW46c29tZS10b2tlbg==',
+ )
+ .reply(200, Fixtures.get('metadata.xml'));
httpMock
.scope(baseUrlARHttps)
@@ -502,25 +443,17 @@ describe('modules/datasource/maven/index', () => {
],
isPrivate: true,
});
- expect(googleAuth).toHaveBeenCalledTimes(5);
+ expect(googleAuth).toHaveBeenCalledTimes(2);
});
it('supports artifactregistry urls without auth', async () => {
- const metadataPaths = [
- '/org/example/package/maven-metadata.xml',
- '/org/example/package/1.0.3-SNAPSHOT/maven-metadata.xml',
- '/org/example/package/1.0.4-SNAPSHOT/maven-metadata.xml',
- '/org/example/package/1.0.5-SNAPSHOT/maven-metadata.xml',
- ];
const pomfilePath = '/org/example/package/2.0.0/package-2.0.0.pom';
hostRules.clear();
- for (const path of metadataPaths) {
- httpMock
- .scope(baseUrlARHttps)
- .get(path)
- .reply(200, Fixtures.get('metadata.xml'));
- }
+ httpMock
+ .scope(baseUrlARHttps)
+ .get('/org/example/package/maven-metadata.xml')
+ .reply(200, Fixtures.get('metadata.xml'));
httpMock
.scope(baseUrlARHttps)
@@ -555,7 +488,7 @@ describe('modules/datasource/maven/index', () => {
],
isPrivate: true,
});
- expect(googleAuth).toHaveBeenCalledTimes(5);
+ expect(googleAuth).toHaveBeenCalledTimes(2);
});
describe('fetching parent info', () => {
@@ -564,8 +497,6 @@ describe('modules/datasource/maven/index', () => {
meta: null,
pom: Fixtures.get('parent-scm-homepage/pom.xml'),
latest: '1.0.0',
- jars: null,
- snapshots: [],
};
it('should get source and homepage from parent', async () => {
@@ -573,8 +504,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-no-info/meta.xml'),
pom: Fixtures.get('child-no-info/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
mockGenericPackage(parentPackage);
@@ -592,8 +521,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-empty/meta.xml'),
pom: Fixtures.get('child-empty/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
@@ -615,8 +542,6 @@ describe('modules/datasource/maven/index', () => {
meta: null,
pom: parentPom,
latest: '2.0.0',
- jars: null,
- snapshots: [],
};
const childMeta = Fixtures.get('child-parent-cycle/child.meta.xml');
@@ -626,15 +551,11 @@ describe('modules/datasource/maven/index', () => {
meta: null,
pom: childPom,
latest: '2.0.0',
- jars: null,
- snapshots: [],
};
mockGenericPackage({
...childPomMock,
meta: childMeta,
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
mockGenericPackage(parentPomMock);
@@ -655,8 +576,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-scm/meta.xml'),
pom: Fixtures.get('child-scm/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
mockGenericPackage(parentPackage);
@@ -674,8 +593,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-url/meta.xml'),
pom: Fixtures.get('child-url/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
mockGenericPackage(parentPackage);
@@ -693,8 +610,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-all-info/meta.xml'),
pom: Fixtures.get('child-all-info/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
@@ -711,8 +626,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-scm-gitatcolon/meta.xml'),
pom: Fixtures.get('child-scm-gitatcolon/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
@@ -728,8 +641,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-scm-gitatslash/meta.xml'),
pom: Fixtures.get('child-scm-gitatslash/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
@@ -745,8 +656,6 @@ describe('modules/datasource/maven/index', () => {
meta: Fixtures.get('child-scm-gitprotocol/meta.xml'),
pom: Fixtures.get('child-scm-gitprotocol/pom.xml'),
latest: '2.0.0',
- jars: { '2.0.0': 200 },
- snapshots: [],
html: null,
});
@@ -757,4 +666,201 @@ describe('modules/datasource/maven/index', () => {
});
});
});
+
+ describe('post-fetch release validation', () => {
+ it('returns null for 404', async () => {
+ httpMock
+ .scope(MAVEN_REPO)
+ .head('/foo/bar/1.2.3/bar-1.2.3.pom')
+ .reply(404);
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
+ { version: '1.2.3' },
+ );
+
+ expect(res).toBeNull();
+ });
+
+ it('returns null for unknown error', async () => {
+ httpMock
+ .scope(MAVEN_REPO)
+ .head('/foo/bar/1.2.3/bar-1.2.3.pom')
+ .replyWithError('unknown error');
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
+ { version: '1.2.3' },
+ );
+
+ expect(res).toBeNull();
+ });
+
+ it('returns original value for 200 response', async () => {
+ httpMock
+ .scope(MAVEN_REPO)
+ .head('/foo/bar/1.2.3/bar-1.2.3.pom')
+ .reply(200);
+ const releaseOrig: Release = { version: '1.2.3' };
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
+ releaseOrig,
+ );
+
+ expect(res).toBe(releaseOrig);
+ });
+
+ it('returns original value for invalid configs', async () => {
+ const releaseOrig: Release = { version: '1.2.3' };
+ expect(
+ await postprocessRelease(
+ { datasource, registryUrl: MAVEN_REPO },
+ releaseOrig,
+ ),
+ ).toBe(releaseOrig);
+ expect(
+ await postprocessRelease(
+ { datasource, packageName: 'foo:bar' },
+ releaseOrig,
+ ),
+ ).toBe(releaseOrig);
+ });
+
+ it('adds releaseTimestamp', async () => {
+ httpMock
+ .scope(MAVEN_REPO)
+ .head('/foo/bar/1.2.3/bar-1.2.3.pom')
+ .reply(200, '', { 'Last-Modified': '2024-01-01T00:00:00.000Z' });
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: MAVEN_REPO },
+ { version: '1.2.3' },
+ );
+
+ expect(res).toEqual({
+ version: '1.2.3',
+ releaseTimestamp: '2024-01-01T00:00:00.000Z',
+ });
+ });
+
+ describe('S3', () => {
+ const s3mock = mockClient(S3Client);
+
+ afterEach(() => {
+ s3mock.reset();
+ });
+
+ it('checks package', async () => {
+ s3mock
+ .on(HeadObjectCommand, {
+ Bucket: 'bucket',
+ Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
+ })
+ .resolvesOnce({});
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
+ { version: '1.2.3' },
+ );
+
+ expect(res).toEqual({ version: '1.2.3' });
+ });
+
+ it('supports timestamp', async () => {
+ s3mock
+ .on(HeadObjectCommand, {
+ Bucket: 'bucket',
+ Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
+ })
+ .resolvesOnce({
+ LastModified: DateTime.fromISO(
+ '2024-01-01T00:00:00.000Z',
+ ).toJSDate(),
+ });
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
+ { version: '1.2.3' },
+ );
+
+ expect(res).toEqual({
+ version: '1.2.3',
+ releaseTimestamp: '2024-01-01T00:00:00.000Z',
+ });
+ });
+
+ it('returns null for deleted object', async () => {
+ s3mock
+ .on(HeadObjectCommand, {
+ Bucket: 'bucket',
+ Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
+ })
+ .resolvesOnce({ DeleteMarker: true });
+
+ const releaseOrig = { version: '1.2.3' };
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
+ releaseOrig,
+ );
+
+ expect(res).toBeNull();
+ });
+
+ it('returns null for NotFound response', async () => {
+ s3mock
+ .on(HeadObjectCommand, {
+ Bucket: 'bucket',
+ Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
+ })
+ .rejectsOnce('NotFound');
+
+ const releaseOrig = { version: '1.2.3' };
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
+ releaseOrig,
+ );
+
+ expect(res).toBeNull();
+ });
+
+ it('returns null for NoSuchKey response', async () => {
+ s3mock
+ .on(HeadObjectCommand, {
+ Bucket: 'bucket',
+ Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
+ })
+ .rejectsOnce('NoSuchKey');
+
+ const releaseOrig = { version: '1.2.3' };
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
+ releaseOrig,
+ );
+
+ expect(res).toBeNull();
+ });
+
+ it('returns null for unknown error', async () => {
+ s3mock
+ .on(HeadObjectCommand, {
+ Bucket: 'bucket',
+ Key: 'foo/bar/1.2.3/bar-1.2.3.pom',
+ })
+ .rejectsOnce('Unknown');
+
+ const releaseOrig = { version: '1.2.3' };
+
+ const res = await postprocessRelease(
+ { datasource, packageName: 'foo:bar', registryUrl: 's3://bucket' },
+ releaseOrig,
+ );
+
+ expect(res).toBeNull();
+ });
+ });
+ });
});
diff --git a/lib/modules/datasource/maven/index.ts b/lib/modules/datasource/maven/index.ts
index b4c4a4f54055ab3105d1ffb57f71f9726d41d23c..62da4ee93750d84a470686763411dfeb93fdc8cd 100644
--- a/lib/modules/datasource/maven/index.ts
+++ b/lib/modules/datasource/maven/index.ts
@@ -4,8 +4,7 @@ import type { XmlDocument } from 'xmldoc';
import { GlobalConfig } from '../../../config/global';
import { logger } from '../../../logger';
import * as packageCache from '../../../util/cache/package';
-import { filterMap } from '../../../util/filter-map';
-import * as p from '../../../util/promises';
+import { cache } from '../../../util/cache/package/decorator';
import { newlineRegex, regEx } from '../../../util/regex';
import { ensureTrailingSlash } from '../../../util/url';
import mavenVersion from '../../versioning/maven';
@@ -14,6 +13,8 @@ import { compare } from '../../versioning/maven/compare';
import { Datasource } from '../datasource';
import type {
GetReleasesConfig,
+ PostprocessReleaseConfig,
+ PostprocessReleaseResult,
RegistryStrategy,
Release,
ReleaseResult,
@@ -182,132 +183,6 @@ export class MavenDatasource extends Datasource {
return releaseMap;
}
- /**
- *
- * Double-check releases using HEAD request and
- * attach timestamps obtained from `Last-Modified` header.
- *
- * Example input:
- *
- * {
- * '1.0.0': {
- * version: '1.0.0',
- * releaseTimestamp: '2020-01-01T01:00:00.000Z',
- * },
- * '1.0.1': null,
- * }
- *
- * Example output:
- *
- * {
- * '1.0.0': {
- * version: '1.0.0',
- * releaseTimestamp: '2020-01-01T01:00:00.000Z',
- * },
- * '1.0.1': {
- * version: '1.0.1',
- * releaseTimestamp: '2021-01-01T01:00:00.000Z',
- * }
- * }
- *
- * It should validate `1.0.0` with HEAD request, but leave `1.0.1` intact.
- *
- */
- async addReleasesUsingHeadRequests(
- inputReleaseMap: ReleaseMap,
- dependency: MavenDependency,
- repoUrl: string,
- ): Promise<ReleaseMap> {
- const releaseMap = { ...inputReleaseMap };
-
- if (process.env.RENOVATE_EXPERIMENTAL_NO_MAVEN_POM_CHECK) {
- return releaseMap;
- }
-
- const cacheNs = 'datasource-maven:head-requests';
- const cacheTimeoutNs = 'datasource-maven:head-requests-timeout';
- const cacheKey = `${repoUrl}${dependency.dependencyUrl}`;
-
- // Store cache validity as the separate flag.
- // This allows both cache updating and resetting.
- //
- // Even if new version is being released each 10 minutes,
- // we still want to reset the whole cache after 24 hours.
- const cacheValid = await packageCache.get<'valid'>(
- cacheTimeoutNs,
- cacheKey,
- );
-
- let cachedReleaseMap: ReleaseMap = {};
- // istanbul ignore if
- if (cacheValid) {
- const cache = await packageCache.get<ReleaseMap>(cacheNs, cacheKey);
- if (cache) {
- cachedReleaseMap = cache;
- }
- }
-
- // List versions to check with HEAD request
- const freshVersions = filterMap(
- Object.entries(releaseMap),
- ([version, release]) => {
- // Release is present in maven-metadata.xml,
- // but haven't been validated yet
- const isValidatedAtPreviousSteps = release !== null;
-
- // Release was validated and cached with HEAD request during previous run
- const isValidatedHere = !is.undefined(cachedReleaseMap[version]);
-
- // istanbul ignore if: not easily testable
- if (isValidatedAtPreviousSteps || isValidatedHere) {
- return null;
- }
-
- // Select only valid releases not yet verified with HEAD request
- return version;
- },
- );
-
- // Update cached data with freshly discovered versions
- if (freshVersions.length) {
- const queue = freshVersions.map((version) => async (): Promise<void> => {
- const pomUrl = await createUrlForDependencyPom(
- this.http,
- version,
- dependency,
- repoUrl,
- );
- const artifactUrl = getMavenUrl(dependency, repoUrl, pomUrl);
- const release: Release = { version };
-
- const res = await checkResource(this.http, artifactUrl);
-
- if (is.date(res)) {
- release.releaseTimestamp = res.toISOString();
- }
-
- cachedReleaseMap[version] =
- res !== 'not-found' && res !== 'error' ? release : null;
- });
-
- await p.all(queue);
-
- if (!cacheValid) {
- // Store new TTL flag for 24 hours if the previous one is invalidated
- await packageCache.set(cacheTimeoutNs, cacheKey, 'valid', 24 * 60);
- }
-
- // Store updated cache object
- await packageCache.set(cacheNs, cacheKey, cachedReleaseMap, 24 * 60);
- }
-
- // Filter releases with the versions validated via HEAD request
- for (const version of Object.keys(releaseMap)) {
- releaseMap[version] = cachedReleaseMap[version] ?? null;
- }
- return releaseMap;
- }
-
getReleasesFromMap(releaseMap: ReleaseMap): Release[] {
const releases = Object.values(releaseMap).filter(is.truthy);
if (releases.length) {
@@ -336,11 +211,6 @@ export class MavenDatasource extends Datasource {
dependency,
repoUrl,
);
- releaseMap = await this.addReleasesUsingHeadRequests(
- releaseMap,
- dependency,
- repoUrl,
- );
const releases = this.getReleasesFromMap(releaseMap);
if (!releases?.length) {
return null;
@@ -372,4 +242,44 @@ export class MavenDatasource extends Datasource {
return result;
}
+
+ @cache({
+ namespace: `datasource-maven`,
+ key: (
+ { registryUrl, packageName }: PostprocessReleaseConfig,
+ { version }: Release,
+ ) => `postprocessRelease:${registryUrl}:${packageName}:${version}`,
+ ttlMinutes: 24 * 60,
+ })
+ override async postprocessRelease(
+ { packageName, registryUrl }: PostprocessReleaseConfig,
+ release: Release,
+ ): Promise<PostprocessReleaseResult> {
+ if (!packageName || !registryUrl) {
+ return release;
+ }
+
+ const dependency = getDependencyParts(packageName);
+
+ const pomUrl = await createUrlForDependencyPom(
+ this.http,
+ release.version,
+ dependency,
+ registryUrl,
+ );
+
+ const artifactUrl = getMavenUrl(dependency, registryUrl, pomUrl);
+
+ const res = await checkResource(this.http, artifactUrl);
+
+ if (res === 'not-found' || res === 'error') {
+ return 'reject';
+ }
+
+ if (is.date(res)) {
+ release.releaseTimestamp = res.toISOString();
+ }
+
+ return release;
+ }
}
diff --git a/lib/modules/datasource/maven/s3.spec.ts b/lib/modules/datasource/maven/s3.spec.ts
index 6aed183e600a1fb3f04d811737e76f644efecf9c..8ec6e5a316463ec374413588db0613fcbfe24836 100644
--- a/lib/modules/datasource/maven/s3.spec.ts
+++ b/lib/modules/datasource/maven/s3.spec.ts
@@ -1,11 +1,6 @@
import { Readable } from 'node:stream';
-import {
- GetObjectCommand,
- HeadObjectCommand,
- S3Client,
-} from '@aws-sdk/client-s3';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { mockClient } from 'aws-sdk-client-mock';
-import { DateTime } from 'luxon';
import type { ReleaseResult } from '..';
import { getPkgReleases } from '..';
import { Fixtures } from '../../../../test/fixtures';
@@ -51,34 +46,7 @@ describe('modules/datasource/maven/s3', () => {
Bucket: 'repobucket',
Key: 'org/example/package/maven-metadata.xml',
})
- .resolvesOnce({ Body: meta as never })
- .on(HeadObjectCommand, {
- Bucket: 'repobucket',
- Key: 'org/example/package/0.0.1/package-0.0.1.pom',
- })
- .resolvesOnce({ DeleteMarker: true })
- .on(HeadObjectCommand, {
- Bucket: 'repobucket',
- Key: 'org/example/package/1.0.0/package-1.0.0.pom',
- })
- .rejectsOnce('NoSuchKey')
- .on(HeadObjectCommand, {
- Bucket: 'repobucket',
- Key: 'org/example/package/1.0.1/package-1.0.1.pom',
- })
- .rejectsOnce('Unknown')
- .on(HeadObjectCommand, {
- Bucket: 'repobucket',
- Key: 'org/example/package/1.0.2/package-1.0.2.pom',
- })
- .resolvesOnce({})
- .on(HeadObjectCommand, {
- Bucket: 'repobucket',
- Key: 'org/example/package/1.0.3/package-1.0.3.pom',
- })
- .resolvesOnce({
- LastModified: DateTime.fromISO(`2020-01-01T00:00:00.000Z`).toJSDate(),
- });
+ .resolvesOnce({ Body: meta as never });
const res = await get('org.example:package', baseUrlS3);
@@ -88,8 +56,11 @@ describe('modules/datasource/maven/s3', () => {
name: 'package',
registryUrl: 's3://repobucket',
releases: [
+ { version: '0.0.1' },
+ { version: '1.0.0' },
+ { version: '1.0.1' },
{ version: '1.0.2' },
- { version: '1.0.3', releaseTimestamp: '2020-01-01T00:00:00.000Z' },
+ { version: '1.0.3' },
],
isPrivate: true,
});
diff --git a/lib/modules/datasource/maven/util.ts b/lib/modules/datasource/maven/util.ts
index e6efa0d451c7df3218207c7c43410e847e3b13b9..1ddc2eaf4abe36a5a22ab8722737e9b6202e5d87 100644
--- a/lib/modules/datasource/maven/util.ts
+++ b/lib/modules/datasource/maven/util.ts
@@ -11,7 +11,7 @@ import { regEx } from '../../../util/regex';
import type { S3UrlParts } from '../../../util/s3';
import { getS3Client, parseS3Url } from '../../../util/s3';
import { streamToString } from '../../../util/streams';
-import { parseUrl } from '../../../util/url';
+import { ensureTrailingSlash, parseUrl } from '../../../util/url';
import { normalizeDate } from '../metadata';
import type { ReleaseResult } from '../types';
import { getGoogleAuthToken } from '../util';
@@ -261,7 +261,10 @@ export function getMavenUrl(
repoUrl: string,
path: string,
): URL {
- return new URL(`${dependency.dependencyUrl}/${path}`, repoUrl);
+ return new URL(
+ `${dependency.dependencyUrl}/${path}`,
+ ensureTrailingSlash(repoUrl),
+ );
}
export async function downloadMavenXml(
@@ -366,6 +369,7 @@ async function getSnapshotFullVersion(
);
const { xml: mavenMetadata } = await downloadMavenXml(http, metadataUrl);
+ // istanbul ignore if: hard to test
if (!mavenMetadata) {
return null;
}
diff --git a/lib/modules/datasource/sbt-package/index.spec.ts b/lib/modules/datasource/sbt-package/index.spec.ts
index 1855156b974661904ce5069fa8b15be1219455c6..5613cdd4c1dbf02265287b748d51fde9eae48994 100644
--- a/lib/modules/datasource/sbt-package/index.spec.ts
+++ b/lib/modules/datasource/sbt-package/index.spec.ts
@@ -263,8 +263,6 @@ describe('modules/datasource/sbt-package/index', () => {
</metadata>
`,
)
- .head('/org/example/example_2.13/1.2.3/example_2.13-1.2.3.pom')
- .reply(200)
.get('/org/example/example_2.13/1.2.3/example_2.13-1.2.3.pom')
.reply(200);
diff --git a/lib/util/cache/package/types.ts b/lib/util/cache/package/types.ts
index 8cb573131bac9c2583208cd763c32e0a643b2a6c..3a2b3938ed7ebec9dca0600d2770e609eba3cdb6 100644
--- a/lib/util/cache/package/types.ts
+++ b/lib/util/cache/package/types.ts
@@ -75,6 +75,7 @@ export type PackageCacheNamespace =
| 'datasource-hexpm-bob'
| 'datasource-java-version'
| 'datasource-jenkins-plugins'
+ | 'datasource-maven'
| 'datasource-maven:head-requests-timeout'
| 'datasource-maven:head-requests'
| 'datasource-maven:index-html-releases'
diff --git a/lib/workers/repository/process/lookup/index.spec.ts b/lib/workers/repository/process/lookup/index.spec.ts
index ff3c06067332807161813fa1d3aefef5d16dd372..5fb42917805072b38242d3e9750ebc6140303175 100644
--- a/lib/workers/repository/process/lookup/index.spec.ts
+++ b/lib/workers/repository/process/lookup/index.spec.ts
@@ -65,6 +65,10 @@ describe('workers/repository/process/lookup/index', () => {
);
const getMavenReleases = jest.spyOn(MavenDatasource.prototype, 'getReleases');
+ const postprocessMavenRelease = jest.spyOn(
+ MavenDatasource.prototype,
+ 'postprocessRelease',
+ );
const getCustomDatasourceReleases = jest.spyOn(
CustomDatasource.prototype,
@@ -3878,6 +3882,9 @@ describe('workers/repository/process/lookup/index', () => {
{ version: '12.6.2.jre11' },
],
});
+ postprocessMavenRelease.mockImplementationOnce((_, x) =>
+ Promise.resolve(x),
+ );
const res = await Result.wrap(
lookup.lookupUpdates(config),