From e0465365039c1ef33746262c476db432f9128236 Mon Sep 17 00:00:00 2001
From: FidoX <ieb.core@gmail.com>
Date: Mon, 7 Jan 2019 09:44:47 +0000
Subject: [PATCH] feat: maven datasource (WIP) (#2995)
feat: maven datasource
---
lib/datasource/index.js | 2 +
lib/datasource/maven.js | 151 ++++++++++++++++
.../mysql-connector-java/maven-metadata.xml | 12 ++
.../mysql-connector-java/maven-metadata.xml | 17 ++
.../hamcrest/hamcrest-core/maven-metadata.xml | 19 ++
test/datasource/maven.spec.js | 171 ++++++++++++++++++
6 files changed, 372 insertions(+)
create mode 100644 lib/datasource/maven.js
create mode 100644 test/_fixtures/gradle/maven/custom_maven_repo/maven2/mysql/mysql-connector-java/maven-metadata.xml
create mode 100644 test/_fixtures/gradle/maven/repo1.maven.org/maven2/mysql/mysql-connector-java/maven-metadata.xml
create mode 100644 test/_fixtures/gradle/maven/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/maven-metadata.xml
create mode 100644 test/datasource/maven.spec.js
diff --git a/lib/datasource/index.js b/lib/datasource/index.js
index 1110167862..d9765f8bb5 100644
--- a/lib/datasource/index.js
+++ b/lib/datasource/index.js
@@ -12,6 +12,7 @@ const pypi = require('./pypi');
const terraform = require('./terraform');
const gitlab = require('./gitlab');
const cargo = require('./cargo');
+const maven = require('./maven');
const { addMetaData } = require('./metadata');
@@ -21,6 +22,7 @@ const datasources = {
github,
gitlab,
go,
+ maven,
npm,
nuget,
orb,
diff --git a/lib/datasource/maven.js b/lib/datasource/maven.js
new file mode 100644
index 0000000000..9a4c197ed9
--- /dev/null
+++ b/lib/datasource/maven.js
@@ -0,0 +1,151 @@
+const _ = require('lodash');
+const got = require('got');
+const url = require('url');
+const fs = require('fs-extra');
+const xmlParser = require('fast-xml-parser');
+
+module.exports = {
+ getPkgReleases,
+};
+
+// eslint-disable-next-line no-unused-vars
+async function getPkgReleases(purl, config) {
+ const versions = [];
+ const dependency = getDependencyParts(purl);
+ const repositories = getRepositories(purl);
+ if (repositories.length < 1) {
+ logger.error(`No repositories defined for ${dependency.display}`);
+ return null;
+ }
+ logger.debug(
+ `Found ${repositories.length} repositories for ${dependency.display}`
+ );
+ for (let i = 0; i < repositories.length; i += 1) {
+ const repoUrl = repositories[i];
+ logger.debug(
+ `Looking up ${dependency.display} in repository #${i} - ${repoUrl}`
+ );
+ const mavenMetadata = await downloadMavenMetadata(dependency, repoUrl);
+ if (mavenMetadata) {
+ const newVersions = extractVersions(mavenMetadata).filter(
+ version => !versions.includes(version)
+ );
+ versions.push(...newVersions);
+ logger.debug(`Found ${newVersions.length} new versions for ${dependency.display} in repository ${repoUrl}`); // prettier-ignore
+ }
+ }
+
+ if (versions.length === 0) {
+ logger.warn(`No versions found for ${dependency.display} in ${repositories.length} repositories`); // prettier-ignore
+ return null;
+ }
+ logger.debug(`Found ${versions.length} versions for ${dependency.display}`);
+
+ return {
+ ...dependency,
+ releases: versions.map(v => ({ version: v })),
+ };
+}
+
+function getDependencyParts(purl) {
+ return {
+ display: `${purl.namespace}:${purl.name}`,
+ group: purl.namespace,
+ name: purl.name,
+ version: purl.version,
+ dependencyUrl: generateMavenUrl(purl),
+ };
+}
+
+function getRepositories(purl) {
+ if (!purl.qualifiers || !purl.qualifiers.repository_url) {
+ return [];
+ }
+ return purl.qualifiers.repository_url.split(',').map(repoUrl => {
+ if (!repoUrl.endsWith('/')) {
+ return repoUrl + '/';
+ }
+ return repoUrl;
+ });
+}
+
+async function downloadMavenMetadata(dependency, repoUrl) {
+ const pkgUrl = new url.URL(
+ `${dependency.dependencyUrl}/maven-metadata.xml`,
+ repoUrl
+ );
+ let mavenMetadata;
+ switch (pkgUrl.protocol) {
+ case 'file:':
+ mavenMetadata = await downloadFileProtocol(pkgUrl);
+ break;
+ case 'http:':
+ case 'https:':
+ mavenMetadata = await downloadHttpProtocol(pkgUrl);
+ break;
+ default:
+ logger.error(
+ `Invalid protocol ${pkgUrl.protocol} in repository ${repoUrl}`
+ );
+ return null;
+ }
+ if (!mavenMetadata) {
+ logger.debug(`${dependency.display} not found in repository ${repoUrl}`);
+ }
+ return mavenMetadata;
+}
+
+function extractVersions(mavenMetadata) {
+ const doc = xmlParser.parse(mavenMetadata);
+ return _.get(doc, 'metadata.versioning.versions.version', []).map(v =>
+ String(v)
+ );
+}
+
+async function downloadFileProtocol(pkgUrl) {
+ const pkgPath = pkgUrl.toString().replace('file://', '');
+ if (!(await fs.exists(pkgPath))) {
+ return null;
+ }
+ return fs.readFile(pkgPath, 'utf8');
+}
+
+async function downloadHttpProtocol(pkgUrl) {
+ let raw;
+ try {
+ raw = await got(pkgUrl);
+ } catch (err) {
+ if (isNotFoundError(err)) {
+ logger.debug(`Url not found ${pkgUrl}`);
+ } else if (isTemporalError(err)) {
+ logger.warn(`Error requesting ${pkgUrl} Error Code: ${err.statusCode}`);
+ if (isMavenCentral(pkgUrl)) {
+ throw new Error('registry-failure');
+ }
+ } else {
+ logger.warn(
+ `Unknown error requesting ${pkgUrl} Error Code: ${err.statusCode}`
+ );
+ }
+ return null;
+ }
+ return raw.body;
+}
+
+function generateMavenUrl(purl) {
+ return purl.namespace.replace(/\./g, '/') + `/${purl.name}`;
+}
+
+function isMavenCentral(pkgUrl) {
+ return pkgUrl.host === 'central.maven.org';
+}
+
+function isTemporalError(err) {
+ return (
+ err.statusCode === 429 || (err.statusCode > 500 && err.statusCode < 600)
+ );
+}
+
+function isNotFoundError(err) {
+ return err.statusCode === 404;
+}
diff --git a/test/_fixtures/gradle/maven/custom_maven_repo/maven2/mysql/mysql-connector-java/maven-metadata.xml b/test/_fixtures/gradle/maven/custom_maven_repo/maven2/mysql/mysql-connector-java/maven-metadata.xml
new file mode 100644
index 0000000000..38781a726f
--- /dev/null
+++ b/test/_fixtures/gradle/maven/custom_maven_repo/maven2/mysql/mysql-connector-java/maven-metadata.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?><metadata>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>6.0.5</version>
+ <versioning>
+ <versions>
+ <version>6.0.5</version>
+ <version>6.0.4</version>
+ </versions>
+ <lastUpdated>20130301200000</lastUpdated>
+ </versioning>
+</metadata>
diff --git a/test/_fixtures/gradle/maven/repo1.maven.org/maven2/mysql/mysql-connector-java/maven-metadata.xml b/test/_fixtures/gradle/maven/repo1.maven.org/maven2/mysql/mysql-connector-java/maven-metadata.xml
new file mode 100644
index 0000000000..4c3bd0fea3
--- /dev/null
+++ b/test/_fixtures/gradle/maven/repo1.maven.org/maven2/mysql/mysql-connector-java/maven-metadata.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?><metadata>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>8.0.12</version>
+ <versioning>
+ <versions>
+ <version>8.0.12</version>
+ <version>8.0.11</version>
+ <version>8.0.9</version>
+ <version>8.0.8</version>
+ <version>8.0.7</version>
+ <version>6.0.6</version>
+ <version>6.0.5</version>
+ </versions>
+ <lastUpdated>20130301200000</lastUpdated>
+ </versioning>
+</metadata>
diff --git a/test/_fixtures/gradle/maven/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/maven-metadata.xml b/test/_fixtures/gradle/maven/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/maven-metadata.xml
new file mode 100644
index 0000000000..027fecd11a
--- /dev/null
+++ b/test/_fixtures/gradle/maven/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/maven-metadata.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata modelVersion="1.1.0">
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <versioning>
+ <latest>2.1-rc3</latest>
+ <release>2.1-rc3</release>
+ <versions>
+ <version>1.1</version>
+ <version>1.2</version>
+ <version>1.2.1</version>
+ <version>1.3.RC2</version>
+ <version>1.3</version>
+ <version>2.1-rc2</version>
+ <version>2.1-rc3</version>
+ </versions>
+ <lastUpdated>20181129182853</lastUpdated>
+ </versioning>
+</metadata>
diff --git a/test/datasource/maven.spec.js b/test/datasource/maven.spec.js
new file mode 100644
index 0000000000..f98d4d6283
--- /dev/null
+++ b/test/datasource/maven.spec.js
@@ -0,0 +1,171 @@
+const nock = require('nock');
+const fs = require('fs');
+
+const datasource = require('../../lib/datasource');
+const { initLogger } = require('../../lib/logger');
+
+initLogger();
+
+const MYSQL_VERSIONS = [
+ '6.0.5',
+ '6.0.6',
+ '8.0.7',
+ '8.0.8',
+ '8.0.9',
+ '8.0.11',
+ '8.0.12',
+];
+
+const MYSQL_MAVEN_METADATA = fs.readFileSync(
+ 'test/_fixtures/gradle/maven/repo1.maven.org/maven2/mysql/mysql-connector-java/maven-metadata.xml',
+ 'utf8'
+);
+
+const config = {
+ versionScheme: 'loose',
+};
+
+describe('datasource/maven', () => {
+ beforeEach(() => {
+ nock('http://central.maven.org')
+ .get('/maven2/mysql/mysql-connector-java/maven-metadata.xml')
+ .reply(200, MYSQL_MAVEN_METADATA);
+ nock('http://failed_repo')
+ .get('/mysql/mysql-connector-java/maven-metadata.xml')
+ .reply(404, null);
+ });
+
+ describe('getPkgReleases', () => {
+ it('should return empty if library is not found', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/unknown/unknown@1.0.5?repository_url=file://test/_fixtures/gradle/maven/repo1.maven.org/maven2/',
+ config
+ );
+ expect(releases).toBeNull();
+ });
+
+ it('should return all versions of a specific library', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/org.hamcrest/hamcrest-core@1.2?repository_url=file://test/_fixtures/gradle/maven/repo1.maven.org/maven2/,file://test/_fixtures/gradle/maven/custom_maven_repo/maven2/',
+ config
+ );
+ expect(releases.releases).toEqual(
+ generateReleases([
+ '1.1',
+ '1.2',
+ '1.2.1',
+ '1.3',
+ '1.3.RC2',
+ '2.1-rc2',
+ '2.1-rc3',
+ ])
+ );
+ });
+
+ it('should return versions in all repositories for a specific library', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5?repository_url=file://test/_fixtures/gradle/maven/repo1.maven.org/maven2/,file://test/_fixtures/gradle/maven/custom_maven_repo/maven2/',
+ config
+ );
+ expect(releases.releases).toEqual(
+ generateReleases(['6.0.4', ...MYSQL_VERSIONS])
+ );
+ });
+
+ it('should return all versions of a specific library for http repositories', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5?repository_url=http://central.maven.org/maven2/',
+ config
+ );
+ expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS));
+ });
+
+ it('should return all versions of a specific library if a repository fails', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5?repository_url=http://central.maven.org/maven2/,http://failed_repo/,http://dns_error_repo',
+ config
+ );
+ expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS));
+ });
+
+ it('should throw registry-failure if maven-central fails', async () => {
+ nock('http://central.maven.org')
+ .get('/maven2/org/artifact/maven-metadata.xml')
+ .times(4)
+ .reply(503);
+
+ expect.assertions(1);
+ try {
+ await datasource.getPkgReleases(
+ 'pkg:maven/org/artifact@6.0.5?repository_url=http://central.maven.org/maven2/',
+ config
+ );
+ } catch (e) {
+ expect(e.message).toEqual('registry-failure');
+ }
+ });
+
+ it('should return all versions of a specific library if a repository fails because invalid protocol', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5?repository_url=http://central.maven.org/maven2/,http://failed_repo/,ftp://protocol_error_repo',
+ config
+ );
+ expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS));
+ });
+
+ it('should return all versions of a specific library if a repository fails because invalid metadata file is found in another repository', async () => {
+ const invalidMavenMetadata = `
+ <?xml version="1.0" encoding="UTF-8"?><metadata>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>8.0.12</version>
+ <versioning>
+ <lastUpdated>20130301200000</lastUpdated>
+ </versioning>
+ </metadata>
+ `;
+ nock('http://invalid_metadata_repo')
+ .get('/maven2/mysql/mysql-connector-java/maven-metadata.xml')
+ .reply(200, invalidMavenMetadata);
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5?repository_url=http://central.maven.org/maven2/,http://invalid_metadata_repo/maven2/',
+ config
+ );
+ expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS));
+ });
+
+ it('should return all versions of a specific library if a repository fails because a metadata file is not xml', async () => {
+ const invalidMavenMetadata = `
+ Invalid XML
+ `;
+ nock('http://invalid_metadata_repo')
+ .get('/maven2/mysql/mysql-connector-java/maven-metadata.xml')
+ .reply(200, invalidMavenMetadata);
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5?repository_url=http://central.maven.org/maven2/,http://invalid_metadata_repo/maven2/',
+ config
+ );
+ expect(releases.releases).toEqual(generateReleases(MYSQL_VERSIONS));
+ });
+
+ it('should return all versions of a specific library if a repository does not end with /', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5?repository_url=http://central.maven.org/maven2',
+ config
+ );
+ expect(releases).not.toBeNull();
+ });
+
+ it('should return null if no repositories defined', async () => {
+ const releases = await datasource.getPkgReleases(
+ 'pkg:maven/mysql/mysql-connector-java@6.0.5',
+ config
+ );
+ expect(releases).toBeNull();
+ });
+ });
+});
+
+function generateReleases(versions) {
+ return versions.map(v => ({ version: v }));
+}
--
GitLab