From 4a19d57c6bb66139fb7f678366fe031b36d3a16b Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Wed, 26 Dec 2018 09:36:24 +0200
Subject: [PATCH] feat: datasource versions filter (#3001)
* add filter and sort to datasource
* remove filter and sort from individual datasources
---
lib/datasource/docker/index.js | 4 +---
lib/datasource/github/index.js | 5 +----
lib/datasource/index.js | 24 +++++++++++++++++++++++-
lib/datasource/npm/index.js | 6 +-----
lib/datasource/nuget/index.js | 6 +-----
lib/datasource/orb/index.js | 3 ---
lib/datasource/packagist/index.js | 28 ++++++++++++----------------
lib/datasource/pypi/index.js | 6 ++----
lib/datasource/terraform/index.js | 10 +++-------
test/datasource/packagist.spec.js | 1 +
10 files changed, 45 insertions(+), 48 deletions(-)
diff --git a/lib/datasource/docker/index.js b/lib/datasource/docker/index.js
index c0a237c2c7..f8a5617b73 100644
--- a/lib/datasource/docker/index.js
+++ b/lib/datasource/docker/index.js
@@ -4,7 +4,6 @@ const URL = require('url');
const is = require('@sindresorhus/is');
const parseLinkHeader = require('parse-link-header');
const wwwAuthenticate = require('www-authenticate');
-const { isVersion, sortVersions } = require('../../versioning/docker');
const hostRules = require('../../util/host-rules');
module.exports = {
@@ -314,8 +313,7 @@ async function getPkgReleases(purl, config = {}) {
if (!tags) {
return null;
}
- const sortedTags = tags.filter(isVersion).sort(sortVersions);
- const releases = sortedTags.map(version => ({ version }));
+ const releases = tags.map(version => ({ version }));
const ret = { releases };
return ret;
}
diff --git a/lib/datasource/github/index.js b/lib/datasource/github/index.js
index dc62649666..37495da76f 100644
--- a/lib/datasource/github/index.js
+++ b/lib/datasource/github/index.js
@@ -87,7 +87,6 @@ async function getDigest(config) {
*
* This function will:
* - Fetch all tags or releases (depending on configuration)
- * - Filter for only tags that meet the version scheme definition of a version
* - Sanitize the versions if desired (e.g. strip out leading 'v')
* - Return a dependency object containing sourceUrl string and releases array
*/
@@ -130,9 +129,7 @@ async function getPkgReleases(purl, config) {
return null;
}
// Filter by version scheme
- const { isVersion, sortVersions } = versioning.get(versionScheme);
- // Return a sorted list of valid Versions
- versions = versions.filter(isVersion).sort(sortVersions);
+ const { isVersion } = versioning.get(versionScheme);
const dependency = {
sourceUrl: 'https://github.com/' + repo,
};
diff --git a/lib/datasource/index.js b/lib/datasource/index.js
index 8704697a5a..f0797c9bda 100644
--- a/lib/datasource/index.js
+++ b/lib/datasource/index.js
@@ -1,4 +1,5 @@
const { parse } = require('../util/purl');
+const versioning = require('../versioning');
const orb = require('./orb');
const docker = require('./docker');
@@ -30,7 +31,28 @@ const datasources = {
const cacheNamespace = 'datasource-releases';
-function getPkgReleases(purlStr, config) {
+async function getPkgReleases(purlStr, config) {
+ const res = await getRawReleases(purlStr, config);
+ if (!res) {
+ return res;
+ }
+ const versionScheme =
+ config && config.versionScheme ? config.versionScheme : 'semver';
+ // Filter by version scheme
+ const { isVersion, sortVersions } = versioning.get(versionScheme);
+ // Return a sorted list of valid Versions
+ function sortReleases(release1, release2) {
+ return sortVersions(release1.version, release2.version);
+ }
+ if (res.releases) {
+ res.releases = res.releases
+ .filter(release => isVersion(release.version))
+ .sort(sortReleases);
+ }
+ return res;
+}
+
+function getRawReleases(purlStr, config) {
const cacheKey = cacheNamespace + purlStr;
// The repoCache is initialized for each repo
// By returning a Promise and reusing it, we should only fetch each package at most once
diff --git a/lib/datasource/npm/index.js b/lib/datasource/npm/index.js
index 3411d0ab4b..424feff12b 100644
--- a/lib/datasource/npm/index.js
+++ b/lib/datasource/npm/index.js
@@ -8,7 +8,6 @@ const getRegistryUrl = require('registry-auth-token/registry-url');
const registryAuthToken = require('registry-auth-token');
const parse = require('github-url-from-git');
const { isBase64 } = require('validator');
-const { isVersion, sortVersions } = require('../../versioning/semver');
const hostRules = require('../../util/host-rules');
module.exports = {
@@ -322,10 +321,7 @@ async function getDependency(name, maxRetries = 5) {
}\`\n\nMarking the latest version of an npm package as deprecated results in the entire package being considered deprecated, so contact the package author you think this is a mistake.`;
dep.deprecationSource = 'npm';
}
- const versions = Object.keys(res.versions)
- .filter(isVersion)
- .sort(sortVersions);
- dep.releases = versions.map(version => {
+ dep.releases = Object.keys(res.versions).map(version => {
const release = {
version,
gitRef: res.versions[version].gitHead,
diff --git a/lib/datasource/nuget/index.js b/lib/datasource/nuget/index.js
index b368f2e18a..4db719b232 100644
--- a/lib/datasource/nuget/index.js
+++ b/lib/datasource/nuget/index.js
@@ -1,7 +1,6 @@
const parse = require('github-url-from-git');
const got = require('got');
const xmlParser = require('fast-xml-parser');
-const { isVersion, sortVersions } = require('../../versioning/semver');
module.exports = {
getPkgReleases,
@@ -19,10 +18,7 @@ async function getPkgReleases(purl) {
const dep = {
name,
};
- dep.releases = (res.versions || [])
- .filter(isVersion)
- .sort(sortVersions)
- .map(version => ({ version }));
+ dep.releases = (res.versions || []).map(version => ({ version }));
// look up nuspec for latest release to get repository
const url = `https://api.nuget.org/v3-flatcontainer/${name.toLowerCase()}/${res.versions.pop()}/${name.toLowerCase()}.nuspec`;
try {
diff --git a/lib/datasource/orb/index.js b/lib/datasource/orb/index.js
index 1d07eaa301..6f7f6b22f8 100644
--- a/lib/datasource/orb/index.js
+++ b/lib/datasource/orb/index.js
@@ -1,5 +1,4 @@
const got = require('got');
-const { isVersion, sortVersions } = require('../../versioning/semver');
module.exports = {
getPkgReleases,
@@ -43,8 +42,6 @@ async function getPkgReleases(purl) {
dep.homepage =
dep.homepage || `https://circleci.com/orbs/registry/orb/${dependency}`;
dep.releases = res.versions.map(v => v.version);
- dep.releases = dep.releases.filter(v => isVersion(v));
- dep.releases = dep.releases.sort(sortVersions);
dep.releases = dep.releases.map(version => ({
version,
}));
diff --git a/lib/datasource/packagist/index.js b/lib/datasource/packagist/index.js
index 9d9bc9626e..0dc04609c5 100644
--- a/lib/datasource/packagist/index.js
+++ b/lib/datasource/packagist/index.js
@@ -4,7 +4,6 @@ const delay = require('delay');
const got = require('got');
const parse = require('github-url-from-git');
const pAll = require('p-all');
-const { isVersion, sortVersions } = require('../../versioning/semver-composer');
const hostRules = require('../../util/host-rules');
module.exports = {
@@ -103,21 +102,18 @@ function extractDepReleases(versions) {
dep.releases = [];
return dep;
}
- dep.releases = Object.keys(versions)
- .filter(isVersion)
- .sort(sortVersions)
- .map(version => {
- const release = versions[version];
- dep.homepage = release.homepage || dep.homepage;
- if (release.source && release.source.url) {
- dep.sourceUrl = parse(release.source.url) || release.source.url;
- }
- return {
- version: version.replace(/^v/, ''),
- gitRef: version,
- releaseTimestamp: release.time,
- };
- });
+ dep.releases = Object.keys(versions).map(version => {
+ const release = versions[version];
+ dep.homepage = release.homepage || dep.homepage;
+ if (release.source && release.source.url) {
+ dep.sourceUrl = parse(release.source.url) || release.source.url;
+ }
+ return {
+ version: version.replace(/^v/, ''),
+ gitRef: version,
+ releaseTimestamp: release.time,
+ };
+ });
return dep;
}
diff --git a/lib/datasource/pypi/index.js b/lib/datasource/pypi/index.js
index a3326b33c3..dde5da30ec 100644
--- a/lib/datasource/pypi/index.js
+++ b/lib/datasource/pypi/index.js
@@ -1,7 +1,7 @@
const got = require('got');
const url = require('url');
const is = require('@sindresorhus/is');
-const { isVersion, sortVersions, matches } = require('../../versioning/pep440');
+const { matches } = require('../../versioning/pep440');
module.exports = {
getPkgReleases,
@@ -12,9 +12,7 @@ function normalizeName(input) {
}
function compatibleVersions(releases, compatibility) {
- const versions = Object.keys(releases)
- .filter(isVersion)
- .sort(sortVersions);
+ const versions = Object.keys(releases);
if (!(compatibility && compatibility.python)) {
return versions;
}
diff --git a/lib/datasource/terraform/index.js b/lib/datasource/terraform/index.js
index c82863f00b..f9587a821a 100644
--- a/lib/datasource/terraform/index.js
+++ b/lib/datasource/terraform/index.js
@@ -1,6 +1,5 @@
const got = require('got');
const parse = require('github-url-from-git');
-const { isVersion, sortVersions } = require('../../versioning/semver');
module.exports = {
getPkgReleases,
@@ -43,12 +42,9 @@ async function getPkgReleases(purl) {
if (res.source) {
dep.sourceUrl = parse(res.source);
}
- dep.releases = res.versions
- .filter(v => isVersion(v))
- .sort(sortVersions)
- .map(version => ({
- version,
- }));
+ dep.releases = res.versions.map(version => ({
+ version,
+ }));
if (pkgUrl.startsWith('https://registry.terraform.io/')) {
dep.homepage = `https://registry.terraform.io/modules/${dependency}`;
}
diff --git a/test/datasource/packagist.spec.js b/test/datasource/packagist.spec.js
index c9a516b5f6..4136104666 100644
--- a/test/datasource/packagist.spec.js
+++ b/test/datasource/packagist.spec.js
@@ -20,6 +20,7 @@ describe('datasource/packagist', () => {
hostRules.find = jest.fn(input => input);
global.repoCache = {};
config = {
+ versionScheme: 'semver-composer',
registryUrls: [
{
type: 'composer',
--
GitLab