From 45328c57b6ea57c9281a17583d29463c17100aba Mon Sep 17 00:00:00 2001
From: Sergei Zharinov <zharinov@users.noreply.github.com>
Date: Fri, 28 Jul 2023 11:09:41 +0300
Subject: [PATCH] refactor: Extract datasource result filters to separate file
(#23587)
---
lib/modules/datasource/common.ts | 171 +++++++++++++++++-
lib/modules/datasource/conan/index.ts | 2 +-
lib/modules/datasource/docker/index.ts | 2 +-
lib/modules/datasource/index.spec.ts | 2 +-
lib/modules/datasource/index.ts | 158 +---------------
lib/modules/datasource/util.ts | 10 +
.../{common.spec.ts => utils.spec.ts} | 4 +-
lib/workers/repository/process/fetch.ts | 6 +-
.../repository/process/lookup/index.ts | 6 +-
.../repository/process/vulnerabilities.ts | 2 +-
10 files changed, 193 insertions(+), 170 deletions(-)
create mode 100644 lib/modules/datasource/util.ts
rename lib/modules/datasource/{common.spec.ts => utils.spec.ts} (85%)
diff --git a/lib/modules/datasource/common.ts b/lib/modules/datasource/common.ts
index 6c7a05f47d..f4a4b9d4e4 100644
--- a/lib/modules/datasource/common.ts
+++ b/lib/modules/datasource/common.ts
@@ -1,6 +1,37 @@
import is from '@sindresorhus/is';
-import type { HttpResponse } from '../../util/http/types';
-import type { GetPkgReleasesConfig } from './types';
+import { logger } from '../../logger';
+import { filterMap } from '../../util/filter-map';
+import { regEx } from '../../util/regex';
+import { defaultVersioning } from '../versioning';
+import * as allVersioning from '../versioning';
+import datasources from './api';
+import { CustomDatasource } from './custom';
+import type {
+ DatasourceApi,
+ GetPkgReleasesConfig,
+ ReleaseResult,
+} from './types';
+
+export function getDatasourceFor(datasource: string): DatasourceApi | null {
+ if (datasource?.startsWith('custom.')) {
+ return getDatasourceFor(CustomDatasource.id);
+ }
+ return datasources.get(datasource) ?? null;
+}
+
+export function getDefaultVersioning(
+ datasourceName: string | undefined
+): string {
+ if (!datasourceName) {
+ return defaultVersioning.id;
+ }
+ const datasource = getDatasourceFor(datasourceName);
+ // istanbul ignore if: wrong regex manager config?
+ if (!datasource) {
+ logger.warn({ datasourceName }, 'Missing datasource!');
+ }
+ return datasource?.defaultVersioning ?? defaultVersioning.id;
+}
export function isGetPkgReleasesConfig(
input: unknown
@@ -15,10 +46,136 @@ export function isGetPkgReleasesConfig(
);
}
-const JFROG_ARTIFACTORY_RES_HEADER = 'x-jfrog-version';
+export function applyExtractVersion<
+ Config extends Pick<GetPkgReleasesConfig, 'extractVersion'>
+>(config: Config, releaseResult: ReleaseResult): ReleaseResult {
+ if (!config.extractVersion) {
+ return releaseResult;
+ }
+
+ const extractVersionRegEx = regEx(config.extractVersion);
+ releaseResult.releases = filterMap(releaseResult.releases, (release) => {
+ const version = extractVersionRegEx.exec(release.version)?.groups?.version;
+ if (!version) {
+ return null;
+ }
+
+ release.version = version;
+ return release;
+ });
+
+ return releaseResult;
+}
+
+export function filterValidVersions<
+ Config extends Pick<GetPkgReleasesConfig, 'versioning' | 'datasource'>
+>(config: Config, releaseResult: ReleaseResult): ReleaseResult {
+ const versioningName =
+ config.versioning ?? getDefaultVersioning(config.datasource);
+ const versioning = allVersioning.get(versioningName);
+
+ releaseResult.releases = filterMap(releaseResult.releases, (release) =>
+ versioning.isVersion(release.version) ? release : null
+ );
+
+ return releaseResult;
+}
+
+export function sortAndRemoveDuplicates<
+ Config extends Pick<GetPkgReleasesConfig, 'versioning' | 'datasource'>
+>(config: Config, releaseResult: ReleaseResult): ReleaseResult {
+ const versioningName =
+ config.versioning ?? getDefaultVersioning(config.datasource);
+ const versioning = allVersioning.get(versioningName);
+
+ releaseResult.releases = releaseResult.releases.sort((a, b) =>
+ versioning.sortVersions(a.version, b.version)
+ );
+
+ // Once releases are sorted, deduplication is straightforward and efficient
+ let previousVersion: string | null = null;
+ releaseResult.releases = filterMap(releaseResult.releases, (release) => {
+ if (previousVersion === release.version) {
+ return null;
+ }
+ previousVersion = release.version;
+ return release;
+ });
+
+ return releaseResult;
+}
+
+export function applyConstraintsFiltering<
+ Config extends Pick<
+ GetPkgReleasesConfig,
+ | 'constraintsFiltering'
+ | 'versioning'
+ | 'datasource'
+ | 'constraints'
+ | 'packageName'
+ >
+>(config: Config, releaseResult: ReleaseResult): ReleaseResult {
+ if (config?.constraintsFiltering !== 'strict') {
+ for (const release of releaseResult.releases) {
+ delete release.constraints;
+ }
+
+ return releaseResult;
+ }
+
+ const versioningName =
+ config.versioning ?? getDefaultVersioning(config.datasource);
+ const versioning = allVersioning.get(versioningName);
+
+ const configConstraints = config.constraints;
+ const filteredReleases: string[] = [];
+ releaseResult.releases = filterMap(releaseResult.releases, (release) => {
+ const releaseConstraints = release.constraints;
+ delete release.constraints;
+
+ // istanbul ignore if
+ if (!configConstraints || !releaseConstraints) {
+ return release;
+ }
+
+ for (const [name, configConstraint] of Object.entries(configConstraints)) {
+ // istanbul ignore if
+ if (!versioning.isValid(configConstraint)) {
+ continue;
+ }
+
+ const constraint = releaseConstraints[name];
+ if (!is.nonEmptyArray(constraint)) {
+ // A release with no constraints is OK
+ continue;
+ }
+
+ const satisfiesConstraints = constraint.some(
+ // If the constraint value is a subset of any release's constraints, then it's OK
+ // fallback to release's constraint match if subset is not supported by versioning
+ (releaseConstraint) =>
+ !releaseConstraint ||
+ (versioning.subset?.(configConstraint, releaseConstraint) ??
+ versioning.matches(configConstraint, releaseConstraint))
+ );
+
+ if (!satisfiesConstraints) {
+ filteredReleases.push(release.version);
+ return null;
+ }
+ }
+
+ return release;
+ });
+
+ if (filteredReleases.length) {
+ const count = filteredReleases.length;
+ const packageName = config.packageName;
+ const releases = filteredReleases.join(', ');
+ logger.debug(
+ `Filtered ${count} releases for ${packageName} due to constraintsFiltering=strict: ${releases}`
+ );
+ }
-export function isArtifactoryServer<T = unknown>(
- res: HttpResponse<T> | undefined
-): boolean {
- return is.string(res?.headers[JFROG_ARTIFACTORY_RES_HEADER]);
+ return releaseResult;
}
diff --git a/lib/modules/datasource/conan/index.ts b/lib/modules/datasource/conan/index.ts
index 9e4f3b05db..44493c1b00 100644
--- a/lib/modules/datasource/conan/index.ts
+++ b/lib/modules/datasource/conan/index.ts
@@ -5,7 +5,6 @@ import { cache } from '../../../util/cache/package/decorator';
import { GithubHttp } from '../../../util/http/github';
import { ensureTrailingSlash, joinUrlParts } from '../../../util/url';
import * as allVersioning from '../../versioning';
-import { isArtifactoryServer } from '../common';
import { Datasource } from '../datasource';
import type {
DigestConfig,
@@ -13,6 +12,7 @@ import type {
Release,
ReleaseResult,
} from '../types';
+import { isArtifactoryServer } from '../util';
import {
conanDatasourceRegex,
datasource,
diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts
index 6bcd11ddaf..f5082fd06e 100644
--- a/lib/modules/datasource/docker/index.ts
+++ b/lib/modules/datasource/docker/index.ts
@@ -14,9 +14,9 @@ import {
parseLinkHeader,
} from '../../../util/url';
import { id as dockerVersioningId } from '../../versioning/docker';
-import { isArtifactoryServer } from '../common';
import { Datasource } from '../datasource';
import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
+import { isArtifactoryServer } from '../util';
import {
DOCKER_HUB,
dockerDatasourceId,
diff --git a/lib/modules/datasource/index.spec.ts b/lib/modules/datasource/index.spec.ts
index e5c91e2c88..a37361384a 100644
--- a/lib/modules/datasource/index.spec.ts
+++ b/lib/modules/datasource/index.spec.ts
@@ -7,6 +7,7 @@ import {
import { ExternalHostError } from '../../types/errors/external-host-error';
import { loadModules } from '../../util/modules';
import datasources from './api';
+import { getDefaultVersioning } from './common';
import { Datasource } from './datasource';
import type {
DatasourceApi,
@@ -17,7 +18,6 @@ import type {
import {
getDatasourceList,
getDatasources,
- getDefaultVersioning,
getDigest,
getPkgReleases,
supportsDigests,
diff --git a/lib/modules/datasource/index.ts b/lib/modules/datasource/index.ts
index ee92733dba..20cbcd9da9 100644
--- a/lib/modules/datasource/index.ts
+++ b/lib/modules/datasource/index.ts
@@ -6,13 +6,15 @@ import { ExternalHostError } from '../../types/errors/external-host-error';
import * as memCache from '../../util/cache/memory';
import * as packageCache from '../../util/cache/package';
import { clone } from '../../util/clone';
-import { filterMap } from '../../util/filter-map';
-import { regEx } from '../../util/regex';
import { trimTrailingSlash } from '../../util/url';
-import { defaultVersioning } from '../versioning';
-import * as allVersioning from '../versioning';
import datasources from './api';
-import { CustomDatasource } from './custom';
+import {
+ applyConstraintsFiltering,
+ applyExtractVersion,
+ filterValidVersions,
+ getDatasourceFor,
+ sortAndRemoveDuplicates,
+} from './common';
import { addMetaData } from './metadata';
import { setNpmrc } from './npm';
import { resolveRegistryUrl } from './npm/npmrc';
@@ -33,13 +35,6 @@ export const getDatasourceList = (): string[] => Array.from(datasources.keys());
const cacheNamespace = 'datasource-releases';
-export function getDatasourceFor(datasource: string): DatasourceApi | null {
- if (datasource?.startsWith('custom.')) {
- return getDatasourceFor(CustomDatasource.id);
- }
- return datasources.get(datasource) ?? null;
-}
-
type GetReleasesInternalConfig = GetReleasesConfig & GetPkgReleasesConfig;
// TODO: fix error Type
@@ -241,20 +236,6 @@ function resolveRegistryUrls(
return massageRegistryUrls(resolvedUrls);
}
-export function getDefaultVersioning(
- datasourceName: string | undefined
-): string {
- if (!datasourceName) {
- return defaultVersioning.id;
- }
- const datasource = getDatasourceFor(datasourceName);
- // istanbul ignore if: wrong regex manager config?
- if (!datasource) {
- logger.warn({ datasourceName }, 'Missing datasource!');
- }
- return datasource?.defaultVersioning ?? defaultVersioning.id;
-}
-
function applyReplacements(
config: GetReleasesInternalConfig
): Pick<ReleaseResult, 'replacementName' | 'replacementVersion'> | undefined {
@@ -346,131 +327,6 @@ function getRawReleases(
return promisedRes;
}
-function applyExtractVersion<
- Config extends Pick<GetPkgReleasesConfig, 'extractVersion'>
->(config: Config, releaseResult: ReleaseResult): void {
- if (!config.extractVersion) {
- return;
- }
-
- const extractVersionRegEx = regEx(config.extractVersion);
- releaseResult.releases = filterMap(releaseResult.releases, (release) => {
- const version = extractVersionRegEx.exec(release.version)?.groups?.version;
- if (!version) {
- return null;
- }
-
- release.version = version;
- return release;
- });
-}
-
-function filterValidVersions<
- Config extends Pick<GetPkgReleasesConfig, 'versioning' | 'datasource'>
->(config: Config, releaseResult: ReleaseResult): void {
- const versioningName =
- config.versioning ?? getDefaultVersioning(config.datasource);
- const versioning = allVersioning.get(versioningName);
-
- releaseResult.releases = filterMap(releaseResult.releases, (release) =>
- versioning.isVersion(release.version) ? release : null
- );
-}
-
-function sortAndRemoveDuplicates<
- Config extends Pick<GetPkgReleasesConfig, 'versioning' | 'datasource'>
->(config: Config, releaseResult: ReleaseResult): void {
- const versioningName =
- config.versioning ?? getDefaultVersioning(config.datasource);
- const versioning = allVersioning.get(versioningName);
-
- releaseResult.releases = releaseResult.releases.sort((a, b) =>
- versioning.sortVersions(a.version, b.version)
- );
-
- // Once releases are sorted, deduplication is straightforward and efficient
- let previousVersion: string | null = null;
- releaseResult.releases = filterMap(releaseResult.releases, (release) => {
- if (previousVersion === release.version) {
- return null;
- }
- previousVersion = release.version;
- return release;
- });
-}
-
-function applyConstraintsFiltering<
- Config extends Pick<
- GetPkgReleasesConfig,
- | 'constraintsFiltering'
- | 'versioning'
- | 'datasource'
- | 'constraints'
- | 'packageName'
- >
->(config: Config, releaseResult: ReleaseResult): void {
- if (config?.constraintsFiltering !== 'strict') {
- for (const release of releaseResult.releases) {
- delete release.constraints;
- }
- return;
- }
-
- const versioningName =
- config.versioning ?? getDefaultVersioning(config.datasource);
- const versioning = allVersioning.get(versioningName);
-
- const configConstraints = config.constraints;
- const filteredReleases: string[] = [];
- releaseResult.releases = filterMap(releaseResult.releases, (release) => {
- const releaseConstraints = release.constraints;
- delete release.constraints;
-
- // istanbul ignore if
- if (!configConstraints || !releaseConstraints) {
- return release;
- }
-
- for (const [name, configConstraint] of Object.entries(configConstraints)) {
- // istanbul ignore if
- if (!versioning.isValid(configConstraint)) {
- continue;
- }
-
- const constraint = releaseConstraints[name];
- if (!is.nonEmptyArray(constraint)) {
- // A release with no constraints is OK
- continue;
- }
-
- const satisfiesConstraints = constraint.some(
- // If the constraint value is a subset of any release's constraints, then it's OK
- // fallback to release's constraint match if subset is not supported by versioning
- (releaseConstraint) =>
- !releaseConstraint ||
- (versioning.subset?.(configConstraint, releaseConstraint) ??
- versioning.matches(configConstraint, releaseConstraint))
- );
-
- if (!satisfiesConstraints) {
- filteredReleases.push(release.version);
- return null;
- }
- }
-
- return release;
- });
-
- if (filteredReleases.length) {
- const count = filteredReleases.length;
- const packageName = config.packageName;
- const releases = filteredReleases.join(', ');
- logger.debug(
- `Filtered ${count} releases for ${packageName} due to constraintsFiltering=strict: ${releases}`
- );
- }
-}
-
export async function getPkgReleases(
config: GetPkgReleasesConfig
): Promise<ReleaseResult | null> {
diff --git a/lib/modules/datasource/util.ts b/lib/modules/datasource/util.ts
new file mode 100644
index 0000000000..7293769a2b
--- /dev/null
+++ b/lib/modules/datasource/util.ts
@@ -0,0 +1,10 @@
+import is from '@sindresorhus/is';
+import type { HttpResponse } from '../../util/http/types';
+
+const JFROG_ARTIFACTORY_RES_HEADER = 'x-jfrog-version';
+
+export function isArtifactoryServer<T = unknown>(
+ res: HttpResponse<T> | undefined
+): boolean {
+ return is.string(res?.headers[JFROG_ARTIFACTORY_RES_HEADER]);
+}
diff --git a/lib/modules/datasource/common.spec.ts b/lib/modules/datasource/utils.spec.ts
similarity index 85%
rename from lib/modules/datasource/common.spec.ts
rename to lib/modules/datasource/utils.spec.ts
index c68448381d..20d8161e84 100644
--- a/lib/modules/datasource/common.spec.ts
+++ b/lib/modules/datasource/utils.spec.ts
@@ -1,7 +1,7 @@
import type { HttpResponse } from '../../util/http/types';
-import { isArtifactoryServer } from './common';
+import { isArtifactoryServer } from './util';
-describe('modules/datasource/common', () => {
+describe('modules/datasource/utils', () => {
it('is artifactory server invalid', () => {
const response: HttpResponse<string> = {
statusCode: 200,
diff --git a/lib/workers/repository/process/fetch.ts b/lib/workers/repository/process/fetch.ts
index e8208445af..4d11ec848f 100644
--- a/lib/workers/repository/process/fetch.ts
+++ b/lib/workers/repository/process/fetch.ts
@@ -3,10 +3,8 @@ import is from '@sindresorhus/is';
import { getManagerConfig, mergeChildConfig } from '../../../config';
import type { RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
-import {
- getDefaultConfig,
- getDefaultVersioning,
-} from '../../../modules/datasource';
+import { getDefaultConfig } from '../../../modules/datasource';
+import { getDefaultVersioning } from '../../../modules/datasource/common';
import type {
PackageDependency,
PackageFile,
diff --git a/lib/workers/repository/process/lookup/index.ts b/lib/workers/repository/process/lookup/index.ts
index be900de83d..b3c8298bd2 100644
--- a/lib/workers/repository/process/lookup/index.ts
+++ b/lib/workers/repository/process/lookup/index.ts
@@ -6,13 +6,15 @@ import { logger } from '../../../../logger';
import {
Release,
ReleaseResult,
- getDatasourceFor,
- getDefaultVersioning,
getDigest,
getPkgReleases,
isGetPkgReleasesConfig,
supportsDigests,
} from '../../../../modules/datasource';
+import {
+ getDatasourceFor,
+ getDefaultVersioning,
+} from '../../../../modules/datasource/common';
import { getRangeStrategy } from '../../../../modules/manager';
import * as allVersioning from '../../../../modules/versioning';
import { ExternalHostError } from '../../../../types/errors/external-host-error';
diff --git a/lib/workers/repository/process/vulnerabilities.ts b/lib/workers/repository/process/vulnerabilities.ts
index 9ff98a803d..15991b863f 100644
--- a/lib/workers/repository/process/vulnerabilities.ts
+++ b/lib/workers/repository/process/vulnerabilities.ts
@@ -6,7 +6,7 @@ import { parseCvssVector } from 'vuln-vects';
import { getManagerConfig, mergeChildConfig } from '../../../config';
import type { PackageRule, RenovateConfig } from '../../../config/types';
import { logger } from '../../../logger';
-import { getDefaultVersioning } from '../../../modules/datasource';
+import { getDefaultVersioning } from '../../../modules/datasource/common';
import type {
PackageDependency,
PackageFile,
--
GitLab