From f6112172891f378bb0d169aed5fb3f99b771a442 Mon Sep 17 00:00:00 2001
From: MartijnLeijssen-TomTom
<46988880+MartijnLeijssen-TomTom@users.noreply.github.com>
Date: Thu, 15 Jun 2023 17:56:54 +0200
Subject: [PATCH] feat(datasource/conan): add sourceUrl when artifactory is
used (#22656)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
---
lib/modules/datasource/common.spec.ts | 22 ++++
lib/modules/datasource/common.ts | 9 ++
lib/modules/datasource/conan/index.spec.ts | 129 +++++++++++++++++++
lib/modules/datasource/conan/index.ts | 55 +++++++-
lib/modules/datasource/conan/types.ts | 14 ++
lib/modules/datasource/docker/artifactory.ts | 10 --
lib/modules/datasource/docker/index.ts | 2 +-
7 files changed, 229 insertions(+), 12 deletions(-)
create mode 100644 lib/modules/datasource/common.spec.ts
delete mode 100644 lib/modules/datasource/docker/artifactory.ts
diff --git a/lib/modules/datasource/common.spec.ts b/lib/modules/datasource/common.spec.ts
new file mode 100644
index 0000000000..c68448381d
--- /dev/null
+++ b/lib/modules/datasource/common.spec.ts
@@ -0,0 +1,22 @@
+import type { HttpResponse } from '../../util/http/types';
+import { isArtifactoryServer } from './common';
+
+describe('modules/datasource/common', () => {
+ it('is artifactory server invalid', () => {
+ const response: HttpResponse<string> = {
+ statusCode: 200,
+ body: 'test',
+ headers: { 'invalid-header': 'version' },
+ };
+ expect(isArtifactoryServer(response)).toBeFalse();
+ });
+
+ it('is artifactory server valid', () => {
+ const response: HttpResponse<string> = {
+ statusCode: 200,
+ body: 'test',
+ headers: { 'x-jfrog-version': 'version' },
+ };
+ expect(isArtifactoryServer(response)).toBeTrue();
+ });
+});
diff --git a/lib/modules/datasource/common.ts b/lib/modules/datasource/common.ts
index f8052e547b..6c7a05f47d 100644
--- a/lib/modules/datasource/common.ts
+++ b/lib/modules/datasource/common.ts
@@ -1,4 +1,5 @@
import is from '@sindresorhus/is';
+import type { HttpResponse } from '../../util/http/types';
import type { GetPkgReleasesConfig } from './types';
export function isGetPkgReleasesConfig(
@@ -13,3 +14,11 @@ export function isGetPkgReleasesConfig(
)
);
}
+
+const JFROG_ARTIFACTORY_RES_HEADER = 'x-jfrog-version';
+
+export function isArtifactoryServer<T = unknown>(
+ res: HttpResponse<T> | undefined
+): boolean {
+ return is.string(res?.headers[JFROG_ARTIFACTORY_RES_HEADER]);
+}
diff --git a/lib/modules/datasource/conan/index.spec.ts b/lib/modules/datasource/conan/index.spec.ts
index 84c1a98c2f..9128a16d66 100644
--- a/lib/modules/datasource/conan/index.spec.ts
+++ b/lib/modules/datasource/conan/index.spec.ts
@@ -254,5 +254,134 @@ describe('modules/datasource/conan/index', () => {
})
).toBeNull();
});
+
+ it('artifactory sourceurl', async () => {
+ httpMock
+ .scope('https://fake.artifactory.com/artifactory/api/conan/test-repo/')
+ .get('/v2/conans/search?q=arti')
+ .reply(
+ 200,
+ { results: ['arti/1.0.0@_/_', 'arti/1.1.1@_/_'] },
+ {
+ 'x-jfrog-version': 'latest',
+ }
+ );
+ httpMock
+ .scope('https://fake.artifactory.com/artifactory/api/conan/test-repo/')
+ .get('/v2/conans/arti/1.1.1/_/_/latest')
+ .reply(200, {
+ revision: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+ time: '2032-06-23T00:00:00.000+0000',
+ });
+ httpMock
+ .scope(
+ 'https://fake.artifactory.com/artifactory/api/storage/test-repo/'
+ )
+ .get(
+ '/_/arti/1.1.1/_/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/export/conanfile.py?properties=conan.package.url'
+ )
+ .reply(200, {
+ properties: {
+ 'conan.package.url': 'https://fake.conan.url.com',
+ },
+ });
+
+ config.registryUrls = [
+ 'https://fake.artifactory.com/artifactory/api/conan/test-repo',
+ ];
+ config.packageName = 'arti';
+ expect(
+ await getPkgReleases({
+ ...config,
+ packageName: 'arti/1.1@_/_',
+ })
+ ).toEqual({
+ registryUrl:
+ 'https://fake.artifactory.com/artifactory/api/conan/test-repo',
+ releases: [
+ {
+ version: '1.0.0',
+ },
+ {
+ version: '1.1.1',
+ },
+ ],
+ });
+ });
+
+ it('artifactory header without api', async () => {
+ httpMock
+ .scope('https://fake.artifactory.com')
+ .get('/v2/conans/search?q=arti')
+ .reply(
+ 200,
+ { results: ['arti/1.0.0@_/_', 'arti/1.1.1@_/_'] },
+ {
+ 'x-jfrog-version': 'latest',
+ }
+ );
+ config.registryUrls = ['https://fake.artifactory.com'];
+ config.packageName = 'arti';
+ expect(
+ await getPkgReleases({
+ ...config,
+ packageName: 'arti/1.1@_/_',
+ })
+ ).toEqual({
+ registryUrl: 'https://fake.artifactory.com',
+ releases: [
+ {
+ version: '1.0.0',
+ },
+ {
+ version: '1.1.1',
+ },
+ ],
+ });
+ });
+
+ it('artifactory invalid version', async () => {
+ httpMock
+ .scope('https://fake.artifactory.com/artifactory/api/conan/test-repo')
+ .get('/v2/conans/search?q=arti')
+ .reply(
+ 200,
+ {
+ results: ['arti/invalid_version@_/_'],
+ },
+ { 'x-jfrog-version': 'latest' }
+ );
+ config.registryUrls = [
+ 'https://fake.artifactory.com/artifactory/api/conan/test-repo',
+ ];
+ config.packageName = 'arti';
+ expect(
+ await getPkgReleases({
+ ...config,
+ packageName: 'arti/1.1@_/_',
+ })
+ ).toEqual({
+ registryUrl:
+ 'https://fake.artifactory.com/artifactory/api/conan/test-repo',
+ releases: [],
+ });
+ });
+
+ it('non artifactory header', async () => {
+ httpMock
+ .scope('https://fake.artifactory.com/artifactory/api/conan/test-repo')
+ .get('/v2/conans/search?q=arti')
+ .reply(200, {}, { 'x-jfrog-invalid-header': 'latest' });
+ config.registryUrls = [
+ 'https://fake.artifactory.com/artifactory/api/conan/test-repo',
+ ];
+ config.packageName = 'arti';
+ expect(
+ await getPkgReleases({
+ ...config,
+ packageName: 'arti/1.1@_/_',
+ })
+ ).toBeNull();
+ });
});
});
diff --git a/lib/modules/datasource/conan/index.ts b/lib/modules/datasource/conan/index.ts
index c77dc6e812..8fa550a0ae 100644
--- a/lib/modules/datasource/conan/index.ts
+++ b/lib/modules/datasource/conan/index.ts
@@ -4,6 +4,8 @@ import { logger } from '../../../logger';
import { cache } from '../../../util/cache/package/decorator';
import { GithubHttp } from '../../../util/http/github';
import { ensureTrailingSlash, joinUrlParts } from '../../../util/url';
+import * as allVersioning from '../../versioning';
+import { isArtifactoryServer } from '../common';
import { Datasource } from '../datasource';
import type {
DigestConfig,
@@ -17,7 +19,13 @@ import {
defaultRegistryUrl,
getConanPackage,
} from './common';
-import type { ConanJSON, ConanRevisionsJSON, ConanYAML } from './types';
+import type {
+ ConanJSON,
+ ConanProperties,
+ ConanRevisionJSON,
+ ConanRevisionsJSON,
+ ConanYAML,
+} from './types';
export class ConanDatasource extends Datasource {
static readonly id = datasource;
@@ -117,6 +125,7 @@ export class ConanDatasource extends Datasource {
{ packageName, registryUrl },
'Looking up conan api dependency'
);
+
if (registryUrl) {
const url = ensureTrailingSlash(registryUrl);
const lookupUrl = joinUrlParts(
@@ -143,6 +152,50 @@ export class ConanDatasource extends Datasource {
}
}
}
+
+ if (isArtifactoryServer(rep)) {
+ const conanApiRegexp =
+ /(?<host>.*)\/artifactory\/api\/conan\/(?<repo>[^/]+)/;
+ const groups = url.match(conanApiRegexp)?.groups;
+ if (!groups) {
+ return dep;
+ }
+ const semver = allVersioning.get('semver');
+
+ const sortedReleases = dep.releases
+ .filter((release) => semver.isVersion(release.version))
+ .sort((a, b) => semver.sortVersions(a.version, b.version));
+
+ const latestVersion = sortedReleases.at(-1)?.version;
+
+ if (!latestVersion) {
+ return dep;
+ }
+ logger.debug(
+ `Conan package ${packageName} has latest version ${latestVersion}`
+ );
+
+ const latestRevisionUrl = joinUrlParts(
+ url,
+ `v2/conans/${conanPackage.conanName}/${latestVersion}/${conanPackage.userAndChannel}/latest`
+ );
+ const revResp = await this.http.getJson<ConanRevisionJSON>(
+ latestRevisionUrl
+ );
+ const packageRev = revResp.body.revision;
+
+ const [user, channel] = conanPackage.userAndChannel.split('/');
+ const packageUrl = joinUrlParts(
+ `${groups.host}/artifactory/api/storage/${groups.repo}`,
+ `${user}/${conanPackage.conanName}/${latestVersion}/${channel}/${packageRev}/export/conanfile.py?properties=conan.package.url`
+ );
+ const packageUrlResp = await this.http.getJson<ConanProperties>(
+ packageUrl
+ );
+ const conanPackageUrl =
+ packageUrlResp.body.properties['conan.package.url'][0];
+ dep.sourceUrl = conanPackageUrl;
+ }
return dep;
}
} catch (err) {
diff --git a/lib/modules/datasource/conan/types.ts b/lib/modules/datasource/conan/types.ts
index 854935c11c..bb65481f50 100644
--- a/lib/modules/datasource/conan/types.ts
+++ b/lib/modules/datasource/conan/types.ts
@@ -19,3 +19,17 @@ export interface ConanPackage {
conanName: string;
userAndChannel: string;
}
+
+export interface ConanRecipeProperties {
+ 'conan.package.channel': string[];
+ 'conan.package.license': string[];
+ 'conan.package.name': string[];
+ 'conan.package.url': string[];
+ 'conan.package.user': string[];
+ 'conan.package.version': string[];
+}
+
+export interface ConanProperties {
+ properties: ConanRecipeProperties;
+ uri: string;
+}
diff --git a/lib/modules/datasource/docker/artifactory.ts b/lib/modules/datasource/docker/artifactory.ts
deleted file mode 100644
index e5c09df01c..0000000000
--- a/lib/modules/datasource/docker/artifactory.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import is from '@sindresorhus/is';
-import type { HttpResponse } from '../../../util/http/types';
-
-const JFROG_ARTIFACTORY_RES_HEADER = 'x-jfrog-version';
-
-export function isArtifactoryServer<T = unknown>(
- res: HttpResponse<T> | undefined
-): boolean {
- return is.string(res?.headers[JFROG_ARTIFACTORY_RES_HEADER]);
-}
diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts
index ca5f5d728a..6bcd11ddaf 100644
--- a/lib/modules/datasource/docker/index.ts
+++ b/lib/modules/datasource/docker/index.ts
@@ -14,9 +14,9 @@ import {
parseLinkHeader,
} from '../../../util/url';
import { id as dockerVersioningId } from '../../versioning/docker';
+import { isArtifactoryServer } from '../common';
import { Datasource } from '../datasource';
import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
-import { isArtifactoryServer } from './artifactory';
import {
DOCKER_HUB,
dockerDatasourceId,
--
GitLab