From 9fea985b950ec2a8bbac1eefc1e986af8016f4b0 Mon Sep 17 00:00:00 2001
From: jjcaballero <juan_fernandez@nuance.com>
Date: Fri, 1 Oct 2021 10:39:29 +0200
Subject: [PATCH] feat: create datasource for artifactory registry (#11602)
---
lib/datasource/api.ts | 2 +
.../__fixtures__/releases-as-files.html | 21 +++
.../__fixtures__/releases-as-folders.html | 21 +++
.../__snapshots__/index.spec.ts.snap | 80 +++++++++
lib/datasource/artifactory/common.ts | 1 +
lib/datasource/artifactory/index.spec.ts | 156 ++++++++++++++++++
lib/datasource/artifactory/index.ts | 113 +++++++++++++
lib/datasource/artifactory/readme.md | 5 +
lib/util/html.spec.ts | 18 ++
lib/util/html.ts | 8 +-
10 files changed, 423 insertions(+), 2 deletions(-)
create mode 100644 lib/datasource/artifactory/__fixtures__/releases-as-files.html
create mode 100644 lib/datasource/artifactory/__fixtures__/releases-as-folders.html
create mode 100644 lib/datasource/artifactory/__snapshots__/index.spec.ts.snap
create mode 100644 lib/datasource/artifactory/common.ts
create mode 100644 lib/datasource/artifactory/index.spec.ts
create mode 100644 lib/datasource/artifactory/index.ts
create mode 100644 lib/datasource/artifactory/readme.md
diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts
index 33631ce7be..3e45c789a1 100644
--- a/lib/datasource/api.ts
+++ b/lib/datasource/api.ts
@@ -1,4 +1,5 @@
import { AdoptiumJavaDatasource } from './adoptium-java';
+import { ArtifactoryDatasource } from './artifactory';
import { BitBucketTagsDatasource } from './bitbucket-tags';
import { CdnJsDatasource } from './cdnjs';
import { ClojureDatasource } from './clojure';
@@ -40,6 +41,7 @@ const api = new Map<string, DatasourceApi>();
export default api;
api.set(AdoptiumJavaDatasource.id, new AdoptiumJavaDatasource());
+api.set(ArtifactoryDatasource.id, new ArtifactoryDatasource());
api.set('bitbucket-tags', new BitBucketTagsDatasource());
api.set('cdnjs', new CdnJsDatasource());
api.set('clojure', new ClojureDatasource());
diff --git a/lib/datasource/artifactory/__fixtures__/releases-as-files.html b/lib/datasource/artifactory/__fixtures__/releases-as-files.html
new file mode 100644
index 0000000000..2bdde58399
--- /dev/null
+++ b/lib/datasource/artifactory/__fixtures__/releases-as-files.html
@@ -0,0 +1,21 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+ <head>
+ <meta name="robots" content="noindex" />
+ <title>Repository Title</title>
+ </head>
+
+ <body>
+ <h1>Index</h1>
+ <pre>Name Last modified Size</pre><hr/>
+ <pre>
+ <a href="..">..</a>
+ <a href="1.0.0">1.0.0</a> 21-Jul-2021 20:08 -
+ <a href="1.0.1">1.0.1</a> 23-Aug-2021 20:03 -
+ <a href="1.0.2">1.0.2</a> 21-Jul-2021 20:09 -
+ <a href="1.0.3">1.0.3</a> 06-Feb-2021 09:54 -
+ </pre>
+ <hr/>
+ <address style="font-size:small;">Artifactory Port 8080</address>
+ </body>
+</html>
diff --git a/lib/datasource/artifactory/__fixtures__/releases-as-folders.html b/lib/datasource/artifactory/__fixtures__/releases-as-folders.html
new file mode 100644
index 0000000000..16b86d87d6
--- /dev/null
+++ b/lib/datasource/artifactory/__fixtures__/releases-as-folders.html
@@ -0,0 +1,21 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+ <head>
+ <meta name="robots" content="noindex" />
+ <title>Repository Title</title>
+ </head>
+
+ <body>
+ <h1>Index</h1>
+ <pre>Name Last modified Size</pre><hr/>
+ <pre>
+ <a href="../">../</a>
+ <a href="1.0.0/">1.0.0/</a> 21-Jul-2021 20:08 -
+ <a href="1.0.1/">1.0.1/</a> 23-Aug-2021 20:03 -
+ <a href="1.0.2/">1.0.2/</a> 21-Jul-2021 20:09 -
+ <a href="1.0.3/">1.0.3/</a> 06-Feb-2021 09:54 -
+ </pre>
+ <hr/>
+ <address style="font-size:small;">Artifactory Port 8080</address>
+ </body>
+</html>
diff --git a/lib/datasource/artifactory/__snapshots__/index.spec.ts.snap b/lib/datasource/artifactory/__snapshots__/index.spec.ts.snap
new file mode 100644
index 0000000000..9c3eaf66a2
--- /dev/null
+++ b/lib/datasource/artifactory/__snapshots__/index.spec.ts.snap
@@ -0,0 +1,80 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`datasource/artifactory/index getReleases parses real data (files): without slash at the end 1`] = `
+Object {
+ "registryUrl": "https://jfrog.company.com/artifactory",
+ "releases": Array [
+ Object {
+ "releaseTimestamp": "2021-07-21T20:08:00.000Z",
+ "version": "1.0.0",
+ },
+ Object {
+ "releaseTimestamp": "2021-08-23T20:03:00.000Z",
+ "version": "1.0.1",
+ },
+ Object {
+ "releaseTimestamp": "2021-07-21T20:09:00.000Z",
+ "version": "1.0.2",
+ },
+ Object {
+ "releaseTimestamp": "2021-02-06T09:54:00.000Z",
+ "version": "1.0.3",
+ },
+ ],
+}
+`;
+
+exports[`datasource/artifactory/index getReleases parses real data (folders): with slash at the end 1`] = `
+Object {
+ "registryUrl": "https://jfrog.company.com/artifactory",
+ "releases": Array [
+ Object {
+ "releaseTimestamp": "2021-07-21T20:08:00.000Z",
+ "version": "1.0.0",
+ },
+ Object {
+ "releaseTimestamp": "2021-08-23T20:03:00.000Z",
+ "version": "1.0.1",
+ },
+ Object {
+ "releaseTimestamp": "2021-07-21T20:09:00.000Z",
+ "version": "1.0.2",
+ },
+ Object {
+ "releaseTimestamp": "2021-02-06T09:54:00.000Z",
+ "version": "1.0.3",
+ },
+ ],
+}
+`;
+
+exports[`datasource/artifactory/index getReleases parses real data (merge strategy with 2 registries) 1`] = `
+Object {
+ "releases": Array [
+ Object {
+ "registryUrl": "https://jfrog.company.com/artifactory",
+ "releaseTimestamp": "2021-07-21T20:08:00.000Z",
+ "version": "1.0.0",
+ },
+ Object {
+ "registryUrl": "https://jfrog.company.com/artifactory",
+ "releaseTimestamp": "2021-08-23T20:03:00.000Z",
+ "version": "1.0.1",
+ },
+ Object {
+ "registryUrl": "https://jfrog.company.com/artifactory",
+ "releaseTimestamp": "2021-07-21T20:09:00.000Z",
+ "version": "1.0.2",
+ },
+ Object {
+ "registryUrl": "https://jfrog.company.com/artifactory",
+ "releaseTimestamp": "2021-02-06T09:54:00.000Z",
+ "version": "1.0.3",
+ },
+ Object {
+ "registryUrl": "https://jfrog.company.com/artifactory/production",
+ "version": "1.3.0",
+ },
+ ],
+}
+`;
diff --git a/lib/datasource/artifactory/common.ts b/lib/datasource/artifactory/common.ts
new file mode 100644
index 0000000000..7dd522f507
--- /dev/null
+++ b/lib/datasource/artifactory/common.ts
@@ -0,0 +1 @@
+export const datasource = 'artifactory';
diff --git a/lib/datasource/artifactory/index.spec.ts b/lib/datasource/artifactory/index.spec.ts
new file mode 100644
index 0000000000..ff16119030
--- /dev/null
+++ b/lib/datasource/artifactory/index.spec.ts
@@ -0,0 +1,156 @@
+import { getPkgReleases } from '..';
+import * as httpMock from '../../../test/http-mock';
+import { loadFixture } from '../../../test/util';
+import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
+import { logger } from '../../logger';
+import { joinUrlParts } from '../../util/url';
+import { ArtifactoryDatasource } from '.';
+
+const datasource = ArtifactoryDatasource.id;
+
+const testRegistryUrl = 'https://jfrog.company.com/artifactory';
+const testLookupName = 'project';
+const testConfig = {
+ registryUrls: [testRegistryUrl],
+ depName: testLookupName,
+};
+const fixtureReleasesAsFolders = loadFixture('releases-as-folders.html');
+const fixtureReleasesAsFiles = loadFixture('releases-as-files.html');
+
+function getPath(folder: string): string {
+ return `/${folder}`;
+}
+
+describe('datasource/artifactory/index', () => {
+ beforeEach(() => {
+ jest.resetAllMocks();
+ });
+
+ describe('getReleases', () => {
+ it('parses real data (folders): with slash at the end', async () => {
+ httpMock
+ .scope(testRegistryUrl)
+ .get(getPath(testLookupName))
+ .reply(200, fixtureReleasesAsFolders);
+ const res = await getPkgReleases({
+ ...testConfig,
+ datasource,
+ lookupName: testLookupName,
+ });
+ expect(res.releases).toHaveLength(4);
+ expect(res).toMatchSnapshot({
+ registryUrl: 'https://jfrog.company.com/artifactory',
+ });
+ });
+
+ it('parses real data (files): without slash at the end', async () => {
+ httpMock
+ .scope(testRegistryUrl)
+ .get(getPath(testLookupName))
+ .reply(200, fixtureReleasesAsFiles);
+ const res = await getPkgReleases({
+ ...testConfig,
+ datasource,
+ lookupName: testLookupName,
+ });
+ expect(res.releases).toHaveLength(4);
+ expect(res).toMatchSnapshot({
+ registryUrl: 'https://jfrog.company.com/artifactory',
+ });
+ });
+
+ it('parses real data (merge strategy with 2 registries)', async () => {
+ const secondRegistryUrl: string = joinUrlParts(
+ testRegistryUrl,
+ 'production'
+ );
+ httpMock
+ .scope(testRegistryUrl)
+ .get(getPath(testLookupName))
+ .reply(200, fixtureReleasesAsFiles);
+ httpMock
+ .scope(secondRegistryUrl)
+ .get(getPath(testLookupName))
+ .reply(200, '<html>\n<h1>Header</h1>\n<a>1.3.0</a>\n<hmtl/>');
+ const res = await getPkgReleases({
+ registryUrls: [testRegistryUrl, secondRegistryUrl],
+ depName: testLookupName,
+ datasource,
+ lookupName: testLookupName,
+ });
+ expect(res.releases).toHaveLength(5);
+ expect(res).toMatchSnapshot();
+ });
+
+ it('returns null without registryUrl + warning', async () => {
+ const res = await getPkgReleases({
+ datasource,
+ depName: testLookupName,
+ lookupName: testLookupName,
+ });
+ expect(logger.warn).toHaveBeenCalledTimes(1);
+ expect(logger.warn).toHaveBeenCalledWith(
+ { lookupName: 'project' },
+ 'artifactory datasource requires custom registryUrl. Skipping datasource'
+ );
+ expect(res).toBeNull();
+ });
+
+ it('returns null for empty 200 OK', async () => {
+ httpMock
+ .scope(testRegistryUrl)
+ .get(getPath(testLookupName))
+ .reply(200, '<html>\n<h1>Header wo. nodes</h1>\n<hmtl/>');
+ expect(
+ await getPkgReleases({
+ ...testConfig,
+ datasource,
+ lookupName: testLookupName,
+ })
+ ).toBeNull();
+ });
+
+ it('404 returns null', async () => {
+ httpMock.scope(testRegistryUrl).get(getPath(testLookupName)).reply(404);
+ expect(
+ await getPkgReleases({
+ ...testConfig,
+ datasource,
+ lookupName: testLookupName,
+ })
+ ).toBeNull();
+ expect(logger.warn).toHaveBeenCalledTimes(1);
+ expect(logger.warn).toHaveBeenCalledWith(
+ {
+ lookupName: 'project',
+ registryUrl: 'https://jfrog.company.com/artifactory',
+ },
+ 'artifactory: `Not Found` error'
+ );
+ });
+
+ it('throws for error diff than 404', async () => {
+ httpMock.scope(testRegistryUrl).get(getPath(testLookupName)).reply(502);
+ await expect(
+ getPkgReleases({
+ ...testConfig,
+ datasource,
+ lookupName: testLookupName,
+ })
+ ).rejects.toThrow(EXTERNAL_HOST_ERROR);
+ });
+
+ it('throws no Http error', async () => {
+ httpMock
+ .scope(testRegistryUrl)
+ .get(getPath(testLookupName))
+ .replyWithError('unknown error');
+ const res = await getPkgReleases({
+ ...testConfig,
+ datasource,
+ lookupName: testLookupName,
+ });
+ expect(res).toBeNull();
+ });
+ });
+});
diff --git a/lib/datasource/artifactory/index.ts b/lib/datasource/artifactory/index.ts
new file mode 100644
index 0000000000..2c462be9c9
--- /dev/null
+++ b/lib/datasource/artifactory/index.ts
@@ -0,0 +1,113 @@
+import { logger } from '../../logger';
+import { cache } from '../../util/cache/package/decorator';
+import { parse } from '../../util/html';
+import { HttpError } from '../../util/http/types';
+import { joinUrlParts } from '../../util/url';
+import { Datasource } from '../datasource';
+import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
+import { datasource } from './common';
+
+export class ArtifactoryDatasource extends Datasource {
+ static readonly id = datasource;
+
+ constructor() {
+ super(datasource);
+ }
+
+ override readonly customRegistrySupport = true;
+
+ override readonly caching = true;
+
+ override readonly registryStrategy = 'merge';
+
+ @cache({
+ namespace: `datasource-${datasource}`,
+ key: ({ registryUrl, lookupName }: GetReleasesConfig) =>
+ `${registryUrl}:${lookupName}`,
+ })
+ async getReleases({
+ lookupName,
+ registryUrl,
+ }: GetReleasesConfig): Promise<ReleaseResult | null> {
+ if (!registryUrl) {
+ logger.warn(
+ { lookupName },
+ 'artifactory datasource requires custom registryUrl. Skipping datasource'
+ );
+ return null;
+ }
+
+ const url = joinUrlParts(registryUrl, lookupName);
+
+ const result: ReleaseResult = {
+ releases: [],
+ };
+ try {
+ const response = await this.http.get(url);
+ const body = parse(response.body, {
+ blockTextElements: {
+ script: true,
+ noscript: true,
+ style: true,
+ },
+ });
+ const nodes = body.querySelectorAll('a');
+
+ nodes
+ .filter(
+ // filter out hyperlink to navigate to parent folder
+ (node) => node.innerHTML !== '../' && node.innerHTML !== '..'
+ )
+ .forEach(
+ // extract version and published time for each node
+ (node) => {
+ const version: string =
+ node.innerHTML.slice(-1) === '/'
+ ? node.innerHTML.slice(0, -1)
+ : node.innerHTML;
+
+ const published = ArtifactoryDatasource.parseReleaseTimestamp(
+ node.nextSibling?.text
+ );
+
+ const thisRelease: Release = {
+ version,
+ releaseTimestamp: published,
+ };
+
+ result.releases.push(thisRelease);
+ }
+ );
+
+ if (result.releases.length) {
+ logger.trace(
+ { registryUrl, lookupName, versions: result.releases.length },
+ 'artifactory: Found versions'
+ );
+ } else {
+ logger.trace(
+ { registryUrl, lookupName },
+ 'artifactory: No versions found'
+ );
+ }
+ } catch (err) {
+ // istanbul ignore else: not testable with nock
+ if (err instanceof HttpError) {
+ if (err.response?.statusCode === 404) {
+ logger.warn(
+ { registryUrl, lookupName },
+ 'artifactory: `Not Found` error'
+ );
+ return null;
+ }
+ }
+ this.handleGenericErrors(err);
+ }
+
+ return result.releases.length ? result : null;
+ }
+
+ private static parseReleaseTimestamp(rawText: string): string {
+ return rawText.trim().replace(/ ?-$/, '');
+ }
+}
diff --git a/lib/datasource/artifactory/readme.md b/lib/datasource/artifactory/readme.md
new file mode 100644
index 0000000000..e8e40c6fe4
--- /dev/null
+++ b/lib/datasource/artifactory/readme.md
@@ -0,0 +1,5 @@
+Artifactory is the recommended registry for Conan packages.
+
+This datasource returns releases from given custom `registryUrl`(s).
+
+The target URL is composed by the `registryUrl` and the `lookupName`, which defaults to `depName` when `lookupName` is not defined.
diff --git a/lib/util/html.spec.ts b/lib/util/html.spec.ts
index c4727d8234..a78a70ae96 100644
--- a/lib/util/html.spec.ts
+++ b/lib/util/html.spec.ts
@@ -13,4 +13,22 @@ describe('util/html', () => {
const body = parse('');
expect(body.childNodes).toHaveLength(0);
});
+
+ it('parses HTML: PRE block hides child nodes', () => {
+ const body = parse('<div>Hello, world!</div>\n<pre><a>node A</a></pre>');
+ const childNodesA = body.querySelectorAll('a');
+ expect(childNodesA).toHaveLength(0);
+ });
+
+ it('parses HTML: use additional options to discover child nodes on PRE blocks', () => {
+ const body = parse('<div>Hello, world!</div>\n<pre><a>node A</a></pre>', {
+ blockTextElements: {},
+ });
+ const childNodesA = body.querySelectorAll('a');
+ expect(childNodesA).toHaveLength(1);
+ const div = childNodesA[0];
+ expect(div.tagName).toBe('A');
+ expect(div.textContent).toBe('node A');
+ expect(div instanceof HTMLElement).toBe(true);
+ });
});
diff --git a/lib/util/html.ts b/lib/util/html.ts
index 26ac555833..d0ee29fd8b 100644
--- a/lib/util/html.ts
+++ b/lib/util/html.ts
@@ -1,7 +1,11 @@
-import { HTMLElement, parse as _parse } from 'node-html-parser';
+import { HTMLElement, Options, parse as _parse } from 'node-html-parser';
export { HTMLElement };
-export function parse(html: string): HTMLElement {
+export function parse(html: string, config?: Partial<Options>): HTMLElement {
+ if (typeof config !== 'undefined') {
+ return _parse(html, config);
+ }
+
return _parse(html);
}
--
GitLab