From c8c8684ea3d957e3b843ec4a620fc54d9c2fb4b8 Mon Sep 17 00:00:00 2001
From: Michael Kriese <michael.kriese@visualon.de>
Date: Mon, 30 Aug 2021 18:28:32 +0200
Subject: [PATCH] fix(datasource): trim trailing slash in registry url (#11392)
Co-authored-by: Rhys Arkins <rhys@arkins.net>
---
lib/datasource/docker/index.spec.ts | 26 ++++++++++
.../__snapshots__/index.spec.ts.snap | 50 ++++++++-----------
lib/datasource/gitlab-tags/index.spec.ts | 31 +++++++++++-
lib/datasource/gitlab-tags/index.ts | 14 ++++--
lib/datasource/index.ts | 3 +-
.../__snapshots__/index.spec.ts.snap | 2 +-
.../nuget/__snapshots__/index.spec.ts.snap | 8 +--
.../pypi/__snapshots__/index.spec.ts.snap | 12 ++---
.../repology/__snapshots__/index.spec.ts.snap | 12 ++---
lib/datasource/repology/index.ts | 8 +--
.../rubygems/__snapshots__/index.spec.ts.snap | 2 +-
lib/util/url.spec.ts | 17 +++++++
lib/util/url.ts | 4 ++
13 files changed, 130 insertions(+), 59 deletions(-)
diff --git a/lib/datasource/docker/index.spec.ts b/lib/datasource/docker/index.spec.ts
index c88d086521..038fde1a0b 100644
--- a/lib/datasource/docker/index.spec.ts
+++ b/lib/datasource/docker/index.spec.ts
@@ -607,6 +607,32 @@ describe('datasource/docker/index', () => {
expect(httpMock.getTrace()).toMatchSnapshot();
});
+ it('strips trailing slash from registry', async () => {
+ httpMock
+ .scope(baseUrl)
+ .get('/')
+ .reply(401, '', {
+ 'www-authenticate':
+ 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:my/node:pull "',
+ })
+ .get('/my/node/tags/list?n=10000')
+ .reply(200, { tags: ['1.0.0'] }, {})
+ .get('/')
+ .reply(200)
+ .get('/my/node/manifests/1.0.0')
+ .reply(200);
+ httpMock
+ .scope(authUrl)
+ .get('/token?service=registry.docker.io&scope=repository:my/node:pull')
+ .reply(200, { token: 'some-token ' });
+ const res = await getPkgReleases({
+ datasource: id,
+ depName: 'my/node',
+ registryUrls: ['https://index.docker.io/'],
+ });
+ expect(res?.releases).toHaveLength(1);
+ });
+
it('returns null if no auth', async () => {
hostRules.find.mockReturnValue({});
httpMock.scope(baseUrl).get('/').reply(401, undefined, {
diff --git a/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap b/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap
index 4ab77e4ae0..4ab20c12a0 100644
--- a/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap
@@ -2,7 +2,7 @@
exports[`datasource/gitlab-tags/index getReleases returns tags from custom registry 1`] = `
Object {
- "registryUrl": "https://gitlab.company.com/api/v4/",
+ "registryUrl": "https://gitlab.company.com/api/v4",
"releases": Array [
Object {
"gitRef": "v1.0.0",
@@ -18,23 +18,30 @@ Object {
"version": "v1.1.1",
},
],
- "sourceUrl": "https://gitlab.company.com/api/v4/some/dep2",
+ "sourceUrl": "https://gitlab.company.com/some/dep2",
}
`;
-exports[`datasource/gitlab-tags/index getReleases returns tags from custom registry 2`] = `
-Array [
- Object {
- "headers": Object {
- "accept": "application/json",
- "accept-encoding": "gzip, deflate, br",
- "host": "gitlab.company.com",
- "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
+exports[`datasource/gitlab-tags/index getReleases returns tags from custom registry in sub path 1`] = `
+Object {
+ "registryUrl": "https://my.company.com/gitlab",
+ "releases": Array [
+ Object {
+ "gitRef": "v1.0.0",
+ "releaseTimestamp": "2020-03-04T18:01:37.000Z",
+ "version": "v1.0.0",
+ },
+ Object {
+ "gitRef": "v1.1.0",
+ "version": "v1.1.0",
},
- "method": "GET",
- "url": "https://gitlab.company.com/api/v4/projects/some%2Fdep2/repository/tags?per_page=100",
- },
-]
+ Object {
+ "gitRef": "v1.1.1",
+ "version": "v1.1.1",
+ },
+ ],
+ "sourceUrl": "https://my.company.com/gitlab/some/dep2",
+}
`;
exports[`datasource/gitlab-tags/index getReleases returns tags with default registry 1`] = `
@@ -53,18 +60,3 @@ Object {
"sourceUrl": "https://gitlab.com/some/dep2",
}
`;
-
-exports[`datasource/gitlab-tags/index getReleases returns tags with default registry 2`] = `
-Array [
- Object {
- "headers": Object {
- "accept": "application/json",
- "accept-encoding": "gzip, deflate, br",
- "host": "gitlab.com",
- "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)",
- },
- "method": "GET",
- "url": "https://gitlab.com/api/v4/projects/some%2Fdep2/repository/tags?per_page=100",
- },
-]
-`;
diff --git a/lib/datasource/gitlab-tags/index.spec.ts b/lib/datasource/gitlab-tags/index.spec.ts
index 76ea7a5933..0f468cf8a7 100644
--- a/lib/datasource/gitlab-tags/index.spec.ts
+++ b/lib/datasource/gitlab-tags/index.spec.ts
@@ -31,7 +31,35 @@ describe('datasource/gitlab-tags/index', () => {
});
expect(res).toMatchSnapshot();
expect(res.releases).toHaveLength(3);
- expect(httpMock.getTrace()).toMatchSnapshot();
+ });
+
+ it('returns tags from custom registry in sub path', async () => {
+ const body = [
+ {
+ name: 'v1.0.0',
+ commit: {
+ created_at: '2020-03-04T12:01:37.000-06:00',
+ },
+ },
+ {
+ name: 'v1.1.0',
+ commit: {},
+ },
+ {
+ name: 'v1.1.1',
+ },
+ ];
+ httpMock
+ .scope('https://my.company.com/gitlab')
+ .get('/api/v4/projects/some%2Fdep2/repository/tags?per_page=100')
+ .reply(200, body);
+ const res = await getPkgReleases({
+ datasource,
+ registryUrls: ['https://my.company.com/gitlab'],
+ depName: 'some/dep2',
+ });
+ expect(res).toMatchSnapshot();
+ expect(res?.releases).toHaveLength(3);
});
it('returns tags with default registry', async () => {
@@ -46,7 +74,6 @@ describe('datasource/gitlab-tags/index', () => {
});
expect(res).toMatchSnapshot();
expect(res.releases).toHaveLength(2);
- expect(httpMock.getTrace()).toMatchSnapshot();
});
});
});
diff --git a/lib/datasource/gitlab-tags/index.ts b/lib/datasource/gitlab-tags/index.ts
index 6c93462dc3..bf3355c07b 100644
--- a/lib/datasource/gitlab-tags/index.ts
+++ b/lib/datasource/gitlab-tags/index.ts
@@ -1,6 +1,6 @@
-import URL from 'url';
import * as packageCache from '../../util/cache/package';
import { GitlabHttp } from '../../util/http/gitlab';
+import { joinUrlParts } from '../../util/url';
import type { GetReleasesConfig, ReleaseResult } from '../types';
import type { GitlabTag } from './types';
@@ -19,9 +19,11 @@ function getCacheKey(depHost: string, repo: string): string {
}
export async function getReleases({
- registryUrl: depHost,
+ registryUrl,
lookupName: repo,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
+ const depHost = registryUrl.replace(/\/api\/v4$/, '');
+
const cachedResult = await packageCache.get<ReleaseResult>(
cacheNamespace,
getCacheKey(depHost, repo)
@@ -34,9 +36,11 @@ export async function getReleases({
const urlEncodedRepo = encodeURIComponent(repo);
// tag
- const url = URL.resolve(
+ const url = joinUrlParts(
depHost,
- `/api/v4/projects/${urlEncodedRepo}/repository/tags?per_page=100`
+ `api/v4/projects`,
+ urlEncodedRepo,
+ `repository/tags?per_page=100`
);
const gitlabTags = (
@@ -46,7 +50,7 @@ export async function getReleases({
).body;
const dependency: ReleaseResult = {
- sourceUrl: URL.resolve(depHost, repo),
+ sourceUrl: joinUrlParts(depHost, repo),
releases: null,
};
dependency.releases = gitlabTags.map(({ name, commit }) => ({
diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts
index 1b12c8b5b8..1d914e800d 100644
--- a/lib/datasource/index.ts
+++ b/lib/datasource/index.ts
@@ -7,6 +7,7 @@ import * as memCache from '../util/cache/memory';
import * as packageCache from '../util/cache/package';
import { clone } from '../util/clone';
import { regEx } from '../util/regex';
+import { trimTrailingSlash } from '../util/url';
import * as allVersioning from '../versioning';
import datasources from './api';
import { addMetaData } from './metadata';
@@ -200,7 +201,7 @@ function resolveRegistryUrls(
} else {
registryUrls = [...defaultRegistryUrls];
}
- return registryUrls.filter(Boolean);
+ return registryUrls.filter(Boolean).map(trimTrailingSlash);
}
export function getDefaultVersioning(datasourceName: string): string {
diff --git a/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap b/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap
index 19e59eebf0..0a310669ee 100644
--- a/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap
@@ -9,7 +9,7 @@ Object {
exports[`datasource/jenkins-plugins/index getReleases returns package releases for a hit for info and releases 1`] = `
Object {
- "registryUrl": "https://updates.jenkins.io/",
+ "registryUrl": "https://updates.jenkins.io",
"releases": Array [
Object {
"downloadUrl": "http://updates.jenkins-ci.org/download/plugins/email-ext/2.10/email-ext.hpi",
diff --git a/lib/datasource/nuget/__snapshots__/index.spec.ts.snap b/lib/datasource/nuget/__snapshots__/index.spec.ts.snap
index 8e5f4830f0..2b976f37d4 100644
--- a/lib/datasource/nuget/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/nuget/__snapshots__/index.spec.ts.snap
@@ -67,7 +67,7 @@ Array [
exports[`datasource/nuget/index getReleases handles paginated results (v2) 1`] = `
Object {
- "registryUrl": "https://www.nuget.org/api/v2/",
+ "registryUrl": "https://www.nuget.org/api/v2",
"releases": Array [
Object {
"version": "1.0.0",
@@ -104,7 +104,7 @@ Array [
exports[`datasource/nuget/index getReleases processes real data (v2) 1`] = `
Object {
- "registryUrl": "https://www.nuget.org/api/v2/",
+ "registryUrl": "https://www.nuget.org/api/v2",
"releases": Array [
Object {
"releaseTimestamp": "2011-01-07T07:57:55.387Z",
@@ -2052,7 +2052,7 @@ Array [
exports[`datasource/nuget/index getReleases processes real data with no github project url (v2) 1`] = `
Object {
- "registryUrl": "https://www.nuget.org/api/v2/",
+ "registryUrl": "https://www.nuget.org/api/v2",
"releases": Array [
Object {
"version": "3.11.0",
@@ -2078,7 +2078,7 @@ Array [
exports[`datasource/nuget/index getReleases processes real data without project url (v2) 1`] = `
Object {
- "registryUrl": "https://www.nuget.org/api/v2/",
+ "registryUrl": "https://www.nuget.org/api/v2",
"releases": Array [
Object {
"version": "2.5.7.10213",
diff --git a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
index 623cd7804a..a3bccdc7c1 100644
--- a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap
@@ -83,7 +83,7 @@ Array [
exports[`datasource/pypi/index getReleases parses data-requires-python and respects constraints from simple endpoint 1`] = `
Object {
- "registryUrl": "https://pypi.org/simple/",
+ "registryUrl": "https://pypi.org/simple",
"releases": Array [
Object {
"version": "0.1.2",
@@ -123,7 +123,7 @@ Array [
exports[`datasource/pypi/index getReleases process data from +simple endpoint 1`] = `
Object {
- "registryUrl": "https://some.registry.org/+simple/",
+ "registryUrl": "https://some.registry.org/+simple",
"releases": Array [
Object {
"version": "0.1.2",
@@ -179,7 +179,7 @@ Array [
exports[`datasource/pypi/index getReleases process data from simple endpoint 1`] = `
Object {
- "registryUrl": "https://pypi.org/simple/",
+ "registryUrl": "https://pypi.org/simple",
"releases": Array [
Object {
"version": "0.1.2",
@@ -235,7 +235,7 @@ Array [
exports[`datasource/pypi/index getReleases process data from simple endpoint with hyphens replaced with underscores 1`] = `
Object {
- "registryUrl": "https://pypi.org/simple/",
+ "registryUrl": "https://pypi.org/simple",
"releases": Array [
Object {
"version": "0.0.5",
@@ -260,7 +260,7 @@ Array [
exports[`datasource/pypi/index getReleases processes real data 1`] = `
Object {
- "registryUrl": "https://pypi.org/pypi/",
+ "registryUrl": "https://pypi.org/pypi",
"releases": Array [
Object {
"releaseTimestamp": "2017-04-03T16:55:14.000Z",
@@ -373,7 +373,7 @@ Array [
exports[`datasource/pypi/index getReleases respects constraints 1`] = `
Object {
- "registryUrl": "https://pypi.org/pypi/",
+ "registryUrl": "https://pypi.org/pypi",
"releases": Array [
Object {
"version": "0.4.0",
diff --git a/lib/datasource/repology/__snapshots__/index.spec.ts.snap b/lib/datasource/repology/__snapshots__/index.spec.ts.snap
index a21d22412b..f31c48d616 100644
--- a/lib/datasource/repology/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/repology/__snapshots__/index.spec.ts.snap
@@ -2,7 +2,7 @@
exports[`datasource/repology/index getReleases returns correct version for api package 1`] = `
Object {
- "registryUrl": "https://repology.org/",
+ "registryUrl": "https://repology.org",
"releases": Array [
Object {
"version": "1.181",
@@ -38,7 +38,7 @@ Array [
exports[`datasource/repology/index getReleases returns correct version for binary package 1`] = `
Object {
- "registryUrl": "https://repology.org/",
+ "registryUrl": "https://repology.org",
"releases": Array [
Object {
"version": "1.14.2-2+deb10u1",
@@ -64,7 +64,7 @@ Array [
exports[`datasource/repology/index getReleases returns correct version for multi-package project with different name 1`] = `
Object {
- "registryUrl": "https://repology.org/",
+ "registryUrl": "https://repology.org",
"releases": Array [
Object {
"version": "12.2-4+deb10u1",
@@ -90,7 +90,7 @@ Array [
exports[`datasource/repology/index getReleases returns correct version for multi-package project with same name 1`] = `
Object {
- "registryUrl": "https://repology.org/",
+ "registryUrl": "https://repology.org",
"releases": Array [
Object {
"version": "9.3.0-r2",
@@ -116,7 +116,7 @@ Array [
exports[`datasource/repology/index getReleases returns correct version for source package 1`] = `
Object {
- "registryUrl": "https://repology.org/",
+ "registryUrl": "https://repology.org",
"releases": Array [
Object {
"version": "1.181",
@@ -152,7 +152,7 @@ Array [
exports[`datasource/repology/index getReleases returns multiple versions if they are present in repository 1`] = `
Object {
- "registryUrl": "https://repology.org/",
+ "registryUrl": "https://repology.org",
"releases": Array [
Object {
"version": "1:11.0.7.10-1.el8_1",
diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts
index fd571da101..53fb346682 100644
--- a/lib/datasource/repology/index.ts
+++ b/lib/datasource/repology/index.ts
@@ -3,7 +3,7 @@ import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
import * as packageCache from '../../util/cache/package';
import { Http } from '../../util/http';
-import { getQueryString } from '../../util/url';
+import { getQueryString, joinUrlParts } from '../../util/url';
import type { GetReleasesConfig, ReleaseResult } from '../types';
import type { RepologyPackage, RepologyPackageType } from './types';
@@ -52,7 +52,7 @@ async function queryPackagesViaResolver(
// Retrieve list of packages by looking up Repology project
const packages = await queryPackages(
- `${registryUrl}tools/project-by?${query}`
+ joinUrlParts(registryUrl, `tools/project-by?${query}`)
);
return packages;
@@ -65,7 +65,7 @@ async function queryPackagesViaAPI(
// Directly query the package via the API. This will only work if `packageName` has the
// same name as the repology project
const packages = await queryPackages(
- `${registryUrl}api/v1/project/${packageName}`
+ joinUrlParts(registryUrl, `api/v1/project`, packageName)
);
return packages;
@@ -170,7 +170,7 @@ async function getCachedPackage(
pkgName: string
): Promise<RepologyPackage[]> {
// Fetch previous result from cache if available
- const cacheKey = `${registryUrl}${repoName}/${pkgName}`;
+ const cacheKey = joinUrlParts(registryUrl, repoName, pkgName);
const cachedResult = await packageCache.get<RepologyPackage[]>(
cacheNamespace,
cacheKey
diff --git a/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap b/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap
index 2f12ed4e3c..11742fb103 100644
--- a/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap
@@ -1461,7 +1461,7 @@ Array [
exports[`datasource/rubygems/index getReleases uses multiple source urls 1`] = `
Object {
"homepage": "http://rubyonrails.org",
- "registryUrl": "https://firstparty.com/basepath/",
+ "registryUrl": "https://firstparty.com/basepath",
"releases": Array [
Object {
"releaseTimestamp": "2009-07-25T18:01:56.000Z",
diff --git a/lib/util/url.spec.ts b/lib/util/url.spec.ts
index c425ef9eb7..d6f959e8b9 100644
--- a/lib/util/url.spec.ts
+++ b/lib/util/url.spec.ts
@@ -1,5 +1,6 @@
import {
ensurePathPrefix,
+ joinUrlParts,
parseUrl,
resolveBaseUrl,
trimTrailingSlash,
@@ -89,4 +90,20 @@ describe('util/url', () => {
ensurePathPrefix('https://index.docker.io/v2/something', '/v2')
).toBe('https://index.docker.io/v2/something');
});
+
+ it('joinUrlParts', () => {
+ const registryUrl = 'https://some.test';
+ expect(joinUrlParts(registryUrl, 'foo')).toBe(`${registryUrl}/foo`);
+ expect(joinUrlParts(registryUrl, '/?foo')).toBe(`${registryUrl}?foo`);
+ expect(joinUrlParts(registryUrl, '/foo/bar/')).toBe(
+ `${registryUrl}/foo/bar/`
+ );
+ expect(joinUrlParts(`${registryUrl}/foo/`, '/foo/bar')).toBe(
+ `${registryUrl}/foo/foo/bar`
+ );
+ expect(joinUrlParts(`${registryUrl}/api/`, '/foo/bar')).toBe(
+ `${registryUrl}/api/foo/bar`
+ );
+ expect(joinUrlParts('foo//////')).toBe('foo/');
+ });
});
diff --git a/lib/util/url.ts b/lib/util/url.ts
index b84548b241..ea9e560eb2 100644
--- a/lib/util/url.ts
+++ b/lib/util/url.ts
@@ -1,5 +1,9 @@
import urlJoin from 'url-join';
+export function joinUrlParts(...parts: string[]): string {
+ return urlJoin(...parts);
+}
+
export function ensurePathPrefix(url: string, prefix: string): string {
const parsed = new URL(url);
const fullPath = url.replace(parsed.origin, '');
--
GitLab