diff --git a/lib/datasource/repology/__snapshots__/index.spec.ts.snap b/lib/datasource/repology/__snapshots__/index.spec.ts.snap index 185e99788df1060941580a490cdac80c8c98e658..45201206761d58b67898dd2524db515812896f5d 100644 --- a/lib/datasource/repology/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/repology/__snapshots__/index.spec.ts.snap @@ -20,7 +20,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", + "url": "https://repology.org/api/v1/project/nginx", }, ] `; @@ -45,7 +45,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=pulseaudio-utils", + "url": "https://repology.org/api/v1/project/pulseaudio-utils", }, ] `; @@ -70,7 +70,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/tools/project-by?repo=alpine_3_12&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=gcc", + "url": "https://repology.org/api/v1/project/gcc", }, ] `; @@ -95,17 +95,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=gcc-defaults", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "repology.org", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=gcc-defaults", + "url": "https://repology.org/api/v1/project/gcc-defaults", }, ] `; @@ -120,17 +110,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "repology.org", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx", + "url": "https://repology.org/api/v1/project/nginx", }, ] `; @@ -145,42 +125,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/tools/project-by?repo=this_should&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=never-exist", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "repology.org", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://repology.org/tools/project-by?repo=this_should&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=never-exist", - }, -] -`; - -exports[`datasource/repology/index getReleases returns null for unsupported repository 1`] = ` -Array [ - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "repology.org", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://repology.org/tools/project-by?repo=unsupported_repo&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "repology.org", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://repology.org/tools/project-by?repo=unsupported_repo&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx", + "url": "https://repology.org/api/v1/project/never-exist", }, ] `; @@ -195,32 +140,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", - }, -] -`; - -exports[`datasource/repology/index getReleases throws error on unexpected response during source package lookup 1`] = ` -Array [ - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "repology.org", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate", - "host": "repology.org", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx", + "url": "https://repology.org/api/v1/project/nginx", }, ] `; diff --git a/lib/datasource/repology/index.spec.ts b/lib/datasource/repology/index.spec.ts index 844388c4c42112a649e675484c15c3311b8b1b1d..314acea523965eae8422c191aba2e0f3e6b366bb 100644 --- a/lib/datasource/repology/index.spec.ts +++ b/lib/datasource/repology/index.spec.ts @@ -6,37 +6,15 @@ import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import { id as versioning } from '../../versioning/loose'; import { RepologyPackage, id as datasource } from '.'; -const repologyApiHost = 'https://repology.org/'; +const repologyApiHost = 'https://repology.org/api/v1/'; type ResponseMock = { status: number; body?: string }; -const mockProjectBy = ( - repo: string, - name: string, - binary: ResponseMock, - source: ResponseMock -) => { - const endpoint = '/tools/project-by'; - const defaultParams = { - target_page: 'api_v1_project', - noautoresolve: 'on', - }; - - if (binary) { - httpMock - .scope(repologyApiHost) - .get(endpoint) - .query({ ...defaultParams, repo, name, name_type: 'binname' }) - .reply(binary.status, binary.body); - } - - if (source) { - httpMock - .scope(repologyApiHost) - .get(endpoint) - .query({ ...defaultParams, repo, name, name_type: 'srcname' }) - .reply(source.status, source.body); - } +const mockProjectBy = (name: string, response: ResponseMock) => { + httpMock + .scope(repologyApiHost) + .get(`/project/${name}`) + .reply(response.status, response.body); }; const fixtureNginx = fs.readFileSync( @@ -65,12 +43,7 @@ describe(getName(__filename), () => { afterEach(() => httpMock.reset()); it('returns null for empty result', async () => { - mockProjectBy( - 'debian_stable', - 'nginx', - { status: 200, body: '[]' }, - { status: 200, body: '[]' } - ); + mockProjectBy('nginx', { status: 200, body: '[]' }); expect( await getPkgReleases({ @@ -83,12 +56,7 @@ describe(getName(__filename), () => { }); it('returns null for missing repository or package', async () => { - mockProjectBy( - 'this_should', - 'never-exist', - { status: 404 }, - { status: 404 } - ); + mockProjectBy('never-exist', { status: 404 }); expect( await getPkgReleases({ @@ -100,39 +68,8 @@ describe(getName(__filename), () => { expect(httpMock.getTrace()).toMatchSnapshot(); }); - it('returns null for unsupported repository', async () => { - mockProjectBy( - 'unsupported_repo', - 'nginx', - { status: 403 }, - { status: 403 } - ); - - expect( - await getPkgReleases({ - datasource, - versioning, - depName: 'unsupported_repo/nginx', - }) - ).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); - }); - it('throws error on unexpected response during binary package lookup', async () => { - mockProjectBy('debian_stable', 'nginx', { status: 500 }, null); - - await expect( - getPkgReleases({ - datasource, - versioning, - depName: 'debian_stable/nginx', - }) - ).rejects.toThrow(EXTERNAL_HOST_ERROR); - expect(httpMock.getTrace()).toMatchSnapshot(); - }); - - it('throws error on unexpected response during source package lookup', async () => { - mockProjectBy('debian_stable', 'nginx', { status: 404 }, { status: 500 }); + mockProjectBy('nginx', { status: 500 }); await expect( getPkgReleases({ @@ -156,12 +93,7 @@ describe(getName(__filename), () => { }); it('returns correct version for binary package', async () => { - mockProjectBy( - 'debian_stable', - 'nginx', - { status: 200, body: fixtureNginx }, - null - ); + mockProjectBy('nginx', { status: 200, body: fixtureNginx }); const res = await getPkgReleases({ datasource, @@ -175,12 +107,7 @@ describe(getName(__filename), () => { }); it('returns correct version for source package', async () => { - mockProjectBy( - 'debian_stable', - 'gcc-defaults', - { status: 404 }, - { status: 200, body: fixtureGccDefaults } - ); + mockProjectBy('gcc-defaults', { status: 200, body: fixtureGccDefaults }); const res = await getPkgReleases({ datasource, @@ -194,12 +121,7 @@ describe(getName(__filename), () => { }); it('returns correct version for multi-package project with same name', async () => { - mockProjectBy( - 'alpine_3_12', - 'gcc', - { status: 200, body: fixtureGcc }, - null - ); + mockProjectBy('gcc', { status: 200, body: fixtureGcc }); const res = await getPkgReleases({ datasource, @@ -213,12 +135,10 @@ describe(getName(__filename), () => { }); it('returns correct version for multi-package project with different name', async () => { - mockProjectBy( - 'debian_stable', - 'pulseaudio-utils', - { status: 200, body: fixturePulseaudio }, - null - ); + mockProjectBy('pulseaudio-utils', { + status: 200, + body: fixturePulseaudio, + }); const res = await getPkgReleases({ datasource, @@ -238,12 +158,7 @@ describe(getName(__filename), () => { ]; const pkgsJSON = JSON.stringify(pkgs); - mockProjectBy( - 'dummy', - 'example', - { status: 200, body: pkgsJSON }, - { status: 200, body: pkgsJSON } - ); + mockProjectBy('example', { status: 200, body: pkgsJSON }); expect( await getPkgReleases({ diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts index 445b98a61479aef97f236af974b1a48c65135063..980d247eda72440d3f2f171c6340265f0184b801 100644 --- a/lib/datasource/repology/index.ts +++ b/lib/datasource/repology/index.ts @@ -1,4 +1,3 @@ -import { URLSearchParams } from 'url'; import { HOST_DISABLED } from '../../constants/error-messages'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; @@ -25,36 +24,30 @@ export interface RepologyPackage { async function queryPackage( repoName: string, - pkgName: string, - pkgType: RepologyPackageType + pkgName: string ): Promise<RepologyPackage> { try { - const query = new URLSearchParams({ - repo: repoName, - name_type: pkgType, - target_page: 'api_v1_project', - noautoresolve: 'on', - name: pkgName, - }).toString(); - // Retrieve list of packages by looking up Repology project - const url = `https://repology.org/tools/project-by?${query}`; + const url = `https://repology.org/api/v1/project/${pkgName}`; const res = await http.getJson<RepologyPackage[]>(url); let pkgs = res.body.filter((pkg) => pkg.repo === repoName); // In some cases Repology bundles multiple packages into a single project, // which would result in ambiguous results. If we have more than one result // left, we should try to determine the correct package by comparing either - // binname or srcname (depending on pkgType) to the given dependency name. + // binname or srcname to the given dependency name. + if (pkgs.length > 1) { + pkgs = pkgs.filter((pkg) => !pkg.binname || pkg.binname === pkgName); + } if (pkgs.length > 1) { - pkgs = pkgs.filter((pkg) => !pkg[pkgType] || pkg[pkgType] === pkgName); + pkgs = pkgs.filter((pkg) => !pkg.srcname || pkg.srcname === pkgName); } // Abort if there is still more than one package left, as the result would // be ambiguous and unreliable. This should usually not happen... if (pkgs.length > 1) { logger.warn( - { repoName, pkgName, pkgType, pkgs }, + { repoName, pkgName, pkgs }, 'Repology lookup returned ambiguous results, ignoring...' ); return null; @@ -64,14 +57,9 @@ async function queryPackage( } catch (err) { if (err.statusCode === 404) { logger.debug( - { repoName, pkgName, pkgType }, + { repoName, pkgName }, 'Repository or package not found on Repology' ); - } else if (err.statusCode === 403) { - logger.debug( - { repoName }, - 'Repology does not support tools/project-by lookups for repository' - ); } else { throw err; } @@ -95,21 +83,14 @@ async function getCachedPackage( return cachedResult; } - // Attempt a binary package lookup and return if successfully - const binPkg = await queryPackage(repoName, pkgName, 'binname'); - if (binPkg) { - await packageCache.set(cacheNamespace, cacheKey, binPkg, cacheMinutes); - return binPkg; - } - - // Otherwise, attempt a source package lookup and return if successfully - const srcPkg = await queryPackage(repoName, pkgName, 'srcname'); - if (srcPkg) { - await packageCache.set(cacheNamespace, cacheKey, srcPkg, cacheMinutes); - return srcPkg; + // Attempt a package lookup and return if successfully + const pkg = await queryPackage(repoName, pkgName); + if (pkg) { + await packageCache.set(cacheNamespace, cacheKey, pkg, cacheMinutes); + return pkg; } - // No binary or source package was found on Repology + // No package was found on Repology return null; } diff --git a/lib/datasource/repology/readme.md b/lib/datasource/repology/readme.md index 5985f7813c9a9b61a7aa119e2aa6681e9a3301c6..d3d014d765aec4e64ae434aa40de26375ba79131 100644 --- a/lib/datasource/repology/readme.md +++ b/lib/datasource/repology/readme.md @@ -6,8 +6,6 @@ A [list of all supported repositories](https://repology.org/repositories/statist As an example, the `Alpine Linux 3.12` repository points to `https://repology.org/repository/alpine_3_12` and therefor has the repository identifier `alpine_3_12`. -Please note that as of today, the Repology API endpoint `/tools/project-by` does not support a small subset of repositories. You can manually double-check [on their website](https://repology.org/tools/project-by) if your desired repository is supported and otherwise raise a request on their side. This datasource will also print a debug message `Repology does not support tools/project-by lookups for repository` if the given repository is unsupported. - **Usage Example** A real world example for this specific datasource would be maintaining system packages within a Dockerfile, as this allows to specifically pin each dependency without having to manually keep the versions up-to-date. This can be achieved by configuring a generic regex manager in `renovate.json` for files named `Dockerfile`: