diff --git a/lib/datasource/repology/__snapshots__/index.spec.ts.snap b/lib/datasource/repology/__snapshots__/index.spec.ts.snap index 45201206761d58b67898dd2524db515812896f5d..92c7f3a7c1cff68a0f8e371f3d891a9e6b9ead75 100644 --- a/lib/datasource/repology/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/repology/__snapshots__/index.spec.ts.snap @@ -1,5 +1,40 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`datasource/repology/index getReleases returns correct version for api package 1`] = ` +Object { + "releases": Array [ + Object { + "version": "1.181", + }, + ], +} +`; + +exports[`datasource/repology/index getReleases returns correct version for api package 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=gcc-defaults", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/api/v1/project/gcc-defaults", + }, +] +`; + exports[`datasource/repology/index getReleases returns correct version for binary package 1`] = ` Object { "releases": Array [ @@ -20,7 +55,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/api/v1/project/nginx", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", }, ] `; @@ -45,7 +80,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/api/v1/project/pulseaudio-utils", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=pulseaudio-utils", }, ] `; @@ -70,7 +105,7 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/api/v1/project/gcc", + "url": "https://repology.org/tools/project-by?repo=alpine_3_12&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=gcc", }, ] `; @@ -95,7 +130,42 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/api/v1/project/gcc-defaults", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=gcc-defaults", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=gcc-defaults", + }, +] +`; + +exports[`datasource/repology/index getReleases returns null for ambiguous package results 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=dummy&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=example", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=dummy&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=example", }, ] `; @@ -110,7 +180,17 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/api/v1/project/nginx", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx", }, ] `; @@ -125,13 +205,93 @@ Array [ "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://repology.org/api/v1/project/never-exist", + "url": "https://repology.org/tools/project-by?repo=this_should&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=never-exist", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=this_should&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=never-exist", + }, +] +`; + +exports[`datasource/repology/index getReleases throws error on API request timeout 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/api/v1/project/nginx", + }, +] +`; + +exports[`datasource/repology/index getReleases throws error on Resolver request timeout 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", }, ] `; -exports[`datasource/repology/index getReleases throws error on unexpected response during binary package lookup 1`] = ` +exports[`datasource/repology/index getReleases throws error on unexpected API response 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, Object { "headers": Object { "accept": "application/json", @@ -145,4 +305,44 @@ Array [ ] `; +exports[`datasource/repology/index getReleases throws error on unexpected Resolver response with binary package 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, +] +`; + +exports[`datasource/repology/index getReleases throws error on unexpected Resolver response with source package 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "repology.org", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx", + }, +] +`; + exports[`datasource/repology/index getReleases throws without repository and package name 1`] = `Array []`; diff --git a/lib/datasource/repology/index.spec.ts b/lib/datasource/repology/index.spec.ts index 3c39b3d9f9ba525f7066ca9def97d8fbb11d1467..e805724f4090f967813701273bd76780164e7da9 100644 --- a/lib/datasource/repology/index.spec.ts +++ b/lib/datasource/repology/index.spec.ts @@ -6,15 +6,44 @@ import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import { id as versioning } from '../../versioning/loose'; import { RepologyPackage, id as datasource } from '.'; -const repologyApiHost = 'https://repology.org/api/v1/'; +const repologyHost = 'https://repology.org/'; -type ResponseMock = { status: number; body?: string }; +type ResponseMock = { status?: number; body?: string; code?: string }; -const mockProjectBy = (name: string, response: ResponseMock) => { - httpMock - .scope(repologyApiHost) - .get(`/project/${name}`) - .reply(response.status, response.body); +const mockApiCall = (name: string, response: ResponseMock) => { + const interceptor = httpMock + .scope(repologyHost) + .get(`/api/v1/project/${name}`); + if (response.status) { + interceptor.reply(response.status, response.body); + } else { + interceptor.replyWithError({ code: response.code }); + } +}; + +const mockResolverCall = ( + repo: string, + name: string, + name_type: string, + response: ResponseMock +) => { + const query = { + repo, + name_type, + target_page: 'api_v1_project', + noautoresolve: 'on', + name, + }; + + const interceptor = httpMock + .scope(repologyHost) + .get('/tools/project-by') + .query(query); + if (response.status) { + interceptor.reply(response.status, response.body); + } else { + interceptor.replyWithError({ code: response.code }); + } }; const fixtureNginx = fs.readFileSync( @@ -43,7 +72,14 @@ describe(getName(__filename), () => { afterEach(() => httpMock.reset()); it('returns null for empty result', async () => { - mockProjectBy('nginx', { status: 200, body: '[]' }); + mockResolverCall('debian_stable', 'nginx', 'binname', { + status: 200, + body: '[]', + }); + mockResolverCall('debian_stable', 'nginx', 'srcname', { + status: 200, + body: '[]', + }); expect( await getPkgReleases({ @@ -56,7 +92,12 @@ describe(getName(__filename), () => { }); it('returns null for missing repository or package', async () => { - mockProjectBy('never-exist', { status: 404 }); + mockResolverCall('this_should', 'never-exist', 'binname', { + status: 404, + }); + mockResolverCall('this_should', 'never-exist', 'srcname', { + status: 404, + }); expect( await getPkgReleases({ @@ -68,8 +109,84 @@ describe(getName(__filename), () => { expect(httpMock.getTrace()).toMatchSnapshot(); }); - it('throws error on unexpected response during binary package lookup', async () => { - mockProjectBy('nginx', { status: 500 }); + it('throws error on unexpected API response', async () => { + mockResolverCall('debian_stable', 'nginx', 'binname', { + status: 200, + body: '[]', + }); + mockResolverCall('debian_stable', 'nginx', 'srcname', { + status: 403, + }); + mockApiCall('nginx', { status: 500 }); + + await expect( + getPkgReleases({ + datasource, + versioning, + depName: 'debian_stable/nginx', + }) + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + + it('throws error on unexpected Resolver response with binary package', async () => { + mockResolverCall('debian_stable', 'nginx', 'binname', { + status: 500, + }); + + await expect( + getPkgReleases({ + datasource, + versioning, + depName: 'debian_stable/nginx', + }) + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + + it('throws error on unexpected Resolver response with source package', async () => { + mockResolverCall('debian_stable', 'nginx', 'binname', { + status: 200, + body: '[]', + }); + mockResolverCall('debian_stable', 'nginx', 'srcname', { + status: 500, + }); + + await expect( + getPkgReleases({ + datasource, + versioning, + depName: 'debian_stable/nginx', + }) + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + + it('throws error on API request timeout', async () => { + mockResolverCall('debian_stable', 'nginx', 'binname', { + status: 200, + body: '[]', + }); + mockResolverCall('debian_stable', 'nginx', 'srcname', { + status: 403, + }); + mockApiCall('nginx', { code: 'ETIMEDOUT' }); + + await expect( + getPkgReleases({ + datasource, + versioning, + depName: 'debian_stable/nginx', + }) + ).rejects.toThrow(EXTERNAL_HOST_ERROR); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + + it('throws error on Resolver request timeout', async () => { + mockResolverCall('debian_stable', 'nginx', 'binname', { + code: 'ETIMEDOUT', + }); await expect( getPkgReleases({ @@ -93,7 +210,10 @@ describe(getName(__filename), () => { }); it('returns correct version for binary package', async () => { - mockProjectBy('nginx', { status: 200, body: fixtureNginx }); + mockResolverCall('debian_stable', 'nginx', 'binname', { + status: 200, + body: fixtureNginx, + }); const res = await getPkgReleases({ datasource, @@ -107,7 +227,30 @@ describe(getName(__filename), () => { }); it('returns correct version for source package', async () => { - mockProjectBy('gcc-defaults', { status: 200, body: fixtureGccDefaults }); + mockResolverCall('debian_stable', 'gcc-defaults', 'binname', { + status: 404, + }); + mockResolverCall('debian_stable', 'gcc-defaults', 'srcname', { + status: 200, + body: fixtureGccDefaults, + }); + + const res = await getPkgReleases({ + datasource, + versioning, + depName: 'debian_stable/gcc-defaults', + }); + expect(res).toMatchSnapshot(); + expect(res.releases).toHaveLength(1); + expect(res.releases[0].version).toEqual('1.181'); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + + it('returns correct version for api package', async () => { + mockResolverCall('debian_stable', 'gcc-defaults', 'binname', { + status: 403, + }); + mockApiCall('gcc-defaults', { status: 200, body: fixtureGccDefaults }); const res = await getPkgReleases({ datasource, @@ -121,7 +264,10 @@ describe(getName(__filename), () => { }); it('returns correct version for multi-package project with same name', async () => { - mockProjectBy('gcc', { status: 200, body: fixtureGcc }); + mockResolverCall('alpine_3_12', 'gcc', 'binname', { + status: 200, + body: fixtureGcc, + }); const res = await getPkgReleases({ datasource, @@ -135,7 +281,7 @@ describe(getName(__filename), () => { }); it('returns correct version for multi-package project with different name', async () => { - mockProjectBy('pulseaudio-utils', { + mockResolverCall('debian_stable', 'pulseaudio-utils', 'binname', { status: 200, body: fixturePulseaudio, }); @@ -158,15 +304,24 @@ describe(getName(__filename), () => { ]; const pkgsJSON = JSON.stringify(pkgs); - mockProjectBy('example', { status: 200, body: pkgsJSON }); + mockResolverCall('dummy', 'example', 'binname', { + status: 200, + body: pkgsJSON, + }); - expect( - await getPkgReleases({ - datasource, - versioning, - depName: 'dummy/example', - }) - ).toBeNull(); + mockResolverCall('dummy', 'example', 'srcname', { + status: 200, + body: pkgsJSON, + }); + + const release = await getPkgReleases({ + datasource, + versioning, + depName: 'dummy/example', + }); + + expect(release).toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); }); }); }); diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts index 980d247eda72440d3f2f171c6340265f0184b801..b79a81e36dc90d2f1fddf534e69cad299851dfdb 100644 --- a/lib/datasource/repology/index.ts +++ b/lib/datasource/repology/index.ts @@ -12,6 +12,7 @@ const cacheNamespace = `datasource-${id}`; const cacheMinutes = 60; export type RepologyPackageType = 'binname' | 'srcname'; +const packageTypes: RepologyPackageType[] = ['binname', 'srcname']; export interface RepologyPackage { repo: string; @@ -22,49 +23,140 @@ export interface RepologyPackage { origversion?: string; } -async function queryPackage( - repoName: string, - pkgName: string -): Promise<RepologyPackage> { +async function queryPackages(url: string): Promise<RepologyPackage[]> { try { - // Retrieve list of packages by looking up Repology project - const url = `https://repology.org/api/v1/project/${pkgName}`; const res = await http.getJson<RepologyPackage[]>(url); - let pkgs = res.body.filter((pkg) => pkg.repo === repoName); - - // In some cases Repology bundles multiple packages into a single project, - // which would result in ambiguous results. If we have more than one result - // left, we should try to determine the correct package by comparing either - // binname or srcname to the given dependency name. - if (pkgs.length > 1) { - pkgs = pkgs.filter((pkg) => !pkg.binname || pkg.binname === pkgName); - } - if (pkgs.length > 1) { - pkgs = pkgs.filter((pkg) => !pkg.srcname || pkg.srcname === pkgName); + return res.body; + } catch (err) { + if (err.statusCode === 404) { + // Return an array here because the api does not return proper http codes + // and instead of an 404 error an empty array with code 200 is returned + // When querying the resolver 404 is thrown if package could not be resolved + // and 403 if the repo is not supported + // 403 is handled later because in this case we are trying the API + return []; } - // Abort if there is still more than one package left, as the result would - // be ambiguous and unreliable. This should usually not happen... - if (pkgs.length > 1) { - logger.warn( - { repoName, pkgName, pkgs }, - 'Repology lookup returned ambiguous results, ignoring...' - ); - return null; + throw err; + } +} + +async function queryPackagesViaResolver( + repoName: string, + packageName: string, + packageType: RepologyPackageType +): Promise<RepologyPackage[]> { + const query = new URLSearchParams({ + repo: repoName, + name_type: packageType, + target_page: 'api_v1_project', + noautoresolve: 'on', + name: packageName, + }).toString(); + + // Retrieve list of packages by looking up Repology project + const packages = await queryPackages( + `https://repology.org/tools/project-by?${query}` + ); + + return packages; +} + +async function queryPackagesViaAPI( + packageName: string +): Promise<RepologyPackage[]> { + // Directly query the package via the API. This will only work if `packageName` has the + // same name as the repology project + const packages = await queryPackages( + `https://repology.org/api/v1/project/${packageName}` + ); + + return packages; +} + +function findPackageInResponse( + response: RepologyPackage[], + repoName: string, + pkgName: string, + types: RepologyPackageType[] +): RepologyPackage | undefined { + let pkgs = response.filter((pkg) => pkg.repo === repoName); + + // In some cases Repology bundles multiple packages into a single project, + // which would result in ambiguous results. If we have more than one result + // left, we should try to determine the correct package by comparing either + // binname or srcname (depending on `types`) to the given dependency name. + if (pkgs.length > 1) { + for (const pkgType of types) { + pkgs = pkgs.filter((pkg) => !pkg[pkgType] || pkg[pkgType] === pkgName); + if (pkgs.length === 1) { + break; + } } + } + + // Abort if there is still more than one package left, as the result would + // be ambiguous and unreliable. This should usually not happen... + if (pkgs.length > 1) { + logger.warn( + { repoName, pkgName, packageTypes, pkgs }, + 'Repology lookup returned ambiguous results, ignoring...' + ); + return null; + } - return pkgs[0]; + // pkgs might be an empty array here and in that case we return undefined + return pkgs[0]; +} + +async function queryPackage( + repoName: string, + pkgName: string +): Promise<RepologyPackage> { + let response: RepologyPackage[]; + let pkg: RepologyPackage; + // Try getting the packages from tools/project-by first for type binname and + // afterwards for srcname. This needs to be done first, because some packages + // resolve to repology projects which have a different name than the package + // e.g. `pulseaudio-utils` resolves to project `pulseaudio`, BUT there is also + // a project called `pulseaudio-utils` but it does not contain the package we + // are looking for. + try { + for (const pkgType of packageTypes) { + response = await queryPackagesViaResolver(repoName, pkgName, pkgType); + + pkg = findPackageInResponse(response, repoName, pkgName, [pkgType]); + if (pkg) { + // exit immediately if package found + return pkg; + } + } } catch (err) { - if (err.statusCode === 404) { + if (err.statusCode === 403) { logger.debug( { repoName, pkgName }, - 'Repository or package not found on Repology' + 'Repology does not support tools/project-by lookups for repository. Will try direct API access now' ); - } else { - throw err; + + // If the repository is not supported in tools/project-by we try directly accessing the + // API. This will support all repositories but requires that the project name is equal to the + // package name. This won't be always the case but for a good portion we might be able to resolve + // the package this way. + response = await queryPackagesViaAPI(pkgName); + pkg = findPackageInResponse(response, repoName, pkgName, packageTypes); + if (pkg) { + // exit immediately if package found + return pkg; + } } + throw err; } + logger.debug( + { repoName, pkgName }, + 'Repository or package not found on Repology' + ); + return null; } @@ -107,7 +199,6 @@ export async function getReleases({ ); } - // Attempt to resolve package version through Repology logger.trace(`repology.getReleases(${repoName}, ${pkgName})`); try { // Attempt to retrieve (cached) package information from Repology