diff --git a/lib/datasource/common.ts b/lib/datasource/common.ts index 30ff87a415828b1ac1c3fc1d3d8dee879063c093..5e75686bdd7f12ef0e35a17450f7e1fb19ff2734 100644 --- a/lib/datasource/common.ts +++ b/lib/datasource/common.ts @@ -69,6 +69,7 @@ export interface ReleaseResult { tags?: Record<string, string>; versions?: any; registryUrl?: string; + isPrivate?: boolean; } export interface DatasourceApi { @@ -79,6 +80,7 @@ export interface DatasourceApi { defaultVersioning?: string; defaultConfig?: Record<string, unknown>; registryStrategy?: 'first' | 'hunt' | 'merge'; + caching?: boolean; } // TODO: remove, only for compatibility diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 875f22aad9b6ac810c7028d106c5f09536c62da7..d907fb674c79013238168352fa2a416587a1779b 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -4,6 +4,7 @@ import { HOST_DISABLED } from '../constants/error-messages'; import { logger } from '../logger'; import { ExternalHostError } from '../types/errors/external-host-error'; import * as memCache from '../util/cache/memory'; +import * as packageCache from '../util/cache/package'; import { clone } from '../util/clone'; import { regEx } from '../util/regex'; import * as allVersioning from '../versioning'; @@ -52,7 +53,25 @@ async function getRegistryReleases( config: GetReleasesConfig, registryUrl: string ): Promise<ReleaseResult> { + const cacheKey = `${datasource.id} ${registryUrl} ${config.lookupName}`; + if (datasource.caching) { + const cachedResult = await packageCache.get<ReleaseResult>( + cacheNamespace, + cacheKey + ); + // istanbul ignore if + if (cachedResult) { + logger.debug({ cacheKey }, 'Returning cached datasource response'); + return cachedResult; + } + } const res = await datasource.getReleases({ ...config, registryUrl }); + // cache non-null responses unless marked as private + if (datasource.caching && res && !res.isPrivate) { + logger.trace({ cacheKey }, 'Caching datasource response'); + const cacheMinutes = 15; + await packageCache.set(cacheNamespace, cacheKey, res, cacheMinutes); + } return res; } diff --git a/lib/datasource/pypi/index.spec.ts b/lib/datasource/pypi/index.spec.ts index 7e17e21ff568b99745ba8a9e93acee03c3a63cfc..faa95e5fa4e414ac5912504dfafb928dbbe5628f 100644 --- a/lib/datasource/pypi/index.spec.ts +++ b/lib/datasource/pypi/index.spec.ts @@ -1,6 +1,7 @@ import fs from 'fs'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/http-mock'; +import * as hostRules from '../../util/host-rules'; import { id as datasource } from '.'; const res1: any = fs.readFileSync( @@ -89,6 +90,23 @@ describe('datasource/pypi', () => { }); expect(httpMock.getTrace()).toMatchSnapshot(); }); + + it('sets private if authorization privided', async () => { + hostRules.add({ hostName: 'customprivate.pypi.net', token: 'abc123' }); + httpMock + .scope('https://customprivate.pypi.net/foo') + .get('/azure-cli-monitor/json') + .reply(200, JSON.parse(res1)); + const config = { + registryUrls: ['https://customprivate.pypi.net/foo'], + }; + const res = await getPkgReleases({ + ...config, + datasource, + depName: 'azure-cli-monitor', + }); + expect(res.isPrivate).toBe(true); + }); it('supports multiple custom datasource urls', async () => { httpMock .scope('https://custom.pypi.net/foo') @@ -248,6 +266,23 @@ describe('datasource/pypi', () => { ).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('sets private simple if authorization provided', async () => { + hostRules.add({ hostName: 'some.private.registry.org', token: 'abc123' }); + httpMock + .scope('https://some.private.registry.org/+simple/') + .get('/dj-database-url') + .reply(200, htmlResponse); + const config = { + registryUrls: ['https://some.private.registry.org/+simple/'], + }; + const res = await getPkgReleases({ + datasource, + ...config, + constraints: { python: '2.7' }, + depName: 'dj-database-url', + }); + expect(res.isPrivate).toBe(true); + }); it('process data from simple endpoint with hyphens replaced with underscores', async () => { httpMock .scope('https://pypi.org/simple/') diff --git a/lib/datasource/pypi/index.ts b/lib/datasource/pypi/index.ts index 8bfcb2a08d04a98424df8a2df3d5a4e1eb934ca6..b588d6d0ac46e82f276f511851239ffad3ac94e0 100644 --- a/lib/datasource/pypi/index.ts +++ b/lib/datasource/pypi/index.ts @@ -13,6 +13,7 @@ export const defaultRegistryUrls = [ ]; export const defaultVersioning = pep440.id; export const registryStrategy = 'merge'; +export const caching = true; const githubRepoPattern = /^https?:\/\/github\.com\/[^\\/]+\/[^\\/]+$/; const http = new Http(id); @@ -50,6 +51,9 @@ async function getDependency( logger.trace({ dependency: packageName }, 'pip package not found'); return null; } + if (rep.authorization) { + dependency.isPrivate = true; + } logger.trace({ lookupUrl }, 'Got pypi api result'); if ( !(dep.info && normalizeName(dep.info.name) === normalizeName(packageName)) @@ -179,6 +183,9 @@ async function getSimpleDependency( logger.trace({ dependency: packageName }, 'pip package not found'); return null; } + if (response.authorization) { + dependency.isPrivate = true; + } const root: HTMLElement = parse(cleanSimpleHtml(dep)); const links = root.querySelectorAll('a'); const releases: Releases = {};