diff --git a/lib/util/http/dns.spec.ts b/lib/util/http/dns.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..b1e1477b572778c4f426ab6b3c40866f68dec62e --- /dev/null +++ b/lib/util/http/dns.spec.ts @@ -0,0 +1,50 @@ +import { logger } from '../../logger'; +import { clearDnsCache, dnsLookup, printDnsStats } from './dns'; + +describe('util/http/dns', () => { + describe('dnsLookup', () => { + it('works', async () => { + clearDnsCache(); + const ip = await new Promise((resolve) => + dnsLookup('api.github.com', 4, (_e, r, _f) => { + resolve(r); + }) + ); + expect(ip).toBeString(); + // uses cache + expect( + await new Promise((resolve) => + dnsLookup('api.github.com', (_e, r, _f) => { + resolve(r); + }) + ) + ).toBe(ip); + expect( + await new Promise((resolve) => + dnsLookup('api.github.com', {}, (_e, r, _f) => { + resolve(r); + }) + ) + ).toBe(ip); + }); + + it('throws', async () => { + clearDnsCache(); + const ip = new Promise((resolve, reject) => + dnsLookup('api.github.comcccccccc', 4, (_e, r, _f) => { + if (_e) { + reject(_e); + } else { + resolve(r); + } + }) + ); + await expect(ip).rejects.toThrow(); + }); + + it('prints stats', () => { + printDnsStats(); + expect(logger.debug).toHaveBeenCalled(); + }); + }); +}); diff --git a/lib/util/http/dns.ts b/lib/util/http/dns.ts index e4c84792791789808319f5a195669c2aee227e80..df49ad448e08e357b02fa5725ccac04a3a3cb63c 100644 --- a/lib/util/http/dns.ts +++ b/lib/util/http/dns.ts @@ -1,6 +1,107 @@ -import CacheableLookup from 'cacheable-lookup'; +import { LookupAllOptions, LookupOneOptions, lookup as _dnsLookup } from 'dns'; +import type { EntryObject, IPFamily, LookupOptions } from 'cacheable-lookup'; import QuickLRU from 'quick-lru'; +import { logger } from '../../logger'; -export const dnsCache = new CacheableLookup({ - cache: new QuickLRU({ maxSize: 1000 }), -}); +const cache = new QuickLRU<string, any>({ maxSize: 1000 }); + +function lookup( + ...[host, options, callback]: + | [ + hostname: string, + family: IPFamily, + callback: ( + error: NodeJS.ErrnoException, + address: string, + family: IPFamily + ) => void + ] + | [ + hostname: string, + callback: ( + error: NodeJS.ErrnoException, + address: string, + family: IPFamily + ) => void + ] + | [ + hostname: string, + options: LookupOptions & { all: true }, + callback: ( + error: NodeJS.ErrnoException, + result: ReadonlyArray<EntryObject> + ) => void + ] + | [ + hostname: string, + options: LookupOptions, + callback: ( + error: NodeJS.ErrnoException, + address: string, + family: IPFamily + ) => void + ] +): void { + let opts: LookupOneOptions | LookupAllOptions; + // TODO: strict null incompatible types (#7154) + let cb: any; + + if (typeof options === 'function') { + opts = {}; + cb = options; + } else if (typeof options === 'number') { + opts = { family: options }; + cb = callback; + } else { + opts = options; + cb = callback; + } + + // istanbul ignore if: not used + if (opts.all) { + const key = `${host}_all`; + if (cache.has(key)) { + logger.trace({ host }, 'dns lookup cache hit'); + cb(null, cache.get(key)); + return; + } + + _dnsLookup(host, opts, (err, res) => { + if (err) { + logger.error({ host, err }, 'dns lookup error'); + cb(err, null, null); + return; + } + logger.trace({ host, opts, res }, 'dns lookup'); + cache.set(key, res); + cb(null, res, null); + }); + } else { + if (cache.has(host)) { + logger.trace({ host }, 'dns lookup cache hit'); + cb(null, ...cache.get(host)); + return; + } + + _dnsLookup(host, opts, (err, ...res) => { + if (err) { + logger.error({ host, err }, 'dns lookup error'); + cb(err); + return; + } + logger.trace({ host, opts, res }, 'dns lookup'); + cache.set(host, res); + cb(null, ...res); + }); + } +} + +export { lookup as dnsLookup }; + +export function printDnsStats(): void { + logger.debug({ hosts: Array.from(cache.keys()) }, 'dns cache'); +} + +export function clearDnsCache(): void { + cache.clear(); +} diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts index f640b3116641755bf2cf7f4ad18b72edb1673127..8be653b0d4786b797bb83f84aff9a1aabe1c9b54 100644 --- a/lib/util/http/host-rules.spec.ts +++ b/lib/util/http/host-rules.spec.ts @@ -1,7 +1,7 @@ import { PlatformId } from '../../constants'; import { bootstrap } from '../../proxy'; import * as hostRules from '../host-rules'; -import { dnsCache } from './dns'; +import { dnsLookup } from './dns'; import { applyHostRules } from './host-rules'; const url = 'https://github.com'; @@ -114,7 +114,7 @@ describe('util/http/host-rules', () => { hostRules.add({ dnsCache: true }); expect(applyHostRules(url, { ...options, token: 'xxx' })).toMatchObject({ hostType: 'github', - dnsCache: dnsCache, + lookup: dnsLookup, token: 'xxx', }); }); diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index fab7ecd3b34f420cba5fbff1fe7fa0424136d511..23310bf9d23bfaa35e4c26237752900a6da0442b 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -8,7 +8,7 @@ import { logger } from '../../logger'; import { hasProxy } from '../../proxy'; import type { HostRule } from '../../types'; import * as hostRules from '../host-rules'; -import { dnsCache } from './dns'; +import { dnsLookup } from './dns'; import type { GotOptions } from './types'; export function findMatchingRules(options: GotOptions, url: string): HostRule { @@ -106,7 +106,7 @@ export function applyHostRules(url: string, inOptions: GotOptions): GotOptions { } if (foundRules.dnsCache) { - options.dnsCache = dnsCache; + options.lookup = dnsLookup; } if (!hasProxy() && foundRules.enableHttp2 === true) { diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index 84b19b6e68832937b57cc9d401633b72d51a54c0..079d3ce86b7e95da9bff0629f693b8ebf9b560a8 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -6,6 +6,7 @@ import { pkg } from '../../expose.cjs'; import { logger, setMeta } from '../../logger'; import { removeDanglingContainers } from '../../util/exec/docker'; import { deleteLocalFile, privateCacheDir } from '../../util/fs'; +import { clearDnsCache, printDnsStats } from '../../util/http/dns'; import * as queue from '../../util/http/queue'; import { addSplit, getSplits, splitInit } from '../../util/split'; import { setBranchCache } from './cache'; @@ -87,5 +88,7 @@ export async function renovateRepository( logger.debug(splits, 'Repository timing splits (milliseconds)'); printRequestStats(); logger.info({ durationMs: splits.total }, 'Repository finished'); + printDnsStats(); + clearDnsCache(); return repoResult; }