From aabb3abf9ab691fb22816aa0e33d4b379e831efd Mon Sep 17 00:00:00 2001
From: Michael Kriese <michael.kriese@visualon.de>
Date: Thu, 1 Sep 2022 17:23:46 +0200
Subject: [PATCH] feat(http): use own dns cache (#17574)
---
lib/util/http/dns.spec.ts | 50 ++++++++++++++
lib/util/http/dns.ts | 109 +++++++++++++++++++++++++++++--
lib/util/http/host-rules.spec.ts | 4 +-
lib/util/http/host-rules.ts | 4 +-
lib/workers/repository/index.ts | 3 +
5 files changed, 162 insertions(+), 8 deletions(-)
create mode 100644 lib/util/http/dns.spec.ts
diff --git a/lib/util/http/dns.spec.ts b/lib/util/http/dns.spec.ts
new file mode 100644
index 0000000000..b1e1477b57
--- /dev/null
+++ b/lib/util/http/dns.spec.ts
@@ -0,0 +1,50 @@
+import { logger } from '../../logger';
+import { clearDnsCache, dnsLookup, printDnsStats } from './dns';
+
+describe('util/http/dns', () => {
+ describe('dnsLookup', () => {
+ it('works', async () => {
+ clearDnsCache();
+ const ip = await new Promise((resolve) =>
+ dnsLookup('api.github.com', 4, (_e, r, _f) => {
+ resolve(r);
+ })
+ );
+ expect(ip).toBeString();
+ // uses cache
+ expect(
+ await new Promise((resolve) =>
+ dnsLookup('api.github.com', (_e, r, _f) => {
+ resolve(r);
+ })
+ )
+ ).toBe(ip);
+ expect(
+ await new Promise((resolve) =>
+ dnsLookup('api.github.com', {}, (_e, r, _f) => {
+ resolve(r);
+ })
+ )
+ ).toBe(ip);
+ });
+
+ it('throws', async () => {
+ clearDnsCache();
+ const ip = new Promise((resolve, reject) =>
+ dnsLookup('api.github.comcccccccc', 4, (_e, r, _f) => {
+ if (_e) {
+ reject(_e);
+ } else {
+ resolve(r);
+ }
+ })
+ );
+ await expect(ip).rejects.toThrow();
+ });
+
+ it('prints stats', () => {
+ printDnsStats();
+ expect(logger.debug).toHaveBeenCalled();
+ });
+ });
+});
diff --git a/lib/util/http/dns.ts b/lib/util/http/dns.ts
index e4c8479279..df49ad448e 100644
--- a/lib/util/http/dns.ts
+++ b/lib/util/http/dns.ts
@@ -1,6 +1,107 @@
-import CacheableLookup from 'cacheable-lookup';
+import { LookupAllOptions, LookupOneOptions, lookup as _dnsLookup } from 'dns';
+import type { EntryObject, IPFamily, LookupOptions } from 'cacheable-lookup';
import QuickLRU from 'quick-lru';
+import { logger } from '../../logger';
-export const dnsCache = new CacheableLookup({
- cache: new QuickLRU({ maxSize: 1000 }),
-});
+const cache = new QuickLRU<string, any>({ maxSize: 1000 });
+
+function lookup(
+ ...[host, options, callback]:
+ | [
+ hostname: string,
+ family: IPFamily,
+ callback: (
+ error: NodeJS.ErrnoException,
+ address: string,
+ family: IPFamily
+ ) => void
+ ]
+ | [
+ hostname: string,
+ callback: (
+ error: NodeJS.ErrnoException,
+ address: string,
+ family: IPFamily
+ ) => void
+ ]
+ | [
+ hostname: string,
+ options: LookupOptions & { all: true },
+ callback: (
+ error: NodeJS.ErrnoException,
+ result: ReadonlyArray<EntryObject>
+ ) => void
+ ]
+ | [
+ hostname: string,
+ options: LookupOptions,
+ callback: (
+ error: NodeJS.ErrnoException,
+ address: string,
+ family: IPFamily
+ ) => void
+ ]
+): void {
+ let opts: LookupOneOptions | LookupAllOptions;
+ // TODO: strict null incompatible types (#7154)
+ let cb: any;
+
+ if (typeof options === 'function') {
+ opts = {};
+ cb = options;
+ } else if (typeof options === 'number') {
+ opts = { family: options };
+ cb = callback;
+ } else {
+ opts = options;
+ cb = callback;
+ }
+
+ // istanbul ignore if: not used
+ if (opts.all) {
+ const key = `${host}_all`;
+ if (cache.has(key)) {
+ logger.trace({ host }, 'dns lookup cache hit');
+ cb(null, cache.get(key));
+ return;
+ }
+
+ _dnsLookup(host, opts, (err, res) => {
+ if (err) {
+ logger.error({ host, err }, 'dns lookup error');
+ cb(err, null, null);
+ return;
+ }
+ logger.trace({ host, opts, res }, 'dns lookup');
+ cache.set(key, res);
+ cb(null, res, null);
+ });
+ } else {
+ if (cache.has(host)) {
+ logger.trace({ host }, 'dns lookup cache hit');
+ cb(null, ...cache.get(host));
+ return;
+ }
+
+ _dnsLookup(host, opts, (err, ...res) => {
+ if (err) {
+ logger.error({ host, err }, 'dns lookup error');
+ cb(err);
+ return;
+ }
+ logger.trace({ host, opts, res }, 'dns lookup');
+ cache.set(host, res);
+ cb(null, ...res);
+ });
+ }
+}
+
+export { lookup as dnsLookup };
+
+export function printDnsStats(): void {
+ logger.debug({ hosts: Array.from(cache.keys()) }, 'dns cache');
+}
+
+export function clearDnsCache(): void {
+ cache.clear();
+}
diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts
index f640b31166..8be653b0d4 100644
--- a/lib/util/http/host-rules.spec.ts
+++ b/lib/util/http/host-rules.spec.ts
@@ -1,7 +1,7 @@
import { PlatformId } from '../../constants';
import { bootstrap } from '../../proxy';
import * as hostRules from '../host-rules';
-import { dnsCache } from './dns';
+import { dnsLookup } from './dns';
import { applyHostRules } from './host-rules';
const url = 'https://github.com';
@@ -114,7 +114,7 @@ describe('util/http/host-rules', () => {
hostRules.add({ dnsCache: true });
expect(applyHostRules(url, { ...options, token: 'xxx' })).toMatchObject({
hostType: 'github',
- dnsCache: dnsCache,
+ lookup: dnsLookup,
token: 'xxx',
});
});
diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts
index fab7ecd3b3..23310bf9d2 100644
--- a/lib/util/http/host-rules.ts
+++ b/lib/util/http/host-rules.ts
@@ -8,7 +8,7 @@ import { logger } from '../../logger';
import { hasProxy } from '../../proxy';
import type { HostRule } from '../../types';
import * as hostRules from '../host-rules';
-import { dnsCache } from './dns';
+import { dnsLookup } from './dns';
import type { GotOptions } from './types';
export function findMatchingRules(options: GotOptions, url: string): HostRule {
@@ -106,7 +106,7 @@ export function applyHostRules(url: string, inOptions: GotOptions): GotOptions {
}
if (foundRules.dnsCache) {
- options.dnsCache = dnsCache;
+ options.lookup = dnsLookup;
}
if (!hasProxy() && foundRules.enableHttp2 === true) {
diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts
index 84b19b6e68..079d3ce86b 100644
--- a/lib/workers/repository/index.ts
+++ b/lib/workers/repository/index.ts
@@ -6,6 +6,7 @@ import { pkg } from '../../expose.cjs';
import { logger, setMeta } from '../../logger';
import { removeDanglingContainers } from '../../util/exec/docker';
import { deleteLocalFile, privateCacheDir } from '../../util/fs';
+import { clearDnsCache, printDnsStats } from '../../util/http/dns';
import * as queue from '../../util/http/queue';
import { addSplit, getSplits, splitInit } from '../../util/split';
import { setBranchCache } from './cache';
@@ -87,5 +88,7 @@ export async function renovateRepository(
logger.debug(splits, 'Repository timing splits (milliseconds)');
printRequestStats();
logger.info({ durationMs: splits.total }, 'Repository finished');
+ printDnsStats();
+ clearDnsCache();
return repoResult;
}
--
GitLab