From 2eb7c7602714624eb6a5f9984b4bce9f764dfd19 Mon Sep 17 00:00:00 2001
From: RahulGautamSingh <rahultesnik@gmail.com>
Date: Wed, 5 Jun 2024 19:35:54 +0545
Subject: [PATCH] feat(http)!: remove `dnsCache` option (#29449)
BREAKING CHANGE: dnsCache is now removed
---
lib/types/host-rules.ts | 1 -
lib/util/http/dns.spec.ts | 50 --------------
lib/util/http/dns.ts | 111 -------------------------------
lib/util/http/host-rules.spec.ts | 17 -----
lib/util/http/host-rules.ts | 5 --
lib/workers/repository/index.ts | 3 -
package.json | 2 -
pnpm-lock.yaml | 22 ++----
8 files changed, 5 insertions(+), 206 deletions(-)
delete mode 100644 lib/util/http/dns.spec.ts
delete mode 100644 lib/util/http/dns.ts
diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts
index 23ac43d7f6..00820c81a8 100644
--- a/lib/types/host-rules.ts
+++ b/lib/types/host-rules.ts
@@ -14,7 +14,6 @@ export interface HostRule {
headers?: Record<string, string>;
maxRetryAfter?: number;
- dnsCache?: boolean;
keepAlive?: boolean;
artifactAuth?: string[] | null;
httpsCertificateAuthority?: string;
diff --git a/lib/util/http/dns.spec.ts b/lib/util/http/dns.spec.ts
deleted file mode 100644
index db574bc10e..0000000000
--- a/lib/util/http/dns.spec.ts
+++ /dev/null
@@ -1,50 +0,0 @@
-import { logger } from '../../logger';
-import { clearDnsCache, dnsLookup, printDnsStats } from './dns';
-
-describe('util/http/dns', () => {
- describe('dnsLookup', () => {
- it('works', async () => {
- clearDnsCache();
- const ip = await new Promise((resolve) =>
- dnsLookup('api.github.com', 4, (_e, r, _f) => {
- resolve(r);
- }),
- );
- expect(ip).toBeString();
- // uses cache
- expect(
- await new Promise((resolve) =>
- dnsLookup('api.github.com', (_e, r, _f) => {
- resolve(r);
- }),
- ),
- ).toBe(ip);
- expect(
- await new Promise((resolve) =>
- dnsLookup('api.github.com', {}, (_e, r, _f) => {
- resolve(r);
- }),
- ),
- ).toBe(ip);
- });
-
- it('throws', async () => {
- clearDnsCache();
- const ip = new Promise((resolve, reject) =>
- dnsLookup('api.github.comcccccccc', 4, (_e, r, _f) => {
- if (_e) {
- reject(_e);
- } else {
- resolve(r);
- }
- }),
- );
- await expect(ip).rejects.toThrow();
- });
-
- it('prints stats', () => {
- printDnsStats();
- expect(logger.debug).toHaveBeenCalled();
- });
- });
-});
diff --git a/lib/util/http/dns.ts b/lib/util/http/dns.ts
deleted file mode 100644
index 808060f0d0..0000000000
--- a/lib/util/http/dns.ts
+++ /dev/null
@@ -1,111 +0,0 @@
-import {
- LookupAllOptions,
- LookupOneOptions,
- lookup as _dnsLookup,
-} from 'node:dns';
-import type { EntryObject, IPFamily, LookupOptions } from 'cacheable-lookup';
-import { LRUCache } from 'lru-cache';
-import { logger } from '../../logger';
-
-const cache = new LRUCache<string, any>({ max: 1000 });
-
-function lookup(
- ...[host, options, callback]:
- | [
- hostname: string,
- family: IPFamily,
- callback: (
- error: NodeJS.ErrnoException,
- address: string,
- family: IPFamily,
- ) => void,
- ]
- | [
- hostname: string,
- callback: (
- error: NodeJS.ErrnoException,
- address: string,
- family: IPFamily,
- ) => void,
- ]
- | [
- hostname: string,
- options: LookupOptions & { all: true },
- callback: (
- error: NodeJS.ErrnoException,
- result: ReadonlyArray<EntryObject>,
- ) => void,
- ]
- | [
- hostname: string,
- options: LookupOptions,
- callback: (
- error: NodeJS.ErrnoException,
- address: string,
- family: IPFamily,
- ) => void,
- ]
-): void {
- let opts: LookupOneOptions | LookupAllOptions;
- // TODO: strict null incompatible types (#22198)
- let cb: any;
-
- if (typeof options === 'function') {
- opts = {};
- cb = options;
- } else if (typeof options === 'number') {
- opts = { family: options };
- cb = callback;
- } else {
- opts = options;
- cb = callback;
- }
-
- // istanbul ignore if: not used
- if (opts.all) {
- const key = `${host}_all`;
- if (cache.has(key)) {
- logger.trace({ host }, 'dns lookup cache hit');
- cb(null, cache.get(key));
- return;
- }
-
- _dnsLookup(host, opts, (err, res) => {
- if (err) {
- logger.debug({ host, err }, 'dns lookup error');
- cb(err, null, null);
- return;
- }
- logger.trace({ host, opts, res }, 'dns lookup');
- cache.set(key, res);
- cb(null, res, null);
- });
- } else {
- if (cache.has(host)) {
- logger.trace({ host }, 'dns lookup cache hit');
- cb(null, ...cache.get(host));
- return;
- }
-
- _dnsLookup(host, opts, (err, ...res) => {
- if (err) {
- logger.debug({ host, err }, 'dns lookup error');
- cb(err);
- return;
- }
- logger.trace({ host, opts, res }, 'dns lookup');
- cache.set(host, res);
- cb(null, ...res);
- });
- }
-}
-
-export { lookup as dnsLookup };
-
-export function printDnsStats(): void {
- logger.debug({ hosts: Array.from(cache.keys()) }, 'dns cache');
-}
-
-export function clearDnsCache(): void {
- cache.clear();
-}
diff --git a/lib/util/http/host-rules.spec.ts b/lib/util/http/host-rules.spec.ts
index 2b45213283..dcbaa06de7 100644
--- a/lib/util/http/host-rules.spec.ts
+++ b/lib/util/http/host-rules.spec.ts
@@ -2,7 +2,6 @@ import { GlobalConfig } from '../../config/global';
import { bootstrap } from '../../proxy';
import type { HostRule } from '../../types';
import * as hostRules from '../host-rules';
-import { dnsLookup } from './dns';
import { applyHostRule, findMatchingRule } from './host-rules';
import type { GotOptions } from './types';
@@ -125,22 +124,6 @@ describe('util/http/host-rules', () => {
});
});
- it('uses dnsCache', () => {
- hostRules.add({ dnsCache: true });
-
- const opts = { ...options, token: 'xxx' };
- const hostRule = findMatchingRule(url, opts);
- expect(hostRule).toEqual({
- dnsCache: true,
- token: 'token',
- });
- expect(applyHostRule(url, opts, hostRule)).toMatchObject({
- hostType: 'github',
- lookup: dnsLookup,
- token: 'xxx',
- });
- });
-
it('uses http keep-alive', () => {
hostRules.add({ keepAlive: true });
diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts
index ebbf1ed339..fa575ee8c8 100644
--- a/lib/util/http/host-rules.ts
+++ b/lib/util/http/host-rules.ts
@@ -12,7 +12,6 @@ import type { HostRule } from '../../types';
import * as hostRules from '../host-rules';
import { matchRegexOrGlobList } from '../string-match';
import { parseUrl } from '../url';
-import { dnsLookup } from './dns';
import { keepAliveAgents } from './keep-alive';
import type { GotOptions, InternalHttpOptions } from './types';
@@ -161,10 +160,6 @@ export function applyHostRule<GotOptions extends HostRulesGotOptions>(
options.timeout = hostRule.timeout;
}
- if (hostRule.dnsCache) {
- options.lookup = dnsLookup;
- }
-
if (hostRule.headers) {
const allowedHeaders = GlobalConfig.get('allowedHeaders', []);
const filteredHeaders: Record<string, string> = {};
diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts
index f9d30bcf78..9b05fd8420 100644
--- a/lib/workers/repository/index.ts
+++ b/lib/workers/repository/index.ts
@@ -16,7 +16,6 @@ import { removeDanglingContainers } from '../../util/exec/docker';
import { deleteLocalFile, privateCacheDir } from '../../util/fs';
import { isCloned } from '../../util/git';
import { detectSemanticCommits } from '../../util/git/semantic';
-import { clearDnsCache, printDnsStats } from '../../util/http/dns';
import * as queue from '../../util/http/queue';
import * as throttle from '../../util/http/throttle';
import { addSplit, getSplits, splitInit } from '../../util/split';
@@ -138,8 +137,6 @@ export async function renovateRepository(
HttpStats.report();
HttpCacheStats.report();
LookupStats.report();
- printDnsStats();
- clearDnsCache();
const cloned = isCloned();
logger.info({ cloned, durationMs: splits.total }, 'Repository finished');
resetRepositoryLogLevelRemaps();
diff --git a/package.json b/package.json
index d4fcdbecdb..b129126d00 100644
--- a/package.json
+++ b/package.json
@@ -175,7 +175,6 @@
"azure-devops-node-api": "14.0.1",
"bunyan": "1.8.15",
"cacache": "18.0.4",
- "cacheable-lookup": "5.0.4",
"chalk": "4.1.2",
"changelog-filename-regex": "2.0.1",
"clean-git-ref": "2.0.1",
@@ -213,7 +212,6 @@
"jsonata": "2.0.5",
"jsonc-parser": "3.3.1",
"klona": "2.0.6",
- "lru-cache": "10.4.3",
"luxon": "3.4.4",
"markdown-it": "14.1.0",
"markdown-table": "2.0.0",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 9ee258ea13..46cc82d1e7 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -111,9 +111,6 @@ importers:
cacache:
specifier: 18.0.4
version: 18.0.4
- cacheable-lookup:
- specifier: 5.0.4
- version: 5.0.4
chalk:
specifier: 4.1.2
version: 4.1.2
@@ -225,9 +222,6 @@ importers:
klona:
specifier: 2.0.6
version: 2.0.6
- lru-cache:
- specifier: 10.4.3
- version: 10.4.3
luxon:
specifier: 3.4.4
version: 3.4.4
@@ -4546,10 +4540,6 @@ packages:
resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==}
engines: {node: '>=8'}
- minipass@7.1.0:
- resolution: {integrity: sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==}
- engines: {node: '>=16 || 14 >=14.17'}
-
minipass@7.1.2:
resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
engines: {node: '>=16 || 14 >=14.17'}
@@ -10095,7 +10085,7 @@ snapshots:
fs-minipass@3.0.3:
dependencies:
- minipass: 7.1.0
+ minipass: 7.1.2
fs.realpath@1.0.0: {}
@@ -11258,7 +11248,7 @@ snapshots:
cacache: 18.0.4
http-cache-semantics: 4.1.1
is-lambda: 1.0.1
- minipass: 7.1.0
+ minipass: 7.1.2
minipass-fetch: 3.0.5
minipass-flush: 1.0.5
minipass-pipeline: 1.2.4
@@ -11458,11 +11448,11 @@ snapshots:
minipass-collect@2.0.1:
dependencies:
- minipass: 7.1.0
+ minipass: 7.1.2
minipass-fetch@3.0.5:
dependencies:
- minipass: 7.1.0
+ minipass: 7.1.2
minipass-sized: 1.0.3
minizlib: 2.1.2
optionalDependencies:
@@ -11490,8 +11480,6 @@ snapshots:
minipass@5.0.0: {}
- minipass@7.1.0: {}
-
minipass@7.1.2: {}
minizlib@2.1.2:
@@ -12468,7 +12456,7 @@ snapshots:
ssri@10.0.6:
dependencies:
- minipass: 7.1.0
+ minipass: 7.1.2
stack-utils@2.0.6:
dependencies:
--
GitLab