feat(http): use own dns cache (#17574)

lib/util/http/dns.spec.ts

@@ -0,0 +1,50 @@
+import { logger } from '../../logger';
+import { clearDnsCache, dnsLookup, printDnsStats } from './dns';
+
+describe('util/http/dns', () => {
+  describe('dnsLookup', () => {
+    it('works', async () => {
+      clearDnsCache();
+      const ip = await new Promise((resolve) =>
+        dnsLookup('api.github.com', 4, (_e, r, _f) => {
+          resolve(r);
+        })
+      );
+      expect(ip).toBeString();
+      // uses cache
+      expect(
+        await new Promise((resolve) =>
+          dnsLookup('api.github.com', (_e, r, _f) => {
+            resolve(r);
+          })
+        )
+      ).toBe(ip);
+      expect(
+        await new Promise((resolve) =>
+          dnsLookup('api.github.com', {}, (_e, r, _f) => {
+            resolve(r);
+          })
+        )
+      ).toBe(ip);
+    });
+
+    it('throws', async () => {
+      clearDnsCache();
+      const ip = new Promise((resolve, reject) =>
+        dnsLookup('api.github.comcccccccc', 4, (_e, r, _f) => {
+          if (_e) {
+            reject(_e);
+          } else {
+            resolve(r);
+          }
+        })
+      );
+      await expect(ip).rejects.toThrow();
+    });
+
+    it('prints stats', () => {
+      printDnsStats();
+      expect(logger.debug).toHaveBeenCalled();
+    });
+  });
+});

lib/util/http/dns.ts

@@ -1,6 +1,107 @@
-import CacheableLookup from 'cacheable-lookup';
+import { LookupAllOptions, LookupOneOptions, lookup as _dnsLookup } from 'dns';
+import type { EntryObject, IPFamily, LookupOptions } from 'cacheable-lookup';
 import QuickLRU from 'quick-lru';
+import { logger } from '../../logger';
 
-export const dnsCache = new CacheableLookup({
-  cache: new QuickLRU({ maxSize: 1000 }),
-});
+const cache = new QuickLRU<string, any>({ maxSize: 1000 });
+
+function lookup(
+  ...[host, options, callback]:
+    | [
+        hostname: string,
+        family: IPFamily,
+        callback: (
+          error: NodeJS.ErrnoException,
+          address: string,
+          family: IPFamily
+        ) => void
+      ]
+    | [
+        hostname: string,
+        callback: (
+          error: NodeJS.ErrnoException,
+          address: string,
+          family: IPFamily
+        ) => void
+      ]
+    | [
+        hostname: string,
+        options: LookupOptions & { all: true },
+        callback: (
+          error: NodeJS.ErrnoException,
+          result: ReadonlyArray<EntryObject>
+        ) => void
+      ]
+    | [
+        hostname: string,
+        options: LookupOptions,
+        callback: (
+          error: NodeJS.ErrnoException,
+          address: string,
+          family: IPFamily
+        ) => void
+      ]
+): void {
+  let opts: LookupOneOptions | LookupAllOptions;
+  // TODO: strict null incompatible types (#7154)
+  let cb: any;
+
+  if (typeof options === 'function') {
+    opts = {};
+    cb = options;
+  } else if (typeof options === 'number') {
+    opts = { family: options };
+    cb = callback;
+  } else {
+    opts = options;
+    cb = callback;
+  }
+
+  // istanbul ignore if: not used
+  if (opts.all) {
+    const key = `${host}_all`;
+    if (cache.has(key)) {
+      logger.trace({ host }, 'dns lookup cache hit');
+      cb(null, cache.get(key));
+      return;
+    }
+
+    _dnsLookup(host, opts, (err, res) => {
+      if (err) {
+        logger.error({ host, err }, 'dns lookup error');
+        cb(err, null, null);
+        return;
+      }
+      logger.trace({ host, opts, res }, 'dns lookup');
+      cache.set(key, res);
+      cb(null, res, null);
+    });
+  } else {
+    if (cache.has(host)) {
+      logger.trace({ host }, 'dns lookup cache hit');
+      cb(null, ...cache.get(host));
+      return;
+    }
+
+    _dnsLookup(host, opts, (err, ...res) => {
+      if (err) {
+        logger.error({ host, err }, 'dns lookup error');
+        cb(err);
+        return;
+      }
+      logger.trace({ host, opts, res }, 'dns lookup');
+      cache.set(host, res);
+      cb(null, ...res);
+    });
+  }
+}
+
+export { lookup as dnsLookup };
+
+export function printDnsStats(): void {
+  logger.debug({ hosts: Array.from(cache.keys()) }, 'dns cache');
+}
+
+export function clearDnsCache(): void {
+  cache.clear();
+}

lib/util/http/host-rules.spec.ts

@@ -1,7 +1,7 @@
 import { PlatformId } from '../../constants';
 import { bootstrap } from '../../proxy';
 import * as hostRules from '../host-rules';
-import { dnsCache } from './dns';
+import { dnsLookup } from './dns';
 import { applyHostRules } from './host-rules';
 
 const url = 'https://github.com';
@@ -114,7 +114,7 @@ describe('util/http/host-rules', () => {
     hostRules.add({ dnsCache: true });
     expect(applyHostRules(url, { ...options, token: 'xxx' })).toMatchObject({
       hostType: 'github',
-      dnsCache: dnsCache,
+      lookup: dnsLookup,
       token: 'xxx',
     });
   });

lib/util/http/host-rules.ts

@@ -8,7 +8,7 @@ import { logger } from '../../logger';
 import { hasProxy } from '../../proxy';
 import type { HostRule } from '../../types';
 import * as hostRules from '../host-rules';
-import { dnsCache } from './dns';
+import { dnsLookup } from './dns';
 import type { GotOptions } from './types';
 
 export function findMatchingRules(options: GotOptions, url: string): HostRule {
@@ -106,7 +106,7 @@ export function applyHostRules(url: string, inOptions: GotOptions): GotOptions {
   }
 
   if (foundRules.dnsCache) {
-    options.dnsCache = dnsCache;
+    options.lookup = dnsLookup;
   }
 
   if (!hasProxy() && foundRules.enableHttp2 === true) {

lib/workers/repository/index.ts

@@ -6,6 +6,7 @@ import { pkg } from '../../expose.cjs';
 import { logger, setMeta } from '../../logger';
 import { removeDanglingContainers } from '../../util/exec/docker';
 import { deleteLocalFile, privateCacheDir } from '../../util/fs';
+import { clearDnsCache, printDnsStats } from '../../util/http/dns';
 import * as queue from '../../util/http/queue';
 import { addSplit, getSplits, splitInit } from '../../util/split';
 import { setBranchCache } from './cache';
@@ -87,5 +88,7 @@ export async function renovateRepository(
   logger.debug(splits, 'Repository timing splits (milliseconds)');
   printRequestStats();
   logger.info({ durationMs: splits.total }, 'Repository finished');
+  printDnsStats();
+  clearDnsCache();
   return repoResult;
 }