From 6e7cfdbfae59a5499b34b288f297c441a413f19b Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Thu, 23 Apr 2020 20:41:36 +0200 Subject: [PATCH 01/12] Add a fallback to dns.lookup --- README.md | 34 +++++++++++++++-------- benchmark.js | 6 ++--- source/index.js | 72 +++++++++++++++++++++++++++++++++---------------- 3 files changed, 75 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 079223e..0ebe53e 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,32 @@ Options used to cache the DNS lookups. Type: `number`
Default: `Infinity` -Limits the cache time (TTL in seconds). +The maximum lifetime of the entries received from the specifed DNS server (TTL in seconds). If set to `0`, it will make a new DNS query each time. +##### options.fallbackTtl + +Type: `number`
+Default: `1` + +The lifetime of the entries received from the OS (TTL in seconds). + +**Note**: This option is independent, `options.maxTtl` does not affect this. + +**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. + +##### options.errorTtl + +Type: `number`
+Default: `0.150` + +The time how long it needs to remember failed queries (TTL in seconds). + +**Note**: This option is independent, `options.maxTtl` does not affect this. + +**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. + ##### options.resolver Type: `dns.Resolver | dns.promises.Resolver`
@@ -146,16 +168,6 @@ Type: `object` The same as the [`dns.lookup(…)`](https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback) options. -##### options.throwNotFound - -Type: `boolean`
-Default: `true` - -If set to `false` and it gets no match, it will return `undefined`. -If set to `true` and it gets no match, it will throw `ENOTFOUND` error. - -**Note**: This option is meant **only** for the asynchronous implementation! The callback version will always pass an error if no match found. - #### query(hostname) An asynchronous function which returns cached DNS lookup entries.
diff --git a/benchmark.js b/benchmark.js index 571324b..978b4ed 100644 --- a/benchmark.js +++ b/benchmark.js @@ -24,12 +24,12 @@ const lookupOptionsADDRCONFIG = { hints: dns.ADDRCONFIG }; -const query = 'example.com'; +const query = 'short'; suite.add('CacheableLookup#lookupAsync', deferred => { // eslint-disable-next-line promise/prefer-await-to-then cacheable.lookupAsync(query).then(() => deferred.resolve()); -}, options).add('CacheableLookup#lookupAsync.all', deferred => { +}, options)/*.add('CacheableLookup#lookupAsync.all', deferred => { // eslint-disable-next-line promise/prefer-await-to-then cacheable.lookupAsync(query, lookupOptions).then(() => deferred.resolve()); }, options).add('CacheableLookup#lookupAsync.all.ADDRCONFIG', deferred => { @@ -54,7 +54,7 @@ suite.add('CacheableLookup#lookupAsync', deferred => { dns.lookup(query, lookupOptions, () => deferred.resolve()); }, options).add('dns#lookup.all.ADDRCONFIG', deferred => { dns.lookup(query, lookupOptionsADDRCONFIG, () => deferred.resolve()); -}, options).on('cycle', event => { +}, options)*/.on('cycle', event => { console.log(String(event.target)); }).on('complete', function () { console.log(`Fastest is ${this.filter('fastest').map('name')}`); diff --git a/source/index.js b/source/index.js index 490d096..4d9be35 100644 --- a/source/index.js +++ b/source/index.js @@ -4,7 +4,10 @@ const {promisify} = require('util'); const os = require('os'); const HostsResolver = require('./hosts-resolver'); -const {Resolver: AsyncResolver} = dnsPromises; +const { + Resolver: AsyncResolver, + lookup: dnsPromiseLookup +} = dnsPromises; const kCacheableLookupCreateConnection = Symbol('cacheableLookupCreateConnection'); const kCacheableLookupInstance = Symbol('cacheableLookupInstance'); @@ -95,9 +98,13 @@ class CacheableLookup { cache = new TTLMap(), maxTtl = Infinity, resolver = new AsyncResolver(), + fallbackTtl = 1, + errorTtl = 0.150, customHostsPath } = {}) { this.maxTtl = maxTtl; + this.fallbackTtl = fallbackTtl; + this.errorTtl = errorTtl; this._cache = cache; this._resolver = resolver; @@ -126,14 +133,14 @@ class CacheableLookup { return this._resolver.getServers(); } - lookup(hostname, options, callback) { + lookup(hostname, options = {}, callback) { if (typeof options === 'function') { callback = options; options = {}; } // eslint-disable-next-line promise/prefer-await-to-then - this.lookupAsync(hostname, options, true).then(result => { + this.lookupAsync(hostname, options).then(result => { if (options.all) { callback(null, result); } else { @@ -142,7 +149,7 @@ class CacheableLookup { }).catch(callback); } - async lookupAsync(hostname, options = {}, throwNotFound = undefined) { + async lookupAsync(hostname, options = {}) { let cached = await this.query(hostname); if (options.family === 6) { @@ -162,16 +169,6 @@ class CacheableLookup { cached = cached.filter(entry => entry.family === 6 ? _iface.has6 : _iface.has4); } - if (cached.length === 0) { - if (throwNotFound || options.throwNotFound !== false) { - const error = new Error(`ENOTFOUND ${hostname}`); - error.code = 'ENOTFOUND'; - error.hostname = hostname; - - throw error; - } - } - if (options.all) { return cached; } @@ -180,16 +177,26 @@ class CacheableLookup { return cached[0]; } - return this._getEntry(cached); + return this._getEntry(cached, hostname); } async query(hostname) { + this.tick(); + let cached = await this._hostsResolver.get(hostname) || await this._cache.get(hostname); - if (!cached || cached.length === 0) { + if (!cached) { cached = await this.queryAndCache(hostname); } + if (cached.length === 0) { + const error = new Error(`ENOTFOUND ${hostname}`); + error.code = 'ENOTFOUND'; + error.hostname = hostname; + + throw error; + } + cached = cached.map(entry => { return {...entry}; }); @@ -198,6 +205,7 @@ class CacheableLookup { } async queryAndCache(hostname) { + // We could make an ANY query, but DNS servers may reject that. const [As, AAAAs] = await Promise.all([this._resolve4(hostname, ttl).catch(() => []), this._resolve6(hostname, ttl).catch(() => [])]); let cacheTtl = 0; @@ -208,6 +216,7 @@ class CacheableLookup { entry.family = 4; entry.expires = now + (entry.ttl * 1000); + // Is TTL the same for all entries? cacheTtl = Math.max(cacheTtl, entry.ttl); } } @@ -217,13 +226,33 @@ class CacheableLookup { entry.family = 6; entry.expires = now + (entry.ttl * 1000); + // Is TTL the same for all entries? cacheTtl = Math.max(cacheTtl, entry.ttl); } } - const entries = [...(As || []), ...(AAAAs || [])]; + let entries = [...(As || []), ...(AAAAs || [])]; + + if (entries.length === 0) { + try { + entries = await dnsPromiseLookup(hostname, { + all: true + }); + + for (const entry of entries) { + entry.ttl = this.fallbackTtl; + entry.expires = now + (entry.ttl * 1000); + } + + cacheTtl = this.fallbackTtl * 1000; + } catch (error) { + await this._cache.set(hostname, [], this.errorTtl * 1000); - cacheTtl = Math.min(this.maxTtl, cacheTtl) * 1000; + throw error; + } + } else { + cacheTtl = Math.min(this.maxTtl, cacheTtl) * 1000; + } if (this.maxTtl > 0 && cacheTtl > 0) { await this._cache.set(hostname, entries, cacheTtl); @@ -232,7 +261,7 @@ class CacheableLookup { return entries; } - _getEntry(entries) { + _getEntry(entries, hostname) { return entries[Math.floor(Math.random() * entries.length)]; } @@ -241,7 +270,7 @@ class CacheableLookup { return; } - if (this._cache instanceof TTLMap) { + if (this._cache instanceof TTLMap && this._cache.size) { const now = Date.now(); for (const [hostname, expiry] of this._cache.expiries) { @@ -273,9 +302,6 @@ class CacheableLookup { agent.createConnection = (options, callback) => { if (!('lookup' in options)) { options.lookup = this.lookup; - - // Make sure the database is up to date - this.tick(); } return agent[kCacheableLookupCreateConnection](options, callback); From 683b1afc085d6a231f26f97edc532384f91c4016 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sat, 25 Apr 2020 13:02:26 +0200 Subject: [PATCH 02/12] Make tests pass --- README.md | 6 +- package.json | 2 +- source/hosts-resolver.js | 6 +- source/index.js | 22 ++++--- tests/test.js | 130 +++++++++++++++++++++++---------------- 5 files changed, 96 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 0ebe53e..c1a21aa 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ The lifetime of the entries received from the OS (TTL in seconds). **Note**: This option is independent, `options.maxTtl` does not affect this. -**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. +**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. ##### options.errorTtl @@ -88,7 +88,7 @@ The time how long it needs to remember failed queries (TTL in seconds). **Note**: This option is independent, `options.maxTtl` does not affect this. -**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. +**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. ##### options.resolver @@ -145,7 +145,7 @@ When `options.all` is `true`, then `callback(error, entries)` is called. Type: `Array` -DNS servers used to make the query. Can be overridden - then the new servers will be used. +The DNS servers used to make queries. Can be overridden - doing so will trigger `cacheableLookup.updateInterfaceInfo()`. #### [lookup(hostname, options, callback)](https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback) diff --git a/package.json b/package.json index 7591241..43429c2 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "homepage": "https://github.com/szmarczak/cacheable-lookup#readme", "devDependencies": { "@types/keyv": "^3.1.1", - "ava": "^3.1.0", + "ava": "^3.7.1", "benchmark": "^2.1.4", "coveralls": "^3.0.9", "keyv": "^4.0.0", diff --git a/source/hosts-resolver.js b/source/hosts-resolver.js index 00b56af..67651dc 100644 --- a/source/hosts-resolver.js +++ b/source/hosts-resolver.js @@ -36,15 +36,15 @@ class HostsResolver { return this._hosts; } - this._lastModifiedTime = mtimeMs; - this._hosts = {}; - let lines = await readFile(_hostsPath, fileOptions); lines = lines.replace(whitespaceRegExp, ' '); lines = lines.replace(tabRegExp, ' '); lines = lines.replace(startsWithWhitespaceRegExp, ''); lines = lines.split('\n'); + this._lastModifiedTime = mtimeMs; + this._hosts = {}; + for (const line of lines) { const parts = line.split(' '); diff --git a/source/index.js b/source/index.js index 4d9be35..7786cdf 100644 --- a/source/index.js +++ b/source/index.js @@ -126,6 +126,8 @@ class CacheableLookup { } set servers(servers) { + this.updateInterfaceInfo(); + this._resolver.setServers(servers); } @@ -146,7 +148,7 @@ class CacheableLookup { } else { callback(null, result.address, result.family, result.expires, result.ttl); } - }).catch(callback); + }, callback); } async lookupAsync(hostname, options = {}) { @@ -169,6 +171,14 @@ class CacheableLookup { cached = cached.filter(entry => entry.family === 6 ? _iface.has6 : _iface.has4); } + if (cached.length === 0) { + const error = new Error(`ENOTFOUND ${hostname}`); + error.code = 'ENOTFOUND'; + error.hostname = hostname; + + throw error; + } + if (options.all) { return cached; } @@ -189,14 +199,6 @@ class CacheableLookup { cached = await this.queryAndCache(hostname); } - if (cached.length === 0) { - const error = new Error(`ENOTFOUND ${hostname}`); - error.code = 'ENOTFOUND'; - error.hostname = hostname; - - throw error; - } - cached = cached.map(entry => { return {...entry}; }); @@ -270,7 +272,7 @@ class CacheableLookup { return; } - if (this._cache instanceof TTLMap && this._cache.size) { + if (this._cache instanceof TTLMap) { const now = Date.now(); for (const [hostname, expiry] of this._cache.expiries) { diff --git a/tests/test.js b/tests/test.js index 1d5db5c..71493bd 100644 --- a/tests/test.js +++ b/tests/test.js @@ -108,6 +108,30 @@ const createResolver = () => { resolve6: (hostname, options, callback) => { return resolver.resolve(hostname, {...options, family: 6}, callback); }, + lookup: (hostname, options, callback) => { + // We don't need to implement hints here + + if (!resolver.lookupData[hostname]) { + const error = new Error(`ENOTFOUND ${hostname}`); + error.code = 'ENOTFOUND'; + error.hostname = hostname; + + callback(error); + return; + } + + let entries = resolver.lookupData[hostname]; + + if (options.family === 4 || options.family === 6) { + entries = entries.filter(entry => entry.family === options.family); + } + + if (options.all) { + return entries; + } + + return entries[0]; + }, data: { '127.0.0.1': { localhost: [ @@ -142,6 +166,12 @@ const createResolver = () => { {address: '127.0.0.1', family: 4, ttl: 60} ] } + }, + lookupData: { + osHostname: [ + {address: '127.0.0.1', family: 4}, + {address: '127.0.0.2', family: 4} + ] } }; @@ -269,8 +299,7 @@ test('V4MAPPED hint', async t => { const cacheable = new CacheableLookup({resolver, customHostsPath: false}); // Make sure default behavior is right - let entries = await cacheable.lookupAsync('static4', {family: 6, throwNotFound: false}); - t.is(entries, undefined); + await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6}), {code: 'ENOTFOUND'}); // V4MAPPED entries = await cacheable.lookupAsync('static4', {family: 6, hints: V4MAPPED}); @@ -283,7 +312,7 @@ test('ADDRCONFIG hint', async t => { const CacheableLookup = mockedInterfaces({has4: true, has6: false}); const cacheable = new CacheableLookup({resolver, customHostsPath: false}); - t.is(await cacheable.lookupAsync('localhost', {family: 6, hints: ADDRCONFIG, throwNotFound: false}), undefined); + await t.throwsAsync(cacheable.lookupAsync('localhost', {family: 6, hints: ADDRCONFIG}), {code: 'ENOTFOUND'}); } //=> has6 = true, family = 6 @@ -302,7 +331,7 @@ test('ADDRCONFIG hint', async t => { const CacheableLookup = mockedInterfaces({has4: false, has6: true}); const cacheable = new CacheableLookup({resolver, customHostsPath: false}); - t.is(await cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG, throwNotFound: false}), undefined); + await t.throwsAsync(cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG}), {code: 'ENOTFOUND'}); } //=> has4 = true, family = 4 @@ -321,7 +350,7 @@ test('ADDRCONFIG hint', async t => { const CacheableLookup = mockedInterfaces({has4: false, has6: true}); const cacheable = new CacheableLookup({resolver, customHostsPath: false}); - t.is(await cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG, throwNotFound: false}), undefined); + await t.throwsAsync(cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG}), {code: 'ENOTFOUND'}); //=> has4 = true, family = 4 CacheableLookup._updateInterfaces({has4: true, has6: true}); // Override os.networkInterfaces() @@ -334,7 +363,7 @@ test('ADDRCONFIG hint', async t => { } }); -test('caching works', async t => { +test.serial('caching works', async t => { const cacheable = new CacheableLookup({resolver, customHostsPath: false}); // Make sure default behavior is right @@ -344,13 +373,18 @@ test('caching works', async t => { ]); // Update DNS data - resolver.data['127.0.0.1'].temporary[0].address = '127.0.0.2'; + const resovlerEntry = resolver.data['127.0.0.1'].temporary[0]; + const {address: resolverAddress} = resovlerEntry; + resovlerEntry.address = '127.0.0.2'; // Lookup again entries = await cacheable.lookupAsync('temporary', {all: true, family: 4}); verify(t, entries, [ {address: '127.0.0.1', family: 4} ]); + + // Restore back + resovlerEntry.address = resolverAddress; }); test('respects ttl', async t => { @@ -363,7 +397,9 @@ test('respects ttl', async t => { ]); // Update DNS data - resolver.data['127.0.0.1'].ttl[0].address = '127.0.0.2'; + const resolverEntry = resolver.data['127.0.0.1'].ttl[0]; + const {address: resolverAddress} = resolverEntry; + resolverEntry.address = '127.0.0.2'; // Wait until it expires await sleep(2000); @@ -373,21 +409,14 @@ test('respects ttl', async t => { verify(t, entries, [ {address: '127.0.0.2', family: 4} ]); -}); -test('`options.throwNotFound` is always `true` when using callback style', async t => { - const cacheable = new CacheableLookup({resolver, customHostsPath: false}); - - const lookup = promisify(cacheable.lookup.bind(cacheable)); - - await t.throwsAsync(lookup('static4', {family: 6, throwNotFound: false}), {code: 'ENOTFOUND'}); + // Restore back + resolverEntry.address = resolverAddress; }); -test('options.throwNotFound', async t => { +test('throw when there are entries available but not for the requested family', async t => { const cacheable = new CacheableLookup({resolver, customHostsPath: false}); - await t.notThrowsAsync(cacheable.lookupAsync('static4', {family: 6, throwNotFound: false})); - await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6, throwNotFound: true}), {code: 'ENOTFOUND'}); await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6}), {code: 'ENOTFOUND'}); }); @@ -396,7 +425,7 @@ test('custom servers', async t => { // .servers (get) t.deepEqual(cacheable.servers, ['127.0.0.1']); - t.is(await cacheable.lookupAsync('unique', {throwNotFound: false}), undefined); + await t.throwsAsync(cacheable.lookupAsync('unique'), {code: 'ENOTFOUND'}); // .servers (set) cacheable.servers = ['127.0.0.1', '192.168.0.100']; @@ -494,6 +523,9 @@ test('options.maxTtl', async t => { address: '127.0.0.2', family: 4 }); + + // Reset + resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.1'; } }); @@ -670,7 +702,7 @@ test('tick() works', async t => { await cacheable.lookupAsync('temporary'); t.is(cacheable._cache.size, 1); - await sleep(1000); + await sleep(1001); cacheable.tick(); t.is(cacheable._cache.size, 0); @@ -731,23 +763,19 @@ test('respects the `hosts` file', async t => { }); const getAddress = async hostname => { - const result = await cacheable.lookupAsync(hostname, {throwNotFound: false}); - - if (result) { - t.is(result.family, 4); - t.is(result.ttl, Infinity); - t.is(result.expires, Infinity); + const result = await cacheable.lookupAsync(hostname); - return result.address; - } + t.is(result.family, 4); + t.is(result.ttl, Infinity); + t.is(result.expires, Infinity); - return result; + return result.address; }; t.is(await getAddress('helloworld'), '127.0.0.1'); t.is(await getAddress('foobar'), '127.0.0.1'); - t.is(await getAddress('woofwoof'), undefined); - t.is(await getAddress('noiphere'), undefined); + await t.throwsAsync(getAddress('woofwoof'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('noiphere'), {code: 'ENOTFOUND'}); t.is(await getAddress('foo1'), '127.0.0.1'); t.is(await getAddress('foo2'), '127.0.0.1'); t.is(await getAddress('manywhitespaces'), '127.0.0.1'); @@ -790,31 +818,27 @@ test('the `hosts` file support can be turned off', async t => { }); const getAddress = async hostname => { - const result = await cacheable.lookupAsync(hostname, {throwNotFound: false}); - - if (result) { - t.is(result.family, 4); - t.is(result.ttl, Infinity); - t.is(result.expires, Infinity); + const result = await cacheable.lookupAsync(hostname); - return result.address; - } + t.is(result.family, 4); + t.is(result.ttl, Infinity); + t.is(result.expires, Infinity); - return result; + return result.address; }; - t.is(await getAddress('helloworld'), undefined); - t.is(await getAddress('foobar'), undefined); - t.is(await getAddress('woofwoof'), undefined); - t.is(await getAddress('noiphere'), undefined); - t.is(await getAddress('foo1'), undefined); - t.is(await getAddress('foo2'), undefined); - t.is(await getAddress('manywhitespaces'), undefined); - t.is(await getAddress('startswithwhitespace'), undefined); - t.is(await getAddress('tab'), undefined); - t.is(await cacheable.lookupAsync('foo3', {family: 4, throwNotFound: false}), undefined); - t.is(await cacheable.lookupAsync('foo3', {family: 6, throwNotFound: false}), undefined); - t.deepEqual(await cacheable.lookupAsync('foo4', {all: true, throwNotFound: false}), []); + await t.throwsAsync(getAddress('helloworld'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('foobar'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('woofwoof'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('noiphere'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('foo1'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('foo2'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('manywhitespaces'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('startswithwhitespace'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('tab'), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('foo3', {family: 4}), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('foo3', {family: 6}), {code: 'ENOTFOUND'}); + await t.throwsAsync(getAddress('foo4', {all: true}), {code: 'ENOTFOUND'}); const {address} = await cacheable.lookupAsync('localhost'); t.is(address, '127.0.0.1'); @@ -837,7 +861,7 @@ test('custom cache support', async t => { t.is(entry.family, 4); t.is(entry.ttl, 1); - await sleep(entry.ttl * 1000); + await sleep(entry.ttl * 1001); cacheable.tick(); From bfe9dbc163adebd8b0c223f4402a9dafdda12a5f Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sat, 25 Apr 2020 15:27:19 +0200 Subject: [PATCH 03/12] Fix linting --- benchmark.js | 16 ++-------------- source/index.js | 9 ++++++--- tests/test.js | 2 +- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/benchmark.js b/benchmark.js index 978b4ed..d441a6c 100644 --- a/benchmark.js +++ b/benchmark.js @@ -4,17 +4,12 @@ const Benchmark = require('benchmark'); const CacheableLookup = require('.'); const cacheable = new CacheableLookup(); -const notCacheable = new CacheableLookup({maxTtl: 0, customHostsPath: false}); const suite = new Benchmark.Suite(); const options = { defer: true }; -const resolve4Options = { - ttl: true -}; - const lookupOptions = { all: true }; @@ -29,7 +24,7 @@ const query = 'short'; suite.add('CacheableLookup#lookupAsync', deferred => { // eslint-disable-next-line promise/prefer-await-to-then cacheable.lookupAsync(query).then(() => deferred.resolve()); -}, options)/*.add('CacheableLookup#lookupAsync.all', deferred => { +}, options).add('CacheableLookup#lookupAsync.all', deferred => { // eslint-disable-next-line promise/prefer-await-to-then cacheable.lookupAsync(query, lookupOptions).then(() => deferred.resolve()); }, options).add('CacheableLookup#lookupAsync.all.ADDRCONFIG', deferred => { @@ -41,20 +36,13 @@ suite.add('CacheableLookup#lookupAsync', deferred => { cacheable.lookup(query, lookupOptions, () => deferred.resolve()); }, options).add('CacheableLookup#lookup.all.ADDRCONFIG', deferred => { cacheable.lookup(query, lookupOptionsADDRCONFIG, () => deferred.resolve()); -}, options).add('CacheableLookup#lookupAsync - zero TTL', deferred => { - // eslint-disable-next-line promise/prefer-await-to-then - notCacheable.lookupAsync(query, lookupOptions).then(() => deferred.resolve()); -}, options).add('CacheableLookup#lookup - zero TTL', deferred => { - notCacheable.lookup(query, lookupOptions, () => deferred.resolve()); -}, options).add('dns#resolve4', deferred => { - dns.resolve4(query, resolve4Options, () => deferred.resolve()); }, options).add('dns#lookup', deferred => { dns.lookup(query, () => deferred.resolve()); }, options).add('dns#lookup.all', deferred => { dns.lookup(query, lookupOptions, () => deferred.resolve()); }, options).add('dns#lookup.all.ADDRCONFIG', deferred => { dns.lookup(query, lookupOptionsADDRCONFIG, () => deferred.resolve()); -}, options)*/.on('cycle', event => { +}, options).on('cycle', event => { console.log(String(event.target)); }).on('complete', function () { console.log(`Fastest is ${this.filter('fastest').map('name')}`); diff --git a/source/index.js b/source/index.js index 7786cdf..15d9dd7 100644 --- a/source/index.js +++ b/source/index.js @@ -95,12 +95,12 @@ const ttl = {ttl: true}; class CacheableLookup { constructor({ + customHostsPath, cache = new TTLMap(), maxTtl = Infinity, resolver = new AsyncResolver(), fallbackTtl = 1, - errorTtl = 0.150, - customHostsPath + errorTtl = 0.15 } = {}) { this.maxTtl = maxTtl; this.fallbackTtl = fallbackTtl; @@ -135,10 +135,12 @@ class CacheableLookup { return this._resolver.getServers(); } - lookup(hostname, options = {}, callback) { + lookup(hostname, options, callback) { if (typeof options === 'function') { callback = options; options = {}; + } else if (!options) { + options = {}; } // eslint-disable-next-line promise/prefer-await-to-then @@ -263,6 +265,7 @@ class CacheableLookup { return entries; } + // eslint-disable-next-line no-unused-vars _getEntry(entries, hostname) { return entries[Math.floor(Math.random() * entries.length)]; } diff --git a/tests/test.js b/tests/test.js index 71493bd..96c8280 100644 --- a/tests/test.js +++ b/tests/test.js @@ -302,7 +302,7 @@ test('V4MAPPED hint', async t => { await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6}), {code: 'ENOTFOUND'}); // V4MAPPED - entries = await cacheable.lookupAsync('static4', {family: 6, hints: V4MAPPED}); + const entries = await cacheable.lookupAsync('static4', {family: 6, hints: V4MAPPED}); verify(t, entries, {address: '::ffff:127.0.0.1', family: 6}); }); From 890d8ab044a356cac674374cf3912f155ded1883 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sat, 25 Apr 2020 16:01:06 +0200 Subject: [PATCH 04/12] Remove redutant code --- README.md | 2 +- benchmark.js | 2 +- source/hosts-resolver.js | 50 ++++++++++++++++------------------ source/index.js | 58 +++++++--------------------------------- 4 files changed, 34 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index c1a21aa..fad3090 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,7 @@ Returns an array of objects with `address`, `family`, `ttl` and `expires` proper #### tick() -Removes outdated entries. +Removes outdated entries. It's automatically called on every lookup with a 1s lock. #### updateInterfaceInfo() diff --git a/benchmark.js b/benchmark.js index d441a6c..464b3b7 100644 --- a/benchmark.js +++ b/benchmark.js @@ -19,7 +19,7 @@ const lookupOptionsADDRCONFIG = { hints: dns.ADDRCONFIG }; -const query = 'short'; +const query = 'example.com'; suite.add('CacheableLookup#lookupAsync', deferred => { // eslint-disable-next-line promise/prefer-await-to-then diff --git a/source/hosts-resolver.js b/source/hosts-resolver.js index 67651dc..31e4496 100644 --- a/source/hosts-resolver.js +++ b/source/hosts-resolver.js @@ -1,5 +1,6 @@ 'use strict'; -const {stat, readFile} = require('fs').promises; +const {watchFile} = require('fs'); +const {readFile} = require('fs').promises; const {isIP} = require('net'); const isWindows = process.platform === 'win32'; @@ -19,30 +20,38 @@ const startsWithWhitespaceRegExp = /^[^\S\r\n]+/gm; class HostsResolver { constructor(customHostsPath = hostsPath) { this._hostsPath = customHostsPath; - this._promise = undefined; this._error = null; this._hosts = {}; - this._lastModifiedTime = 0; - this.update(); - } + this._promise = (async () => { + if (typeof this._hostsPath !== 'string') { + return; + } - async _update() { - try { - const {_hostsPath} = this; - const {mtimeMs} = await stat(_hostsPath); + await this._update(); - if (mtimeMs === this._lastModifiedTime) { - return this._hosts; + if (this._error) { + return; } - let lines = await readFile(_hostsPath, fileOptions); + watchFile(this._hostsPath, (currentTime, previousTime) => { + if (currentTime > previousTime) { + this._update(); + } + }); + + this._promise = null; + })(); + } + + async _update() { + try { + let lines = await readFile(this._hostsPath, fileOptions); lines = lines.replace(whitespaceRegExp, ' '); lines = lines.replace(tabRegExp, ' '); lines = lines.replace(startsWithWhitespaceRegExp, ''); lines = lines.split('\n'); - this._lastModifiedTime = mtimeMs; this._hosts = {}; for (const line of lines) { @@ -75,6 +84,7 @@ class HostsResolver { } } else { this._hosts[hostname] = []; + this._hosts[hostname].expires = Infinity; } this._hosts[hostname].push({ @@ -91,20 +101,6 @@ class HostsResolver { } } - async update() { - if (this._error || this._hostsPath === false) { - return this._hosts; - } - - const promise = this._update(); - - this._promise = promise; - await promise; - this._promise = undefined; - - return this._hosts; - } - async get(hostname) { if (this._promise) { await this._promise; diff --git a/source/index.js b/source/index.js index 15d9dd7..a17050a 100644 --- a/source/index.js +++ b/source/index.js @@ -50,53 +50,12 @@ const getIfaceInfo = () => { return {has4, has6}; }; -class TTLMap { - constructor() { - this.values = new Map(); - this.expiries = new Map(); - } - - set(key, value, ttl) { - this.values.set(key, value); - this.expiries.set(key, ttl && (ttl + Date.now())); - } - - get(key) { - const expiry = this.expiries.get(key); - - if (typeof expiry === 'number') { - if (Date.now() > expiry) { - this.values.delete(key); - this.expiries.delete(key); - - return; - } - } - - return this.values.get(key); - } - - delete(key) { - this.values.delete(key); - return this.expiries.delete(key); - } - - clear() { - this.values.clear(); - this.expiries.clear(); - } - - get size() { - return this.values.size; - } -} - const ttl = {ttl: true}; class CacheableLookup { constructor({ customHostsPath, - cache = new TTLMap(), + cache = new Map(), maxTtl = Infinity, resolver = new AsyncResolver(), fallbackTtl = 1, @@ -250,7 +209,10 @@ class CacheableLookup { cacheTtl = this.fallbackTtl * 1000; } catch (error) { - await this._cache.set(hostname, [], this.errorTtl * 1000); + cacheTtl = this.errorTtl * 1000; + + entries.expires = cacheTtl; + await this._cache.set(hostname, entries, cacheTtl); throw error; } @@ -259,6 +221,7 @@ class CacheableLookup { } if (this.maxTtl > 0 && cacheTtl > 0) { + entries.expires = cacheTtl; await this._cache.set(hostname, entries, cacheTtl); } @@ -275,18 +238,16 @@ class CacheableLookup { return; } - if (this._cache instanceof TTLMap) { + if (this._cache instanceof Map) { const now = Date.now(); - for (const [hostname, expiry] of this._cache.expiries) { - if (now > expiry) { + for (const [hostname, {expires}] of this._cache) { + if (now > expires) { this._cache.delete(hostname); } } } - this._hostsResolver.update(); - this._tickLocked = true; setTimeout(() => { @@ -330,7 +291,6 @@ class CacheableLookup { updateInterfaceInfo() { this._iface = getIfaceInfo(); - this._hostsResolver.update(); this._cache.clear(); } From 7ad3b31d107cd5af71ffb6537b9a7afcba9a2a9a Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sat, 25 Apr 2020 17:50:29 +0200 Subject: [PATCH 05/12] Fixes --- README.md | 8 ++--- source/index.js | 42 +++++++++++++---------- tests/test.js | 89 +++++++++++++++++++++++++++++++++++++------------ 3 files changed, 97 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index fad3090..a0b9d46 100644 --- a/README.md +++ b/README.md @@ -77,18 +77,18 @@ The lifetime of the entries received from the OS (TTL in seconds). **Note**: This option is independent, `options.maxTtl` does not affect this. -**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. +**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`. ##### options.errorTtl Type: `number`
-Default: `0.150` +Default: `0.15` The time how long it needs to remember failed queries (TTL in seconds). **Note**: This option is independent, `options.maxTtl` does not affect this. -**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. +**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`. ##### options.resolver @@ -186,7 +186,7 @@ Returns an array of objects with `address`, `family`, `ttl` and `expires` proper #### tick() -Removes outdated entries. It's automatically called on every lookup with a 1s lock. +Removes outdated entries. It's automatically called on every lookup. #### updateInterfaceInfo() diff --git a/source/index.js b/source/index.js index a17050a..44e2cf2 100644 --- a/source/index.js +++ b/source/index.js @@ -1,14 +1,16 @@ 'use strict'; -const {V4MAPPED, ADDRCONFIG, promises: dnsPromises} = require('dns'); +const { + V4MAPPED, + ADDRCONFIG, + promises: { + Resolver: AsyncResolver + }, + lookup +} = require('dns'); const {promisify} = require('util'); const os = require('os'); const HostsResolver = require('./hosts-resolver'); -const { - Resolver: AsyncResolver, - lookup: dnsPromiseLookup -} = dnsPromises; - const kCacheableLookupCreateConnection = Symbol('cacheableLookupCreateConnection'); const kCacheableLookupInstance = Symbol('cacheableLookupInstance'); @@ -65,9 +67,14 @@ class CacheableLookup { this.fallbackTtl = fallbackTtl; this.errorTtl = errorTtl; + // This value is in milliseconds + this._lockTime = Math.max(Math.floor(Math.min(this.fallbackTtl * 1000, this.errorTtl * 1000)), 10); + this._cache = cache; this._resolver = resolver; + this._lookup = promisify(lookup); + if (this._resolver instanceof AsyncResolver) { this._resolve4 = this._resolver.resolve4.bind(this._resolver); this._resolve6 = this._resolver.resolve6.bind(this._resolver); @@ -98,8 +105,10 @@ class CacheableLookup { if (typeof options === 'function') { callback = options; options = {}; - } else if (!options) { - options = {}; + } + + if (!callback) { + throw new Error('Callback must be a function.'); } // eslint-disable-next-line promise/prefer-await-to-then @@ -172,12 +181,11 @@ class CacheableLookup { const [As, AAAAs] = await Promise.all([this._resolve4(hostname, ttl).catch(() => []), this._resolve6(hostname, ttl).catch(() => [])]); let cacheTtl = 0; - const now = Date.now(); if (As) { for (const entry of As) { entry.family = 4; - entry.expires = now + (entry.ttl * 1000); + entry.expires = Date.now() + (entry.ttl * 1000); // Is TTL the same for all entries? cacheTtl = Math.max(cacheTtl, entry.ttl); @@ -187,7 +195,7 @@ class CacheableLookup { if (AAAAs) { for (const entry of AAAAs) { entry.family = 6; - entry.expires = now + (entry.ttl * 1000); + entry.expires = Date.now() + (entry.ttl * 1000); // Is TTL the same for all entries? cacheTtl = Math.max(cacheTtl, entry.ttl); @@ -198,20 +206,20 @@ class CacheableLookup { if (entries.length === 0) { try { - entries = await dnsPromiseLookup(hostname, { + entries = await this._lookup(hostname, { all: true }); for (const entry of entries) { entry.ttl = this.fallbackTtl; - entry.expires = now + (entry.ttl * 1000); + entry.expires = Date.now() + (entry.ttl * 1000); } cacheTtl = this.fallbackTtl * 1000; } catch (error) { cacheTtl = this.errorTtl * 1000; - entries.expires = cacheTtl; + entries.expires = Date.now() + cacheTtl; await this._cache.set(hostname, entries, cacheTtl); throw error; @@ -221,7 +229,7 @@ class CacheableLookup { } if (this.maxTtl > 0 && cacheTtl > 0) { - entries.expires = cacheTtl; + entries.expires = Date.now() + cacheTtl; await this._cache.set(hostname, entries, cacheTtl); } @@ -242,7 +250,7 @@ class CacheableLookup { const now = Date.now(); for (const [hostname, {expires}] of this._cache) { - if (now > expires) { + if (now >= expires) { this._cache.delete(hostname); } } @@ -252,7 +260,7 @@ class CacheableLookup { setTimeout(() => { this._tickLocked = false; - }, 1000).unref(); + }, this._lockTime).unref(); } install(agent) { diff --git a/tests/test.js b/tests/test.js index 96c8280..67a2d7b 100644 --- a/tests/test.js +++ b/tests/test.js @@ -127,10 +127,11 @@ const createResolver = () => { } if (options.all) { - return entries; + callback(null, entries); + return; } - return entries[0]; + callback(null, entries[0]); }, data: { '127.0.0.1': { @@ -402,7 +403,7 @@ test('respects ttl', async t => { resolverEntry.address = '127.0.0.2'; // Wait until it expires - await sleep(2000); + await sleep((resolverEntry.ttl * 1000) + 1); // Lookup again entries = await cacheable.lookupAsync('ttl', {all: true, family: 4}); @@ -487,10 +488,11 @@ test('options.maxTtl', async t => { }); // Update DNS data - resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.2'; + const resolverEntry = resolver.data['127.0.0.1'].maxTtl[0]; + resolverEntry.address = '127.0.0.2'; // Wait until it expires - await sleep(2000); + await sleep((cacheable.maxTtl * 1000) + 1); // Lookup again verify(t, await cacheable.lookupAsync('maxTtl'), { @@ -499,7 +501,7 @@ test('options.maxTtl', async t => { }); // Reset - resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.1'; + resolverEntry.address = '127.0.0.1'; } //=> maxTtl = 0 @@ -513,10 +515,11 @@ test('options.maxTtl', async t => { }); // Update DNS data - resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.2'; + const resolverEntry = resolver.data['127.0.0.1'].maxTtl[0]; + resolverEntry.address = '127.0.0.2'; // Wait until it expires - await sleep(1); + await sleep((cacheable.maxTtl * 1000) + 1); // Lookup again verify(t, await cacheable.lookupAsync('maxTtl'), { @@ -525,7 +528,7 @@ test('options.maxTtl', async t => { }); // Reset - resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.1'; + resolverEntry.address = '127.0.0.1'; } }); @@ -567,6 +570,10 @@ test('.lookup() and .lookupAsync() are automatically bounded', async t => { await t.notThrowsAsync(cacheable.lookupAsync('localhost')); await t.notThrowsAsync(promisify(cacheable.lookup)('localhost')); + + t.throws(() => cacheable.lookup('localhost'), { + message: 'Callback must be a function.' + }); }); test('works (Internet connection)', async t => { @@ -600,7 +607,7 @@ test.serial('install & uninstall', async t => { }); test('`.install()` throws if no Agent provided', t => { - const cacheable = new CacheableLookup(); + const cacheable = new CacheableLookup({customHostsPath: false}); t.throws(() => cacheable.install(), { message: 'Expected an Agent instance as the first argument' @@ -612,7 +619,7 @@ test('`.install()` throws if no Agent provided', t => { }); test('`.uninstall()` throws if no Agent provided', t => { - const cacheable = new CacheableLookup(); + const cacheable = new CacheableLookup({customHostsPath: false}); t.throws(() => cacheable.uninstall(), { message: 'Expected an Agent instance as the first argument' @@ -624,7 +631,7 @@ test('`.uninstall()` throws if no Agent provided', t => { }); test.serial('`.uninstall()` does not alter unmodified Agents', t => { - const cacheable = new CacheableLookup(); + const cacheable = new CacheableLookup({customHostsPath: false}); const {createConnection} = http.globalAgent; cacheable.uninstall(http.globalAgent); @@ -633,7 +640,7 @@ test.serial('`.uninstall()` does not alter unmodified Agents', t => { }); test.serial('throws if double-installing CacheableLookup', t => { - const cacheable = new CacheableLookup(); + const cacheable = new CacheableLookup({customHostsPath: false}); cacheable.install(http.globalAgent); t.throws(() => cacheable.install(http.globalAgent), { @@ -663,7 +670,7 @@ test.serial('install - providing custom lookup function anyway', async t => { }); test.serial('throws when calling `.uninstall()` on the wrong instance', t => { - const a = new CacheableLookup(); + const a = new CacheableLookup({customHostsPath: false}); const b = new CacheableLookup({resolver, customHostsPath: false}); a.install(http.globalAgent); @@ -702,7 +709,7 @@ test('tick() works', async t => { await cacheable.lookupAsync('temporary'); t.is(cacheable._cache.size, 1); - await sleep(1001); + await sleep((resolver.data['127.0.0.1'].temporary[0].ttl * 1000) + 1); cacheable.tick(); t.is(cacheable._cache.size, 0); @@ -710,25 +717,26 @@ test('tick() works', async t => { test('tick() doesn\'t delete active entries', async t => { const cacheable = new CacheableLookup({resolver, customHostsPath: false}); + cacheable.tick(); await cacheable.lookupAsync('temporary'); t.is(cacheable._cache.size, 1); - await sleep(500); + await sleep((cacheable._lockTime) + 1); cacheable.tick(); t.is(cacheable._cache.size, 1); }); -test('tick() is locked for 1s', async t => { - const cacheable = new CacheableLookup(); +test('tick() works properly', async t => { + const cacheable = new CacheableLookup({customHostsPath: false}); cacheable.tick(); t.true(cacheable._tickLocked); - const sleepPromise = sleep(1000); + const sleepPromise = sleep((cacheable._lockTime) + 1); - await sleep(800); + await sleep((cacheable._lockTime) - 10); t.true(cacheable._tickLocked); await sleepPromise; @@ -736,7 +744,7 @@ test('tick() is locked for 1s', async t => { }); test.serial('double tick() has no effect', t => { - const cacheable = new CacheableLookup(); + const cacheable = new CacheableLookup({customHostsPath: false}); const _setTimeout = setTimeout; global.setTimeout = (...args) => { @@ -906,3 +914,42 @@ test('lookup throws if failed to retrieve the `hosts` file', async t => { } ); }); + +test('fallback works', async t => { + const cacheable = new CacheableLookup({resolver, customHostsPath: false, fallbackTtl: 1}); + cacheable._lookup = promisify(resolver.lookup.bind(resolver)); + + const entries = await cacheable.lookupAsync('osHostname', {all: true}); + t.is(entries.length, 2); + + t.is(entries[0].address, '127.0.0.1'); + t.is(entries[0].family, 4); + + t.is(entries[1].address, '127.0.0.2'); + t.is(entries[1].family, 4); + + t.is(cacheable._cache.size, 1); + + await sleep((entries[0].ttl * 1000) + 1); + + cacheable.tick(); + + t.is(cacheable._cache.size, 0); +}); + +test('errors are cached', async t => { + const cacheable = new CacheableLookup({resolver, customHostsPath: false, errorTtl: 0.1}); + cacheable._lookup = promisify(resolver.lookup.bind(resolver)); + + await t.throwsAsync(cacheable.lookupAsync('doesNotExist'), { + code: 'ENOTFOUND' + }); + + t.is(cacheable._cache.size, 1); + + await sleep((cacheable.errorTtl * 1000) + 1); + + cacheable.tick(); + + t.is(cacheable._cache.size, 0); +}); From dae3c15d80e212669678ddee6fbf1cd6e0a14308 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sat, 25 Apr 2020 17:56:34 +0200 Subject: [PATCH 06/12] update readme --- README.md | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index a0b9d46..03104fb 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ Returns a new instance of `CacheableLookup`. #### cache -Type: [`TTLMap`](index.d.ts) | [`Keyv`](https://github.com/lukechilds/keyv/)
-Default: `new TTLMap()` +Type: `Map` | [`Keyv`](https://github.com/lukechilds/keyv/)
+Default: `new Map()` Custom cache instance. If `undefined`, it will create a new one. @@ -206,18 +206,13 @@ Performed on: - CPU governor: performance ``` -CacheableLookup#lookupAsync x 2,024,888 ops/sec ±0.84% (87 runs sampled) -CacheableLookup#lookupAsync.all x 2,093,860 ops/sec ±1.00% (88 runs sampled) -CacheableLookup#lookupAsync.all.ADDRCONFIG x 1,898,088 ops/sec ±0.61% (89 runs sampled) -CacheableLookup#lookup x 1,905,060 ops/sec ±0.76% (90 runs sampled) -CacheableLookup#lookup.all x 1,889,284 ops/sec ±1.37% (87 runs sampled) -CacheableLookup#lookup.all.ADDRCONFIG x 1,740,616 ops/sec ±0.83% (89 runs sampled) -CacheableLookup#lookupAsync - zero TTL x 226 ops/sec ±3.55% (56 runs sampled) -CacheableLookup#lookup - zero TTL x 228 ops/sec ±2.48% (62 runs sampled) -dns#resolve4 x 346 ops/sec ±3.58% (55 runs sampled) -dns#lookup x 20,368 ops/sec ±38.31% (53 runs sampled) -dns#lookup.all x 13,529 ops/sec ±31.35% (29 runs sampled) -dns#lookup.all.ADDRCONFIG x 6,211 ops/sec ±22.92% (26 runs sampled) +CacheableLookup#lookupAsync x 2,319,803 ops/sec ±0.82% (84 runs sampled) +CacheableLookup#lookupAsync.all x 2,419,856 ops/sec ±0.66% (89 runs sampled) +CacheableLookup#lookupAsync.all.ADDRCONFIG x 2,127,545 ops/sec ±1.04% (89 runs sampled) +CacheableLookup#lookup x 2,217,960 ops/sec ±1.15% (88 runs sampled) +CacheableLookup#lookup.all x 2,218,162 ops/sec ±0.71% (89 runs sampled) +CacheableLookup#lookup.all.ADDRCONFIG x 1,998,112 ops/sec ±0.75% (88 runs sampled) +!!! MISSING DNS.LOOKUP !!! Fastest is CacheableLookup#lookupAsync.all ``` From 44f0f360c5bd56d00ffa9a0f409db4cc62b06485 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sat, 25 Apr 2020 18:04:27 +0200 Subject: [PATCH 07/12] Proxy dns.lookup in tests --- tests/test.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test.js b/tests/test.js index 67a2d7b..408123a 100644 --- a/tests/test.js +++ b/tests/test.js @@ -6,7 +6,6 @@ const path = require('path'); const test = require('ava'); const Keyv = require('keyv'); const proxyquire = require('proxyquire'); -const CacheableLookup = require('../source'); const makeRequest = options => new Promise((resolve, reject) => { http.get(options, resolve).once('error', reject); @@ -181,6 +180,12 @@ const createResolver = () => { const resolver = createResolver(); +const CacheableLookup = proxyquire('../source', { + dns: { + lookup: resolver.lookup + } +}); + const verify = (t, entry, value) => { if (Array.isArray(value)) { // eslint-disable-next-line guard-for-in @@ -917,7 +922,6 @@ test('lookup throws if failed to retrieve the `hosts` file', async t => { test('fallback works', async t => { const cacheable = new CacheableLookup({resolver, customHostsPath: false, fallbackTtl: 1}); - cacheable._lookup = promisify(resolver.lookup.bind(resolver)); const entries = await cacheable.lookupAsync('osHostname', {all: true}); t.is(entries.length, 2); @@ -939,7 +943,6 @@ test('fallback works', async t => { test('errors are cached', async t => { const cacheable = new CacheableLookup({resolver, customHostsPath: false, errorTtl: 0.1}); - cacheable._lookup = promisify(resolver.lookup.bind(resolver)); await t.throwsAsync(cacheable.lookupAsync('doesNotExist'), { code: 'ENOTFOUND' From ad9f9540f95572ff1aae9a63f4bff60eea586ba5 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sat, 25 Apr 2020 18:15:07 +0200 Subject: [PATCH 08/12] nitpick --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 03104fb..714a4f8 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ The maximum lifetime of the entries received from the specifed DNS server (TTL i If set to `0`, it will make a new DNS query each time. +**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`. + ##### options.fallbackTtl Type: `number`
From 2511a66857b8a787eadb075689c78502c657b994 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sun, 26 Apr 2020 13:03:46 +0200 Subject: [PATCH 09/12] Add dns.lookup benchmarks --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 714a4f8..34cc83a 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,9 @@ CacheableLookup#lookupAsync.all.ADDRCONFIG x 2,127,545 ops/sec ±1.04% (89 runs CacheableLookup#lookup x 2,217,960 ops/sec ±1.15% (88 runs sampled) CacheableLookup#lookup.all x 2,218,162 ops/sec ±0.71% (89 runs sampled) CacheableLookup#lookup.all.ADDRCONFIG x 1,998,112 ops/sec ±0.75% (88 runs sampled) -!!! MISSING DNS.LOOKUP !!! +dns#lookup x 7,272 ops/sec ±0.36% (86 runs sampled) +dns#lookup.all x 7,249 ops/sec ±0.40% (86 runs sampled) +dns#lookup.all.ADDRCONFIG x 5,693 ops/sec ±0.28% (85 runs sampled) Fastest is CacheableLookup#lookupAsync.all ``` From eab270e8bec2ab61cf081e1f96445617287f8d71 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Sun, 26 Apr 2020 17:42:36 +0200 Subject: [PATCH 10/12] Update test.js --- tests/test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test.js b/tests/test.js index 408123a..18eab45 100644 --- a/tests/test.js +++ b/tests/test.js @@ -741,7 +741,9 @@ test('tick() works properly', async t => { const sleepPromise = sleep((cacheable._lockTime) + 1); - await sleep((cacheable._lockTime) - 10); + // This sometimes fails on GitHub Actions on Windows + // I suspect it's I/O is poor + await sleep((cacheable._lockTime) - 15); t.true(cacheable._tickLocked); await sleepPromise; From e93d1ea02b8da167ebb6a52af2b5bd6b0a1431da Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Mon, 27 Apr 2020 10:26:01 +0200 Subject: [PATCH 11/12] Improve types and increase Node.js compatibility --- README.md | 4 ++-- index.d.ts | 47 ++++++++++++++++++++++++++++++----------------- source/index.js | 21 ++++++++++++++++++--- tests/test.js | 24 ++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 34cc83a..a17b8de 100644 --- a/README.md +++ b/README.md @@ -196,9 +196,9 @@ Updates interface info. For example, you need to run this when you plug or unplu **Note:** Running `updateInterfaceInfo()` will also trigger `clear()`! -#### clear() +#### clear(hostname?) -Clears the cache. +Clears the cache for the given hostname. If the hostname argument is not present, the entire cache will be cleared. ## High performance diff --git a/index.d.ts b/index.d.ts index e0553fc..99be154 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,9 +1,11 @@ -import {Resolver, LookupAddress, promises as dnsPromises} from 'dns'; +import {Resolver, promises as dnsPromises} from 'dns'; import {Agent} from 'http'; -type AsyncResolver = dnsPromises.Resolver; +export {Resolver}; -type IPFamily = 4 | 6; +export type AsyncResolver = dnsPromises.Resolver; + +export type IPFamily = 4 | 6; type TPromise = T | Promise; @@ -35,9 +37,23 @@ export interface Options { * @default '/etc/hosts' */ customHostsPath?: string | false; + /** + * The lifetime of the entries received from the OS (TTL in seconds). + * + * **Note**: This option is independent, `options.maxTtl` does not affect this. + * @default 1 + */ + fallbackTtl?: number; + /** + * The time how long it needs to remember failed queries (TTL in seconds). + * + * **Note**: This option is independent, `options.maxTtl` does not affect this. + * @default 0.15 + */ + errorTtl?: number; } -interface EntryObject { +export interface EntryObject { /** * The IP address (can be an IPv4 or IPv5 address). */ @@ -56,7 +72,7 @@ interface EntryObject { readonly expires: number; } -interface LookupOptions { +export interface LookupOptions { /** * One or more supported getaddrinfo flags. Multiple flags may be passed by bitwise ORing their values. */ @@ -72,14 +88,6 @@ interface LookupOptions { all?: boolean; } -interface AsyncLookupOptions extends LookupOptions { - /** - * Throw when there's no match. If set to `false` and it gets no match, it will return `undefined`. - * @default false - */ - throwNotFound?: boolean; -} - export default class CacheableLookup { constructor(options?: Options); /** @@ -96,8 +104,8 @@ export default class CacheableLookup { /** * The asynchronous version of `dns.lookup(…)`. */ - lookupAsync(hostname: string, options: AsyncLookupOptions & {all: true}): Promise>; - lookupAsync(hostname: string, options: AsyncLookupOptions): Promise; + lookupAsync(hostname: string, options: LookupOptions & {all: true}): Promise>; + lookupAsync(hostname: string, options: LookupOptions): Promise; lookupAsync(hostname: string): Promise; lookupAsync(hostname: string, family: IPFamily): Promise; /** @@ -108,6 +116,11 @@ export default class CacheableLookup { * An asynchronous function which makes a new DNS lookup query and updates the database. This is used by `query(hostname, family)` if no entry in the database is present. Returns an array of objects with `address`, `family`, `ttl` and `expires` properties. */ queryAndCache(hostname: string): Promise>; + /** + * Returns an entry from the array for the given hostname. + * Useful to implement a round-robin algorithm. + */ + _getEntry(entries: ReadonlyArray, hostname: string): EntryObject; /** * Removes outdated entries. */ @@ -125,7 +138,7 @@ export default class CacheableLookup { */ updateInterfaceInfo(): void; /** - * Clears the cache. + * Clears the cache for the given hostname. If the hostname argument is not present, the entire cache will be cleared. */ - clear(): void; + clear(hostname?: string): void; } diff --git a/source/index.js b/source/index.js index 44e2cf2..6902d1f 100644 --- a/source/index.js +++ b/source/index.js @@ -105,6 +105,10 @@ class CacheableLookup { if (typeof options === 'function') { callback = options; options = {}; + } else if (typeof options === 'number') { + options = { + family: options + }; } if (!callback) { @@ -122,6 +126,12 @@ class CacheableLookup { } async lookupAsync(hostname, options = {}) { + if (typeof options === 'number') { + options = { + family: options + }; + } + let cached = await this.query(hostname); if (options.family === 6) { @@ -187,7 +197,7 @@ class CacheableLookup { entry.family = 4; entry.expires = Date.now() + (entry.ttl * 1000); - // Is TTL the same for all entries? + // Is the TTL the same for all entries? cacheTtl = Math.max(cacheTtl, entry.ttl); } } @@ -197,7 +207,7 @@ class CacheableLookup { entry.family = 6; entry.expires = Date.now() + (entry.ttl * 1000); - // Is TTL the same for all entries? + // Is the TTL the same for all entries? cacheTtl = Math.max(cacheTtl, entry.ttl); } } @@ -302,7 +312,12 @@ class CacheableLookup { this._cache.clear(); } - clear() { + clear(hostname) { + if (hostname) { + this._cache.delete(hostname); + return; + } + this._cache.clear(); } } diff --git a/tests/test.js b/tests/test.js index 18eab45..c2b5a9d 100644 --- a/tests/test.js +++ b/tests/test.js @@ -958,3 +958,27 @@ test('errors are cached', async t => { t.is(cacheable._cache.size, 0); }); + +test('passing family as options', async t => { + const cacheable = new CacheableLookup({resolver, customHostsPath: false}); + + const promisified = promisify(cacheable.lookup); + + const entry = await cacheable.lookupAsync('localhost', 6); + t.is(entry.address, '::ffff:127.0.0.2'); + t.is(entry.family, 6); + + const address = await promisified('localhost', 6); + t.is(address, '::ffff:127.0.0.2'); +}); + +test('clear(hostname) works', async t => { + const cacheable = new CacheableLookup({resolver, customHostsPath: false}); + + await cacheable.lookupAsync('localhost'); + await cacheable.lookupAsync('temporary'); + + cacheable.clear('localhost'); + + t.is(cacheable._cache.size, 1); +}); From 5f3afcb2d8f3b795eabf6cc940bab6df93ddd607 Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Mon, 27 Apr 2020 10:35:27 +0200 Subject: [PATCH 12/12] Improve TS tests --- index.d.ts | 4 +--- index.test-d.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/index.d.ts b/index.d.ts index 99be154..3b5f4e6 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,9 +1,7 @@ import {Resolver, promises as dnsPromises} from 'dns'; import {Agent} from 'http'; -export {Resolver}; - -export type AsyncResolver = dnsPromises.Resolver; +type AsyncResolver = dnsPromises.Resolver; export type IPFamily = 4 | 6; diff --git a/index.test-d.ts b/index.test-d.ts index 57f63fe..2ea6467 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -1,12 +1,20 @@ +import {Resolver} from 'dns'; +import {Agent} from 'https'; import {expectType} from 'tsd'; import Keyv = require('keyv'); import CacheableLookup, {EntryObject} from '.'; (async () => { const cacheable = new CacheableLookup(); + const agent = new Agent(); new CacheableLookup({ - cache: new Keyv() + cache: new Keyv(), + customHostsPath: false, + fallbackTtl: 0, + errorTtl: 0, + maxTtl: 0, + resolver: new Resolver() }); expectType(cacheable.servers); @@ -37,5 +45,8 @@ import CacheableLookup, {EntryObject} from '.'; expectType(cacheable.updateInterfaceInfo()); expectType(cacheable.tick()); + expectType(cacheable.install(agent)); + expectType(cacheable.uninstall(agent)); + expectType(cacheable.clear('localhost')); expectType(cacheable.clear()); })();