From 6e7cfdbfae59a5499b34b288f297c441a413f19b Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Thu, 23 Apr 2020 20:41:36 +0200
Subject: [PATCH 01/12] Add a fallback to dns.lookup
---
README.md | 34 +++++++++++++++--------
benchmark.js | 6 ++---
source/index.js | 72 +++++++++++++++++++++++++++++++++----------------
3 files changed, 75 insertions(+), 37 deletions(-)
diff --git a/README.md b/README.md
index 079223e..0ebe53e 100644
--- a/README.md
+++ b/README.md
@@ -64,10 +64,32 @@ Options used to cache the DNS lookups.
Type: `number`
Default: `Infinity`
-Limits the cache time (TTL in seconds).
+The maximum lifetime of the entries received from the specifed DNS server (TTL in seconds).
If set to `0`, it will make a new DNS query each time.
+##### options.fallbackTtl
+
+Type: `number`
+Default: `1`
+
+The lifetime of the entries received from the OS (TTL in seconds).
+
+**Note**: This option is independent, `options.maxTtl` does not affect this.
+
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
+
+##### options.errorTtl
+
+Type: `number`
+Default: `0.150`
+
+The time how long it needs to remember failed queries (TTL in seconds).
+
+**Note**: This option is independent, `options.maxTtl` does not affect this.
+
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
+
##### options.resolver
Type: `dns.Resolver | dns.promises.Resolver`
@@ -146,16 +168,6 @@ Type: `object`
The same as the [`dns.lookup(…)`](https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback) options.
-##### options.throwNotFound
-
-Type: `boolean`
-Default: `true`
-
-If set to `false` and it gets no match, it will return `undefined`.
-If set to `true` and it gets no match, it will throw `ENOTFOUND` error.
-
-**Note**: This option is meant **only** for the asynchronous implementation! The callback version will always pass an error if no match found.
-
#### query(hostname)
An asynchronous function which returns cached DNS lookup entries.
diff --git a/benchmark.js b/benchmark.js
index 571324b..978b4ed 100644
--- a/benchmark.js
+++ b/benchmark.js
@@ -24,12 +24,12 @@ const lookupOptionsADDRCONFIG = {
hints: dns.ADDRCONFIG
};
-const query = 'example.com';
+const query = 'short';
suite.add('CacheableLookup#lookupAsync', deferred => {
// eslint-disable-next-line promise/prefer-await-to-then
cacheable.lookupAsync(query).then(() => deferred.resolve());
-}, options).add('CacheableLookup#lookupAsync.all', deferred => {
+}, options)/*.add('CacheableLookup#lookupAsync.all', deferred => {
// eslint-disable-next-line promise/prefer-await-to-then
cacheable.lookupAsync(query, lookupOptions).then(() => deferred.resolve());
}, options).add('CacheableLookup#lookupAsync.all.ADDRCONFIG', deferred => {
@@ -54,7 +54,7 @@ suite.add('CacheableLookup#lookupAsync', deferred => {
dns.lookup(query, lookupOptions, () => deferred.resolve());
}, options).add('dns#lookup.all.ADDRCONFIG', deferred => {
dns.lookup(query, lookupOptionsADDRCONFIG, () => deferred.resolve());
-}, options).on('cycle', event => {
+}, options)*/.on('cycle', event => {
console.log(String(event.target));
}).on('complete', function () {
console.log(`Fastest is ${this.filter('fastest').map('name')}`);
diff --git a/source/index.js b/source/index.js
index 490d096..4d9be35 100644
--- a/source/index.js
+++ b/source/index.js
@@ -4,7 +4,10 @@ const {promisify} = require('util');
const os = require('os');
const HostsResolver = require('./hosts-resolver');
-const {Resolver: AsyncResolver} = dnsPromises;
+const {
+ Resolver: AsyncResolver,
+ lookup: dnsPromiseLookup
+} = dnsPromises;
const kCacheableLookupCreateConnection = Symbol('cacheableLookupCreateConnection');
const kCacheableLookupInstance = Symbol('cacheableLookupInstance');
@@ -95,9 +98,13 @@ class CacheableLookup {
cache = new TTLMap(),
maxTtl = Infinity,
resolver = new AsyncResolver(),
+ fallbackTtl = 1,
+ errorTtl = 0.150,
customHostsPath
} = {}) {
this.maxTtl = maxTtl;
+ this.fallbackTtl = fallbackTtl;
+ this.errorTtl = errorTtl;
this._cache = cache;
this._resolver = resolver;
@@ -126,14 +133,14 @@ class CacheableLookup {
return this._resolver.getServers();
}
- lookup(hostname, options, callback) {
+ lookup(hostname, options = {}, callback) {
if (typeof options === 'function') {
callback = options;
options = {};
}
// eslint-disable-next-line promise/prefer-await-to-then
- this.lookupAsync(hostname, options, true).then(result => {
+ this.lookupAsync(hostname, options).then(result => {
if (options.all) {
callback(null, result);
} else {
@@ -142,7 +149,7 @@ class CacheableLookup {
}).catch(callback);
}
- async lookupAsync(hostname, options = {}, throwNotFound = undefined) {
+ async lookupAsync(hostname, options = {}) {
let cached = await this.query(hostname);
if (options.family === 6) {
@@ -162,16 +169,6 @@ class CacheableLookup {
cached = cached.filter(entry => entry.family === 6 ? _iface.has6 : _iface.has4);
}
- if (cached.length === 0) {
- if (throwNotFound || options.throwNotFound !== false) {
- const error = new Error(`ENOTFOUND ${hostname}`);
- error.code = 'ENOTFOUND';
- error.hostname = hostname;
-
- throw error;
- }
- }
-
if (options.all) {
return cached;
}
@@ -180,16 +177,26 @@ class CacheableLookup {
return cached[0];
}
- return this._getEntry(cached);
+ return this._getEntry(cached, hostname);
}
async query(hostname) {
+ this.tick();
+
let cached = await this._hostsResolver.get(hostname) || await this._cache.get(hostname);
- if (!cached || cached.length === 0) {
+ if (!cached) {
cached = await this.queryAndCache(hostname);
}
+ if (cached.length === 0) {
+ const error = new Error(`ENOTFOUND ${hostname}`);
+ error.code = 'ENOTFOUND';
+ error.hostname = hostname;
+
+ throw error;
+ }
+
cached = cached.map(entry => {
return {...entry};
});
@@ -198,6 +205,7 @@ class CacheableLookup {
}
async queryAndCache(hostname) {
+ // We could make an ANY query, but DNS servers may reject that.
const [As, AAAAs] = await Promise.all([this._resolve4(hostname, ttl).catch(() => []), this._resolve6(hostname, ttl).catch(() => [])]);
let cacheTtl = 0;
@@ -208,6 +216,7 @@ class CacheableLookup {
entry.family = 4;
entry.expires = now + (entry.ttl * 1000);
+ // Is TTL the same for all entries?
cacheTtl = Math.max(cacheTtl, entry.ttl);
}
}
@@ -217,13 +226,33 @@ class CacheableLookup {
entry.family = 6;
entry.expires = now + (entry.ttl * 1000);
+ // Is TTL the same for all entries?
cacheTtl = Math.max(cacheTtl, entry.ttl);
}
}
- const entries = [...(As || []), ...(AAAAs || [])];
+ let entries = [...(As || []), ...(AAAAs || [])];
+
+ if (entries.length === 0) {
+ try {
+ entries = await dnsPromiseLookup(hostname, {
+ all: true
+ });
+
+ for (const entry of entries) {
+ entry.ttl = this.fallbackTtl;
+ entry.expires = now + (entry.ttl * 1000);
+ }
+
+ cacheTtl = this.fallbackTtl * 1000;
+ } catch (error) {
+ await this._cache.set(hostname, [], this.errorTtl * 1000);
- cacheTtl = Math.min(this.maxTtl, cacheTtl) * 1000;
+ throw error;
+ }
+ } else {
+ cacheTtl = Math.min(this.maxTtl, cacheTtl) * 1000;
+ }
if (this.maxTtl > 0 && cacheTtl > 0) {
await this._cache.set(hostname, entries, cacheTtl);
@@ -232,7 +261,7 @@ class CacheableLookup {
return entries;
}
- _getEntry(entries) {
+ _getEntry(entries, hostname) {
return entries[Math.floor(Math.random() * entries.length)];
}
@@ -241,7 +270,7 @@ class CacheableLookup {
return;
}
- if (this._cache instanceof TTLMap) {
+ if (this._cache instanceof TTLMap && this._cache.size) {
const now = Date.now();
for (const [hostname, expiry] of this._cache.expiries) {
@@ -273,9 +302,6 @@ class CacheableLookup {
agent.createConnection = (options, callback) => {
if (!('lookup' in options)) {
options.lookup = this.lookup;
-
- // Make sure the database is up to date
- this.tick();
}
return agent[kCacheableLookupCreateConnection](options, callback);
From 683b1afc085d6a231f26f97edc532384f91c4016 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sat, 25 Apr 2020 13:02:26 +0200
Subject: [PATCH 02/12] Make tests pass
---
README.md | 6 +-
package.json | 2 +-
source/hosts-resolver.js | 6 +-
source/index.js | 22 ++++---
tests/test.js | 130 +++++++++++++++++++++++----------------
5 files changed, 96 insertions(+), 70 deletions(-)
diff --git a/README.md b/README.md
index 0ebe53e..c1a21aa 100644
--- a/README.md
+++ b/README.md
@@ -77,7 +77,7 @@ The lifetime of the entries received from the OS (TTL in seconds).
**Note**: This option is independent, `options.maxTtl` does not affect this.
-**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
##### options.errorTtl
@@ -88,7 +88,7 @@ The time how long it needs to remember failed queries (TTL in seconds).
**Note**: This option is independent, `options.maxTtl` does not affect this.
-**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
##### options.resolver
@@ -145,7 +145,7 @@ When `options.all` is `true`, then `callback(error, entries)` is called.
Type: `Array`
-DNS servers used to make the query. Can be overridden - then the new servers will be used.
+The DNS servers used to make queries. Can be overridden - doing so will trigger `cacheableLookup.updateInterfaceInfo()`.
#### [lookup(hostname, options, callback)](https://nodejs.org/api/dns.html#dns_dns_lookup_hostname_options_callback)
diff --git a/package.json b/package.json
index 7591241..43429c2 100644
--- a/package.json
+++ b/package.json
@@ -32,7 +32,7 @@
"homepage": "https://github.com/szmarczak/cacheable-lookup#readme",
"devDependencies": {
"@types/keyv": "^3.1.1",
- "ava": "^3.1.0",
+ "ava": "^3.7.1",
"benchmark": "^2.1.4",
"coveralls": "^3.0.9",
"keyv": "^4.0.0",
diff --git a/source/hosts-resolver.js b/source/hosts-resolver.js
index 00b56af..67651dc 100644
--- a/source/hosts-resolver.js
+++ b/source/hosts-resolver.js
@@ -36,15 +36,15 @@ class HostsResolver {
return this._hosts;
}
- this._lastModifiedTime = mtimeMs;
- this._hosts = {};
-
let lines = await readFile(_hostsPath, fileOptions);
lines = lines.replace(whitespaceRegExp, ' ');
lines = lines.replace(tabRegExp, ' ');
lines = lines.replace(startsWithWhitespaceRegExp, '');
lines = lines.split('\n');
+ this._lastModifiedTime = mtimeMs;
+ this._hosts = {};
+
for (const line of lines) {
const parts = line.split(' ');
diff --git a/source/index.js b/source/index.js
index 4d9be35..7786cdf 100644
--- a/source/index.js
+++ b/source/index.js
@@ -126,6 +126,8 @@ class CacheableLookup {
}
set servers(servers) {
+ this.updateInterfaceInfo();
+
this._resolver.setServers(servers);
}
@@ -146,7 +148,7 @@ class CacheableLookup {
} else {
callback(null, result.address, result.family, result.expires, result.ttl);
}
- }).catch(callback);
+ }, callback);
}
async lookupAsync(hostname, options = {}) {
@@ -169,6 +171,14 @@ class CacheableLookup {
cached = cached.filter(entry => entry.family === 6 ? _iface.has6 : _iface.has4);
}
+ if (cached.length === 0) {
+ const error = new Error(`ENOTFOUND ${hostname}`);
+ error.code = 'ENOTFOUND';
+ error.hostname = hostname;
+
+ throw error;
+ }
+
if (options.all) {
return cached;
}
@@ -189,14 +199,6 @@ class CacheableLookup {
cached = await this.queryAndCache(hostname);
}
- if (cached.length === 0) {
- const error = new Error(`ENOTFOUND ${hostname}`);
- error.code = 'ENOTFOUND';
- error.hostname = hostname;
-
- throw error;
- }
-
cached = cached.map(entry => {
return {...entry};
});
@@ -270,7 +272,7 @@ class CacheableLookup {
return;
}
- if (this._cache instanceof TTLMap && this._cache.size) {
+ if (this._cache instanceof TTLMap) {
const now = Date.now();
for (const [hostname, expiry] of this._cache.expiries) {
diff --git a/tests/test.js b/tests/test.js
index 1d5db5c..71493bd 100644
--- a/tests/test.js
+++ b/tests/test.js
@@ -108,6 +108,30 @@ const createResolver = () => {
resolve6: (hostname, options, callback) => {
return resolver.resolve(hostname, {...options, family: 6}, callback);
},
+ lookup: (hostname, options, callback) => {
+ // We don't need to implement hints here
+
+ if (!resolver.lookupData[hostname]) {
+ const error = new Error(`ENOTFOUND ${hostname}`);
+ error.code = 'ENOTFOUND';
+ error.hostname = hostname;
+
+ callback(error);
+ return;
+ }
+
+ let entries = resolver.lookupData[hostname];
+
+ if (options.family === 4 || options.family === 6) {
+ entries = entries.filter(entry => entry.family === options.family);
+ }
+
+ if (options.all) {
+ return entries;
+ }
+
+ return entries[0];
+ },
data: {
'127.0.0.1': {
localhost: [
@@ -142,6 +166,12 @@ const createResolver = () => {
{address: '127.0.0.1', family: 4, ttl: 60}
]
}
+ },
+ lookupData: {
+ osHostname: [
+ {address: '127.0.0.1', family: 4},
+ {address: '127.0.0.2', family: 4}
+ ]
}
};
@@ -269,8 +299,7 @@ test('V4MAPPED hint', async t => {
const cacheable = new CacheableLookup({resolver, customHostsPath: false});
// Make sure default behavior is right
- let entries = await cacheable.lookupAsync('static4', {family: 6, throwNotFound: false});
- t.is(entries, undefined);
+ await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6}), {code: 'ENOTFOUND'});
// V4MAPPED
entries = await cacheable.lookupAsync('static4', {family: 6, hints: V4MAPPED});
@@ -283,7 +312,7 @@ test('ADDRCONFIG hint', async t => {
const CacheableLookup = mockedInterfaces({has4: true, has6: false});
const cacheable = new CacheableLookup({resolver, customHostsPath: false});
- t.is(await cacheable.lookupAsync('localhost', {family: 6, hints: ADDRCONFIG, throwNotFound: false}), undefined);
+ await t.throwsAsync(cacheable.lookupAsync('localhost', {family: 6, hints: ADDRCONFIG}), {code: 'ENOTFOUND'});
}
//=> has6 = true, family = 6
@@ -302,7 +331,7 @@ test('ADDRCONFIG hint', async t => {
const CacheableLookup = mockedInterfaces({has4: false, has6: true});
const cacheable = new CacheableLookup({resolver, customHostsPath: false});
- t.is(await cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG, throwNotFound: false}), undefined);
+ await t.throwsAsync(cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG}), {code: 'ENOTFOUND'});
}
//=> has4 = true, family = 4
@@ -321,7 +350,7 @@ test('ADDRCONFIG hint', async t => {
const CacheableLookup = mockedInterfaces({has4: false, has6: true});
const cacheable = new CacheableLookup({resolver, customHostsPath: false});
- t.is(await cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG, throwNotFound: false}), undefined);
+ await t.throwsAsync(cacheable.lookupAsync('localhost', {family: 4, hints: ADDRCONFIG}), {code: 'ENOTFOUND'});
//=> has4 = true, family = 4
CacheableLookup._updateInterfaces({has4: true, has6: true}); // Override os.networkInterfaces()
@@ -334,7 +363,7 @@ test('ADDRCONFIG hint', async t => {
}
});
-test('caching works', async t => {
+test.serial('caching works', async t => {
const cacheable = new CacheableLookup({resolver, customHostsPath: false});
// Make sure default behavior is right
@@ -344,13 +373,18 @@ test('caching works', async t => {
]);
// Update DNS data
- resolver.data['127.0.0.1'].temporary[0].address = '127.0.0.2';
+ const resovlerEntry = resolver.data['127.0.0.1'].temporary[0];
+ const {address: resolverAddress} = resovlerEntry;
+ resovlerEntry.address = '127.0.0.2';
// Lookup again
entries = await cacheable.lookupAsync('temporary', {all: true, family: 4});
verify(t, entries, [
{address: '127.0.0.1', family: 4}
]);
+
+ // Restore back
+ resovlerEntry.address = resolverAddress;
});
test('respects ttl', async t => {
@@ -363,7 +397,9 @@ test('respects ttl', async t => {
]);
// Update DNS data
- resolver.data['127.0.0.1'].ttl[0].address = '127.0.0.2';
+ const resolverEntry = resolver.data['127.0.0.1'].ttl[0];
+ const {address: resolverAddress} = resolverEntry;
+ resolverEntry.address = '127.0.0.2';
// Wait until it expires
await sleep(2000);
@@ -373,21 +409,14 @@ test('respects ttl', async t => {
verify(t, entries, [
{address: '127.0.0.2', family: 4}
]);
-});
-test('`options.throwNotFound` is always `true` when using callback style', async t => {
- const cacheable = new CacheableLookup({resolver, customHostsPath: false});
-
- const lookup = promisify(cacheable.lookup.bind(cacheable));
-
- await t.throwsAsync(lookup('static4', {family: 6, throwNotFound: false}), {code: 'ENOTFOUND'});
+ // Restore back
+ resolverEntry.address = resolverAddress;
});
-test('options.throwNotFound', async t => {
+test('throw when there are entries available but not for the requested family', async t => {
const cacheable = new CacheableLookup({resolver, customHostsPath: false});
- await t.notThrowsAsync(cacheable.lookupAsync('static4', {family: 6, throwNotFound: false}));
- await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6, throwNotFound: true}), {code: 'ENOTFOUND'});
await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6}), {code: 'ENOTFOUND'});
});
@@ -396,7 +425,7 @@ test('custom servers', async t => {
// .servers (get)
t.deepEqual(cacheable.servers, ['127.0.0.1']);
- t.is(await cacheable.lookupAsync('unique', {throwNotFound: false}), undefined);
+ await t.throwsAsync(cacheable.lookupAsync('unique'), {code: 'ENOTFOUND'});
// .servers (set)
cacheable.servers = ['127.0.0.1', '192.168.0.100'];
@@ -494,6 +523,9 @@ test('options.maxTtl', async t => {
address: '127.0.0.2',
family: 4
});
+
+ // Reset
+ resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.1';
}
});
@@ -670,7 +702,7 @@ test('tick() works', async t => {
await cacheable.lookupAsync('temporary');
t.is(cacheable._cache.size, 1);
- await sleep(1000);
+ await sleep(1001);
cacheable.tick();
t.is(cacheable._cache.size, 0);
@@ -731,23 +763,19 @@ test('respects the `hosts` file', async t => {
});
const getAddress = async hostname => {
- const result = await cacheable.lookupAsync(hostname, {throwNotFound: false});
-
- if (result) {
- t.is(result.family, 4);
- t.is(result.ttl, Infinity);
- t.is(result.expires, Infinity);
+ const result = await cacheable.lookupAsync(hostname);
- return result.address;
- }
+ t.is(result.family, 4);
+ t.is(result.ttl, Infinity);
+ t.is(result.expires, Infinity);
- return result;
+ return result.address;
};
t.is(await getAddress('helloworld'), '127.0.0.1');
t.is(await getAddress('foobar'), '127.0.0.1');
- t.is(await getAddress('woofwoof'), undefined);
- t.is(await getAddress('noiphere'), undefined);
+ await t.throwsAsync(getAddress('woofwoof'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('noiphere'), {code: 'ENOTFOUND'});
t.is(await getAddress('foo1'), '127.0.0.1');
t.is(await getAddress('foo2'), '127.0.0.1');
t.is(await getAddress('manywhitespaces'), '127.0.0.1');
@@ -790,31 +818,27 @@ test('the `hosts` file support can be turned off', async t => {
});
const getAddress = async hostname => {
- const result = await cacheable.lookupAsync(hostname, {throwNotFound: false});
-
- if (result) {
- t.is(result.family, 4);
- t.is(result.ttl, Infinity);
- t.is(result.expires, Infinity);
+ const result = await cacheable.lookupAsync(hostname);
- return result.address;
- }
+ t.is(result.family, 4);
+ t.is(result.ttl, Infinity);
+ t.is(result.expires, Infinity);
- return result;
+ return result.address;
};
- t.is(await getAddress('helloworld'), undefined);
- t.is(await getAddress('foobar'), undefined);
- t.is(await getAddress('woofwoof'), undefined);
- t.is(await getAddress('noiphere'), undefined);
- t.is(await getAddress('foo1'), undefined);
- t.is(await getAddress('foo2'), undefined);
- t.is(await getAddress('manywhitespaces'), undefined);
- t.is(await getAddress('startswithwhitespace'), undefined);
- t.is(await getAddress('tab'), undefined);
- t.is(await cacheable.lookupAsync('foo3', {family: 4, throwNotFound: false}), undefined);
- t.is(await cacheable.lookupAsync('foo3', {family: 6, throwNotFound: false}), undefined);
- t.deepEqual(await cacheable.lookupAsync('foo4', {all: true, throwNotFound: false}), []);
+ await t.throwsAsync(getAddress('helloworld'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('foobar'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('woofwoof'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('noiphere'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('foo1'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('foo2'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('manywhitespaces'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('startswithwhitespace'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('tab'), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('foo3', {family: 4}), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('foo3', {family: 6}), {code: 'ENOTFOUND'});
+ await t.throwsAsync(getAddress('foo4', {all: true}), {code: 'ENOTFOUND'});
const {address} = await cacheable.lookupAsync('localhost');
t.is(address, '127.0.0.1');
@@ -837,7 +861,7 @@ test('custom cache support', async t => {
t.is(entry.family, 4);
t.is(entry.ttl, 1);
- await sleep(entry.ttl * 1000);
+ await sleep(entry.ttl * 1001);
cacheable.tick();
From bfe9dbc163adebd8b0c223f4402a9dafdda12a5f Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sat, 25 Apr 2020 15:27:19 +0200
Subject: [PATCH 03/12] Fix linting
---
benchmark.js | 16 ++--------------
source/index.js | 9 ++++++---
tests/test.js | 2 +-
3 files changed, 9 insertions(+), 18 deletions(-)
diff --git a/benchmark.js b/benchmark.js
index 978b4ed..d441a6c 100644
--- a/benchmark.js
+++ b/benchmark.js
@@ -4,17 +4,12 @@ const Benchmark = require('benchmark');
const CacheableLookup = require('.');
const cacheable = new CacheableLookup();
-const notCacheable = new CacheableLookup({maxTtl: 0, customHostsPath: false});
const suite = new Benchmark.Suite();
const options = {
defer: true
};
-const resolve4Options = {
- ttl: true
-};
-
const lookupOptions = {
all: true
};
@@ -29,7 +24,7 @@ const query = 'short';
suite.add('CacheableLookup#lookupAsync', deferred => {
// eslint-disable-next-line promise/prefer-await-to-then
cacheable.lookupAsync(query).then(() => deferred.resolve());
-}, options)/*.add('CacheableLookup#lookupAsync.all', deferred => {
+}, options).add('CacheableLookup#lookupAsync.all', deferred => {
// eslint-disable-next-line promise/prefer-await-to-then
cacheable.lookupAsync(query, lookupOptions).then(() => deferred.resolve());
}, options).add('CacheableLookup#lookupAsync.all.ADDRCONFIG', deferred => {
@@ -41,20 +36,13 @@ suite.add('CacheableLookup#lookupAsync', deferred => {
cacheable.lookup(query, lookupOptions, () => deferred.resolve());
}, options).add('CacheableLookup#lookup.all.ADDRCONFIG', deferred => {
cacheable.lookup(query, lookupOptionsADDRCONFIG, () => deferred.resolve());
-}, options).add('CacheableLookup#lookupAsync - zero TTL', deferred => {
- // eslint-disable-next-line promise/prefer-await-to-then
- notCacheable.lookupAsync(query, lookupOptions).then(() => deferred.resolve());
-}, options).add('CacheableLookup#lookup - zero TTL', deferred => {
- notCacheable.lookup(query, lookupOptions, () => deferred.resolve());
-}, options).add('dns#resolve4', deferred => {
- dns.resolve4(query, resolve4Options, () => deferred.resolve());
}, options).add('dns#lookup', deferred => {
dns.lookup(query, () => deferred.resolve());
}, options).add('dns#lookup.all', deferred => {
dns.lookup(query, lookupOptions, () => deferred.resolve());
}, options).add('dns#lookup.all.ADDRCONFIG', deferred => {
dns.lookup(query, lookupOptionsADDRCONFIG, () => deferred.resolve());
-}, options)*/.on('cycle', event => {
+}, options).on('cycle', event => {
console.log(String(event.target));
}).on('complete', function () {
console.log(`Fastest is ${this.filter('fastest').map('name')}`);
diff --git a/source/index.js b/source/index.js
index 7786cdf..15d9dd7 100644
--- a/source/index.js
+++ b/source/index.js
@@ -95,12 +95,12 @@ const ttl = {ttl: true};
class CacheableLookup {
constructor({
+ customHostsPath,
cache = new TTLMap(),
maxTtl = Infinity,
resolver = new AsyncResolver(),
fallbackTtl = 1,
- errorTtl = 0.150,
- customHostsPath
+ errorTtl = 0.15
} = {}) {
this.maxTtl = maxTtl;
this.fallbackTtl = fallbackTtl;
@@ -135,10 +135,12 @@ class CacheableLookup {
return this._resolver.getServers();
}
- lookup(hostname, options = {}, callback) {
+ lookup(hostname, options, callback) {
if (typeof options === 'function') {
callback = options;
options = {};
+ } else if (!options) {
+ options = {};
}
// eslint-disable-next-line promise/prefer-await-to-then
@@ -263,6 +265,7 @@ class CacheableLookup {
return entries;
}
+ // eslint-disable-next-line no-unused-vars
_getEntry(entries, hostname) {
return entries[Math.floor(Math.random() * entries.length)];
}
diff --git a/tests/test.js b/tests/test.js
index 71493bd..96c8280 100644
--- a/tests/test.js
+++ b/tests/test.js
@@ -302,7 +302,7 @@ test('V4MAPPED hint', async t => {
await t.throwsAsync(cacheable.lookupAsync('static4', {family: 6}), {code: 'ENOTFOUND'});
// V4MAPPED
- entries = await cacheable.lookupAsync('static4', {family: 6, hints: V4MAPPED});
+ const entries = await cacheable.lookupAsync('static4', {family: 6, hints: V4MAPPED});
verify(t, entries, {address: '::ffff:127.0.0.1', family: 6});
});
From 890d8ab044a356cac674374cf3912f155ded1883 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sat, 25 Apr 2020 16:01:06 +0200
Subject: [PATCH 04/12] Remove redutant code
---
README.md | 2 +-
benchmark.js | 2 +-
source/hosts-resolver.js | 50 ++++++++++++++++------------------
source/index.js | 58 +++++++---------------------------------
4 files changed, 34 insertions(+), 78 deletions(-)
diff --git a/README.md b/README.md
index c1a21aa..fad3090 100644
--- a/README.md
+++ b/README.md
@@ -186,7 +186,7 @@ Returns an array of objects with `address`, `family`, `ttl` and `expires` proper
#### tick()
-Removes outdated entries.
+Removes outdated entries. It's automatically called on every lookup with a 1s lock.
#### updateInterfaceInfo()
diff --git a/benchmark.js b/benchmark.js
index d441a6c..464b3b7 100644
--- a/benchmark.js
+++ b/benchmark.js
@@ -19,7 +19,7 @@ const lookupOptionsADDRCONFIG = {
hints: dns.ADDRCONFIG
};
-const query = 'short';
+const query = 'example.com';
suite.add('CacheableLookup#lookupAsync', deferred => {
// eslint-disable-next-line promise/prefer-await-to-then
diff --git a/source/hosts-resolver.js b/source/hosts-resolver.js
index 67651dc..31e4496 100644
--- a/source/hosts-resolver.js
+++ b/source/hosts-resolver.js
@@ -1,5 +1,6 @@
'use strict';
-const {stat, readFile} = require('fs').promises;
+const {watchFile} = require('fs');
+const {readFile} = require('fs').promises;
const {isIP} = require('net');
const isWindows = process.platform === 'win32';
@@ -19,30 +20,38 @@ const startsWithWhitespaceRegExp = /^[^\S\r\n]+/gm;
class HostsResolver {
constructor(customHostsPath = hostsPath) {
this._hostsPath = customHostsPath;
- this._promise = undefined;
this._error = null;
this._hosts = {};
- this._lastModifiedTime = 0;
- this.update();
- }
+ this._promise = (async () => {
+ if (typeof this._hostsPath !== 'string') {
+ return;
+ }
- async _update() {
- try {
- const {_hostsPath} = this;
- const {mtimeMs} = await stat(_hostsPath);
+ await this._update();
- if (mtimeMs === this._lastModifiedTime) {
- return this._hosts;
+ if (this._error) {
+ return;
}
- let lines = await readFile(_hostsPath, fileOptions);
+ watchFile(this._hostsPath, (currentTime, previousTime) => {
+ if (currentTime > previousTime) {
+ this._update();
+ }
+ });
+
+ this._promise = null;
+ })();
+ }
+
+ async _update() {
+ try {
+ let lines = await readFile(this._hostsPath, fileOptions);
lines = lines.replace(whitespaceRegExp, ' ');
lines = lines.replace(tabRegExp, ' ');
lines = lines.replace(startsWithWhitespaceRegExp, '');
lines = lines.split('\n');
- this._lastModifiedTime = mtimeMs;
this._hosts = {};
for (const line of lines) {
@@ -75,6 +84,7 @@ class HostsResolver {
}
} else {
this._hosts[hostname] = [];
+ this._hosts[hostname].expires = Infinity;
}
this._hosts[hostname].push({
@@ -91,20 +101,6 @@ class HostsResolver {
}
}
- async update() {
- if (this._error || this._hostsPath === false) {
- return this._hosts;
- }
-
- const promise = this._update();
-
- this._promise = promise;
- await promise;
- this._promise = undefined;
-
- return this._hosts;
- }
-
async get(hostname) {
if (this._promise) {
await this._promise;
diff --git a/source/index.js b/source/index.js
index 15d9dd7..a17050a 100644
--- a/source/index.js
+++ b/source/index.js
@@ -50,53 +50,12 @@ const getIfaceInfo = () => {
return {has4, has6};
};
-class TTLMap {
- constructor() {
- this.values = new Map();
- this.expiries = new Map();
- }
-
- set(key, value, ttl) {
- this.values.set(key, value);
- this.expiries.set(key, ttl && (ttl + Date.now()));
- }
-
- get(key) {
- const expiry = this.expiries.get(key);
-
- if (typeof expiry === 'number') {
- if (Date.now() > expiry) {
- this.values.delete(key);
- this.expiries.delete(key);
-
- return;
- }
- }
-
- return this.values.get(key);
- }
-
- delete(key) {
- this.values.delete(key);
- return this.expiries.delete(key);
- }
-
- clear() {
- this.values.clear();
- this.expiries.clear();
- }
-
- get size() {
- return this.values.size;
- }
-}
-
const ttl = {ttl: true};
class CacheableLookup {
constructor({
customHostsPath,
- cache = new TTLMap(),
+ cache = new Map(),
maxTtl = Infinity,
resolver = new AsyncResolver(),
fallbackTtl = 1,
@@ -250,7 +209,10 @@ class CacheableLookup {
cacheTtl = this.fallbackTtl * 1000;
} catch (error) {
- await this._cache.set(hostname, [], this.errorTtl * 1000);
+ cacheTtl = this.errorTtl * 1000;
+
+ entries.expires = cacheTtl;
+ await this._cache.set(hostname, entries, cacheTtl);
throw error;
}
@@ -259,6 +221,7 @@ class CacheableLookup {
}
if (this.maxTtl > 0 && cacheTtl > 0) {
+ entries.expires = cacheTtl;
await this._cache.set(hostname, entries, cacheTtl);
}
@@ -275,18 +238,16 @@ class CacheableLookup {
return;
}
- if (this._cache instanceof TTLMap) {
+ if (this._cache instanceof Map) {
const now = Date.now();
- for (const [hostname, expiry] of this._cache.expiries) {
- if (now > expiry) {
+ for (const [hostname, {expires}] of this._cache) {
+ if (now > expires) {
this._cache.delete(hostname);
}
}
}
- this._hostsResolver.update();
-
this._tickLocked = true;
setTimeout(() => {
@@ -330,7 +291,6 @@ class CacheableLookup {
updateInterfaceInfo() {
this._iface = getIfaceInfo();
- this._hostsResolver.update();
this._cache.clear();
}
From 7ad3b31d107cd5af71ffb6537b9a7afcba9a2a9a Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sat, 25 Apr 2020 17:50:29 +0200
Subject: [PATCH 05/12] Fixes
---
README.md | 8 ++---
source/index.js | 42 +++++++++++++----------
tests/test.js | 89 +++++++++++++++++++++++++++++++++++++------------
3 files changed, 97 insertions(+), 42 deletions(-)
diff --git a/README.md b/README.md
index fad3090..a0b9d46 100644
--- a/README.md
+++ b/README.md
@@ -77,18 +77,18 @@ The lifetime of the entries received from the OS (TTL in seconds).
**Note**: This option is independent, `options.maxTtl` does not affect this.
-**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`.
##### options.errorTtl
Type: `number`
-Default: `0.150`
+Default: `0.15`
The time how long it needs to remember failed queries (TTL in seconds).
**Note**: This option is independent, `options.maxTtl` does not affect this.
-**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks.
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`.
##### options.resolver
@@ -186,7 +186,7 @@ Returns an array of objects with `address`, `family`, `ttl` and `expires` proper
#### tick()
-Removes outdated entries. It's automatically called on every lookup with a 1s lock.
+Removes outdated entries. It's automatically called on every lookup.
#### updateInterfaceInfo()
diff --git a/source/index.js b/source/index.js
index a17050a..44e2cf2 100644
--- a/source/index.js
+++ b/source/index.js
@@ -1,14 +1,16 @@
'use strict';
-const {V4MAPPED, ADDRCONFIG, promises: dnsPromises} = require('dns');
+const {
+ V4MAPPED,
+ ADDRCONFIG,
+ promises: {
+ Resolver: AsyncResolver
+ },
+ lookup
+} = require('dns');
const {promisify} = require('util');
const os = require('os');
const HostsResolver = require('./hosts-resolver');
-const {
- Resolver: AsyncResolver,
- lookup: dnsPromiseLookup
-} = dnsPromises;
-
const kCacheableLookupCreateConnection = Symbol('cacheableLookupCreateConnection');
const kCacheableLookupInstance = Symbol('cacheableLookupInstance');
@@ -65,9 +67,14 @@ class CacheableLookup {
this.fallbackTtl = fallbackTtl;
this.errorTtl = errorTtl;
+ // This value is in milliseconds
+ this._lockTime = Math.max(Math.floor(Math.min(this.fallbackTtl * 1000, this.errorTtl * 1000)), 10);
+
this._cache = cache;
this._resolver = resolver;
+ this._lookup = promisify(lookup);
+
if (this._resolver instanceof AsyncResolver) {
this._resolve4 = this._resolver.resolve4.bind(this._resolver);
this._resolve6 = this._resolver.resolve6.bind(this._resolver);
@@ -98,8 +105,10 @@ class CacheableLookup {
if (typeof options === 'function') {
callback = options;
options = {};
- } else if (!options) {
- options = {};
+ }
+
+ if (!callback) {
+ throw new Error('Callback must be a function.');
}
// eslint-disable-next-line promise/prefer-await-to-then
@@ -172,12 +181,11 @@ class CacheableLookup {
const [As, AAAAs] = await Promise.all([this._resolve4(hostname, ttl).catch(() => []), this._resolve6(hostname, ttl).catch(() => [])]);
let cacheTtl = 0;
- const now = Date.now();
if (As) {
for (const entry of As) {
entry.family = 4;
- entry.expires = now + (entry.ttl * 1000);
+ entry.expires = Date.now() + (entry.ttl * 1000);
// Is TTL the same for all entries?
cacheTtl = Math.max(cacheTtl, entry.ttl);
@@ -187,7 +195,7 @@ class CacheableLookup {
if (AAAAs) {
for (const entry of AAAAs) {
entry.family = 6;
- entry.expires = now + (entry.ttl * 1000);
+ entry.expires = Date.now() + (entry.ttl * 1000);
// Is TTL the same for all entries?
cacheTtl = Math.max(cacheTtl, entry.ttl);
@@ -198,20 +206,20 @@ class CacheableLookup {
if (entries.length === 0) {
try {
- entries = await dnsPromiseLookup(hostname, {
+ entries = await this._lookup(hostname, {
all: true
});
for (const entry of entries) {
entry.ttl = this.fallbackTtl;
- entry.expires = now + (entry.ttl * 1000);
+ entry.expires = Date.now() + (entry.ttl * 1000);
}
cacheTtl = this.fallbackTtl * 1000;
} catch (error) {
cacheTtl = this.errorTtl * 1000;
- entries.expires = cacheTtl;
+ entries.expires = Date.now() + cacheTtl;
await this._cache.set(hostname, entries, cacheTtl);
throw error;
@@ -221,7 +229,7 @@ class CacheableLookup {
}
if (this.maxTtl > 0 && cacheTtl > 0) {
- entries.expires = cacheTtl;
+ entries.expires = Date.now() + cacheTtl;
await this._cache.set(hostname, entries, cacheTtl);
}
@@ -242,7 +250,7 @@ class CacheableLookup {
const now = Date.now();
for (const [hostname, {expires}] of this._cache) {
- if (now > expires) {
+ if (now >= expires) {
this._cache.delete(hostname);
}
}
@@ -252,7 +260,7 @@ class CacheableLookup {
setTimeout(() => {
this._tickLocked = false;
- }, 1000).unref();
+ }, this._lockTime).unref();
}
install(agent) {
diff --git a/tests/test.js b/tests/test.js
index 96c8280..67a2d7b 100644
--- a/tests/test.js
+++ b/tests/test.js
@@ -127,10 +127,11 @@ const createResolver = () => {
}
if (options.all) {
- return entries;
+ callback(null, entries);
+ return;
}
- return entries[0];
+ callback(null, entries[0]);
},
data: {
'127.0.0.1': {
@@ -402,7 +403,7 @@ test('respects ttl', async t => {
resolverEntry.address = '127.0.0.2';
// Wait until it expires
- await sleep(2000);
+ await sleep((resolverEntry.ttl * 1000) + 1);
// Lookup again
entries = await cacheable.lookupAsync('ttl', {all: true, family: 4});
@@ -487,10 +488,11 @@ test('options.maxTtl', async t => {
});
// Update DNS data
- resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.2';
+ const resolverEntry = resolver.data['127.0.0.1'].maxTtl[0];
+ resolverEntry.address = '127.0.0.2';
// Wait until it expires
- await sleep(2000);
+ await sleep((cacheable.maxTtl * 1000) + 1);
// Lookup again
verify(t, await cacheable.lookupAsync('maxTtl'), {
@@ -499,7 +501,7 @@ test('options.maxTtl', async t => {
});
// Reset
- resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.1';
+ resolverEntry.address = '127.0.0.1';
}
//=> maxTtl = 0
@@ -513,10 +515,11 @@ test('options.maxTtl', async t => {
});
// Update DNS data
- resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.2';
+ const resolverEntry = resolver.data['127.0.0.1'].maxTtl[0];
+ resolverEntry.address = '127.0.0.2';
// Wait until it expires
- await sleep(1);
+ await sleep((cacheable.maxTtl * 1000) + 1);
// Lookup again
verify(t, await cacheable.lookupAsync('maxTtl'), {
@@ -525,7 +528,7 @@ test('options.maxTtl', async t => {
});
// Reset
- resolver.data['127.0.0.1'].maxTtl[0].address = '127.0.0.1';
+ resolverEntry.address = '127.0.0.1';
}
});
@@ -567,6 +570,10 @@ test('.lookup() and .lookupAsync() are automatically bounded', async t => {
await t.notThrowsAsync(cacheable.lookupAsync('localhost'));
await t.notThrowsAsync(promisify(cacheable.lookup)('localhost'));
+
+ t.throws(() => cacheable.lookup('localhost'), {
+ message: 'Callback must be a function.'
+ });
});
test('works (Internet connection)', async t => {
@@ -600,7 +607,7 @@ test.serial('install & uninstall', async t => {
});
test('`.install()` throws if no Agent provided', t => {
- const cacheable = new CacheableLookup();
+ const cacheable = new CacheableLookup({customHostsPath: false});
t.throws(() => cacheable.install(), {
message: 'Expected an Agent instance as the first argument'
@@ -612,7 +619,7 @@ test('`.install()` throws if no Agent provided', t => {
});
test('`.uninstall()` throws if no Agent provided', t => {
- const cacheable = new CacheableLookup();
+ const cacheable = new CacheableLookup({customHostsPath: false});
t.throws(() => cacheable.uninstall(), {
message: 'Expected an Agent instance as the first argument'
@@ -624,7 +631,7 @@ test('`.uninstall()` throws if no Agent provided', t => {
});
test.serial('`.uninstall()` does not alter unmodified Agents', t => {
- const cacheable = new CacheableLookup();
+ const cacheable = new CacheableLookup({customHostsPath: false});
const {createConnection} = http.globalAgent;
cacheable.uninstall(http.globalAgent);
@@ -633,7 +640,7 @@ test.serial('`.uninstall()` does not alter unmodified Agents', t => {
});
test.serial('throws if double-installing CacheableLookup', t => {
- const cacheable = new CacheableLookup();
+ const cacheable = new CacheableLookup({customHostsPath: false});
cacheable.install(http.globalAgent);
t.throws(() => cacheable.install(http.globalAgent), {
@@ -663,7 +670,7 @@ test.serial('install - providing custom lookup function anyway', async t => {
});
test.serial('throws when calling `.uninstall()` on the wrong instance', t => {
- const a = new CacheableLookup();
+ const a = new CacheableLookup({customHostsPath: false});
const b = new CacheableLookup({resolver, customHostsPath: false});
a.install(http.globalAgent);
@@ -702,7 +709,7 @@ test('tick() works', async t => {
await cacheable.lookupAsync('temporary');
t.is(cacheable._cache.size, 1);
- await sleep(1001);
+ await sleep((resolver.data['127.0.0.1'].temporary[0].ttl * 1000) + 1);
cacheable.tick();
t.is(cacheable._cache.size, 0);
@@ -710,25 +717,26 @@ test('tick() works', async t => {
test('tick() doesn\'t delete active entries', async t => {
const cacheable = new CacheableLookup({resolver, customHostsPath: false});
+ cacheable.tick();
await cacheable.lookupAsync('temporary');
t.is(cacheable._cache.size, 1);
- await sleep(500);
+ await sleep((cacheable._lockTime) + 1);
cacheable.tick();
t.is(cacheable._cache.size, 1);
});
-test('tick() is locked for 1s', async t => {
- const cacheable = new CacheableLookup();
+test('tick() works properly', async t => {
+ const cacheable = new CacheableLookup({customHostsPath: false});
cacheable.tick();
t.true(cacheable._tickLocked);
- const sleepPromise = sleep(1000);
+ const sleepPromise = sleep((cacheable._lockTime) + 1);
- await sleep(800);
+ await sleep((cacheable._lockTime) - 10);
t.true(cacheable._tickLocked);
await sleepPromise;
@@ -736,7 +744,7 @@ test('tick() is locked for 1s', async t => {
});
test.serial('double tick() has no effect', t => {
- const cacheable = new CacheableLookup();
+ const cacheable = new CacheableLookup({customHostsPath: false});
const _setTimeout = setTimeout;
global.setTimeout = (...args) => {
@@ -906,3 +914,42 @@ test('lookup throws if failed to retrieve the `hosts` file', async t => {
}
);
});
+
+test('fallback works', async t => {
+ const cacheable = new CacheableLookup({resolver, customHostsPath: false, fallbackTtl: 1});
+ cacheable._lookup = promisify(resolver.lookup.bind(resolver));
+
+ const entries = await cacheable.lookupAsync('osHostname', {all: true});
+ t.is(entries.length, 2);
+
+ t.is(entries[0].address, '127.0.0.1');
+ t.is(entries[0].family, 4);
+
+ t.is(entries[1].address, '127.0.0.2');
+ t.is(entries[1].family, 4);
+
+ t.is(cacheable._cache.size, 1);
+
+ await sleep((entries[0].ttl * 1000) + 1);
+
+ cacheable.tick();
+
+ t.is(cacheable._cache.size, 0);
+});
+
+test('errors are cached', async t => {
+ const cacheable = new CacheableLookup({resolver, customHostsPath: false, errorTtl: 0.1});
+ cacheable._lookup = promisify(resolver.lookup.bind(resolver));
+
+ await t.throwsAsync(cacheable.lookupAsync('doesNotExist'), {
+ code: 'ENOTFOUND'
+ });
+
+ t.is(cacheable._cache.size, 1);
+
+ await sleep((cacheable.errorTtl * 1000) + 1);
+
+ cacheable.tick();
+
+ t.is(cacheable._cache.size, 0);
+});
From dae3c15d80e212669678ddee6fbf1cd6e0a14308 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sat, 25 Apr 2020 17:56:34 +0200
Subject: [PATCH 06/12] update readme
---
README.md | 23 +++++++++--------------
1 file changed, 9 insertions(+), 14 deletions(-)
diff --git a/README.md b/README.md
index a0b9d46..03104fb 100644
--- a/README.md
+++ b/README.md
@@ -45,8 +45,8 @@ Returns a new instance of `CacheableLookup`.
#### cache
-Type: [`TTLMap`](index.d.ts) | [`Keyv`](https://github.com/lukechilds/keyv/)
-Default: `new TTLMap()`
+Type: `Map` | [`Keyv`](https://github.com/lukechilds/keyv/)
+Default: `new Map()`
Custom cache instance. If `undefined`, it will create a new one.
@@ -206,18 +206,13 @@ Performed on:
- CPU governor: performance
```
-CacheableLookup#lookupAsync x 2,024,888 ops/sec ±0.84% (87 runs sampled)
-CacheableLookup#lookupAsync.all x 2,093,860 ops/sec ±1.00% (88 runs sampled)
-CacheableLookup#lookupAsync.all.ADDRCONFIG x 1,898,088 ops/sec ±0.61% (89 runs sampled)
-CacheableLookup#lookup x 1,905,060 ops/sec ±0.76% (90 runs sampled)
-CacheableLookup#lookup.all x 1,889,284 ops/sec ±1.37% (87 runs sampled)
-CacheableLookup#lookup.all.ADDRCONFIG x 1,740,616 ops/sec ±0.83% (89 runs sampled)
-CacheableLookup#lookupAsync - zero TTL x 226 ops/sec ±3.55% (56 runs sampled)
-CacheableLookup#lookup - zero TTL x 228 ops/sec ±2.48% (62 runs sampled)
-dns#resolve4 x 346 ops/sec ±3.58% (55 runs sampled)
-dns#lookup x 20,368 ops/sec ±38.31% (53 runs sampled)
-dns#lookup.all x 13,529 ops/sec ±31.35% (29 runs sampled)
-dns#lookup.all.ADDRCONFIG x 6,211 ops/sec ±22.92% (26 runs sampled)
+CacheableLookup#lookupAsync x 2,319,803 ops/sec ±0.82% (84 runs sampled)
+CacheableLookup#lookupAsync.all x 2,419,856 ops/sec ±0.66% (89 runs sampled)
+CacheableLookup#lookupAsync.all.ADDRCONFIG x 2,127,545 ops/sec ±1.04% (89 runs sampled)
+CacheableLookup#lookup x 2,217,960 ops/sec ±1.15% (88 runs sampled)
+CacheableLookup#lookup.all x 2,218,162 ops/sec ±0.71% (89 runs sampled)
+CacheableLookup#lookup.all.ADDRCONFIG x 1,998,112 ops/sec ±0.75% (88 runs sampled)
+!!! MISSING DNS.LOOKUP !!!
Fastest is CacheableLookup#lookupAsync.all
```
From 44f0f360c5bd56d00ffa9a0f409db4cc62b06485 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sat, 25 Apr 2020 18:04:27 +0200
Subject: [PATCH 07/12] Proxy dns.lookup in tests
---
tests/test.js | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/tests/test.js b/tests/test.js
index 67a2d7b..408123a 100644
--- a/tests/test.js
+++ b/tests/test.js
@@ -6,7 +6,6 @@ const path = require('path');
const test = require('ava');
const Keyv = require('keyv');
const proxyquire = require('proxyquire');
-const CacheableLookup = require('../source');
const makeRequest = options => new Promise((resolve, reject) => {
http.get(options, resolve).once('error', reject);
@@ -181,6 +180,12 @@ const createResolver = () => {
const resolver = createResolver();
+const CacheableLookup = proxyquire('../source', {
+ dns: {
+ lookup: resolver.lookup
+ }
+});
+
const verify = (t, entry, value) => {
if (Array.isArray(value)) {
// eslint-disable-next-line guard-for-in
@@ -917,7 +922,6 @@ test('lookup throws if failed to retrieve the `hosts` file', async t => {
test('fallback works', async t => {
const cacheable = new CacheableLookup({resolver, customHostsPath: false, fallbackTtl: 1});
- cacheable._lookup = promisify(resolver.lookup.bind(resolver));
const entries = await cacheable.lookupAsync('osHostname', {all: true});
t.is(entries.length, 2);
@@ -939,7 +943,6 @@ test('fallback works', async t => {
test('errors are cached', async t => {
const cacheable = new CacheableLookup({resolver, customHostsPath: false, errorTtl: 0.1});
- cacheable._lookup = promisify(resolver.lookup.bind(resolver));
await t.throwsAsync(cacheable.lookupAsync('doesNotExist'), {
code: 'ENOTFOUND'
From ad9f9540f95572ff1aae9a63f4bff60eea586ba5 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sat, 25 Apr 2020 18:15:07 +0200
Subject: [PATCH 08/12] nitpick
---
README.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/README.md b/README.md
index 03104fb..714a4f8 100644
--- a/README.md
+++ b/README.md
@@ -68,6 +68,8 @@ The maximum lifetime of the entries received from the specifed DNS server (TTL i
If set to `0`, it will make a new DNS query each time.
+**Pro Tip**: This shouldn't be lower than your DNS server response time in order to prevent bottlenecks. For example, if you use Cloudflare, this value should be greater than `0.01`.
+
##### options.fallbackTtl
Type: `number`
From 2511a66857b8a787eadb075689c78502c657b994 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sun, 26 Apr 2020 13:03:46 +0200
Subject: [PATCH 09/12] Add dns.lookup benchmarks
---
README.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 714a4f8..34cc83a 100644
--- a/README.md
+++ b/README.md
@@ -214,7 +214,9 @@ CacheableLookup#lookupAsync.all.ADDRCONFIG x 2,127,545 ops/sec ±1.04% (89 runs
CacheableLookup#lookup x 2,217,960 ops/sec ±1.15% (88 runs sampled)
CacheableLookup#lookup.all x 2,218,162 ops/sec ±0.71% (89 runs sampled)
CacheableLookup#lookup.all.ADDRCONFIG x 1,998,112 ops/sec ±0.75% (88 runs sampled)
-!!! MISSING DNS.LOOKUP !!!
+dns#lookup x 7,272 ops/sec ±0.36% (86 runs sampled)
+dns#lookup.all x 7,249 ops/sec ±0.40% (86 runs sampled)
+dns#lookup.all.ADDRCONFIG x 5,693 ops/sec ±0.28% (85 runs sampled)
Fastest is CacheableLookup#lookupAsync.all
```
From eab270e8bec2ab61cf081e1f96445617287f8d71 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Sun, 26 Apr 2020 17:42:36 +0200
Subject: [PATCH 10/12] Update test.js
---
tests/test.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tests/test.js b/tests/test.js
index 408123a..18eab45 100644
--- a/tests/test.js
+++ b/tests/test.js
@@ -741,7 +741,9 @@ test('tick() works properly', async t => {
const sleepPromise = sleep((cacheable._lockTime) + 1);
- await sleep((cacheable._lockTime) - 10);
+ // This sometimes fails on GitHub Actions on Windows
+ // I suspect it's I/O is poor
+ await sleep((cacheable._lockTime) - 15);
t.true(cacheable._tickLocked);
await sleepPromise;
From e93d1ea02b8da167ebb6a52af2b5bd6b0a1431da Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Mon, 27 Apr 2020 10:26:01 +0200
Subject: [PATCH 11/12] Improve types and increase Node.js compatibility
---
README.md | 4 ++--
index.d.ts | 47 ++++++++++++++++++++++++++++++-----------------
source/index.js | 21 ++++++++++++++++++---
tests/test.js | 24 ++++++++++++++++++++++++
4 files changed, 74 insertions(+), 22 deletions(-)
diff --git a/README.md b/README.md
index 34cc83a..a17b8de 100644
--- a/README.md
+++ b/README.md
@@ -196,9 +196,9 @@ Updates interface info. For example, you need to run this when you plug or unplu
**Note:** Running `updateInterfaceInfo()` will also trigger `clear()`!
-#### clear()
+#### clear(hostname?)
-Clears the cache.
+Clears the cache for the given hostname. If the hostname argument is not present, the entire cache will be cleared.
## High performance
diff --git a/index.d.ts b/index.d.ts
index e0553fc..99be154 100644
--- a/index.d.ts
+++ b/index.d.ts
@@ -1,9 +1,11 @@
-import {Resolver, LookupAddress, promises as dnsPromises} from 'dns';
+import {Resolver, promises as dnsPromises} from 'dns';
import {Agent} from 'http';
-type AsyncResolver = dnsPromises.Resolver;
+export {Resolver};
-type IPFamily = 4 | 6;
+export type AsyncResolver = dnsPromises.Resolver;
+
+export type IPFamily = 4 | 6;
type TPromise = T | Promise;
@@ -35,9 +37,23 @@ export interface Options {
* @default '/etc/hosts'
*/
customHostsPath?: string | false;
+ /**
+ * The lifetime of the entries received from the OS (TTL in seconds).
+ *
+ * **Note**: This option is independent, `options.maxTtl` does not affect this.
+ * @default 1
+ */
+ fallbackTtl?: number;
+ /**
+ * The time how long it needs to remember failed queries (TTL in seconds).
+ *
+ * **Note**: This option is independent, `options.maxTtl` does not affect this.
+ * @default 0.15
+ */
+ errorTtl?: number;
}
-interface EntryObject {
+export interface EntryObject {
/**
* The IP address (can be an IPv4 or IPv5 address).
*/
@@ -56,7 +72,7 @@ interface EntryObject {
readonly expires: number;
}
-interface LookupOptions {
+export interface LookupOptions {
/**
* One or more supported getaddrinfo flags. Multiple flags may be passed by bitwise ORing their values.
*/
@@ -72,14 +88,6 @@ interface LookupOptions {
all?: boolean;
}
-interface AsyncLookupOptions extends LookupOptions {
- /**
- * Throw when there's no match. If set to `false` and it gets no match, it will return `undefined`.
- * @default false
- */
- throwNotFound?: boolean;
-}
-
export default class CacheableLookup {
constructor(options?: Options);
/**
@@ -96,8 +104,8 @@ export default class CacheableLookup {
/**
* The asynchronous version of `dns.lookup(…)`.
*/
- lookupAsync(hostname: string, options: AsyncLookupOptions & {all: true}): Promise>;
- lookupAsync(hostname: string, options: AsyncLookupOptions): Promise;
+ lookupAsync(hostname: string, options: LookupOptions & {all: true}): Promise>;
+ lookupAsync(hostname: string, options: LookupOptions): Promise;
lookupAsync(hostname: string): Promise;
lookupAsync(hostname: string, family: IPFamily): Promise;
/**
@@ -108,6 +116,11 @@ export default class CacheableLookup {
* An asynchronous function which makes a new DNS lookup query and updates the database. This is used by `query(hostname, family)` if no entry in the database is present. Returns an array of objects with `address`, `family`, `ttl` and `expires` properties.
*/
queryAndCache(hostname: string): Promise>;
+ /**
+ * Returns an entry from the array for the given hostname.
+ * Useful to implement a round-robin algorithm.
+ */
+ _getEntry(entries: ReadonlyArray, hostname: string): EntryObject;
/**
* Removes outdated entries.
*/
@@ -125,7 +138,7 @@ export default class CacheableLookup {
*/
updateInterfaceInfo(): void;
/**
- * Clears the cache.
+ * Clears the cache for the given hostname. If the hostname argument is not present, the entire cache will be cleared.
*/
- clear(): void;
+ clear(hostname?: string): void;
}
diff --git a/source/index.js b/source/index.js
index 44e2cf2..6902d1f 100644
--- a/source/index.js
+++ b/source/index.js
@@ -105,6 +105,10 @@ class CacheableLookup {
if (typeof options === 'function') {
callback = options;
options = {};
+ } else if (typeof options === 'number') {
+ options = {
+ family: options
+ };
}
if (!callback) {
@@ -122,6 +126,12 @@ class CacheableLookup {
}
async lookupAsync(hostname, options = {}) {
+ if (typeof options === 'number') {
+ options = {
+ family: options
+ };
+ }
+
let cached = await this.query(hostname);
if (options.family === 6) {
@@ -187,7 +197,7 @@ class CacheableLookup {
entry.family = 4;
entry.expires = Date.now() + (entry.ttl * 1000);
- // Is TTL the same for all entries?
+ // Is the TTL the same for all entries?
cacheTtl = Math.max(cacheTtl, entry.ttl);
}
}
@@ -197,7 +207,7 @@ class CacheableLookup {
entry.family = 6;
entry.expires = Date.now() + (entry.ttl * 1000);
- // Is TTL the same for all entries?
+ // Is the TTL the same for all entries?
cacheTtl = Math.max(cacheTtl, entry.ttl);
}
}
@@ -302,7 +312,12 @@ class CacheableLookup {
this._cache.clear();
}
- clear() {
+ clear(hostname) {
+ if (hostname) {
+ this._cache.delete(hostname);
+ return;
+ }
+
this._cache.clear();
}
}
diff --git a/tests/test.js b/tests/test.js
index 18eab45..c2b5a9d 100644
--- a/tests/test.js
+++ b/tests/test.js
@@ -958,3 +958,27 @@ test('errors are cached', async t => {
t.is(cacheable._cache.size, 0);
});
+
+test('passing family as options', async t => {
+ const cacheable = new CacheableLookup({resolver, customHostsPath: false});
+
+ const promisified = promisify(cacheable.lookup);
+
+ const entry = await cacheable.lookupAsync('localhost', 6);
+ t.is(entry.address, '::ffff:127.0.0.2');
+ t.is(entry.family, 6);
+
+ const address = await promisified('localhost', 6);
+ t.is(address, '::ffff:127.0.0.2');
+});
+
+test('clear(hostname) works', async t => {
+ const cacheable = new CacheableLookup({resolver, customHostsPath: false});
+
+ await cacheable.lookupAsync('localhost');
+ await cacheable.lookupAsync('temporary');
+
+ cacheable.clear('localhost');
+
+ t.is(cacheable._cache.size, 1);
+});
From 5f3afcb2d8f3b795eabf6cc940bab6df93ddd607 Mon Sep 17 00:00:00 2001
From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>
Date: Mon, 27 Apr 2020 10:35:27 +0200
Subject: [PATCH 12/12] Improve TS tests
---
index.d.ts | 4 +---
index.test-d.ts | 13 ++++++++++++-
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/index.d.ts b/index.d.ts
index 99be154..3b5f4e6 100644
--- a/index.d.ts
+++ b/index.d.ts
@@ -1,9 +1,7 @@
import {Resolver, promises as dnsPromises} from 'dns';
import {Agent} from 'http';
-export {Resolver};
-
-export type AsyncResolver = dnsPromises.Resolver;
+type AsyncResolver = dnsPromises.Resolver;
export type IPFamily = 4 | 6;
diff --git a/index.test-d.ts b/index.test-d.ts
index 57f63fe..2ea6467 100644
--- a/index.test-d.ts
+++ b/index.test-d.ts
@@ -1,12 +1,20 @@
+import {Resolver} from 'dns';
+import {Agent} from 'https';
import {expectType} from 'tsd';
import Keyv = require('keyv');
import CacheableLookup, {EntryObject} from '.';
(async () => {
const cacheable = new CacheableLookup();
+ const agent = new Agent();
new CacheableLookup({
- cache: new Keyv()
+ cache: new Keyv(),
+ customHostsPath: false,
+ fallbackTtl: 0,
+ errorTtl: 0,
+ maxTtl: 0,
+ resolver: new Resolver()
});
expectType(cacheable.servers);
@@ -37,5 +45,8 @@ import CacheableLookup, {EntryObject} from '.';
expectType(cacheable.updateInterfaceInfo());
expectType(cacheable.tick());
+ expectType(cacheable.install(agent));
+ expectType(cacheable.uninstall(agent));
+ expectType(cacheable.clear('localhost'));
expectType(cacheable.clear());
})();