blocklist-wrapper as a dep not a plugin

blocklist-wrapper is a plugin, and so it gets initialized like one, processed
like one, which is sequentially (as all plugins are) one after the other.

This prevents an opportunity for an optimization where the blocklist files
downloaded by blocklist-wrapper can be done in parallel with upstreaming dns
requests.
Blocklist files take anywhere between 100ms (from cloudflare caches) to 2s
(cloudfront caches) to 5s (s3); while upstreaming doh requests take 10ms to
30ms+. On Workers, up to 6 network-requests can be done in-parallel. But
because blocklist-wrapper is a plugin that runs before dns-operation,
these are ran sequentially (but offer no benefit).

Turning blocklist-wrapper into a dependency of dns-operation makes it easier
to parllalelize these network-requests.

src/commons/envutil.js

@@ -68,14 +68,15 @@ export function tdParts() {
   return env.tdParts;
 }
 
-export function dohResolver() {
+export function dohResolvers() {
   if (!env) return null;
-  return env.dnsResolverUrl;
-}
 
-export function secondaryDohResolver() {
-  if (!env) return null;
-  return env.secondaryDohResolver;
+  if (isWorkers()) {
+    // upstream to two resolvers on workers; since egress is free,
+    // faster among the 2 should help lower tail latencies at zero-cost
+    return [env.dnsResolverUrl, env.secondaryDohResolver];
+  }
+  return [env.dnsResolverUrl];
 }
 
 export function tlsCrtPath() {

src/core/deno/blocklists.ts

@@ -29,13 +29,7 @@ export async function setup(bw: any) {
     return true;
   }
 
-  console.info(
-    "dowloading bl u/ts/nc/parts",
-    url,
-    timestamp,
-    nodecount,
-    tdparts
-  );
+  console.info("dowloading bl u/ts/nc/parts", url, timestamp);
   await bw.initBlocklistConstruction(
     /* rxid*/ "bl-download",
     now,

src/core/plugin.js

@@ -53,14 +53,6 @@ export default class RethinkPlugin {
       false
     );
 
-    this.registerPlugin(
-      "blocklistFilter",
-      services.blocklistWrapper,
-      ["rxid"],
-      this.blocklistFilterCallBack,
-      false
-    );
-
     this.registerPlugin(
       "commandControl",
       services.commandControl,
@@ -137,29 +129,6 @@ export default class RethinkPlugin {
     this.log.endTime(t);
   }
 
-  /**
-   * Adds "blocklistFilter" to RethinkPlugin params
-   * @param {*} response - Contains `data` which is `blocklistFilter`
-   * @param {*} currentRequest
-   */
-  blocklistFilterCallBack(response, currentRequest) {
-    const rxid = this.parameter.get("rxid");
-    const r = response.data;
-    this.log.d(rxid, "blocklist-filter response");
-
-    if (
-      response.isException ||
-      util.emptyObj(r) ||
-      // FIXME: check if blocklist-filter has t/ft vars set?
-      // ref: blocklistWrapper:isBlocklistFilterSetup
-      util.emptyObj(r.blocklistFilter)
-    ) {
-      this.log.e(rxid, "err building blocklist-filter", response);
-      this.loadException(rxid, response, currentRequest);
-      return;
-    } // else: blocklist-filter setup complete
-  }
-
   /**
    * params
    * @param {*} response

src/core/svc.js

@@ -36,21 +36,20 @@ async function systemReady() {
 
   log.i("svc: systemReady");
 
+  const bw = new BlocklistWrapper();
   const cache = new DnsCache(dnsutil.cacheSize());
-  const blf = new BlocklistFilter();
 
-  services.blocklistWrapper = new BlocklistWrapper(blf);
-  services.commandControl = new CommandControl(blf);
   services.userOperation = new UserOperation();
-  services.dnsResolver = new DNSResolver(blf, cache);
-  services.dnsCacheHandler = new DNSCacheResponder(blf, cache);
+  services.dnsCacheHandler = new DNSCacheResponder(bw, cache);
+  services.commandControl = new CommandControl(bw);
+  services.dnsResolver = new DNSResolver(bw, cache);
 
   if (envutil.isNode()) {
     const b = await import("./node/blocklists.js");
-    await b.setup(services.blocklistWrapper);
+    await b.setup(bw);
   } else if (envutil.isDeno()) {
     const b = await import("./deno/blocklists.ts");
-    await b.setup(services.blocklistWrapper);
+    await b.setup(bw);
   } // else: setup blocklists on-demand; for ex, on workers
 
   done();

src/plugins/blocklist-wrapper/blocklistFilter.js

@@ -6,7 +6,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-import { customTagToFlag as _customTagToFlag } from "./radixTrie.js";
+import { customTagToFlag } from "./radixTrie.js";
 import * as dnsutil from "../../commons/dnsutil.js";
 import * as dnsBlockUtil from "../dnsblockutil.js";
 
@@ -20,22 +20,20 @@ export class BlocklistFilter {
     this.enc = new TextEncoder();
   }
 
-  loadFilter(t, ft, blocklistBasicConfig, blocklistFileTag) {
+  load(t, ft, blocklistBasicConfig, blocklistFileTag) {
     this.t = t;
     this.ft = ft;
     this.blocklistBasicConfig = blocklistBasicConfig;
     this.blocklistFileTag = blocklistFileTag;
   }
 
-  getDomainInfo(domainName) {
+  blockstamp(domainName) {
     const n = dnsutil.normalizeName(domainName);
 
-    return {
-      searchResult: this.hadDomainName(n),
-    };
+    return this.lookup(n);
   }
 
-  hadDomainName(n) {
+  lookup(n) {
     return this.ft.lookup(this.reverseUtf8(n));
   }
 
@@ -47,12 +45,8 @@ export class BlocklistFilter {
     return this.t.flagsToTag(uintFlag);
   }
 
-  customTagToFlag(tagList) {
-    return _customTagToFlag(tagList, this.blocklistFileTag);
-  }
-
   getB64FlagFromTag(tagList, flagVersion) {
-    const uintFlag = this.customTagToFlag(tagList);
+    const uintFlag = customTagToFlag(tagList, this.blocklistFileTag);
     return dnsBlockUtil.getB64Flag(uintFlag, flagVersion);
   }
 }

src/plugins/blocklist-wrapper/main.js

@@ -14,8 +14,8 @@ import * as envutil from "../../commons/envutil.js";
 import * as rdnsutil from "../dnsblockutil.js";
 
 class BlocklistWrapper {
-  constructor(blf) {
-    this.blocklistFilter = blf;
+  constructor() {
+    this.blocklistFilter = new BlocklistFilter();
     this.td = null; // trie
     this.rd = null; // rank-dir
     this.ft = null; // file-tags
@@ -26,17 +26,11 @@ class BlocklistWrapper {
     this.log = log.withTags("BlocklistWrapper");
   }
 
-  /**
-   * @param {*} param
-   * @param {String} param.rxid
-   * @returns
-   */
-  async RethinkModule(param) {
-    let response = util.emptyResponse();
-
+  async init(rxid) {
     if (this.isBlocklistFilterSetup()) {
-      response.data.blocklistFilter = this.blocklistFilter;
-      return response;
+      const blres = util.emptyResponse();
+      blres.data.blocklistFilter = this.blocklistFilter;
+      return blres;
     }
 
     try {
@@ -47,9 +41,9 @@ class BlocklistWrapper {
         // it has been a while, queue another blocklist-construction
         now - this.startTime > envutil.downloadTimeout() * 2
       ) {
-        this.log.i(param.rxid, "download blocklists", now, this.startTime);
-        return await this.initBlocklistConstruction(
-          param.rxid,
+        this.log.i(rxid, "download blocklists", now, this.startTime);
+        return this.initBlocklistConstruction(
+          rxid,
           now,
           envutil.blocklistUrl(),
           envutil.timestamp(),
@@ -58,53 +52,61 @@ class BlocklistWrapper {
         );
       } else {
         // someone's constructing... wait till finished
-        // res.arrayBuffer() is the most expensive op, taking anywhere
-        // between 700ms to 1.2s for trie. But: We don't want all incoming
-        // reqs to wait until the trie becomes available. 400ms is 1/3rd of
-        // 1.2s and 2x 250ms; both of these values have cost implications:
-        // 250ms (0.28GB-sec or 218ms wall time) in unbound usage per req
-        // equals cost of one bundled req.
-        let totalWaitms = 0;
-        const waitms = 50;
-        while (totalWaitms < envutil.downloadTimeout()) {
-          if (this.isBlocklistFilterSetup()) {
-            response.data.blocklistFilter = this.blocklistFilter;
-            return response;
-          }
-          await util.sleep(waitms);
-          totalWaitms += waitms;
-        }
-
-        response.isException = true;
-        response.exceptionStack = this.exceptionStack || "download timeout";
-        response.exceptionFrom = this.exceptionFrom || "blocklistWrapper.js";
+        return this.waitUntilDone();
       }
     } catch (e) {
-      this.log.e(param.rxid, "main", e.stack);
-      response = util.errResponse("blocklistWrapper", e);
+      this.log.e(rxid, "main", e.stack);
+      return util.errResponse("blocklistWrapper", e);
     }
+  }
 
-    return response;
+  getBlocklistFilter() {
+    return this.blocklistFilter;
   }
 
   isBlocklistFilterSetup() {
     return rdnsutil.isBlocklistFilterSetup(this.blocklistFilter);
   }
 
-  initBlocklistFilterConstruction(td, rd, ft, config) {
+  async waitUntilDone() {
+    // res.arrayBuffer() is the most expensive op, taking anywhere
+    // between 700ms to 1.2s for trie. But: We don't want all incoming
+    // reqs to wait until the trie becomes available. 400ms is 1/3rd of
+    // 1.2s and 2x 250ms; both of these values have cost implications:
+    // 250ms (0.28GB-sec or 218ms wall time) in unbound usage per req
+    // equals cost of one bundled req.
+    let totalWaitms = 0;
+    const waitms = 50;
+    const response = util.emptyResponse();
+    while (totalWaitms < envutil.downloadTimeout()) {
+      if (this.isBlocklistFilterSetup()) {
+        response.data.blocklistFilter = this.blocklistFilter;
+        return response;
+      }
+      await util.sleep(waitms);
+      totalWaitms += waitms;
+    }
+
+    response.isException = true;
+    response.exceptionStack = this.exceptionStack || "download timeout";
+    response.exceptionFrom = this.exceptionFrom || "blocklistWrapper.js";
+    return response;
+  }
+
+  initBlocklistFilterConstruction(td, rd, ftags, bconfig) {
     this.isBlocklistUnderConstruction = true;
     this.startTime = Date.now();
     const filter = createBlocklistFilter(
       /* trie*/ td,
       /* rank-dir*/ rd,
-      /* file-tags*/ ft,
-      /* basic-config*/ config
+      /* file-tags*/ ftags,
+      /* basic-config*/ bconfig
     );
-    this.blocklistFilter.loadFilter(
+    this.blocklistFilter.load(
       /* trie*/ filter.t,
       /* frozen-trie*/ filter.ft,
-      /* basic-config*/ filter.blocklistBasicConfig,
-      /* file-tags*/ filter.blocklistFileTag
+      /* basic-config*/ bconfig,
+      /* file-tags*/ ftags
     );
     this.isBlocklistUnderConstruction = false;
   }
@@ -130,7 +132,7 @@ class BlocklistWrapper {
         tdParts
       );
 
-      this.blocklistFilter.loadFilter(
+      this.blocklistFilter.load(
         bl.t,
         bl.ft,
         bl.blocklistBasicConfig,
@@ -140,7 +142,7 @@ class BlocklistWrapper {
       this.log.i(rxid, "blocklist-filter setup");
       if (false) {
         // test
-        const result = this.blocklistFilter.getDomainInfo("google.com");
+        const result = this.blocklistFilter.blockstamp("google.com");
         this.log.d(rxid, JSON.stringify(result));
       }
 

src/plugins/command-control/cc.js

@@ -10,10 +10,10 @@ import * as util from "../../commons/util.js";
 import * as rdnsutil from "../dnsblockutil.js";
 
 export class CommandControl {
-  constructor(blf) {
+  constructor(blocklistWrapper) {
     this.latestTimestamp = envutil.timestamp();
     this.log = log.withTags("CommandControl");
-    this.blocklistFilter = blf;
+    this.bw = blocklistWrapper;
   }
 
   /**
@@ -26,7 +26,7 @@ export class CommandControl {
   async RethinkModule(param) {
     // process only GET requests, ignore all others
     if (util.isGetRequest(param.request)) {
-      return this.commandOperation(
+      return await this.commandOperation(
         param.rxid,
         param.request.url,
         param.isDnsMsg
@@ -69,9 +69,7 @@ export class CommandControl {
     return d.length > 1 ? d[0] : emptyFlag;
   }
 
-  commandOperation(rxid, url, isDnsMsg) {
-    // note: this.blocklistFilter is assumed to have been setup!
-    const blf = this.blocklistFilter;
+  async commandOperation(rxid, url, isDnsMsg) {
     let response = util.emptyResponse();
 
     try {
@@ -96,6 +94,10 @@ export class CommandControl {
 
       this.log.d(rxid, "processing...", url, command, b64UserFlag);
 
+      // blocklistFilter may not to have been setup, so set it up
+      await this.bw.init();
+      const blf = this.bw.getBlocklistFilter();
+
       if (command === "listtob64") {
         response.data.httpResponse = listToB64(queryString, blf);
       } else if (command === "b64tolist") {
@@ -152,7 +154,7 @@ function domainNameToList(queryString, blocklistFilter, latestTimestamp) {
     listDetail: {},
   };
 
-  const searchResult = blocklistFilter.hadDomainName(domainName);
+  const searchResult = blocklistFilter.lookup(domainName);
   if (!searchResult) {
     return jsonResponse(r);
   }
@@ -176,7 +178,7 @@ function domainNameToUint(queryString, blocklistFilter) {
     list: {},
   };
 
-  const searchResult = blocklistFilter.hadDomainName(domainName);
+  const searchResult = blocklistFilter.lookup(domainName);
   if (!searchResult) {
     return jsonResponse(r);
   }