Event bus to streamline process up (serverless-dns#43)

* Event bus to streamline process up Since nodejs doesn't respect import sequence (unlike deno), conditionals are required to initialize global deps like log, env, plugin.services etc. This is solved with a pub-sub mechanism where-in modules subscribe to events to know when their deps are up and running. Today, the "config" module, imported for side-effects from the entrypoints (server-workers.js for workers, server-deno.ts for deno, and server-node.js for node) of respective runtimes, initializes "log" and "env" objs, and kicks-off other basic setup. It then publishes a "ready" event which triggers the "plugin" module, that then inits essential "services", which includes downloading the blocklists where a disk is available (like on fly / dev machines). Once the service init is done, "plugin" publishes a "go" event. The entrypoints listening on the "go" event now bring up their endpoints as required and start serving DoH/DoT requests. pub/sub could have been a state-machine, but it isn't. All subs run as micro-tasks on all runtimes. To account for time delay between blocklist download (during service init) and entrypoint bring up (in response to "go" event), fly's grace-period for health-checks is increased to 15s. In TypeScript, the Window global object is extended to include "log" and "env" global properties. This Window is aliased to globalThis in JavaScript files. Even though globalThis support was added in TypeScript v3.4, there seems to be no way to extend globalThis, per se. Co-authored-by: Amith Mohanan <[email protected]>
CaptainFrosty · Jan 1, 2022 · 95dfdb4 · 95dfdb4
1 parent 147b56e
commit 95dfdb4
Show file tree

Hide file tree

Showing 33 changed files with 1,181 additions and 787 deletions.
diff --git a/.eslintrc.cjs b/.eslintrc.cjs
@@ -37,6 +37,7 @@ module.exports = {
     "valid-jsdoc": 0,
     "quotes": ["error", "double", { allowTemplateLiterals: true }],
     "no-unused-vars": "warn",
+    "new-cap": ["error", { "properties": false }],
 
     // Enforces rules from .prettierrc file.
     // These should be fixed automatically with formatting.

diff --git a/.github/workflows/deno-deploy.yml b/.github/workflows/deno-deploy.yml
@@ -28,8 +28,8 @@ env:
     ${{ github.event_name == 'workflow_dispatch' &&
       github.event.inputs.deployment-type == 'live' &&
       'build/deno-deploy/live' || 'build/deno-deploy/dev' }}
-  IN_FILE: "src/http.ts"
-  OUT_FILE: "http.bundle.js"
+  IN_FILE: "src/server-deno.ts"
+  OUT_FILE: "index.bundle.js"
 
 jobs:
   deploy:

diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ dist/
 node_modules/
 worker/
 test/data/cache/
+blocklists__/
 
 package-lock.json
 

diff --git a/deno.Dockerfile b/deno.Dockerfile
@@ -28,7 +28,7 @@ RUN ls -Fla
 ENTRYPOINT ["/bin/deno"]
 
 # This is only used while building, on fly.io
-CMD ["run", "--allow-net", "--allow-env", "--allow-read", "src/http.ts"]
+CMD ["run", "--allow-net", "--allow-env", "--allow-read", "src/server-deno.ts"]
 
 # Run port process as a root privilege user. For say port 53
 # USER root
diff --git a/fly.toml b/fly.toml
@@ -24,8 +24,8 @@ auto_rollback = true
 # Don't use `[processes]`, undocumented and causes problems with [auto]scaling.
 # Instead, use CMD in Dockerfile.
 # [processes]
-# app = "run --allow-net --allow-env --import-map=import_map.json http.ts"
-# app = "node server.js"
+# app = "run --allow-net --allow-env --import-map=import_map.json server-deno.ts"
+# app = "node server-node.js"
 
 # DNS over HTTP[S]
 [[services]]
@@ -45,8 +45,10 @@ script_checks = []
   port = 443
 
   [[services.tcp_checks]]
-  grace_period = "1s"
-  interval = "15s"
+  # account for delay due to blocklists download that
+  # happens on process startup: plugin.js:systemReady
+  grace_period = "15s"
+  interval = "30s"
   restart_limit = 6
   timeout = "2s"
 
@@ -68,7 +70,9 @@ script_checks = []
   port = 853
 
   [[services.tcp_checks]]
-  grace_period = "1s"
-  interval = "15s"
+  # account for delay due to blocklists download that
+  # happens on process startup: plugin.js:systemReady
+  grace_period = "15s"
+  interval = "30s"
   restart_limit = 6
   timeout = "2s"
diff --git a/node.Dockerfile b/node.Dockerfile
@@ -22,4 +22,4 @@ RUN rm -f *Dockerfile .dockerignore
 
 RUN ls -Fla
 
-CMD ["node", "src/server.js"]
+CMD ["node", "src/server-node.js"]
diff --git a/src/basic/userOperation.js b/src/basic/userOperation.js
@@ -15,6 +15,7 @@ export class UserOperation {
     this.userConfigCache = new UserCache(1000);
     this.blocklistFilter = new BlocklistFilter();
   }
+
   /**
    * @param {*} param
    * @param {*} param.dnsResolverUrl
@@ -27,7 +28,7 @@ export class UserOperation {
   }
 
   loadUser(param) {
-    let response = {};
+    const response = {};
     response.isException = false;
     response.exceptionStack = "";
     response.exceptionFrom = "";
@@ -38,7 +39,7 @@ export class UserOperation {
       if (!param.isDnsMsg) {
         return response;
       }
-      let userBlocklistInfo = {};
+      const userBlocklistInfo = {};
       userBlocklistInfo.from = "Cache";
       let blocklistFlag = getBlocklistFlag(param.request.url);
       let currentUser = this.userConfigCache.get(blocklistFlag);
@@ -48,14 +49,14 @@ export class UserOperation {
         currentUser.flagVersion = 0;
         currentUser.userServiceListUint = false;
 
-        let response = this.blocklistFilter.unstamp(blocklistFlag);
+        const response = this.blocklistFilter.unstamp(blocklistFlag);
         currentUser.userBlocklistFlagUint = response.userBlocklistFlagUint;
         currentUser.flagVersion = response.flagVersion;
 
         if (!util.emptyString(currentUser.userBlocklistFlagUint)) {
           currentUser.userServiceListUint = dnsBlockUtil.flagIntersection(
             currentUser.userBlocklistFlagUint,
-            this.blocklistFilter.wildCardUint,
+            this.blocklistFilter.wildCardUint
           );
         } else {
           blocklistFlag = "";
@@ -80,6 +81,7 @@ export class UserOperation {
     return response;
   }
 }
+
 /**
  * Get the blocklist flag from `Request` URL
  * DNS over TLS flag from SNI should be rewritten to `url`'s pathname
@@ -88,12 +90,12 @@ export class UserOperation {
  */
 function getBlocklistFlag(url) {
   let blocklistFlag = "";
-  let reqUrl = new URL(url);
+  const reqUrl = new URL(url);
 
   // Check if pathname has `/dns-query`
-  let tmpsplit = reqUrl.pathname.split("/");
+  const tmpsplit = reqUrl.pathname.split("/");
   if (tmpsplit.length > 1) {
-    if (tmpsplit[1].toLowerCase() == "dns-query") {
+    if (tmpsplit[1].toLowerCase() === "dns-query") {
       blocklistFlag = tmpsplit[2] || "";
     } else {
       blocklistFlag = tmpsplit[1] || "";

diff --git a/src/blocklist-wrapper/blocklistFilter.js b/src/blocklist-wrapper/blocklistFilter.js
@@ -9,7 +9,7 @@
 import { Buffer } from "buffer";
 import { DomainNameCache } from "../cache-wrapper/cache-wrapper.js";
 import { customTagToFlag as _customTagToFlag } from "./radixTrie.js";
-import { base32, rbase32 } from "./b32.js";
+import { rbase32 } from "./b32.js";
 
 export class BlocklistFilter {
   constructor() {
@@ -18,30 +18,10 @@ export class BlocklistFilter {
     this.blocklistBasicConfig = null;
     this.blocklistFileTag = null;
     this.domainNameCache = null;
-    //following wildCard array is hardcoded to avoid the usage of blocklistFileTag download from s3
-    //the hard coded array contains the list of blocklist files mentioned at setWildcardlist()
-    //TODO is future version wildcard list should be downloaded from KV or from env
+    // TODO: wildcard list should be fetched from S3/KV
     this.wildCardUint = new Uint16Array([
-      64544,
-      18431,
-      8191,
-      65535,
-      64640,
-      1,
-      128,
-      16320,
+      64544, 18431, 8191, 65535, 64640, 1, 128, 16320,
     ]);
-    /*
-    this.wildCardLists = new Set();
-    setWildcardlist.call(this);
-    const str = _customTagToFlag(
-      this.wildCardLists,
-      this.blocklistFileTag,
-    );
-    this.wildCardUint = new Uint16Array(str.length);
-    for (let i = 0; i < this.wildCardUint.length; i++) {
-      this.wildCardUint[i] = str.charCodeAt(i);
-    }*/
   }
 
   loadFilter(t, ft, blocklistBasicConfig, blocklistFileTag) {
@@ -84,24 +64,23 @@ export class BlocklistFilter {
 
   getB64FlagFromTag(tagList, flagVersion) {
     try {
-      if (flagVersion == "0") {
+      if (flagVersion === "0") {
         return encodeURIComponent(
           Buffer.from(
-            _customTagToFlag(tagList, this.blocklistFileTag),
-          ).toString("base64"),
+            _customTagToFlag(tagList, this.blocklistFileTag)
+          ).toString("base64")
         );
-      } else if (flagVersion == "1") {
-        return "1:" +
+      } else if (flagVersion === "1") {
+        return (
+          "1:" +
           encodeURI(
             btoa(
-              encodeToBinary(
-                _customTagToFlag(
-                  tagList,
-                  this.blocklistFileTag,
-                ),
-              ),
-            ).replace(/\//g, "_").replace(/\+/g, "-"),
-          );
+              encodeToBinary(_customTagToFlag(tagList, this.blocklistFileTag))
+            )
+              .replace(/\//g, "_")
+              .replace(/\+/g, "-")
+          )
+        );
       }
     } catch (e) {
       throw e;
@@ -136,25 +115,26 @@ function toUint(flag) {
     const response = {};
     response.userBlocklistFlagUint = "";
     response.flagVersion = "0";
-    //added to check if UserFlag is empty for changing dns request flow
-    flag = (flag) ? flag.trim() : "";
+    // added to check if UserFlag is empty for changing dns request flow
+    flag = flag ? flag.trim() : "";
 
     if (flag.length <= 0) {
       return response;
     }
 
     const isFlagB32 = isB32(flag);
     // "v:b64" or "v+b32" or "uriencoded(b64)", where v is uint version
-    let s = flag.split(isFlagB32 ? b32delim : b64delim);
+    const s = flag.split(isFlagB32 ? b32delim : b64delim);
     let convertor = (x) => ""; // empty convertor
     let f = ""; // stamp flag
     const v = version(s);
 
-    if (v == "0") { // version 0
+    if (v === "0") {
+      // version 0
       convertor = Base64ToUint;
       f = s[0];
-    } else if (v == "1") {
-      convertor = (isFlagB32) ? Base32ToUint_v1 : Base64ToUint_v1;
+    } else if (v === "1") {
+      convertor = isFlagB32 ? Base32ToUintV1 : Base64ToUintV1;
       f = s[1];
     } else {
       throw new Error("unknown blocklist stamp version in " + s);
@@ -179,7 +159,7 @@ function Base64ToUint(b64Flag) {
   return uint;
 }
 
-function Base64ToUint_v1(b64Flag) {
+function Base64ToUintV1(b64Flag) {
   let str = decodeURI(b64Flag);
   str = decodeFromBinary(atob(str.replace(/_/g, "/").replace(/-/g, "+")));
   const uint = [];
@@ -189,7 +169,7 @@ function Base64ToUint_v1(b64Flag) {
   return uint;
 }
 
-function Base32ToUint_v1(flag) {
+function Base32ToUintV1(flag) {
   let str = decodeURI(flag);
   str = decodeFromBinaryArray(rbase32(str));
   const uint = [];
@@ -212,64 +192,3 @@ function decodeFromBinaryArray(b) {
   const u8 = true;
   return decodeFromBinary(b, u8);
 }
-
-function setWildcardlist() {
-  this.wildCardLists.add("KBI"); // safe-search-not-supported
-  this.wildCardLists.add("YWG"); // nextdns dht-bootstrap-nodes
-  this.wildCardLists.add("SMQ"); // nextdns file-hosting
-  this.wildCardLists.add("AQX"); // nextdns proxies
-  this.wildCardLists.add("BTG"); // nextdns streaming audio
-  this.wildCardLists.add("GUN"); // nextdns streaming video
-  this.wildCardLists.add("KSH"); // nextdns torrent clients
-  this.wildCardLists.add("WAS"); // nextdns torrent trackers
-  this.wildCardLists.add("AZY"); // nextdns torrent websites
-  this.wildCardLists.add("GWB"); // nextdns usenet
-  this.wildCardLists.add("YMG"); // nextdns warez
-  this.wildCardLists.add("CZM"); // tiuxo porn
-  this.wildCardLists.add("ZVO"); // oblat social-networks
-  this.wildCardLists.add("YOM"); // 9gag srv
-  this.wildCardLists.add("THR"); // amazon srv
-  this.wildCardLists.add("RPW"); // blizzard srv
-  this.wildCardLists.add("AMG"); // dailymotion srv
-  this.wildCardLists.add("WTJ"); // discord srv
-  this.wildCardLists.add("ZXU"); // disney+ srv
-  this.wildCardLists.add("FJG"); // ebay srv
-  this.wildCardLists.add("NYS"); // facebook srv
-  this.wildCardLists.add("OKG"); // fortnite srv
-  this.wildCardLists.add("KNP"); // hulu srv
-  this.wildCardLists.add("FLI"); // imgur srv
-  this.wildCardLists.add("RYX"); // instagram srv
-  this.wildCardLists.add("CIH"); // leagueoflegends srv
-  this.wildCardLists.add("PTE"); // messenger srv
-  this.wildCardLists.add("KEA"); // minecraft srv
-  this.wildCardLists.add("CMR"); // netflix srv
-  this.wildCardLists.add("DDO"); // pinterest srv
-  this.wildCardLists.add("VLM"); // reddit srv
-  this.wildCardLists.add("JEH"); // roblox srv
-  this.wildCardLists.add("XLX"); // skype srv
-  this.wildCardLists.add("OQW"); // snapchat srv
-  this.wildCardLists.add("FXC"); // spotify srv
-  this.wildCardLists.add("HZJ"); // steam srv
-  this.wildCardLists.add("SWK"); // telegram srv
-  this.wildCardLists.add("VAM"); // tiktok srv
-  this.wildCardLists.add("AOS"); // tinder srv
-  this.wildCardLists.add("FAL"); // tumblr srv
-  this.wildCardLists.add("CZK"); // twitch srv
-  this.wildCardLists.add("FZB"); // twitter srv
-  this.wildCardLists.add("PYW"); // vimeo srv
-  this.wildCardLists.add("JXA"); // vk srv
-  this.wildCardLists.add("KOR"); // whatsapp srv
-  this.wildCardLists.add("DEP"); // youtube srv
-  this.wildCardLists.add("RFX"); // zoom srv
-  this.wildCardLists.add("RAF"); // parked-domains
-  this.wildCardLists.add("RKG"); // infosec.cert-pa.it
-  this.wildCardLists.add("GLV"); // covid malware sophos labs
-  this.wildCardLists.add("FHW"); // alexa native
-  this.wildCardLists.add("AGZ"); // apple native
-  this.wildCardLists.add("IVN"); // huawei native
-  this.wildCardLists.add("FIB"); // roku native
-  this.wildCardLists.add("FGF"); // samsung native
-  this.wildCardLists.add("FLL"); // sonos native
-  this.wildCardLists.add("IVO"); // windows native
-  this.wildCardLists.add("ALQ"); // xiaomi native
-}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -22,4 +22,4 @@ RUN rm -f *Dockerfile .dockerignore

		RUN ls -Fla

		CMD ["node", "src/server.js"]
		CMD ["node", "src/server-node.js"]