diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 54e5e7a83e..0b7802db99 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -37,6 +37,7 @@ module.exports = { "valid-jsdoc": 0, "quotes": ["error", "double", { allowTemplateLiterals: true }], "no-unused-vars": "warn", + "new-cap": ["error", { "properties": false }], // Enforces rules from .prettierrc file. // These should be fixed automatically with formatting. diff --git a/.github/workflows/deno-deploy.yml b/.github/workflows/deno-deploy.yml index 7bbcaa6d71..9e9e916a27 100644 --- a/.github/workflows/deno-deploy.yml +++ b/.github/workflows/deno-deploy.yml @@ -28,8 +28,8 @@ env: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.deployment-type == 'live' && 'build/deno-deploy/live' || 'build/deno-deploy/dev' }} - IN_FILE: "src/http.ts" - OUT_FILE: "http.bundle.js" + IN_FILE: "src/server-deno.ts" + OUT_FILE: "index.bundle.js" jobs: deploy: diff --git a/.gitignore b/.gitignore index 737bb49029..a1f4e02f07 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ dist/ node_modules/ worker/ test/data/cache/ +blocklists__/ package-lock.json diff --git a/deno.Dockerfile b/deno.Dockerfile index 626d788fde..d2b46fae06 100644 --- a/deno.Dockerfile +++ b/deno.Dockerfile @@ -28,7 +28,7 @@ RUN ls -Fla ENTRYPOINT ["/bin/deno"] # This is only used while building, on fly.io -CMD ["run", "--allow-net", "--allow-env", "--allow-read", "src/http.ts"] +CMD ["run", "--allow-net", "--allow-env", "--allow-read", "src/server-deno.ts"] # Run port process as a root privilege user. For say port 53 # USER root diff --git a/fly.toml b/fly.toml index 8db5335f3b..dd6c181e99 100644 --- a/fly.toml +++ b/fly.toml @@ -24,8 +24,8 @@ auto_rollback = true # Don't use `[processes]`, undocumented and causes problems with [auto]scaling. # Instead, use CMD in Dockerfile. # [processes] -# app = "run --allow-net --allow-env --import-map=import_map.json http.ts" -# app = "node server.js" +# app = "run --allow-net --allow-env --import-map=import_map.json server-deno.ts" +# app = "node server-node.js" # DNS over HTTP[S] [[services]] @@ -45,8 +45,10 @@ script_checks = [] port = 443 [[services.tcp_checks]] - grace_period = "1s" - interval = "15s" + # account for delay due to blocklists download that + # happens on process startup: plugin.js:systemReady + grace_period = "15s" + interval = "30s" restart_limit = 6 timeout = "2s" @@ -68,7 +70,9 @@ script_checks = [] port = 853 [[services.tcp_checks]] - grace_period = "1s" - interval = "15s" + # account for delay due to blocklists download that + # happens on process startup: plugin.js:systemReady + grace_period = "15s" + interval = "30s" restart_limit = 6 timeout = "2s" diff --git a/node.Dockerfile b/node.Dockerfile index 32f4818984..834f4bc89d 100644 --- a/node.Dockerfile +++ b/node.Dockerfile @@ -22,4 +22,4 @@ RUN rm -f *Dockerfile .dockerignore RUN ls -Fla -CMD ["node", "src/server.js"] +CMD ["node", "src/server-node.js"] diff --git a/src/basic/userOperation.js b/src/basic/userOperation.js index b9eaa49d6c..203471b7b4 100644 --- a/src/basic/userOperation.js +++ b/src/basic/userOperation.js @@ -15,6 +15,7 @@ export class UserOperation { this.userConfigCache = new UserCache(1000); this.blocklistFilter = new BlocklistFilter(); } + /** * @param {*} param * @param {*} param.dnsResolverUrl @@ -27,7 +28,7 @@ export class UserOperation { } loadUser(param) { - let response = {}; + const response = {}; response.isException = false; response.exceptionStack = ""; response.exceptionFrom = ""; @@ -38,7 +39,7 @@ export class UserOperation { if (!param.isDnsMsg) { return response; } - let userBlocklistInfo = {}; + const userBlocklistInfo = {}; userBlocklistInfo.from = "Cache"; let blocklistFlag = getBlocklistFlag(param.request.url); let currentUser = this.userConfigCache.get(blocklistFlag); @@ -48,14 +49,14 @@ export class UserOperation { currentUser.flagVersion = 0; currentUser.userServiceListUint = false; - let response = this.blocklistFilter.unstamp(blocklistFlag); + const response = this.blocklistFilter.unstamp(blocklistFlag); currentUser.userBlocklistFlagUint = response.userBlocklistFlagUint; currentUser.flagVersion = response.flagVersion; if (!util.emptyString(currentUser.userBlocklistFlagUint)) { currentUser.userServiceListUint = dnsBlockUtil.flagIntersection( currentUser.userBlocklistFlagUint, - this.blocklistFilter.wildCardUint, + this.blocklistFilter.wildCardUint ); } else { blocklistFlag = ""; @@ -80,6 +81,7 @@ export class UserOperation { return response; } } + /** * Get the blocklist flag from `Request` URL * DNS over TLS flag from SNI should be rewritten to `url`'s pathname @@ -88,12 +90,12 @@ export class UserOperation { */ function getBlocklistFlag(url) { let blocklistFlag = ""; - let reqUrl = new URL(url); + const reqUrl = new URL(url); // Check if pathname has `/dns-query` - let tmpsplit = reqUrl.pathname.split("/"); + const tmpsplit = reqUrl.pathname.split("/"); if (tmpsplit.length > 1) { - if (tmpsplit[1].toLowerCase() == "dns-query") { + if (tmpsplit[1].toLowerCase() === "dns-query") { blocklistFlag = tmpsplit[2] || ""; } else { blocklistFlag = tmpsplit[1] || ""; diff --git a/src/blocklist-wrapper/blocklistFilter.js b/src/blocklist-wrapper/blocklistFilter.js index 78b0c78ac0..dbec315c25 100644 --- a/src/blocklist-wrapper/blocklistFilter.js +++ b/src/blocklist-wrapper/blocklistFilter.js @@ -9,7 +9,7 @@ import { Buffer } from "buffer"; import { DomainNameCache } from "../cache-wrapper/cache-wrapper.js"; import { customTagToFlag as _customTagToFlag } from "./radixTrie.js"; -import { base32, rbase32 } from "./b32.js"; +import { rbase32 } from "./b32.js"; export class BlocklistFilter { constructor() { @@ -18,30 +18,10 @@ export class BlocklistFilter { this.blocklistBasicConfig = null; this.blocklistFileTag = null; this.domainNameCache = null; - //following wildCard array is hardcoded to avoid the usage of blocklistFileTag download from s3 - //the hard coded array contains the list of blocklist files mentioned at setWildcardlist() - //TODO is future version wildcard list should be downloaded from KV or from env + // TODO: wildcard list should be fetched from S3/KV this.wildCardUint = new Uint16Array([ - 64544, - 18431, - 8191, - 65535, - 64640, - 1, - 128, - 16320, + 64544, 18431, 8191, 65535, 64640, 1, 128, 16320, ]); - /* - this.wildCardLists = new Set(); - setWildcardlist.call(this); - const str = _customTagToFlag( - this.wildCardLists, - this.blocklistFileTag, - ); - this.wildCardUint = new Uint16Array(str.length); - for (let i = 0; i < this.wildCardUint.length; i++) { - this.wildCardUint[i] = str.charCodeAt(i); - }*/ } loadFilter(t, ft, blocklistBasicConfig, blocklistFileTag) { @@ -84,24 +64,23 @@ export class BlocklistFilter { getB64FlagFromTag(tagList, flagVersion) { try { - if (flagVersion == "0") { + if (flagVersion === "0") { return encodeURIComponent( Buffer.from( - _customTagToFlag(tagList, this.blocklistFileTag), - ).toString("base64"), + _customTagToFlag(tagList, this.blocklistFileTag) + ).toString("base64") ); - } else if (flagVersion == "1") { - return "1:" + + } else if (flagVersion === "1") { + return ( + "1:" + encodeURI( btoa( - encodeToBinary( - _customTagToFlag( - tagList, - this.blocklistFileTag, - ), - ), - ).replace(/\//g, "_").replace(/\+/g, "-"), - ); + encodeToBinary(_customTagToFlag(tagList, this.blocklistFileTag)) + ) + .replace(/\//g, "_") + .replace(/\+/g, "-") + ) + ); } } catch (e) { throw e; @@ -136,8 +115,8 @@ function toUint(flag) { const response = {}; response.userBlocklistFlagUint = ""; response.flagVersion = "0"; - //added to check if UserFlag is empty for changing dns request flow - flag = (flag) ? flag.trim() : ""; + // added to check if UserFlag is empty for changing dns request flow + flag = flag ? flag.trim() : ""; if (flag.length <= 0) { return response; @@ -145,16 +124,17 @@ function toUint(flag) { const isFlagB32 = isB32(flag); // "v:b64" or "v+b32" or "uriencoded(b64)", where v is uint version - let s = flag.split(isFlagB32 ? b32delim : b64delim); + const s = flag.split(isFlagB32 ? b32delim : b64delim); let convertor = (x) => ""; // empty convertor let f = ""; // stamp flag const v = version(s); - if (v == "0") { // version 0 + if (v === "0") { + // version 0 convertor = Base64ToUint; f = s[0]; - } else if (v == "1") { - convertor = (isFlagB32) ? Base32ToUint_v1 : Base64ToUint_v1; + } else if (v === "1") { + convertor = isFlagB32 ? Base32ToUintV1 : Base64ToUintV1; f = s[1]; } else { throw new Error("unknown blocklist stamp version in " + s); @@ -179,7 +159,7 @@ function Base64ToUint(b64Flag) { return uint; } -function Base64ToUint_v1(b64Flag) { +function Base64ToUintV1(b64Flag) { let str = decodeURI(b64Flag); str = decodeFromBinary(atob(str.replace(/_/g, "/").replace(/-/g, "+"))); const uint = []; @@ -189,7 +169,7 @@ function Base64ToUint_v1(b64Flag) { return uint; } -function Base32ToUint_v1(flag) { +function Base32ToUintV1(flag) { let str = decodeURI(flag); str = decodeFromBinaryArray(rbase32(str)); const uint = []; @@ -212,64 +192,3 @@ function decodeFromBinaryArray(b) { const u8 = true; return decodeFromBinary(b, u8); } - -function setWildcardlist() { - this.wildCardLists.add("KBI"); // safe-search-not-supported - this.wildCardLists.add("YWG"); // nextdns dht-bootstrap-nodes - this.wildCardLists.add("SMQ"); // nextdns file-hosting - this.wildCardLists.add("AQX"); // nextdns proxies - this.wildCardLists.add("BTG"); // nextdns streaming audio - this.wildCardLists.add("GUN"); // nextdns streaming video - this.wildCardLists.add("KSH"); // nextdns torrent clients - this.wildCardLists.add("WAS"); // nextdns torrent trackers - this.wildCardLists.add("AZY"); // nextdns torrent websites - this.wildCardLists.add("GWB"); // nextdns usenet - this.wildCardLists.add("YMG"); // nextdns warez - this.wildCardLists.add("CZM"); // tiuxo porn - this.wildCardLists.add("ZVO"); // oblat social-networks - this.wildCardLists.add("YOM"); // 9gag srv - this.wildCardLists.add("THR"); // amazon srv - this.wildCardLists.add("RPW"); // blizzard srv - this.wildCardLists.add("AMG"); // dailymotion srv - this.wildCardLists.add("WTJ"); // discord srv - this.wildCardLists.add("ZXU"); // disney+ srv - this.wildCardLists.add("FJG"); // ebay srv - this.wildCardLists.add("NYS"); // facebook srv - this.wildCardLists.add("OKG"); // fortnite srv - this.wildCardLists.add("KNP"); // hulu srv - this.wildCardLists.add("FLI"); // imgur srv - this.wildCardLists.add("RYX"); // instagram srv - this.wildCardLists.add("CIH"); // leagueoflegends srv - this.wildCardLists.add("PTE"); // messenger srv - this.wildCardLists.add("KEA"); // minecraft srv - this.wildCardLists.add("CMR"); // netflix srv - this.wildCardLists.add("DDO"); // pinterest srv - this.wildCardLists.add("VLM"); // reddit srv - this.wildCardLists.add("JEH"); // roblox srv - this.wildCardLists.add("XLX"); // skype srv - this.wildCardLists.add("OQW"); // snapchat srv - this.wildCardLists.add("FXC"); // spotify srv - this.wildCardLists.add("HZJ"); // steam srv - this.wildCardLists.add("SWK"); // telegram srv - this.wildCardLists.add("VAM"); // tiktok srv - this.wildCardLists.add("AOS"); // tinder srv - this.wildCardLists.add("FAL"); // tumblr srv - this.wildCardLists.add("CZK"); // twitch srv - this.wildCardLists.add("FZB"); // twitter srv - this.wildCardLists.add("PYW"); // vimeo srv - this.wildCardLists.add("JXA"); // vk srv - this.wildCardLists.add("KOR"); // whatsapp srv - this.wildCardLists.add("DEP"); // youtube srv - this.wildCardLists.add("RFX"); // zoom srv - this.wildCardLists.add("RAF"); // parked-domains - this.wildCardLists.add("RKG"); // infosec.cert-pa.it - this.wildCardLists.add("GLV"); // covid malware sophos labs - this.wildCardLists.add("FHW"); // alexa native - this.wildCardLists.add("AGZ"); // apple native - this.wildCardLists.add("IVN"); // huawei native - this.wildCardLists.add("FIB"); // roku native - this.wildCardLists.add("FGF"); // samsung native - this.wildCardLists.add("FLL"); // sonos native - this.wildCardLists.add("IVO"); // windows native - this.wildCardLists.add("ALQ"); // xiaomi native -} diff --git a/src/blocklist-wrapper/blocklistWrapper.js b/src/blocklist-wrapper/blocklistWrapper.js index 6c6c3975a2..586cc55098 100644 --- a/src/blocklist-wrapper/blocklistWrapper.js +++ b/src/blocklist-wrapper/blocklistWrapper.js @@ -13,6 +13,9 @@ class BlocklistWrapper { constructor() { this.blocklistFilter = new BlocklistFilter(); this.startTime; + this.td = null; // trie + this.rd = null; // rank-dir + this.ft = null; // file-tags this.isBlocklistUnderConstruction = false; this.exceptionFrom = ""; this.exceptionStack = ""; @@ -29,42 +32,45 @@ class BlocklistWrapper { * @returns */ async RethinkModule(param) { - let response = {}; + const response = {}; response.isException = false; response.exceptionStack = ""; response.exceptionFrom = ""; response.data = {}; - if (this.blocklistFilter.t !== null) { + + if (this.isBlocklistFilterSetup()) { response.data.blocklistFilter = this.blocklistFilter; return response; } + try { const now = Date.now(); + if (this.isBlocklistUnderConstruction === false) { return await this.initBlocklistConstruction( now, param.blocklistUrl, param.latestTimestamp, param.tdNodecount, - param.tdParts, + param.tdParts ); - } else if ((now - this.startTime) > (param.workerTimeout * 2)) { + } else if (now - this.startTime > param.workerTimeout * 2) { + // it has been a while, queue another blocklist-construction return await this.initBlocklistConstruction( now, param.blocklistUrl, param.latestTimestamp, param.tdNodecount, - param.tdParts, + param.tdParts ); - } else { // someone's constructing... wait till finished + } else { + // someone's constructing... wait till finished // res.arrayBuffer() is the most expensive op, taking anywhere // between 700ms to 1.2s for trie. But: We don't want all incoming // reqs to wait until the trie becomes available. 400ms is 1/3rd of // 1.2s and 2x 250ms; both of these values have cost implications: // 250ms (0.28GB-sec or 218ms wall time) in unbound usage per req // equals cost of one bundled req. - // going back to direct-s3 download as worker-bundled blocklist files download - // gets triggered for 10% of requests. let totalWaitms = 0; const waitms = 50; while (totalWaitms < param.fetchTimeout) { @@ -76,12 +82,10 @@ class BlocklistWrapper { totalWaitms += waitms; } response.isException = true; - response.exceptionStack = (this.exceptionStack) - ? this.exceptionStack - : "Problem in loading blocklistFilter - Waiting Timeout"; - response.exceptionFrom = (this.exceptionFrom) - ? this.exceptionFrom - : "blocklistWrapper.js RethinkModule"; + response.exceptionStack = + this.exceptionStack || "blocklist filter not ready"; + response.exceptionFrom = + this.exceptionFrom || "blocklistWrapper.js RethinkModule"; } } catch (e) { response.isException = true; @@ -92,47 +96,67 @@ class BlocklistWrapper { return response; } + isBlocklistFilterSetup() { + return this.blocklistFilter && this.blocklistFilter.t; + } + + initBlocklistFilterConstruction(td, rd, ft, config) { + this.isBlocklistUnderConstruction = true; + const filter = createBlocklistFilter( + /* trie*/ td, + /* rank-dir*/ rd, + /* file-tags*/ ft, + /* basic-config*/ config + ); + this.blocklistFilter.loadFilter( + /* trie*/ filter.t, + /* frozen-trie*/ filter.ft, + /* basic-config*/ filter.blocklistBasicConfig, + /* file-tags*/ filter.blocklistFileTag + ); + this.isBlocklistUnderConstruction = false; + } + async initBlocklistConstruction( when, blocklistUrl, latestTimestamp, tdNodecount, - tdParts, + tdParts ) { this.isBlocklistUnderConstruction = true; this.startTime = when; - let response = {}; + const response = {}; response.isException = false; response.exceptionStack = ""; response.exceptionFrom = ""; response.data = {}; try { - let bl = await downloadBuildBlocklist( + const bl = await this.downloadBuildBlocklist( blocklistUrl, latestTimestamp, tdNodecount, - tdParts, + tdParts ); this.blocklistFilter.loadFilter( bl.t, bl.ft, bl.blocklistBasicConfig, - bl.blocklistFileTag, + bl.blocklistFileTag ); log.d("done blocklist filter"); - if (false) { // test + if (false) { + // test const result = this.blocklistFilter.getDomainInfo("google.com"); log.d(JSON.stringify(result)); } - this.isBlocklistUnderConstruction = false; response.data.blocklistFilter = this.blocklistFilter; } catch (e) { - this.isBlocklistUnderConstruction = false; response.isException = true; response.exceptionStack = e.stack; response.exceptionFrom = "blocklistWrapper.js initBlocklistConstruction"; @@ -140,48 +164,55 @@ class BlocklistWrapper { this.exceptionStack = response.exceptionStack; log.e(e); } + + this.isBlocklistUnderConstruction = false; + return response; } -} -//Add needed env variables to param -async function downloadBuildBlocklist( - blocklistUrl, - latestTimestamp, - tdNodecount, - tdParts, -) { - try { - let resp = {}; - const baseurl = blocklistUrl + latestTimestamp; - let blocklistBasicConfig = { - nodecount: tdNodecount || -1, - tdparts: tdParts || -1, - }; - - tdNodecount == null && log.e("tdNodecount missing!"); - const buf0 = fileFetch(baseurl + "/filetag.json", "json"); - const buf1 = makeTd(baseurl, blocklistBasicConfig.tdparts); - const buf2 = fileFetch(baseurl + "/rd.txt", "buffer"); - - let downloads = await Promise.all([buf0, buf1, buf2]); - - log.d("call createBlocklistFilter", blocklistBasicConfig); - - let trie = createBlocklistFilter( - downloads[1], - downloads[2], - downloads[0], - blocklistBasicConfig, - ); + async downloadBuildBlocklist( + blocklistUrl, + latestTimestamp, + tdNodecount, + tdParts + ) { + try { + !tdNodecount && log.e("tdNodecount zero or missing!"); + + const resp = {}; + const baseurl = blocklistUrl + latestTimestamp; + const blocklistBasicConfig = { + nodecount: tdNodecount || -1, + tdparts: tdParts || -1, + }; + + const buf0 = fileFetch(baseurl + "/filetag.json", "json"); + const buf1 = makeTd(baseurl, blocklistBasicConfig.tdparts); + const buf2 = fileFetch(baseurl + "/rd.txt", "buffer"); + + const downloads = await Promise.all([buf0, buf1, buf2]); - resp.t = trie.t; - resp.ft = trie.ft; - resp.blocklistBasicConfig = blocklistBasicConfig; - resp.blocklistFileTag = downloads[0]; - return resp; - } catch (e) { - throw e; + log.d("call createBlocklistFilter", blocklistBasicConfig); + + this.td = downloads[1]; + this.rd = downloads[2]; + this.ft = downloads[0]; + + const trie = createBlocklistFilter( + /* trie*/ this.td, + /* rank-dir*/ this.rd, + /* file-tags*/ this.ft, + /* basic-config*/ blocklistBasicConfig + ); + + resp.t = trie.t; // tags + resp.ft = trie.ft; // frozen-trie + resp.blocklistBasicConfig = blocklistBasicConfig; + resp.blocklistFileTag = this.ft; + return resp; + } catch (e) { + throw e; + } } } @@ -190,18 +221,16 @@ async function fileFetch(url, typ) { throw new Error("Unknown conversion type at fileFetch"); } log.d("Start Downloading : " + url); - const res = await fetch(url, { cf: { cacheTtl: /*2w*/ 1209600 } }); - if (res.status == 200) { - if (typ == "buffer") { + const res = await fetch(url, { cf: { cacheTtl: /* 2w*/ 1209600 } }); + if (res.status === 200) { + if (typ === "buffer") { return await res.arrayBuffer(); - } else if (typ == "json") { + } else if (typ === "json") { return await res.json(); } } else { log.e(url, res); - throw new Error( - JSON.stringify([url, res, "response status unsuccessful at fileFetch"]), - ); + throw new Error(JSON.stringify([url, res, "fileFetch fail"])); } } @@ -221,11 +250,14 @@ async function makeTd(baseurl, n) { const tdpromises = []; for (let i = 0; i <= n; i++) { // td00.txt, td01.txt, td02.txt, ... , td98.txt, td100.txt, ... - const f = baseurl + "/td" + - (i).toLocaleString("en-US", { + const f = + baseurl + + "/td" + + i.toLocaleString("en-US", { minimumIntegerDigits: 2, useGrouping: false, - }) + ".txt"; + }) + + ".txt"; tdpromises.push(fileFetch(f, "buffer")); } const tds = await Promise.all(tdpromises); @@ -240,14 +272,11 @@ async function makeTd(baseurl, n) { // stackoverflow.com/a/40108543/ // Concatenate a mix of typed arrays function concat(arraybuffers) { - let sz = arraybuffers.reduce( - (sum, a) => sum + a.byteLength, - 0, - ); - let buf = new ArrayBuffer(sz); - let cat = new Uint8Array(buf); + const sz = arraybuffers.reduce((sum, a) => sum + a.byteLength, 0); + const buf = new ArrayBuffer(sz); + const cat = new Uint8Array(buf); let offset = 0; - for (let a of arraybuffers) { + for (const a of arraybuffers) { // github: jessetane/array-buffer-concat/blob/7d79d5ebf/index.js#L17 const v = new Uint8Array(a); cat.set(v, offset); diff --git a/src/command-control/cc.js b/src/command-control/cc.js index 2afb147edf..f379ccaa81 100644 --- a/src/command-control/cc.js +++ b/src/command-control/cc.js @@ -100,25 +100,25 @@ export class CommandControl { return response; } - let command = pathSplit[1]; + const command = pathSplit[1]; const b64UserFlag = this.userFlag(reqUrl, isDnsCmd); - if (command == "listtob64") { + if (command === "listtob64") { response.data.httpResponse = listToB64(queryString, blocklistFilter); - } else if (command == "b64tolist") { + } else if (command === "b64tolist") { response.data.httpResponse = b64ToList(queryString, blocklistFilter); - } else if (command == "dntolist") { + } else if (command === "dntolist") { response.data.httpResponse = domainNameToList( queryString, blocklistFilter, this.latestTimestamp ); - } else if (command == "dntouint") { + } else if (command === "dntouint") { response.data.httpResponse = domainNameToUint( queryString, blocklistFilter ); - } else if (command == "config" || command == "configure" || !isDnsCmd) { + } else if (command === "config" || command === "configure" || !isDnsCmd) { response.data.httpResponse = configRedirect( b64UserFlag, reqUrl.origin, @@ -157,19 +157,19 @@ function configRedirect(userFlag, origin, timestamp, highlight) { } function domainNameToList(queryString, blocklistFilter, latestTimestamp) { - let domainName = queryString.get("dn") || ""; - let returndata = {}; + const domainName = queryString.get("dn") || ""; + const returndata = {}; returndata.domainName = domainName; returndata.version = latestTimestamp; returndata.list = {}; - var searchResult = blocklistFilter.hadDomainName(domainName); + const searchResult = blocklistFilter.hadDomainName(domainName); if (searchResult) { let list; let listDetail = {}; - for (let entry of searchResult) { + for (const entry of searchResult) { list = blocklistFilter.getTag(entry[1]); listDetail = {}; - for (let listValue of list) { + for (const listValue of list) { listDetail[listValue] = blocklistFilter.blocklistFileTag[listValue]; } returndata.list[entry[0]] = listDetail; @@ -182,13 +182,13 @@ function domainNameToList(queryString, blocklistFilter, latestTimestamp) { } function domainNameToUint(queryString, blocklistFilter) { - let domainName = queryString.get("dn") || ""; - let returndata = {}; + const domainName = queryString.get("dn") || ""; + const returndata = {}; returndata.domainName = domainName; returndata.list = {}; - var searchResult = blocklistFilter.hadDomainName(domainName); + const searchResult = blocklistFilter.hadDomainName(domainName); if (searchResult) { - for (let entry of searchResult) { + for (const entry of searchResult) { returndata.list[entry[0]] = entry[1]; } } else { @@ -199,9 +199,9 @@ function domainNameToUint(queryString, blocklistFilter) { } function listToB64(queryString, blocklistFilter) { - let list = queryString.get("list") || []; - let flagVersion = parseInt(queryString.get("flagversion")) || 0; - let returndata = {}; + const list = queryString.get("list") || []; + const flagVersion = queryString.get("flagversion") || "0"; + const returndata = {}; returndata.command = "List To B64String"; returndata.inputList = list; returndata.flagVersion = flagVersion; @@ -213,15 +213,15 @@ function listToB64(queryString, blocklistFilter) { } function b64ToList(queryString, blocklistFilter) { - let b64 = queryString.get("b64") || ""; - let returndata = {}; + const b64 = queryString.get("b64") || ""; + const returndata = {}; returndata.command = "Base64 To List"; returndata.inputB64 = b64; - let response = blocklistFilter.unstamp(b64); + const response = blocklistFilter.unstamp(b64); if (response.userBlocklistFlagUint.length > 0) { returndata.list = blocklistFilter.getTag(response.userBlocklistFlagUint); returndata.listDetail = {}; - for (let listValue of returndata.list) { + for (const listValue of returndata.list) { returndata.listDetail[listValue] = blocklistFilter.blocklistFileTag[listValue]; } diff --git a/src/dns-operation/cacheResponse.js b/src/dns-operation/cacheResponse.js index 69163f34f6..573f68e5ae 100644 --- a/src/dns-operation/cacheResponse.js +++ b/src/dns-operation/cacheResponse.js @@ -7,10 +7,11 @@ */ import * as dnsutil from "../helpers/dnsutil.js"; +import * as cacheutil from "../helpers/cacheutil.js"; export default class DNSCacheResponse { - constructor() { - } + constructor() {} + /** * @param {*} param * @param {*} param.userBlocklistInfo @@ -23,7 +24,7 @@ export default class DNSCacheResponse { * @returns */ async RethinkModule(param) { - let response = {}; + const response = {}; response.isException = false; response.exceptionStack = ""; response.exceptionFrom = ""; @@ -44,9 +45,9 @@ export default class DNSCacheResponse { } async resolveFromCache(param) { - const key = dnsutil.cacheKey(param.requestDecodedDnsPacket); + const key = cacheutil.cacheKey(param.requestDecodedDnsPacket); if (!key) return false; - let cacheResponse = await param.dnsCache.get(key, param.request.url); + const cacheResponse = await param.dnsCache.get(key, param.request.url); console.debug("cache key : ", key); console.debug("Cache Response", JSON.stringify(cacheResponse)); if (!cacheResponse) return false; @@ -55,37 +56,38 @@ export default class DNSCacheResponse { param.userBlocklistInfo, param.requestDecodedDnsPacket, param.dnsQuestionBlock, - param.dnsResponseBlock, + param.dnsResponseBlock ); } } + async function parseCacheResponse(cr, blockInfo, reqDnsPacket, qb, rb) { - //check dns-block for incoming request against blocklist metadata from cache + // check dns-block for incoming request against blocklist metadata from cache let response = checkDnsBlock( qb, reqDnsPacket, cr.metaData.cacheFilter, - blockInfo, + blockInfo ); - console.debug("question block ",JSON.stringify(response)) + console.debug("question block ", JSON.stringify(response)); if (response && response.isBlocked) { return response; } - //cache response contains only metadata information - //return false to resolve dns request. + // cache response contains only metadata information + // return false to resolve dns request. if (!cr.metaData.bodyUsed) { return false; } - //answer block check + // answer block check response = checkDnsBlock( rb, cr.dnsPacket, cr.metaData.cacheFilter, - blockInfo, + blockInfo ); - console.debug("answer block ",JSON.stringify(response)) + console.debug("answer block ", JSON.stringify(response)); if (response && response.isBlocked) { return response; } @@ -100,11 +102,11 @@ function checkDnsBlock(qb, dnsPacket, cf, blockInfo) { } function generateResponse(cr, qid) { - let response = {}; + const response = {}; response.dnsPacket = cr.dnsPacket; - dnsutil.updateQueryId(response.dnsPacket, qid); - dnsutil.updateTtl(response.dnsPacket, cr.metaData.ttlEndTime); + cacheutil.updateQueryId(response.dnsPacket, qid); + cacheutil.updateTtl(response.dnsPacket, cr.metaData.ttlEndTime); response.dnsBuffer = dnsutil.encode(response.dnsPacket); diff --git a/src/dns-operation/dnsBlock.js b/src/dns-operation/dnsBlock.js index 4be6c51a2a..ad36710a31 100644 --- a/src/dns-operation/dnsBlock.js +++ b/src/dns-operation/dnsBlock.js @@ -6,12 +6,12 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import * as dnsutil from "../helpers/dnsutil.js"; -import * as dnsCacheUtil from "../helpers/cacheutil.js" -import * as dnsBlockUtil from "../helpers/dnsblockutil.js" +import * as dnsCacheUtil from "../helpers/cacheutil.js"; +import * as dnsBlockUtil from "../helpers/dnsblockutil.js"; export default class DNSQuestionBlock { - constructor() { - } + constructor() {} + /** * @param {*} param * @param {*} param.userBlocklistInfo @@ -23,7 +23,7 @@ export default class DNSQuestionBlock { * @returns */ async RethinkModule(param) { - let response = {}; + const response = {}; response.isException = false; response.exceptionStack = ""; response.exceptionFrom = ""; @@ -41,11 +41,11 @@ export default class DNSQuestionBlock { } dnsBlock(param) { - let response = this.performBlocking( + const response = this.performBlocking( param.userBlocklistInfo, param.requestDecodedDnsPacket, param.blocklistFilter, - false, + false ); if (response && response.isBlocked) { console.debug("add block response to cache"); @@ -55,7 +55,7 @@ export default class DNSQuestionBlock { param.blocklistFilter, param.requestDecodedDnsPacket, "", - param.event, + param.event ); } return response; @@ -63,21 +63,22 @@ export default class DNSQuestionBlock { performBlocking(blockInfo, dnsPacket, blf, cf) { if ( - !blockInfo.userBlocklistFlagUint || blockInfo.userBlocklistFlagUint === "" || + !blockInfo.userBlocklistFlagUint || + blockInfo.userBlocklistFlagUint === "" || !dnsutil.isBlockable(dnsPacket) ) { return false; } - const qn = dnsutil.getQueryName(dnsPacket.questions) - if(!qn) return false; + const qn = dnsutil.getQueryName(dnsPacket.questions); + if (!qn) return false; return dnsBlockUtil.doBlock(blf, blockInfo, qn, cf); } } function putCache(cache, url, blf, dnsPacket, buf, event) { - const key = dnsutil.cacheKey(dnsPacket); + const key = dnsCacheUtil.cacheKey(dnsPacket); if (!key) return; - let input = dnsCacheUtil.createCacheInput(dnsPacket, blf, false); + const input = dnsCacheUtil.createCacheInput(dnsPacket, blf, false); cache.put(key, input, url, buf, event); -} \ No newline at end of file +} diff --git a/src/dns-operation/dnsOperation.js b/src/dns-operation/dnsOperation.js index 8196bd0905..d159943287 100644 --- a/src/dns-operation/dnsOperation.js +++ b/src/dns-operation/dnsOperation.js @@ -10,5 +10,12 @@ import DNSParserWrap from "./dnsParserWrap.js"; import DNSQuestionBlock from "./dnsBlock.js"; import DNSResponseBlock from "./dnsResponseBlock.js"; import DNSResolver from "./dnsResolver.js"; -import DNSCacheResponse from "./cacheResponse.js" -export { DNSQuestionBlock, DNSParserWrap, DNSResolver, DNSResponseBlock, DNSCacheResponse }; +import DNSCacheResponse from "./cacheResponse.js"; + +export { + DNSQuestionBlock, + DNSParserWrap, + DNSResolver, + DNSResponseBlock, + DNSCacheResponse, +}; diff --git a/src/dns-operation/dnsParserWrap.js b/src/dns-operation/dnsParserWrap.js index d2a42e73fe..92bd63ee9f 100644 --- a/src/dns-operation/dnsParserWrap.js +++ b/src/dns-operation/dnsParserWrap.js @@ -19,6 +19,7 @@ export default class DNSParserWrap { throw e; } } + encode(DecodedDnsPacket) { try { return DnsParser.encode(DecodedDnsPacket); diff --git a/src/dns-operation/dnsResolver.js b/src/dns-operation/dnsResolver.js index 5a24986e03..52a1fa0514 100644 --- a/src/dns-operation/dnsResolver.js +++ b/src/dns-operation/dnsResolver.js @@ -11,7 +11,7 @@ import * as dnsutil from "../helpers/dnsutil.js"; import * as util from "../helpers/util.js"; import * as envutil from "../helpers/envutil.js"; import { DNSParserWrap as Dns } from "../dns-operation/dnsOperation.js"; -const quad1 = "1.1.1.2"; + export default class DNSResolver { constructor() { this.http2 = null; @@ -23,12 +23,18 @@ export default class DNSResolver { async lazyInit() { if (envutil.isNode() && !this.http2) { this.http2 = await import("http2"); + log.i("created custom http2 client"); + } + if (envutil.isNode() && !this.nodeUtil) { this.nodeUtil = await import("../helpers/node/util.js"); + log.i("imported node-util"); } if (envutil.isNode() && !this.transport) { + const plainOldDnsIp = dnsutil.dnsIpv4(); this.transport = new ( await import("../helpers/node/dns-transport.js") - ).Transport(quad1, 53); + ).Transport(plainOldDnsIp, 53); + log.i("created udp/tcp dns transport", plainOldDnsIp); } } @@ -60,9 +66,9 @@ export default class DNSResolver { const upRes = await this.resolveDnsUpstream( param.request, param.dnsResolverUrl, - param.requestBodyBuffer, + param.requestBodyBuffer ); - resp = await decodeResponse(upRes, this.dnsParser); + resp = await decodeResponse(upRes, this.dnsParser); return resp; } } @@ -75,17 +81,15 @@ async function decodeResponse(response, dnsParser) { log.d("!OK", response.status, response.statusText, await response.text()); throw new Error(response.status + " http err: " + response.statusText); } - let data = {}; + const data = {}; data.dnsBuffer = await response.arrayBuffer(); if (!dnsutil.validResponseSize(data.dnsBuffer)) { throw new Error("Null / invalid response from upstream"); } try { - //Todo: call dnsutil.encode makes answer[0].ttl as some big number need to debug. - data.dnsPacket = dnsParser.decode( - data.dnsBuffer, - ); + // TODO: at times, dnsutil.encode sets answer[0].ttl to some large number + data.dnsPacket = dnsParser.decode(data.dnsBuffer); } catch (e) { log.e("decode fail " + response.status + " cache? " + data.dnsBuffer); throw e; @@ -102,7 +106,7 @@ async function decodeResponse(response, dnsParser) { DNSResolver.prototype.resolveDnsUpstream = async function ( request, resolverUrl, - requestBodyBuffer, + requestBodyBuffer ) { try { // for now, upstream plain-old dns on fly @@ -120,9 +124,9 @@ DNSResolver.prototype.resolveDnsUpstream = async function ( : new Response(null, { status: 503 }); } - let u = new URL(request.url); - let dnsResolverUrl = new URL(resolverUrl); - u.hostname = dnsResolverUrl.hostname; // override host, default cloudflare-dns.com + const u = new URL(request.url); + const dnsResolverUrl = new URL(resolverUrl); + u.hostname = dnsResolverUrl.hostname; // default cloudflare-dns.com u.pathname = dnsResolverUrl.pathname; // override path, default /dns-query u.port = dnsResolverUrl.port; // override port, default 443 u.protocol = dnsResolverUrl.protocol; // override proto, default https @@ -161,7 +165,11 @@ DNSResolver.prototype.resolveDnsUpstream = async function ( * @returns {Promise} */ DNSResolver.prototype.doh2 = async function (request) { - console.debug("upstream using h2"); + if (!this.http2 || !this.nodeUtil) { + throw new Error("h2 / node-util not setup, bailing"); + } + + log.d("upstream with doh2"); const http2 = this.http2; const transformPseudoHeaders = this.nodeUtil.transformPseudoHeaders; diff --git a/src/dns-operation/dnsResponseBlock.js b/src/dns-operation/dnsResponseBlock.js index 43708dda14..cf57eea524 100644 --- a/src/dns-operation/dnsResponseBlock.js +++ b/src/dns-operation/dnsResponseBlock.js @@ -10,8 +10,7 @@ import * as dnsCacheUtil from "../helpers/cacheutil.js"; import * as dnsBlockUtil from "../helpers/dnsblockutil.js"; export default class DNSResponseBlock { - constructor() { - } + constructor() {} /** * @param {*} param @@ -25,7 +24,7 @@ export default class DNSResponseBlock { * @returns */ async RethinkModule(param) { - let response = {}; + const response = {}; response.isException = false; response.exceptionStack = ""; response.exceptionFrom = ""; @@ -35,7 +34,7 @@ export default class DNSResponseBlock { param.userBlocklistInfo, param.responseDecodedDnsPacket, param.blocklistFilter, - false, + false ); putCache( @@ -44,7 +43,7 @@ export default class DNSResponseBlock { param.blocklistFilter, param.responseDecodedDnsPacket, param.responseBodyBuffer, - param.event, + param.event ); } catch (e) { response.isException = true; @@ -57,33 +56,30 @@ export default class DNSResponseBlock { } performBlocking(blockInfo, dnsPacket, blf, cf) { - if ( - !blockInfo.userBlocklistFlagUint || blockInfo.userBlocklistFlagUint === "" - ) { + if (!hasBlockstamp(blockInfo)) { return false; - } - if (dnsutil.isCname(dnsPacket)) { + } else if (dnsutil.isCname(dnsPacket)) { return doCnameBlock(dnsPacket, blf, blockInfo, cf); - } - if (dnsutil.isHttps(dnsPacket)) { + } else if (dnsutil.isHttps(dnsPacket)) { return doHttpsBlock(dnsPacket, blf, blockInfo, cf); } + return false; } } function doHttpsBlock(dnsPacket, blf, blockInfo, cf) { console.debug("At Https-Svcb dns Block"); - let tn = dnsutil.getTargetName(dnsPacket.answers); + const tn = dnsutil.getTargetName(dnsPacket.answers); if (!tn) return false; return dnsBlockUtil.doBlock(blf, blockInfo, tn, cf); } function doCnameBlock(dnsPacket, blf, blockInfo, cf) { console.debug("At Cname dns Block"); - let cn = dnsutil.getCname(dnsPacket.answers); + const cn = dnsutil.getCname(dnsPacket.answers); let response = false; - for (let n of cn) { + for (const n of cn) { response = dnsBlockUtil.doBlock(blf, blockInfo, n, cf); if (response.isBlocked) break; } @@ -92,9 +88,15 @@ function doCnameBlock(dnsPacket, blf, blockInfo, cf) { function putCache(cache, url, blf, dnsPacket, buf, event) { if (!dnsCacheUtil.isCacheable(dnsPacket)) return; - const key = dnsutil.cacheKey(dnsPacket); + const key = dnsCacheUtil.cacheKey(dnsPacket); if (!key) return; - let input = dnsCacheUtil.createCacheInput(dnsPacket, blf, true); + const input = dnsCacheUtil.createCacheInput(dnsPacket, blf, true); console.debug("Cache Input ", JSON.stringify(input)); cache.put(key, input, url, buf, event); -} \ No newline at end of file +} + +function hasBlockstamp(blockInfo) { + return ( + blockInfo.userBlocklistFlagUint && blockInfo.userBlocklistFlagUint !== "" + ); +} diff --git a/src/helpers/cacheutil.js b/src/helpers/cacheutil.js index 6f9d20e1ae..6638790380 100644 --- a/src/helpers/cacheutil.js +++ b/src/helpers/cacheutil.js @@ -8,7 +8,9 @@ import * as util from "./util.js"; import * as dnsutil from "./dnsutil.js"; + const ttlGraceSec = 30; // 30s cache extra time + export function generateQuestionFilter(cf, blf, dnsPacket) { const q = dnsPacket.questions[0].name; cf[q] = util.objOf(blf.getDomainInfo(q).searchResult); @@ -28,7 +30,7 @@ export function generateAnswerFilter(cf, blf, dnsPacket) { } export function addCacheFilter(cf, blf, li) { - for (let name of li) { + for (const name of li) { cf[name] = util.objOf(blf.getDomainInfo(name).searchResult); } } @@ -49,20 +51,20 @@ export function determineCacheExpiry(dnsPacket) { if (!dnsutil.hasAnswers(dnsPacket)) return expiresImmediately; // set min(ttl) among all answers, but at least ttlGraceSec let minttl = 1 << 30; // some abnormally high ttl - for (let a of dnsPacket.answers) { + for (const a of dnsPacket.answers) { minttl = Math.min(a.ttl || minttl, minttl); } if (minttl === 1 << 30) return expiresImmediately; minttl = Math.max(minttl + ttlGraceSec, ttlGraceSec); - const expiry = Date.now() + (minttl * 1000); + const expiry = Date.now() + minttl * 1000; return expiry; } export function cacheMetadata(dnsPacket, ttlEndTime, blf, bodyUsed) { - let cf = {}; + const cf = {}; generateAnswerFilter(cf, blf, dnsPacket); generateQuestionFilter(cf, blf, dnsPacket); return { @@ -74,14 +76,35 @@ export function cacheMetadata(dnsPacket, ttlEndTime, blf, bodyUsed) { export function createCacheInput(dnsPacket, blf, bodyUsed) { const ttlEndTime = determineCacheExpiry(dnsPacket); - let cacheInput = {}; - cacheInput.metaData = cacheMetadata( - dnsPacket, - ttlEndTime, - blf, - bodyUsed, - ); + const cacheInput = {}; + cacheInput.metaData = cacheMetadata(dnsPacket, ttlEndTime, blf, bodyUsed); cacheInput.dnsPacket = dnsPacket; return cacheInput; } +export function updateTtl(decodedDnsPacket, end) { + const now = Date.now(); + const outttl = Math.max( + Math.floor((end - now) / 1000) - ttlGraceSec, + ttlGraceSec + ); + for (const a of decodedDnsPacket.answers) { + if (!dnsutil.optAnswer(a)) a.ttl = outttl; + } +} + +export function cacheKey(packet) { + // multiple questions are kind of an undefined behaviour + // stackoverflow.com/a/55093896 + if (!dnsutil.hasSingleQuestion(packet)) return null; + + const name = packet.questions[0].name.trim().toLowerCase(); + const type = packet.questions[0].type; + return name + ":" + type; +} + +export function updateQueryId(decodedDnsPacket, queryId) { + if (queryId === decodedDnsPacket.id) return false; // no change + decodedDnsPacket.id = queryId; + return true; +} diff --git a/src/helpers/deno/config.ts b/src/helpers/deno/config.ts index 786a3ebe98..484f8958f8 100644 --- a/src/helpers/deno/config.ts +++ b/src/helpers/deno/config.ts @@ -1,10 +1,41 @@ import { config as dotEnvConfig } from "dotenv"; +import * as system from "../../system.js"; +import Log from "../log.js"; +import EnvManager from "../env.js"; -// Load env variables from .env file to Deno.env (if file exists) -try { - dotEnvConfig({ export: true }); - Deno.env.set("RUNTIME", "deno"); -} catch (e) { - // throws without --allow-read flag - console.warn(".env file may not be loaded => ", e.name, ":", e.message); +// In global scope. +declare global { + // TypeScript compiler needs to know type of every variable / property. + // So, we extend the window object (globalThis) with declaration merging. + // See: https://www.typescriptlang.org/docs/handbook/declaration-merging.html + interface Window { + envManager?: EnvManager; + log?: Log; + env?: any; + } } + +((main) => { + if (!Deno) throw new Error("failed loading deno-specific config"); + + const isProd = Deno.env.get("DENO_ENV") === "production"; + + // Load env variables from .env file to Deno.env (if file exists) + try { + dotEnvConfig({ export: true }); + // override: if we are running this file, then we're on Deno + Deno.env.set("RUNTIME", "deno"); + } catch (e) { + // throws without --allow-read flag + console.warn(".env file may not be loaded => ", e.name, ":", e.message); + } + + window.envManager = new EnvManager(); + + window.log = new Log( + window.env.logLevel, + isProd, // set console level only in prod. + ); + + system.pub("ready"); +})(); diff --git a/src/helpers/dnsblockutil.js b/src/helpers/dnsblockutil.js index fcc1c8efdf..5e12132ce5 100644 --- a/src/helpers/dnsblockutil.js +++ b/src/helpers/dnsblockutil.js @@ -7,34 +7,34 @@ */ import * as util from "../helpers/util.js"; -export function isBlocklistFiter(blf) { - return (blf && blf.t && blf.ft) +export function isBlocklistFilterSetup(blf) { + return blf && blf.t && blf.ft; } export function doBlock(blf, userBlInfo, key, cf) { - let blInfo = getDomainInfo(blf, cf, key); - console.debug("blocklist filter result : ",JSON.stringify(blInfo)) + const blInfo = getDomainInfo(blf, cf, key); + console.debug("blocklist filter result : ", JSON.stringify(blInfo)); if (!blInfo) return false; - let response = checkDomainBlocking( + const response = checkDomainBlocking( userBlInfo.userBlocklistFlagUint, userBlInfo.flagVersion, blInfo, - key, + key ); if (response && response.isBlocked) return response; - if(!userBlInfo.userServiceListUint) return response; + if (!userBlInfo.userServiceListUint) return response; return checkWildcardBlocking( userBlInfo.userServiceListUint, userBlInfo.flagVersion, blInfo, - key, + key ); } function getDomainInfo(blf, cf, key) { - if (isBlocklistFiter(blf)) { + if (isBlocklistFilterSetup(blf)) { return blf.getDomainInfo(key).searchResult; } if (!cf && !cf.hasOwnProperty(key)) return false; @@ -43,7 +43,7 @@ function getDomainInfo(blf, cf, key) { function checkDomainBlocking(ufUint, flagVersion, blocklistMap, dn) { try { - let dnUint = blocklistMap.get(dn); + const dnUint = blocklistMap.get(dn); if (!dnUint) return false; return checkFlagIntersection(ufUint, dnUint, flagVersion, blocklistMap, dn); } catch (e) { @@ -52,11 +52,11 @@ function checkDomainBlocking(ufUint, flagVersion, blocklistMap, dn) { } function checkWildcardBlocking(wcUint, flagVersion, blocklistMap, dn) { - let dnSplit = dn.split("."); + const dnSplit = dn.split("."); let dnJoin = ""; let response = {}; let dnUint; - while (dnSplit.shift() != undefined) { + while (dnSplit.shift() !== undefined) { dnJoin = dnSplit.join("."); dnUint = blocklistMap.get(dn); if (!dnUint) return false; @@ -65,7 +65,7 @@ function checkWildcardBlocking(wcUint, flagVersion, blocklistMap, dn) { dnUint, flagVersion, blocklistMap, - dnJoin, + dnJoin ); if (response && response.isBlocked) { return response; @@ -76,54 +76,49 @@ function checkWildcardBlocking(wcUint, flagVersion, blocklistMap, dn) { function checkFlagIntersection(uint1, uint2, flagVersion, blocklistMap, key) { try { - let response = {}; + const response = {}; response.isBlocked = false; response.blockedB64Flag = ""; response.blockedTag = []; - let dnUint = blocklistMap.get(key); + const dnUint = blocklistMap.get(key); let blockedUint = flagIntersection(uint1, uint2); if (blockedUint) { response.isBlocked = true; - response.blockedB64Flag = getB64Flag( - blockedUint, - flagVersion, - ); + response.blockedB64Flag = getB64Flag(blockedUint, flagVersion); } else { blockedUint = new Uint16Array(dnUint); - response.blockedB64Flag = getB64Flag( - blockedUint, - flagVersion, - ); + response.blockedB64Flag = getB64Flag(blockedUint, flagVersion); } - return response; - //response.blockedTag = blocklistFilter.getTag(blockedUint); + return response; + // response.blockedTag = blocklistFilter.getTag(blockedUint); } catch (e) { throw e; } } -export function flagIntersection(flag1, flag2) { +export function flagIntersection(flag1, flag2) { try { if (util.emptyString(flag1) || util.emptyString(flag2)) return false; + let flag1Header = flag1[0]; let flag2Header = flag2[0]; let intersectHeader = flag1Header & flag2Header; - if (intersectHeader == 0) { - //console.log("first return") + + if (intersectHeader === 0) { return false; } + let flag1Length = flag1.length - 1; let flag2Length = flag2.length - 1; const intersectBody = []; let tmpInterectHeader = intersectHeader; let maskHeaderForBodyEmpty = 1; let tmpBodyIntersect; - for (; tmpInterectHeader != 0;) { - if ((flag1Header & 1) == 1) { - if ((tmpInterectHeader & 1) == 1) { + for (; tmpInterectHeader !== 0; ) { + if ((flag1Header & 1) === 1) { + if ((tmpInterectHeader & 1) === 1) { tmpBodyIntersect = flag1[flag1Length] & flag2[flag2Length]; - //console.log(flag1[flag1Length] + " :&: " + flag2[flag2Length] + " -- " + tmpBodyIntersect) - if (tmpBodyIntersect == 0) { + if (tmpBodyIntersect === 0) { intersectHeader = intersectHeader ^ maskHeaderForBodyEmpty; } else { intersectBody.push(tmpBodyIntersect); @@ -131,7 +126,7 @@ export function flagIntersection(flag1, flag2) { } flag1Length = flag1Length - 1; } - if ((flag2Header & 1) == 1) { + if ((flag2Header & 1) === 1) { flag2Length = flag2Length - 1; } flag1Header = flag1Header >>> 1; @@ -139,16 +134,16 @@ export function flagIntersection(flag1, flag2) { flag2Header = flag2Header >>> 1; maskHeaderForBodyEmpty = maskHeaderForBodyEmpty * 2; } - //console.log(intersectBody) - if (intersectHeader == 0) { - //console.log("Second Return") + + if (intersectHeader === 0) { return false; } + const intersectFlag = new Uint16Array(intersectBody.length + 1); let count = 0; intersectFlag[count++] = intersectHeader; let bodyData; - while ((bodyData = intersectBody.pop()) != undefined) { + while ((bodyData = intersectBody.pop()) !== undefined) { intersectFlag[count++] = bodyData; } return intersectFlag; @@ -159,16 +154,17 @@ export function flagIntersection(flag1, flag2) { function getB64Flag(uint16Arr, flagVersion) { try { - if (flagVersion == "0") { + if (flagVersion === "0") { return encodeURIComponent(Buffer.from(uint16Arr).toString("base64")); - } else if (flagVersion == "1") { - return "1:" + + } else if (flagVersion === "1") { + return ( + "1:" + encodeURI( - btoa(encodeUint16arrToBinary(uint16Arr)).replace(/\//g, "_").replace( - /\+/g, - "-", - ), - ); + btoa(encodeUint16arrToBinary(uint16Arr)) + .replace(/\//g, "_") + .replace(/\+/g, "-") + ) + ); } } catch (e) { throw e; diff --git a/src/helpers/dnsutil.js b/src/helpers/dnsutil.js index 89dda40ff1..b54874a8ce 100644 --- a/src/helpers/dnsutil.js +++ b/src/helpers/dnsutil.js @@ -6,37 +6,49 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { DNSParserWrap as Dns } from "../dns-operation/dnsOperation.js" -import * as envutil from "./envutil.js" +import { DNSParserWrap as Dns } from "../dns-operation/dnsOperation.js"; +import * as envutil from "./envutil.js"; // dns packet constants (in bytes) // A dns message over TCP stream has a header indicating length. -export const dnsHeaderSize = 2 -export const dnsPacketHeaderSize = 12 -export const minDNSPacketSize = dnsPacketHeaderSize + 5 -export const maxDNSPacketSize = 4096 +export const dnsHeaderSize = 2; +export const dnsPacketHeaderSize = 12; +export const minDNSPacketSize = dnsPacketHeaderSize + 5; +export const maxDNSPacketSize = 4096; -const minRequestTimeout = 5000 // 7s -const defaultRequestTimeout = 15000 // 15s -const maxRequestTimeout = 30000 // 30s -const dns = new Dns() +const _dnsCloudflareSec = "1.1.1.2"; +const _dnsCacheSize = 10000; + +const _minRequestTimeout = 5000; // 7s +const _defaultRequestTimeout = 15000; // 15s +const _maxRequestTimeout = 30000; // 30s + +const dns = new Dns(); + +export function dnsIpv4() { + return _dnsCloudflareSec; +} + +export function cacheSize() { + return _dnsCacheSize; +} export function servfail(qid, qs) { - if (!qid || !qs) return null + if (!qid || !qs) return null; return encode({ id: qid, type: "response", flags: 4098, // servfail questions: qs, - }) + }); } export function requestTimeout() { - const t = envutil.workersTimeout(defaultRequestTimeout) - return (t > minRequestTimeout) ? - Math.min(t, maxRequestTimeout) : - minRequestTimeout + const t = envutil.workersTimeout(_defaultRequestTimeout); + return t > _minRequestTimeout + ? Math.min(t, _maxRequestTimeout) + : _minRequestTimeout; } export function truncated(ans) { @@ -49,12 +61,11 @@ export function truncated(ans) { } export function validResponseSize(r) { - return r && validateSize(r.byteLength) + return r && validateSize(r.byteLength); } export function validateSize(sz) { - return sz >= minDNSPacketSize && - sz <= maxDNSPacketSize + return sz >= minDNSPacketSize && sz <= maxDNSPacketSize; } export function hasAnswers(packet) { @@ -91,72 +102,49 @@ export function decode(buf) { } export function isBlockable(packet) { - return hasSingleQuestion(packet) && (packet.questions[0].type == "A" || - packet.questions[0].type == "AAAA" || - packet.questions[0].type == "CNAME" || - packet.questions[0].type == "HTTPS" || - packet.questions[0].type == "SVCB"); -} - -export function cacheKey(packet) { - // multiple questions are kind of an undefined behaviour - // stackoverflow.com/a/55093896 - if (!hasSingleQuestion(packet)) return null; - - const name = packet.questions[0].name - .trim() - .toLowerCase(); - const type = packet.questions[0].type; - return name + ":" + type; -} - -export function updateTtl(decodedDnsPacket, end) { - const now = Date.now(); - const outttl = Math.max(Math.floor((end - now) / 1000), 30); // ttl grace already set during cache put - for (let a of decodedDnsPacket.answers) { - if (!optAnswer(a)) a.ttl = outttl; - } -} - -export function updateQueryId(decodedDnsPacket, queryId) { - if (queryId === 0) return false; // doh reqs are qid free - if (queryId === decodedDnsPacket.id) return false; // no change - decodedDnsPacket.id = queryId; - return true; + return ( + hasSingleQuestion(packet) && + (packet.questions[0].type === "A" || + packet.questions[0].type === "AAAA" || + packet.questions[0].type === "CNAME" || + packet.questions[0].type === "HTTPS" || + packet.questions[0].type === "SVCB") + ); } export function isCname(packet) { - return (hasAnswers(packet) && packet.answers[0].type == "CNAME"); + return hasAnswers(packet) && packet.answers[0].type === "CNAME"; } export function isHttps(packet) { - return (hasAnswers(packet) && - (packet.answers[0].type == "HTTPS" || packet.answers[0].type == "SVCB")); + return ( + hasAnswers(packet) && + (packet.answers[0].type === "HTTPS" || packet.answers[0].type === "SVCB") + ); } export function getCname(answers) { - let li = []; + const li = []; li[0] = answers[0].data.trim().toLowerCase(); - li[1] = answers[answers.length - 1].name.trim() - .toLowerCase(); + li[1] = answers[answers.length - 1].name.trim().toLowerCase(); return li; } export function dohStatusCode(b) { - if (!b || !b.byteLength) return 412 - if (b.byteLength > maxDNSPacketSize) return 413 - if (b.byteLength < minDNSPacketSize) return 400 - return 200 + if (!b || !b.byteLength) return 412; + if (b.byteLength > maxDNSPacketSize) return 413; + if (b.byteLength < minDNSPacketSize) return 400; + return 200; } export function getTargetName(answers) { - let tn = answers[0].data.targetName.trim().toLowerCase(); + const tn = answers[0].data.targetName.trim().toLowerCase(); if (tn === ".") return false; return tn; } export function getQueryName(questions) { - let qn = questions[0].name.trim().toLowerCase(); + const qn = questions[0].name.trim().toLowerCase(); if (qn === "") return false; return qn; } diff --git a/src/helpers/env.js b/src/helpers/env.js index 95ec9d91a5..cc5bf1ee15 100644 --- a/src/helpers/env.js +++ b/src/helpers/env.js @@ -23,44 +23,68 @@ * runtimes or / and specify a type, if not string. */ const _ENV_VAR_MAPPINGS = { - runTime: "RUNTIME", + // "internal-name": "Runtime specific variable name(s)". + runTime: { + name: "RUNTIME", + type: "string", + }, runTimeEnv: { - worker: "WORKER_ENV", - node: "NODE_ENV", - deno: "DENO_ENV", + name: { + worker: "WORKER_ENV", + node: "NODE_ENV", + deno: "DENO_ENV", + }, + type: "string", + }, + cloudPlatform: { + name: "CLOUD_PLATFORM", + type: "string", + }, + logLevel: { + name: "LOG_LEVEL", + type: "string", + }, + blocklistUrl: { + name: "CF_BLOCKLIST_URL", + type: "string", + }, + latestTimestamp: { + name: "CF_LATEST_BLOCKLIST_TIMESTAMP", + type: "string", + }, + dnsResolverUrl: { + name: "CF_DNS_RESOLVER_URL", + type: "string", }, - cloudPlatform: "CLOUD_PLATFORM", - logLevel: "LOG_LEVEL", - blocklistUrl: "CF_BLOCKLIST_URL", - latestTimestamp: "CF_LATEST_BLOCKLIST_TIMESTAMP", - dnsResolverUrl: "CF_DNS_RESOLVER_URL", onInvalidFlagStopProcessing: { + name: "CF_ON_INVALID_FLAG_STOPPROCESSING", type: "boolean", - all: "CF_ON_INVALID_FLAG_STOPPROCESSING", }, - - // parallel request wait timeout for download blocklist from s3 + workerTimeout: { + name: "WORKER_TIMEOUT", + type: "number", + }, fetchTimeout: { + name: "CF_BLOCKLIST_DOWNLOAD_TIMEOUT", type: "number", - all: "CF_BLOCKLIST_DOWNLOAD_TIMEOUT", }, - - // env variables for td file split tdNodecount: { + name: "TD_NODE_COUNT", type: "number", - all: "TD_NODE_COUNT", }, tdParts: { + name: "TD_PARTS", type: "number", - all: "TD_PARTS", }, // set to on - off aggressive cache plugin // as of now Cache-api is available only on worker - // so _getRuntimeEnv will set this to false for other runtime. + // so load() will set this to false for other runtime. isAggCacheReq: { + name: { + worker: "IS_AGGRESSIVE_CACHE_REQ", + }, type: "boolean", - worker: "IS_AGGRESSIVE_CACHE_REQ", }, }; @@ -73,37 +97,53 @@ function _getRuntimeEnv(runtime) { console.info("Loading env. from runtime:", runtime); const env = {}; - for (const [key, mapping] of Object.entries(_ENV_VAR_MAPPINGS)) { + for (const [key, mappedKey] of Object.entries(_ENV_VAR_MAPPINGS)) { let name = null; - let type = "string"; + let type = null; + + if (typeof mappedKey !== "object") continue; + + if (typeof mappedKey.name === "object") { + name = mappedKey.name[runtime]; + } else { + name = mappedKey.name; + } + type = mappedKey.type; - if (typeof mapping === "string") { - name = mapping; - } else if (typeof mapping === "object") { - name = mapping.all || mapping[runtime]; - type = mapping.type || type; - } else throw new Error("Unfamiliar mapping"); + if (!name || !type) { + console.debug(runtime, "unnamed / untyped env mapping", key, mappedKey); + continue; + } if (runtime === "node") env[key] = process.env[name]; - else if (runtime === "deno") env[key] = name && Deno.env.get(name); + else if (runtime === "deno") env[key] = Deno.env.get(name); else if (runtime === "worker") env[key] = globalThis[name]; - else throw new Error(`Unknown runtime: ${runtime}`); + else throw new Error(`unsupported runtime: ${runtime}`); - // All env are assumed to be strings, so typecast them. + // env vars are strings by default, typecast to their respective types if (type === "boolean") env[key] = !!env[key]; else if (type === "number") env[key] = Number(env[key]); else if (type === "string") env[key] = env[key] || ""; - else throw new Error(`Unsupported type: ${type}`); + else throw new Error(`unsupported type: ${type}`); + + console.debug("added", key, mappedKey, env[key]); } return env; } -function _getRuntime() { - // As `process` also exists in worker, we need to check for worker first. - if (globalThis.RUNTIME === "worker") return "worker"; - if (typeof Deno !== "undefined") return "deno"; - if (typeof process !== "undefined") return "node"; +function _determineRuntimeIfPossible() { + if (typeof Deno !== "undefined") { + return Deno.env.get("RUNTIME") || "deno"; + } + + if (typeof process !== "undefined") { + // process also exists in Workers, where RUNTIME is defined + if (globalThis.RUNTIME) return globalThis.RUNTIME; + if (process.env) return process.env.RUNTIME || "node"; + } + + return null; } export default class EnvManager { @@ -111,40 +151,25 @@ export default class EnvManager { * Initializes the env manager. */ constructor() { - if (globalThis.env) throw new Error("envManager is already initialized."); - - globalThis.env = {}; + this.runtime = _determineRuntimeIfPossible(); this.envMap = new Map(); - this.isLoaded = false; + this.load(); } /** * Loads env variables from runtime env. and is made globally available * through `env` namespace. Existing env variables will be overwritten. */ - loadEnv() { - const runtime = _getRuntime(); - const env = _getRuntimeEnv(runtime); - for (const [key, value] of Object.entries(env)) { - this.envMap.set(key, value); + load() { + const renv = _getRuntimeEnv(this.runtime); + + globalThis.env = renv; // global `env` namespace. + + for (const [k, v] of Object.entries(renv)) { + this.envMap.set(k, v); } - // adding download timeout with worker time to determine worker's overall - // timeout - runtime === "worker" && - this.envMap.set( - "workerTimeout", - Number(WORKER_TIMEOUT) + Number(CF_BLOCKLIST_DOWNLOAD_TIMEOUT) - ); - - console.debug( - "Loaded env: ", - (runtime === "worker" && JSON.stringify(this.toObject())) || - this.toObject() - ); - - globalThis.env = this.toObject(); // Global `env` namespace. - this.isLoaded = true; + console.debug("env loaded: ", JSON.stringify(renv)); } /** @@ -166,7 +191,18 @@ export default class EnvManager { * @return {*} - env variable value */ get(key) { - return this.envMap.get(key); + const v = this.envMap.get(key); + if (v) return v; + + if (this.runtime === "node") { + return process.env[key]; + } else if (this.runtime === "deno") { + return Deno.env.get(key); + } else if (this.runtime === "worker") { + return globalThis[key]; + } + + return null; } /** @@ -175,6 +211,6 @@ export default class EnvManager { */ set(key, value) { this.envMap.set(key, value); - globalThis.env = this.toObject(); + globalThis.env[key] = value; } } diff --git a/src/helpers/envutil.js b/src/helpers/envutil.js index bcf364cd85..797c3887c6 100644 --- a/src/helpers/envutil.js +++ b/src/helpers/envutil.js @@ -10,6 +10,15 @@ export function onFly() { return env && env.cloudPlatform === "fly"; } +export function hasDisk() { + // got disk on test nodejs envs and on fly + return onFly() || (isNode() && !isProd()); +} + +export function isProd() { + return env && env.runTimeEnv === "production"; +} + export function isWorkers() { return env && env.runTime === "worker"; } @@ -21,3 +30,24 @@ export function isNode() { export function workersTimeout(defaultValue = 0) { return (env && env.workerTimeout) || defaultValue; } + +export function blocklistUrl() { + if (!env) return null; + return env.blocklistUrl; +} + +export function timestamp() { + if (!env) return null; + return env.latestTimestamp; +} + +export function tdNodeCount() { + if (!env) return null; + return env.tdNodecount; +} + +export function tdParts() { + if (!env) return null; + return env.tdParts; +} + diff --git a/src/helpers/node/blocklists.js b/src/helpers/node/blocklists.js new file mode 100644 index 0000000000..36b61030d0 --- /dev/null +++ b/src/helpers/node/blocklists.js @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2021 RethinkDNS and its authors. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import * as fs from "fs"; +import * as path from "path"; +import * as util from "../util.js"; +import * as envutil from "../envutil.js"; + +const blocklistsDir = "./blocklists__"; +const tdFile = "td.txt"; +const rdFile = "rd.txt"; +const ftFile = "filetag.json"; + +export async function setup(bw) { + + if (!bw || !envutil.hasDisk()) return false; + + const now = Date.now(); + const url = envutil.blocklistUrl(); + const timestamp = envutil.timestamp(); + const nodecount = envutil.tdNodeCount(); + const tdparts = envutil.tdParts(); + + const ok = setupLocally(bw, timestamp, nodecount); + if (ok) { + log.i("bl setup locally tstamp/nc", timestamp, nodecount); + return true; + } + + log.i("dowloading bl tstamp/nc/parts", timestamp, nodecount, tdparts); + await bw.initBlocklistConstruction( + now, + url, + timestamp, + nodecount, + tdparts + ); + + save(bw, timestamp); +} + +function save(bw, timestamp) { + if (!bw.isBlocklistFilterSetup()) return false; + + mkdirsIfNeeded(timestamp); + + const [tdfp, rdfp, ftfp] = getFilePaths(timestamp); + + // write out array-buffers to disk + fs.writeFileSync(tdfp, util.bufferOf(bw.td)); + fs.writeFileSync(rdfp, util.bufferOf(bw.rd)); + fs.writeFileSync(ftfp, JSON.stringify(bw.ft)); + + log.i("blocklists written to disk"); + + return true; +} + +function setupLocally(bw, timestamp, nodecount) { + if (!hasBlocklistFiles(timestamp)) return false; + + const [td, rd, ft] = getFilePaths(timestamp); + log.i("on-disk td/rd/ft", td, rd, ft); + + const tdbuf = fs.readFileSync(td); + const rdbuf = fs.readFileSync(rd); + const ftbuf = fs.readFileSync(ft, "utf-8"); + + // TODO: file integrity checks + const ab0 = util.arrayBufferOf(tdbuf); + const ab1 = util.arrayBufferOf(rdbuf); + const json1 = JSON.parse(ftbuf); + const json2 = { nodecount: nodecount }; + + bw.initBlocklistFilterConstruction( + /*trie*/ ab0, + /*rank-dir*/ ab1, + /*file-tag*/ json1, + /*basic-config*/ json2 + ); + + return true; +} + +function hasBlocklistFiles(timestamp) { + const [td, rd, ft] = getFilePaths(timestamp); + + return fs.existsSync(td) && fs.existsSync(rd) && fs.existsSync(ft); +} + +function getFilePaths(t) { + const td = path.normalize(blocklistsDir + "/" + t + "/" + tdFile); + const rd = path.normalize(blocklistsDir + "/" + t + "/" + rdFile); + const ft = path.normalize(blocklistsDir + "/" + t + "/" + ftFile); + + return [td, rd, ft]; +} + +function getDirPaths(t) { + const bldir = path.normalize(blocklistsDir); + const tsdir = path.normalize(blocklistsDir + "/" + t); + + return [bldir, tsdir]; +} + +function mkdirsIfNeeded(timestamp) { + const [dir1, dir2] = getDirPaths(timestamp); + + if (!fs.existsSync(dir1)) { + log.i("creating blocklist dir", dir1) + fs.mkdirSync(dir1); + } + + if (!fs.existsSync(dir2)) { + log.i("creating timestamp dir", dir2) + fs.mkdirSync(dir2); + } +} + diff --git a/src/helpers/node/config.js b/src/helpers/node/config.js index c68f0306db..090399c87b 100644 --- a/src/helpers/node/config.js +++ b/src/helpers/node/config.js @@ -1,64 +1,94 @@ /** * Configuration file for node runtime - * TODO?: Remove all side-effects and use a constructor? - * - This module has side effects, sequentially setting up the environment. - * - Only variables may be exported from this module. - * - Don't define functions here, import functions if required. + * TODO: Remove all side-effects and use a constructor? + * This module has side effects, sequentially setting up the environment. */ import { atob, btoa } from "buffer"; import fetch, { Headers, Request, Response } from "node-fetch"; import { getTLSfromEnv } from "./util.js"; -import EnvManager from "../env.js"; import Log from "../log.js"; +import * as system from "../../system.js"; +import EnvManager from "../env.js"; +import * as swap from "../linux/swap.js"; + +(async (main) => { + // if this file execs... assume we're on nodejs. + const runtime = "node"; + const isProd = process.env.NODE_ENV === "production"; + const onFly = process.env.CLOUD_PLATFORM === "fly"; + let devutils = null; + let dotenv = null; + + // dev utilities + if (!isProd) { + devutils = await import("./util-dev.js"); + dotenv = await import("dotenv"); + } + + /** Environment Variables */ + // Load env variables from .env file to process.env (if file exists) + // NOTE: this won't overwrite existing + if (dotenv) { + dotenv.config(); + console.log("loading local .env"); + } -/** Environment Variables */ -// Load env variables from .env file to process.env (if file exists) -// NOTE: this won't overwrite existing -if (process.env.NODE_ENV !== "production") (await import("dotenv")).config(); -process.env.RUNTIME = "node"; + console.log("override runtime, from", process.env.RUNTIME, "to", runtime); + process.env.RUNTIME = runtime; // must call before creating env-manager -if (!globalThis.envManager) globalThis.envManager = new EnvManager(); -envManager.loadEnv(); + globalThis.envManager = new EnvManager(); -/** Logging level */ -globalThis.log = new Log( - env.logLevel, + /** Logger */ + globalThis.log = new Log( + env.logLevel, + isProd // set console level only in prod. + ); - // set console level only in production - !console.level && env.runTimeEnv === "production" -); + /** TLS crt and key */ + // Raw TLS CERT and KEY are stored (base64) in an env var for fly deploys + // (fly deploys are dev/prod nodejs deploys where env TLS_CN or TLS_ is set). + // Otherwise, retrieve KEY and CERT from the filesystem (this is the case + // for local non-prod nodejs deploys with self-signed certs). + const _TLS_CRT_AND_KEY = + eval(`process.env.TLS_${process.env.TLS_CN}`) || process.env.TLS_; -/** Polyfills */ -if (!globalThis.fetch) { - globalThis.fetch = - process.env.NODE_ENV !== "production" - ? (await import("./util-dev.js")).fetchPlus - : fetch; - globalThis.Headers = Headers; - globalThis.Request = Request; - globalThis.Response = Response; -} + if (isProd || _TLS_CRT_AND_KEY) { + const [tlsKey, tlsCrt] = getTLSfromEnv(_TLS_CRT_AND_KEY); + envManager.set("tlsKey", tlsKey); + envManager.set("tlsCrt", tlsCrt); + console.log("env (fly) tls setup"); + } else { + const [tlsKey, tlsCrt] = devutils.getTLSfromFile( + process.env.TLS_KEY_PATH, + process.env.TLS_CRT_PATH + ); + envManager.set("tlsKey", tlsKey); + envManager.set("tlsCrt", tlsCrt); + console.info("dev (local) tls setup"); + } -if (!globalThis.atob || !globalThis.btoa) { - globalThis.atob = atob; - globalThis.btoa = btoa; -} + /** Polyfills */ + if (!globalThis.fetch) { + globalThis.fetch = isProd ? fetch : devutils.fetchPlus; + globalThis.Headers = Headers; + globalThis.Request = Request; + globalThis.Response = Response; + log.i("polyfill fetch web api"); + } -/** TLS crt and key */ -const _TLS_CRT_KEY = - eval(`process.env.TLS_${process.env.TLS_CN}`) || process.env.TLS_; + if (!globalThis.atob || !globalThis.btoa) { + globalThis.atob = atob; + globalThis.btoa = btoa; + log.i("polyfill atob / btoa"); + } -export const [TLS_KEY, TLS_CRT] = - process.env.NODE_ENV === "production" || _TLS_CRT_KEY != null - ? getTLSfromEnv(_TLS_CRT_KEY) - : (await import("./util-dev.js")).getTLSfromFile( - process.env.TLS_KEY_PATH, - process.env.TLS_CRT_PATH - ); + /** Swap on Fly */ + if (onFly) { + const ok = swap.mkswap(); + console.info("mkswap done?", ok); + } -/** Swap on fly */ -if (process.env.CLOUD_PLATFORM === "fly") { - const ok = (await import("../linux/swap.js")).mkswap(); - console.info("mkswap done?", ok); -} + /** signal up */ + system.pub("ready"); +})(); diff --git a/src/helpers/plugin.js b/src/helpers/plugin.js index ae4c6d757d..4ea04bd469 100644 --- a/src/helpers/plugin.js +++ b/src/helpers/plugin.js @@ -15,19 +15,41 @@ import { DNSResponseBlock, } from "../dns-operation/dnsOperation.js"; import * as util from "./util.js"; +import * as envutil from "./envutil.js"; +import * as system from "../system.js"; import { DnsCache } from "../cache-wrapper/cache-wrapper.js"; import * as dnsutil from "../helpers/dnsutil.js"; -const blocklistWrapper = new BlocklistWrapper(); -const commandControl = new CommandControl(); -const userOperation = new UserOperation(); -const dnsQuestionBlock = new DNSQuestionBlock(); -const dnsResolver = new DNSResolver(); -const dnsResponseBlock = new DNSResponseBlock(); -const dnsCacheHandler = new DNSCacheResponse(); -//dns cache used accross 3plugins so passed as parameter -const dnsCache = new DnsCache(10000); +const services = {}; + +((main) => { + system.sub("ready", systemReady); +})(); + +async function systemReady() { + if (services.ready) return; + + log.i("plugin.js: systemReady"); + + services.blocklistWrapper = new BlocklistWrapper(); + services.commandControl = new CommandControl(); + services.userOperation = new UserOperation(); + services.dnsQuestionBlock = new DNSQuestionBlock(); + services.dnsResolver = new DNSResolver(); + services.dnsResponseBlock = new DNSResponseBlock(); + services.dnsCacheHandler = new DNSCacheResponse(); + services.dnsCache = new DnsCache(dnsutil.cacheSize()); + + if (envutil.isNode()) { + const blocklists = await import("./node/blocklists.js"); + await blocklists.setup(services.blocklistWrapper); + } + + system.pub("go"); + + services.ready = true; +} export default class RethinkPlugin { /** @@ -40,23 +62,23 @@ export default class RethinkPlugin { this.parameter = new Map(envManager.getMap()); this.registerParameter("request", event.request); this.registerParameter("event", event); - this.registerParameter("dnsQuestionBlock", dnsQuestionBlock); - this.registerParameter("dnsResponseBlock", dnsResponseBlock); - this.registerParameter("dnsCache", dnsCache); + this.registerParameter("dnsQuestionBlock", services.dnsQuestionBlock); + this.registerParameter("dnsResponseBlock", services.dnsResponseBlock); + this.registerParameter("dnsCache", services.dnsCache); this.plugin = []; this.registerPlugin( "userOperation", - userOperation, + services.userOperation, ["dnsResolverUrl", "request", "isDnsMsg"], - userOperationCallBack, - false, + this.userOperationCallBack, + false ); this.registerPlugin( "AggressiveCaching", - dnsCacheHandler, + services.dnsCacheHandler, [ "userBlocklistInfo", "request", @@ -66,13 +88,13 @@ export default class RethinkPlugin { "dnsQuestionBlock", "dnsResponseBlock", ], - dnsAggCacheCallBack, - false, + this.dnsAggCacheCallBack, + false ); this.registerPlugin( "blocklistFilter", - blocklistWrapper, + services.blocklistWrapper, [ "blocklistUrl", "latestTimestamp", @@ -81,20 +103,20 @@ export default class RethinkPlugin { "tdNodecount", "fetchTimeout", ], - blocklistFilterCallBack, - false, + this.blocklistFilterCallBack, + false ); this.registerPlugin( "commandControl", - commandControl, + services.commandControl, ["request", "blocklistFilter", "latestTimestamp", "isDnsMsg"], - commandControlCallBack, - false, + this.commandControlCallBack, + false ); this.registerPlugin( "dnsQuestionBlock", - dnsQuestionBlock, + services.dnsQuestionBlock, [ "requestDecodedDnsPacket", "blocklistFilter", @@ -103,12 +125,12 @@ export default class RethinkPlugin { "request", "dnsCache", ], - dnsQuestionBlockCallBack, - false, + this.dnsQuestionBlockCallBack, + false ); this.registerPlugin( "dnsResolver", - dnsResolver, + services.dnsResolver, [ "requestBodyBuffer", "request", @@ -118,12 +140,12 @@ export default class RethinkPlugin { "blocklistFilter", "dnsCache", ], - dnsResolverCallBack, - false, + this.dnsResolverCallBack, + false ); this.registerPlugin( "DNSResponseBlock", - dnsResponseBlock, + services.dnsResponseBlock, [ "userBlocklistInfo", "blocklistFilter", @@ -133,8 +155,8 @@ export default class RethinkPlugin { "request", "dnsCache", ], - dnsResponseBlockCallBack, - false, + this.dnsResponseBlockCallBack, + false ); } @@ -147,7 +169,7 @@ export default class RethinkPlugin { module, parameter, callBack, - continueOnStopProcess, + continueOnStopProcess ) { this.plugin.push({ name: pluginName, @@ -169,7 +191,7 @@ export default class RethinkPlugin { log.lapTime(t, p.name, "send-req"); const res = await p.module.RethinkModule( - generateParam(this.parameter, p.param), + generateParam(this.parameter, p.param) ); log.lapTime(t, p.name, "got-res"); @@ -182,132 +204,137 @@ export default class RethinkPlugin { } log.endTime(t); } -} -/** - * Adds "blocklistFilter" to RethinkPlugin params - * @param {*} response - Contains `data` which is `blocklistFilter` - * @param {*} currentRequest - */ -function blocklistFilterCallBack(response, currentRequest) { - log.d("In blocklistFilterCallBack"); + /** + * Adds "blocklistFilter" to RethinkPlugin params + * @param {*} response - Contains `data` which is `blocklistFilter` + * @param {*} currentRequest + */ + blocklistFilterCallBack(response, currentRequest) { + log.d("In blocklistFilterCallBack"); - if (response.isException) { - loadException(response, currentRequest); - } else { - this.registerParameter("blocklistFilter", response.data.blocklistFilter); + if (response.isException) { + loadException(response, currentRequest); + } else { + this.registerParameter("blocklistFilter", response.data.blocklistFilter); + } } -} -/** - * params - * @param {*} response - * @param {*} currentRequest - */ -async function commandControlCallBack(response, currentRequest) { - log.d("In commandControlCallBack", JSON.stringify(response.data)); + /** + * params + * @param {*} response + * @param {*} currentRequest + */ + async commandControlCallBack(response, currentRequest) { + log.d("In commandControlCallBack"); - if (response.data.stopProcessing) { - currentRequest.httpResponse = response.data.httpResponse; - currentRequest.stopProcessing = true; + if (response.data.stopProcessing) { + currentRequest.httpResponse = response.data.httpResponse; + currentRequest.stopProcessing = true; + } } -} -/** - * Adds "userBlocklistInfo" and "dnsResolverUrl" to RethinkPlugin params - * @param {*} response - Contains `data` which is `userBlocklistInfo` - * @param {*} currentRequest - */ -async function userOperationCallBack(response, currentRequest) { - log.d("In userOperationCallBack", JSON.stringify(response.data)); - - if (response.isException) { - loadException(response, currentRequest); - } else { - this.registerParameter( - "userBlocklistInfo", - response.data.userBlocklistInfo, - ); - this.registerParameter( - "dnsResolverUrl", - response.data.dnsResolverUrl, - ); - } -} + /** + * Adds "userBlocklistInfo" and "dnsResolverUrl" to RethinkPlugin params + * @param {*} response - Contains `data` which is `userBlocklistInfo` + * @param {*} currentRequest + */ + async userOperationCallBack(response, currentRequest) { + log.d("In userOperationCallBack"); -function dnsAggCacheCallBack(response, currentRequest) { - log.d("In dnsAggCacheCallBack", JSON.stringify(response.data)); - - if (response.isException) { - loadException(response, currentRequest); - } else if (response.data && response.data.isBlocked) { - currentRequest.isDnsBlock = response.data.isBlocked; - currentRequest.blockedB64Flag = response.data.blockedB64Flag; - currentRequest.stopProcessing = true; - currentRequest.dnsBlockResponse(); - } else if (response.data && response.data.dnsBuffer) { - this.registerParameter("responseDecodedDnsPacket", response.data.dnsPacket); - currentRequest.dnsResponse(response.data.dnsBuffer); - currentRequest.decodedDnsPacket = response.data.dnsPacket; - currentRequest.stopProcessing = true; + if (response.isException) { + loadException(response, currentRequest); + } else { + this.registerParameter( + "userBlocklistInfo", + response.data.userBlocklistInfo + ); + this.registerParameter("dnsResolverUrl", response.data.dnsResolverUrl); + } } -} -function dnsQuestionBlockCallBack(response, currentRequest) { - log.d("In dnsQuestionBlockCallBack", JSON.stringify(response.data)); + dnsAggCacheCallBack(response, currentRequest) { + log.d("In dnsAggCacheCallBack"); - if (response.isException) { - loadException(response, currentRequest); - } else if (response.data) { - currentRequest.isDnsBlock = response.data.isBlocked; - currentRequest.blockedB64Flag = response.data.blockedB64Flag; - if (currentRequest.isDnsBlock) { + if (response.isException) { + loadException(response, currentRequest); + } else if (response.data && response.data.isBlocked) { + currentRequest.isDnsBlock = response.data.isBlocked; + currentRequest.blockedB64Flag = response.data.blockedB64Flag; currentRequest.stopProcessing = true; currentRequest.dnsBlockResponse(); + } else if (response.data && response.data.dnsBuffer) { + this.registerParameter( + "responseDecodedDnsPacket", + response.data.dnsPacket + ); + currentRequest.dnsResponse(response.data.dnsBuffer); + currentRequest.decodedDnsPacket = response.data.dnsPacket; + currentRequest.stopProcessing = true; } } -} -/** - * Adds "responseBodyBuffer" (arrayBuffer of dns response from upstream - * resolver) to RethinkPlugin params - * @param {*} response - * @param {*} currentRequest - */ -function dnsResolverCallBack(response, currentRequest) { - log.d("In dnsResolverCallBack", JSON.stringify(response.data)); - if (response.isException) { - loadException(response, currentRequest); - } else { - this.registerParameter("responseBodyBuffer", response.data.dnsBuffer); + dnsQuestionBlockCallBack(response, currentRequest) { + log.d("In dnsQuestionBlockCallBack"); + + if (response.isException) { + loadException(response, currentRequest); + } else if (response.data) { + currentRequest.isDnsBlock = response.data.isBlocked; + currentRequest.blockedB64Flag = response.data.blockedB64Flag; + if (currentRequest.isDnsBlock) { + currentRequest.stopProcessing = true; + currentRequest.dnsBlockResponse(); + } + } + } + + /** + * Adds "responseBodyBuffer" (arrayBuffer of dns response from upstream + * resolver) to RethinkPlugin params + * @param {*} response + * @param {*} currentRequest + */ + dnsResolverCallBack(response, currentRequest) { + log.d("In dnsResolverCallBack", JSON.stringify(response.data)); - this.registerParameter("responseDecodedDnsPacket", response.data.dnsPacket); + if (response.isException) { + loadException(response, currentRequest); + } else { + this.registerParameter("responseBodyBuffer", response.data.dnsBuffer); + + this.registerParameter( + "responseDecodedDnsPacket", + response.data.dnsPacket + ); + } } -} -/** - * Adds "dnsCnameBlockResponse" to RethinkPlugin params - * @param {*} response - - * @param {*} currentRequest - */ -function dnsResponseBlockCallBack(response, currentRequest) { - log.d("In dnsResponseBlockCallBack", JSON.stringify(response.data)); - - if (response.isException) { - loadException(response, currentRequest); - } else if (response.data && response.data.isBlocked) { - currentRequest.isDnsBlock = response.data.isBlocked; - currentRequest.blockedB64Flag = response.data.blockedB64Flag !== "" - ? response.data.blockedB64Flag - : currentRequest.blockedB64Flag; - currentRequest.stopProcessing = true; - currentRequest.dnsBlockResponse(); - } else { - currentRequest.dnsResponse(this.parameter.get("responseBodyBuffer")); - currentRequest.decodedDnsPacket = this.parameter.get( - "responseDecodedDnsPacket", - ); - currentRequest.stopProcessing = true; + /** + * Adds "dnsCnameBlockResponse" to RethinkPlugin params + * @param {*} response - + * @param {*} currentRequest + */ + dnsResponseBlockCallBack(response, currentRequest) { + log.d("In dnsResponseBlockCallBack"); + + if (response.isException) { + loadException(response, currentRequest); + } else if (response.data && response.data.isBlocked) { + currentRequest.isDnsBlock = response.data.isBlocked; + currentRequest.blockedB64Flag = + response.data.blockedB64Flag !== "" + ? response.data.blockedB64Flag + : currentRequest.blockedB64Flag; + currentRequest.stopProcessing = true; + currentRequest.dnsBlockResponse(); + } else { + currentRequest.dnsResponse(this.parameter.get("responseBodyBuffer")); + currentRequest.decodedDnsPacket = this.parameter.get( + "responseDecodedDnsPacket" + ); + currentRequest.stopProcessing = true; + } } } @@ -336,8 +363,8 @@ function generateParam(parameter, list) { } async function setRequest(parameter, currentRequest) { - let request = parameter.get("request"); - parameter.set("isDnsMsg", util.isDnsMsg(request)) + const request = parameter.get("request"); + parameter.set("isDnsMsg", util.isDnsMsg(request)); const isDnsMsg = parameter.get("isDnsMsg"); if (!isValidRequest(isDnsMsg, request)) { @@ -349,7 +376,7 @@ async function setRequest(parameter, currentRequest) { return; } - let buf = await getBodyBuffer(request); + const buf = await getBodyBuffer(request); parameter.set("requestBodyBuffer", buf); parameter.set("requestDecodedDnsPacket", dnsutil.decode(buf)); currentRequest.decodedDnsPacket = parameter.get("requestDecodedDnsPacket"); @@ -357,9 +384,9 @@ async function setRequest(parameter, currentRequest) { async function getBodyBuffer(request) { if (request.method.toUpperCase() === "GET") { - const QueryString = (new URL(request.url)).searchParams; + const QueryString = new URL(request.url).searchParams; return base64ToArrayBuffer( - decodeURI(QueryString.get("dns")).replace(/-/g, "+").replace(/_/g, "/"), + decodeURI(QueryString.get("dns")).replace(/-/g, "+").replace(/_/g, "/") ); } else { return await request.arrayBuffer(); diff --git a/src/helpers/util.js b/src/helpers/util.js index 02c5e0eed4..12cd49a8e8 100644 --- a/src/helpers/util.js +++ b/src/helpers/util.js @@ -11,7 +11,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { Buffer } from "buffer" +import { Buffer } from "buffer"; /** * Encodes a number to an Uint8Array of length `n` in Big Endian byte order. @@ -23,40 +23,40 @@ import { Buffer } from "buffer" export function encodeUint8ArrayBE(n, len) { const o = n; - if (!n) return new Uint8Array(len) + if (!n) return new Uint8Array(len); const a = []; - a.unshift(n & 255) + a.unshift(n & 255); while (n >= 256) { n = n >>> 8; - a.unshift(n & 255) + a.unshift(n & 255); } if (a.length > len) { - throw new RangeError(`Cannot encode ${o} in ${len} len Uint8Array`) + throw new RangeError(`Cannot encode ${o} in ${len} len Uint8Array`); } - let fill = len - a.length - while (fill--) a.unshift(0) + let fill = len - a.length; + while (fill--) a.unshift(0); - return new Uint8Array(a) + return new Uint8Array(a); } export function fromBrowser(ua) { - return ua && ua.startsWith("Mozilla/5.0") + return ua && ua.startsWith("Mozilla/5.0"); } export function jsonHeaders() { return { "Content-Type": "application/json", - } + }; } export function dnsHeaders() { return { "Accept": "application/dns-message", "Content-Type": "application/dns-message", - } + }; } export function corsHeaders() { @@ -73,14 +73,11 @@ export function corsHeaders() { */ export function corsHeadersIfNeeded(ua) { // allow cors when user agents claiming to be browsers - return (fromBrowser(ua)) ? corsHeaders() : {} + return fromBrowser(ua) ? corsHeaders() : {}; } export function browserHeaders() { - return Object.assign( - jsonHeaders(), - corsHeaders(), - ); + return Object.assign(jsonHeaders(), corsHeaders()); } /** @@ -88,19 +85,16 @@ export function browserHeaders() { * @return {Object} - Headers */ export function dohHeaders(ua) { - return Object.assign( - dnsHeaders(), - corsHeadersIfNeeded(ua), - ) + return Object.assign(dnsHeaders(), corsHeadersIfNeeded(ua)); } export function contentLengthHeader(b) { - const len = (!b || !b.byteLength) ? "0" : b.byteLength.toString() - return { "Content-Length" : len } + const len = !b || !b.byteLength ? "0" : b.byteLength.toString(); + return { "Content-Length": len }; } -export function concatHeaders() { - return Object.assign(...arguments) +export function concatHeaders(...args) { + return Object.assign(...args); } /** @@ -108,14 +102,14 @@ export function concatHeaders() { * @return {Object} - Headers */ export function copyHeaders(request) { - const headers = {} - if (!request || !request.headers) return headers + const headers = {}; + if (!request || !request.headers) return headers; // Object.assign, Object spread, etc don't work request.headers.forEach((val, name) => { - headers[name] = val - }) - return headers + headers[name] = val; + }); + return headers; } export function copyNonPseudoHeaders(req) { const headers = {}; @@ -136,37 +130,37 @@ export function copyNonPseudoHeaders(req) { */ export function sleep(ms) { return new Promise((resolve) => { - setTimeout(resolve, ms) + setTimeout(resolve, ms); }); } export function objOf(map) { - return map.entries ? Object.fromEntries(map) : false + return map.entries ? Object.fromEntries(map) : false; } // stackoverflow.com/a/31394257 export function arrayBufferOf(buf) { - if (!buf) return null + if (!buf) return null; - const offset = buf.byteOffset - const len = buf.byteLength - return buf.buffer.slice(offset, offset + len) + const offset = buf.byteOffset; + const len = buf.byteLength; + return buf.buffer.slice(offset, offset + len); } // stackoverflow.com/a/17064149 export function bufferOf(arrayBuf) { - if (!arrayBuf) return null + if (!arrayBuf) return null; - return Buffer.from(new Uint8Array(arrayBuf)) + return Buffer.from(new Uint8Array(arrayBuf)); } export function recycleBuffer(b) { - b.fill(0) - return 0 + b.fill(0); + return 0; } export function createBuffer(size) { - return Buffer.allocUnsafe(size) + return Buffer.allocUnsafe(size); } export function timedOp(op, ms, cleanup) { @@ -207,16 +201,41 @@ export function timedOp(op, ms, cleanup) { } export function timeout(ms, callback) { - return setTimeout(callback, ms) + if (typeof callback !== "function") return -1; + return setTimeout(callback, ms); } // stackoverflow.com/a/8084248 export function uid() { // ex: ".ww8ja208it" - return (Math.random() + 1).toString(36).slice(1) + return (Math.random() + 1).toString(36).slice(1); +} + +// queues fn in a macro-task queue of the event-loop +// exec order: github.com/nodejs/node/issues/22257 +export function taskBox(fn) { + timeout(/* with 0ms delay*/ 0, () => safeBox(fn)); +} + +// queues fn in a micro-task queue +// ref: MDN: Web/API/HTML_DOM_API/Microtask_guide/In_depth +// queue-task polyfill: stackoverflow.com/a/61605098 +export function microtaskBox(...fns) { + let enqueue = null; + if (typeof queueMicroTask === "function") { + enqueue = queueMicroTask; + } else { + const p = Promise.resolve(); + enqueue = p.then.bind(p); + } + + for (const f of fns) { + enqueue(() => safeBox(f)); + } } export function safeBox(fn, defaultResponse = null) { + if (typeof fn !== "function") return defaultResponse; try { return fn(); } catch (ignore) {} @@ -228,8 +247,10 @@ export function safeBox(fn, defaultResponse = null) { * @return {Boolean} */ export function isDnsMsg(req) { - return req.headers.get("Accept") === "application/dns-message" || + return ( + req.headers.get("Accept") === "application/dns-message" || req.headers.get("Content-Type") === "application/dns-message" + ); } export function emptyResponse() { @@ -255,6 +276,19 @@ export function mapOf(obj) { } export function emptyString(str) { - return !str || str.length === 0 + return !str || str.length === 0; } +export function respond204() { + return new Response(null, { + status: 204, // no content + headers: corsHeaders(), + }); +} + +export function respond503() { + return new Response(null, { + status: 503, // unavailable + headers: dnsHeaders(), + }); +} diff --git a/src/helpers/workers/config.js b/src/helpers/workers/config.js new file mode 100644 index 0000000000..d3896d60dd --- /dev/null +++ b/src/helpers/workers/config.js @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 RethinkDNS and its authors. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import EnvManager from "../env.js"; +import * as system from "../../system.js"; +import Log from "../log.js"; + +((main) => { + // if we're executing this file, we're on workers + globalThis.RUNTIME = "worker"; + + const isProd = globalThis.WORKER_ENV === "production"; + + if (!globalThis.envManager) { + globalThis.envManager = new EnvManager(); + } + + globalThis.log = new Log( + env.logLevel, + isProd // set console level only in prod. + ); + + system.pub("ready"); +})(); diff --git a/src/index.js b/src/index.js index ce343ac5d0..442e82e439 100644 --- a/src/index.js +++ b/src/index.js @@ -12,33 +12,9 @@ import EnvManager from "./helpers/env.js"; import Log from "./helpers/log.js"; import * as util from "./helpers/util.js"; import * as dnsutil from "./helpers/dnsutil.js"; - -if (!globalThis.envManager) globalThis.envManager = new EnvManager(); - -if (typeof addEventListener !== "undefined") { - addEventListener("fetch", (event) => { - event.respondWith(handleRequest(event)); - }); -} - -function initEnvIfNeeded() { - if (!envManager.isLoaded) { - envManager.loadEnv(); - } - - if (!globalThis.log) { - globalThis.log = new Log( - env.logLevel, - - // set console level only in production - !console.level && env.runTimeEnv === "production" - ); - } -} +import * as system from "./system.js"; export function handleRequest(event) { - initEnvIfNeeded(); - return Promise.race([ new Promise((accept, _) => { accept(proxyRequest(event)); @@ -55,7 +31,7 @@ export function handleRequest(event) { async function proxyRequest(event) { try { - if (optionsRequest(event.request)) return respond204(); + if (optionsRequest(event.request)) return util.respond204(); const currentRequest = new CurrentRequest(); const plugin = new RethinkPlugin(event); @@ -75,13 +51,6 @@ function optionsRequest(request) { return request.method === "OPTIONS"; } -function respond204() { - return new Response(null, { - status: 204, // no content - headers: util.corsHeaders(), - }); -} - function errorOrServfail(request, err) { const UA = request.headers.get("User-Agent"); if (!util.fromBrowser(UA)) return servfail(); @@ -94,11 +63,5 @@ function errorOrServfail(request, err) { } function servfail() { - return new Response( - dnsutil.servfail(), // null response - { - status: 503, // unavailable - headers: util.dnsHeaders(), - } - ); + return util.respond503(); } diff --git a/src/http.ts b/src/server-deno.ts similarity index 51% rename from src/http.ts rename to src/server-deno.ts index 01b5dec899..a4a5195f56 100644 --- a/src/http.ts +++ b/src/server-deno.ts @@ -2,29 +2,35 @@ // other modules. import "./helpers/deno/config.ts"; import { handleRequest } from "./index.js"; +import * as system from "./system.js"; -const { TERMINATE_TLS, TLS_CRT_PATH, TLS_KEY_PATH } = Deno.env.toObject(); -const HTTP_PORT = 8080; +((main) => { + system.sub("go", systemUp); +})(); -const l = (TERMINATE_TLS == "true") - ? Deno.listenTls({ - port: HTTP_PORT, - certFile: TLS_CRT_PATH, - keyFile: TLS_KEY_PATH, - }) - : Deno.listen({ - port: HTTP_PORT, - }); -console.log( - `Running HTTP webserver at: http://${(l.addr as Deno.NetAddr).hostname}:${ - (l.addr as Deno.NetAddr).port - }/`, -); +async function systemUp() { + const { TERMINATE_TLS, TLS_CRT_PATH, TLS_KEY_PATH } = Deno.env.toObject(); + const HTTP_PORT = 8080; -// Connections to the listener will be yielded up as an async iterable. -for await (const conn of l) { - // To not be blocking, handle each connection without awaiting - handleHttp(conn); + const l = (TERMINATE_TLS == "true") + ? Deno.listenTls({ + port: HTTP_PORT, + certFile: TLS_CRT_PATH, + keyFile: TLS_KEY_PATH, + }) + : Deno.listen({ + port: HTTP_PORT, + }); + console.log(`deno up at: http://${(l.addr as Deno.NetAddr).hostname}:${ + (l.addr as Deno.NetAddr).port + }/`, + ); + + // Connections to the listener will be yielded up as an async iterable. + for await (const conn of l) { + // To not be blocking, handle each connection without awaiting + handleHttp(conn); + } } async function handleHttp(conn: Deno.Conn) { diff --git a/src/server.js b/src/server-node.js similarity index 97% rename from src/server.js rename to src/server-node.js index 26d00877e8..bad1bdfbf6 100644 --- a/src/server.js +++ b/src/server-node.js @@ -11,12 +11,12 @@ import net, { isIPv6, Socket } from "net"; import tls, { TLSSocket } from "tls"; import http2, { Http2ServerRequest, Http2ServerResponse } from "http2"; import { V1ProxyProtocol } from "proxy-protocol-js"; - +import * as system from "./system.js"; import { handleRequest } from "./index.js"; import * as dnsutil from "./helpers/dnsutil.js"; import * as util from "./helpers/util.js"; import { copyNonPseudoHeaders } from "./helpers/node/util.js"; -import { TLS_CRT, TLS_KEY } from "./helpers/node/config.js"; +import "./helpers/node/config.js"; /* eslint-enable no-unused-vars */ // Ports which the services are exposed on. Corresponds to fly.toml ports. @@ -31,18 +31,21 @@ const DOT_PORT = DOT_IS_PROXY_PROTO : DOT_ENTRY_PORT; const DOH_PORT = DOH_ENTRY_PORT; -const tlsOptions = { - key: TLS_KEY, - cert: TLS_CRT, -}; - let OUR_RG_DN_RE = null; // regular dns name match let OUR_WC_DN_RE = null; // wildcard dns name match -// main -((_) => { +((main) => { + system.sub("go", systemUp); +})(); + +function systemUp() { + const tlsOpts = { + key: env.tlsKey, + cert: env.tlsCrt, + }; + const dot1 = tls - .createServer(tlsOptions, serveTLS) + .createServer(tlsOpts, serveTLS) .listen(DOT_PORT, () => up("DoT", dot1.address())); const dot2 = @@ -52,13 +55,13 @@ let OUR_WC_DN_RE = null; // wildcard dns name match .listen(DOT_PROXY_PORT, () => up("DoT ProxyProto", dot2.address())); const doh = http2 - .createSecureServer({ ...tlsOptions, allowHTTP1: true }, serveHTTPS) + .createSecureServer({ ...tlsOpts, allowHTTP1: true }, serveHTTPS) .listen(DOH_PORT, () => up("DoH", doh.address())); function up(server, addr) { log.i(server, `listening on: [${addr.address}]:${addr.port}`); } -})(); +} function close(sock) { util.safeBox(() => sock.destroy()); diff --git a/src/server-workers.js b/src/server-workers.js new file mode 100644 index 0000000000..6a067a313c --- /dev/null +++ b/src/server-workers.js @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 RethinkDNS and its authors. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import "./helpers/workers/config.js"; +import { handleRequest } from "./index.js"; +import * as system from "./system.js"; +import * as util from "./helpers/util.js"; + +let up = false; + +((main) => { + if (typeof addEventListener === "undefined") { + throw new Error("workers env missing addEventListener"); + } + + system.sub("go", systemUp); + + addEventListener("fetch", serveDoh); +})(); + +function systemUp() { + up = true; +} + +function serveDoh(event) { + if (!up) { + event.respondWith(util.respond503()); + return; + } + + event.respondWith(handleRequest(event)); +} diff --git a/src/system.js b/src/system.js new file mode 100644 index 0000000000..eac2e46541 --- /dev/null +++ b/src/system.js @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 RethinkDNS and its authors. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import * as util from "./helpers/util.js"; + +// once emitted, they stick; firing off new listeners forever, just the once. +const stickyEvents = new Set([ + // when env setup is done + "ready", + // when plugin setup is done + "go", +]); + +const events = new Set(); + +const listeners = new Map(); + +(() => { + for (const e of events) { + listeners.set(e, new Set()); + } + + for (const se of stickyEvents) { + listeners.set(se, new Set()); + } +})(); + +export function pub(event) { + const eventCallbacks = listeners.get(event); + + if (!eventCallbacks) return; + + // listeners valid just the once for stickyEvents + if (stickyEvents.has(event)) { + listeners.delete(event); + } + + // callbacks are queued async and don't block the caller + util.microtaskBox(...eventCallbacks); +} + +export function sub(event, cb) { + const callbacks = listeners.get(event); + + if (!callbacks) { + // if event is sticky, fire off the listener at once + if (stickyEvents.has(event)) { + util.microtaskBox(cb); + return true; + } + return false; + } + + callbacks.add(cb); + + return true; +} diff --git a/webpack.config.cjs b/webpack.config.cjs index e2e17332f4..f69e008b79 100644 --- a/webpack.config.cjs +++ b/webpack.config.cjs @@ -1,10 +1,11 @@ const webpack = require("webpack"); module.exports = { + entry: "./src/server-workers.js", target: "webworker", plugins: [ new webpack.IgnorePlugin({ resourceRegExp: - /(^dgram$)|(^http2$)|(\/node\/.*\.js$)|(.*_node\.js$)|(.*\.node\.js$)/, + /(^dgram$)|(^http2$)|(\/node\/.*\.js$)|(.*-node\.js$)|(.*\.node\.js$)/, }), ], optimization: {