From 2a5ca851cf9fd48e5b18eb80ef8b7532ba19b71f Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:52:00 +0800 Subject: [PATCH 1/9] fix: add the missing `version` key in `GM.info` --- src/ext/content-scripts/entry-userscripts.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ext/content-scripts/entry-userscripts.js b/src/ext/content-scripts/entry-userscripts.js index eab82138..df16d1d6 100644 --- a/src/ext/content-scripts/entry-userscripts.js +++ b/src/ext/content-scripts/entry-userscripts.js @@ -164,6 +164,7 @@ async function injection() { scriptHandler: data.scriptHandler, scriptHandlerVersion: data.scriptHandlerVersion, scriptMetaStr: userscript.scriptMetaStr, + version: data.scriptHandlerVersion, }; // add GM_info userscript.apis.GM_info = userscript.apis.GM.info; From b307c420b24df503fdd7a9f8795882de718e4855 Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:44:57 +0800 Subject: [PATCH 2/9] feat: add `document` response type support in xhr --- src/ext/background/main.js | 4 +++- src/ext/content-scripts/api.js | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/ext/background/main.js b/src/ext/background/main.js index 8f716472..cfe32313 100644 --- a/src/ext/background/main.js +++ b/src/ext/background/main.js @@ -466,7 +466,9 @@ async function handleMessage(message, sender) { }; } xhr.open(method, details.url, true, user, password); - xhr.responseType = details.responseType || ""; + xhr.responseType = details.responseType; + // transfer to content script via text and then parse to document + if (xhr.responseType === "document") xhr.responseType = "text"; if (details.headers) { for (const key in details.headers) { if (!key.startsWith("Proxy-") && !key.startsWith("Sec-")) { diff --git a/src/ext/content-scripts/api.js b/src/ext/content-scripts/api.js index 6865b7d9..a827d263 100644 --- a/src/ext/content-scripts/api.js +++ b/src/ext/content-scripts/api.js @@ -150,13 +150,28 @@ function xhr(details) { } catch (err) { console.error("error parsing xhr arraybuffer", err); } - } else if (r.responseType === "blob" && r.response.data) { + } + if (r.responseType === "blob" && r.response.data) { // blob responses had their data converted in background // convert it back to blob const resp = await fetch(r.response.data); const b = await resp.blob(); r.response = b; } + if (r.responseType === "document") { + // document responses had their data converted in background + // convert it back to blob + try { + const parser = new DOMParser(); + const mimeType = r.contentType.includes("text/html") + ? "text/html" + : "text/xml"; + r.response = parser.parseFromString(r.response, mimeType); + r.responseXML = r.response; + } catch (err) { + console.error("error parsing xhr document", err); + } + } } // call userscript method details[msg.name](msg.response); From 51c03a7104030ef08cba52c976b2a5139e5b56d6 Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Wed, 4 Sep 2024 17:56:43 +0800 Subject: [PATCH 3/9] feat: add `contentType` key in xhr response object For convenience, and is also required when decode the response in content scripts side. --- src/ext/background/main.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ext/background/main.js b/src/ext/background/main.js index cfe32313..b2b5d9ba 100644 --- a/src/ext/background/main.js +++ b/src/ext/background/main.js @@ -432,6 +432,7 @@ async function handleMessage(message, sender) { // can not send xhr through postMessage // construct new object to be sent as "response" const x = { + contentType: undefined, // non-standard readyState: xhr.readyState, response: xhr.response, responseHeaders: xhr.getAllResponseHeaders(), @@ -442,6 +443,10 @@ async function handleMessage(message, sender) { timeout: xhr.timeout, withCredentials: xhr.withCredentials, }; + // get content-type when headers received + if (xhr.readyState >= xhr.HEADERS_RECEIVED) { + x.contentType = xhr.getResponseHeader("Content-Type"); + } // only include responseText when needed if (["", "text"].indexOf(xhr.responseType) !== -1) { x.responseText = xhr.responseText; From b83dfbb0330684e88a616fa40de81c842f0df8fe Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:17:10 +0800 Subject: [PATCH 4/9] perf: transfer arraybuffer instead of blob base64 Avoid the calculation amount, memory consumption and data expansion caused by encoding, decoding and fetch. --- src/ext/background/main.js | 25 ++++++------------------- src/ext/content-scripts/api.js | 17 ++++++++++------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/ext/background/main.js b/src/ext/background/main.js index b2b5d9ba..ecf849b6 100644 --- a/src/ext/background/main.js +++ b/src/ext/background/main.js @@ -21,14 +21,6 @@ function userscriptSort(a, b) { return Number(a.scriptObject.weight) < Number(b.scriptObject.weight); } -async function readAsDataURL(blob) { - return new Promise((resolve) => { - const reader = new FileReader(); - reader.readAsDataURL(blob); - reader.onloadend = () => resolve(reader.result); // base64data - }); -} - async function getPlatform() { let platform = localStorage.getItem("platform"); if (!platform) { @@ -452,19 +444,12 @@ async function handleMessage(message, sender) { x.responseText = xhr.responseText; } // only process when xhr is complete and data exist - if (xhr.readyState === 4 && xhr.response !== null) { + if (xhr.readyState === xhr.DONE && xhr.response !== null) { // need to convert arraybuffer data to postMessage if (xhr.responseType === "arraybuffer") { - const arr = Array.from(new Uint8Array(xhr.response)); - x.response = arr; - } - // need to convert blob data to postMessage - if (xhr.responseType === "blob") { - const base64data = await readAsDataURL(xhr.response); - x.response = { - data: base64data, - type: xhr.responseType, - }; + /** @type {ArrayBuffer} */ + const buffer = xhr.response; + x.response = Array.from(new Uint8Array(buffer)); } } port.postMessage({ name: e, event, response: x }); @@ -472,6 +457,8 @@ async function handleMessage(message, sender) { } xhr.open(method, details.url, true, user, password); xhr.responseType = details.responseType; + // transfer to content script via arraybuffer and then parse to blob + if (xhr.responseType === "blob") xhr.responseType = "arraybuffer"; // transfer to content script via text and then parse to document if (xhr.responseType === "document") xhr.responseType = "text"; if (details.headers) { diff --git a/src/ext/content-scripts/api.js b/src/ext/content-scripts/api.js index a827d263..65a49599 100644 --- a/src/ext/content-scripts/api.js +++ b/src/ext/content-scripts/api.js @@ -145,22 +145,25 @@ function xhr(details) { // arraybuffer responses had their data converted in background // convert it back to arraybuffer try { - const buffer = new Uint8Array(r.response).buffer; - r.response = buffer; + r.response = new Uint8Array(r.response).buffer; } catch (err) { console.error("error parsing xhr arraybuffer", err); } } - if (r.responseType === "blob" && r.response.data) { + if (r.responseType === "blob") { // blob responses had their data converted in background // convert it back to blob - const resp = await fetch(r.response.data); - const b = await resp.blob(); - r.response = b; + try { + const typedArray = new Uint8Array(r.response); + const type = r.contentType ?? ""; + r.response = new Blob([typedArray], { type }); + } catch (err) { + console.error("error parsing xhr blob", err); + } } if (r.responseType === "document") { // document responses had their data converted in background - // convert it back to blob + // convert it back to document try { const parser = new DOMParser(); const mimeType = r.contentType.includes("text/html") From e05b2e5656c6a0012fef2ad3247a5fa8457fd328 Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:24:44 +0800 Subject: [PATCH 5/9] perf: handle `responseText` on content scripts side Avoid double memory consumption and transfer costs. --- src/ext/background/main.js | 4 ---- src/ext/content-scripts/api.js | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ext/background/main.js b/src/ext/background/main.js index ecf849b6..527301b8 100644 --- a/src/ext/background/main.js +++ b/src/ext/background/main.js @@ -439,10 +439,6 @@ async function handleMessage(message, sender) { if (xhr.readyState >= xhr.HEADERS_RECEIVED) { x.contentType = xhr.getResponseHeader("Content-Type"); } - // only include responseText when needed - if (["", "text"].indexOf(xhr.responseType) !== -1) { - x.responseText = xhr.responseText; - } // only process when xhr is complete and data exist if (xhr.readyState === xhr.DONE && xhr.response !== null) { // need to convert arraybuffer data to postMessage diff --git a/src/ext/content-scripts/api.js b/src/ext/content-scripts/api.js index 65a49599..c22552a7 100644 --- a/src/ext/content-scripts/api.js +++ b/src/ext/content-scripts/api.js @@ -139,6 +139,10 @@ function xhr(details) { ) { // process xhr response const r = msg.response; + // only include responseText when needed + if (["", "text"].includes(r.responseType)) { + r.responseText = r.response; + } // only process when xhr is complete and data exist if (r.readyState === 4 && r.response !== null) { if (r.responseType === "arraybuffer") { From 1578a91132cbf39ae460bdff7bf878d9ed035a59 Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Wed, 4 Sep 2024 18:46:20 +0800 Subject: [PATCH 6/9] feat: add responseXML to match real XMLHttpRequest behavior Only add implementation, not enable, to avoid unnecessary calculations, and this legacy behavior is not recommended, users should explicitly use `responseType: "document"` to obtain it. --- src/ext/content-scripts/api.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/ext/content-scripts/api.js b/src/ext/content-scripts/api.js index c22552a7..02f8a960 100644 --- a/src/ext/content-scripts/api.js +++ b/src/ext/content-scripts/api.js @@ -143,6 +143,28 @@ function xhr(details) { if (["", "text"].includes(r.responseType)) { r.responseText = r.response; } + /** + * only include responseXML when needed + * NOTE: Only add implementation at this time, not enable, to avoid + * unnecessary calculations, and this legacy default behavior is not + * recommended, users should explicitly use `responseType: "document"` + * to obtain it. + if (r.responseType === "") { + const mimeTypes = [ + "text/xml", + "application/xml", + "application/xhtml+xml", + "image/svg+xml", + ]; + for (const mimeType of mimeTypes) { + if (r.contentType.includes(mimeType)) { + const parser = new DOMParser(); + r.responseXML = parser.parseFromString(r.response, "text/xml"); + break; + } + } + } + */ // only process when xhr is complete and data exist if (r.readyState === 4 && r.response !== null) { if (r.responseType === "arraybuffer") { From e653d406eb3be64d931d57e68233e937542ab2ef Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Wed, 4 Sep 2024 21:54:32 +0800 Subject: [PATCH 7/9] perf: replace `responseType` default value with `text` --- src/ext/background/main.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ext/background/main.js b/src/ext/background/main.js index 527301b8..c25e038d 100644 --- a/src/ext/background/main.js +++ b/src/ext/background/main.js @@ -453,6 +453,8 @@ async function handleMessage(message, sender) { } xhr.open(method, details.url, true, user, password); xhr.responseType = details.responseType; + // avoid unexpected behavior of legacy defaults such as parsing XML + if (xhr.responseType === "") xhr.responseType = "text"; // transfer to content script via arraybuffer and then parse to blob if (xhr.responseType === "blob") xhr.responseType = "arraybuffer"; // transfer to content script via text and then parse to document From 0882bb94a7ca6dba9248976b19fa9697801be249 Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Thu, 5 Sep 2024 21:19:01 +0800 Subject: [PATCH 8/9] refactor: adjust xhr code logic and sequence --- README.md | 1 - src/ext/background/main.js | 98 +++++++++++++++++----------------- src/ext/content-scripts/api.js | 12 +++-- 3 files changed, 57 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index f0b3fc85..8591027c 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,6 @@ Userscripts currently supports the following api methods. All methods are asynch - `status` - `statusText` - `timeout` - - `withCredentials` - `responseText` (when `responseType` is `text`) - returns an object with a single property, `abort`, which is a `Function` - usage: `const foo = GM.xmlHttpRequest({...});` ... `foo.abort();` to abort the request diff --git a/src/ext/background/main.js b/src/ext/background/main.js index c25e038d..844b3f7e 100644 --- a/src/ext/background/main.js +++ b/src/ext/background/main.js @@ -392,13 +392,32 @@ async function handleMessage(message, sender) { return { status: "fulfilled", result }; } case "API_XHR": { - // parse details and set up for XMLHttpRequest + // initializing an xhr instance + const xhr = new XMLHttpRequest(); + // establish a long-lived port connection to content script + const port = browser.tabs.connect(sender.tab.id, { + name: message.xhrPortName, + }); + // receive messages from content script and process them + port.onMessage.addListener((msg) => { + if (msg.name === "ABORT") xhr.abort(); + if (msg.name === "DISCONNECT") port.disconnect(); + }); + // handle port disconnect and clean tasks + port.onDisconnect.addListener((p) => { + if (p?.error) { + console.error( + `port disconnected due to an error: ${p.error.message}`, + ); + } + }); + // parse details and set up for xhr instance const details = message.details; - const method = details.method ? details.method : "GET"; + const method = details.method || "GET"; const user = details.user || null; const password = details.password || null; let body = details.data || null; - if (body != null && details.binary != null) { + if (typeof body === "string" && details.binary) { const len = body.length; const arr = new Uint8Array(len); for (let i = 0; i < len; i++) { @@ -406,17 +425,18 @@ async function handleMessage(message, sender) { } body = new Blob([arr], { type: "text/plain" }); } - // establish a long-lived port connection to content script - const port = browser.tabs.connect(sender.tab.id, { - name: message.xhrPortName, - }); - // set up XMLHttpRequest - const xhr = new XMLHttpRequest(); - xhr.withCredentials = details.user && details.password; - xhr.timeout = details.timeout || 0; - if (details.overrideMimeType) { - xhr.overrideMimeType(details.overrideMimeType); - } + + // xhr instances automatically filter out unexpected user values + xhr.timeout = details.timeout; + xhr.responseType = details.responseType; + // record parsed values for subsequent use + const responseType = xhr.responseType; + // avoid unexpected behavior of legacy defaults such as parsing XML + if (responseType === "") xhr.responseType = "text"; + // transfer to content script via arraybuffer and then parse to blob + if (responseType === "blob") xhr.responseType = "arraybuffer"; + // transfer to content script via text and then parse to document + if (responseType === "document") xhr.responseType = "text"; // add required listeners and send result back to the content script for (const e of message.events) { if (!details[e]) continue; @@ -428,12 +448,11 @@ async function handleMessage(message, sender) { readyState: xhr.readyState, response: xhr.response, responseHeaders: xhr.getAllResponseHeaders(), - responseType: xhr.responseType, + responseType, responseURL: xhr.responseURL, status: xhr.status, statusText: xhr.statusText, timeout: xhr.timeout, - withCredentials: xhr.withCredentials, }; // get content-type when headers received if (xhr.readyState >= xhr.HEADERS_RECEIVED) { @@ -442,8 +461,10 @@ async function handleMessage(message, sender) { // only process when xhr is complete and data exist if (xhr.readyState === xhr.DONE && xhr.response !== null) { // need to convert arraybuffer data to postMessage - if (xhr.responseType === "arraybuffer") { - /** @type {ArrayBuffer} */ + if ( + xhr.responseType === "arraybuffer" && + xhr.response instanceof ArrayBuffer + ) { const buffer = xhr.response; x.response = Array.from(new Uint8Array(buffer)); } @@ -451,36 +472,6 @@ async function handleMessage(message, sender) { port.postMessage({ name: e, event, response: x }); }; } - xhr.open(method, details.url, true, user, password); - xhr.responseType = details.responseType; - // avoid unexpected behavior of legacy defaults such as parsing XML - if (xhr.responseType === "") xhr.responseType = "text"; - // transfer to content script via arraybuffer and then parse to blob - if (xhr.responseType === "blob") xhr.responseType = "arraybuffer"; - // transfer to content script via text and then parse to document - if (xhr.responseType === "document") xhr.responseType = "text"; - if (details.headers) { - for (const key in details.headers) { - if (!key.startsWith("Proxy-") && !key.startsWith("Sec-")) { - const val = details.headers[key]; - xhr.setRequestHeader(key, val); - } - } - } - // receive messages from content script and process them - port.onMessage.addListener((msg) => { - if (msg.name === "ABORT") xhr.abort(); - if (msg.name === "DISCONNECT") port.disconnect(); - }); - // handle port disconnect and clean tasks - port.onDisconnect.addListener((p) => { - if (p?.error) { - console.error( - `port disconnected due to an error: ${p.error.message}`, - ); - } - }); - xhr.send(body); // if onloadend not set in xhr details // onloadend event won't be passed to content script // if that happens port DISCONNECT message won't be posted @@ -490,6 +481,17 @@ async function handleMessage(message, sender) { port.postMessage({ name: "onloadend", event }); }; } + if (details.overrideMimeType) { + xhr.overrideMimeType(details.overrideMimeType); + } + xhr.open(method, details.url, true, user, password); + // must set headers after `xhr.open()`, but before `xhr.send()` + if (typeof details.headers === "object") { + for (const [key, val] of Object.entries(details.headers)) { + xhr.setRequestHeader(key, val); + } + } + xhr.send(body); return { status: "fulfilled" }; } case "REFRESH_DNR_RULES": { diff --git a/src/ext/content-scripts/api.js b/src/ext/content-scripts/api.js index 02f8a960..0988c063 100644 --- a/src/ext/content-scripts/api.js +++ b/src/ext/content-scripts/api.js @@ -129,7 +129,10 @@ function xhr(details) { const response = { abort: () => console.error("xhr has not yet been initialized"), }; - // port listener, most of the messaging logic goes here + /** + * port listener, most of the messaging logic goes here + * @type {Parameters[0]} + */ const listener = (port) => { if (port.name !== xhrPortName) return; port.onMessage.addListener(async (msg) => { @@ -167,7 +170,7 @@ function xhr(details) { */ // only process when xhr is complete and data exist if (r.readyState === 4 && r.response !== null) { - if (r.responseType === "arraybuffer") { + if (r.responseType === "arraybuffer" && Array.isArray(r.response)) { // arraybuffer responses had their data converted in background // convert it back to arraybuffer try { @@ -176,7 +179,7 @@ function xhr(details) { console.error("error parsing xhr arraybuffer", err); } } - if (r.responseType === "blob") { + if (r.responseType === "blob" && Array.isArray(r.response)) { // blob responses had their data converted in background // convert it back to blob try { @@ -187,7 +190,7 @@ function xhr(details) { console.error("error parsing xhr blob", err); } } - if (r.responseType === "document") { + if (r.responseType === "document" && typeof r.response === "string") { // document responses had their data converted in background // convert it back to document try { @@ -211,7 +214,6 @@ function xhr(details) { port.postMessage({ name: "DISCONNECT" }); } }); - // handle port disconnect and clean tasks port.onDisconnect.addListener((p) => { if (p?.error) { From 1ef027ba7ce85af041a7e8c22f8ca0c604227d83 Mon Sep 17 00:00:00 2001 From: ACTCD <101378590+ACTCD@users.noreply.github.com> Date: Thu, 5 Sep 2024 21:38:19 +0800 Subject: [PATCH 9/9] perf: use `TextEncoder` instead of `charCodeAt` loop Use `TypedArray` directly without constructing a new `Blob`. --- src/ext/background/main.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/ext/background/main.js b/src/ext/background/main.js index 844b3f7e..f584ed5a 100644 --- a/src/ext/background/main.js +++ b/src/ext/background/main.js @@ -417,15 +417,11 @@ async function handleMessage(message, sender) { const user = details.user || null; const password = details.password || null; let body = details.data || null; + // deprecate once body supports more data types + // the `binary` key will no longer needed if (typeof body === "string" && details.binary) { - const len = body.length; - const arr = new Uint8Array(len); - for (let i = 0; i < len; i++) { - arr[i] = body.charCodeAt(i); - } - body = new Blob([arr], { type: "text/plain" }); + body = new TextEncoder().encode(body); } - // xhr instances automatically filter out unexpected user values xhr.timeout = details.timeout; xhr.responseType = details.responseType;