From 64b37bf6616ad85b23a796039d7f1e2f0eedde89 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 10 Nov 2022 21:24:17 -0800 Subject: [PATCH 1/3] add CI badge to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fd53a9b..33b3c03 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # monaco-tm +![TypeScript build](https://github.com/bolinfest/monaco-tm/actions/workflows/verify-build.yml/badge.svg) + This gets TextMate grammars working in standalone Monaco by leveraging `vscode-oniguruma` and `vscode-textmate`. For more context, see: https://github.com/microsoft/monaco-editor/issues/1915. From 356963aab0a4f19debb5b41812a4415c0b97babe Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 10 Nov 2022 21:24:24 -0800 Subject: [PATCH 2/3] update vscode-oniguruma and vscode-textmate deps --- package.json | 4 ++-- yarn.lock | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 532a50e..60f7d0d 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,8 @@ }, "dependencies": { "monaco-editor": "0.21.2", - "vscode-oniguruma": "1.3.0", - "vscode-textmate": "5.1.1" + "vscode-oniguruma": "^1.6.1", + "vscode-textmate": "^6.0.0" }, "devDependencies": { "css-loader": "3.5.3", diff --git a/yarn.lock b/yarn.lock index 3eafd6f..cce3e81 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4085,15 +4085,15 @@ vm-browserify@^1.0.1: resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0" integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ== -vscode-oniguruma@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.3.0.tgz#6788a9db2f8b0781243b4eb8c7a1dd25f6c0e2c8" - integrity sha512-m4Br19v6XD4MRbVrgsLNSZgQrBzk1BCMCleL8+GrcoGxKEJJd62zOFcTaoQR3hCrSlLqoxWmJ7Cc0VieVV3iTQ== +vscode-oniguruma@^1.6.1: + version "1.6.2" + resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.2.tgz#aeb9771a2f1dbfc9083c8a7fdd9cccaa3f386607" + integrity sha512-KH8+KKov5eS/9WhofZR8M8dMHWN2gTxjMsG4jd04YhpbPR91fUj7rYQ2/XjeHCJWbg7X++ApRIU9NUwM2vTvLA== -vscode-textmate@5.1.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.1.1.tgz#d88dbf271bee7cede455a21bd4894ba5724a4a7e" - integrity sha512-5VHjF+Fglf9d2JI5OyQ7FHutK6/29G0qYyD920K0SWO7uY8JTWbqyKAHEtfB/ZDk2fOe/E23n3wz9fHXKi63yg== +vscode-textmate@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210" + integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ== wasm-dce@^1.0.0: version "1.0.2" From 45d4563410623920507e2466cdd8e21c0186f386 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Fri, 11 Nov 2022 09:32:31 -0800 Subject: [PATCH 3/3] impose timeout when running tokenizeLine2() --- src/providers.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/providers.ts b/src/providers.ts index e59ed77..2070ab6 100644 --- a/src/providers.ts +++ b/src/providers.ts @@ -152,6 +152,13 @@ export class SimpleLanguageInfoProvider { } } +/** + * Specify a timeout when tokenizing a line to prevent a long line from locking + * up the main thread. Note this is used in VS Code: + * https://github.com/microsoft/vscode/blob/504c5a768a001b2099dd2b44e9dc39e10ccdfb56/src/vs/workbench/services/textMate/common/TMTokenization.ts#L39 + */ +const TOKENIZE_TIMEOUT_MS = 500; + class TokensProviderCache { private scopeNameToGrammar: Map> = new Map(); @@ -172,7 +179,19 @@ class TokensProviderCache { line: string, state: monaco.languages.IState, ): monaco.languages.IEncodedLineTokens { - const tokenizeLineResult2 = grammar.tokenizeLine2(line, state as StackElement); + const tokenizeLineResult2 = grammar.tokenizeLine2( + line, + state as StackElement, + TOKENIZE_TIMEOUT_MS, + ); + + // This is the strategy used by VS Code: + if (tokenizeLineResult2.stoppedEarly) { + console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`); + // return the state at the beginning of the line + return {tokens: tokenizeLineResult2.tokens, endState: state}; + } + const {tokens, ruleStack: endState} = tokenizeLineResult2; return {tokens, endState}; },