feat(proj): init highlighter (#402)

* dev(proj): chore fix

* feat(proj): init highlighter

projects/cetz-editor/index.html

@@ -6,11 +6,6 @@
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Cetz Editor</title>
-    <script>
-        window.script$jsYamlLoad = new Promise(resolve => {
-            window.script$jsYamlLoadResolve = resolve;
-        });
-    </script>
     <script id="script-main" type="module" src="/dist/esm/index.mjs"></script>
     <link rel="stylesheet" href="typst-style.css" />
     <script src="https://cdn.jsdelivr.net/npm/[email protected]/dev/vs/loader.min.js"></script>
@@ -40,19 +35,16 @@
                         <span>from</span>
                         <div class="flex-row" style="display: inline-flex; justify-content:
             space-around; gap: 5px">
-                            <select id="definition-source-selector"
-                            class="editor-code-select" style="width: 5em;">
+                            <select id="definition-source-selector" class="editor-code-select" style="width: 5em;">
                                 <option value="builtin">Builtin</option>
                                 <option value="url">Url</option>
                                 <option value="file">File</option>
                             </select>
-                            <input id="definition-source-value"
-                            class="editor-input-box" type="text"
-                            placeholder="https://..." />
-                            <button id="definition-source-confirm"
-                            class="editor-input-button">Pull</button>
-                        </div>
+                            <input id="definition-source-value" class="editor-input-box" type="text"
+                                placeholder="https://..." />
+                            <button id="definition-source-confirm" class="editor-input-button">Pull</button>
                         </div>
+                    </div>
                     <div class="editor-action-group">
                         <span>element:</span>
                         <div class="flex-row" style="display: inline-flex; justify-content:
@@ -144,14 +136,14 @@
                     language: 'typst',
                     theme: 'tokyo-night',
                     'semanticHighlighting.enabled': true,
-                     "bracketPairColorization.enabled": true,
+                    "bracketPairColorization.enabled": true,
                 });
                 const mainEditor = monaco.editor.create(document.getElementById('main-editor'), {
                     value: '',
                     language: 'typst',
                     theme: 'tokyo-night',
                     'semanticHighlighting.enabled': true,
-                     "bracketPairColorization.enabled": true,
+                    "bracketPairColorization.enabled": true,
                 });
                 $preview.bindElement(document.getElementById('preview-panel'), v => mainEditor.setValue(v));
                 document.getElementById('export-svg').addEventListener('click', () => {
@@ -288,8 +280,8 @@
   line(start, end, name: "t", mark: mark)
   content("t", inner-text)
 }`;
-                definitionEditor.setValue(builtinDefinitions);
-                mainEditor.setValue(`#{
+                    definitionEditor.setValue(builtinDefinitions);
+                    mainEditor.setValue(`#{
   make-ins("c0", (0, 0), "x-circle", (rad: 50))
   make-ins("c1", (50, 50), "x-rect", (x: 100))
   make-ins("c2", (200, 0), "x-circle", (rad: 50))
@@ -302,8 +294,8 @@
                 triggerSyncMain();
 
                 previewSelector.onchange = () => {
-                  console.log('previewSelector.value', previewSelector.value);
-                  $preview.doSelectDef(previewSelector.value);
+                    console.log('previewSelector.value', previewSelector.value);
+                    $preview.doSelectDef(previewSelector.value);
                 };
 
                 $preview.doSelectDef('');
@@ -312,7 +304,7 @@
                 definitionSourceSelector.onchange = () => {
                     if (definitionSourceSelector.value === 'builtin' || definitionSourceSelector.value === 'file') {
                         checkDefinitionSource();
-                    } 
+                    }
                 };
             });
         };

projects/cetz-editor/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@myriaddreamin/cetz-editor",
   "private": true,
-  "version": "0.4.0",
+  "version": "0.4.1-rc1",
   "description": "Simple visualized cetz editor",
   "author": "Myriad Dreamin <[email protected]>",
   "license": "Apache-2.0",
@@ -21,6 +21,9 @@
     "@myriaddreamin/typst-ts-web-compiler": "*"
   },
   "devDependencies": {
+    "@myriaddreamin/typst.ts": "*",
+    "@myriaddreamin/typst-ts-renderer": "*",
+    "@myriaddreamin/typst-ts-web-compiler": "*",
     "vite": "^4.3.9",
     "vite-plugin-singlefile": "^0.13.5",
     "vitest": "^0.32.2"

projects/cetz-editor/src/index.mts

@@ -682,9 +682,11 @@ import cetz.draw: *
       let args = '';
       const argEntries = Object.entries(ins.args ?? {});
       if (argEntries.length) {
-        args = '  args:';
         for (const [k, v] of argEntries) {
-          args += `\n    ${k}: ${v}`;
+          if (args) {
+            args += ', ';
+          }
+          args += `${k}: ${v}`;
         }
       }
       let [x, y] = ins.pos;
@@ -696,7 +698,7 @@ import cetz.draw: *
       // [${x}, ${y}]`);
       x = Math.round(x * 1000) / 1000;
       y = Math.round(y * 1000) / 1000;
-      data.push(`make-ins("${ins.name}", (${x}, ${y}), ${ins.type}, (${args}))`);
+      data.push(`make-ins("${ins.name}", (${x}, ${y}), "${ins.type}", (${args}))`);
     }
     this.updateMainContent(`#{
   ${data.join(INEW_LINE)}
@@ -971,7 +973,7 @@ import cetz.draw: *
 
         let j = skipIdent(i);
         if (i === j) {
-          throw new Error(`expected ident at ${i}`);
+          throw new Error(`expected ident at ${i} ${s.slice(i)}`);
         }
 
         const ident = s.slice(i, j);
@@ -1083,7 +1085,7 @@ import cetz.draw: *
     }
 
     function parseMainContent() {
-      // console.log('parseMainContent', s);
+      console.log('parseMainContent', s);
       const instances: any[] = [];
       let i = 0;
       i = eatToken(skipSpaces(i), '#');
@@ -1111,7 +1113,7 @@ import cetz.draw: *
         }
         j = skipIdent(i);
         if (i === j) {
-          throw new Error(`expected ident at ${i}`);
+          throw new Error(`expected ident at ${i} ${s.slice(i)}`);
         }
         let ident = s.slice(i, j);
         i = skipSpaces(j);

projects/highlighter/esbuild.config.mjs

@@ -0,0 +1,83 @@
+import esbuild from 'esbuild';
+import process from 'process';
+import builtins from 'builtin-modules';
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+const IS_COMMON_JS = process.argv[2] === 'commonjs';
+console.log(IS_COMMON_JS);
+const IS_PRODUCTION = process.argv[2] === 'production' || process.argv[3] === 'production';
+const banner = `/*
+THIS IS A GENERATED/BUNDLED FILE BY ESBUILD
+if you want to view the source, please visit the github repository https://github.com/Myriad-Dreamin/typst.ts/blob/main/packages/typst.ts
+*/
+`;
+
+let wasmPlugin = {
+  name: 'wasm',
+  setup(build) {
+    // Resolve ".wasm" files to a path with a namespace
+    build.onResolve({ filter: /\.wasm$/ }, args => {
+      if (args.resolveDir === '') {
+        return; // Ignore unresolvable paths
+      }
+      let p = args.path;
+      return {
+        path: path.isAbsolute(p) ? p : path.join(args.resolveDir, p),
+        namespace: 'wasm-binary',
+      };
+    });
+
+    // Virtual modules in the "wasm-binary" namespace contain the
+    // actual bytes of the WebAssembly file. This uses esbuild's
+    // built-in "binary" loader instead of manually embedding the
+    // binary data inside JavaScript code ourselves.
+    build.onLoad({ filter: /.*/, namespace: 'wasm-binary' }, async args => {
+      let contents = new Uint8Array();
+
+      try {
+        contents = await fs.promises.readFile(args.path);
+      } catch (e) {
+        if (args.importer.includes('contrib/')) {
+          console.log('error while importing:', args, e);
+        }
+      }
+      return {
+        contents,
+        loader: 'binary',
+      };
+    });
+  },
+};
+
+const context = await esbuild.context({
+  banner: {
+    js: banner,
+  },
+  outdir: IS_COMMON_JS ? 'dist/cjs/contrib/hljs/' : 'dist/esm/contrib/hljs/',
+  outExtension: {
+    '.js': '.bundle.js',
+  },
+  entryPoints: [
+    'src/contrib/hljs/typst-lite.mts',
+    'src/contrib/hljs/typst.mts',
+  ],
+  bundle: true,
+  format: IS_COMMON_JS ? 'cjs' : 'esm',
+  tsconfig: IS_COMMON_JS ? 'tsconfig.cjs.json' : 'tsconfig.json',
+  platform: 'browser',
+  external: [...builtins],
+  target: 'es2020',
+  logLevel: 'info',
+  sourcemap: IS_PRODUCTION ? false : 'inline',
+  treeShaking: true,
+  plugins: [wasmPlugin],
+});
+
+if (IS_PRODUCTION) {
+  await context.rebuild();
+  process.exit(0);
+} else {
+  await context.watch();
+}

projects/highlighter/index.html

@@ -0,0 +1,124 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <link rel="icon" href="favicon.svg" />
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Highlighting Demo</title>
+    <script>
+      // window.$typst$parserModuleSource = 'local';
+    </script>
+    <!-- <script id="script-main" type="module" src="/dist/esm/contrib/hljs/typst.bundle.js"></script> -->
+    <!-- <script id="script-main" type="module" src="/dist/esm/contrib/hljs/typst-lite.mjs"></script> -->
+    <!-- <script id="script-main" type="module" src="/dist/esm/contrib/hljs/typst-lite.bundle.js"></script> -->
+    <!-- <script id="script-main" src="/dist/cjs/contrib/hljs/typst-lite.bundle.js"></script> -->
+    <script id="script-main" src="/dist/cjs/contrib/hljs/typst.bundle.js"></script>
+    <link rel="stylesheet" href="typst-style.css" />
+    <link
+      rel="stylesheet"
+      href="https://cdn.jsdelivr.net/gh/highlightjs/[email protected]/build/styles/atom-one-dark.min.css"
+    />
+    <style>
+      body,
+      html {
+        margin: 0;
+        padding: 0;
+      }
+      body {
+        background-color: hsl(39, 77%, 95%);
+      }
+      pre,
+      p {
+        padding: 0;
+        margin: 1.5em 0;
+        white-space: normal;
+      }
+      pre code {
+        white-space: pre;
+        border-radius: 3px;
+      }
+    </style>
+    <script src="https://cdn.jsdelivr.net/gh/highlightjs/[email protected]/build/highlight.min.js"></script>
+    <script>
+      let loaded = fetch('http://localhost:20810/corpus/math/undergradmath.typ')
+        .then(res => res.text())
+        .then(text => {
+          // pre, code
+          const pre = document.createElement('pre');
+          const code = document.createElement('code');
+          code.classList.add('language-typst');
+          code.textContent = text;
+          pre.appendChild(code);
+          document.body.appendChild(pre);
+        });
+
+      let contentLoaded = new Promise(resolve => {
+        document.addEventListener('DOMContentLoaded', resolve);
+      });
+      const run = () => {
+        if (!$typst$parserModule) {
+          return;
+        }
+        $typst$parserModule.then(contentLoaded).then(() => {
+          hljs.registerLanguage(
+            'typst',
+            window.hljsTypst({
+              codeBlockDefaultLanguage: 'typst',
+            }),
+          );
+          const h = () =>
+            document.querySelectorAll('pre code').forEach(el => {
+              if (!el.getAttribute('data-highlighted')) {
+                hljs.highlightElement(el);
+              }
+            });
+          h();
+          loaded.then(h);
+        });
+      };
+      document.getElementById('script-main').onload = run;
+      run();
+    </script>
+  </head>
+  <body style="margin: 20px">
+    <div style="margin: 40px"></div>
+    <p>Usage:</p>
+    <pre>
+      <code class="language-typescript">export interface TypstHljsOptions {
+  handleCodeBlocks?: boolean | ((code: string, emitter: any) => /*handled*/ boolean);
+  codeBlockDefaultLanguage?: string;
+}
+
+$typst$parserModule.then(() => {
+  hljs.registerLanguage(
+    'typst',
+    window.hljsTypst({ // TypstHljsOptions
+      codeBlockDefaultLanguage: 'typst',
+    }),
+  );</code>
+    </pre>
+    <p>Example:</p>
+    <pre>
+      <code class="language-typst">#let x = 1;
+#import cetz.canvas;
+```js
+function nestedCodeRender() {}
+```
+Example Code:
+```typst
+#let x = 1;
+#import cetz.canvas;
+```
+
+#raw(lang: typst, `
+#let nestedCodeRender() {}`)</code>
+    </pre>
+    <p>Set Default language to typst:</p>
+    <pre>
+      <code class="language-typst">#let x = 1;
+```
+* I'm an inferred typst code *
+```</code>
+    </pre>
+  </body>
+</html>