dev(pkg::compiler): load font asset from remote

compiler/Cargo.toml

@@ -102,12 +102,5 @@ __web = [
 web-render = ["__web"]
 browser-compile = ["__web", "web-render"]
 browser-embedded-fonts = ["__web"]
-web = [
-    "__web",
-    "web-render",
-    "browser-compile",
-    "browser-embedded-fonts",
-    "cjk",
-    "emoji",
-]
+web = ["__web", "web-render", "browser-compile"]
 default = ["system", "dynamic-layout", "cjk", "emoji"]

packages/templates/node.js-compiler-next/package.json

@@ -9,7 +9,8 @@
   "dependencies": {
     "@myriaddreamin/typst-ts-renderer": "^0.4.0-rc6",
     "@myriaddreamin/typst-ts-web-compiler": "^0.4.0-rc6",
-    "@myriaddreamin/typst.ts": "^0.4.0-rc6"
+    "@myriaddreamin/typst.ts": "^0.4.0-rc6",
+    "node-fetch": "3.3.2"
   },
   "devDependencies": {
     "@types/web": "^0.0.99",

packages/templates/node.js-compiler-next/src/main.ts

@@ -4,9 +4,58 @@ import * as _2 from '@myriaddreamin/typst-ts-web-compiler';
 
 import { createTypstCompiler, createTypstRenderer } from '@myriaddreamin/typst.ts';
 
+import { preloadFontAssets } from '@myriaddreamin/typst.ts/dist/cjs/options.init.cjs';
+import { existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from 'fs';
+import * as path from 'path';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+
 async function main() {
+  const fetcher = (await import('node-fetch')).default;
+  const dataDir =
+    process.env.APPDATA ||
+    (process.platform == 'darwin'
+      ? process.env.HOME + '/Library/Preferences'
+      : process.env.HOME + '/.local/share');
+
+  const cacheDir = path.join(dataDir, 'typst/fonts');
+
   const compiler = createTypstCompiler();
-  await compiler.init();
+  await compiler.init({
+    beforeBuild: [
+      preloadFontAssets({
+        assets: ['text', 'cjk', 'emoji'],
+        fetcher: async (url: URL | RequestInfo, init?: RequestInit | undefined) => {
+          const cachePath = path.join(cacheDir, url.toString().replace(/[^a-zA-Z0-9]/g, '_'));
+          if (existsSync(cachePath)) {
+            const font_res = {
+              arrayBuffer: async () => {
+                return readFileSync(cachePath).buffer;
+              },
+            };
+
+            return font_res as any;
+          }
+
+          console.log('loading remote font:', url);
+          const proxyOption = process.env.HTTPS_PROXY
+            ? { agent: new HttpsProxyAgent(process.env.HTTPS_PROXY) }
+            : {};
+
+          const font_res = await fetcher(url as any, {
+            ...proxyOption,
+            ...((init as any) || {}),
+          });
+          const buffer = await font_res.arrayBuffer();
+          mkdirSync(path.dirname(cachePath), { recursive: true });
+          writeFileSync(cachePath, Buffer.from(buffer));
+          font_res.arrayBuffer = async () => {
+            return buffer;
+          };
+          return font_res as any;
+        },
+      }),
+    ],
+  });
 
   compiler.addSource('/main.typ', 'Hello, typst!');
   const artifactData = await compiler.compile({

packages/typst.ts/src/compiler.mts

@@ -3,7 +3,7 @@ import type * as typst from '@myriaddreamin/typst-ts-web-compiler/pkg/wasm-pack-
 import { buildComponent, globalFontPromises } from './init.mjs';
 import { FsAccessModel } from './internal.types.mjs';
 
-import type { InitOptions } from './options.init.mjs';
+import { preloadRemoteFonts, type InitOptions } from './options.init.mjs';
 import { LazyWasmModule } from './wasm.mjs';
 
 export interface CompileOptions {
@@ -112,6 +112,26 @@ class TypstCompilerDriver {
   async init(options?: Partial<InitOptions>): Promise<void> {
     this.compilerJs = await import('@myriaddreamin/typst-ts-web-compiler/pkg/wasm-pack-shim.mjs');
     const TypstCompilerBuilder = this.compilerJs.TypstCompilerBuilder;
+
+    const compilerOptions = { ...(options || {}) };
+    const hasPreloadRemoteFonts = compilerOptions.beforeBuild?.some(
+      (fn: any) => fn._preloadRemoteFontOptions !== undefined,
+    );
+    const hasSpecifiedAssets = compilerOptions.beforeBuild?.some(
+      (fn: any) => fn._preloadRemoteFontOptions?.assets !== undefined,
+    );
+    const hasDisableAssets = compilerOptions.beforeBuild?.some(
+      (fn: any) => fn._preloadRemoteFontOptions?.assets === false,
+    );
+
+    if (!hasPreloadRemoteFonts || (!hasSpecifiedAssets && !hasDisableAssets)) {
+      compilerOptions.beforeBuild?.push(
+        preloadRemoteFonts([], {
+          assets: ['text'],
+        }),
+      );
+    }
+
     this.compiler = await buildComponent(options, gCompilerModule, TypstCompilerBuilder, {});
   }
 

packages/typst.ts/src/init.mts

@@ -20,7 +20,7 @@ export interface ComponentBuildHooks {
 
 interface InitContext<T> {
   ref: {
-    loadFont(builder: TypstCommonBuilder<T>, fontPath: string): Promise<void>;
+    loadFonts(builder: TypstCommonBuilder<T>, fonts: (string | Uint8Array)[]): Promise<void>;
   };
   builder: TypstCommonBuilder<T>;
   hooks: ComponentBuildHooks;
@@ -109,10 +109,47 @@ async function addPartialFonts<T>({ builder, hooks }: InitContext<T>): Promise<v
 }
 
 class ComponentBuilder<T> {
-  async loadFont(builder: TypstCommonBuilder<T>, fontPath: string): Promise<void> {
-    const response = await fetch(fontPath);
-    const fontBuffer = new Uint8Array(await response.arrayBuffer());
-    await builder.add_raw_font(fontBuffer);
+  loadedFonts = new Set<string>();
+  fetcher?: typeof fetch = fetch;
+
+  setFetcher(fetcher: typeof fetch): void {
+    this.fetcher = fetcher;
+  }
+
+  async loadFonts(builder: TypstCommonBuilder<T>, fonts: (string | Uint8Array)[]): Promise<void> {
+    const escapeImport = (t: string) => import(t);
+    const fetcher = (this.fetcher ||= await escapeImport('node-fetch'));
+
+    const fontsToLoad = fonts.filter(font => {
+      if (font instanceof Uint8Array) {
+        return true;
+      }
+
+      if (this.loadedFonts.has(font)) {
+        return false;
+      }
+
+      this.loadedFonts.add(font);
+      return true;
+    });
+
+    const fontLists = await Promise.all(
+      fontsToLoad.map(async font => {
+        if (font instanceof Uint8Array) {
+          await builder.add_raw_font(font);
+          return;
+        }
+
+        return new Uint8Array(await (await fetcher(font)).arrayBuffer());
+      }),
+    );
+
+    for (const font of fontLists) {
+      if (!font) {
+        continue;
+      }
+      await builder.add_raw_font(font);
+    }
   }
 
   async build(

packages/typst.ts/src/options.init.mts

@@ -58,6 +58,70 @@ export interface InitOptions {
   getModule(): WebAssemblyModuleRef | Promise<WebAssemblyModuleRef>;
 }
 
+/** @internal */
+const _textFonts: string[] = [
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/LinLibertine_R.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/LinLibertine_RB.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/LinLibertine_RBI.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/LinLibertine_RI.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NewCMMath-Book.otf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NewCMMath-Regular.otf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NewCM10-Regular.otf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NewCM10-Bold.otf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NewCM10-Italic.otf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NewCM10-BoldItalic.otf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/DejaVuSansMono.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/DejaVuSansMono-Bold.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/DejaVuSansMono-Oblique.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/DejaVuSansMono-BoldOblique.ttf',
+];
+/** @internal */
+const _cjkFonts: string[] = [
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/InriaSerif-Bold.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/InriaSerif-BoldItalic.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/InriaSerif-Italic.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/InriaSerif-Regular.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/Roboto-Regular.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NotoSerifCJKsc-Regular.otf',
+];
+/** @internal */
+const _emojiFonts: string[] = [
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/TwitterColorEmoji.ttf',
+  'https://raw.githubusercontent.com/Myriad-Dreamin/typst/assets-fonts/NotoColorEmoji.ttf',
+];
+
+type AvailableFontAsset = 'text' | 'cjk' | 'emoji';
+
+export interface LoadRemoteAssetsOptions {
+  /**
+   * preload font assets or don't preload any font assets
+   * @default ['text']
+   */
+  assets?: AvailableFontAsset[] | false;
+  /**
+   * custom fetcher
+   * Note: the default fetcher for node.js does not cache any fonts
+   * @default fetch
+   */
+  fetcher?: typeof fetch;
+}
+
+export interface LoadRemoteFontsOptions extends LoadRemoteAssetsOptions {}
+
+/**
+ * disable default font assets
+ */
+export function disableDefaultFontAssets(): BeforeBuildFn {
+  return preloadRemoteFonts([], { assets: false });
+}
+
+/**
+ * preload font assets
+ */
+export function preloadFontAssets(options?: LoadRemoteAssetsOptions): BeforeBuildFn {
+  return preloadRemoteFonts([], options);
+}
+
 /**
  * preload remote fonts
  *
@@ -76,10 +140,40 @@ export interface InitOptions {
  * });
  * ```
  */
-export function preloadRemoteFonts(fonts: string[]): BeforeBuildFn {
-  return async (_, { ref, builder }: InitContext) => {
-    await Promise.all(fonts.map(font => ref.loadFont(builder, font)));
+export function preloadRemoteFonts(
+  userFonts: (string | Uint8Array)[],
+  options?: LoadRemoteFontsOptions,
+): BeforeBuildFn {
+  const fonts = [...userFonts];
+  if (
+    options &&
+    options?.assets !== false &&
+    options?.assets?.length &&
+    options?.assets?.length > 0
+  ) {
+    for (const asset of options.assets) {
+      switch (asset) {
+        case 'text':
+          fonts.push(..._textFonts);
+          break;
+        case 'cjk':
+          fonts.push(..._cjkFonts);
+          break;
+        case 'emoji':
+          fonts.push(..._emojiFonts);
+          break;
+      }
+    }
+  }
+
+  const loader = async (_: BeforeBuildMark, { ref, builder }: InitContext) => {
+    if (options?.fetcher) {
+      ref.setFetcher(options.fetcher);
+    }
+    await ref.loadFonts(builder, fonts);
   };
+  loader._preloadRemoteFontOptions = options;
+  return loader;
 }
 
 /**
@@ -169,7 +263,8 @@ type Builder = typstRenderer.TypstRendererBuilder & typstCompiler.TypstCompilerB
  */
 interface InitContext {
   ref: {
-    loadFont(builder: Builder, fontPath: string): Promise<void>;
+    setFetcher(fetcher: typeof fetch): void;
+    loadFonts(builder: Builder, fonts: (string | Uint8Array)[]): Promise<void>;
   };
   builder: Builder;
 }

yarn.lock

@@ -4186,6 +4186,11 @@ damerau-levenshtein@^1.0.8:
   resolved "https://registry.yarnpkg.com/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz#b43d286ccbd36bc5b2f7ed41caf2d0aba1f8a6e7"
   integrity sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==
 
+data-uri-to-buffer@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e"
+  integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==
+
 data-urls@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b"
@@ -5298,6 +5303,14 @@ faye-websocket@^0.11.3:
   dependencies:
     websocket-driver ">=0.5.1"
 
+fetch-blob@^3.1.2, fetch-blob@^3.1.4:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9"
+  integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==
+  dependencies:
+    node-domexception "^1.0.0"
+    web-streams-polyfill "^3.0.3"
+
 [email protected], figures@^3.0.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af"
@@ -5450,6 +5463,13 @@ form-data@^4.0.0:
     combined-stream "^1.0.8"
     mime-types "^2.1.12"
 
+formdata-polyfill@^4.0.10:
+  version "4.0.10"
+  resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423"
+  integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==
+  dependencies:
+    fetch-blob "^3.1.2"
+
 [email protected]:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811"
@@ -7335,6 +7355,20 @@ node-addon-api@^3.0.0, node-addon-api@^3.2.1:
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
   integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==
 
+node-domexception@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
+  integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
+
+[email protected]:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.2.tgz#d1e889bacdf733b4ff3b2b243eb7a12866a0b78b"
+  integrity sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==
+  dependencies:
+    data-uri-to-buffer "^4.0.0"
+    fetch-blob "^3.1.4"
+    formdata-polyfill "^4.0.10"
+
 node-forge@^1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -9458,7 +9492,7 @@ typescript@=5.0.4:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.0.4.tgz#b217fd20119bd61a94d4011274e0ab369058da3b"
   integrity sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==
 
-typescript@^5.0.2, typescript@^5.0.4:
+typescript@^5.0.2:
   version "5.2.2"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
   integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
@@ -9699,6 +9733,11 @@ wcwidth@^1.0.1:
   dependencies:
     defaults "^1.0.3"
 
+web-streams-polyfill@^3.0.3:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6"
+  integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==
+
 webidl-conversions@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff"