From 41ad9285f0edcb9173d65424d8833bdb7ffd4a7a Mon Sep 17 00:00:00 2001
From: zhzluke96 <zhzluke96@outlook.com>
Date: Mon, 21 Oct 2024 20:19:51 +0800
Subject: [PATCH] feat: upgrade to v3.0.3

---
 package.json                                         | 2 +-
 pnpm-lock.yaml                                       | 8 ++++----
 src/TokenizersHub.ts                                 | 1 +
 src/components/TokenizerPanel/TokenizerConfigure.tsx | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/package.json b/package.json
index 4bf4e10..187533b 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@lenml/tokenizers": "^3.0.1",
+    "@lenml/tokenizers": "^3.0.3",
     "@quik-fe/stand": "^1.1.2",
     "@types/draft-js": "^0.11.18",
     "@types/styled-components": "^5.1.34",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 830a478..8807c24 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -6,8 +6,8 @@ settings:
 
 dependencies:
   '@lenml/tokenizers':
-    specifier: ^3.0.1
-    version: 3.0.1
+    specifier: ^3.0.3
+    version: 3.0.3
   '@quik-fe/stand':
     specifier: ^1.1.2
     version: 1.1.2
@@ -603,8 +603,8 @@ packages:
       '@jridgewell/sourcemap-codec': 1.4.15
     dev: true
 
-  /@lenml/tokenizers@3.0.1:
-    resolution: {integrity: sha512-rmj9gdhpVsZsWyKoO3q/uDWhzMjNOiIUEfH1bq8Ft0Y1kcmjE+hHtm8RcpanA5pPIJ5cz0+46K/s7Qh/J/b4IQ==}
+  /@lenml/tokenizers@3.0.3:
+    resolution: {integrity: sha512-gjIgkc/TEA+vDE4LwLD1u5OkArGxoFCqd3OB6dZbGVsXhv0FpElrNyrhHLaryLCXuY7SkjocI89fbc7ibcIjxA==}
     dev: false
 
   /@nodelib/fs.scandir@2.1.5:
diff --git a/src/TokenizersHub.ts b/src/TokenizersHub.ts
index e873378..1cc2753 100644
--- a/src/TokenizersHub.ts
+++ b/src/TokenizersHub.ts
@@ -4,6 +4,7 @@ import EventEmitter from "eventemitter3";
 
 type PreTrainedTokenizer = tokenizers.PreTrainedTokenizer;
 
+// NOTE: 这里的 package version 并不是 tokenizers 的 version，而是打包版本的 version，并不一定和 `tokenizers` 版本一致
 const package_version = "3.0.1";
 export const packages = [
   "llama3_1",
diff --git a/src/components/TokenizerPanel/TokenizerConfigure.tsx b/src/components/TokenizerPanel/TokenizerConfigure.tsx
index 3248eb8..eccc773 100644
--- a/src/components/TokenizerPanel/TokenizerConfigure.tsx
+++ b/src/components/TokenizerPanel/TokenizerConfigure.tsx
@@ -56,7 +56,7 @@ const TokenizerFromUrls = ({ onConfigChange }: Props) => {
             onChange={(e) => {
               setFormData((prev) => ({ ...prev, config_url: e.target.value }));
             }}
-            placeholder="https://huggingface.co/.../config.json"
+            placeholder="https://huggingface.co/.../tokenizer_config.json"
           />
         </label>
       </div>