From 96db3e47eb3feb74ede7947961077154c0b8c682 Mon Sep 17 00:00:00 2001
From: zhzluke96 <zhzluke96@outlook.com>
Date: Sun, 6 Oct 2024 04:17:37 +0800
Subject: [PATCH] feat: upgrade to v3

---
 package.json         |  2 +-
 pnpm-lock.yaml       | 21 +++++++++++----------
 src/TokenizersHub.ts |  2 +-
 3 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/package.json b/package.json
index 2014bad..4bf4e10 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@lenml/tokenizers": "^1.0.5",
+    "@lenml/tokenizers": "^3.0.1",
     "@quik-fe/stand": "^1.1.2",
     "@types/draft-js": "^0.11.18",
     "@types/styled-components": "^5.1.34",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index bf794b1..830a478 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -6,8 +6,8 @@ settings:
 
 dependencies:
   '@lenml/tokenizers':
-    specifier: ^1.0.5
-    version: 1.0.5
+    specifier: ^3.0.1
+    version: 3.0.1
   '@quik-fe/stand':
     specifier: ^1.1.2
     version: 1.1.2
@@ -603,8 +603,8 @@ packages:
       '@jridgewell/sourcemap-codec': 1.4.15
     dev: true
 
-  /@lenml/tokenizers@1.0.5:
-    resolution: {integrity: sha512-Xx3V3O/xHt+AnCCOwtw0YuDIoEWEBzqkdbuZEXFAcPM7JLSHUmrq5pjwqJMX4n5o+58ZUnSwX19+cCLQGQpmBA==}
+  /@lenml/tokenizers@3.0.1:
+    resolution: {integrity: sha512-rmj9gdhpVsZsWyKoO3q/uDWhzMjNOiIUEfH1bq8Ft0Y1kcmjE+hHtm8RcpanA5pPIJ5cz0+46K/s7Qh/J/b4IQ==}
     dev: false
 
   /@nodelib/fs.scandir@2.1.5:
@@ -1139,8 +1139,8 @@ packages:
     resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==}
     dev: true
 
-  /core-js@3.37.1:
-    resolution: {integrity: sha512-Xn6qmxrQZyB0FFY8E3bgRXei3lWDJHhvI+u0q9TKIYM49G8pAr0FgnnrFRAmsbptZL1yxRADVXn+x5AGsbBfyw==}
+  /core-js@3.38.1:
+    resolution: {integrity: sha512-OP35aUorbU3Zvlx7pjsFdu1rGNnD4pgw/CWoYzRY3t2EzoVT7shKHY1dlAy3f41cGIO7ZDPQimhGFTlEYkG/Hw==}
     requiresBuild: true
     dev: false
 
@@ -1422,14 +1422,14 @@ packages:
   /fbjs@2.0.0:
     resolution: {integrity: sha512-8XA8ny9ifxrAWlyhAbexXcs3rRMtxWcs3M0lctLfB49jRDHiaxj+Mo0XxbwE7nKZYzgCFoq64FS+WFd4IycPPQ==}
     dependencies:
-      core-js: 3.37.1
+      core-js: 3.38.1
       cross-fetch: 3.1.8
       fbjs-css-vars: 1.0.2
       loose-envify: 1.4.0
       object-assign: 4.1.1
       promise: 7.3.1
       setimmediate: 1.0.5
-      ua-parser-js: 0.7.38
+      ua-parser-js: 0.7.39
     transitivePeerDependencies:
       - encoding
     dev: false
@@ -2106,8 +2106,9 @@ packages:
     hasBin: true
     dev: true
 
-  /ua-parser-js@0.7.38:
-    resolution: {integrity: sha512-fYmIy7fKTSFAhG3fuPlubeGaMoAd6r0rSnfEsO5nEY55i26KSLt9EH7PLQiiqPUhNqYIJvSkTy1oArIcXAbPbA==}
+  /ua-parser-js@0.7.39:
+    resolution: {integrity: sha512-IZ6acm6RhQHNibSt7+c09hhvsKy9WUr4DVbeq9U8o71qxyYtJpQeDxQnMrVqnIFMLcQjHO0I9wgfO2vIahht4w==}
+    hasBin: true
     dev: false
 
   /update-browserslist-db@1.0.16(browserslist@4.23.0):
diff --git a/src/TokenizersHub.ts b/src/TokenizersHub.ts
index fe4a611..61ef6e0 100644
--- a/src/TokenizersHub.ts
+++ b/src/TokenizersHub.ts
@@ -4,7 +4,7 @@ import EventEmitter from "eventemitter3";
 
 type PreTrainedTokenizer = tokenizers.PreTrainedTokenizer;
 
-const package_version = "1.1.1";
+const package_version = "3.0.1";
 export const packages = [
   "llama3_1",
   "llama2",