fix: tokenize OpenAI encode exception #1250 (#1253)

Opla · Sep 16, 2024 · 4f5cc4c · 4f5cc4c
1 parent bf3f1b6
commit 4f5cc4c
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/crates/tokenizer/src/lib.rs b/crates/tokenizer/src/lib.rs
@@ -27,7 +27,7 @@ pub fn encode_gpt(text: String) -> Result<Vec<Rank>, String> {
 }
 
 pub fn encode(text: String, model: String, encoding: Option<String>) -> Result<Vec<Rank>, String> {
-    if model.starts_with("gpt") {
+    if model.to_lowercase().starts_with("gpt") {
         return encode_gpt(text);
     }
     if encoding.is_some() {

diff --git a/webapp/native/src/providers/mod.rs b/webapp/native/src/providers/mod.rs
@@ -529,7 +529,7 @@ impl ProvidersManager {
             let encoded = match encode(text, model, None) {
                 Ok(e) => { e }
                 Err(err) => {
-                    return Err(format!("LLM encode error: {:?}", err));
+                    return Err(format!("LLM OpenAI encode error: {:?}", err));
                 }
             };
             let tokens: Vec<u64> = encoded