reduce max token length to 255 to allow for more tokens in tokenizer

guidance-ai · Dec 18, 2024 · 389755b · 389755b
1 parent 53f9968
commit 389755b
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/toktrie/src/toktree.rs b/toktrie/src/toktree.rs
@@ -263,8 +263,8 @@ impl TrieNode {
     }
 }
 
-// max length of token is 1023 bytes
-const LEN_BITS: u32 = 10;
+// max length of token is 255 bytes
+const LEN_BITS: u32 = 8;
 
 impl TokTrie {
     // see https://github.com/microsoft/llguidance/blob/main/docs/special_tokens.md