Add tokenizer type

ggerganov · Nov 26, 2024 · 445904e · 445904e
1 parent 8d243b6
commit 445904e
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/include/llama.h b/include/llama.h
@@ -104,6 +104,7 @@ extern "C" {
         LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH   = 24,
         LLAMA_VOCAB_PRE_TYPE_EXAONE         = 25,
         LLAMA_VOCAB_PRE_TYPE_CHAMELEON      = 26,
+        LLAMA_VOCAB_PRE_TYPE_OLMO2          = 27,
     };
 
     enum llama_rope_type {

diff --git a/src/llama.cpp b/src/llama.cpp
@@ -6406,6 +6406,9 @@ static void llm_load_vocab(
                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CHAMELEON;
                 vocab.tokenizer_add_bos = true;
                 vocab.tokenizer_clean_spaces = false;
+            } else if (
+                tokenizer_pre == "olmo2") {
+                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_OLMO2;
             } else {
                 throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
             }