Add olmo2 tokenizer to convert script

ggerganov · Nov 26, 2024 · 8d243b6 · 8d243b6
1 parent c9b00a7
commit 8d243b6
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
@@ -658,6 +658,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
         if chkhsh == "60824e3c0d9401f89943cbb2fff727f0e2d4c545ba4df2d6e4f09a6db0f5b450":
             # ref: https://huggingface.co/facebook/chameleon-7b
             res = "chameleon"
+        if chkhsh == "53e325976a6e142379c19b09afcae354f2f496f147afa8f9e189a33fe4e3024e":
+            # ref: https://huggingface.co/allenai/OLMo-2-1124-7B-Instruct
+            res = "olmo2"
 
         if res is None:
             logger.warning("\n")

diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
@@ -102,6 +102,7 @@ class TOKENIZER_TYPE(IntEnum):
     {"name": "exaone",         "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", },
     {"name": "phi-2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/microsoft/phi-2", },
     {"name": "chameleon",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/facebook/chameleon-7b", },
+    {"name": "olmo2",          "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-2-1124-7B-Instruct", },
 ]