diff --git a/tests/torchtune/models/qwen2_5/test_tokenizer.py b/tests/torchtune/models/qwen2_5/test_tokenizer.py index 332fef2b92..8649d6d45e 100644 --- a/tests/torchtune/models/qwen2_5/test_tokenizer.py +++ b/tests/torchtune/models/qwen2_5/test_tokenizer.py @@ -24,50 +24,15 @@ def test_tokenize_messages(self): Message(role="user", content="Give me a short introduction to LLMs."), Message(role="assistant", content=""), ] + + # fmt: off expected_tokens = [ - 151644, - 82, - 88, - 479, - 94, - 56, - 119, - 230, - 98, - 374, - 494, - 1318, - 249, - 13, - 151645, - 94, - 151644, - 273, - 105, - 94, - 38, - 229, - 362, - 98, - 1695, - 310, - 1305, - 165, - 128, - 432, - 43, - 44, - 82, - 13, - 151645, - 94, - 151644, - 397, - 251, - 249, - 94, + 151644, 82, 88, 479, 94, 56, 119, 230, 98, 374, 494, 1318, 249, 13, 151645, 94, 151644, 273, 105, 94, + 38, 229, 362, 98, 1695, 310, 1305, 165, 128, 432, 43, 44, 82, 13, 151645, 94, 151644, 397, 251, 249, 94, 151643, - ] + ] # noqa + # fmt: on + expected_formatted_messages = ( "<|im_start|>system\n" "You are a helpful assistant.<|im_end|>\n" @@ -92,75 +57,15 @@ def test_tool_call(self): Message(role="ipython", content="test response"), Message(role="assistant", content=""), ] + # fmt: off expected_tokens = [ - 151644, - 82, - 88, - 479, - 94, - 64, - 151645, - 94, - 151644, - 273, - 105, - 94, - 65, - 151645, - 94, - 151644, - 397, - 251, - 249, - 94, - 151657, - 94, - 83, - 269, - 107, - 330, - 94, - 151658, - 151645, - 94, - 151644, - 273, - 105, - 94, - 27, - 83, - 1364, - 62, - 237, - 79, - 102, - 182, - 29, - 94, - 83, - 269, - 706, - 102, - 182, - 94, - 1932, - 83, - 1364, - 62, - 237, - 79, - 102, - 182, - 29, - 151645, - 94, - 151644, - 397, - 251, - 249, - 94, - 151643, - ] + 151644, 82, 88, 479, 94, 64, 151645, 94, 151644, 273, 105, 94, 65, 151645, 94, 151644, 397, 251, 249, + 94, 151657, 94, 83, 269, 107, 330, 94, 151658, 151645, 94, 151644, 273, 105, 94, 27, 83, 1364, + 62, 237, 79, 102, 182, 29, 94, 83, 269, 706, 102, 182, 94, 1932, 83, 1364, 62, 237, 79, 102, + 182, 29, 151645, 94, 151644, 397, 251, 249, 94, 151643, + ] # noqa + # fmt: on + expected_formatted_messages = ( "<|im_start|>system\n" "a<|im_end|>\n"