From 2b5b560b17129b8e699dad800939caee41619004 Mon Sep 17 00:00:00 2001
From: "sweep-ai[bot]" <128439645+sweep-ai[bot]@users.noreply.github.com>
Date: Wed, 3 Jan 2024 18:00:42 +0000
Subject: [PATCH] feat: Add unit tests for jieba dependency

---
 tests/test_jieba.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 tests/test_jieba.py

diff --git a/tests/test_jieba.py b/tests/test_jieba.py
new file mode 100644
index 00000000..5c4245c2
--- /dev/null
+++ b/tests/test_jieba.py
@@ -0,0 +1,16 @@
+import jieba
+import pytest
+
+
+def test_jieba_segmentation():
+    text = "我爱自然语言处理"
+    seg_list = jieba.cut(text, cut_all=False)
+    assert list(seg_list) == ['我', '爱', '自然语言处理']
+
+def test_jieba_import():
+    assert jieba is not None
+
+def test_jieba_tokenization():
+    text = "我爱自然语言处理"
+    tokens = jieba.tokenize(text)
+    assert list(tokens) == [('我', 0, 1), ('爱', 1, 2), ('自然语言处理', 2, 8)]