diff --git a/minlp-tokenizer/minlptokenizer/tokenizer.py b/minlp-tokenizer/minlptokenizer/tokenizer.py index 63bced23..1052bf02 100644 --- a/minlp-tokenizer/minlptokenizer/tokenizer.py +++ b/minlp-tokenizer/minlptokenizer/tokenizer.py @@ -125,7 +125,7 @@ def tag2words(self, text, y_pred_result): word = '' if word: words.append(word) - return regex.split(r'\s+', ' '.join(words)) + return words def set_interfere_factor(self, interfere_factor): """ diff --git a/minlp-tokenizer/requirements.txt b/minlp-tokenizer/requirements.txt index 06bb0e78..142f16be 100644 --- a/minlp-tokenizer/requirements.txt +++ b/minlp-tokenizer/requirements.txt @@ -1,3 +1,3 @@ -tensorflow>=1.15,<2 +tensorflow>=1.14 pyahocorasick regex diff --git a/minlp-tokenizer/setup.py b/minlp-tokenizer/setup.py index be7c2ba6..9724b009 100644 --- a/minlp-tokenizer/setup.py +++ b/minlp-tokenizer/setup.py @@ -32,7 +32,7 @@ setup( name='minlp-tokenizer', - version='3.2.3', + version='3.2.4', description='MiNLP-Tokenizer中文分词工具', author='Yuankai Guo, Liang Shi, Yupeng Chen', author_email='guoyuankai@xiaomi.com, shiliang1@xiaomi.com',