fix TypeError in nagisa.decode when running in Python2

taishi-i · Dec 30, 2019 · db7bca6 · db7bca6
1 parent 45342af
commit db7bca6
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/nagisa/tagger.py b/nagisa/tagger.py
@@ -160,7 +160,8 @@ def decode(self, words, lower=False):
         """
         if not type(words) == list:
             raise AssertionError("Please input a list of words.")
-        words = ["　" if w == " " or w == "　" else utils.preprocess(w) for w in words]
+        words = [utils.preprocess_without_rstrip(w) if w == " " or w == "　"
+                 else utils.preprocess(w) for w in words]
         postags = self._postagging(words, lower)
         return postags
 

diff --git a/nagisa/utils.pyx b/nagisa/utils.pyx
@@ -46,6 +46,14 @@ cpdef unicode preprocess(text):
     return text
 
 
+cpdef unicode preprocess_without_rstrip(text):
+    if type(text) != unicode:
+        text = unicode(text, 'utf-8')
+    text = normalize(text)
+    text = text.replace(' ', '　')
+    return text
+
+
 cpdef list get_unigram(unicode text):
     cdef unicode uni
     return [uni for uni in text]