fix white space bug in nagisa.decode

taishi-i · Dec 30, 2019 · 6897b07 · 6897b07
1 parent 29e93c6
commit 6897b07
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 3 deletions.
diff --git a/nagisa/tagger.py b/nagisa/tagger.py
@@ -160,7 +160,7 @@ def decode(self, words, lower=False):
         """
         if not type(words) == list:
             raise AssertionError("Please input a list of words.")
-        words = [utils.preprocess(w) for w in words]
+        words = ["　" if w == " " or w == "　" else utils.preprocess(w) for w in words]
         postags = self._postagging(words, lower)
         return postags
 

diff --git a/test/nagisa_test.py b/test/nagisa_test.py
@@ -76,17 +76,39 @@ def test_tagging(self):
         postags = nagisa.decode(words)
         self.assertEqual(output, postags)
 
+        # test_13
+        words  = [" (人•ᴗ•♡)", "　", "こんばんは","♪"]
+        output = ['補助記号', "空白", '感動詞', '補助記号']
+        postags = nagisa.postagging(words)
+
+        self.assertEqual(output, postags)
+
+        # test_14
+        postags = nagisa.decode(words)
+        self.assertEqual(output, postags)
+
+        # test_15
+        words  = [" (人•ᴗ•♡)", " ", "こんばんは","♪"]
+        output = ['補助記号', "空白", '感動詞', '補助記号']
+        postags = nagisa.postagging(words)
+
+        self.assertEqual(output, postags)
+
+        # test_16
+        postags = nagisa.decode(words)
+        self.assertEqual(output, postags)
+
 
     def test_fit(self):
-        # test_13
+        # test_17
         nagisa.fit(
             train_file="nagisa/data/sample_datasets/sample.train",
             dev_file="nagisa/data/sample_datasets/sample.dev",
             test_file="nagisa/data/sample_datasets/sample.test",
             model_name="sample",
         )
 
-        # test_14
+        # test_18
         nagisa.fit(
             train_file="nagisa/data/sample_datasets/sample.train",
             dev_file="nagisa/data/sample_datasets/sample.dev",