Update code

PyThaiNLP · Oct 12, 2022 · be25a73 · be25a73
1 parent 101ad5a
commit be25a73
Show file tree

Hide file tree

Showing 6 changed files with 1 addition and 172 deletions.
diff --git a/pythainlp/tag/perceptron.py b/pythainlp/tag/perceptron.py
@@ -36,7 +36,7 @@ def _pud_tagger():
 def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
     """
     :param list words: a list of tokenized words
-    :param str corpus: corpus name (orchid, pud, or lst20)
+    :param str corpus: corpus name (orchid, pud)
     :return: a list of tuples (word, POS tag)
     :rtype: list[tuple[str, str]]
     """

diff --git a/pythainlp/wangchanberta/core.py b/pythainlp/wangchanberta/core.py
@@ -81,13 +81,6 @@ def get_ner(
                     i['word'].replace("<_>", " ").replace('▁', ''), i['entity']
                 ) for i in self.json_ner if i['word'] != '▁'
             ]
-        elif self.grouped_entities and self.dataset_name == "lst20":
-            self.sent_ner = [
-                (
-                    i['word'].replace("<_>", " ").replace('▁', ''),
-                    i['entity_group'].replace('_', '-').replace('E-', 'I-')
-                ) for i in self.json_ner
-            ]
         else:
             self.sent_ner = [
                 (

diff --git a/pythainlp/wangchanberta/postag.py b/pythainlp/wangchanberta/postag.py
diff --git a/tests/test_tag.py b/tests/test_tag.py
@@ -48,8 +48,6 @@ def test_pos_tag(self):
         self.assertEqual(unigram.tag([], corpus="pud"), [])
         self.assertEqual(unigram.tag(None, corpus="orchid"), [])
         self.assertEqual(unigram.tag([], corpus="orchid"), [])
-        self.assertEqual(unigram.tag(None, corpus="lst20"), [])
-        self.assertEqual(unigram.tag([], corpus="lst20"), [])
         self.assertIsNotNone(
             pos_tag(tokens, engine="unigram", corpus="orchid")
         )
@@ -58,11 +56,6 @@ def test_pos_tag(self):
         )
         self.assertIsNotNone(pos_tag(tokens, engine="unigram", corpus="pud"))
         self.assertIsNotNone(pos_tag([""], engine="unigram", corpus="pud"))
-        self.assertIsNotNone(pos_tag(tokens, engine="unigram", corpus="lst20"))
-        self.assertIsNotNone(pos_tag([""], engine="unigram", corpus="lst20"))
-        self.assertIsNotNone(
-            pos_tag([""], engine="unigram", corpus="lst20_ud")
-        )
         self.assertEqual(
             pos_tag(["คุณ", "กำลัง", "ประชุม"], engine="unigram"),
             [("คุณ", "PPRS"), ("กำลัง", "XVBM"), ("ประชุม", "VACT")],
@@ -81,8 +74,6 @@ def test_pos_tag(self):
         self.assertEqual(perceptron.tag([], corpus="orchid_ud"), [])
         self.assertEqual(perceptron.tag(None, corpus="pud"), [])
         self.assertEqual(perceptron.tag([], corpus="pud"), [])
-        self.assertEqual(perceptron.tag(None, corpus="lst20"), [])
-        self.assertEqual(perceptron.tag([], corpus="lst20"), [])
         self.assertIsNotNone(
             pos_tag(tokens, engine="perceptron", corpus="orchid")
         )
@@ -92,18 +83,6 @@ def test_pos_tag(self):
         self.assertIsNotNone(
             pos_tag(tokens, engine="perceptron", corpus="pud")
         )
-        self.assertIsNotNone(
-            pos_tag(tokens, engine="perceptron", corpus="lst20")
-        )
-        self.assertIsNotNone(
-            pos_tag(tokens, engine="perceptron", corpus="lst20_ud")
-        )
-        self.assertEqual(
-            pos_tag([], engine="wangchanberta", corpus="lst20"), []
-        )
-        self.assertIsNotNone(
-            pos_tag(tokens, engine="wangchanberta", corpus="lst20")
-        )
         self.assertIsNotNone(
             pos_tag(tokens, engine="tltk")
         )
@@ -117,14 +96,6 @@ def test_pos_tag(self):
                 [("แมว", "NCMN"), ("วิ่ง", "VACT")],
             ],
         )
-        with self.assertRaises(ValueError):
-            self.assertIsNotNone(
-                pos_tag(tokens, engine="wangchanberta", corpus="lst20_ud")
-            )
-        with self.assertRaises(ValueError):
-            self.assertIsNotNone(
-                tltk.pos_tag(tokens, corpus="lst20")
-            )
 
     # ### pythainlp.tag.PerceptronTagger
 
@@ -362,9 +333,6 @@ def test_NER_class(self):
         self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า"))
         self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False))
         self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True))
-        ner = NER(engine="lst20_onnx")
-        self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า"))
-        self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", tag=True))
         ner = NER(engine="tltk")
         self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า"))
         self.assertIsNotNone(ner.tag("แมวทำอะไรตอนห้าโมงเช้า", pos=False))

diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py
@@ -591,7 +591,6 @@ def test_nercut(self):
         self.assertEqual(nercut.segment("ทันแน่ๆ"), ['ทัน', 'แน่ๆ'])
         self.assertEqual(nercut.segment("%1ครั้ง"), ['%', '1', 'ครั้ง'])
         self.assertEqual(nercut.segment("ทุ๊กกโคนน"), ['ทุ๊กกโคนน'])
-        self.assertEqual(nercut.segment("อือหือ"), ['อือหือ'])
         self.assertEqual(
             nercut.segment("อย่าลืมอัพการ์ดนะจ๊ะ"),
             ['อย่าลืมอัพการ์ดนะจ๊ะ']

diff --git a/tests/test_wangchanberta.py b/tests/test_wangchanberta.py
@@ -28,29 +28,6 @@ def test_thainer_wangchanberta(self):
             ner.get_ner("I คิด therefore I am ผ็ฎ์", tag=True)
         )
 
-    def test_lst20_ner_wangchanberta(self):
-        ner = ThaiNameTagger(dataset_name="lst20")
-        self.assertIsNotNone(
-            ner.get_ner("I คิด therefore I am ผ็ฎ์")
-        )
-        self.assertIsNotNone(
-            ner.get_ner("I คิด therefore I am ผ็ฎ์", tag=True)
-        )
-        self.assertIsNotNone(
-            ner.get_ner(
-                "โรงเรียนสวนกุหลาบเป็นโรงเรียนที่ดี แต่ไม่มีสวนกุหลาบ",
-                tag=True
-            )
-        )
-
-        ner = ThaiNameTagger(
-            dataset_name="lst20",
-            grouped_entities=False
-        )
-        self.assertIsNotNone(
-            ner.get_ner("I คิด therefore I am ผ็ฎ์", tag=True)
-        )
-
     def test_segment_wangchanberta(self):
         self.assertIsNotNone(
             segment("I คิด therefore I am ผ็ฎ์")