[#41] fix tokenize method to call right attribute

aisingapore · Dec 13, 2021 · a519024 · a519024
1 parent aa3e3d5
commit a519024
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/sgnlp/models/sentic_asgcn/tokenization.py b/sgnlp/models/sentic_asgcn/tokenization.py
@@ -83,12 +83,11 @@ def create_vocab(self, save_directory: str):
         return vocab
 
     def _tokenize(self, text, **kwargs):
-        text = text.lower()
+        if self.do_lower_case:
+            text = text.lower()
         words = text.split()
         unknownidx = 1
-        sequence = [
-            self.word2idx[w] if w in self.word2idx else unknownidx for w in words
-        ]
+        sequence = [self.vocab[w] if w in self.vocab else unknownidx for w in words]
         if len(sequence) == 0:
             sequence = [0]
         return sequence