Vythai (#706)

Fixed Vietnamese nonbreaking space
stanfordnlp · May 28, 2021 · 1972122 · 1972122
1 parent e3a0c95
commit 1972122
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/stanza/models/common/vocab.py b/stanza/models/common/vocab.py
@@ -47,6 +47,7 @@ def load_state_dict(cls, state_dict):
         return new
 
     def normalize_unit(self, unit):
+        unit = unit.replace(" ","\xa0")
         if self.lower:
             return unit.lower()
         return unit
@@ -79,7 +80,7 @@ def __getitem__(self, key):
             raise TypeError("Vocab key must be one of str, list, or int")
 
     def __contains__(self, key):
-        return key in self._unit2id
+        return self.normalize_unit(key) in self._unit2id
 
     @property
     def size(self):