small fix to tokenizer

tacchinotacchi · Oct 30, 2019 · d01f173 · d01f173
1 parent be35422
commit d01f173
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/utils.py b/utils.py
@@ -9,7 +9,10 @@
 from torchtext.vocab import pretrained_aliases, Vocab
 
 import spacy
+from spacy.symbols import ORTH
+
 spacy_en = spacy.load("en")
+spacy_en.tokenizer.add_special_case("<mask>", [{ORTH: "<mask>"}])
 
 def set_seed(seed):
     np.random.seed(seed)