Updated so that padding is removed during decode. Fixes: fastai#2910

marii-moe · Nov 3, 2020 · f86d8fa · f86d8fa
1 parent 580742b
commit f86d8fa
Show file tree

Hide file tree

Showing 2 changed files with 313 additions and 54 deletions.
diff --git a/fastai/text/data.py b/fastai/text/data.py
@@ -32,7 +32,7 @@ class LMTensorText(TensorText): pass
 # Cell
 class Numericalize(Transform):
     "Reversible transform of tokenized texts to numericalized ids"
-    def __init__(self, vocab=None, min_freq=3, max_vocab=60000, special_toks=None, pad_tok=None):
+    def __init__(self, vocab=None, min_freq=3, max_vocab=60000, special_toks=None, pad_tok=PAD):
         store_attr('vocab,min_freq,max_vocab,special_toks,pad_tok')
         self.o2i = None if vocab is None else defaultdict(int, {v:k for k,v in enumerate(vocab)})