use const for tokenizer regex

vecto-ai · Apr 25, 2018 · 0749248 · 0749248
1 parent 991db74
commit 0749248
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/vecto/corpus/corpus.py b/vecto/corpus/corpus.py
@@ -58,7 +58,7 @@ def gen(self, re_pattern):
                     yield(token)
 
 
-def load_file_as_ids(path, vocabulary, gzipped=None, downcase=True, re_pattern=r"[\w\-']+|[.,!?…]"):
+def load_file_as_ids(path, vocabulary, gzipped=None, downcase=True, re_pattern=_default_tokenizer_patter):
     # use proper tokenizer from cooc
     # options to ignore sentence bounbdaries
     # specify what to do with missing words