Merge pull request #13 from hugo-quantmetry/master

fixed tokenizer from conf file
MAIF · Aug 28, 2019 · 038641b · 038641b
2 parents 9a9d54d + 40aef80
commit 038641b
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/melusine/nlp_tools/tokenizer.py b/melusine/nlp_tools/tokenizer.py
@@ -121,7 +121,7 @@ def tokenize(self, row):
     def _tokenize(self, text, pattern=regex_tokenize):
         """Returns list of tokens from text."""
         if isinstance(text, str):
-            tokens = re.findall("\w+(?:[\?\-\"_]\w+)*", text, re.M+re.DOTALL)
+            tokens = re.findall(pattern, text, re.M+re.DOTALL)
             tokens = self._remove_stopwords(tokens)
         else:
             tokens = []