feat(tokenizer): add flatten option to MecabTokenizer config

entelecheia · Jul 23, 2023 · 070740b · 070740b
1 parent 191a5fd
commit 070740b
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/corprep/conf/tokenizer/mecab.yaml b/src/corprep/conf/tokenizer/mecab.yaml
@@ -3,6 +3,7 @@ defaults:
   - tagger: mecab
 
 _target_: corprep.tokenizer.MecabTokenizer
+flatten: true
 noun_postags:
   - NNG
   - NNP