feat(tokenizer): add strip_pos option to SimpleTokenizer configuration

entelecheia · Jul 23, 2023 · 7fc3991 · 7fc3991
1 parent 4691e71
commit 7fc3991
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/corprep/conf/tokenizer/simple.yaml b/src/corprep/conf/tokenizer/simple.yaml
@@ -2,3 +2,4 @@ defaults:
   - __init__
 
 _target_: corprep.tokenizer.SimpleTokenizer
+strip_pos: true