feat(corprep): add tokenizer_config_name and token_col to dataset tok…

…enize configuration
entelecheia · Jul 23, 2023 · 4691e71 · 4691e71
1 parent ea92375
commit 4691e71
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/corprep/conf/pipe/dataset_tokenize.yaml b/src/corprep/conf/pipe/dataset_tokenize.yaml
@@ -3,9 +3,11 @@ defaults:
 
 run: corprep.datasets.tokenize.tokenize_dataset
 run_with:
+  tokenizer_config_name: simple
   num_proc: 1
   batched: true
   text_col: bodyText
+  token_col: tokenizedText
   verbose: ${..verbose}
 use_pipe_obj: true
 return_pipe_obj: false