feat(corprep): add new file for tokenizing dataset

entelecheia · Jul 17, 2023 · 6fb827e · 6fb827e
1 parent 2db1824
commit 6fb827e
Showing 1 changed file with 11 additions and 0 deletions.
diff --git a/src/corprep/conf/pipe/tokenize_dataset.yaml b/src/corprep/conf/pipe/tokenize_dataset.yaml
@@ -0,0 +1,11 @@
+defaults:
+  - __general_external_funcs__
+
+run: corprep.datasets.preprocessing.tokenize.tokenize_dataset
+run_with:
+  num_proc: 1
+  batched: true
+  text_col: bodyText
+  verbose: false
+use_pipe_obj: true
+return_pipe_obj: false