feat(corprep): add filter_dataset configuration files

entelecheia · Aug 3, 2023 · 6edb66f · 6edb66f
1 parent 3a2835a
commit 6edb66f
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 0 deletions.
diff --git a/src/corprep/conf/pipe/filter_dataset.yaml b/src/corprep/conf/pipe/filter_dataset.yaml
@@ -0,0 +1,6 @@
+defaults:
+- __general_external_funcs__
+- /run: filter_dataset
+use_pipe_obj: true
+pipe_obj_arg_name: null
+return_pipe_obj: false
diff --git a/src/corprep/conf/run/filter_dataset.yaml b/src/corprep/conf/run/filter_dataset.yaml
@@ -0,0 +1,9 @@
+_target_: corprep.datasets.filter.filter_dataset
+queries: null
+sample_size: 100
+sample_seed: 42
+output_dir: .
+sample_filename: sample.parquet
+train_filename: train.parquet
+discard_filename: discard.parquet
+verbose: false