fix(pipeline): rename dataset_load_raw to load_raw_dataset, add file_…

…pattern and set verbose to false
entelecheia · Jul 28, 2023 · 74cd0c2 · 74cd0c2
1 parent fa0b9be
commit 74cd0c2
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/config/pipeline/datasets.yaml b/config/pipeline/datasets.yaml
@@ -1,6 +1,6 @@
 defaults:
   - __init__
-  - /pipe@pipe_load_raw: dataset_load_raw
+  - /pipe@pipe_load_raw: load_raw_dataset
   - /pipe@pipe_load: load_dataset_from_disk
   - /pipe@pipe_save: save_dataset_to_disk
   - /pipe@pipe_tokenize: tokenize_dataset

diff --git a/src/corprep/conf/run/load_raw_dataset.yaml b/src/corprep/conf/run/load_raw_dataset.yaml
@@ -1,4 +1,5 @@
 _target_: corprep.datasets.io.load_raw_dataset
-raw_dataset_dir:
+raw_dataset_dir: null
 path: json
-verbose: ${oc.select:..verbose, false}
+file_pattern: '*.dat'
+verbose: false