feat(pipeline): add new dataset filter and load steps

config/pipeline/datasets-filter.yaml

@@ -0,0 +1,21 @@
+defaults:
+  - datasets
+
+use_task_as_initial_object: true
+steps:
+  - uses: pipe_load_dfs
+    with:
+      data_files: datasets/processed/kakao_nouns_similar_6.parquet
+      split: train
+      verbose: true
+  - uses: pipe_head
+    verbose: true
+  - uses: pipe_filter
+    with:
+      queries:
+        - "duplicate == False"
+        - "cleaned_text.str.split().str.len() > 15"
+      sample_size: 100
+      sample_seed: 123
+      output_dir: datasets/filtered/kakao_deduped
+      verbose: true

config/pipeline/datasets.yaml

@@ -10,6 +10,8 @@ defaults:
   - /pipe@pipe_head: pandas_print_head
   - /pipe@pipe_save_df: save_dataframes
   - /pipe@pipe_find_similar: find_similar_docs_ac
+  - /pipe@pipe_filter: filter_dataset
+  - /pipe@pipe_load_dfs: load_dataframes
 
 use_task_as_initial_object: true
 steps: