feat(pipeline): add datasets save predictions configuration

entelecheia · Sep 9, 2023 · d29c1d2 · d29c1d2
1 parent bb93396
commit d29c1d2
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/config/pipeline/datasets-save-predictions.yaml b/config/pipeline/datasets-save-predictions.yaml
@@ -0,0 +1,15 @@
+defaults:
+  - datasets
+
+use_task_as_initial_object: true
+steps:
+  - uses: convert_absa_output_to_results
+    with:
+      output_file: datasets/predictions/corprep-gpt4-train_2023/QUAD_revision2_gpt-4_20230905.jsonl
+    verbose: true
+  - uses: records_to_dataframe
+  - uses: save_dataframes
+    with:
+      data_file: datasets/absa_outputs/corprep-gpt4-train_${variables.filter_year}.parquet
+      verbose: true
+    verbose: true