feat(config): add new configuration files for absa, pipeline, task, a…

…nd workflow

fix(corprep): modify verbose setting and absa_task in default.yaml of absa in corprep
refactor(corprep): remove hyfi_user_config_path and hydra settings in config.yaml and hconf.yaml of corprep
feat(corprep): add new pipeline configurations for absa and datasets in corprep
fix(corprep): adjust pipeline setting in task/absa.yaml and task/datasets.yaml in corprep
refactor(corprep): un-comment ds_task in workflow/corprep.yaml of corprep
chore(files): remove obsolete dataset-test.yaml, absa.yaml, and datasets.yaml files in corprep
chore(files): relocate gpt35.yaml, absa-kakao-gpt35.yaml, absa-kakao.yaml, datasets-daum.yaml, and datasets-kakao.yaml files from corprep to config directory

src/corprep/conf/absa/gpt35.yaml → config/absa/gpt35.yaml

@@ -8,6 +8,4 @@ rate_limit_per_minute: 3500
 absa_task: QUAD
 prompt_name: base
 temperature: 0.0
-output_dir: outputs/preds
-save_filename:
 verbose: true

config/absa/gpt4.yaml

@@ -0,0 +1,10 @@
+defaults:
+  - default
+
+api_key:
+model_name: gpt-3.5-turbo
+rate_limit_per_minute: 3500
+absa_task: QUAD
+prompt_name: base
+temperature: 0.0
+verbose: true

config/pipeline/datasets-test.yaml

@@ -0,0 +1,15 @@
+defaults:
+  - datasets
+  - /pipe@pipe_sample: dataset_sample
+
+use_task_as_initial_object: true
+steps:
+  - uses: pipe_load
+    with:
+      dataset_path: datasets/processed/kakao
+      verbose: true
+  - uses: pipe_sample
+    with:
+      num_samples: 100
+      randomize: false
+      verbose: true

config/task/absa-test.yaml

@@ -0,0 +1,11 @@
+defaults:
+  - __init__
+  - /pipeline@absa_pipeline: absa-kakao
+  - /pipeline@absa_pipeline_gpt35: absa-kakao-gpt35
+
+_config_name_: absa
+task_name: absa
+verbose: true
+pipelines:
+  # - absa_pipeline
+  - absa_pipeline_gpt35

config/task/datasets-test.yaml

@@ -0,0 +1,10 @@
+defaults:
+  - __init__
+  - /pipeline@test_pipeline: datasets-test
+
+_config_name_: datasets-test
+task_name: datasets
+verbose: true
+pipelines:
+  # - ds_pipeline
+  - test_pipeline

config/workflow/absa.yaml

@@ -0,0 +1,8 @@
+# @package _global_
+defaults:
+  - corprep
+- override /task@absa_task: absa-test
+
+tasks:
+  # - ds_task
+  - absa_task

config/workflow/datasets.yaml

@@ -0,0 +1,8 @@
+# @package _global_
+defaults:
+  - corprep
+  - override /task@ds_task: datasets-test
+
+tasks:
+  - ds_task
+  # - absa_task

src/corprep/conf/absa/default.yaml

@@ -5,9 +5,9 @@ _target_: corprep.absa.config.AbsaConfig
 api_key:
 model_name: gpt-4-0613 # gpt-3.5-turbo-0613
 rate_limit_per_minute: 20
-absa_task: TRIPLE
+absa_task: QUAD
 prompt_name: base
 temperature: 0.0
 output_dir: outputs/preds
 save_filename:
-verbose: true
+verbose: false

src/corprep/conf/config.yaml

@@ -8,8 +8,3 @@ defaults:
   - override hydra/help: help
   - override hydra/hydra_logging: colorlog
   - override hydra/job_logging: colorlog
-
-hyfi_user_config_path: ${oc.env:HYFI_USER_CONFIG_PATH,./conf}
-hydra:
-  searchpath:
-    - ${hyfi_user_config_path}

src/corprep/conf/hconf.yaml

@@ -8,8 +8,3 @@ defaults:
   - override hydra/help: help
   - override hydra/hydra_logging: colorlog
   - override hydra/job_logging: colorlog
-
-hyfi_user_config_path: ${oc.env:HYFI_USER_CONFIG_PATH,conf}
-hydra:
-  searchpath:
-    - ${hyfi_user_config_path}

src/corprep/conf/pipeline/absa.yaml

@@ -0,0 +1,33 @@
+defaults:
+  - __init__
+  - /pipe@pipe_load: dataset_load
+  - /pipe@pipe_predict: absa_agent_predict
+  - /pipe@pipe_sample: dataset_sample
+  - /pipe@pipe_save: dataset_save
+
+use_task_as_initial_object: true
+steps:
+  - uses: pipe_load
+    with:
+      dataset_path: datasets/processed/kakao
+      verbose: true
+  - uses: pipe_sample
+    with:
+      num_samples: 100
+      randomize: false
+      verbose: true
+    verbose: true
+  - uses: pipe_predict
+    with:
+      tasks: [QUAD]
+      absa_config_name: default
+      text_col: bodyText
+      batch_size: 5
+      num_workers: 5
+      remove_columns: [lastModifiedDt, bodyText, writers, serviceUrl]
+      load_from_cache_file: false
+      verbose: true
+  - uses: pipe_save
+    with:
+      dataset_path: outputs/preds/kakao_quad_100
+    verbose: true

src/corprep/conf/pipeline/datasets.yaml

@@ -0,0 +1,19 @@
+defaults:
+  - __init__
+  - /pipe@pipe_load_raw: dataset_load_raw
+  - /pipe@pipe_load: dataset_load
+  - /pipe@pipe_save: dataset_save
+
+use_task_as_initial_object: true
+steps:
+  - uses: pipe_load_raw
+    with:
+      raw_dataset_dir: workspace/datasets/raw/kakao_news
+      path: parquet
+      file_pattern: "*.parquet"
+      verbose: true
+    verbose: true
+  - uses: pipe_save
+    with:
+      dataset_path: datasets/processed/kakao
+    verbose: true

src/corprep/conf/task/absa.yaml

@@ -1,11 +1,9 @@
 defaults:
   - __init__
-  - /pipeline@absa_pipeline: absa-kakao
-  - /pipeline@absa_pipeline_gpt35: absa-kakao-gpt35
+  - /pipeline@absa_pipeline: absa
 
 _config_name_: absa
 task_name: absa
 verbose: true
 pipelines:
-  # - absa_pipeline
-  - absa_pipeline_gpt35
+  - absa_pipeline

src/corprep/conf/task/datasets.yaml

@@ -7,5 +7,5 @@ _config_name_: datasets
 task_name: datasets
 verbose: true
 pipelines:
-  # - ds_pipeline
+  - ds_pipeline
   - test_pipeline

src/corprep/conf/workflow/corprep.yaml

@@ -8,5 +8,5 @@ defaults:
 name: corprep
 verbose: true
 tasks:
-  # - ds_task
+  - ds_task
   - absa_task