feat(pipeline): add extract tokens step with kakao tokenizer config

entelecheia · Jul 25, 2023 · 04e66d2 · 04e66d2
1 parent 98e0228
commit 04e66d2
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/config/pipeline/datasets-tokenize.yaml b/config/pipeline/datasets-tokenize.yaml
@@ -1,6 +1,7 @@
 defaults:
   - datasets
   - /pipe@pipe_sample: dataset_sample
+  - /pipe@pipe_extract: dataset_extract_tokens
 
 use_task_as_initial_object: true
 steps:
@@ -20,6 +21,13 @@ steps:
       token_col: tokenizedText
       load_from_cache_file: false
       verbose: true
+  - uses: pipe_extract
+    with:
+      tokenizer_config_name: kakao
+      token_col: tokenizedText
+      nouns_only: true
+      load_from_cache_file: false
+      verbose: true
   - uses: pipe_save
     with:
       dataset_path: datasets/processed/kakao_tokenized