feat(pipeline): create datasets-tokenize.yaml for tokenization in pip…

…eline

config/pipeline/datasets-tokenize.yaml

@@ -0,0 +1,25 @@
+defaults:
+  - datasets
+  - /pipe@pipe_sample: dataset_sample
+
+use_task_as_initial_object: true
+steps:
+  - uses: pipe_load
+    with:
+      dataset_path: datasets/processed/kakao
+      verbose: true
+  - uses: pipe_sample
+    with:
+      num_samples: 11
+      randomize: false
+      verbose: true
+  - uses: pipe_tokenize
+    with:
+      tokenizer_config_name: simple
+      text_col: bodyText
+      token_col: tokenizedText
+      verbose: true
+  - uses: pipe_save
+    with:
+      dataset_path: datasets/processed/kakao_tokenized
+    verbose: true