feat(pipeline): add load from cache option in tokenizer config

entelecheia · Jul 24, 2023 · 9617708 · 9617708
1 parent 82a0198
commit 9617708
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/config/pipeline/datasets-tokenize.yaml b/config/pipeline/datasets-tokenize.yaml
@@ -18,6 +18,7 @@ steps:
       tokenizer_config_name: mecab
       text_col: bodyText
       token_col: tokenizedText
+      load_from_cache_file: false
       verbose: true
   - uses: pipe_save
     with: