feat(corprep): add find_similar_docs configuration

entelecheia · Jul 27, 2023 · c468597 · c468597
1 parent 3710f93
commit c468597
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 0 deletions.
diff --git a/src/corprep/conf/pipe/find_similar_docs.yaml b/src/corprep/conf/pipe/find_similar_docs.yaml
@@ -0,0 +1,6 @@
+defaults:
+  - __general_external_funcs__
+  - /run: find_similar_docs
+
+use_pipe_obj: true
+return_pipe_obj: false
diff --git a/src/corprep/conf/run/find_similar_docs.yaml b/src/corprep/conf/run/find_similar_docs.yaml
@@ -0,0 +1,18 @@
+_target_: corprep.datasets.similarity.find_similar_docs
+num_workers: 2
+min_num_docs: 5
+percentile: 80
+distance_threshold:
+linkage: average
+grouping_freq: W
+grouping_name: Week
+date_col: createdDt
+token_col: nouns
+id_col: newsId
+ordering_col: createdDt_int
+duplicate_col: duplicate
+fig_col: fig_filename
+output_dir: .
+show_fig: false
+save_fig: false
+verbose: false