Merge pull request #68 from vecto-ai/cli-bofang

Cli bofang

vecto/benchmarks/categorization/categorization.py

@@ -7,6 +7,7 @@
 import csv
 import numpy as np
 from scipy.spatial import distance
+import os
 
 OTHER_EXT = 'None'
 BENCHMARK = 'benchmark'
@@ -106,7 +107,8 @@ def evaluate(self, embs, data):
         # add experiment_setup and result entry for result
         result["experiment_setup"] = {}
         result["result"] = result['global_stats']['scores']
-        result["experiment_setup"]['default_measurement'] = {'Purity'}
+        result["experiment_setup"]['default_measurement'] = 'Purity'
+
         return result
 
     def read_datasets_from_dir(self, path_to_dir):
@@ -127,6 +129,9 @@ def run(self, embs, path_dataset):
         datasets = self.read_datasets_from_dir(path_dataset)
         for dataset_name, dataset_data in datasets.items():
             result = self.evaluate(embs, dataset_data)
+            result['experiment_setup']['dataset'] = os.path.basename(os.path.normpath(path_dataset))
+            result['experiment_setup']['embeddings'] = embs.metadata
+            result['experiment_setup']['method'] = self.method
             results.append(result)
         return results
 

vecto/benchmarks/visualize.py

@@ -22,7 +22,7 @@ def clean_dic(data):
     return data_clean
 
 
-def df_from_file(path):
+def df_from_file_bak(path):
     logger.debug(f"processing {path}")
     data = load_json(path)
     data_clean = [clean_dic(x) for x in data]
@@ -43,6 +43,24 @@ def df_from_file(path):
     # df["reciprocal_rank"] = 1 / (df["rank"] + 1)
     return dframe
 
+def df_from_file(path):
+    data = load_json(path)
+    meta = [["experiment_setup", "task"],
+            ["experiment_setup", "subcategory"],
+            ["experiment_setup", "method"],
+            ["experiment_setup", "embeddings"]]
+    dframe = json_normalize(data, meta=meta)
+    if "details" in dframe:
+        dframe.drop("details", axis="columns", inplace=True)
+    default_measurement = "accuracy"
+    try:
+        default_measurement = dframe["experiment_setup.default_measurement"].unique()[0]
+    except:
+        logger.warning(f"default_measurement not specified in {path}")
+    dframe["result"] = dframe["result." + default_measurement]
+    # df["reciprocal_rank"] = 1 / (df["rank"] + 1)
+    return dframe
+
 
 def df_from_dir(path):
     dfs = []