update the result computation

Author: terryyz

analysis/get_results.py

@@ -14,6 +14,7 @@
 from cuml.linear_model import LogisticRegression
 import cupy as cp
 
+
 def update_model_info(model_info):
     for model, info in model_info.items():
         if "https://huggingface.co/" in info["link"]:
@@ -56,8 +57,8 @@ def get_results(tids):
     for model, info in model_info.items():
         model = model.replace("/", "--")
         hf_model = ""
-        files = glob(f"results/{model}--bigcodebench-*.json")
-        assert files, f"No files found for results/{model}--bigcodebench-*.json"
+        files = glob(f"results/{model}--bigcodebench-*_eval_results.json")
+        assert files, f"No files found for results/{model}--bigcodebench-*_eval_results.json"
         for file in files:
             try:
                 _, suffix = os.path.basename(file).split("--bigcodebench-hard-")
@@ -86,7 +87,7 @@ def get_results(tids):
                 raise ValueError("Unknown task")
 
             mode = ""
-            if "-sanitized-calibrate" in file:
+            if "calibrated" in file:
                 mode = "-cal"
 
             results[info["name"]][f"pass@1"][f"{task}{mode}"] = round(mean(status)*100,1)
@@ -141,17 +142,17 @@ def split_gen():
                 if "calibrated" in file:
                     if info["prompted"]:
                         if suffix.startswith("complete"):
-                            with open(f"sanitized_calibrated_samples/complete/{model}--bigcodebench-{suffix}", "w") as f:
+                            with open(f"sanitized_calibrated_samples/complete/{model}--bigcodebench*-{suffix}", "w") as f:
                                 f.writelines(data)
                         else:
-                            with open(f"sanitized_calibrated_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f:
+                            with open(f"sanitized_calibrated_samples/instruct/{model}--bigcodebench*-{suffix}", "w") as f:
                                 f.writelines(data)
                 else:
                     if suffix.startswith("complete"):
-                        with open(f"sanitized_samples/complete/{model}--bigcodebench-{suffix}", "w") as f:
+                        with open(f"sanitized_samples/complete/{model}--bigcodebench*-{suffix}", "w") as f:
                             f.writelines(data)
                     else:
-                        with open(f"sanitized_samples/instruct/{model}--bigcodebench-{suffix}", "w") as f:
+                        with open(f"sanitized_samples/instruct/{model}--bigcodebench*-{suffix}", "w") as f:
                             f.writelines(data)
 
 
@@ -168,7 +169,7 @@ def read_task_perf(tids, task="complete"):
             try:
                 try:
                     if info["prompted"]:
-                        files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_eval_results.json")
+                        files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized*calibrated_eval_results.json")
                         if files:
                             file = files[0]
                         else:
@@ -177,7 +178,7 @@ def read_task_perf(tids, task="complete"):
                         file = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized_eval_results.json")[0]
                 except:
                     if info["prompted"]:# and not info["direct_complete"]:
-                        files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized-calibrated_hard_eval_results.json")
+                        files = glob(f"results/{model}--bigcodebench-{task}*-0-1-sanitized*calibrated_hard_eval_results.json")
                         if files:
                             file = files[0]
                         else:
@@ -187,7 +188,7 @@ def read_task_perf(tids, task="complete"):
             except:
                 try:
                     if info["prompted"]:# and not info["direct_complete"]:
-                        files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized-calibrated_hard_eval_results.json")
+                        files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized*calibrated_hard_eval_results.json")
                         if files:
                             file = files[0]
                         else:
@@ -196,7 +197,7 @@ def read_task_perf(tids, task="complete"):
                         file = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized_hard_eval_results.json")[0]
                 except:
                     if info["prompted"]:
-                        files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized-calibrated_eval_results.json")
+                        files = glob(f"results/{model}--bigcodebench-hard-{task}*-0-1-sanitized*calibrated_eval_results.json")
                         if files:
                             file = files[0]
                         else:
@@ -394,7 +395,7 @@ def get_perf_df(data_dict):
 
 
 if __name__ == "__main__":
-    
+    split_gen()
     bcb_orig = load_dataset("bigcode/bigcodebench", split="v0.1.1")
     bcb_hard = load_dataset("bigcode/bigcodebench-hard", split="v0.1.1")
     bcb_config = {
@@ -408,7 +409,7 @@ def get_perf_df(data_dict):
         instruct_data, instruct_files = read_task_perf(bcb["task_id"], "instruct")
         complete_df = get_perf_df(complete_data)
         instruct_df = get_perf_df(instruct_data)
-        
+
         push_ds(DatasetDict({"complete": Dataset.from_pandas(complete_df), "instruct": Dataset.from_pandas(instruct_df)}), f"bigcode/bigcodebench{suffix}-perf")
 
         with open("task2domain.json", "r") as f: