From a6565e570489137fcc2fc1c9ddd22a69ac8d0335 Mon Sep 17 00:00:00 2001
From: Fanjia Yan <fanjiayan@berkeley.edu>
Date: Sun, 10 Mar 2024 16:39:13 -0700
Subject: [PATCH] [Bug Fix] Executable Checker

---
 .../openfunctions_executable_checker.py               | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/berkeley-function-call-leaderboard/openfunctions_executable_checker.py b/berkeley-function-call-leaderboard/openfunctions_executable_checker.py
index 076ff9877..32dbf7cb3 100644
--- a/berkeley-function-call-leaderboard/openfunctions_executable_checker.py
+++ b/berkeley-function-call-leaderboard/openfunctions_executable_checker.py
@@ -51,7 +51,10 @@ def convert_to_function_call(data_str):
         params_str = params_str.replace("'", '"')
         params_str = params_str.replace("\\" + "n", "")
         # Step 6: Load the parameters string as a dictionary
-        params = json.loads(params_str)
+        try:
+            params = json.loads(params_str)
+        except:
+            continue
         function_string = func_name + "("
         for k,v in params.items():
             if isinstance(v, str):
@@ -117,6 +120,8 @@ def convert_to_function_call(data_str):
         execution_result_type = testing_data[i]["execution_result_type"]
         if type(execution_result_type) is str and len(execution_result) > 1:
                 execution_result_type = [execution_result_type] * len(execution_result)
+        elif type(execution_result_type) is str and len(execution_result) == 1:
+            execution_result_type = [execution_result_type]
     if ("gpt" in model_name or "fire" in model_name or "mistral-large-latest" in model_name) and input_file is None:
         try:
             result = convert_to_function_call(result_data[i]["result"])
@@ -229,7 +234,7 @@ def convert_to_function_call(data_str):
     if len(result) != len(execution_result):
         total += 1
         continue
-    for function_call, exec_result in zip(result, execution_result):
+    for function_call, exec_result, exec_type in zip(result, execution_result, execution_result_type):
         try:
             exec(
                 "from data.function.gorilla_openfunctions_v1_test_function import *"
@@ -242,7 +247,7 @@ def convert_to_function_call(data_str):
             test_result_list.append(False)
             reason_list.append(str(e))
             continue
-        if execution_result_type == "exact_match":
+        if exec_type == "exact_match":
             test_result_list.append(output == exec_result or [output] == exec_result)
         else:
             try: