rebase updates

llm_demo/evaluation/__init__.py

@@ -13,6 +13,6 @@
 # limitations under the License.
 
 from .eval_golden import goldens
-from .evaluation import run_llm_for_eval
+from .evaluation import evaluate_retrieval_phase, run_llm_for_eval
 
-__ALL__ = ["run_llm_for_eval", "goldens"]
+__ALL__ = ["run_llm_for_eval", "goldens", "evaluate_retrieval_phase"]

llm_demo/evaluation/eval_golden.py

@@ -35,6 +35,7 @@ class EvalData(BaseModel):
         default=True, description="determine to reset the chat after invoke"
     )
 
+
 goldens = [
     EvalData(
         category="Search Airport Tool",

llm_demo/evaluation/evaluation.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import asyncio
 import json
 from typing import Dict, List
 
@@ -35,6 +36,7 @@ async def run_llm_for_eval(
     agent = orc.get_user_session(session_id)
     for eval_data in eval_list:
         query_response = await agent.invoke(eval_data.query)
+
         # Retrieve prediction_tool_calls from query response
         prediction_tool_calls = []
         for step in query_response.get("intermediate_steps"):
@@ -52,6 +54,7 @@ async def run_llm_for_eval(
             orc.user_session_reset(session, session_id)
     return eval_list
 
+
 def evaluate_task(
     eval_dataset: "pd.DataFrame", metrics: List[str], experiment_name: str
 ) -> evaluation_base.EvalResult:
@@ -64,22 +67,20 @@ def evaluate_task(
         metrics=metrics,
         experiment=experiment_name,
     )
+
     eval_result = eval_task.evaluate()
     return eval_result
 
+
 def evaluate_retrieval_phase(eval_datas: List[EvalData]) -> evaluation_base.EvalResult:
     RETRIEVAL_EXPERIMENT_NAME = "retrieval-phase-eval"
     metrics = ["tool_call_quality"]
     responses = []
     references = []
     for e in eval_datas:
-        responses.append(
-            json.dumps({"content": e.content, "tool_calls": e.tool_calls})
-        )
+        responses.append(json.dumps({"content": e.content, "tool_calls": e.tool_calls}))
         references.append(
-            json.dumps(
-                {"content": e.content, "tool_calls": e.prediction_tool_calls}
-            )
+            json.dumps({"content": e.content, "tool_calls": e.prediction_tool_calls})
         )
     eval_dataset = pd.DataFrame(
         {