From a34447b02451f0355e48f8de77a1d9cba80e0439 Mon Sep 17 00:00:00 2001 From: Billy Hu Date: Mon, 10 Jun 2024 16:37:53 -0700 Subject: [PATCH] Handle the error case --- .../evals/evaluate/_code_client/proxy_client.py | 10 ++++++++-- .../tests/evals/unittests/test_evaluate.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/promptflow-evals/promptflow/evals/evaluate/_code_client/proxy_client.py b/src/promptflow-evals/promptflow/evals/evaluate/_code_client/proxy_client.py index 044a4628225..d1c1b054b2a 100644 --- a/src/promptflow-evals/promptflow/evals/evaluate/_code_client/proxy_client.py +++ b/src/promptflow-evals/promptflow/evals/evaluate/_code_client/proxy_client.py @@ -3,6 +3,8 @@ # --------------------------------------------------------- import logging +import numpy as np + from promptflow.client import PFClient from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor @@ -20,9 +22,13 @@ def __init__(self, pf_client: PFClient): self._thread_pool = ThreadPoolExecutor(thread_name_prefix="evaluators_thread") def run(self, flow, data, column_mapping=None, **kwargs): - eval_future = self._thread_pool.submit(self._pf_client.run, flow, data=data, column_mapping=column_mapping) + eval_future = self._thread_pool.submit( + self._pf_client.run, flow, data=data, column_mapping=column_mapping, **kwargs + ) return ProxyRun(run=eval_future) def get_details(self, proxy_run, all_results=False): run = proxy_run.run.result(timeout=60 * 60) - return self._pf_client.get_details(run, all_results=all_results) + result_df = self._pf_client.get_details(run, all_results=all_results) + result_df.replace("(Failed)", np.nan, inplace=True) + return result_df diff --git a/src/promptflow-evals/tests/evals/unittests/test_evaluate.py b/src/promptflow-evals/tests/evals/unittests/test_evaluate.py index d28384f5dc6..17dfd141daf 100644 --- a/src/promptflow-evals/tests/evals/unittests/test_evaluate.py +++ b/src/promptflow-evals/tests/evals/unittests/test_evaluate.py @@ -371,7 +371,7 @@ def test_evaluate_output_path(self, evaluate_test_data_jsonl_file, tmpdir, use_t def test_evaluate_with_errors(self): """Test evaluate_handle_errors""" data = _get_file("yeti_questions.jsonl") - result = evaluate(data=data, evaluators={"yeti": _yeti_evaluator}, _use_thread_pool=True) + result = evaluate(data=data, evaluators={"yeti": _yeti_evaluator}) result_df = pd.DataFrame(result["rows"]) expected = pd.read_json(data, lines=True) expected.rename(columns={"question": "inputs.question", "answer": "inputs.answer"}, inplace=True)