From 753978e3db5faf3b7beeba2d95aae109e7f4a331 Mon Sep 17 00:00:00 2001 From: Kumar Anirudha Date: Mon, 6 Oct 2025 21:14:33 +0530 Subject: [PATCH] fix: correct noncommittal detection and filter empty questions in answer_relevancy --- src/ragas/metrics/_answer_relevance.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/ragas/metrics/_answer_relevance.py b/src/ragas/metrics/_answer_relevance.py index 42be40afb..5661304f5 100644 --- a/src/ragas/metrics/_answer_relevance.py +++ b/src/ragas/metrics/_answer_relevance.py @@ -116,15 +116,21 @@ def _calculate_score( ) -> float: question = row["user_input"] gen_questions = [answer.question for answer in answers] - all_noncommittal = np.all([answer.noncommittal for answer in answers]) - if all(q == "" for q in gen_questions): - logger.warning( - "Invalid JSON response. Expected dictionary with key 'question'" - ) - score = np.nan + any_noncommittal = np.any([answer.noncommittal for answer in answers]) + + if any_noncommittal: + score = 0.0 else: - cosine_sim = self.calculate_similarity(question, gen_questions) - score = cosine_sim.mean() * int(not all_noncommittal) + valid_questions = [q for q in gen_questions if q.strip() != ""] + + if len(valid_questions) == 0: + logger.warning( + "No valid questions generated. All questions were empty." + ) + score = np.nan + else: + cosine_sim = self.calculate_similarity(question, valid_questions) + score = cosine_sim.mean() return score