From cc2d19d5ac175652e12cd2638abff3be72c90684 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Wed, 30 Aug 2023 09:14:36 +0530 Subject: [PATCH 1/4] limit score to 1 --- src/ragas/metrics/context_relevance.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ragas/metrics/context_relevance.py b/src/ragas/metrics/context_relevance.py index a016e97e6..288a80dd3 100644 --- a/src/ragas/metrics/context_relevance.py +++ b/src/ragas/metrics/context_relevance.py @@ -159,13 +159,9 @@ def _score_batch( overlap_scores = [] context_sents = sent_tokenize(context) for output in n_response: - indices = [ - context.find(sent) - for sent in sent_tokenize(output) - if context.find(sent) != -1 - ] indices = sent_tokenize(output) - overlap_scores.append(len(indices) / len(context_sents)) + score = min(len(indices) / len(context_sents), 1) + overlap_scores.append(score) if self.strictness > 1: agr_score = self.sent_agreement.evaluate(n_response) else: From 12e277674a8f2465d1313c13d5b6ff31d78cc8b4 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Tue, 5 Sep 2023 12:08:23 +0530 Subject: [PATCH 2/4] improve context relevancy --- src/ragas/metrics/context_relevance.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/ragas/metrics/context_relevance.py b/src/ragas/metrics/context_relevance.py index 288a80dd3..7205d385d 100644 --- a/src/ragas/metrics/context_relevance.py +++ b/src/ragas/metrics/context_relevance.py @@ -16,25 +16,11 @@ CONTEXT_RELEVANCE = HumanMessagePromptTemplate.from_template( """\ -Task: Candidate sentence extraction. -Given the question and context, extract minimum number of sentences from context required to answer the question. If the context do not contain information required to answer the question return "No candidate sentences found". - -question: Which equation is known as worlds most famous equation? -context:\nAlbert Einstein (14 March 1879 – 18 April 1955) was a German-born theoretical physicist,[5] widely ranked among the greatest and most influential scientists of all time. Best known for developing the theory of relativity, he also made important contributions to quantum mechanics, and was thus a central figure in the revolutionary reshaping of the scientific understanding of nature that modern physics accomplished in the first decades of the twentieth century. -His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called "the world's most famous equation". -sentences:His mass–energy equivalence formula E = mc2, which arises from relativity theory, has been called "the world's most famous equation". - -question: Were Scott Derrickson and Ed Wood of the same nationality? -context :\nScott Derrickson (born July 16, 1966) is an American director, screenwriter and producer He lives in Los Angeles, California He is best known for directing horror films such as "Sinister", "The Exorcism of Emily Rose", and "Deliver Us From Evil", as well as the 2016 Marvel Cinematic Universe installment, "Doctor Strange"Tyler Bates is an American musician, music producer, and composer for films, television, and video games. Adam Collis is an American filmmaker and actor.Conrad Brooks is an American actor.Edward Davis Wood Jr. (October 10, 1924 – December 10, 1978) was an American filmmaker, actor, writer, producer, and director. -sentences:Scott Derrickson (born July 16, 1966) is an American director, screenwriter and producer. Edward Davis Wood Jr. (October 10, 1924 – December 10, 1978) was an American filmmaker, actor, writer, producer, and director. - -question: How many were killed in the Tiananmen Square incident? -context:\nTiananmen Square incident, also called June Fourth incident or 6/4, series of protests and demonstrations in China in the spring of 1989 that culminated on the night of June 3–4 with a government crackdown on the demonstrators in Tiananmen Square in Beijing. -sentences: No candidate sentences found. +Please extract relevant sentences from the provided context that can potentially help answer the following question. If no relevant sentences are found, or if you believe the question cannot be answered from the given context, return the phrase "Insufficient Information". While extracting candidate sentences you're not allowed to make any changes to sentences from given context. question:{question} context:\n{context} -sentences:""" # noqa: E501 +candidate sentences:\n""" # noqa: E501 ) @@ -110,7 +96,7 @@ class ContextRelevancy(MetricWithLLM): name: str = "context_ relevancy" evaluation_mode: EvaluationMode = EvaluationMode.qc batch_size: int = 15 - strictness: int = 2 + strictness: int = 1 agreement_metric: str = "bert_score" model_name: str = "cross-encoder/stsb-TinyBERT-L-4" @@ -159,7 +145,11 @@ def _score_batch( overlap_scores = [] context_sents = sent_tokenize(context) for output in n_response: - indices = sent_tokenize(output) + indices = ( + output.split("\n") + if output.lower() != "insufficient information" + else [] + ) score = min(len(indices) / len(context_sents), 1) overlap_scores.append(score) if self.strictness > 1: From 31145d8f7d9812560c4faa6cee741c4c0813d08e Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Tue, 5 Sep 2023 12:14:02 +0530 Subject: [PATCH 3/4] change strictness to 2 --- src/ragas/metrics/context_relevance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ragas/metrics/context_relevance.py b/src/ragas/metrics/context_relevance.py index 7205d385d..35a74eb7a 100644 --- a/src/ragas/metrics/context_relevance.py +++ b/src/ragas/metrics/context_relevance.py @@ -96,7 +96,7 @@ class ContextRelevancy(MetricWithLLM): name: str = "context_ relevancy" evaluation_mode: EvaluationMode = EvaluationMode.qc batch_size: int = 15 - strictness: int = 1 + strictness: int = 2 agreement_metric: str = "bert_score" model_name: str = "cross-encoder/stsb-TinyBERT-L-4" From e76d93da4967ec176c4b24501a43351ab8d62f4a Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Wed, 6 Sep 2023 09:27:06 +0530 Subject: [PATCH 4/4] fix typo --- src/ragas/metrics/context_relevance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ragas/metrics/context_relevance.py b/src/ragas/metrics/context_relevance.py index 35a74eb7a..d8130ec4c 100644 --- a/src/ragas/metrics/context_relevance.py +++ b/src/ragas/metrics/context_relevance.py @@ -147,7 +147,7 @@ def _score_batch( for output in n_response: indices = ( output.split("\n") - if output.lower() != "insufficient information" + if output.lower() != "insufficient information." else [] ) score = min(len(indices) / len(context_sents), 1)