From ca9e6b06f67bb9211962f5c954dc089ad7343343 Mon Sep 17 00:00:00 2001 From: lucasiscovici Date: Thu, 14 Dec 2023 17:05:28 +0100 Subject: [PATCH 1/5] Updated NLI Verdicts to Binary Format this commit changes the verdict system in our natural language inference module from 'Yes/No' to a binary '1/0' format. This modification aims to streamline decision-making and ensure consistency across various languages, enhancing the system's efficiency and reliability in multilingual contexts --- src/ragas/metrics/_faithfulness.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ragas/metrics/_faithfulness.py b/src/ragas/metrics/_faithfulness.py index 3cd5c1e7c..b67daf2dc 100644 --- a/src/ragas/metrics/_faithfulness.py +++ b/src/ragas/metrics/_faithfulness.py @@ -53,7 +53,7 @@ NLI_STATEMENTS_MESSAGE = HumanMessagePromptTemplate.from_template( """ - Natural language inference. Only use "Yes" or "No" as verdict. + Natural language inference. Use only "Yes" (1) or "No" (0) as a binary verdict. Context: John is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects. @@ -66,22 +66,22 @@ {{ "statement_1": "John is majoring in Biology.", "reason": "John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.", - "verdict": "No" + "verdict": "0" }}, {{ "statement_2": "John is taking a course on Artificial Intelligence.", "reason": "The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI.", - "verdict": "No" + "verdict": "0" }}, {{ "statement_3": "John is a dedicated student.", "reason": "The context states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication.", - "verdict": "Yes" + "verdict": "1" }}, {{ "statement_4": "John has a part-time job.", "reason": "There is no information given in the context about John having a part-time job.", - "verdict": "No" + "verdict": "0" }} ] @@ -93,7 +93,7 @@ {{ "statement_1": "Albert Einstein was a genius.", "reason": "The context and statement are unrelated" - "verdict": "No" + "verdict": "0" }} ] @@ -105,7 +105,7 @@ {{ "statement_1": "Nil", "reason": "The statement is invalid", - "verdict": "No" + "verdict": "0" }} ] @@ -169,7 +169,7 @@ def _score_batch( result = self.llm.generate(prompts, callbacks=batch_group) outputs = result.generations - verdict_score_map = {"yes": 1, "no": 0, "null": np.nan} + verdict_score_map = {"1": 1, "0": 0, "null": np.nan} scores = [] for output in outputs: output = json_loader.safe_load(output[0].text, self.llm) From 89d01adb5fd867ed82055af82d6c730709e4df3e Mon Sep 17 00:00:00 2001 From: lucasiscovici Date: Thu, 14 Dec 2023 18:20:26 +0100 Subject: [PATCH 2/5] Updated NLI Verdicts to Binary Format --- src/ragas/metrics/_context_precision.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ragas/metrics/_context_precision.py b/src/ragas/metrics/_context_precision.py index 328623338..68641c956 100644 --- a/src/ragas/metrics/_context_precision.py +++ b/src/ragas/metrics/_context_precision.py @@ -16,19 +16,19 @@ CONTEXT_PRECISION = HumanMessagePromptTemplate.from_template( """\ -Verify if the information in the given context is useful in answering the question. +Verify if the information in the given context is useful in answering the question. Use only "Yes" (1) or "No" (0) as a binary verdict. question: What are the health benefits of green tea? context: This article explores the rich history of tea cultivation in China, tracing its roots back to the ancient dynasties. It discusses how different regions have developed their unique tea varieties and brewing techniques. The article also delves into the cultural significance of tea in Chinese society and how it has become a symbol of hospitality and relaxation. verification: -{{"reason":"The context, while informative about the history and cultural significance of tea in China, does not provide specific information about the health benefits of green tea. Thus, it is not useful for answering the question about health benefits.", "verdict":"No"}} +{{"reason":"The context, while informative about the history and cultural significance of tea in China, does not provide specific information about the health benefits of green tea. Thus, it is not useful for answering the question about health benefits.", "verdict":"0"}} question: How does photosynthesis work in plants? context: Photosynthesis in plants is a complex process involving multiple steps. This paper details how chlorophyll within the chloroplasts absorbs sunlight, which then drives the chemical reaction converting carbon dioxide and water into glucose and oxygen. It explains the role of light and dark reactions and how ATP and NADPH are produced during these processes. verification: -{{"reason":"This context is extremely relevant and useful for answering the question. It directly addresses the mechanisms of photosynthesis, explaining the key components and processes involved.", "verdict":"Yes"}} +{{"reason":"This context is extremely relevant and useful for answering the question. It directly addresses the mechanisms of photosynthesis, explaining the key components and processes involved.", "verdict":"1"}} question:{question} context: @@ -98,7 +98,7 @@ def _score_batch( json_loader.safe_load(item, self.llm) for item in sum(response, []) ] response = [ - int("yes" in resp.get("verdict", " ").lower()) + "1" == resp.get("verdict", " ").strip() if resp.get("verdict") else np.nan for resp in response From 73ccff3419bbadce68a28107016d12931b99e53b Mon Sep 17 00:00:00 2001 From: lucasiscovici Date: Thu, 14 Dec 2023 18:23:26 +0100 Subject: [PATCH 3/5] Updated NLI Verdicts to Binary Format --- src/ragas/metrics/_context_recall.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ragas/metrics/_context_recall.py b/src/ragas/metrics/_context_recall.py index 79f684d57..22ee2b928 100644 --- a/src/ragas/metrics/_context_recall.py +++ b/src/ragas/metrics/_context_recall.py @@ -16,7 +16,7 @@ CONTEXT_RECALL_RA = HumanMessagePromptTemplate.from_template( """ -Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Output json with reason. +Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Use only "Yes" (1) or "No" (0) as a binary classification. Output json with reason. question: What can you tell me about albert Albert Einstein? @@ -26,22 +26,22 @@ [ {{ "statement_1":"Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time.", "reason": "The date of birth of Einstein is mentioned clearly in the context.", - "Attributed": "Yes" + "Attributed": "1" }}, {{ "statement_2":"He received the 1921 Nobel Prize in Physics 'for his services to theoretical physics.", "reason": "The exact sentence is present in the given context.", - "Attributed": "Yes" + "Attributed": "1" }}, {{ "statement_3": "He published 4 papers in 1905.", "reason": "There is no mention about papers he wrote in the given context.", - "Attributed": "No" + "Attributed": "0" }}, {{ "statement_4":"Einstein moved to Switzerland in 1895.", "reason": "There is no supporting evidence for this in the given context.", - "Attributed": "No" + "Attributed": "0" }} ] @@ -54,7 +54,7 @@ {{ "statement_1":"England won the 2022 ICC Men's T20 World Cup.", "reason": "From context it is clear that England defeated Pakistan to win the World Cup.", - "Attributed": "Yes" + "Attributed": "1" }} ] @@ -121,7 +121,7 @@ def _score_batch( response = json_loader.safe_load(response[0], self.llm) if response: response = [ - int(item.get("Attributed", "").lower() == "yes") + int(item.get("Attributed", "0").strip() == "1") if item.get("Attributed") else np.nan for item in response From e65cabdd0abfe38930f57a259f121f6bbbd9322c Mon Sep 17 00:00:00 2001 From: lucasiscovici Date: Thu, 14 Dec 2023 18:25:45 +0100 Subject: [PATCH 4/5] int --- src/ragas/metrics/_context_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ragas/metrics/_context_precision.py b/src/ragas/metrics/_context_precision.py index 68641c956..67aae5300 100644 --- a/src/ragas/metrics/_context_precision.py +++ b/src/ragas/metrics/_context_precision.py @@ -98,7 +98,7 @@ def _score_batch( json_loader.safe_load(item, self.llm) for item in sum(response, []) ] response = [ - "1" == resp.get("verdict", " ").strip() + int("1" == resp.get("verdict", " ").strip()) if resp.get("verdict") else np.nan for resp in response From 349e19ff1b8f2404ecd02986a3bf37cea32b479c Mon Sep 17 00:00:00 2001 From: lucasiscovici Date: Thu, 14 Dec 2023 18:26:35 +0100 Subject: [PATCH 5/5] default verdict to 0 --- src/ragas/metrics/_context_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ragas/metrics/_context_precision.py b/src/ragas/metrics/_context_precision.py index 67aae5300..28152b36f 100644 --- a/src/ragas/metrics/_context_precision.py +++ b/src/ragas/metrics/_context_precision.py @@ -98,7 +98,7 @@ def _score_batch( json_loader.safe_load(item, self.llm) for item in sum(response, []) ] response = [ - int("1" == resp.get("verdict", " ").strip()) + int("1" == resp.get("verdict", "0").strip()) if resp.get("verdict") else np.nan for resp in response