Updated NLI Verdicts to Binary Format

src/ragas/metrics/_context_precision.py

@@ -16,19 +16,19 @@
 
 CONTEXT_PRECISION = HumanMessagePromptTemplate.from_template(
     """\
-Verify if the information in the given context is useful in answering the question.
+Verify if the information in the given context is useful in answering the question. Use only "Yes" (1) or "No" (0) as a binary verdict.
 
 question: What are the health benefits of green tea?
 context: 
 This article explores the rich history of tea cultivation in China, tracing its roots back to the ancient dynasties. It discusses how different regions have developed their unique tea varieties and brewing techniques. The article also delves into the cultural significance of tea in Chinese society and how it has become a symbol of hospitality and relaxation.
 verification:
-{{"reason":"The context, while informative about the history and cultural significance of tea in China, does not provide specific information about the health benefits of green tea. Thus, it is not useful for answering the question about health benefits.", "verdict":"No"}}
+{{"reason":"The context, while informative about the history and cultural significance of tea in China, does not provide specific information about the health benefits of green tea. Thus, it is not useful for answering the question about health benefits.", "verdict":"0"}}
 
 question: How does photosynthesis work in plants?
 context:
 Photosynthesis in plants is a complex process involving multiple steps. This paper details how chlorophyll within the chloroplasts absorbs sunlight, which then drives the chemical reaction converting carbon dioxide and water into glucose and oxygen. It explains the role of light and dark reactions and how ATP and NADPH are produced during these processes.
 verification:
-{{"reason":"This context is extremely relevant and useful for answering the question. It directly addresses the mechanisms of photosynthesis, explaining the key components and processes involved.", "verdict":"Yes"}}
+{{"reason":"This context is extremely relevant and useful for answering the question. It directly addresses the mechanisms of photosynthesis, explaining the key components and processes involved.", "verdict":"1"}}
 
 question:{question}
 context:
@@ -98,7 +98,7 @@ def _score_batch(
                     json_loader.safe_load(item, self.llm) for item in sum(response, [])
                 ]
                 response = [
-                    int("yes" in resp.get("verdict", " ").lower())
+                    int("1" == resp.get("verdict", "0").strip())
                     if resp.get("verdict")
                     else np.nan
                     for resp in response

src/ragas/metrics/_context_recall.py

@@ -16,7 +16,7 @@
 
 CONTEXT_RECALL_RA = HumanMessagePromptTemplate.from_template(
     """
-Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Output json with reason.
+Given a context, and an answer, analyze each sentence in the answer and classify if the sentence can be attributed to the given context or not. Use only "Yes" (1) or "No" (0) as a binary classification. Output json with reason.
 
 
 question: What can you tell me about albert Albert Einstein?
@@ -26,22 +26,22 @@
 [
     {{  "statement_1":"Albert Einstein, born on 14 March 1879, was a German-born theoretical physicist, widely held to be one of the greatest and most influential scientists of all time.",
         "reason": "The date of birth of Einstein is mentioned clearly in the context.",
-        "Attributed": "Yes"
+        "Attributed": "1"
     }},
     {{
         "statement_2":"He received the 1921 Nobel Prize in Physics 'for his services to theoretical physics.",
         "reason": "The exact sentence is present in the given context.",
-        "Attributed": "Yes"
+        "Attributed": "1"
     }},
     {{
         "statement_3": "He published 4 papers in 1905.",
         "reason": "There is no mention about papers he wrote in the given context.",
-        "Attributed": "No"
+        "Attributed": "0"
     }},
     {{
         "statement_4":"Einstein moved to Switzerland in 1895.",
         "reason": "There is no supporting evidence for this in the given context.",
-        "Attributed": "No"
+        "Attributed": "0"
     }}
 ]
 
@@ -54,7 +54,7 @@
     {{
         "statement_1":"England won the 2022 ICC Men's T20 World Cup.",
         "reason": "From context it is clear that England defeated Pakistan to win the World Cup.",
-         "Attributed": "Yes"
+         "Attributed": "1"
     }}
 ]
 
@@ -121,7 +121,7 @@ def _score_batch(
                 response = json_loader.safe_load(response[0], self.llm)
                 if response:
                     response = [
-                        int(item.get("Attributed", "").lower() == "yes")
+                        int(item.get("Attributed", "0").strip() == "1")
                         if item.get("Attributed")
                         else np.nan
                         for item in response

src/ragas/metrics/_faithfulness.py

@@ -53,7 +53,7 @@
 
 NLI_STATEMENTS_MESSAGE = HumanMessagePromptTemplate.from_template(
     """
- Natural language inference. Only use "Yes" or "No" as verdict.
+ Natural language inference. Use only "Yes" (1) or "No" (0) as a binary verdict.
 
 Context:
 John is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects.
@@ -66,22 +66,22 @@
     {{
         "statement_1": "John is majoring in Biology.",
         "reason": "John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.",
-        "verdict": "No"
+        "verdict": "0"
     }},
     {{
         "statement_2": "John is taking a course on Artificial Intelligence.",
         "reason": "The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI.",
-        "verdict": "No"
+        "verdict": "0"
     }},
     {{
         "statement_3": "John is a dedicated student.",
         "reason": "The context states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication.",
-        "verdict": "Yes"
+        "verdict": "1"
     }},
     {{
         "statement_4": "John has a part-time job.",
         "reason": "There is no information given in the context about John having a part-time job.",
-        "verdict": "No"
+        "verdict": "0"
     }}
 ]
 
@@ -93,7 +93,7 @@
      {{
         "statement_1": "Albert Einstein was a genius.",
         "reason": "The context and statement are unrelated"
-        "verdict": "No"
+        "verdict": "0"
     }}
 ]
 
@@ -105,7 +105,7 @@
      {{
         "statement_1": "Nil",
         "reason": "The statement is invalid",
-        "verdict": "No"
+        "verdict": "0"
     }}
 ]
 
@@ -169,7 +169,7 @@ def _score_batch(
 
             result = self.llm.generate(prompts, callbacks=batch_group)
             outputs = result.generations
-            verdict_score_map = {"yes": 1, "no": 0, "null": np.nan}
+            verdict_score_map = {"1": 1, "0": 0, "null": np.nan}
             scores = []
             for output in outputs:
                 output = json_loader.safe_load(output[0].text, self.llm)