diff --git a/src/ragas/llms/prompt.py b/src/ragas/llms/prompt.py index 666ff555..505849c7 100644 --- a/src/ragas/llms/prompt.py +++ b/src/ragas/llms/prompt.py @@ -49,7 +49,7 @@ class Prompt(BaseModel): input_keys: t.List[str] output_key: str output_type: str = "json" - language = "en" + language: str = "english" @root_validator def validate_prompt(cls, values: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index 6bd5f697..5fea5094 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -2,33 +2,24 @@ import typing as t from abc import ABC, abstractmethod from dataclasses import dataclass, field -from random import choice from fsspec.exceptions import asyncio -from langchain.prompts import ChatPromptTemplate from numpy.random import default_rng from ragas.llms import BaseRagasLLM from ragas.llms.json_load import load_as_json -from ragas.llms.prompt import PromptValue from ragas.testset.docstore import Direction, Document, DocumentStore, Node from ragas.testset.prompts import ( - FILTER_QUESTION, - MULTICONTEXT_QUESTION, - SCORE_CONTEXT, - SEED_QUESTION, - TABLE_QA, - demonstrations, + context_scoring_prompt, + filter_question_prompt, + multi_context_question_prompt, + seed_question_prompt, ) rng = default_rng() logger = logging.getLogger(__name__) -def to_pv(prompt: ChatPromptTemplate) -> PromptValue: - return PromptValue(prompt_str=prompt.format()) - - @dataclass class Filter(ABC): ... @@ -43,9 +34,8 @@ def filter(self, node: Node) -> t.Dict: return asyncio.get_event_loop().run_until_complete(self.afilter(node)) async def afilter(self, node: Node) -> t.Dict: - human_prompt = SCORE_CONTEXT.format(context=node.page_content) - prompt = ChatPromptTemplate.from_messages([human_prompt]) - results = await self.llm.agenerate_text(prompt=to_pv(prompt)) + prompt = context_scoring_prompt.format(context=node.page_content) + results = await self.llm.agenerate_text(prompt=prompt) output = results.generations[0][0].text.strip() score = load_as_json(output) score.update({"score": score.get("score", 0) >= self.threshold}) @@ -60,10 +50,8 @@ def filter(self, question: str) -> bool: return asyncio.get_event_loop().run_until_complete(self.afilter(question)) async def afilter(self, question: str) -> bool: - human_prompt = FILTER_QUESTION.format(question=question) - prompt = ChatPromptTemplate.from_messages([human_prompt]) - - results = await self.llm.agenerate_text(prompt=to_pv(prompt)) + prompt = filter_question_prompt.format(question=question) + results = await self.llm.agenerate_text(prompt=prompt) results = results.generations[0][0].text.strip() json_results = load_as_json(results) logger.debug("filtered question: %s", json_results) @@ -137,16 +125,15 @@ async def aevolve(self, llm: BaseRagasLLM, docstore: DocumentStore): self.nodes = docstore.get_random_nodes(k=1) self._root_node = self.nodes[0] merged_node = self.merged_nodes() - passed, table_is_present = await self.node_filter.afilter(self.nodes[0]) - if not passed: + passed = await self.node_filter.afilter(self.nodes[0]) + if not passed["score"]: self.nodes = docstore.get_random_nodes(k=1) return await self.aretry_evolve(llm, docstore, update_count=False) # frame a basic question with with node - seed_questions = await simple_evolution(llm, merged_node, table_is_present) + seed_question = await simple_evolution(llm, merged_node) # NOTE: might need improvement # select only one seed question here - seed_question = choice(seed_questions) is_valid_question = await self.question_filter.afilter(seed_question) if not is_valid_question: # get more context to rewrite question @@ -158,46 +145,21 @@ async def aevolve(self, llm: BaseRagasLLM, docstore: DocumentStore): return seed_question -async def simple_evolution( - llm: BaseRagasLLM, seed_doc: Document, is_table_present: bool = False -): - if is_table_present: - human_prompt = TABLE_QA.format(context=seed_doc.page_content) - else: - sample = rng.choice(demonstrations, 1)[0] # type: ignore - questions = rng.choice(sample["questions"], 2, replace=False) - questions = ( - "{" - + str({k: v for dic in questions.tolist() for k, v in dic.items()}).replace( - "'", '"' - ) - + "}" - ) - demo = f'Context:{sample["context"]}\nQuestions:{questions}' - human_prompt = SEED_QUESTION.format( - demonstration=demo, context=seed_doc.page_content - ) - - prompt = ChatPromptTemplate.from_messages([human_prompt]) - results = llm.generate_text_with_hmpt(prompts=[prompt]) +async def simple_evolution(llm: BaseRagasLLM, seed_doc: Document): + prompt = seed_question_prompt.format(context=seed_doc.page_content) + results = llm.generate_text(prompt=prompt) results = results.generations[0][0].text - if is_table_present: - return [results] - else: - results = load_as_json(results) - return [v for v in results.values()] + return results async def multi_context_evolution( llm: BaseRagasLLM, seed_node: Node, doc_store: DocumentStore ): question = simple_evolution(llm, seed_node) - print(question) similar_context = doc_store.get_similar(seed_node)[0] - human_prompt = MULTICONTEXT_QUESTION.format( + prompt = multi_context_question_prompt.format( question=question, context1=seed_node.page_content, context2=similar_context ) - prompt = ChatPromptTemplate.from_messages([human_prompt]) - results = await llm.agenerate_text(prompt=to_pv(prompt)) + results = await llm.agenerate_text(prompt=prompt) question = results.generations[0][0].text.strip() return question diff --git a/src/ragas/testset/prompts.py b/src/ragas/testset/prompts.py index 04fe50a5..16269f36 100644 --- a/src/ragas/testset/prompts.py +++ b/src/ragas/testset/prompts.py @@ -1,5 +1,308 @@ from langchain.prompts import HumanMessagePromptTemplate +from ragas.llms.prompt import Prompt + +seed_question_prompt = Prompt( + name="seed_question", + instruction="""Generate a question from given context satisfying the rules given below: + 2.The question should be framed such that it must be clearly understood without providing context. + 3.The question should be fully answerable from information present in given context.""", + examples=[ + { + "context": "The Eiffel Tower in Paris was originally intended as a temporary structure, built for the 1889 World's Fair. It was almost dismantled in 1909 but was saved because it was repurposed as a giant radio antenna.", + "output": "Who built the Eiffel Tower?", + }, + ], + input_keys=["context"], + output_key="output", + output_type="string", + language="english", +) + + +reasoning_question_prompt = Prompt( + name="reasoning_question", + instruction="""Complicate the given question by rewriting question into a multi-hop reasoning question based on the provided context. + Answering the question should require the reader to make multiple logical connections or inferences using the information available in given context. + Rules to follow when rewriting question: + 1. Ensure that the rewritten question can be answered entirely from the information present in the contexts. + 2. Do not frame questions that contains more than 15 words. Use abbreviation wherever possible. + 3. Make sure the question is clear and unambiguous. + 4. phrases like 'based on the provided context','according to the context',etc are not allowed to appear in the question.""", + examples=[ + { + "question": "What is the capital of France?", + "context": "France is a country in Western Europe. It has several cities, including Paris, Lyon, and Marseille. Paris is not only known for its cultural landmarks like the Eiffel Tower and the Louvre Museum but also as the administrative center.", + "output": "Linking the Eiffel Tower and administrative center, which city stands as both?", + }, + { + "question": "What does the append() method do in Python?", + "context": "In Python, lists are used to store multiple items in a single variable. Lists are one of 4 built-in data types used to store collections of data. The append() method adds a single item to the end of a list.", + "output": "If a list represents a variable collection, what method extends it by one item?", + }, + ], + input_keys=["question", "context"], + output_key="output", + output_type="string", + language="english", +) + + +multi_context_question_prompt = Prompt( + name="multi_context_question", + instruction=""" + The task is to rewrite and complicate the given question in a way that answering it requires information derived from both context1 and context2. + Follow the rules given below while rewriting the question. + 1. The rewritten question should not be very long. Use abbreviation wherever possible. + 2. The rewritten question must be reasonable and must be understood and responded by humans. + 3. The rewritten question must be fully answerable from information present in context1 and context2. + 4. Read and understand both contexts and rewrite the question so that answering requires insight from both context1 and context2. + 5. phrases like 'based on the provided context','according to the context?',etc are not allowed to appear in the question.""", + examples=[ + { + "question": "What process turns plants green?", + "context1": "Chlorophyll is the pigment that gives plants their green color and helps them photosynthesize.", + "context2": "Photosynthesis in plants typically occurs in the leaves where chloroplasts are concentrated.", + "output": "In which plant structures does the pigment responsible for their verdancy facilitate energy production?", + }, + { + "question": "How do you calculate the area of a rectangle?", + "context1": "The area of a shape is calculated based on the shape's dimensions. For rectangles, this involves multiplying the length and width.", + "context2": "Rectangles have four sides with opposite sides being equal in length. They are a type of quadrilateral.", + "output": "What multiplication involving equal opposites yields a quadrilateral's area?", + }, + ], + input_keys=["question", "context1", "context2"], + output_key="output", + output_type="string", + language="english", +) + +conditional_question_prompt = Prompt( + name="conditional_question", + instruction="""Rewrite the provided question to increase its complexity by introducing a conditional element. + The goal is to make the question more intricate by incorporating a scenario or condition that affects the context of the question. + Follow the rules given below while rewriting the question. + 1. The rewritten question should not be longer than 25 words. Use abbreviation wherever possible. + 2. The rewritten question must be reasonable and must be understood and responded by humans. + 3. The rewritten question must be fully answerable from information present context. + 4. phrases like 'provided context','according to the context?',etc are not allowed to appear in the question.""", + examples=[ + { + "question": "What is the function of the roots of a plant?", + "context": "The roots of a plant absorb water and nutrients from the soil, anchor the plant in the ground, and store food.", + "output": "What dual purpose do plant roots serve concerning soil nutrients and stability?", + }, + { + "question": "How do vaccines protect against diseases?", + "context": "Vaccines protect against diseases by stimulating the body's immune response to produce antibodies, which recognize and combat pathogens.", + "output": "How do vaccines utilize the body's immune system to defend against pathogens?", + }, + ], + input_keys=["question", "context"], + output_key="output", + output_type="string", + language="english", +) + + +compress_question_prompt = Prompt( + name="compress_question", + instruction="""Rewrite the following question to make it more indirect and shorter while retaining the essence of the original question. + The goal is to create a question that conveys the same meaning but in a less direct manner. The rewritten question should shorter so use abbreviation wherever possible.""", + examples=[ + { + "question": "What is the distance between the Earth and the Moon?", + "output": "How far is the Moon from Earth?", + }, + { + "question": "What ingredients are required to bake a chocolate cake?", + "output": "What's needed for a chocolate cake?", + }, + ], + input_keys=["question"], + output_key="output", + output_type="string", + language="english", +) + + +conversational_question_prompt = Prompt( + name="conversation_question", + instruction="""Reformat the provided question into two separate questions as if it were to be part of a conversation. Each question should focus on a specific aspect or subtopic related to the original question. + Follow the rules given below while rewriting the question. + 1. The rewritten question should not be longer than 25 words. Use abbreviation wherever possible. + 2. The rewritten question must be reasonable and must be understood and responded by humans. + 3. The rewritten question must be fully answerable from information present context. + 4. phrases like 'provided context','according to the context?',etc are not allowed to appear in the question.""", + examples=[ + { + "question": "What are the advantages and disadvantages of remote work?", + "output": { + "first_question": "What are the benefits of remote work?", + "second_question": "On the flip side, what challenges are encountered when working remotely?", + }, + } + ], + input_keys=["question"], + output_key="output", + output_type="json", + language="english", +) + +context_scoring_prompt = Prompt( + name="score_context", + instruction="""Given a context, complete the two following tasks and output answer valid json format +1.Evaluate the provided context and assign a numerical score between 0 and 10 based on the following criteria: + - Award a high score to context that thoroughly delves into and explains concepts. + - Assign a lower score to context that contains excessive references, acknowledgments, personal information, or other non-essential elements.""", + examples=[ + { + "context": "Albert Einstein (14 March 1879 - 18 April 1955) was a German-born theoretical physicist who is widely held to be one of the greatest and most influential scientists of all time.", + "output": {"score": 6.0}, + } + ], + input_keys=["context"], + output_key="output", + output_type="json", + language="english", +) + +question_rewrite_prompt = Prompt( + name="rewrite_question", + instruction="""Given a context, transform the given question to be clear and standalone by replacing its coreferences with specific details from the context:""", + examples=[ + { + "context": "The Eiffel Tower was constructed using iron and was originally intended as a temporary exhibit for the 1889 World's Fair held in Paris. Despite its initial temporary purpose, the Eiffel Tower quickly became a symbol of Parisian ingenuity and an iconic landmark of the city, attracting millions of visitors each year. The tower's design, created by Gustave Eiffel, was initially met with criticism from some French artists and intellectuals, but it has since been celebrated as a masterpiece of structural engineering and architectural design.", + "question": "Who created the design for the Tower?", + "output": "Who created the design for the Eiffel Tower?", + }, + { + "context": "'Exploring Zero-Shot Learning in Neural Networks' was published by Smith and Lee in 2021, focusing on the application of zero-shot learning techniques in artificial intelligence.", + "question": "What datasets were used for the zero-shot evaluations in this study?", + "output": "What datasets were used for the zero-shot evaluations Exploring Zero-Shot Learning in Neural Networks paper?", + }, + ], + input_keys=["context", "question"], + output_key="output", + output_type="string", + language="english", +) + + +filter_question_prompt = Prompt( + name="filter_question", + instruction="""Given a question, classify it based on clarity and specificity""", + examples=[ + { + "question": "What is the discovery about space?", + "output": { + "reason": "The question is too vague and does not specify which discovery about space it is referring to.", + "verdit": "No", + }, + }, + { + "question": "What caused the Great Depression?", + "output": { + "reason": "The question is specific and refers to a well-known historical economic event, making it clear and answerable.", + "verdict": "Yes", + }, + }, + { + "question": "What is the keyword that best describes the paper's focus in natural language understanding tasks?", + "output": { + "reason": "The question mentions a 'paper' in it without referring it's name which makes it unclear without it", + "verdict": "No", + }, + }, + { + "question": "Who wrote 'Romeo and Juliet'?", + "output": { + "reason": "The question is clear and refers to a specific work by name therefore it is clear", + "verdict": "Yes", + }, + }, + { + "question": "What did the study mention?", + "output": { + "reason": "The question is vague and does not specify which study it is referring to", + "verdict": "No", + }, + }, + { + "question": "What is the focus of the REPLUG paper?", + "output": { + "reason": "The question refers to a specific work by it's name hence can be understood", + "verdict": "Yes", + }, + }, + { + "question": "What is the purpose of the reward-driven stage in the training process?", + "output": { + "reason": "The question lacks specific context regarding the type of training process, making it potentially ambiguous and open to multiple interpretations.", + "verdict": "No", + }, + }, + ], + input_keys=["question"], + output_key="output", + output_type="json", + language="english", +) + +evolution_elimination_prompt = Prompt( + name="evolution_elimination", + instruction="""Check if the given two questions are equal based on following requirements: + 1. They have same constraints and requirements. + 2. They have same depth and breadth of the inquiry.""", + examples=[ + { + "question1": "What are the primary causes of climate change?", + "question2": "What factors contribute to global warming?", + "output": { + "reason": "While both questions deal with environmental issues, 'climate change' encompasses broader changes than 'global warming', leading to different depths of inquiry.", + "verdict": "Not Equal", + }, + }, + { + "question1": "How does photosynthesis work in plants?", + "question2": "Can you explain the process of photosynthesis in plants?", + "output": { + "reason": "Both questions ask for an explanation of the photosynthesis process in plants, sharing the same depth, breadth, and requirements for the answer.", + "verdict": "Equal", + }, + }, + ], + input_keys=["question1", "question2"], + output_key="output", + output_type="json", + language="english", +) + +question_answer_prompt = Prompt( + name="answer_formulate", + instruction="""Answer the question using the information from the given context. Answer '-1' if answer is not present in the context.""", + examples=[ + { + "context": """The novel '1984' by George Orwell is set in a dystopian future where the world is divided into three superstates. The story follows the life of Winston Smith, who lives in Oceania, a superstate constantly at war.""", + "question": "In which superstate does Winston Smith live in the novel '1984'?", + "answer": "Winston Smith lives in the superstate of Oceania in the novel '1984'.", + }, + { + "context": """The novel "Pride and Prejudice" by Jane Austen revolves around the character Elizabeth Bennet and her family. The story is set in the 19th century in rural England and deals with issues of marriage, morality, and misconceptions.""", + "question": "What year was 'Pride and Prejudice' published?", + "answer": "-1", + }, + ], + input_keys=["context", "question"], + output_key="answer", + output_type="string", + language="english", +) + + +## TODO: remove this + SEED_QUESTION = HumanMessagePromptTemplate.from_template( """ Generate two questions from given context satisfying the rules given below: