diff --git a/docs/howtos/customizations/index.md b/docs/howtos/customizations/index.md index 61a59a92e..a95efe75f 100644 --- a/docs/howtos/customizations/index.md +++ b/docs/howtos/customizations/index.md @@ -16,5 +16,7 @@ How to customize various aspects of Ragas to suit your needs. ## Testset Generation -- [Add your own test cases](testgenerator/index.md) +- [Configure or automatically generate Personas](testgenerator/_persona_generator.md) +- [Customize single-hop queries for RAG evaluation](testgenerator/_testgen-custom-single-hop.md) +- [Create custom multi-hop queries for RAG evaluation](testgenerator/_testgen-customisation.md) - [Seed generations using production data](testgenerator/index.md) diff --git a/docs/howtos/customizations/testgenerator/_testgen-custom-single-hop.md b/docs/howtos/customizations/testgenerator/_testgen-custom-single-hop.md new file mode 100644 index 000000000..2e843b1e2 --- /dev/null +++ b/docs/howtos/customizations/testgenerator/_testgen-custom-single-hop.md @@ -0,0 +1,272 @@ +# Create custom single-hop queries from your documents + +### Load sample documents +I am using documents from [gitlab handbook](https://huggingface.co/datasets/explodinggradients/Sample_Docs_Markdown). You can download it by running the below command. + + +```python +from langchain_community.document_loaders import DirectoryLoader + + +path = "Sample_Docs_Markdown/" +loader = DirectoryLoader(path, glob="**/*.md") +docs = loader.load() +``` + +### Create KG + +Create a base knowledge graph with the documents + + +```python +from ragas.testset.graph import KnowledgeGraph +from ragas.testset.graph import Node, NodeType + + +kg = KnowledgeGraph() +for doc in docs: + kg.nodes.append( + Node( + type=NodeType.DOCUMENT, + properties={ + "page_content": doc.page_content, + "document_metadata": doc.metadata, + }, + ) + ) +``` + + /opt/homebrew/Caskroom/miniforge/base/envs/ragas/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html + from .autonotebook import tqdm as notebook_tqdm + + +### Set up the LLM and Embedding Model +You may use any of [your choice](/docs/howtos/customizations/customize_models.md), here I am using models from open-ai. + + +```python +from ragas.llms.base import llm_factory +from ragas.embeddings.base import embedding_factory + +llm = llm_factory() +embedding = embedding_factory() +``` + +### Setup the transforms + + +Here we are using 2 extractors and 2 relationship builders. +- Headline extrator: Extracts headlines from the documents +- Keyphrase extractor: Extracts keyphrases from the documents +- Headline splitter: Splits the document into nodes based on headlines + + + +```python +from ragas.testset.transforms import apply_transforms +from ragas.testset.transforms import ( + HeadlinesExtractor, + HeadlineSplitter, + KeyphrasesExtractor, +) + + +headline_extractor = HeadlinesExtractor(llm=llm) +headline_splitter = HeadlineSplitter(min_tokens=300, max_tokens=1000) +keyphrase_extractor = KeyphrasesExtractor( + llm=llm, property_name="keyphrases", max_num=10 +) +``` + + +```python +transforms = [ + headline_extractor, + headline_splitter, + keyphrase_extractor, +] + +apply_transforms(kg, transforms=transforms) +``` + + Applying KeyphrasesExtractor: 6%| | 2/36 [00:01<00:20, 1Property 'keyphrases' already exists in node '514fdc'. Skipping! + Applying KeyphrasesExtractor: 11%| | 4/36 [00:01<00:10, 2Property 'keyphrases' already exists in node '84a0f6'. Skipping! + Applying KeyphrasesExtractor: 64%|▋| 23/36 [00:03<00:01, Property 'keyphrases' already exists in node '93f19d'. Skipping! + Applying KeyphrasesExtractor: 72%|▋| 26/36 [00:04<00:00, 1Property 'keyphrases' already exists in node 'a126bf'. Skipping! + Applying KeyphrasesExtractor: 81%|▊| 29/36 [00:04<00:00, Property 'keyphrases' already exists in node 'c230df'. Skipping! + Applying KeyphrasesExtractor: 89%|▉| 32/36 [00:04<00:00, 1Property 'keyphrases' already exists in node '4f2765'. Skipping! + Property 'keyphrases' already exists in node '4a4777'. Skipping! + + +### Configure personas + +You can also do this automatically by using the [automatic persona generator](/docs/howtos/customizations/testgenerator/_persona_generator.md) + + +```python +from ragas.testset.persona import Persona + +person1 = Persona( + name="gitlab employee", + role_description="A junior gitlab employee curious on workings on gitlab", +) +persona2 = Persona( + name="Hiring manager at gitlab", + role_description="A hiring manager at gitlab trying to underestand hiring policies in gitlab", +) +persona_list = [person1, persona2] +``` + +## + +## SingleHop Query + +Inherit from `SingleHopQuerySynthesizer` and modify the function that generates scenarios for query creation. + +**Steps**: +- find qualified set of nodes for the query creation. Here I am selecting all nodes with keyphrases extracted. +- For each qualified set + - Match the keyphrase with one or more persona. + - Create all possible combinations of (Node, Persona, Query Style, Query Length) + - Samples the required number of queries from the combinations + + +```python +from ragas.testset.synthesizers.single_hop import ( + SingleHopQuerySynthesizer, + SingleHopScenario, +) +from dataclasses import dataclass +from ragas.testset.synthesizers.prompts import ( + ThemesPersonasInput, + ThemesPersonasMatchingPrompt, +) + + +@dataclass +class MySingleHopScenario(SingleHopQuerySynthesizer): + + theme_persona_matching_prompt = ThemesPersonasMatchingPrompt() + + async def _generate_scenarios(self, n, knowledge_graph, persona_list, callbacks): + + property_name = "keyphrases" + nodes = [] + for node in knowledge_graph.nodes: + if node.type.name == "CHUNK" and node.get_property(property_name): + nodes.append(node) + + number_of_samples_per_node = max(1, n // len(nodes)) + + scenarios = [] + for node in nodes: + if len(scenarios) >= n: + break + themes = node.properties.get(property_name, [""]) + prompt_input = ThemesPersonasInput(themes=themes, personas=persona_list) + persona_concepts = await self.theme_persona_matching_prompt.generate( + data=prompt_input, llm=self.llm, callbacks=callbacks + ) + base_scenarios = self.prepare_combinations( + node, + themes, + personas=persona_list, + persona_concepts=persona_concepts.mapping, + ) + scenarios.extend( + self.sample_combinations(base_scenarios, number_of_samples_per_node) + ) + + return scenarios +``` + + +```python +query = MySingleHopScenario(llm=llm) +``` + + +```python +scenarios = await query.generate_scenarios( + n=5, knowledge_graph=kg, persona_list=persona_list +) +``` + + +```python +scenarios[0] +``` + + + + + SingleHopScenario( + nodes=1 + term=what is an ally + persona=name='Hiring manager at gitlab' role_description='A hiring manager at gitlab trying to underestand hiring policies in gitlab' + style=Web search like queries + length=long) + + + + +```python +result = await query.generate_sample(scenario=scenarios[-1]) +``` + +### Modify prompt to customize the query style +Here I am replacing the default prompt with an instruction to generate only Yes/No questions. This is an optional step. + + +```python +instruction = """Generate a Yes/No query and answer based on the specified conditions (persona, term, style, length) +and the provided context. Ensure the answer is entirely faithful to the context, using only the information +directly from the provided context. + +### Instructions: +1. **Generate a Yes/No Query**: Based on the context, persona, term, style, and length, create a question +that aligns with the persona's perspective, incorporates the term, and can be answered with 'Yes' or 'No'. +2. **Generate an Answer**: Using only the content from the provided context, provide a 'Yes' or 'No' answer +to the query. Do not add any information not included in or inferable from the context.""" +``` + + +```python +prompt = query.get_prompts()["generate_query_reference_prompt"] +prompt.instruction = instruction +query.set_prompts(**{"generate_query_reference_prompt": prompt}) +``` + + +```python +result = await query.generate_sample(scenario=scenarios[-1]) +``` + + +```python +result.user_input +``` + + + + + 'Does the Diversity, Inclusion & Belonging (DIB) Team at GitLab have a structured approach to encourage collaborations among team members through various communication methods?' + + + + +```python +result.reference +``` + + + + + 'Yes' + + + + +```python + +``` diff --git a/docs/howtos/customizations/testgenerator/_testgen-customisation.md b/docs/howtos/customizations/testgenerator/_testgen-customisation.md new file mode 100644 index 000000000..39bb2958e --- /dev/null +++ b/docs/howtos/customizations/testgenerator/_testgen-customisation.md @@ -0,0 +1,273 @@ +# Create custom multi-hop queries from your documents + +In this tutorial you will get to learn how to create custom multi-hop queries from your documents. This is a very powerful feature that allows you to create queries that are not possible with the standard query types. This also helps you to create queries that are more specific to your use case. + +### Load sample documents +I am using documents from [gitlab handbook](https://huggingface.co/datasets/explodinggradients/Sample_Docs_Markdown). You can download it by running the below command. + + +```python +! git clone https://huggingface.co/datasets/explodinggradients/Sample_Docs_Markdown +``` + + +```python +from langchain_community.document_loaders import DirectoryLoader, TextLoader + +path = "Sample_Docs_Markdown/" +loader = DirectoryLoader(path, glob="**/*.md") +docs = loader.load() +``` + +### Create KG + +Create a base knowledge graph with the documents + + +```python +from ragas.testset.graph import KnowledgeGraph +from ragas.testset.graph import Node, NodeType + + +kg = KnowledgeGraph() +for doc in docs: + kg.nodes.append( + Node( + type=NodeType.DOCUMENT, + properties={ + "page_content": doc.page_content, + "document_metadata": doc.metadata, + }, + ) + ) +``` + + /opt/homebrew/Caskroom/miniforge/base/envs/ragas/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html + from .autonotebook import tqdm as notebook_tqdm + + +### Set up the LLM and Embedding Model +You may use any of [your choice](/docs/howtos/customizations/customize_models.md), here I am using models from open-ai. + + +```python +from ragas.llms.base import llm_factory +from ragas.embeddings.base import embedding_factory + +llm = llm_factory() +embedding = embedding_factory() +``` + +### Setup Extractors and Relationship builders + +To create multi-hop queries you need to undestand the set of documents that can be used for it. Ragas uses relationships between documents/nodes to quality nodes for creating multi-hop queries. To concretize, if Node A and Node B and conencted by a relationship (say entity or keyphrase overlap) then you can create a multi-hop query between them. + +Here we are using 2 extractors and 2 relationship builders. +- Headline extrator: Extracts headlines from the documents +- Keyphrase extractor: Extracts keyphrases from the documents +- Headline splitter: Splits the document into nodes based on headlines +- OverlapScore Builder: Builds relationship between nodes based on keyphrase overlap + + +```python +from ragas.testset.transforms import Parallel, apply_transforms +from ragas.testset.transforms import ( + HeadlinesExtractor, + HeadlineSplitter, + KeyphrasesExtractor, + OverlapScoreBuilder, +) + + +headline_extractor = HeadlinesExtractor(llm=llm) +headline_splitter = HeadlineSplitter(min_tokens=300, max_tokens=1000) +keyphrase_extractor = KeyphrasesExtractor( + llm=llm, property_name="keyphrases", max_num=10 +) +relation_builder = OverlapScoreBuilder( + property_name="keyphrases", + new_property_name="overlap_score", + threshold=0.01, + distance_threshold=0.9, +) +``` + + +```python +transforms = [ + headline_extractor, + headline_splitter, + keyphrase_extractor, + relation_builder, +] + +apply_transforms(kg, transforms=transforms) +``` + + Applying KeyphrasesExtractor: 6%|██████▏ | 2/36 [00:01<00:17, 1.94it/s]Property 'keyphrases' already exists in node 'a2f389'. Skipping! + Applying KeyphrasesExtractor: 17%|██████████████████▋ | 6/36 [00:01<00:04, 6.37it/s]Property 'keyphrases' already exists in node '3068c0'. Skipping! + Applying KeyphrasesExtractor: 53%|██████████████████████████████████████████████████████████▌ | 19/36 [00:02<00:01, 8.88it/s]Property 'keyphrases' already exists in node '854bf7'. Skipping! + Applying KeyphrasesExtractor: 78%|██████████████████████████████████████████████████████████████████████████████████████▎ | 28/36 [00:03<00:00, 9.73it/s]Property 'keyphrases' already exists in node '2eeb07'. Skipping! + Property 'keyphrases' already exists in node 'd68f83'. Skipping! + Applying KeyphrasesExtractor: 83%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 30/36 [00:03<00:00, 9.35it/s]Property 'keyphrases' already exists in node '8fdbea'. Skipping! + Applying KeyphrasesExtractor: 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32/36 [00:04<00:00, 7.76it/s]Property 'keyphrases' already exists in node 'ef6ae0'. Skipping! + + +### Configure personas + +You can also do this automatically by using the [automatic persona generator](/docs/howtos/customizations/testgenerator/_persona_generator.md) + + +```python +from ragas.testset.persona import Persona + +person1 = Persona( + name="gitlab employee", + role_description="A junior gitlab employee curious on workings on gitlab", +) +persona2 = Persona( + name="Hiring manager at gitlab", + role_description="A hiring manager at gitlab trying to underestand hiring policies in gitlab", +) +persona_list = [person1, persona2] +``` + +### Create multi-hop query + +Inherit from `MultiHopQuerySynthesizer` and modify the function that generates scenarios for query creation. + +**Steps**: +- find qualified set of (nodeA, relationship, nodeB) based on the relationships between nodes +- For each qualified set + - Match the keyphrase with one or more persona. + - Create all possible combinations of (Nodes, Persona, Query Style, Query Length) + - Samples the required number of queries from the combinations + + + +```python +from dataclasses import dataclass +import typing as t +from ragas.testset.synthesizers.multi_hop.base import ( + MultiHopQuerySynthesizer, + MultiHopScenario, +) +from ragas.testset.synthesizers.prompts import ( + ThemesPersonasInput, + ThemesPersonasMatchingPrompt, +) + + +@dataclass +class MyMultiHopQuery(MultiHopQuerySynthesizer): + + theme_persona_matching_prompt = ThemesPersonasMatchingPrompt() + + async def _generate_scenarios( + self, + n: int, + knowledge_graph, + persona_list, + callbacks, + ) -> t.List[MultiHopScenario]: + + # query and get (node_a, rel, node_b) to create multi-hop queries + results = kg.find_two_nodes_single_rel( + relationship_condition=lambda rel: ( + True if rel.type == "keyphrases_overlap" else False + ) + ) + + num_sample_per_triplet = max(1, n // len(results)) + + scenarios = [] + for triplet in results: + if len(scenarios) < n: + node_a, node_b = triplet[0], triplet[-1] + overlapped_keywords = triplet[1].properties["overlapped_items"] + if overlapped_keywords: + + # match the keyword with a persona for query creation + themes = list(dict(overlapped_keywords).keys()) + prompt_input = ThemesPersonasInput( + themes=themes, personas=persona_list + ) + persona_concepts = ( + await self.theme_persona_matching_prompt.generate( + data=prompt_input, llm=self.llm, callbacks=callbacks + ) + ) + + overlapped_keywords = [list(item) for item in overlapped_keywords] + + # prepare and sample possible combinations + base_scenarios = self.prepare_combinations( + [node_a, node_b], + overlapped_keywords, + personas=persona_list, + persona_item_mapping=persona_concepts.mapping, + property_name="keyphrases", + ) + + # get number of required samples from this triplet + base_scenarios = self.sample_diverse_combinations( + base_scenarios, num_sample_per_triplet + ) + + scenarios.extend(base_scenarios) + + return scenarios +``` + + +```python +query = MyMultiHopQuery(llm=llm) +scenarios = await query.generate_scenarios( + n=10, knowledge_graph=kg, persona_list=persona_list +) +``` + + +```python +scenarios[4] +``` + + + + + MultiHopScenario( + nodes=2 + combinations=['Diversity Inclusion & Belonging', 'Diversity, Inclusion & Belonging Goals'] + style=Web search like queries + length=short + persona=name='Hiring manager at gitlab' role_description='A hiring manager at gitlab trying to underestand hiring policies in gitlab') + + + + +```python + +``` + +### Run the multi-hop query + + +```python +result = await query.generate_sample(scenario=scenarios[-1]) +``` + + +```python +result.user_input +``` + + + + + 'How does GitLab ensure that its DIB roundtables are effective in promoting diversity and inclusion?' + + + +Yay! You have created a multi-hop query. Now you can create any such queries by creating and exploring relationships between documents. + +## diff --git a/docs/howtos/customizations/testgenerator/testgen-custom-single-hop.ipynb b/docs/howtos/customizations/testgenerator/testgen-custom-single-hop.ipynb new file mode 100644 index 000000000..7829dd073 --- /dev/null +++ b/docs/howtos/customizations/testgenerator/testgen-custom-single-hop.ipynb @@ -0,0 +1,457 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "51c3407b-6041-4217-9ef9-a0e619a51603", + "metadata": {}, + "source": [ + "# Create custom single-hop queries from your documents" + ] + }, + { + "cell_type": "markdown", + "id": "5fc18fe5", + "metadata": {}, + "source": [ + "### Load sample documents\n", + "I am using documents from [gitlab handbook](https://huggingface.co/datasets/explodinggradients/Sample_Docs_Markdown). You can download it by running the below command." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5e3647cd-f754-4f05-a5ea-488b6a6affaf", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import DirectoryLoader\n", + "\n", + "\n", + "path = \"Sample_Docs_Markdown/\"\n", + "loader = DirectoryLoader(path, glob=\"**/*.md\")\n", + "docs = loader.load()" + ] + }, + { + "cell_type": "markdown", + "id": "ba780919", + "metadata": {}, + "source": [ + "### Create KG\n", + "\n", + "Create a base knowledge graph with the documents" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9034eaf0-e6d8-41d1-943b-594331972f69", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/Caskroom/miniforge/base/envs/ragas/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from ragas.testset.graph import KnowledgeGraph\n", + "from ragas.testset.graph import Node, NodeType\n", + "\n", + "\n", + "kg = KnowledgeGraph()\n", + "for doc in docs:\n", + " kg.nodes.append(\n", + " Node(\n", + " type=NodeType.DOCUMENT,\n", + " properties={\n", + " \"page_content\": doc.page_content,\n", + " \"document_metadata\": doc.metadata,\n", + " },\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "575e5725", + "metadata": {}, + "source": [ + "### Set up the LLM and Embedding Model\n", + "You may use any of [your choice](/docs/howtos/customizations/customize_models.md), here I am using models from open-ai." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "52f6d1ae-c9ed-4d82-99d7-d130a36e41e8", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.llms.base import llm_factory\n", + "from ragas.embeddings.base import embedding_factory\n", + "\n", + "llm = llm_factory()\n", + "embedding = embedding_factory()" + ] + }, + { + "cell_type": "markdown", + "id": "af7f9eaa", + "metadata": {}, + "source": [ + "### Setup the transforms\n", + "\n", + "\n", + "Here we are using 2 extractors and 2 relationship builders.\n", + "- Headline extrator: Extracts headlines from the documents\n", + "- Keyphrase extractor: Extracts keyphrases from the documents\n", + "- Headline splitter: Splits the document into nodes based on headlines\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1308cf70-486c-4fc3-be9a-2401e9455312", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset.transforms import apply_transforms\n", + "from ragas.testset.transforms import (\n", + " HeadlinesExtractor,\n", + " HeadlineSplitter,\n", + " KeyphrasesExtractor,\n", + ")\n", + "\n", + "\n", + "headline_extractor = HeadlinesExtractor(llm=llm)\n", + "headline_splitter = HeadlineSplitter(min_tokens=300, max_tokens=1000)\n", + "keyphrase_extractor = KeyphrasesExtractor(\n", + " llm=llm, property_name=\"keyphrases\", max_num=10\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7eb5f52e-4f9f-4333-bc71-ec795bf5dfff", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Applying KeyphrasesExtractor: 6%| | 2/36 [00:01<00:20, 1Property 'keyphrases' already exists in node '514fdc'. Skipping!\n", + "Applying KeyphrasesExtractor: 11%| | 4/36 [00:01<00:10, 2Property 'keyphrases' already exists in node '84a0f6'. Skipping!\n", + "Applying KeyphrasesExtractor: 64%|▋| 23/36 [00:03<00:01, Property 'keyphrases' already exists in node '93f19d'. Skipping!\n", + "Applying KeyphrasesExtractor: 72%|▋| 26/36 [00:04<00:00, 1Property 'keyphrases' already exists in node 'a126bf'. Skipping!\n", + "Applying KeyphrasesExtractor: 81%|▊| 29/36 [00:04<00:00, Property 'keyphrases' already exists in node 'c230df'. Skipping!\n", + "Applying KeyphrasesExtractor: 89%|▉| 32/36 [00:04<00:00, 1Property 'keyphrases' already exists in node '4f2765'. Skipping!\n", + "Property 'keyphrases' already exists in node '4a4777'. Skipping!\n", + " \r" + ] + } + ], + "source": [ + "transforms = [\n", + " headline_extractor,\n", + " headline_splitter,\n", + " keyphrase_extractor,\n", + "]\n", + "\n", + "apply_transforms(kg, transforms=transforms)" + ] + }, + { + "cell_type": "markdown", + "id": "40503f3c", + "metadata": {}, + "source": [ + "### Configure personas\n", + "\n", + "You can also do this automatically by using the [automatic persona generator](/docs/howtos/customizations/testgenerator/_persona_generator.md)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "213d93e7-1233-4df7-8022-4827b683f0b3", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset.persona import Persona\n", + "\n", + "person1 = Persona(\n", + " name=\"gitlab employee\",\n", + " role_description=\"A junior gitlab employee curious on workings on gitlab\",\n", + ")\n", + "persona2 = Persona(\n", + " name=\"Hiring manager at gitlab\",\n", + " role_description=\"A hiring manager at gitlab trying to underestand hiring policies in gitlab\",\n", + ")\n", + "persona_list = [person1, persona2]" + ] + }, + { + "cell_type": "markdown", + "id": "d5088c18-a8eb-4180-b066-46a8a795553b", + "metadata": {}, + "source": [ + "## " + ] + }, + { + "cell_type": "markdown", + "id": "e3c756d2-1131-4fde-b3a7-b81589d15929", + "metadata": {}, + "source": [ + "## SingleHop Query\n", + "\n", + "Inherit from `SingleHopQuerySynthesizer` and modify the function that generates scenarios for query creation. \n", + "\n", + "**Steps**:\n", + "- find qualified set of nodes for the query creation. Here I am selecting all nodes with keyphrases extracted.\n", + "- For each qualified set\n", + " - Match the keyphrase with one or more persona. \n", + " - Create all possible combinations of (Node, Persona, Query Style, Query Length)\n", + " - Samples the required number of queries from the combinations" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "c0a7128c-3840-434d-a1df-9e0835c2eb9b", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset.synthesizers.single_hop import (\n", + " SingleHopQuerySynthesizer,\n", + " SingleHopScenario,\n", + ")\n", + "from dataclasses import dataclass\n", + "from ragas.testset.synthesizers.prompts import (\n", + " ThemesPersonasInput,\n", + " ThemesPersonasMatchingPrompt,\n", + ")\n", + "\n", + "\n", + "@dataclass\n", + "class MySingleHopScenario(SingleHopQuerySynthesizer):\n", + "\n", + " theme_persona_matching_prompt = ThemesPersonasMatchingPrompt()\n", + "\n", + " async def _generate_scenarios(self, n, knowledge_graph, persona_list, callbacks):\n", + "\n", + " property_name = \"keyphrases\"\n", + " nodes = []\n", + " for node in knowledge_graph.nodes:\n", + " if node.type.name == \"CHUNK\" and node.get_property(property_name):\n", + " nodes.append(node)\n", + "\n", + " number_of_samples_per_node = max(1, n // len(nodes))\n", + "\n", + " scenarios = []\n", + " for node in nodes:\n", + " if len(scenarios) >= n:\n", + " break\n", + " themes = node.properties.get(property_name, [\"\"])\n", + " prompt_input = ThemesPersonasInput(themes=themes, personas=persona_list)\n", + " persona_concepts = await self.theme_persona_matching_prompt.generate(\n", + " data=prompt_input, llm=self.llm, callbacks=callbacks\n", + " )\n", + " base_scenarios = self.prepare_combinations(\n", + " node,\n", + " themes,\n", + " personas=persona_list,\n", + " persona_concepts=persona_concepts.mapping,\n", + " )\n", + " scenarios.extend(\n", + " self.sample_combinations(base_scenarios, number_of_samples_per_node)\n", + " )\n", + "\n", + " return scenarios" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "6613ade2-b2bb-466a-800a-9ab8cad61661", + "metadata": {}, + "outputs": [], + "source": [ + "query = MySingleHopScenario(llm=llm)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ca6f997f-355b-423f-8559-d20acfd11a53", + "metadata": {}, + "outputs": [], + "source": [ + "scenarios = await query.generate_scenarios(\n", + " n=5, knowledge_graph=kg, persona_list=persona_list\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "6622721d-74e1-4922-b68d-ce4c29a00c02", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "SingleHopScenario(\n", + "nodes=1\n", + "term=what is an ally\n", + "persona=name='Hiring manager at gitlab' role_description='A hiring manager at gitlab trying to underestand hiring policies in gitlab'\n", + "style=Web search like queries\n", + "length=long)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scenarios[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff32bf81", + "metadata": {}, + "outputs": [], + "source": [ + "result = await query.generate_sample(scenario=scenarios[-1])" + ] + }, + { + "cell_type": "markdown", + "id": "bc5c0fb1", + "metadata": {}, + "source": [ + "### Modify prompt to customize the query style\n", + "Here I am replacing the default prompt with an instruction to generate only Yes/No questions. This is an optional step. " + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "6c5d43df-43ad-4ef4-9c52-37a943198400", + "metadata": {}, + "outputs": [], + "source": [ + "instruction = \"\"\"Generate a Yes/No query and answer based on the specified conditions (persona, term, style, length) \n", + "and the provided context. Ensure the answer is entirely faithful to the context, using only the information \n", + "directly from the provided context.\n", + "\n", + "### Instructions:\n", + "1. **Generate a Yes/No Query**: Based on the context, persona, term, style, and length, create a question \n", + "that aligns with the persona's perspective, incorporates the term, and can be answered with 'Yes' or 'No'.\n", + "2. **Generate an Answer**: Using only the content from the provided context, provide a 'Yes' or 'No' answer \n", + "to the query. Do not add any information not included in or inferable from the context.\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "4d20f2e7-7870-4dfe-acf1-05feb84adfe7", + "metadata": {}, + "outputs": [], + "source": [ + "prompt = query.get_prompts()[\"generate_query_reference_prompt\"]\n", + "prompt.instruction = instruction\n", + "query.set_prompts(**{\"generate_query_reference_prompt\": prompt})" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "855770c7-577b-41df-98c2-d366dd927008", + "metadata": {}, + "outputs": [], + "source": [ + "result = await query.generate_sample(scenario=scenarios[-1])" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "40254484-4e1d-450e-8d8b-3b9a20a00467", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Does the Diversity, Inclusion & Belonging (DIB) Team at GitLab have a structured approach to encourage collaborations among team members through various communication methods?'" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.user_input" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "916c1c5b-c92b-40cc-a1e8-d608e7c080f7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Yes'" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.reference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d5fc423-e9e5-4493-b109-d3f5baac7eca", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ragas", + "language": "python", + "name": "ragas" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/howtos/customizations/testgenerator/testgen-customisation.ipynb b/docs/howtos/customizations/testgenerator/testgen-customisation.ipynb new file mode 100644 index 000000000..ec835aea3 --- /dev/null +++ b/docs/howtos/customizations/testgenerator/testgen-customisation.ipynb @@ -0,0 +1,444 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "51c3407b-6041-4217-9ef9-a0e619a51603", + "metadata": {}, + "source": [ + "# Create custom multi-hop queries from your documents\n", + "\n", + "In this tutorial you will get to learn how to create custom multi-hop queries from your documents. This is a very powerful feature that allows you to create queries that are not possible with the standard query types. This also helps you to create queries that are more specific to your use case." + ] + }, + { + "cell_type": "markdown", + "id": "6d0a971b", + "metadata": {}, + "source": [ + "### Load sample documents\n", + "I am using documents from [gitlab handbook](https://huggingface.co/datasets/explodinggradients/Sample_Docs_Markdown). You can download it by running the below command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd7e01c8", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "! git clone https://huggingface.co/datasets/explodinggradients/Sample_Docs_Markdown" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5e3647cd-f754-4f05-a5ea-488b6a6affaf", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n", + "\n", + "path = \"Sample_Docs_Markdown/\"\n", + "loader = DirectoryLoader(path, glob=\"**/*.md\")\n", + "docs = loader.load()" + ] + }, + { + "cell_type": "markdown", + "id": "7db0c75d", + "metadata": {}, + "source": [ + "### Create KG\n", + "\n", + "Create a base knowledge graph with the documents" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "9034eaf0-e6d8-41d1-943b-594331972f69", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/Caskroom/miniforge/base/envs/ragas/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from ragas.testset.graph import KnowledgeGraph\n", + "from ragas.testset.graph import Node, NodeType\n", + "\n", + "\n", + "kg = KnowledgeGraph()\n", + "for doc in docs:\n", + " kg.nodes.append(\n", + " Node(\n", + " type=NodeType.DOCUMENT,\n", + " properties={\n", + " \"page_content\": doc.page_content,\n", + " \"document_metadata\": doc.metadata,\n", + " },\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "fa9b3f77", + "metadata": {}, + "source": [ + "### Set up the LLM and Embedding Model\n", + "You may use any of [your choice](/docs/howtos/customizations/customize_models.md), here I am using models from open-ai." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "52f6d1ae-c9ed-4d82-99d7-d130a36e41e8", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.llms.base import llm_factory\n", + "from ragas.embeddings.base import embedding_factory\n", + "\n", + "llm = llm_factory()\n", + "embedding = embedding_factory()" + ] + }, + { + "cell_type": "markdown", + "id": "f22a543f", + "metadata": {}, + "source": [ + "### Setup Extractors and Relationship builders\n", + "\n", + "To create multi-hop queries you need to undestand the set of documents that can be used for it. Ragas uses relationships between documents/nodes to quality nodes for creating multi-hop queries. To concretize, if Node A and Node B and conencted by a relationship (say entity or keyphrase overlap) then you can create a multi-hop query between them.\n", + "\n", + "Here we are using 2 extractors and 2 relationship builders.\n", + "- Headline extrator: Extracts headlines from the documents\n", + "- Keyphrase extractor: Extracts keyphrases from the documents\n", + "- Headline splitter: Splits the document into nodes based on headlines\n", + "- OverlapScore Builder: Builds relationship between nodes based on keyphrase overlap" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "1308cf70-486c-4fc3-be9a-2401e9455312", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset.transforms import Parallel, apply_transforms\n", + "from ragas.testset.transforms import (\n", + " HeadlinesExtractor,\n", + " HeadlineSplitter,\n", + " KeyphrasesExtractor,\n", + " OverlapScoreBuilder,\n", + ")\n", + "\n", + "\n", + "headline_extractor = HeadlinesExtractor(llm=llm)\n", + "headline_splitter = HeadlineSplitter(min_tokens=300, max_tokens=1000)\n", + "keyphrase_extractor = KeyphrasesExtractor(\n", + " llm=llm, property_name=\"keyphrases\", max_num=10\n", + ")\n", + "relation_builder = OverlapScoreBuilder(\n", + " property_name=\"keyphrases\",\n", + " new_property_name=\"overlap_score\",\n", + " threshold=0.01,\n", + " distance_threshold=0.9,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "7eb5f52e-4f9f-4333-bc71-ec795bf5dfff", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Applying KeyphrasesExtractor: 6%|██████▏ | 2/36 [00:01<00:17, 1.94it/s]Property 'keyphrases' already exists in node 'a2f389'. Skipping!\n", + "Applying KeyphrasesExtractor: 17%|██████████████████▋ | 6/36 [00:01<00:04, 6.37it/s]Property 'keyphrases' already exists in node '3068c0'. Skipping!\n", + "Applying KeyphrasesExtractor: 53%|██████████████████████████████████████████████████████████▌ | 19/36 [00:02<00:01, 8.88it/s]Property 'keyphrases' already exists in node '854bf7'. Skipping!\n", + "Applying KeyphrasesExtractor: 78%|██████████████████████████████████████████████████████████████████████████████████████▎ | 28/36 [00:03<00:00, 9.73it/s]Property 'keyphrases' already exists in node '2eeb07'. Skipping!\n", + "Property 'keyphrases' already exists in node 'd68f83'. Skipping!\n", + "Applying KeyphrasesExtractor: 83%|████████████████████████████████████████████████████████████████████████████████████████████▌ | 30/36 [00:03<00:00, 9.35it/s]Property 'keyphrases' already exists in node '8fdbea'. Skipping!\n", + "Applying KeyphrasesExtractor: 89%|██████████████████████████████████████████████████████████████████████████████████████████████████▋ | 32/36 [00:04<00:00, 7.76it/s]Property 'keyphrases' already exists in node 'ef6ae0'. Skipping!\n", + " \r" + ] + } + ], + "source": [ + "transforms = [\n", + " headline_extractor,\n", + " headline_splitter,\n", + " keyphrase_extractor,\n", + " relation_builder,\n", + "]\n", + "\n", + "apply_transforms(kg, transforms=transforms)" + ] + }, + { + "cell_type": "markdown", + "id": "b7da1d6d", + "metadata": {}, + "source": [ + "### Configure personas\n", + "\n", + "You can also do this automatically by using the [automatic persona generator](/docs/howtos/customizations/testgenerator/_persona_generator.md)" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "213d93e7-1233-4df7-8022-4827b683f0b3", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset.persona import Persona\n", + "\n", + "person1 = Persona(\n", + " name=\"gitlab employee\",\n", + " role_description=\"A junior gitlab employee curious on workings on gitlab\",\n", + ")\n", + "persona2 = Persona(\n", + " name=\"Hiring manager at gitlab\",\n", + " role_description=\"A hiring manager at gitlab trying to underestand hiring policies in gitlab\",\n", + ")\n", + "persona_list = [person1, persona2]" + ] + }, + { + "cell_type": "markdown", + "id": "ced43cb5", + "metadata": {}, + "source": [ + "### Create multi-hop query \n", + "\n", + "Inherit from `MultiHopQuerySynthesizer` and modify the function that generates scenarios for query creation. \n", + "\n", + "**Steps**:\n", + "- find qualified set of (nodeA, relationship, nodeB) based on the relationships between nodes\n", + "- For each qualified set\n", + " - Match the keyphrase with one or more persona. \n", + " - Create all possible combinations of (Nodes, Persona, Query Style, Query Length)\n", + " - Samples the required number of queries from the combinations\n" + ] + }, + { + "cell_type": "code", + "execution_count": 137, + "id": "08db4335-4b00-4f06-b855-4c847675a801", + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "import typing as t\n", + "from ragas.testset.synthesizers.multi_hop.base import (\n", + " MultiHopQuerySynthesizer,\n", + " MultiHopScenario,\n", + ")\n", + "from ragas.testset.synthesizers.prompts import (\n", + " ThemesPersonasInput,\n", + " ThemesPersonasMatchingPrompt,\n", + ")\n", + "\n", + "\n", + "@dataclass\n", + "class MyMultiHopQuery(MultiHopQuerySynthesizer):\n", + "\n", + " theme_persona_matching_prompt = ThemesPersonasMatchingPrompt()\n", + "\n", + " async def _generate_scenarios(\n", + " self,\n", + " n: int,\n", + " knowledge_graph,\n", + " persona_list,\n", + " callbacks,\n", + " ) -> t.List[MultiHopScenario]:\n", + "\n", + " # query and get (node_a, rel, node_b) to create multi-hop queries\n", + " results = kg.find_two_nodes_single_rel(\n", + " relationship_condition=lambda rel: (\n", + " True if rel.type == \"keyphrases_overlap\" else False\n", + " )\n", + " )\n", + "\n", + " num_sample_per_triplet = max(1, n // len(results))\n", + "\n", + " scenarios = []\n", + " for triplet in results:\n", + " if len(scenarios) < n:\n", + " node_a, node_b = triplet[0], triplet[-1]\n", + " overlapped_keywords = triplet[1].properties[\"overlapped_items\"]\n", + " if overlapped_keywords:\n", + "\n", + " # match the keyword with a persona for query creation\n", + " themes = list(dict(overlapped_keywords).keys())\n", + " prompt_input = ThemesPersonasInput(\n", + " themes=themes, personas=persona_list\n", + " )\n", + " persona_concepts = (\n", + " await self.theme_persona_matching_prompt.generate(\n", + " data=prompt_input, llm=self.llm, callbacks=callbacks\n", + " )\n", + " )\n", + "\n", + " overlapped_keywords = [list(item) for item in overlapped_keywords]\n", + "\n", + " # prepare and sample possible combinations\n", + " base_scenarios = self.prepare_combinations(\n", + " [node_a, node_b],\n", + " overlapped_keywords,\n", + " personas=persona_list,\n", + " persona_item_mapping=persona_concepts.mapping,\n", + " property_name=\"keyphrases\",\n", + " )\n", + "\n", + " # get number of required samples from this triplet\n", + " base_scenarios = self.sample_diverse_combinations(\n", + " base_scenarios, num_sample_per_triplet\n", + " )\n", + "\n", + " scenarios.extend(base_scenarios)\n", + "\n", + " return scenarios" + ] + }, + { + "cell_type": "code", + "execution_count": 138, + "id": "6935cdde-99c0-4893-8bd1-f72dc398eaee", + "metadata": {}, + "outputs": [], + "source": [ + "query = MyMultiHopQuery(llm=llm)\n", + "scenarios = await query.generate_scenarios(\n", + " n=10, knowledge_graph=kg, persona_list=persona_list\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 151, + "id": "78fec1b9-f8a1-4237-9721-65bdae7059f8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "MultiHopScenario(\n", + "nodes=2\n", + "combinations=['Diversity Inclusion & Belonging', 'Diversity, Inclusion & Belonging Goals']\n", + "style=Web search like queries\n", + "length=short\n", + "persona=name='Hiring manager at gitlab' role_description='A hiring manager at gitlab trying to underestand hiring policies in gitlab')" + ] + }, + "execution_count": 151, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scenarios[4]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49a38d27", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "61ae1d99", + "metadata": {}, + "source": [ + "### Run the multi-hop query" + ] + }, + { + "cell_type": "code", + "execution_count": 143, + "id": "da42bfb0-5122-4094-be22-6d6e74a9c0c0", + "metadata": {}, + "outputs": [], + "source": [ + "result = await query.generate_sample(scenario=scenarios[-1])" + ] + }, + { + "cell_type": "code", + "execution_count": 144, + "id": "d4a865a7-b14b-4aa0-8def-128120cebae9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'How does GitLab ensure that its DIB roundtables are effective in promoting diversity and inclusion?'" + ] + }, + "execution_count": 144, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.user_input" + ] + }, + { + "cell_type": "markdown", + "id": "b716f1a5", + "metadata": {}, + "source": [ + "Yay! You have created a multi-hop query. Now you can create any such queries by creating and exploring relationships between documents." + ] + }, + { + "cell_type": "markdown", + "id": "d5088c18-a8eb-4180-b066-46a8a795553b", + "metadata": {}, + "source": [ + "## " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ragas", + "language": "python", + "name": "ragas" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mkdocs.yml b/mkdocs.yml index 04a6b81ce..2b0782c9d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -85,6 +85,8 @@ nav: - Testset Generation: - howtos/customizations/testgenerator/index.md - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md + - Custom Single-hop Query: howtos/customizations/testgenerator/_testgen-custom-single-hop.md + - Custom Multi-hop Query: howtos/customizations/testgenerator/_testgen-customisation.md - Applications: - howtos/applications/index.md - Metrics: diff --git a/src/ragas/testset/graph.py b/src/ragas/testset/graph.py index fbc73c765..1e5e7fa08 100644 --- a/src/ragas/testset/graph.py +++ b/src/ragas/testset/graph.py @@ -316,12 +316,12 @@ def remove_node( ] return new_graph - def find_direct_clusters( + def find_two_nodes_single_rel( self, relationship_condition: t.Callable[[Relationship], bool] = lambda _: True - ) -> t.Dict[Node, t.List[t.Set[Node]]]: + ) -> t.List[t.Tuple[Node, Relationship, Node]]: """ - Finds direct clusters of nodes in the knowledge graph based on a relationship condition. - Here if A->B, and A->C, then A, B, and C form a cluster. + Finds nodes in the knowledge graph based on a relationship condition. + (NodeA, NodeB, Rel) triples are considered as multi-hop nodes. Parameters ---------- @@ -330,40 +330,34 @@ def find_direct_clusters( Returns ------- - List[Set[Node]] - A list of sets, where each set contains nodes that form a cluster. + List[Set[Node, Relationship, Node]] + A list of sets, where each set contains two nodes and a relationship forming a multi-hop node. """ - clusters = [] relationships = [ - rel for rel in self.relationships if relationship_condition(rel) + relationship + for relationship in self.relationships + if relationship_condition(relationship) ] - for node in self.nodes: - cluster = set() - cluster.add(node) - for rel in relationships: - if rel.bidirectional: - if rel.source == node: - cluster.add(rel.target) - elif rel.target == node: - cluster.add(rel.source) - else: - if rel.source == node: - cluster.add(rel.target) - if len(cluster) > 1: - if cluster not in clusters: - clusters.append(cluster) + triplets = set() - # Remove subsets from clusters - unique_clusters = [] - for cluster in clusters: - if not any(cluster < other for other in clusters): - unique_clusters.append(cluster) - clusters = unique_clusters + for relationship in relationships: + if relationship.source != relationship.target: + node_a = relationship.source + node_b = relationship.target + # Ensure the smaller ID node is always first + if node_a.id < node_b.id: + normalized_tuple = (node_a, relationship, node_b) + else: + normalized_relationship = Relationship( + source=node_b, + target=node_a, + type=relationship.type, + properties=relationship.properties, + ) + normalized_tuple = (node_b, normalized_relationship, node_a) - cluster_dict = {} - for cluster in clusters: - cluster_dict.update({cluster.pop(): cluster}) + triplets.add(normalized_tuple) - return cluster_dict + return list(triplets) diff --git a/src/ragas/testset/synthesizers/multi_hop/abstract.py b/src/ragas/testset/synthesizers/multi_hop/abstract.py index bd8d78a70..c464a9d99 100644 --- a/src/ragas/testset/synthesizers/multi_hop/abstract.py +++ b/src/ragas/testset/synthesizers/multi_hop/abstract.py @@ -9,7 +9,7 @@ from ragas.prompt import PydanticPrompt from ragas.testset.graph import KnowledgeGraph, Node from ragas.testset.graph_queries import get_child_nodes -from ragas.testset.persona import Persona, PersonaList +from ragas.testset.persona import Persona from ragas.testset.synthesizers.multi_hop.base import ( MultiHopQuerySynthesizer, MultiHopScenario, @@ -115,8 +115,8 @@ async def _generate_scenarios( base_scenarios = self.prepare_combinations( nodes, concept_combination.combinations, - PersonaList(personas=persona_list), - persona_concepts, + personas=persona_list, + persona_item_mapping=persona_concepts.mappping, property_name="themes", ) base_scenarios = self.sample_diverse_combinations( diff --git a/src/ragas/testset/synthesizers/multi_hop/base.py b/src/ragas/testset/synthesizers/multi_hop/base.py index e51a14623..7d8be7ea8 100644 --- a/src/ragas/testset/synthesizers/multi_hop/base.py +++ b/src/ragas/testset/synthesizers/multi_hop/base.py @@ -8,7 +8,7 @@ from ragas import SingleTurnSample from ragas.prompt import PydanticPrompt -from ragas.testset.persona import PersonaList +from ragas.testset.persona import Persona, PersonaList from ragas.testset.synthesizers.base import ( BaseScenario, BaseSynthesizer, @@ -43,6 +43,9 @@ class MultiHopScenario(BaseScenario): combinations: t.List[str] + def __repr__(self) -> str: + return f"MultiHopScenario(\nnodes={len(self.nodes)}\ncombinations={self.combinations}\nstyle={self.style}\nlength={self.length}\npersona={self.persona})" + @dataclass class MultiHopQuerySynthesizer(BaseSynthesizer[Scenario]): @@ -53,16 +56,17 @@ def prepare_combinations( self, nodes, combinations: t.List[t.List[str]], - persona_list: PersonaList, - persona_concepts, + personas: t.List[Persona], + persona_item_mapping: t.Dict[str, t.List[str]], property_name: str, ) -> t.List[t.Dict[str, t.Any]]: + persona_list = PersonaList(personas=personas) possible_combinations = [] for combination in combinations: dict = {"combination": combination} valid_personas = [] - for persona, concept_list in persona_concepts.mapping.items(): + for persona, concept_list in persona_item_mapping.items(): concept_list = [c.lower() for c in concept_list] if ( any(concept.lower() in concept_list for concept in combination) @@ -91,6 +95,9 @@ def sample_diverse_combinations( self, data: t.List[t.Dict[str, t.Any]], num_samples: int ) -> t.List[MultiHopScenario]: + if num_samples < 1: + raise ValueError("number of samples to generate should be greater than 0") + selected_samples = [] combination_persona_count = defaultdict(set) style_count = defaultdict(int) diff --git a/src/ragas/testset/synthesizers/multi_hop/prompts.py b/src/ragas/testset/synthesizers/multi_hop/prompts.py index a701eb6c6..279347856 100644 --- a/src/ragas/testset/synthesizers/multi_hop/prompts.py +++ b/src/ragas/testset/synthesizers/multi_hop/prompts.py @@ -80,7 +80,6 @@ class QueryAnswerGenerationPrompt( "that aligns with the persona’s perspective and reflects the themes.\n" "2. **Generate an Answer**: Using only the content from the provided context, create a faithful and detailed answer to " "the query. Do not include any information that not in or cannot be inferred from the given context.\n" - "### Example Outputs:\n\n" ) input_model: t.Type[QueryConditions] = QueryConditions output_model: t.Type[GeneratedQueryAnswer] = GeneratedQueryAnswer diff --git a/src/ragas/testset/synthesizers/multi_hop/specific.py b/src/ragas/testset/synthesizers/multi_hop/specific.py index 02d483278..bee082242 100644 --- a/src/ragas/testset/synthesizers/multi_hop/specific.py +++ b/src/ragas/testset/synthesizers/multi_hop/specific.py @@ -7,8 +7,8 @@ import numpy as np from ragas.prompt import PydanticPrompt -from ragas.testset.graph import KnowledgeGraph, Node -from ragas.testset.persona import Persona, PersonaList +from ragas.testset.graph import KnowledgeGraph +from ragas.testset.persona import Persona from ragas.testset.synthesizers.multi_hop.base import ( MultiHopQuerySynthesizer, MultiHopScenario, @@ -43,21 +43,6 @@ class MultiHopSpecificQuerySynthesizer(MultiHopQuerySynthesizer): theme_persona_matching_prompt: PydanticPrompt = ThemesPersonasMatchingPrompt() generate_query_reference_prompt: PydanticPrompt = QueryAnswerGenerationPrompt() - def get_node_clusters(self, knowledge_graph: KnowledgeGraph) -> t.List[t.Set[Node]]: - - cluster_dict = knowledge_graph.find_direct_clusters( - relationship_condition=lambda rel: ( - True if rel.type == self.relation_type else False - ) - ) - logger.info("found %d clusters", len(cluster_dict)) - node_clusters = [] - for key_node, list_of_nodes in cluster_dict.items(): - for node in list_of_nodes: - node_clusters.append((key_node, node)) - - return node_clusters - async def _generate_scenarios( self, n: int, @@ -78,30 +63,25 @@ async def _generate_scenarios( 4. Return the list of scenarios of length n """ - node_clusters = self.get_node_clusters(knowledge_graph) + triplets = knowledge_graph.find_two_nodes_single_rel( + relationship_condition=lambda rel: ( + True if rel.type == self.relation_type else False + ) + ) - if len(node_clusters) == 0: + if len(triplets) == 0: raise ValueError( "No clusters found in the knowledge graph. Try changing the relationship condition." ) - num_sample_per_cluster = int(np.ceil(n / len(node_clusters))) - - valid_relationships = [ - rel - for rel in knowledge_graph.relationships - if rel.type == self.relation_type - ] + num_sample_per_cluster = int(np.ceil(n / len(triplets))) scenarios = [] - for cluster in node_clusters: + for triplet in triplets: if len(scenarios) < n: - key_node, node = cluster + node_a, node_b = triplet[0], triplet[-1] overlapped_items = [] - for rel in valid_relationships: - if rel.source == key_node and rel.target == node: - overlapped_items = rel.get_property("overlapped_items") - break + overlapped_items = triplet[1].properties["overlapped_items"] if overlapped_items: themes = list(dict(overlapped_items).keys()) prompt_input = ThemesPersonasInput( @@ -114,10 +94,10 @@ async def _generate_scenarios( ) overlapped_items = [list(item) for item in overlapped_items] base_scenarios = self.prepare_combinations( - [key_node, node], + [node_a, node_b], overlapped_items, - PersonaList(personas=persona_list), - persona_concepts, + personas=persona_list, + persona_item_mapping=persona_concepts.mappping, property_name=self.property_name, ) base_scenarios = self.sample_diverse_combinations( diff --git a/src/ragas/testset/synthesizers/single_hop/base.py b/src/ragas/testset/synthesizers/single_hop/base.py index da8ecf368..a958117be 100644 --- a/src/ragas/testset/synthesizers/single_hop/base.py +++ b/src/ragas/testset/synthesizers/single_hop/base.py @@ -8,7 +8,7 @@ from ragas.dataset_schema import SingleTurnSample from ragas.prompt import PydanticPrompt from ragas.testset.graph import Node -from ragas.testset.persona import PersonaList +from ragas.testset.persona import Persona, PersonaList from ragas.testset.synthesizers.base import ( BaseScenario, BaseSynthesizer, @@ -39,6 +39,9 @@ class SingleHopScenario(BaseScenario): term: str + def __repr__(self) -> str: + return f"SingleHopScenario(\nnodes={len(self.nodes)}\nterm={self.term}\npersona={self.persona}\nstyle={self.style}\nlength={self.length})" + @dataclass class SingleHopQuerySynthesizer(BaseSynthesizer[Scenario]): @@ -49,13 +52,14 @@ def prepare_combinations( self, node: Node, terms: t.List[str], - persona_list: PersonaList, - persona_concepts, + personas: t.List[Persona], + persona_concepts: t.Dict[str, t.List[str]], ) -> t.List[t.Dict[str, t.Any]]: sample = {"terms": terms, "node": node} valid_personas = [] - for persona, concepts in persona_concepts.mapping.items(): + persona_list = PersonaList(personas=personas) + for persona, concepts in persona_concepts.items(): concepts = [concept.lower() for concept in concepts] if any(term.lower() in concepts for term in terms): if persona_list[persona]: diff --git a/src/ragas/testset/synthesizers/single_hop/prompts.py b/src/ragas/testset/synthesizers/single_hop/prompts.py index e01e2b65e..281a86c51 100644 --- a/src/ragas/testset/synthesizers/single_hop/prompts.py +++ b/src/ragas/testset/synthesizers/single_hop/prompts.py @@ -29,7 +29,6 @@ class QueryAnswerGenerationPrompt(PydanticPrompt[QueryCondition, GeneratedQueryA "that aligns with the persona's perspective and incorporates the term.\n" "2. **Generate an Answer**: Using only the content from the provided context, construct a detailed answer " "to the query. Do not add any information not included in or inferable from the context.\n" - "### Example Outputs:\n\n" ) input_model: t.Type[QueryCondition] = QueryCondition output_model: t.Type[GeneratedQueryAnswer] = GeneratedQueryAnswer diff --git a/src/ragas/testset/synthesizers/single_hop/specific.py b/src/ragas/testset/synthesizers/single_hop/specific.py index 7561ac7b7..ac0f1b367 100644 --- a/src/ragas/testset/synthesizers/single_hop/specific.py +++ b/src/ragas/testset/synthesizers/single_hop/specific.py @@ -9,7 +9,7 @@ from ragas.prompt import PydanticPrompt from ragas.testset.graph import KnowledgeGraph, Node -from ragas.testset.persona import Persona, PersonaList +from ragas.testset.persona import Persona from ragas.testset.synthesizers.base import BaseScenario from ragas.testset.synthesizers.prompts import ( ThemesPersonasInput, @@ -108,7 +108,10 @@ async def _generate_scenarios( data=prompt_input, llm=self.llm, callbacks=callbacks ) base_scenarios = self.prepare_combinations( - node, themes, PersonaList(personas=persona_list), persona_concepts + node, + themes, + personas=persona_list, + persona_concepts=persona_concepts.mapping, ) scenarios.extend(self.sample_combinations(base_scenarios, samples_per_node)) diff --git a/src/ragas/testset/transforms/__init__.py b/src/ragas/testset/transforms/__init__.py index f8e22040b..02fdb5716 100644 --- a/src/ragas/testset/transforms/__init__.py +++ b/src/ragas/testset/transforms/__init__.py @@ -19,6 +19,10 @@ CosineSimilarityBuilder, SummaryCosineSimilarityBuilder, ) +from .relationship_builders.traditional import ( + JaccardSimilarityBuilder, + OverlapScoreBuilder, +) from .splitters import HeadlineSplitter __all__ = [ @@ -46,4 +50,6 @@ "HeadlineSplitter", "CustomNodeFilter", "NodeFilter", + "JaccardSimilarityBuilder", + "OverlapScoreBuilder", ] diff --git a/src/ragas/testset/transforms/relationship_builders/traditional.py b/src/ragas/testset/transforms/relationship_builders/traditional.py index d7ae6ebb0..ad33ea42f 100644 --- a/src/ragas/testset/transforms/relationship_builders/traditional.py +++ b/src/ragas/testset/transforms/relationship_builders/traditional.py @@ -60,7 +60,7 @@ class OverlapScoreBuilder(RelationshipBuilder): new_property_name: str = "overlap_score" distance_measure: DistanceMeasure = DistanceMeasure.JARO_WINKLER distance_threshold: float = 0.9 - threshold: float = 0.5 + threshold: float = 0.01 def __post_init__(self): try: diff --git a/src/ragas/testset/transforms/splitters/headline.py b/src/ragas/testset/transforms/splitters/headline.py index 2f0a58910..8affc65e4 100644 --- a/src/ragas/testset/transforms/splitters/headline.py +++ b/src/ragas/testset/transforms/splitters/headline.py @@ -52,6 +52,8 @@ async def split(self, node: Node) -> t.Tuple[t.List[Node], t.List[Relationship]] if headlines is None: raise ValueError("'headlines' property not found in this node") + if len(text.split()) < self.min_tokens: + return [node], [] # create the chunks for the different sections indices = [0] for headline in headlines: