From 55ccaa588a5b6f2e03a43150dc7056af9a5779d1 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Fri, 12 Jan 2024 18:25:26 -0800 Subject: [PATCH 01/10] added alternate testset generator --- src/ragas/llms/__init__.py | 1 + src/ragas/testset/generator.py | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 src/ragas/testset/generator.py diff --git a/src/ragas/llms/__init__.py b/src/ragas/llms/__init__.py index 9d6285b6..82c6b887 100644 --- a/src/ragas/llms/__init__.py +++ b/src/ragas/llms/__init__.py @@ -4,6 +4,7 @@ __all__ = [ "BaseRagasLLM", + "LangchainLLMWrapper", "llm_factory", ] diff --git a/src/ragas/testset/generator.py b/src/ragas/testset/generator.py new file mode 100644 index 00000000..9544b477 --- /dev/null +++ b/src/ragas/testset/generator.py @@ -0,0 +1,38 @@ +import typing as t +from dataclasses import dataclass + +from langchain.chat_models import ChatOpenAI +from langchain.embeddings import OpenAIEmbeddings +from ragas.llms import BaseRagasLLM, LangchainLLMWrapper +from ragas.embeddings import BaseRagasEmbeddings + +from llama_index.readers.schema import Document as LlamaindexDocument + + +@dataclass +class TestsetGenerator: + generator_llm: BaseRagasLLM + critic_llm: BaseRagasLLM + embeddings: BaseRagasEmbeddings + + @classmethod + def with_openai( + cls, + generator_llm: str = "gpt-3.5-turbo", + critic_llm: str = "gpt-4", + embeddings: str = "text-embedding-ada-002", + ) -> "TestsetGenerator": + generator_llm_model = LangchainLLMWrapper(ChatOpenAI(model=generator_llm)) + critic_llm_model = LangchainLLMWrapper(ChatOpenAI(model=critic_llm)) + embeddings_model = OpenAIEmbeddings(model=embeddings) + return cls( + generator_llm=generator_llm_model, + critic_llm=critic_llm_model, + embeddings=embeddings_model, + ) + + def generate_with_llamaindex_docs(self, documents: t.Sequence[LlamaindexDocument]): + print(len(documents)) + # chunk documents and add to docstore + # create evolutions and add to executor queue + # run till completion - keep updating progress bar From 0e6b4f0d5b17a01ca49d29cdd82b57cf504aafd1 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sat, 13 Jan 2024 10:31:31 -0800 Subject: [PATCH 02/10] add_documents complete --- src/ragas/testset/docstore.py | 154 ++++++++++++++++++++------------ src/ragas/testset/evolutions.py | 53 ++++++++--- 2 files changed, 141 insertions(+), 66 deletions(-) diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index f87e5112..6cf84351 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -1,5 +1,6 @@ import heapq import typing as t +import logging import uuid from abc import ABC, abstractmethod from dataclasses import dataclass, field @@ -14,6 +15,7 @@ from ragas.async_utils import run_async_tasks from ragas.embeddings.base import BaseRagasEmbeddings, embedding_factory +logger = logging.getLogger(__name__) Embedding = t.Union[t.List[float], npt.NDArray[np.float64]] @@ -22,28 +24,57 @@ class Document(LCDocument): filename: t.Optional[str] = None embedding: t.Optional[t.List[float]] = Field(default=None, repr=False) + @classmethod + def from_langchain_document(cls, doc: LCDocument): + doc_id = str(uuid.uuid4()) + if doc.metadata.get("filename"): + filename = doc.metadata["filename"] + else: + logger.info( + "Document [ID: %s] has no filename. Using doc_id as filename.", doc_id + ) + filename = doc_id + return cls( + page_content=doc.page_content, + metadata=doc.metadata, + doc_id=doc_id, + filename=filename, + ) + + +class Node(Document): + ... + class DocumentStore(ABC): def __init__(self): self.documents = {} @abstractmethod - def add(self, doc: t.Union[Document, t.Sequence[Document]], show_progress=True): + def add_documents(self, docs: t.Sequence[Document], show_progress=True): + ... + + @abstractmethod + def add_nodes(self, nodes: t.Sequence[Node], show_progress=True): + ... + + @abstractmethod + def get_document(self, doc_id: str) -> Document: ... @abstractmethod - def get(self, doc_id: str) -> Document: + def get_node(self, node_id: str) -> Node: ... @abstractmethod def get_similar( - self, doc: Document, threshold: float = 0.7, top_k: int = 3 - ) -> t.List[Document]: + self, item: t.Union[Document, Node], threshold: float = 0.7, top_k: int = 3 + ) -> t.Union[t.List[Document], t.List[Node]]: ... @abstractmethod def get_adjascent( - self, doc: Document, direction: str = "next" + self, item: t.Union[Document, Node], direction: str = "next" ) -> t.Optional[Document]: ... @@ -117,75 +148,86 @@ class InMemoryDocumentStore(DocumentStore): embeddings: BaseRagasEmbeddings = field( default_factory=embedding_factory, repr=False ) - documents_list: t.List[Document] = field(default_factory=list) + nodes: t.List[Document] = field(default_factory=list) embeddings_list: t.List[Embedding] = field(default_factory=list) - documents_map: t.Dict[str, Document] = field(default_factory=dict) + node_map: t.Dict[str, Document] = field(default_factory=dict) - def _add_documents_batch(self, docs: t.Sequence[Document], show_progress=True): + def _embed_items(self, items: t.Union[t.Sequence[Document], t.Sequence[Node]]): + ... + + def add_documents(self, docs: t.Sequence[Document], show_progress=True): """ Add documents in batch mode. """ # NOTE: Adds everything in async mode for now. embed_tasks = [] docs_to_embed = [] - for doc in docs: - if doc.embedding is None: - embed_tasks.append(self.embeddings.aembed_query(doc.page_content)) - docs_to_embed.append(doc) + + # split documents with self.splitter + nodes = [ + Document.from_langchain_document(d) + for d in self.splitter.transform_documents(docs) + ] + + # get embeddings for the docs + for n in nodes: + if n.embedding is None: + embed_tasks.append(self.embeddings.aembed_query(n.page_content)) + docs_to_embed.append(n) else: - self.documents_list.append(doc) - self.documents_map[doc.doc_id] = doc - self.embeddings_list.append(doc.embedding) + self.nodes.append(n) + self.node_map[n.doc_id] = n + self.embeddings_list.append(n.embedding) embeddings = run_async_tasks(embed_tasks, show_progress=show_progress) - for doc, embedding in zip(docs_to_embed, embeddings): - doc.embedding = embedding - self.documents_list.append(doc) - self.documents_map[doc.doc_id] = doc - self.embeddings_list.append(doc.embedding) - - def add(self, doc: t.Union[Document, t.Sequence[Document]], show_progress=True): - if isinstance(doc, list) or isinstance(doc, tuple): - self._add_documents_batch(doc) - elif isinstance(doc, Document): - self.documents_list.append(doc) - self.documents_map[doc.doc_id] = doc - if doc.embedding is None: - doc.embedding = self.embeddings.embed_query(doc.page_content) - self.embeddings_list.append(doc.embedding) - else: - raise ValueError("add() method only supports Document or List[Document]") + for n, embedding in zip(docs_to_embed, embeddings): + n.embedding = embedding + self.nodes.append(n) + self.node_map[n.doc_id] = n + self.embeddings_list.append(n.embedding) - def get(self, doc_id: str) -> Document: - return self.documents_map[doc_id] + def add_nodes(self, nodes: t.Sequence[Node], show_progress=True): + raise NotImplementedError + + def get_document(self, doc_id: str) -> Document: + return self.node_map[doc_id] + + def get_node(self, node_id: str) -> Document: + ... def get_similar( - self, doc: Document, threshold: float = 0.7, top_k: int = 3 - ) -> t.List[Document]: - if doc.embedding is None: - raise ValueError("Document has no embedding.") - scores, doc_ids = get_top_k_embeddings( - query_embedding=doc.embedding, - embeddings=self.embeddings_list, - similarity_fn=similarity, - similarity_cutoff=threshold, - # we need to return k+1 docs here as the top result is the input doc itself - similarity_top_k=top_k + 1, - ) - # remove the query doc itself from results - scores, doc_ids = scores[1:], doc_ids[1:] - return [self.documents_list[doc_id] for doc_id in doc_ids] + self, item: t.Union[Document, Node], threshold: float = 0.7, top_k: int = 3 + ) -> t.Union[t.List[Document], t.List[Node]]: + items = [] + if isinstance(item, Node): + ... + elif isinstance(item, Document): + doc = item + if doc.embedding is None: + raise ValueError("Document has no embedding.") + scores, doc_ids = get_top_k_embeddings( + query_embedding=doc.embedding, + embeddings=self.embeddings_list, + similarity_fn=similarity, + similarity_cutoff=threshold, + # we need to return k+1 docs here as the top result is the input doc itself + similarity_top_k=top_k + 1, + ) + # remove the query doc itself from results + scores, doc_ids = scores[1:], doc_ids[1:] + items = [self.nodes[doc_id] for doc_id in doc_ids] + return items def get_adjascent( - self, doc: Document, direction: str = "next" + self, item: t.Union[Document, Node], direction: str = "next" ) -> t.Optional[Document]: # linear search for doc_id of doc in documents_list - index = self.documents_list.index(doc) + index = self.nodes.index(item) if direction == "next": - if len(self.documents_list) > index + 1: - next_doc = self.documents_list[index + 1] - if next_doc.filename == doc.filename: + if len(self.nodes) > index + 1: + next_doc = self.nodes[index + 1] + if next_doc.filename == item.filename: return next_doc else: return None @@ -193,8 +235,8 @@ def get_adjascent( return None if direction == "prev": if index > 0: - prev_doc = self.documents_list[index - 1] - if prev_doc.filename == doc.filename: + prev_doc = self.nodes[index - 1] + if prev_doc.filename == item.filename: return prev_doc else: return None diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index 31521225..a68ea5f8 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -1,7 +1,10 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +import logging +import typing as t from langchain.prompts import ChatPromptTemplate +from numpy.random import default_rng from ragas.llms import BaseRagasLLM from ragas.llms.json_load import load_as_json @@ -12,8 +15,13 @@ MULTICONTEXT_QUESTION, SCORE_CONTEXT, SEED_QUESTION, + TABLE_QA, + demonstrations, ) +rng = default_rng() +logger = logging.getLogger(__name__) + @dataclass class Filter(ABC): @@ -30,21 +38,23 @@ def to_pv(prompt: ChatPromptTemplate) -> PromptValue: return PromptValue(prompt_str=prompt.format()) -async def filter_context( - llm: BaseRagasLLM, context: str, threshold: float = 7.5 -) -> bool: +async def filter_node( + llm: BaseRagasLLM, node: Document, threshold: float = 7.5 +) -> t.Dict: """ context: str The input context Checks if the context is has enough information to frame a question """ - human_prompt = SCORE_CONTEXT.format(context=context) + human_prompt = SCORE_CONTEXT.format(context=node.page_content) prompt = ChatPromptTemplate.from_messages([human_prompt]) results = await llm.agenerate_text(prompt=to_pv(prompt)) output = results.generations[0][0].text.strip() score = load_as_json(output) - return score >= threshold # type: ignore + # TODO: instead of updating score add a new "pass" key + score.update({"score": score.get("score", 0) >= threshold}) + return score async def filter_question(llm: BaseRagasLLM, question: str) -> bool: @@ -54,6 +64,7 @@ async def filter_question(llm: BaseRagasLLM, question: str) -> bool: results = await llm.agenerate_text(prompt=to_pv(prompt)) results = results.generations[0][0].text.strip() json_results = load_as_json(results) + logger.debug("filtered question: %s", json_results) return json_results.get("verdict") != "No" @@ -66,12 +77,34 @@ async def aevolve(self): ... -async def simple_evolution(llm: BaseRagasLLM, seed_doc: Document): - human_prompt = SEED_QUESTION.format(context=seed_doc.page_content) +async def simple_evolution( + llm: BaseRagasLLM, seed_doc: Document, is_table_present: bool = False +): + if is_table_present: + human_prompt = TABLE_QA.format(context=seed_doc.page_content) + else: + sample = rng.choice(demonstrations, 1)[0] # type: ignore + questions = rng.choice(sample["questions"], 2, replace=False) + questions = ( + "{" + + str({k: v for dic in questions.tolist() for k, v in dic.items()}).replace( + "'", '"' + ) + + "}" + ) + demo = f'Context:{sample["context"]}\nQuestions:{questions}' + human_prompt = SEED_QUESTION.format( + demonstration=demo, context=seed_doc.page_content + ) + prompt = ChatPromptTemplate.from_messages([human_prompt]) - results = await llm.agenerate_text(prompt=to_pv(prompt)) - question = results.generations[0][0].text.strip() - return question + results = llm.generate_text_with_hmpt(prompts=[prompt]) + results = results.generations[0][0].text + if is_table_present: + return [results] + else: + results = load_as_json(results) + return [v for v in results.values()] async def multi_context_evolution( From 217a46b78a5dd165288cc679fc5874f241df607c Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sat, 13 Jan 2024 17:45:14 -0800 Subject: [PATCH 03/10] added the concept of nodes in docstore --- src/ragas/testset/docstore.py | 124 +++++++++++------- tests/unit/testset_generator/test_docstore.py | 74 +++++------ 2 files changed, 114 insertions(+), 84 deletions(-) diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index 6cf84351..49b96015 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -5,18 +5,21 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from enum import Enum +from random import choices import numpy as np import numpy.typing as npt from langchain.text_splitter import TextSplitter from langchain_core.documents import Document as LCDocument -from pydantic import Field +from llama_index.readers.schema import Document as LlamaindexDocument +from langchain_core.pydantic_v1 import Field from ragas.async_utils import run_async_tasks from ragas.embeddings.base import BaseRagasEmbeddings, embedding_factory -logger = logging.getLogger(__name__) Embedding = t.Union[t.List[float], npt.NDArray[np.float64]] +logger = logging.getLogger(__name__) +rng = np.random.default_rng() class Document(LCDocument): @@ -41,11 +44,39 @@ def from_langchain_document(cls, doc: LCDocument): filename=filename, ) + @classmethod + def from_llamaindex_document(cls, doc: LlamaindexDocument): + doc_id = str(uuid.uuid4()) + if doc.metadata.get("filename"): + filename = doc.metadata["filename"] + else: + logger.info( + "Document [ID: %s] has no filename. Using doc_id as filename.", doc_id + ) + filename = doc_id + return cls( + page_content=doc.text, + metadata=doc.metadata, + doc_id=doc_id, + filename=filename, + ) + class Node(Document): ... +class Direction(str, Enum): + """ + Direction for getting adjascent nodes. + """ + + NEXT = "next" + PREV = "prev" + UP = "up" + DOWN = "down" + + class DocumentStore(ABC): def __init__(self): self.documents = {} @@ -59,22 +90,22 @@ def add_nodes(self, nodes: t.Sequence[Node], show_progress=True): ... @abstractmethod - def get_document(self, doc_id: str) -> Document: + def get_node(self, node_id: str) -> Node: ... @abstractmethod - def get_node(self, node_id: str) -> Node: + def get_random_nodes(self, k=1) -> t.List[Node]: ... @abstractmethod def get_similar( - self, item: t.Union[Document, Node], threshold: float = 0.7, top_k: int = 3 + self, node: Node, threshold: float = 0.7, top_k: int = 3 ) -> t.Union[t.List[Document], t.List[Node]]: ... @abstractmethod def get_adjascent( - self, item: t.Union[Document, Node], direction: str = "next" + self, node: Node, direction: Direction = Direction.NEXT ) -> t.Optional[Document]: ... @@ -148,9 +179,9 @@ class InMemoryDocumentStore(DocumentStore): embeddings: BaseRagasEmbeddings = field( default_factory=embedding_factory, repr=False ) - nodes: t.List[Document] = field(default_factory=list) - embeddings_list: t.List[Embedding] = field(default_factory=list) - node_map: t.Dict[str, Document] = field(default_factory=dict) + nodes: t.List[Node] = field(default_factory=list) + node_embeddings_list: t.List[Embedding] = field(default_factory=list) + node_map: t.Dict[str, Node] = field(default_factory=dict) def _embed_items(self, items: t.Union[t.Sequence[Document], t.Sequence[Node]]): ... @@ -159,16 +190,18 @@ def add_documents(self, docs: t.Sequence[Document], show_progress=True): """ Add documents in batch mode. """ - # NOTE: Adds everything in async mode for now. - embed_tasks = [] - docs_to_embed = [] - - # split documents with self.splitter + # split documents with self.splitter into smaller nodes nodes = [ - Document.from_langchain_document(d) + Node.from_langchain_document(d) for d in self.splitter.transform_documents(docs) ] + self.add_nodes(nodes, show_progress=show_progress) + + def add_nodes(self, nodes: t.Sequence[Node], show_progress=True): + # NOTE: Adds everything in async mode for now. + embed_tasks = [] + docs_to_embed = [] # get embeddings for the docs for n in nodes: if n.embedding is None: @@ -177,66 +210,63 @@ def add_documents(self, docs: t.Sequence[Document], show_progress=True): else: self.nodes.append(n) self.node_map[n.doc_id] = n - self.embeddings_list.append(n.embedding) + self.node_embeddings_list.append(n.embedding) embeddings = run_async_tasks(embed_tasks, show_progress=show_progress) for n, embedding in zip(docs_to_embed, embeddings): n.embedding = embedding self.nodes.append(n) self.node_map[n.doc_id] = n - self.embeddings_list.append(n.embedding) + self.node_embeddings_list.append(n.embedding) - def add_nodes(self, nodes: t.Sequence[Node], show_progress=True): - raise NotImplementedError + def get_node(self, node_id: str) -> Node: + return self.node_map[node_id] - def get_document(self, doc_id: str) -> Document: - return self.node_map[doc_id] + def get_document(self, doc_id: str) -> Node: + raise NotImplementedError - def get_node(self, node_id: str) -> Document: - ... + def get_random_nodes(self, k=1) -> t.List[Node]: + return choices(self.nodes, k=k) def get_similar( - self, item: t.Union[Document, Node], threshold: float = 0.7, top_k: int = 3 + self, node: Node, threshold: float = 0.7, top_k: int = 3 ) -> t.Union[t.List[Document], t.List[Node]]: items = [] - if isinstance(item, Node): - ... - elif isinstance(item, Document): - doc = item - if doc.embedding is None: - raise ValueError("Document has no embedding.") - scores, doc_ids = get_top_k_embeddings( - query_embedding=doc.embedding, - embeddings=self.embeddings_list, - similarity_fn=similarity, - similarity_cutoff=threshold, - # we need to return k+1 docs here as the top result is the input doc itself - similarity_top_k=top_k + 1, - ) - # remove the query doc itself from results - scores, doc_ids = scores[1:], doc_ids[1:] - items = [self.nodes[doc_id] for doc_id in doc_ids] + doc = node + if doc.embedding is None: + raise ValueError("Document has no embedding.") + scores, doc_ids = get_top_k_embeddings( + query_embedding=doc.embedding, + embeddings=self.node_embeddings_list, + similarity_fn=similarity, + similarity_cutoff=threshold, + # we need to return k+1 docs here as the top result is the input doc itself + similarity_top_k=top_k + 1, + ) + # remove the query doc itself from results + scores, doc_ids = scores[1:], doc_ids[1:] + items = [self.nodes[doc_id] for doc_id in doc_ids] return items def get_adjascent( - self, item: t.Union[Document, Node], direction: str = "next" + self, node: Node, direction: Direction = Direction.NEXT ) -> t.Optional[Document]: # linear search for doc_id of doc in documents_list - index = self.nodes.index(item) + index = self.nodes.index(node) - if direction == "next": + if direction == Direction.NEXT: if len(self.nodes) > index + 1: next_doc = self.nodes[index + 1] - if next_doc.filename == item.filename: + if next_doc.filename == node.filename: return next_doc else: return None else: return None - if direction == "prev": + if direction == Direction.PREV: if index > 0: prev_doc = self.nodes[index - 1] - if prev_doc.filename == item.filename: + if prev_doc.filename == node.filename: return prev_doc else: return None diff --git a/tests/unit/testset_generator/test_docstore.py b/tests/unit/testset_generator/test_docstore.py index 338067a3..954e2f9e 100644 --- a/tests/unit/testset_generator/test_docstore.py +++ b/tests/unit/testset_generator/test_docstore.py @@ -5,28 +5,28 @@ import pytest from langchain_core.embeddings import Embeddings -from ragas.testset.docstore import Document, InMemoryDocumentStore +from ragas.testset.docstore import InMemoryDocumentStore, Node, Direction def test_adjacent_nodes(): - a1 = Document(doc_id="a1", page_content="a1", filename="a") - a2 = Document(doc_id="a2", page_content="a2", filename="a") - b = Document(doc_id="b", page_content="b", filename="b") + a1 = Node(doc_id="a1", page_content="a1", filename="a") + a2 = Node(doc_id="a2", page_content="a2", filename="a") + b = Node(doc_id="b", page_content="b", filename="b") store = InMemoryDocumentStore(splitter=None) # type: ignore - store.documents_list = [a1, a2, b] + store.nodes = [a1, a2, b] assert store.get_adjascent(a1) == a2 - assert store.get_adjascent(a2, "prev") == a1 - assert store.get_adjascent(a2, "next") is None - assert store.get_adjascent(b, "prev") is None + assert store.get_adjascent(a2, Direction.PREV) == a1 + assert store.get_adjascent(a2, Direction.NEXT) is None + assert store.get_adjascent(b, Direction.PREV) is None # raise ValueError if doc not in store - c = Document(doc_id="c", page_content="c", filename="c") + c = Node(doc_id="c", page_content="c", filename="c") pytest.raises(ValueError, store.get_adjascent, c) -def create_test_documents(with_embeddings=True): +def create_test_nodes(with_embeddings=True): if with_embeddings: path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_embs.pkl") with open(path, "rb") as f: @@ -35,13 +35,13 @@ def create_test_documents(with_embeddings=True): from collections import defaultdict embeddings = defaultdict(lambda: None) - a1 = Document( + a1 = Node( doc_id="a1", page_content="cat", filename="a", embedding=embeddings["cat"] ) - a2 = Document( + a2 = Node( doc_id="a2", page_content="mouse", filename="a", embedding=embeddings["mouse"] ) - b = Document( + b = Node( doc_id="b", page_content="solar_system", filename="b", @@ -52,10 +52,10 @@ def create_test_documents(with_embeddings=True): def test_similar_nodes(): - a1, a2, b = create_test_documents() + a1, a2, b = create_test_nodes() store = InMemoryDocumentStore(splitter=None) # type: ignore - store.documents_list = [a1, a2, b] - store.embeddings_list = [d.embedding for d in store.documents_list] + store.nodes = [a1, a2, b] + store.node_embeddings_list = [d.embedding for d in store.nodes] assert store.get_similar(a1)[0] == a2 assert store.get_similar(a2)[0] == a1 @@ -65,10 +65,10 @@ def test_similar_nodes(): def test_similar_nodes_scaled(): - a1, a2, b = create_test_documents() - store = InMemoryDocumentStore(splitter=None) # type: ignore - store.documents_list = [a1, a2, b] + [b] * 100 - store.embeddings_list = [d.embedding for d in store.documents_list] + a1, a2, b = create_test_nodes() + store = InMemoryDocumentStore(splitter=None) # type: ignore (None type is not Splitter) + store.nodes = [a1, a2, b] + [b] * 100 + store.node_embeddings_list = [d.embedding for d in store.nodes] assert len(store.get_similar(a1, top_k=3)) == 3 assert store.get_similar(a1)[0] == a2 @@ -76,16 +76,16 @@ def test_similar_nodes_scaled(): def test_docstore_add(): - a1, a2, b = create_test_documents() + a1, a2, b = create_test_nodes() store = InMemoryDocumentStore(splitter=None) # type: ignore docs_added = [] for doc in [a1, a2, b]: - store.add(doc) + store.add_nodes([doc]) docs_added.append(doc) - assert store.documents_list == docs_added - assert store.embeddings_list == [d.embedding for d in docs_added] + assert store.nodes == docs_added + assert store.node_embeddings_list == [d.embedding for d in docs_added] - assert store.get(a1.doc_id) == a1 + assert store.get_node(a1.doc_id) == a1 class FakeEmbeddings(Embeddings): @@ -130,20 +130,20 @@ def test_docstore_add_batch(): store = InMemoryDocumentStore(splitter=None, embeddings=fake_embeddings) # type: ignore # add documents in batch - docs = create_test_documents(with_embeddings=False) - store.add(docs) + nodes = create_test_nodes(with_embeddings=False) + store.add_nodes(nodes) assert ( - store.documents_map[docs[0].doc_id].embedding - == fake_embeddings.embeddings[docs[0].page_content] + store.node_map[nodes[0].doc_id].embedding + == fake_embeddings.embeddings[nodes[0].page_content] ) # add documents in batch that have some embeddings - c = Document(doc_id="c", page_content="c", filename="c", embedding=[0.0] * 768) - d = Document(doc_id="d", page_content="d", filename="d", embedding=[0.0] * 768) - store.add([c, d]) + c = Node(doc_id="c", page_content="c", filename="c", embedding=[0.0] * 768) + d = Node(doc_id="d", page_content="d", filename="d", embedding=[0.0] * 768) + store.add_nodes([c, d]) # test get() and that embeddings are correct - assert store.get(c.doc_id).embedding == [0.0] * 768 - assert store.get(d.doc_id).embedding == [0.0] * 768 - assert len(store.documents_list) == 5 - assert len(store.embeddings_list) == 5 - assert len(store.documents_map) == 5 + assert store.get_node(c.doc_id).embedding == [0.0] * 768 + assert store.get_node(d.doc_id).embedding == [0.0] * 768 + assert len(store.nodes) == 5 + assert len(store.node_embeddings_list) == 5 + assert len(store.node_map) == 5 From 43dd01fe38cee9ea368bb24f25ccccf512ba8177 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 14 Jan 2024 08:01:51 -0800 Subject: [PATCH 04/10] fixed handling of nan values in executor --- src/ragas/executor.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ragas/executor.py b/src/ragas/executor.py index ca66c6a9..54d88f33 100644 --- a/src/ragas/executor.py +++ b/src/ragas/executor.py @@ -9,6 +9,7 @@ @dataclass class Executor: + desc: str = "Evaluating" is_async: bool = True max_workers: t.Optional[int] = None futures: t.List[t.Any] = field(default_factory=list, repr=False) @@ -71,10 +72,10 @@ async def _aresults(self) -> t.List[t.Any]: results = [] for future in tqdm( asyncio.as_completed(self.futures), - desc="Evaluating", + desc=self.desc, total=len(self.futures), ): - r = np.nan + r = (-1, np.nan) try: r = await future except Exception as e: @@ -106,14 +107,14 @@ def results(self) -> t.List[t.Any]: try: for future in tqdm( as_completed(self.futures), - desc="Evaluating", + desc=self.desc, total=len(self.futures), ): - r = np.nan + r = (-1, np.nan) try: r = future.result() except Exception as e: - r = np.nan + r = (-1, np.nan) if self.raise_exceptions: raise e finally: @@ -121,5 +122,6 @@ def results(self) -> t.List[t.Any]: finally: self.executor.shutdown(wait=False) + print(results) sorted_results = sorted(results, key=lambda x: x[0]) return [r[1] for r in sorted_results] From 02254fe96f330c9005b4d582d1f3e92b2e659282 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 14 Jan 2024 08:02:11 -0800 Subject: [PATCH 05/10] basic simple evaluation working --- src/ragas/executor.py | 6 +- src/ragas/testset/docstore.py | 8 +- src/ragas/testset/evolutions.py | 143 ++++++++++++++++++++++++++------ src/ragas/testset/generator.py | 7 +- src/ragas/testset/prompts.py | 3 +- 5 files changed, 131 insertions(+), 36 deletions(-) diff --git a/src/ragas/executor.py b/src/ragas/executor.py index 54d88f33..112263a9 100644 --- a/src/ragas/executor.py +++ b/src/ragas/executor.py @@ -75,7 +75,7 @@ async def _aresults(self) -> t.List[t.Any]: desc=self.desc, total=len(self.futures), ): - r = (-1, np.nan) + r = (-1, None) try: r = await future except Exception as e: @@ -110,11 +110,11 @@ def results(self) -> t.List[t.Any]: desc=self.desc, total=len(self.futures), ): - r = (-1, np.nan) + r = (-1, None) try: r = future.result() except Exception as e: - r = (-1, np.nan) + r = (-1, None) if self.raise_exceptions: raise e finally: diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index 49b96015..ad9ab7d9 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -104,9 +104,9 @@ def get_similar( ... @abstractmethod - def get_adjascent( + def get_adjacent( self, node: Node, direction: Direction = Direction.NEXT - ) -> t.Optional[Document]: + ) -> t.Optional[Node]: ... @@ -248,9 +248,9 @@ def get_similar( items = [self.nodes[doc_id] for doc_id in doc_ids] return items - def get_adjascent( + def get_adjacent( self, node: Node, direction: Direction = Direction.NEXT - ) -> t.Optional[Document]: + ) -> t.Optional[Node]: # linear search for doc_id of doc in documents_list index = self.nodes.index(node) diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index a68ea5f8..7b4a3801 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -1,7 +1,9 @@ from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field import logging import typing as t +from fsspec.exceptions import asyncio +from random import choice, choices from langchain.prompts import ChatPromptTemplate from numpy.random import default_rng @@ -9,7 +11,7 @@ from ragas.llms import BaseRagasLLM from ragas.llms.json_load import load_as_json from ragas.llms.prompt import PromptValue -from ragas.testset.docstore import Document, DocumentStore +from ragas.testset.docstore import Document, DocumentStore, Direction from ragas.testset.prompts import ( FILTER_QUESTION, MULTICONTEXT_QUESTION, @@ -18,24 +20,46 @@ TABLE_QA, demonstrations, ) +from ragas.testset.docstore import Node rng = default_rng() logger = logging.getLogger(__name__) +def to_pv(prompt: ChatPromptTemplate) -> PromptValue: + return PromptValue(prompt_str=prompt.format()) + + @dataclass class Filter(ABC): - @abstractmethod - def filter(self) -> bool: - ... + ... + # TODO: talk with @shahul about how we can unify the type - @abstractmethod - async def afilter(self) -> bool: - ... + # @abstractmethod + # def filter(self) -> t.Any: + # ... + # + # @abstractmethod + # async def afilter(self) -> t.Any: + # ... -def to_pv(prompt: ChatPromptTemplate) -> PromptValue: - return PromptValue(prompt_str=prompt.format()) +@dataclass +class NodeFilter(Filter): + llm: BaseRagasLLM + threshold: float = 7.5 + + def filter(self, node: Node) -> t.Dict: + return asyncio.get_event_loop().run_until_complete(self.afilter(node)) + + async def afilter(self, node: Node) -> t.Dict: + human_prompt = SCORE_CONTEXT.format(context=node.page_content) + prompt = ChatPromptTemplate.from_messages([human_prompt]) + results = await self.llm.agenerate_text(prompt=to_pv(prompt)) + output = results.generations[0][0].text.strip() + score = load_as_json(output) + score.update({"score": score.get("score", 0) >= self.threshold}) + return score async def filter_node( @@ -57,24 +81,91 @@ async def filter_node( return score -async def filter_question(llm: BaseRagasLLM, question: str) -> bool: - human_prompt = FILTER_QUESTION.format(question=question) - prompt = ChatPromptTemplate.from_messages([human_prompt]) +@dataclass +class QuestionFilter(Filter): + llm: BaseRagasLLM - results = await llm.agenerate_text(prompt=to_pv(prompt)) - results = results.generations[0][0].text.strip() - json_results = load_as_json(results) - logger.debug("filtered question: %s", json_results) - return json_results.get("verdict") != "No" + def filter(self, question: str) -> bool: + return asyncio.get_event_loop().run_until_complete(self.afilter(question)) + + async def afilter(self, question: str) -> bool: + human_prompt = FILTER_QUESTION.format(question=question) + prompt = ChatPromptTemplate.from_messages([human_prompt]) + + results = await self.llm.agenerate_text(prompt=to_pv(prompt)) + results = results.generations[0][0].text.strip() + json_results = load_as_json(results) + logger.debug("filtered question: %s", json_results) + return json_results.get("verdict") != "No" @dataclass class Evolution: - def evolve(self): - ... + ... + + +@dataclass +class SimpleEvolution(Evolution): + node_filter: NodeFilter + question_filter: QuestionFilter + nodes: t.List[Node] = field(default_factory=list) + max_ties: int = 5 + _tries: int = field(default=0, init=False, repr=False) + + def merged_nodes(self) -> Node: + return Node( + doc_id="merged", page_content=" ".join(n.page_content for n in self.nodes) + ) - async def aevolve(self): - ... + def evolve(self, llm: BaseRagasLLM, docstore: DocumentStore): + logger.info("evolving question") + return asyncio.get_event_loop().run_until_complete(self.aevolve(llm, docstore)) + + async def aretry_evolve( + self, llm: BaseRagasLLM, docstore: DocumentStore, update_count: bool = True + ): + if update_count: + self._tries += 1 + print("retrying evolution: %s times", self._tries) + if self._tries > self.max_ties: + # TODO: make this into a custom exception + raise ValueError("Max tries reached") + return await self.aevolve(llm, docstore) + + async def aevolve(self, llm: BaseRagasLLM, docstore: DocumentStore): + # can the node be used to frame a question? + if self._tries == 0: + self.nodes = docstore.get_random_nodes(k=1) + merged_node = self.merged_nodes() + passed, table_is_present = await self.node_filter.afilter(self.nodes[0]) + if not passed: + self.nodes = docstore.get_random_nodes(k=1) + return await self.aretry_evolve(llm, docstore, update_count=False) + + # frame a basic question with with node + seed_questions = await simple_evolution(llm, merged_node, table_is_present) + # NOTE: might need improvement + # select only one seed question here + seed_question = choice(seed_questions) + is_valid_question = await self.question_filter.afilter(seed_question) + if not is_valid_question: + # get more context to rewrite question + prev_adjacent_node = docstore.get_adjacent(self.nodes[0], Direction.PREV) + if prev_adjacent_node is None: + next_adjacent_node = docstore.get_adjacent( + self.nodes[-1], Direction.NEXT + ) + if next_adjacent_node is not None: + # add nodes + self.nodes.append(next_adjacent_node) + else: + # add prev nodes + self.nodes.insert(0, prev_adjacent_node) + # retry with new nodes added + return await self.aretry_evolve(llm, docstore) + else: + # if valid question + return seed_question async def simple_evolution( @@ -108,13 +199,13 @@ async def simple_evolution( async def multi_context_evolution( - llm: BaseRagasLLM, seed_doc: Document, doc_store: DocumentStore + llm: BaseRagasLLM, seed_node: Node, doc_store: DocumentStore ): - question = simple_evolution(llm, seed_doc) + question = simple_evolution(llm, seed_node) print(question) - similar_context = doc_store.get_similar(seed_doc)[0] + similar_context = doc_store.get_similar(seed_node)[0] human_prompt = MULTICONTEXT_QUESTION.format( - question=question, context1=seed_doc.page_content, context2=similar_context + question=question, context1=seed_node.page_content, context2=similar_context ) prompt = ChatPromptTemplate.from_messages([human_prompt]) results = await llm.agenerate_text(prompt=to_pv(prompt)) diff --git a/src/ragas/testset/generator.py b/src/ragas/testset/generator.py index 9544b477..8b90cfc1 100644 --- a/src/ragas/testset/generator.py +++ b/src/ragas/testset/generator.py @@ -5,6 +5,8 @@ from langchain.embeddings import OpenAIEmbeddings from ragas.llms import BaseRagasLLM, LangchainLLMWrapper from ragas.embeddings import BaseRagasEmbeddings +from ragas.testset.docstore import DocumentStore, Document +from ragas.testset.evolutions import SimpleEvolution from llama_index.readers.schema import Document as LlamaindexDocument @@ -14,6 +16,7 @@ class TestsetGenerator: generator_llm: BaseRagasLLM critic_llm: BaseRagasLLM embeddings: BaseRagasEmbeddings + docstore: DocumentStore @classmethod def with_openai( @@ -32,7 +35,9 @@ def with_openai( ) def generate_with_llamaindex_docs(self, documents: t.Sequence[LlamaindexDocument]): - print(len(documents)) # chunk documents and add to docstore + self.docstore.add_documents( + [Document.from_llamaindex_document(doc) for doc in documents] + ) # create evolutions and add to executor queue # run till completion - keep updating progress bar diff --git a/src/ragas/testset/prompts.py b/src/ragas/testset/prompts.py index e3882083..04fe50a5 100644 --- a/src/ragas/testset/prompts.py +++ b/src/ragas/testset/prompts.py @@ -213,8 +213,7 @@ ) REWRITE_QUESTION = HumanMessagePromptTemplate.from_template( - """ - + """\ Given a context, transform the given question to be clear and standalone by replacing its coreferences with specific details from the context: Contexts: From e9158fd4640dd7b69f9e2574892556c3d969f0d6 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 14 Jan 2024 10:16:38 -0800 Subject: [PATCH 06/10] simplify filter --- src/ragas/testset/evolutions.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index 7b4a3801..6a9fff1b 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -33,15 +33,6 @@ def to_pv(prompt: ChatPromptTemplate) -> PromptValue: @dataclass class Filter(ABC): ... - # TODO: talk with @shahul about how we can unify the type - - # @abstractmethod - # def filter(self) -> t.Any: - # ... - # - # @abstractmethod - # async def afilter(self) -> t.Any: - # ... @dataclass From 16f2c93932fae62078849209460fb3d1b1d8495b Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 14 Jan 2024 12:00:16 -0800 Subject: [PATCH 07/10] updated simple --- src/ragas/testset/docstore.py | 8 ++++++-- src/ragas/testset/evolutions.py | 26 +++++--------------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index ad9ab7d9..79a6381c 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -198,7 +198,9 @@ def add_documents(self, docs: t.Sequence[Document], show_progress=True): self.add_nodes(nodes, show_progress=show_progress) - def add_nodes(self, nodes: t.Sequence[Node], show_progress=True): + def add_nodes( + self, nodes: t.Sequence[Node], show_progress=True, desc: str = "embedding nodes" + ): # NOTE: Adds everything in async mode for now. embed_tasks = [] docs_to_embed = [] @@ -212,7 +214,9 @@ def add_nodes(self, nodes: t.Sequence[Node], show_progress=True): self.node_map[n.doc_id] = n self.node_embeddings_list.append(n.embedding) - embeddings = run_async_tasks(embed_tasks, show_progress=show_progress) + embeddings = run_async_tasks( + embed_tasks, show_progress=show_progress, progress_bar_desc=desc + ) for n, embedding in zip(docs_to_embed, embeddings): n.embedding = embedding self.nodes.append(n) diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index 6a9fff1b..a771cffa 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -53,25 +53,6 @@ async def afilter(self, node: Node) -> t.Dict: return score -async def filter_node( - llm: BaseRagasLLM, node: Document, threshold: float = 7.5 -) -> t.Dict: - """ - context: str - The input context - - Checks if the context is has enough information to frame a question - """ - human_prompt = SCORE_CONTEXT.format(context=node.page_content) - prompt = ChatPromptTemplate.from_messages([human_prompt]) - results = await llm.agenerate_text(prompt=to_pv(prompt)) - output = results.generations[0][0].text.strip() - score = load_as_json(output) - # TODO: instead of updating score add a new "pass" key - score.update({"score": score.get("score", 0) >= threshold}) - return score - - @dataclass class QuestionFilter(Filter): llm: BaseRagasLLM @@ -100,7 +81,7 @@ class SimpleEvolution(Evolution): node_filter: NodeFilter question_filter: QuestionFilter nodes: t.List[Node] = field(default_factory=list) - max_ties: int = 5 + max_tries: int = 5 _tries: int = field(default=0, init=False, repr=False) def merged_nodes(self) -> Node: @@ -118,7 +99,7 @@ async def aretry_evolve( if update_count: self._tries += 1 print("retrying evolution: %s times", self._tries) - if self._tries > self.max_ties: + if self._tries > self.max_tries: # TODO: make this into a custom exception raise ValueError("Max tries reached") return await self.aevolve(llm, docstore) @@ -149,6 +130,9 @@ async def aevolve(self, llm: BaseRagasLLM, docstore: DocumentStore): if next_adjacent_node is not None: # add nodes self.nodes.append(next_adjacent_node) + else: + # retry with new base node + self.nodes = docstore.get_random_nodes(k=1) else: # add prev nodes self.nodes.insert(0, prev_adjacent_node) From c65ef27cdbba25e75a52c6931db24ae36bf59800 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 14 Jan 2024 14:25:35 -0800 Subject: [PATCH 08/10] refactor simplenode --- src/ragas/testset/evolutions.py | 62 ++++++++++++------- tests/unit/testset_generator/test_docstore.py | 10 +-- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index a771cffa..843fe93d 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -73,15 +73,11 @@ async def afilter(self, question: str) -> bool: @dataclass class Evolution: - ... - - -@dataclass -class SimpleEvolution(Evolution): node_filter: NodeFilter question_filter: QuestionFilter nodes: t.List[Node] = field(default_factory=list) max_tries: int = 5 + _root_node: t.Optional[Node] = field(default=None, init=False, repr=False) _tries: int = field(default=0, init=False, repr=False) def merged_nodes(self) -> Node: @@ -89,10 +85,6 @@ def merged_nodes(self) -> Node: doc_id="merged", page_content=" ".join(n.page_content for n in self.nodes) ) - def evolve(self, llm: BaseRagasLLM, docstore: DocumentStore): - logger.info("evolving question") - return asyncio.get_event_loop().run_until_complete(self.aevolve(llm, docstore)) - async def aretry_evolve( self, llm: BaseRagasLLM, docstore: DocumentStore, update_count: bool = True ): @@ -104,10 +96,47 @@ async def aretry_evolve( raise ValueError("Max tries reached") return await self.aevolve(llm, docstore) + @abstractmethod + def evolve(self, llm: BaseRagasLLM, docstore: DocumentStore) -> str: + ... + + @abstractmethod + async def aevolve(self, llm: BaseRagasLLM, docstore: DocumentStore) -> str: + ... + + +@dataclass +class SimpleEvolution(Evolution): + def evolve(self, llm: BaseRagasLLM, docstore: DocumentStore): + logger.info("evolving question") + return asyncio.get_event_loop().run_until_complete(self.aevolve(llm, docstore)) + + def _get_more_adjacent_nodes(self, docstore: DocumentStore): + """ + if the evolutions doesn't have enough nodes to frame a question, get more nodes + """ + assert self._root_node is not None, "root node cannot be None" + # get more nodes from above the context window + prev_adjacent_node = docstore.get_adjacent(self._root_node, Direction.PREV) + if prev_adjacent_node is None: + # get more nodes from below the context window + next_adjacent_node = docstore.get_adjacent(self._root_node, Direction.NEXT) + if next_adjacent_node is not None: + # add next nodes towards the end + self.nodes.append(next_adjacent_node) + else: + # retry with new base node + self.nodes = docstore.get_random_nodes(k=1) + self._root_node = self.nodes[0] + else: + # add prev nodes in index 0 + self.nodes.insert(0, prev_adjacent_node) + async def aevolve(self, llm: BaseRagasLLM, docstore: DocumentStore): # can the node be used to frame a question? if self._tries == 0: self.nodes = docstore.get_random_nodes(k=1) + self._root_node = self.nodes[0] merged_node = self.merged_nodes() passed, table_is_present = await self.node_filter.afilter(self.nodes[0]) if not passed: @@ -122,20 +151,7 @@ async def aevolve(self, llm: BaseRagasLLM, docstore: DocumentStore): is_valid_question = await self.question_filter.afilter(seed_question) if not is_valid_question: # get more context to rewrite question - prev_adjacent_node = docstore.get_adjacent(self.nodes[0], Direction.PREV) - if prev_adjacent_node is None: - next_adjacent_node = docstore.get_adjacent( - self.nodes[-1], Direction.NEXT - ) - if next_adjacent_node is not None: - # add nodes - self.nodes.append(next_adjacent_node) - else: - # retry with new base node - self.nodes = docstore.get_random_nodes(k=1) - else: - # add prev nodes - self.nodes.insert(0, prev_adjacent_node) + self._get_more_adjacent_nodes(docstore) # retry with new nodes added return await self.aretry_evolve(llm, docstore) else: diff --git a/tests/unit/testset_generator/test_docstore.py b/tests/unit/testset_generator/test_docstore.py index 954e2f9e..db4a5018 100644 --- a/tests/unit/testset_generator/test_docstore.py +++ b/tests/unit/testset_generator/test_docstore.py @@ -16,14 +16,14 @@ def test_adjacent_nodes(): store = InMemoryDocumentStore(splitter=None) # type: ignore store.nodes = [a1, a2, b] - assert store.get_adjascent(a1) == a2 - assert store.get_adjascent(a2, Direction.PREV) == a1 - assert store.get_adjascent(a2, Direction.NEXT) is None - assert store.get_adjascent(b, Direction.PREV) is None + assert store.get_adjacent(a1) == a2 + assert store.get_adjacent(a2, Direction.PREV) == a1 + assert store.get_adjacent(a2, Direction.NEXT) is None + assert store.get_adjacent(b, Direction.PREV) is None # raise ValueError if doc not in store c = Node(doc_id="c", page_content="c", filename="c") - pytest.raises(ValueError, store.get_adjascent, c) + pytest.raises(ValueError, store.get_adjacent, c) def create_test_nodes(with_embeddings=True): From 3f4def77e681b0d6a3f7fc4a4b03340c780ecdda Mon Sep 17 00:00:00 2001 From: jjmachan Date: Sun, 14 Jan 2024 14:38:43 -0800 Subject: [PATCH 09/10] fix formating --- src/ragas/executor.py | 1 - src/ragas/llms/base.py | 3 +- src/ragas/testset/docstore.py | 4 +-- src/ragas/testset/evolutions.py | 11 +++--- src/ragas/testset/generator.py | 35 +++++++++++++------ src/ragas/testset/testset_generator.py | 2 +- tests/e2e/test_adaptation.py | 1 - tests/unit/testset_generator/test_docstore.py | 2 +- 8 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/ragas/executor.py b/src/ragas/executor.py index 112263a9..d260ccda 100644 --- a/src/ragas/executor.py +++ b/src/ragas/executor.py @@ -3,7 +3,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field -import numpy as np from tqdm.auto import tqdm diff --git a/src/ragas/llms/base.py b/src/ragas/llms/base.py index 9f26b44d..6224a86a 100644 --- a/src/ragas/llms/base.py +++ b/src/ragas/llms/base.py @@ -36,11 +36,10 @@ def is_multiple_completion_supported(llm: BaseLanguageModel) -> bool: @dataclass class BaseRagasLLM(ABC): - def get_temperature(self, n: int) -> float: """Return the temperature to use for completion based on n.""" return 0.3 if n > 1 else 1e-8 - + @abstractmethod def generate_text( self, diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index 79a6381c..313bec4a 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -1,6 +1,6 @@ import heapq -import typing as t import logging +import typing as t import uuid from abc import ABC, abstractmethod from dataclasses import dataclass, field @@ -11,8 +11,8 @@ import numpy.typing as npt from langchain.text_splitter import TextSplitter from langchain_core.documents import Document as LCDocument -from llama_index.readers.schema import Document as LlamaindexDocument from langchain_core.pydantic_v1 import Field +from llama_index.readers.schema import Document as LlamaindexDocument from ragas.async_utils import run_async_tasks from ragas.embeddings.base import BaseRagasEmbeddings, embedding_factory diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index 843fe93d..6bd5f697 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -1,17 +1,17 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass, field import logging import typing as t -from fsspec.exceptions import asyncio -from random import choice, choices +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from random import choice +from fsspec.exceptions import asyncio from langchain.prompts import ChatPromptTemplate from numpy.random import default_rng from ragas.llms import BaseRagasLLM from ragas.llms.json_load import load_as_json from ragas.llms.prompt import PromptValue -from ragas.testset.docstore import Document, DocumentStore, Direction +from ragas.testset.docstore import Direction, Document, DocumentStore, Node from ragas.testset.prompts import ( FILTER_QUESTION, MULTICONTEXT_QUESTION, @@ -20,7 +20,6 @@ TABLE_QA, demonstrations, ) -from ragas.testset.docstore import Node rng = default_rng() logger = logging.getLogger(__name__) diff --git a/src/ragas/testset/generator.py b/src/ragas/testset/generator.py index 8b90cfc1..d9d86b1c 100644 --- a/src/ragas/testset/generator.py +++ b/src/ragas/testset/generator.py @@ -3,13 +3,12 @@ from langchain.chat_models import ChatOpenAI from langchain.embeddings import OpenAIEmbeddings -from ragas.llms import BaseRagasLLM, LangchainLLMWrapper -from ragas.embeddings import BaseRagasEmbeddings -from ragas.testset.docstore import DocumentStore, Document -from ragas.testset.evolutions import SimpleEvolution - from llama_index.readers.schema import Document as LlamaindexDocument +from ragas.embeddings import BaseRagasEmbeddings +from ragas.llms import BaseRagasLLM, LangchainLLMWrapper +from ragas.testset.docstore import Document, DocumentStore, InMemoryDocumentStore + @dataclass class TestsetGenerator: @@ -24,15 +23,31 @@ def with_openai( generator_llm: str = "gpt-3.5-turbo", critic_llm: str = "gpt-4", embeddings: str = "text-embedding-ada-002", + docstore: t.Optional[DocumentStore] = None, + chunk_size: int = 512, ) -> "TestsetGenerator": generator_llm_model = LangchainLLMWrapper(ChatOpenAI(model=generator_llm)) critic_llm_model = LangchainLLMWrapper(ChatOpenAI(model=critic_llm)) embeddings_model = OpenAIEmbeddings(model=embeddings) - return cls( - generator_llm=generator_llm_model, - critic_llm=critic_llm_model, - embeddings=embeddings_model, - ) + if docstore is None: + from langchain.text_splitter import TokenTextSplitter + + splitter = TokenTextSplitter(chunk_size=chunk_size, chunk_overlap=0) + docstore = InMemoryDocumentStore(splitter) + return cls( + generator_llm=generator_llm_model, + critic_llm=critic_llm_model, + # TODO: remove type ignore after fixing embeddigns + embeddings=embeddings_model, # type: ignore + docstore=docstore, + ) + else: + return cls( + generator_llm=generator_llm_model, + critic_llm=critic_llm_model, + embeddings=embeddings_model, # type: ignore + docstore=docstore, + ) def generate_with_llamaindex_docs(self, documents: t.Sequence[LlamaindexDocument]): # chunk documents and add to docstore diff --git a/src/ragas/testset/testset_generator.py b/src/ragas/testset/testset_generator.py index 8e7d0e12..907fc92a 100644 --- a/src/ragas/testset/testset_generator.py +++ b/src/ragas/testset/testset_generator.py @@ -544,4 +544,4 @@ def generate( count += 1 pbar.update(count) - return TestDataset(test_data=samples) \ No newline at end of file + return TestDataset(test_data=samples) diff --git a/tests/e2e/test_adaptation.py b/tests/e2e/test_adaptation.py index 269233f6..f2b07149 100644 --- a/tests/e2e/test_adaptation.py +++ b/tests/e2e/test_adaptation.py @@ -1,4 +1,3 @@ - from ragas import adapt from ragas.metrics import context_recall diff --git a/tests/unit/testset_generator/test_docstore.py b/tests/unit/testset_generator/test_docstore.py index db4a5018..b5a2ca78 100644 --- a/tests/unit/testset_generator/test_docstore.py +++ b/tests/unit/testset_generator/test_docstore.py @@ -5,7 +5,7 @@ import pytest from langchain_core.embeddings import Embeddings -from ragas.testset.docstore import InMemoryDocumentStore, Node, Direction +from ragas.testset.docstore import Direction, InMemoryDocumentStore, Node def test_adjacent_nodes(): From 9e9c01659f3f5365696c0c52bba40f7dfa75ba79 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Mon, 15 Jan 2024 00:05:07 -0800 Subject: [PATCH 10/10] fix types --- src/ragas/llms/__init__.py | 2 +- src/ragas/testset/generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ragas/llms/__init__.py b/src/ragas/llms/__init__.py index 82c6b887..f4d5513a 100644 --- a/src/ragas/llms/__init__.py +++ b/src/ragas/llms/__init__.py @@ -1,4 +1,4 @@ -from langchain.chat_models import ChatOpenAI +from langchain_community.chat_models import ChatOpenAI from ragas.llms.base import BaseRagasLLM, LangchainLLMWrapper diff --git a/src/ragas/testset/generator.py b/src/ragas/testset/generator.py index d9d86b1c..6131764b 100644 --- a/src/ragas/testset/generator.py +++ b/src/ragas/testset/generator.py @@ -1,8 +1,8 @@ import typing as t from dataclasses import dataclass -from langchain.chat_models import ChatOpenAI from langchain.embeddings import OpenAIEmbeddings +from langchain_community.chat_models import ChatOpenAI from llama_index.readers.schema import Document as LlamaindexDocument from ragas.embeddings import BaseRagasEmbeddings