From 55064c2a9423e855757ab9e6532274b9d50e19e0 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Thu, 8 Feb 2024 16:47:29 -0800 Subject: [PATCH 1/4] ensure dict type --- src/ragas/testset/evolutions.py | 7 +++++-- src/ragas/testset/extractor.py | 1 + src/ragas/testset/filters.py | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/ragas/testset/evolutions.py b/src/ragas/testset/evolutions.py index 09802a188..90c275e63 100644 --- a/src/ragas/testset/evolutions.py +++ b/src/ragas/testset/evolutions.py @@ -180,9 +180,12 @@ async def generate_datarow( relevent_contexts_result = await json_loader.safe_load( results.generations[0][0].text.strip(), llm=self.generator_llm ) - relevant_context_indices = relevent_contexts_result.get( - "relevant_context", None + relevant_context_indices = ( + relevent_contexts_result.get("relevant_context", None) + if isinstance(relevent_contexts_result, dict) + else None ) + if relevant_context_indices is None: relevant_context = CurrentNodes( root_node=current_nodes.root_node, nodes=current_nodes.nodes diff --git a/src/ragas/testset/extractor.py b/src/ragas/testset/extractor.py index 77c586c2e..09154d8cf 100644 --- a/src/ragas/testset/extractor.py +++ b/src/ragas/testset/extractor.py @@ -50,6 +50,7 @@ async def extract(self, node: Node, is_async: bool = True) -> t.List[str]: keyphrases = await json_loader.safe_load( results.generations[0][0].text.strip(), llm=self.llm, is_async=is_async ) + keyphrases = keyphrases if isinstance(keyphrases, dict) else {} logger.debug("keyphrases: %s", keyphrases) return keyphrases.get("keyphrases", []) diff --git a/src/ragas/testset/filters.py b/src/ragas/testset/filters.py index 0eb06b77b..9d017b7e5 100644 --- a/src/ragas/testset/filters.py +++ b/src/ragas/testset/filters.py @@ -54,6 +54,7 @@ async def filter(self, node: Node) -> t.Dict: results = await self.llm.generate(prompt=prompt) output = results.generations[0][0].text.strip() score = await json_loader.safe_load(output, llm=self.llm) + score = score if isinstance(score, dict) else {} logger.debug("node filter: %s", score) score.update({"score": score.get("score", 0) >= self.threshold}) return score @@ -85,6 +86,7 @@ async def filter(self, question: str) -> bool: results = await self.llm.generate(prompt=prompt) results = results.generations[0][0].text.strip() json_results = await json_loader.safe_load(results, llm=self.llm) + json_results = json_results if isinstance(json_results, dict) else {} logger.debug("filtered question: %s", json_results) return json_results.get("verdict") == "1" @@ -117,6 +119,7 @@ async def filter(self, simple_question: str, compressed_question: str) -> bool: results = await self.llm.generate(prompt=prompt) results = results.generations[0][0].text.strip() json_results = await json_loader.safe_load(results, llm=self.llm) + json_results = json_results if isinstance(json_results, dict) else {} logger.debug("evolution filter: %s", json_results) return json_results.get("verdict") == "1" From 3a9ea8d668098c47b7e26c4a05168e3b44ed48f9 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Wed, 14 Feb 2024 17:16:07 -0800 Subject: [PATCH 2/4] pin llama-index version --- requirements/docs.txt | 2 +- requirements/test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index 94e3f229f..84b2aa5a2 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -4,4 +4,4 @@ myst-parser[linkify] sphinx_design astroid<3 myst-nb -llama_index +llama_index==0.9.44 diff --git a/requirements/test.txt b/requirements/test.txt index e2814f467..439220fc8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,4 @@ pytest pytest-xdist[psutil] pytest-asyncio -llama_index +llama_index==0.9.44 From 193de918eda6be246407283669bfe683a7b2c5e6 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Wed, 14 Feb 2024 17:31:30 -0800 Subject: [PATCH 3/4] migrate to latest llama-index --- requirements/test.txt | 2 +- src/ragas/testset/docstore.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index 439220fc8..e2814f467 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,4 @@ pytest pytest-xdist[psutil] pytest-asyncio -llama_index==0.9.44 +llama_index diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index d2d1c140c..ee22f6011 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -21,7 +21,7 @@ from ragas.testset.utils import rng if t.TYPE_CHECKING: - from llama_index.readers.schema import Document as LlamaindexDocument + from llama_index.core.schema import Document as LlamaindexDocument from ragas.testset.extractor import Extractor From b7fd87c048693a65f69efd488a2cde3e4e046ad7 Mon Sep 17 00:00:00 2001 From: Shahules786 Date: Wed, 14 Feb 2024 17:31:53 -0800 Subject: [PATCH 4/4] migrate to latest llama-index --- requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index 84b2aa5a2..94e3f229f 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -4,4 +4,4 @@ myst-parser[linkify] sphinx_design astroid<3 myst-nb -llama_index==0.9.44 +llama_index