From 7aeeb1a21cc52cd7ba7312d6c850edc06e9aa5a5 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Wed, 17 Jan 2024 15:14:34 -0800 Subject: [PATCH 1/8] azure-docs --- docs/howtos/customisations/azure-openai.ipynb | 217 +++++++++--------- 1 file changed, 107 insertions(+), 110 deletions(-) diff --git a/docs/howtos/customisations/azure-openai.ipynb b/docs/howtos/customisations/azure-openai.ipynb index 23fe19a24..8908f4114 100644 --- a/docs/howtos/customisations/azure-openai.ipynb +++ b/docs/howtos/customisations/azure-openai.ipynb @@ -30,7 +30,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 1, "id": "b658e02f", "metadata": {}, "outputs": [ @@ -44,7 +44,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5584ed49477f4e788fc4e9b8ab3dc50d", + "model_id": "8cfcf43797d746c6a35d2c9eb9512abc", "version_major": 2, "version_minor": 0 }, @@ -66,7 +66,7 @@ "})" ] }, - "execution_count": 11, + "execution_count": 1, "metadata": {}, "output_type": "execute_result" } @@ -79,37 +79,12 @@ "fiqa_eval" ] }, - { - "cell_type": "markdown", - "id": "c77789bb", - "metadata": {}, - "source": [ - "### Configuring them for Azure OpenAI endpoints\n", - "\n", - "Ragas also uses AzureOpenAI for running some metrics so make sure you have your Azure OpenAI key, base URL and other information available in your environment. You can check the [langchain docs](https://python.langchain.com/docs/integrations/llms/azure_openai) or the [Azure docs](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/switching-endpoints) for more information." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "0b7179f7", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ[\"OPENAI_API_TYPE\"] = \"azure\"\n", - "os.environ[\"OPENAI_API_VERSION\"] = \"2023-05-15\"\n", - "os.environ[\"OPENAI_API_BASE\"] = \"...\"\n", - "os.environ[\"OPENAI_API_KEY\"] = \"your-openai-key\"" - ] - }, { "cell_type": "markdown", "id": "d4b8a69c", "metadata": {}, "source": [ - "Lets import metrics that we are going to use" + "Lets import metrics that we are going to use. To learn more about what each metrics do, check out this [doc](https://docs.ragas.io/en/latest/concepts/metrics/index.html)" ] }, { @@ -139,68 +114,88 @@ }, { "cell_type": "markdown", - "id": "f1201199", + "id": "c77789bb", "metadata": {}, "source": [ - "Now lets swap out the default `ChatOpenAI` with `AzureChatOpenAI`. Init a new instance of `AzureChatOpenAI` with the `deployment_name` of the model you want to use. You will also have to change the `OpenAIEmbeddings` in the metrics that use them, which in our case is `answer_relevance`.\n", + "### Configuring them for Azure OpenAI endpoints\n", + "\n", + "Ragas also uses AzureOpenAI for running some metrics so make sure you have your Azure OpenAI key, base URL and other information available in your environment. You can check the [langchain docs](https://python.langchain.com/docs/integrations/llms/azure_openai) or the [Azure docs](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/switching-endpoints) for more information.\n", "\n", - "Now in order to use the new `AzureChatOpenAI` llm instance with Ragas metrics, you have to create a new instance of `RagasLLM` using the `ragas.llms.LangchainLLM` wrapper. Its a simple wrapper around langchain that make Langchain LLM/Chat instances compatible with how Ragas metrics will use them." + "\n", + "But basically you need the following information." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "68c6f3be-7935-401a-abdf-5eab73d7fe41", + "metadata": {}, + "outputs": [], + "source": [ + "azure_configs = {\n", + " \"base_url\": \"https://.openai.azure.com/\",\n", + " \"model_deployment\": \"your-deployment-name\",\n", + " \"model_name\": \"your-model-name\",\n", + " \"embedding_deployment\": \"your-deployment-name\",\n", + " \"embedding_name\": \"text-embedding-ada-002\", # most likely\n", + "}" ] }, { "cell_type": "code", "execution_count": null, - "id": "40406a26", + "id": "be0540bb-98c5-4bc9-89dc-0ee10a330e2c", "metadata": {}, "outputs": [], "source": [ - "from langchain.chat_models import AzureChatOpenAI\n", - "from langchain.embeddings import AzureOpenAIEmbeddings\n", - "from ragas.llms import LangchainLLM\n", - "\n", - "# Import evaluate before patching the RagasLLM instance\n", - "from ragas import evaluate\n", - "\n", - "azure_model = AzureChatOpenAI(\n", - " deployment_name=\"your-deployment-name\",\n", - " model=\"your-model-name\",\n", - " openai_api_base=\"https://your-endpoint.openai.azure.com/\",\n", - " openai_api_type=\"azure\",\n", - ")\n", - "# wrapper around azure_model\n", - "ragas_azure_model = LangchainLLM(azure_model)\n", - "# patch the new RagasLLM instance\n", - "answer_relevancy.llm = ragas_azure_model\n", + "import os\n", "\n", - "# init and change the embeddings\n", - "# only for answer_relevancy\n", - "azure_embeddings = AzureOpenAIEmbeddings(\n", - " deployment=\"your-embeddings-deployment-name\",\n", - " model=\"your-embeddings-model-name\",\n", - " openai_api_base=\"https://your-endpoint.openai.azure.com/\",\n", - " openai_api_type=\"azure\",\n", - ")\n", - "# embeddings can be used as it is\n", - "answer_relevancy.embeddings = azure_embeddings" + "# assuming you already have you key available via your environment variable. If not use this\n", + "#os.environ[\"AZURE_OPENAI_API_KEY\"] = \"...\"" ] }, { "cell_type": "markdown", - "id": "44641e41", + "id": "36bb5f1e-14bd-4648-a6e2-72ff980550e0", "metadata": {}, "source": [ - "This replaces the default llm of `answer_relevency` with the Azure OpenAI endpoint. Now with some `__setattr__` magic lets change it for all other metrics." + "Now lets create the chat model and embedding model instances so that ragas can use it for evaluation." ] }, { "cell_type": "code", - "execution_count": null, - "id": "52d9f5f3", + "execution_count": 6, + "id": "50110d32-8ac7-47ae-a75f-53f4dee694e3", "metadata": {}, "outputs": [], "source": [ - "for m in metrics:\n", - " m.__setattr__(\"llm\", ragas_azure_model)" + "from langchain_openai.chat_models import AzureChatOpenAI\n", + "from langchain_openai.embeddings import AzureOpenAIEmbeddings\n", + "from ragas import evaluate\n", + "\n", + "azure_model = AzureChatOpenAI(\n", + " openai_api_version=\"2023-05-15\",\n", + " azure_endpoint=azure_configs[\"base_url\"],\n", + " azure_deployment=azure_configs[\"model_deployment\"],\n", + " model=azure_configs[\"model_name\"],\n", + " validate_base_url=False\n", + ")\n", + "\n", + "# init the embeddings for answer_relevancy, answer_correctness and answer_similarity\n", + "azure_embeddings = AzureOpenAIEmbeddings(\n", + " openai_api_version=\"2023-05-15\",\n", + " azure_endpoint=azure_configs[\"base_url\"],\n", + " azure_deployment=azure_configs[\"embedding_deployment\"],\n", + " model=azure_configs[\"embedding_name\"],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "44641e41", + "metadata": {}, + "source": [ + "In case of any doubts on how to configure the Azure endpont through langchain do reffer to the [AzureChatOpenai](https://python.langchain.com/docs/integrations/chat/azure_chat_openai) and [AzureOpenAIEmbeddings](https://python.langchain.com/docs/integrations/text_embedding/azureopenai) documentations from the langchain docs." ] }, { @@ -215,31 +210,31 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 8, "id": "22eb6f97", "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "evaluating with [context_recall]\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|█████████████████████████████████████████████████████████████| 1/1 [00:51<00:00, 51.87s/it]\n" - ] + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e87bf94ac59448c48faf5bcc03667522", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Evaluating: 0%| | 0/150 [00:00ground_truths\n", " answer\n", " contexts\n", - " context_relevancy\n", " faithfulness\n", " answer_relevancy\n", " context_recall\n", + " context_precision\n", " harmfulness\n", " \n", " \n", @@ -308,10 +305,10 @@ " [Have the check reissued to the proper payee.J...\n", " \\nThe best way to deposit a cheque issued to a...\n", " [Just have the associate sign the back and the...\n", - " 0.088301\n", - " 0.666667\n", - " 0.976247\n", - " 0.111111\n", + " 1.0\n", + " 0.982491\n", + " 0.888889\n", + " 1.0\n", " 0\n", " \n", " \n", @@ -320,10 +317,10 @@ " [Sure you can. You can fill in whatever you w...\n", " \\nYes, you can send a money order from USPS as...\n", " [Sure you can. You can fill in whatever you w...\n", - " 0.191611\n", + " 1.0\n", + " 0.995249\n", " 1.000000\n", - " 0.883586\n", - " 0.800000\n", + " 1.0\n", " 0\n", " \n", " \n", @@ -332,10 +329,10 @@ " [You're confusing a lot of things here. Compan...\n", " \\nYes, it is possible to have one EIN doing bu...\n", " [You're confusing a lot of things here. Compan...\n", - " 0.069420\n", - " 1.000000\n", - " 0.928548\n", + " 1.0\n", + " 0.948876\n", " 1.000000\n", + " 1.0\n", " 0\n", " \n", " \n", @@ -344,10 +341,10 @@ " [\"I'm afraid the great myth of limited liabili...\n", " \\nApplying for and receiving business credit c...\n", " [Set up a meeting with the bank that handles y...\n", - " 0.408924\n", + " 1.0\n", + " 0.813285\n", " 1.000000\n", - " 0.906223\n", - " 0.187500\n", + " 1.0\n", " 0\n", " \n", " \n", @@ -356,10 +353,10 @@ " [You should probably consult an attorney. Howe...\n", " \\nIf your employer has closed and you need to ...\n", " [The time horizon for your 401K/IRA is essenti...\n", - " 0.064802\n", - " 0.666667\n", - " 0.889312\n", + " 0.0\n", + " 0.894836\n", " 0.000000\n", + " 0.0\n", " 0\n", " \n", " \n", @@ -388,19 +385,19 @@ "3 \\nApplying for and receiving business credit c... \n", "4 \\nIf your employer has closed and you need to ... \n", "\n", - " contexts context_relevancy \\\n", - "0 [Just have the associate sign the back and the... 0.088301 \n", - "1 [Sure you can. You can fill in whatever you w... 0.191611 \n", - "2 [You're confusing a lot of things here. Compan... 0.069420 \n", - "3 [Set up a meeting with the bank that handles y... 0.408924 \n", - "4 [The time horizon for your 401K/IRA is essenti... 0.064802 \n", + " contexts faithfulness \\\n", + "0 [Just have the associate sign the back and the... 1.0 \n", + "1 [Sure you can. You can fill in whatever you w... 1.0 \n", + "2 [You're confusing a lot of things here. Compan... 1.0 \n", + "3 [Set up a meeting with the bank that handles y... 1.0 \n", + "4 [The time horizon for your 401K/IRA is essenti... 0.0 \n", "\n", - " faithfulness answer_relevancy context_recall harmfulness \n", - "0 0.666667 0.976247 0.111111 0 \n", - "1 1.000000 0.883586 0.800000 0 \n", - "2 1.000000 0.928548 1.000000 0 \n", - "3 1.000000 0.906223 0.187500 0 \n", - "4 0.666667 0.889312 0.000000 0 " + " answer_relevancy context_recall context_precision harmfulness \n", + "0 0.982491 0.888889 1.0 0 \n", + "1 0.995249 1.000000 1.0 0 \n", + "2 0.948876 1.000000 1.0 0 \n", + "3 0.813285 1.000000 1.0 0 \n", + "4 0.894836 0.000000 0.0 0 " ] }, "execution_count": 9, From 8dbb87a8a9be8cc67f7c5a7e09c58e629cbb7363 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Wed, 17 Jan 2024 15:14:50 -0800 Subject: [PATCH 2/8] clean up embeddings --- src/ragas/embeddings/__init__.py | 6 --- src/ragas/embeddings/base.py | 91 +------------------------------- src/ragas/exceptions.py | 14 ----- src/ragas/utils.py | 2 - 4 files changed, 2 insertions(+), 111 deletions(-) diff --git a/src/ragas/embeddings/__init__.py b/src/ragas/embeddings/__init__.py index 1de1ff06d..db5b8236f 100644 --- a/src/ragas/embeddings/__init__.py +++ b/src/ragas/embeddings/__init__.py @@ -1,15 +1,9 @@ from ragas.embeddings.base import ( - AzureOpenAIEmbeddings, BaseRagasEmbeddings, - FastEmbedEmbeddings, HuggingfaceEmbeddings, - OpenAIEmbeddings, ) __all__ = [ "HuggingfaceEmbeddings", - "OpenAIEmbeddings", - "AzureOpenAIEmbeddings", "BaseRagasEmbeddings", - "FastEmbedEmbeddings", ] diff --git a/src/ragas/embeddings/base.py b/src/ragas/embeddings/base.py index 8dc27e5e5..77922d843 100644 --- a/src/ragas/embeddings/base.py +++ b/src/ragas/embeddings/base.py @@ -1,104 +1,17 @@ from __future__ import annotations -import os import typing as t from dataclasses import field from typing import List import numpy as np -from langchain.embeddings import AzureOpenAIEmbeddings as BaseAzureOpenAIEmbeddings -from langchain.embeddings import FastEmbedEmbeddings as BaseFastEmbedEmbeddings -from langchain.embeddings import OpenAIEmbeddings as BaseOpenAIEmbeddings -from langchain.schema.embeddings import Embeddings +from langchain_core.embeddings import Embeddings as BaseRagasEmbeddings +from langchain_openai.embeddings import OpenAIEmbeddings from pydantic.dataclasses import dataclass -from ragas.exceptions import AzureOpenAIKeyNotFound, OpenAIKeyNotFound -from ragas.utils import NO_KEY - DEFAULT_MODEL_NAME = "BAAI/bge-small-en-v1.5" -class BaseRagasEmbeddings(Embeddings): - ... - - -class OpenAIEmbeddings(BaseOpenAIEmbeddings, BaseRagasEmbeddings): - api_key: str = NO_KEY - - def __init__(self, api_key: str = NO_KEY): - # api key - key_from_env = os.getenv("OPENAI_API_KEY", NO_KEY) - if key_from_env != NO_KEY: - openai_api_key = key_from_env - else: - openai_api_key = api_key - super(BaseOpenAIEmbeddings, self).__init__(openai_api_key=openai_api_key) - self.api_key = openai_api_key - - def validate_api_key(self): - if self.openai_api_key == NO_KEY: - os_env_key = os.getenv("OPENAI_API_KEY", NO_KEY) - if os_env_key != NO_KEY: - self.api_key = os_env_key - else: - raise OpenAIKeyNotFound - - -class FastEmbedEmbeddings(BaseFastEmbedEmbeddings, BaseRagasEmbeddings): - """ - Find the list of supported models at: - https://qdrant.github.io/fastembed/examples/Supported_Models/ - """ - - model_name: str = DEFAULT_MODEL_NAME - """Model name to use.""" - cache_folder: t.Optional[str] = None - """Path to store models.""" - - def validate_api_key(self): - """ - Validates that the api key is set for the Embeddings - """ - pass - - -class AzureOpenAIEmbeddings(BaseAzureOpenAIEmbeddings, BaseRagasEmbeddings): - azure_endpoint: t.Optional[str] = None - deployment: t.Optional[str] = None - api_version: t.Optional[str] = None - api_key: str = NO_KEY - - def __init__( - self, - api_version: t.Optional[str] = None, - azure_endpoint: t.Optional[str] = None, - deployment: t.Optional[str] = None, - api_key: str = NO_KEY, - ): - # api key - key_from_env = os.getenv("AZURE_OPENAI_API_KEY", NO_KEY) - if key_from_env != NO_KEY: - openai_api_key = key_from_env - else: - openai_api_key = api_key - - super(BaseAzureOpenAIEmbeddings, self).__init__( - azure_endpoint=azure_endpoint, # type: ignore (pydantic bug I think) - deployment=deployment, - api_version=api_version, - api_key=openai_api_key, - ) - self.api_key = openai_api_key - - def validate_api_key(self): - if self.openai_api_key == NO_KEY: - os_env_key = os.getenv("AZURE_OPENAI_API_KEY", NO_KEY) - if os_env_key != NO_KEY: - self.api_key = os_env_key - else: - raise AzureOpenAIKeyNotFound - - @dataclass class HuggingfaceEmbeddings(BaseRagasEmbeddings): model_name: str = DEFAULT_MODEL_NAME diff --git a/src/ragas/exceptions.py b/src/ragas/exceptions.py index 6459d9e8a..3c631c3ec 100644 --- a/src/ragas/exceptions.py +++ b/src/ragas/exceptions.py @@ -9,17 +9,3 @@ class RagasException(Exception): def __init__(self, message: str): self.message = message super().__init__(message) - - -class OpenAIKeyNotFound(RagasException): - message: str = "OpenAI API key not found! Seems like your trying to use Ragas metrics with OpenAI endpoints. Please set 'OPENAI_API_KEY' environment variable" # noqa - - def __init__(self): - super().__init__(self.message) - - -class AzureOpenAIKeyNotFound(RagasException): - message: str = "AzureOpenAI API key not found! Seems like your trying to use Ragas metrics with AzureOpenAI endpoints. Please set 'AZURE_OPENAI_API_KEY' environment variable" # noqa - - def __init__(self): - super().__init__(self.message) diff --git a/src/ragas/utils.py b/src/ragas/utils.py index 49a1b3423..137f8f118 100644 --- a/src/ragas/utils.py +++ b/src/ragas/utils.py @@ -4,8 +4,6 @@ from functools import lru_cache DEBUG_ENV_VAR = "RAGAS_DEBUG" -# constant to tell us that there is no key passed to the llm/embeddings -NO_KEY = "no-key" @lru_cache(maxsize=1) From 821d909d13a83e36309821a145de60f0e40bd052 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Wed, 17 Jan 2024 15:24:57 -0800 Subject: [PATCH 3/8] fix fmt --- docs/howtos/customisations/azure-openai.ipynb | 11 ++++------- src/ragas/embeddings/__init__.py | 5 +---- src/ragas/llms/__init__.py | 2 +- src/ragas/llms/base.py | 3 +-- src/ragas/testset/testset_generator.py | 4 ++-- tests/e2e/test_adaptation.py | 1 - 6 files changed, 9 insertions(+), 17 deletions(-) diff --git a/docs/howtos/customisations/azure-openai.ipynb b/docs/howtos/customisations/azure-openai.ipynb index 8908f4114..4a11b9bbd 100644 --- a/docs/howtos/customisations/azure-openai.ipynb +++ b/docs/howtos/customisations/azure-openai.ipynb @@ -137,7 +137,7 @@ " \"model_deployment\": \"your-deployment-name\",\n", " \"model_name\": \"your-model-name\",\n", " \"embedding_deployment\": \"your-deployment-name\",\n", - " \"embedding_name\": \"text-embedding-ada-002\", # most likely\n", + " \"embedding_name\": \"text-embedding-ada-002\", # most likely\n", "}" ] }, @@ -151,7 +151,7 @@ "import os\n", "\n", "# assuming you already have you key available via your environment variable. If not use this\n", - "#os.environ[\"AZURE_OPENAI_API_KEY\"] = \"...\"" + "# os.environ[\"AZURE_OPENAI_API_KEY\"] = \"...\"" ] }, { @@ -178,7 +178,7 @@ " azure_endpoint=azure_configs[\"base_url\"],\n", " azure_deployment=azure_configs[\"model_deployment\"],\n", " model=azure_configs[\"model_name\"],\n", - " validate_base_url=False\n", + " validate_base_url=False,\n", ")\n", "\n", "# init the embeddings for answer_relevancy, answer_correctness and answer_similarity\n", @@ -241,10 +241,7 @@ ], "source": [ "result = evaluate(\n", - " fiqa_eval[\"baseline\"],\n", - " metrics=metrics,\n", - " llm=azure_model,\n", - " embeddings=azure_embeddings\n", + " fiqa_eval[\"baseline\"], metrics=metrics, llm=azure_model, embeddings=azure_embeddings\n", ")\n", "\n", "result" diff --git a/src/ragas/embeddings/__init__.py b/src/ragas/embeddings/__init__.py index db5b8236f..feb7b1546 100644 --- a/src/ragas/embeddings/__init__.py +++ b/src/ragas/embeddings/__init__.py @@ -1,7 +1,4 @@ -from ragas.embeddings.base import ( - BaseRagasEmbeddings, - HuggingfaceEmbeddings, -) +from ragas.embeddings.base import BaseRagasEmbeddings, HuggingfaceEmbeddings __all__ = [ "HuggingfaceEmbeddings", diff --git a/src/ragas/llms/__init__.py b/src/ragas/llms/__init__.py index 9d6285b6e..306d6d233 100644 --- a/src/ragas/llms/__init__.py +++ b/src/ragas/llms/__init__.py @@ -1,4 +1,4 @@ -from langchain.chat_models import ChatOpenAI +from langchain_openai.chat_models import ChatOpenAI from ragas.llms.base import BaseRagasLLM, LangchainLLMWrapper diff --git a/src/ragas/llms/base.py b/src/ragas/llms/base.py index 9f26b44d1..6224a86ad 100644 --- a/src/ragas/llms/base.py +++ b/src/ragas/llms/base.py @@ -36,11 +36,10 @@ def is_multiple_completion_supported(llm: BaseLanguageModel) -> bool: @dataclass class BaseRagasLLM(ABC): - def get_temperature(self, n: int) -> float: """Return the temperature to use for completion based on n.""" return 0.3 if n > 1 else 1e-8 - + @abstractmethod def generate_text( self, diff --git a/src/ragas/testset/testset_generator.py b/src/ragas/testset/testset_generator.py index 8e7d0e124..0d9481b83 100644 --- a/src/ragas/testset/testset_generator.py +++ b/src/ragas/testset/testset_generator.py @@ -364,7 +364,7 @@ def _get_neighbour_node( nodes.append(related_nodes[idx]) # type: ignore idx += inc # type: ignore # TODO: replace split with tikitoken - tokens += len(related_nodes[idx].get_content().split()) + tokens += len(related_nodes[idx].get_content().split()) # type: ignore return nodes if after else nodes[::-1] return [node] @@ -544,4 +544,4 @@ def generate( count += 1 pbar.update(count) - return TestDataset(test_data=samples) \ No newline at end of file + return TestDataset(test_data=samples) diff --git a/tests/e2e/test_adaptation.py b/tests/e2e/test_adaptation.py index 269233f66..f2b071499 100644 --- a/tests/e2e/test_adaptation.py +++ b/tests/e2e/test_adaptation.py @@ -1,4 +1,3 @@ - from ragas import adapt from ragas.metrics import context_recall From 52def04712a01ec6e4349f0c4c4768b8a94ce251 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Wed, 17 Jan 2024 15:27:45 -0800 Subject: [PATCH 4/8] added langchain_openai as dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 76ff166f7..d73b3b813 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ dependencies = [ "tiktoken", "langchain", "langchain-core", + "langchain_openai", "openai>1", "pysbd>=0.3.4", "nest-asyncio", From f60dfc6917e647dee1934abcdb450731b4ee46ce Mon Sep 17 00:00:00 2001 From: jjmachan Date: Wed, 17 Jan 2024 15:47:40 -0800 Subject: [PATCH 5/8] made embeddings optional --- src/ragas/evaluation.py | 10 +++++++--- src/ragas/metrics/_answer_correctness.py | 6 +++--- src/ragas/metrics/_answer_relevance.py | 7 ++----- src/ragas/metrics/_answer_similarity.py | 14 ++++++++------ src/ragas/metrics/base.py | 17 +++++++++++++++++ 5 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/ragas/evaluation.py b/src/ragas/evaluation.py index 6aefe7204..27ca9336c 100644 --- a/src/ragas/evaluation.py +++ b/src/ragas/evaluation.py @@ -12,7 +12,8 @@ from ragas.embeddings.base import BaseRagasEmbeddings from ragas.executor import Executor from ragas.llms.base import BaseRagasLLM, LangchainLLMWrapper -from ragas.metrics.base import Metric, MetricWithLLM +from ragas.metrics.base import Metric, MetricWithEmbeddings, MetricWithLLM +from ragas.metrics.critique import AspectCritique # from ragas.metrics.critique import AspectCritique from ragas.validation import ( @@ -116,11 +117,14 @@ def evaluate( binary_metrics = [] for metric in metrics: - # if isinstance(metric, AspectCritique): - # binary_metrics.append(metric.name) + if isinstance(metric, AspectCritique): + binary_metrics.append(metric.name) if isinstance(metric, MetricWithLLM): if metric.llm is None: metric.llm = llm + if isinstance(metric, MetricWithEmbeddings): + if metric.embeddings is None: + metric.embeddings = embeddings # initialize all the models in the metrics [m.init_model() for m in metrics] diff --git a/src/ragas/metrics/_answer_correctness.py b/src/ragas/metrics/_answer_correctness.py index 87ac4174b..88cdf4895 100644 --- a/src/ragas/metrics/_answer_correctness.py +++ b/src/ragas/metrics/_answer_correctness.py @@ -9,7 +9,7 @@ from ragas.llms.json_load import json_loader from ragas.llms.prompt import Prompt from ragas.metrics._answer_similarity import AnswerSimilarity -from ragas.metrics.base import EvaluationMode, MetricWithLLM +from ragas.metrics.base import EvaluationMode, MetricWithEmbeddings, MetricWithLLM logger = logging.getLogger(__name__) @@ -68,7 +68,7 @@ @dataclass -class AnswerCorrectness(MetricWithLLM): +class AnswerCorrectness(MetricWithLLM, MetricWithEmbeddings): """ Measures answer correctness compared to ground truth as a combination of @@ -106,7 +106,7 @@ def __post_init__(self: t.Self): if self.answer_similarity is None and self.weights[1] != 0: self.answer_similarity = AnswerSimilarity( - llm=self.llm, batch_size=self.batch_size + llm=self.llm, batch_size=self.batch_size, embeddings=self.embeddings ) def _compute_statement_presence(self, prediction: t.Any) -> float: diff --git a/src/ragas/metrics/_answer_relevance.py b/src/ragas/metrics/_answer_relevance.py index 96acbe0d3..0c7795c03 100644 --- a/src/ragas/metrics/_answer_relevance.py +++ b/src/ragas/metrics/_answer_relevance.py @@ -6,17 +6,15 @@ import numpy as np -from ragas.embeddings.base import embedding_factory from ragas.llms.json_load import json_loader from ragas.llms.prompt import Prompt -from ragas.metrics.base import EvaluationMode, MetricWithLLM +from ragas.metrics.base import EvaluationMode, MetricWithEmbeddings, MetricWithLLM logger = logging.getLogger(__name__) if t.TYPE_CHECKING: from langchain_core.callbacks import Callbacks - from ragas.embeddings.base import BaseRagasEmbeddings from ragas.llms.prompt import PromptValue QUESTION_GEN = Prompt( @@ -51,7 +49,7 @@ @dataclass -class AnswerRelevancy(MetricWithLLM): +class AnswerRelevancy(MetricWithLLM, MetricWithEmbeddings): """ Scores the relevancy of the answer according to the given question. Answers with incomplete, redundant or unnecessary information is penalized. @@ -76,7 +74,6 @@ class AnswerRelevancy(MetricWithLLM): question_generation: Prompt = field(default_factory=lambda: QUESTION_GEN) batch_size: int = 15 strictness: int = 3 - embeddings: BaseRagasEmbeddings = field(default_factory=embedding_factory) def init_model(self): super().init_model() diff --git a/src/ragas/metrics/_answer_similarity.py b/src/ragas/metrics/_answer_similarity.py index 09b13de9b..3a07782a3 100644 --- a/src/ragas/metrics/_answer_similarity.py +++ b/src/ragas/metrics/_answer_similarity.py @@ -2,23 +2,22 @@ import logging import typing as t -from dataclasses import dataclass, field +from dataclasses import dataclass import numpy as np -from ragas.embeddings.base import HuggingfaceEmbeddings, embedding_factory -from ragas.metrics.base import EvaluationMode, MetricWithLLM +from ragas.embeddings.base import HuggingfaceEmbeddings +from ragas.metrics.base import EvaluationMode, MetricWithEmbeddings, MetricWithLLM if t.TYPE_CHECKING: from langchain.callbacks.base import Callbacks - from ragas.embeddings.base import BaseRagasEmbeddings logger = logging.getLogger(__name__) @dataclass -class AnswerSimilarity(MetricWithLLM): +class AnswerSimilarity(MetricWithLLM, MetricWithEmbeddings): """ Scores the semantic similarity of ground truth with generated answer. cross encoder score is used to quantify semantic similarity. @@ -42,7 +41,6 @@ class AnswerSimilarity(MetricWithLLM): name: str = "answer_similarity" # type: ignore evaluation_mode: EvaluationMode = EvaluationMode.ga # type: ignore batch_size: int = 15 - embeddings: BaseRagasEmbeddings = field(default_factory=embedding_factory) is_cross_encoder: bool = False threshold: t.Optional[float] = None @@ -59,6 +57,8 @@ def init_model(self): super().init_model() def _score(self, row: t.Dict, callbacks: Callbacks) -> float: + assert self.embeddings is not None, "embeddings must be set" + ground_truths, answers = row["ground_truths"], row["answer"] ground_truths = [item[0] for item in ground_truths] @@ -79,6 +79,8 @@ def _score(self, row: t.Dict, callbacks: Callbacks) -> float: return scores.tolist()[0] async def _ascore(self: t.Self, row: t.Dict, callbacks: Callbacks = []) -> float: + assert self.embeddings is not None, "embeddings must be set" + ground_truths, answers = row["ground_truths"], row["answer"] ground_truths = [item[0] for item in ground_truths] diff --git a/src/ragas/metrics/base.py b/src/ragas/metrics/base.py index 79e832ab7..59a76c512 100644 --- a/src/ragas/metrics/base.py +++ b/src/ragas/metrics/base.py @@ -16,6 +16,7 @@ if t.TYPE_CHECKING: from langchain_core.callbacks import Callbacks + from ragas.embeddings import BaseRagasEmbeddings from ragas.llms import BaseRagasLLM @@ -116,3 +117,19 @@ def init_model(self): raise ValueError( f"Metric '{self.name}' has no valid LLM provided (self.llm is None). Please initantiate a the metric with an LLM to run." # noqa ) + + +@dataclass +class MetricWithEmbeddings(Metric): + embeddings: t.Optional[BaseRagasEmbeddings] = None + + def init_model(self): + """ + Init any models in the metric, this is invoked before evaluate() + to load all the models + Also check if the api key is valid for OpenAI and AzureOpenAI + """ + if self.embeddings is None: + raise ValueError( + f"Metric '{self.name}' has no valid embeddings provided (self.embeddings is None). Please initantiate a the metric with an embeddings to run." # noqa + ) From 50c405796597747fb56be8f13c0b982e54308d65 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Wed, 17 Jan 2024 15:56:23 -0800 Subject: [PATCH 6/8] make it optional in docstore as well --- src/ragas/testset/docstore.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index f87e51122..633f244cf 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -114,7 +114,7 @@ def get_top_k_embeddings( @dataclass class InMemoryDocumentStore(DocumentStore): splitter: TextSplitter - embeddings: BaseRagasEmbeddings = field( + embeddings: t.Optional[BaseRagasEmbeddings] = field( default_factory=embedding_factory, repr=False ) documents_list: t.List[Document] = field(default_factory=list) @@ -125,6 +125,8 @@ def _add_documents_batch(self, docs: t.Sequence[Document], show_progress=True): """ Add documents in batch mode. """ + assert self.embeddings is not None, "Embeddings must be set" + # NOTE: Adds everything in async mode for now. embed_tasks = [] docs_to_embed = [] @@ -145,6 +147,8 @@ def _add_documents_batch(self, docs: t.Sequence[Document], show_progress=True): self.embeddings_list.append(doc.embedding) def add(self, doc: t.Union[Document, t.Sequence[Document]], show_progress=True): + assert self.embeddings is not None, "Embeddings must be set" + if isinstance(doc, list) or isinstance(doc, tuple): self._add_documents_batch(doc) elif isinstance(doc, Document): From cf53163cb2c83056234f3e7678c78a76cf355716 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Wed, 17 Jan 2024 16:18:24 -0800 Subject: [PATCH 7/8] fix tests --- src/ragas/testset/docstore.py | 6 +- tests/unit/testset_generator/test_docstore.py | 64 +++++++++++-------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/ragas/testset/docstore.py b/src/ragas/testset/docstore.py index 633f244cf..03c2b6ced 100644 --- a/src/ragas/testset/docstore.py +++ b/src/ragas/testset/docstore.py @@ -12,7 +12,7 @@ from pydantic import Field from ragas.async_utils import run_async_tasks -from ragas.embeddings.base import BaseRagasEmbeddings, embedding_factory +from ragas.embeddings.base import BaseRagasEmbeddings Embedding = t.Union[t.List[float], npt.NDArray[np.float64]] @@ -114,9 +114,7 @@ def get_top_k_embeddings( @dataclass class InMemoryDocumentStore(DocumentStore): splitter: TextSplitter - embeddings: t.Optional[BaseRagasEmbeddings] = field( - default_factory=embedding_factory, repr=False - ) + embeddings: t.Optional[BaseRagasEmbeddings] = field(default=None, repr=False) documents_list: t.List[Document] = field(default_factory=list) embeddings_list: t.List[Embedding] = field(default_factory=list) documents_map: t.Dict[str, Document] = field(default_factory=dict) diff --git a/tests/unit/testset_generator/test_docstore.py b/tests/unit/testset_generator/test_docstore.py index 338067a36..2f8e12904 100644 --- a/tests/unit/testset_generator/test_docstore.py +++ b/tests/unit/testset_generator/test_docstore.py @@ -3,17 +3,43 @@ import typing as t import pytest +from langchain.text_splitter import TokenTextSplitter from langchain_core.embeddings import Embeddings from ragas.testset.docstore import Document, InMemoryDocumentStore +class FakeEmbeddings(Embeddings): + def __init__(self): + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_embs.pkl") + with open(path, "rb") as f: + self.embeddings: dict[str, t.Any] = pickle.load(f) + + def _get_embedding(self, text: str) -> t.List[float]: + if text in self.embeddings: + return self.embeddings[text] + else: + return [0] * 768 + + def embed_documents(self, texts: t.List[str]) -> t.List[t.List[float]]: + return [self._get_embedding(text) for text in texts] + + def embed_query(self, text: str) -> t.List[float]: + return self._get_embedding(text) + + async def aembed_query(self, text: str) -> t.List[float]: + return self._get_embedding(text) + + def test_adjacent_nodes(): a1 = Document(doc_id="a1", page_content="a1", filename="a") a2 = Document(doc_id="a2", page_content="a2", filename="a") b = Document(doc_id="b", page_content="b", filename="b") - store = InMemoryDocumentStore(splitter=None) # type: ignore + fake_embeddings = FakeEmbeddings() + splitter = TokenTextSplitter(chunk_size=100, chunk_overlap=0) + + store = InMemoryDocumentStore(splitter=splitter, embeddings=fake_embeddings) store.documents_list = [a1, a2, b] assert store.get_adjascent(a1) == a2 @@ -53,7 +79,10 @@ def create_test_documents(with_embeddings=True): def test_similar_nodes(): a1, a2, b = create_test_documents() - store = InMemoryDocumentStore(splitter=None) # type: ignore + + fake_embeddings = FakeEmbeddings() + splitter = TokenTextSplitter(chunk_size=100, chunk_overlap=0) + store = InMemoryDocumentStore(splitter=splitter, embeddings=fake_embeddings) store.documents_list = [a1, a2, b] store.embeddings_list = [d.embedding for d in store.documents_list] @@ -66,7 +95,9 @@ def test_similar_nodes(): def test_similar_nodes_scaled(): a1, a2, b = create_test_documents() - store = InMemoryDocumentStore(splitter=None) # type: ignore + fake_embeddings = FakeEmbeddings() + splitter = TokenTextSplitter(chunk_size=100, chunk_overlap=0) + store = InMemoryDocumentStore(splitter=splitter, embeddings=fake_embeddings) store.documents_list = [a1, a2, b] + [b] * 100 store.embeddings_list = [d.embedding for d in store.documents_list] @@ -77,7 +108,10 @@ def test_similar_nodes_scaled(): def test_docstore_add(): a1, a2, b = create_test_documents() - store = InMemoryDocumentStore(splitter=None) # type: ignore + + fake_embeddings = FakeEmbeddings() + splitter = TokenTextSplitter(chunk_size=100, chunk_overlap=0) + store = InMemoryDocumentStore(splitter=splitter, embeddings=fake_embeddings) docs_added = [] for doc in [a1, a2, b]: store.add(doc) @@ -88,28 +122,6 @@ def test_docstore_add(): assert store.get(a1.doc_id) == a1 -class FakeEmbeddings(Embeddings): - def __init__(self): - path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_embs.pkl") - with open(path, "rb") as f: - self.embeddings: dict[str, t.Any] = pickle.load(f) - - def _get_embedding(self, text: str) -> t.List[float]: - if text in self.embeddings: - return self.embeddings[text] - else: - return [0] * 768 - - def embed_documents(self, texts: t.List[str]) -> t.List[t.List[float]]: - return [self._get_embedding(text) for text in texts] - - def embed_query(self, text: str) -> t.List[float]: - return self._get_embedding(text) - - async def aembed_query(self, text: str) -> t.List[float]: - return self._get_embedding(text) - - @pytest.mark.asyncio async def test_fake_embeddings(): fake_embeddings = FakeEmbeddings() From 7cbb70a7de221d09c7a2cab73cd598b4d914a0fc Mon Sep 17 00:00:00 2001 From: jjmachan Date: Thu, 18 Jan 2024 17:03:26 -0800 Subject: [PATCH 8/8] fix tests --- tests/unit/testset_generator/test_docstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/testset_generator/test_docstore.py b/tests/unit/testset_generator/test_docstore.py index af1f4b185..de911a233 100644 --- a/tests/unit/testset_generator/test_docstore.py +++ b/tests/unit/testset_generator/test_docstore.py @@ -40,7 +40,7 @@ def test_adjacent_nodes(): splitter = TokenTextSplitter(chunk_size=100, chunk_overlap=0) store = InMemoryDocumentStore(splitter=splitter, embeddings=fake_embeddings) - store.documents_list = [a1, a2, b] + store.nodes = [a1, a2, b] assert store.get_adjacent(a1) == a2 assert store.get_adjacent(a2, Direction.PREV) == a1