From 842e8a51af531f11572f6cc255a9bb6687c0f4f8 Mon Sep 17 00:00:00 2001
From: Tino Max <tinothayil@gmail.com>
Date: Wed, 24 Jan 2024 19:36:23 -0800
Subject: [PATCH 1/2] added docs for customisation

---
 docs/howtos/customisations/aws-bedrock.ipynb | 281 +++++++------------
 docs/howtos/customisations/embeddings.ipynb  | 164 ++++-------
 docs/howtos/customisations/llms.ipynb        | 133 ++-------
 3 files changed, 176 insertions(+), 402 deletions(-)

diff --git a/docs/howtos/customisations/aws-bedrock.ipynb b/docs/howtos/customisations/aws-bedrock.ipynb
index 82ccfa921..9d1daee65 100644
--- a/docs/howtos/customisations/aws-bedrock.ipynb
+++ b/docs/howtos/customisations/aws-bedrock.ipynb
@@ -32,16 +32,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 26,
    "id": "b658e02f",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Repo card metadata block was not found. Setting CardData to empty.\n"
+     ]
+    }
+   ],
    "source": [
     "# data\n",
     "from datasets import load_dataset\n",
     "\n",
-    "fiqa_eval = load_dataset(\"explodinggradients/fiqa\", \"ragas_eval\")\n",
-    "fiqa_eval"
+    "amnesty_qa = load_dataset(\"explodinggradients/amnesty_qa\", \"english\")"
    ]
   },
   {
@@ -54,14 +61,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 38,
    "id": "f17bcf9d",
    "metadata": {},
    "outputs": [],
    "source": [
     "from ragas.metrics import (\n",
     "    context_precision,\n",
-    "    answer_relevancy,  # AnswerRelevancy\n",
     "    faithfulness,\n",
     "    context_recall,\n",
     ")\n",
@@ -70,7 +76,6 @@
     "# list of metrics we're going to use\n",
     "metrics = [\n",
     "    faithfulness,\n",
-    "    answer_relevancy,\n",
     "    context_recall,\n",
     "    context_precision,\n",
     "    harmfulness,\n",
@@ -82,21 +87,18 @@
    "id": "f1201199",
    "metadata": {},
    "source": [
-    "Now lets swap out the default `ChatOpenAI` with `BedrockChat`. Init a new instance of `BedrockChat` with the `model_id` of the model you want to use. You will also have to change the `BedrockEmbeddings` in the metrics that use them, which in our case is `answer_relevance`.\n",
-    "\n",
-    "Now in order to use the new `BedrockChat` llm instance with Ragas metrics, you have to create a new instance of `RagasLLM` using the `ragas.llms.LangchainLLM` wrapper. Its a simple wrapper around langchain that make Langchain LLM/Chat instances compatible with how Ragas metrics will use them."
+    "Now lets use the llm from Bedrock using `BedrockChat` class from Langchain. Init a new instance of `BedrockChat` with the `model_id` of the model you want to use. You will also have to change the `BedrockEmbeddings` in the evaluate function with the metrics that we use."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 18,
    "id": "40406a26",
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.llms import LangchainLLM\n",
-    "from langchain.chat_models import BedrockChat\n",
-    "from langchain.embeddings import BedrockEmbeddings\n",
+    "from langchain_community.chat_models import BedrockChat\n",
+    "from langchain_community.embeddings import BedrockEmbeddings\n",
     "\n",
     "config = {\n",
     "    \"credentials_profile_name\": \"your-profile-name\",  # E.g \"default\"\n",
@@ -112,19 +114,12 @@
     "    model_id=config[\"model_id\"],\n",
     "    model_kwargs=config[\"model_kwargs\"],\n",
     ")\n",
-    "# wrapper around bedrock_model\n",
-    "ragas_bedrock_model = LangchainLLM(bedrock_model)\n",
-    "# patch the new RagasLLM instance\n",
-    "answer_relevancy.llm = ragas_bedrock_model\n",
     "\n",
-    "# init and change the embeddings\n",
-    "# only for answer_relevancy\n",
+    "# init the embeddings\n",
     "bedrock_embeddings = BedrockEmbeddings(\n",
     "    credentials_profile_name=config[\"credentials_profile_name\"],\n",
     "    region_name=config[\"region_name\"],\n",
-    ")\n",
-    "# embeddings can be used as it is\n",
-    "answer_relevancy.embeddings = bedrock_embeddings"
+    ")"
    ]
   },
   {
@@ -132,18 +127,7 @@
    "id": "44641e41",
    "metadata": {},
    "source": [
-    "This replaces the default llm of `answer_relevancy` with the Amazon Bedrock endpoint. Now with some `__setattr__` magic lets change it for all other metrics."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "52d9f5f3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "for m in metrics:\n",
-    "    m.__setattr__(\"llm\", ragas_bedrock_model)"
+    "Now we can use the llm and embeddings with `Bedrock` by passing it in the evaluate function."
    ]
   },
   {
@@ -158,87 +142,50 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "22eb6f97",
+   "execution_count": 27,
+   "id": "5d0aac10",
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "evaluating with [faithfulness]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 2/2 [01:22<00:00, 41.24s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "evaluating with [answer_relevancy]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 2/2 [01:21<00:00, 40.59s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "evaluating with [context_recall]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 2/2 [00:46<00:00, 23.22s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "evaluating with [context_precision]\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 2/2 [00:59<00:00, 29.85s/it]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "evaluating with [harmfulness]\n"
-     ]
-    },
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    eval: Dataset({\n",
+       "        features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
+       "        num_rows: 20\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "amnesty_qa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "22eb6f97",
+   "metadata": {},
+   "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 2/2 [00:33<00:00, 16.96s/it]\n"
+      "Evaluating: 100%|██████████| 12/12 [01:05<00:00,  5.48s/it]\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "{'faithfulness': 0.9428, 'answer_relevancy': 0.7860, 'context_recall': 0.2296, 'context_precision': 0.0000, 'harmfulness': 0.0000}"
+       "{'faithfulness': 0.6250, 'context_recall': 1.0000, 'context_precision': 1.0000, 'harmfulness': 0.0000}"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 41,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -251,8 +198,10 @@
     "nest_asyncio.apply()\n",
     "\n",
     "result = evaluate(\n",
-    "    fiqa_eval[\"baseline\"],\n",
+    "    amnesty_qa[\"eval\"].select(range(3)),\n",
     "    metrics=metrics,\n",
+    "    llm=bedrock_model,\n",
+    "    embeddings=bedrock_embeddings,\n",
     ")\n",
     "\n",
     "result"
@@ -270,7 +219,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 40,
    "id": "8686bf53",
    "metadata": {},
    "outputs": [
@@ -296,11 +245,10 @@
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>question</th>\n",
-       "      <th>contexts</th>\n",
-       "      <th>answer</th>\n",
        "      <th>ground_truths</th>\n",
+       "      <th>answer</th>\n",
+       "      <th>contexts</th>\n",
        "      <th>faithfulness</th>\n",
-       "      <th>answer_relevancy</th>\n",
        "      <th>context_recall</th>\n",
        "      <th>context_precision</th>\n",
        "      <th>harmfulness</th>\n",
@@ -309,63 +257,36 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
-       "      <td>How to deposit a cheque issued to an associate...</td>\n",
-       "      <td>[Just have the associate sign the back and the...</td>\n",
-       "      <td>\\nThe best way to deposit a cheque issued to a...</td>\n",
-       "      <td>[Have the check reissued to the proper payee.J...</td>\n",
+       "      <td>What are the global implications of the USA Su...</td>\n",
+       "      <td>[The global implications of the USA Supreme Co...</td>\n",
+       "      <td>The global implications of the USA Supreme Cou...</td>\n",
+       "      <td>[- In 2022, the USA Supreme Court handed down ...</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
        "      <td>1.0</td>\n",
-       "      <td>0.930311</td>\n",
-       "      <td>0.263158</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
-       "      <td>Can I send a money order from USPS as a business?</td>\n",
-       "      <td>[Sure you can.  You can fill in whatever you w...</td>\n",
-       "      <td>\\nYes, you can send a money order from USPS as...</td>\n",
-       "      <td>[Sure you can.  You can fill in whatever you w...</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.984122</td>\n",
-       "      <td>0.363636</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0</td>\n",
+       "      <td>Which companies are the main contributors to G...</td>\n",
+       "      <td>[According to the Carbon Majors database, the ...</td>\n",
+       "      <td>According to the Carbon Majors database, the m...</td>\n",
+       "      <td>[- Fossil fuel companies, whether state or pri...</td>\n",
+       "      <td>0.00</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
-       "      <td>1 EIN doing business under multiple business n...</td>\n",
-       "      <td>[You're confusing a lot of things here. Compan...</td>\n",
-       "      <td>\\nYes, it is possible to have one EIN doing bu...</td>\n",
-       "      <td>[You're confusing a lot of things here. Compan...</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.883872</td>\n",
-       "      <td>0.363636</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>Applying for and receiving business credit</td>\n",
-       "      <td>[Set up a meeting with the bank that handles y...</td>\n",
-       "      <td>\\nApplying for and receiving business credit c...</td>\n",
-       "      <td>[\"I'm afraid the great myth of limited liabili...</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.518287</td>\n",
-       "      <td>0.363636</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>0</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>401k Transfer After Business Closure</td>\n",
-       "      <td>[The time horizon for your 401K/IRA is essenti...</td>\n",
-       "      <td>\\nIf your employer has closed and you need to ...</td>\n",
-       "      <td>[You should probably consult an attorney. Howe...</td>\n",
+       "      <td>Which private companies in the Americas are th...</td>\n",
+       "      <td>[The largest private companies in the Americas...</td>\n",
+       "      <td>According to the Carbon Majors database, the l...</td>\n",
+       "      <td>[The private companies responsible for the mos...</td>\n",
+       "      <td>0.25</td>\n",
        "      <td>1.0</td>\n",
-       "      <td>0.779471</td>\n",
-       "      <td>0.000000</td>\n",
+       "      <td>NaN</td>\n",
        "      <td>0.0</td>\n",
-       "      <td>0</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -373,42 +294,32 @@
       ],
       "text/plain": [
        "                                            question  \\\n",
-       "0  How to deposit a cheque issued to an associate...   \n",
-       "1  Can I send a money order from USPS as a business?   \n",
-       "2  1 EIN doing business under multiple business n...   \n",
-       "3         Applying for and receiving business credit   \n",
-       "4               401k Transfer After Business Closure   \n",
+       "0  What are the global implications of the USA Su...   \n",
+       "1  Which companies are the main contributors to G...   \n",
+       "2  Which private companies in the Americas are th...   \n",
        "\n",
-       "                                            contexts  \\\n",
-       "0  [Just have the associate sign the back and the...   \n",
-       "1  [Sure you can.  You can fill in whatever you w...   \n",
-       "2  [You're confusing a lot of things here. Compan...   \n",
-       "3  [Set up a meeting with the bank that handles y...   \n",
-       "4  [The time horizon for your 401K/IRA is essenti...   \n",
+       "                                       ground_truths  \\\n",
+       "0  [The global implications of the USA Supreme Co...   \n",
+       "1  [According to the Carbon Majors database, the ...   \n",
+       "2  [The largest private companies in the Americas...   \n",
        "\n",
        "                                              answer  \\\n",
-       "0  \\nThe best way to deposit a cheque issued to a...   \n",
-       "1  \\nYes, you can send a money order from USPS as...   \n",
-       "2  \\nYes, it is possible to have one EIN doing bu...   \n",
-       "3  \\nApplying for and receiving business credit c...   \n",
-       "4  \\nIf your employer has closed and you need to ...   \n",
+       "0  The global implications of the USA Supreme Cou...   \n",
+       "1  According to the Carbon Majors database, the m...   \n",
+       "2  According to the Carbon Majors database, the l...   \n",
        "\n",
-       "                                       ground_truths  faithfulness  \\\n",
-       "0  [Have the check reissued to the proper payee.J...           1.0   \n",
-       "1  [Sure you can.  You can fill in whatever you w...           1.0   \n",
-       "2  [You're confusing a lot of things here. Compan...           1.0   \n",
-       "3  [\"I'm afraid the great myth of limited liabili...           1.0   \n",
-       "4  [You should probably consult an attorney. Howe...           1.0   \n",
+       "                                            contexts  faithfulness  \\\n",
+       "0  [- In 2022, the USA Supreme Court handed down ...           NaN   \n",
+       "1  [- Fossil fuel companies, whether state or pri...          0.00   \n",
+       "2  [The private companies responsible for the mos...          0.25   \n",
        "\n",
-       "   answer_relevancy  context_recall  context_precision  harmfulness  \n",
-       "0          0.930311        0.263158                0.0            0  \n",
-       "1          0.984122        0.363636                0.0            0  \n",
-       "2          0.883872        0.363636                0.0            0  \n",
-       "3          0.518287        0.363636                0.0            0  \n",
-       "4          0.779471        0.000000                0.0            0  "
+       "   context_recall  context_precision  harmfulness  \n",
+       "0             NaN                1.0          NaN  \n",
+       "1             NaN                NaN          NaN  \n",
+       "2             1.0                NaN          0.0  "
       ]
      },
-     "execution_count": 9,
+     "execution_count": 40,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -445,7 +356,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "version": "3.10.8"
   }
  },
  "nbformat": 4,
diff --git a/docs/howtos/customisations/embeddings.ipynb b/docs/howtos/customisations/embeddings.ipynb
index 8c31c816f..7e405b88b 100644
--- a/docs/howtos/customisations/embeddings.ipynb
+++ b/docs/howtos/customisations/embeddings.ipynb
@@ -26,14 +26,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 5,
    "id": "25c72521-3372-4663-81e4-c159b0edde40",
    "metadata": {},
    "outputs": [],
    "source": [
     "import os\n",
-    "from langchain.chat_models import AzureChatOpenAI\n",
-    "from ragas.llms import LangchainLLM\n",
+    "from langchain_openai.chat_models import AzureChatOpenAI\n",
+    "from ragas.llms.base import LangchainLLMWrapper\n",
     "\n",
     "os.environ[\"OPENAI_API_VERSION\"] = \"2023-05-15\"\n",
     "os.environ[\"OPENAI_API_KEY\"] = \"your-openai-key\"\n",
@@ -43,9 +43,7 @@
     "    model=\"gpt-35-turbo-16k\",\n",
     "    azure_endpoint=\"https://your-endpoint.openai.azure.com/\",\n",
     "    openai_api_type=\"azure\",\n",
-    ")\n",
-    "# wrapper around azure_model\n",
-    "ragas_azure_model = LangchainLLM(azure_model)"
+    ")"
    ]
   },
   {
@@ -53,16 +51,16 @@
    "id": "f1fdb48b",
    "metadata": {},
    "source": [
-    "In order to use the Azure Open AI embedding, we have to instantiate an object of the `AzureOpenAIEmbeddings` class in Ragas."
+    "In order to use the Azure Open AI embedding, we have to instantiate an object of the `AzureOpenAIEmbeddings` class from Langchain."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.embeddings import AzureOpenAIEmbeddings\n",
+    "from langchain_openai.embeddings import AzureOpenAIEmbeddings\n",
     "\n",
     "azure_embeddings = AzureOpenAIEmbeddings(\n",
     "    deployment=\"your-deployment-name\",\n",
@@ -72,40 +70,10 @@
     ")"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "62645da8-6a52-4cbb-bec7-59f7e153cd38",
-   "metadata": {},
-   "source": [
-    "To use the AzureOpenAIEmbeddings with the AnswerSimilarity metric, create an object of AnswerSimilarity by passing the azure_embeddings and llm as parameters."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "307321ed",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from ragas.metrics import AnswerSimilarity\n",
-    "\n",
-    "answer_similarity = AnswerSimilarity(llm=ragas_azure_model, embeddings=azure_embeddings)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "1930dd49",
-   "metadata": {},
-   "source": [
-    "That's it! answer_similarity will now be using AzureOpenAIEmbeddings under the hood for evaluations.\n",
-    "\n",
-    "Now lets run the evaluations using the example from [quickstart](../../getstarted/evaluation.md)."
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "id": "62c0eadb",
+   "execution_count": 7,
+   "id": "c4f0adec",
    "metadata": {},
    "outputs": [
     {
@@ -119,62 +87,72 @@
        "})"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "# data\n",
+    "#dataset\n",
     "from datasets import load_dataset\n",
     "\n",
     "fiqa_eval = load_dataset(\"explodinggradients/fiqa\", \"ragas_eval\")\n",
     "fiqa_eval"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "62645da8-6a52-4cbb-bec7-59f7e153cd38",
+   "metadata": {},
+   "source": [
+    "To use the AzureOpenAIEmbeddings with the `AnswerSimilarity` metric, we first import the answer_similarity metric and pass in the embeddings in the evaluate function."
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "id": "c4396f6e",
+   "execution_count": 9,
+   "id": "307321ed",
    "metadata": {},
    "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "evaluating with [answer_similarity]\n"
-     ]
-    },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 1/1 [00:01<00:00,  1.04s/it]\n"
+      "Evaluating: 100%|██████████| 5/5 [00:19<00:00,  3.86s/it]\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "{'answer_similarity': 0.8878}"
+       "{'answer_similarity': 0.8494}"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "# evaluate\n",
+    "from ragas.metrics import answer_similarity\n",
     "from ragas import evaluate\n",
     "\n",
     "result = evaluate(\n",
     "    fiqa_eval[\"baseline\"].select(range(5)),  # showing only 5 for demonstration\n",
     "    metrics=[answer_similarity],\n",
+    "    llm=azure_model,\n",
+    "    embeddings=azure_embeddings,\n",
     ")\n",
-    "\n",
     "result"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "d5f48c16",
+   "metadata": {},
+   "source": [
+    "That's it! answer_similarity will now be using AzureOpenAIEmbeddings under the hood for evaluations."
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "f490031e-fb73-4170-8762-61cadb4031e6",
@@ -182,12 +160,12 @@
    "source": [
     "## Evaluating with FastEmbed Embeddings\n",
     "\n",
-    "`FastEmbed` is a Python library built for embedding generation and has support for popular text models. Ragas has integration with FastEmbed and can be used by instantiating an object of the FastEmbedEmbeddings class. More information regarding FastEmbed and supported models can be found [here](https://github.com/qdrant/fastembed)."
+    "`FastEmbed` is a Python library built for embedding generation and has support for popular text models. FastEmbed models can be used by instantiating an object of the FastEmbedEmbeddings class from Langchain. More information regarding FastEmbed and supported models can be found [here](https://github.com/qdrant/fastembed)."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 10,
    "id": "85e313f2-e45c-4551-ab20-4e526e098740",
    "metadata": {},
    "outputs": [
@@ -195,12 +173,12 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 252M/252M [00:10<00:00, 25.0MiB/s] \n"
+      "100%|██████████| 252M/252M [00:09<00:00, 25.7MiB/s] \n"
      ]
     }
    ],
    "source": [
-    "from ragas.embeddings import FastEmbedEmbeddings\n",
+    "from langchain_community.embeddings.fastembed import FastEmbedEmbeddings\n",
     "\n",
     "fast_embeddings = FastEmbedEmbeddings(model_name=\"BAAI/bge-base-en\")"
    ]
@@ -210,25 +188,7 @@
    "id": "c9ddf74a-9830-4e1a-a4dd-7e5ec17a71e4",
    "metadata": {},
    "source": [
-    "Now lets create the metric object for AnswerSimilarity by passing the llm and embedding as the `FastEmbedEmbeddings` object that we created."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "2fd4adf3-db15-4c95-bf7c-407266517214",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "answer_similarity2 = AnswerSimilarity(llm=ragas_azure_model, embeddings=fast_embeddings)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "58a610f2-19e5-40ec-bb7d-760c1d608a85",
-   "metadata": {},
-   "source": [
-    "Now you can run the evaluations with and analyse the results."
+    "Now lets change the embedding model used for AnswerSimilarity by passing the embedding as the `FastEmbedEmbeddings` object that we created in the evaluation function."
    ]
   },
   {
@@ -237,24 +197,17 @@
    "id": "20882d05-1b54-4d17-88a0-f7ada2d6a576",
    "metadata": {},
    "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "evaluating with [answer_similarity]\n"
-     ]
-    },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 1/1 [00:03<00:00,  3.26s/it]\n"
+      "Evaluating: 100%|██████████| 5/5 [00:21<00:00,  4.24s/it]\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "{'answer_similarity': 0.8938}"
+       "{'answer_similarity': 0.8494}"
       ]
      },
      "execution_count": 11,
@@ -265,7 +218,9 @@
    "source": [
     "result2 = evaluate(\n",
     "    fiqa_eval[\"baseline\"].select(range(5)),  # showing only 5 for demonstration\n",
-    "    metrics=[answer_similarity2],\n",
+    "    metrics=[answer_similarity],\n",
+    "    llm=azure_model,\n",
+    "    embeddings=fast_embeddings,\n",
     ")\n",
     "\n",
     "result2"
@@ -292,7 +247,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -305,44 +260,35 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Now we follow the same steps as above to use the HuggingFace Embeddings in the ragas metrics and evaluate."
+    "Now we follow the same steps as above to use the HuggingFace Embeddings in the evaluate function with the metrics."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "answer_similarity3 = AnswerSimilarity(llm=ragas_azure_model, embeddings=hf_embeddings)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "evaluating with [answer_similarity]\n"
+      "Evaluating:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 1/1 [00:01<00:00,  1.35s/it]\n"
+      "Evaluating: 100%|██████████| 5/5 [00:21<00:00,  4.24s/it]\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "{'answer_similarity': 0.9156}"
+       "{'answer_similarity': 0.8494}"
       ]
      },
-     "execution_count": 21,
+     "execution_count": 15,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -350,7 +296,9 @@
    "source": [
     "result3 = evaluate(\n",
     "    fiqa_eval[\"baseline\"].select(range(5)),  # showing only 5 for demonstration\n",
-    "    metrics=[answer_similarity3],\n",
+    "    metrics=[answer_similarity],\n",
+    "    llm=azure_model,\n",
+    "    embeddings=hf_embeddings,\n",
     ")\n",
     "\n",
     "result3"
diff --git a/docs/howtos/customisations/llms.ipynb b/docs/howtos/customisations/llms.ipynb
index 14689df7a..eff17553f 100644
--- a/docs/howtos/customisations/llms.ipynb
+++ b/docs/howtos/customisations/llms.ipynb
@@ -52,65 +52,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 8,
    "id": "6906a4d6",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/anaconda3/envs/ragas/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:189: LangChainDeprecationWarning: The class `ChatOpenAI` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use langchain_openai.ChatOpenAI instead.\n",
-      "  warn_deprecated(\n"
-     ]
-    }
-   ],
-   "source": [
-    "from langchain.chat_models import ChatOpenAI\n",
-    "\n",
-    "gpt4 = ChatOpenAI(model_name=\"gpt-4\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "f1fdb48b",
-   "metadata": {},
-   "source": [
-    "`LangchainLLMWrapper` wrapper is required to use any Langchain llm. This helps Ragas library specify the interfaces that will be used internally by the metrics and what is exposed via the Langchain library. \n",
-    "\n",
-    "You can also use other LLM APIs in tools like LlamaIndex and LiteLLM but creating your own implementation of `BaseRagasLLM` that supports it."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "25c72521-3372-4663-81e4-c159b0edde40",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from ragas.llms.base import LangchainLLMWrapper\n",
-    "\n",
-    "gpt4_wrapper = LangchainLLMWrapper(langchain_llm=gpt4)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "62645da8-6a52-4cbb-bec7-59f7e153cd38",
-   "metadata": {},
-   "source": [
-    "Substitute the llm in `Metric` instance with the newly create GPT4 model."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "307321ed",
-   "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.metrics import faithfulness\n",
+    "from langchain_openai.chat_models import ChatOpenAI\n",
     "\n",
-    "faithfulness.llm = gpt4_wrapper"
+    "gpt4 = ChatOpenAI(model_name=\"gpt-4\")"
    ]
   },
   {
@@ -118,14 +67,14 @@
    "id": "1930dd49",
    "metadata": {},
    "source": [
-    "That's it! `faithfulness` will now be using GPT-4 under the hood for evaluations.\n",
+    "Now that we have setup the llm we can use `faithfulness` with GPT-4 under the hood for evaluations.\n",
     "\n",
-    "Now lets run the evaluations using the example from [quickstart](../quickstart.ipnb)."
+    "Now lets run the evaluations using the example from [quickstart](../quickstart.ipynb)."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 10,
    "id": "62c0eadb",
    "metadata": {},
    "outputs": [
@@ -133,22 +82,21 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Found cached dataset amnesty_qa (/Users/shahules/.cache/huggingface/datasets/explodinggradients___amnesty_qa/english/1.0.0/f7ca2ec9440beb13bf4f80092e9574ee0aa220cc6f37a702e55514dad061d6b6)\n",
-      "100%|████████████████████████████████████████████████████| 1/1 [00:00<00:00, 221.09it/s]\n"
+      "Repo card metadata block was not found. Setting CardData to empty.\n"
      ]
     },
     {
      "data": {
       "text/plain": [
        "DatasetDict({\n",
-       "    train: Dataset({\n",
+       "    eval: Dataset({\n",
        "        features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
        "        num_rows: 20\n",
        "    })\n",
        "})"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -163,7 +111,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 11,
    "id": "c4396f6e",
    "metadata": {},
    "outputs": [
@@ -171,18 +119,16 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/opt/anaconda3/envs/ragas/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:189: LangChainDeprecationWarning: The class `ChatOpenAI` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use langchain_openai.ChatOpenAI instead.\n",
-      "  warn_deprecated(\n",
-      "Evaluating: 100%|█████████████████████████████████████████| 5/5 [02:02<00:00, 24.41s/it]\n"
+      "Evaluating: 100%|██████████| 10/10 [01:21<00:00,  8.18s/it]\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "{'faithfulness': 0.1844}"
+       "{'faithfulness': 0.3389}"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -190,10 +136,12 @@
    "source": [
     "# evaluate\n",
     "from ragas import evaluate\n",
+    "from ragas.metrics import faithfulness\n",
     "\n",
     "result = evaluate(\n",
-    "    fiqa_eval[\"train\"].select(range(5)),  # showing only 5 for demonstration\n",
+    "    fiqa_eval[\"eval\"].select(range(10)),  # showing only 10 for demonstration\n",
     "    metrics=[faithfulness],\n",
+    "    llm=gpt4,\n",
     ")\n",
     "\n",
     "result"
@@ -250,7 +198,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain_openai.chat_models import ChatOpenAI\n",
     "from ragas.llms.base import LangchainLLMWrapper\n",
     "\n",
     "inference_server_url = \"http://localhost:8080/v1\"\n",
@@ -262,10 +210,7 @@
     "    openai_api_base=inference_server_url,\n",
     "    max_tokens=5,\n",
     "    temperature=0,\n",
-    ")\n",
-    "\n",
-    "# use the Ragas LangchainLLM wrapper to create a RagasLLM instance\n",
-    "vllm = LangchainLLMWrapper(chat)"
+    ")\n"
    ]
   },
   {
@@ -273,39 +218,7 @@
    "id": "2dd7932a-7933-4de8-a6af-2830457e02a0",
    "metadata": {},
    "source": [
-    "Now lets import all the metrics you want to use and change the llm."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "20882d05-1b54-4d17-88a0-f7ada2d6a576",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from ragas.metrics import (\n",
-    "    context_precision,\n",
-    "    answer_relevancy,\n",
-    "    faithfulness,\n",
-    "    context_recall,\n",
-    ")\n",
-    "from ragas.metrics.critique import harmfulness\n",
-    "\n",
-    "# change the LLM\n",
-    "\n",
-    "faithfulness.llm = vllm\n",
-    "answer_relevancy.llm = vllm\n",
-    "context_precision.llm = vllm\n",
-    "context_recall.llm = vllm\n",
-    "harmfulness.llm = vllm"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "58a610f2-19e5-40ec-bb7d-760c1d608a85",
-   "metadata": {},
-   "source": [
-    "Now you can run the evaluations with and analyse the results."
+    "Now lets import the metrics you want to use and change the llm in the evaluation."
    ]
   },
   {
@@ -341,11 +254,13 @@
    ],
    "source": [
     "# evaluate\n",
+    "from ragas.metrics import faithfulness\n",
     "from ragas import evaluate\n",
     "\n",
     "result = evaluate(\n",
-    "    fiqa_eval[\"baseline\"].select(range(5)),  # showing only 5 for demonstration\n",
+    "    fiqa_eval[\"baseline\"].select(range(1)),  # showing only 1 for demonstration\n",
     "    metrics=[faithfulness],\n",
+    "    llm=chat,\n",
     ")\n",
     "\n",
     "result"
@@ -354,9 +269,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "ragas",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "ragas"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {

From 9627fe8f74048ad7110b735f357fb723138c046b Mon Sep 17 00:00:00 2001
From: Tino Max <tinothayil@gmail.com>
Date: Wed, 24 Jan 2024 19:47:16 -0800
Subject: [PATCH 2/2] minor fix for dataset

---
 docs/howtos/customisations/embeddings.ipynb | 11 +++++------
 docs/howtos/customisations/llms.ipynb       |  8 ++++----
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/docs/howtos/customisations/embeddings.ipynb b/docs/howtos/customisations/embeddings.ipynb
index 7e405b88b..861ae788d 100644
--- a/docs/howtos/customisations/embeddings.ipynb
+++ b/docs/howtos/customisations/embeddings.ipynb
@@ -33,7 +33,6 @@
    "source": [
     "import os\n",
     "from langchain_openai.chat_models import AzureChatOpenAI\n",
-    "from ragas.llms.base import LangchainLLMWrapper\n",
     "\n",
     "os.environ[\"OPENAI_API_VERSION\"] = \"2023-05-15\"\n",
     "os.environ[\"OPENAI_API_KEY\"] = \"your-openai-key\"\n",
@@ -96,8 +95,8 @@
     "#dataset\n",
     "from datasets import load_dataset\n",
     "\n",
-    "fiqa_eval = load_dataset(\"explodinggradients/fiqa\", \"ragas_eval\")\n",
-    "fiqa_eval"
+    "amnesty_qa = load_dataset(\"explodinggradients/amnesty_qa\", \"english\")\n",
+    "amnesty_qa"
    ]
   },
   {
@@ -137,7 +136,7 @@
     "from ragas import evaluate\n",
     "\n",
     "result = evaluate(\n",
-    "    fiqa_eval[\"baseline\"].select(range(5)),  # showing only 5 for demonstration\n",
+    "    amnesty_qa[\"eval\"].select(range(5)),  # showing only 5 for demonstration\n",
     "    metrics=[answer_similarity],\n",
     "    llm=azure_model,\n",
     "    embeddings=azure_embeddings,\n",
@@ -217,7 +216,7 @@
    ],
    "source": [
     "result2 = evaluate(\n",
-    "    fiqa_eval[\"baseline\"].select(range(5)),  # showing only 5 for demonstration\n",
+    "    amnesty_qa[\"eval\"].select(range(5)),  # showing only 5 for demonstration\n",
     "    metrics=[answer_similarity],\n",
     "    llm=azure_model,\n",
     "    embeddings=fast_embeddings,\n",
@@ -295,7 +294,7 @@
    ],
    "source": [
     "result3 = evaluate(\n",
-    "    fiqa_eval[\"baseline\"].select(range(5)),  # showing only 5 for demonstration\n",
+    "    amnesty_qa[\"eval\"].select(range(5)),  # showing only 5 for demonstration\n",
     "    metrics=[answer_similarity],\n",
     "    llm=azure_model,\n",
     "    embeddings=hf_embeddings,\n",
diff --git a/docs/howtos/customisations/llms.ipynb b/docs/howtos/customisations/llms.ipynb
index eff17553f..27462c077 100644
--- a/docs/howtos/customisations/llms.ipynb
+++ b/docs/howtos/customisations/llms.ipynb
@@ -105,8 +105,8 @@
     "# data\n",
     "from datasets import load_dataset\n",
     "\n",
-    "fiqa_eval = load_dataset(\"explodinggradients/amnesty_qa\", \"english\")\n",
-    "fiqa_eval"
+    "amnesty_qa = load_dataset(\"explodinggradients/amnesty_qa\", \"english\")\n",
+    "amnesty_qa"
    ]
   },
   {
@@ -139,7 +139,7 @@
     "from ragas.metrics import faithfulness\n",
     "\n",
     "result = evaluate(\n",
-    "    fiqa_eval[\"eval\"].select(range(10)),  # showing only 10 for demonstration\n",
+    "    amnesty_qa[\"eval\"].select(range(10)),  # showing only 10 for demonstration\n",
     "    metrics=[faithfulness],\n",
     "    llm=gpt4,\n",
     ")\n",
@@ -258,7 +258,7 @@
     "from ragas import evaluate\n",
     "\n",
     "result = evaluate(\n",
-    "    fiqa_eval[\"baseline\"].select(range(1)),  # showing only 1 for demonstration\n",
+    "    amnesty_qa[\"eval\"].select(range(1)),  # showing only 1 for demonstration\n",
     "    metrics=[faithfulness],\n",
     "    llm=chat,\n",
     ")\n",