llm_observability_ragas.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Intro\n",
        "From https://towardsdatascience.com/evaluating-rag-applications-with-ragas-81d67b0ee31a"
      ],
      "metadata": {
        "id": "e8Io82jX2uvK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Prerequisites\n",
        " * `langchain`, `openai`, and `weaviate-client` for the RAG pipeline\n",
        " * `ragas` for evaluating the RAG pipeline"
      ],
      "metadata": {
        "id": "_3N-53L823Zz"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JAYpi-oM2cOW"
      },
      "outputs": [],
      "source": [
        "!pip install langchain langchain-openai openai weaviate-client ragas"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from getpass import getpass\n",
        "\n",
        "if os.environ.get(\"OPENAI_API_KEY\") is None:\n",
        "    openai_api_key = getpass(\"🔑 Enter your OpenAI API key: \")\n",
        "    os.environ[\"OPENAI_API_KEY\"] = openai_api_key"
      ],
      "metadata": {
        "id": "6y400-tj3MXg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Prepare Data"
      ],
      "metadata": {
        "id": "UOc4u1743Yd6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from langchain.document_loaders import TextLoader\n",
        "from langchain.text_splitter import CharacterTextSplitter\n",
        "\n",
        "url = \"https://raw.githubusercontent.com/langchain-ai/langchain/master/docs/docs/modules/state_of_the_union.txt\"\n",
        "res = requests.get(url)\n",
        "with open(\"state_of_the_union.txt\", \"w\") as f:\n",
        "    f.write(res.text)\n",
        "\n",
        "# Load the data\n",
        "loader = TextLoader('./state_of_the_union.txt')\n",
        "documents = loader.load()\n",
        "\n",
        "# Chunk the data\n",
        "text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=50)\n",
        "chunks = text_splitter.split_documents(documents)\n"
      ],
      "metadata": {
        "id": "5CKrsIIi3aGx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Vector Embedding\n",
        "Generate the vector embeddings for each chunk with the OpenAI embedding model and store them in the vector database"
      ],
      "metadata": {
        "id": "SXzPdjyY32Qw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain_openai import OpenAIEmbeddings\n",
        "from langchain.vectorstores import Weaviate\n",
        "import weaviate\n",
        "from weaviate.embedded import EmbeddedOptions\n",
        "#from dotenv import load_dotenv,find_dotenv\n",
        "\n",
        "# Load OpenAI API key from .env file\n",
        "#load_dotenv(find_dotenv())\n",
        "\n",
        "# Setup vector database\n",
        "client = weaviate.Client(\n",
        "  embedded_options = EmbeddedOptions()\n",
        ")\n",
        "\n",
        "# Populate vector database\n",
        "vectorstore = Weaviate.from_documents(\n",
        "    client = client,\n",
        "    documents = chunks,\n",
        "    embedding = OpenAIEmbeddings(),\n",
        "    by_text = False\n",
        ")\n",
        "\n",
        "# Define vectorstore as retriever to enable semantic search\n",
        "retriever = vectorstore.as_retriever()"
      ],
      "metadata": {
        "id": "6pVbBIaX3tdu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Combine into RAG\n",
        "set up a prompt template and the OpenAI LLM and combine them with the retriever component to a RAG pipeline"
      ],
      "metadata": {
        "id": "TRPOK5gN37yI"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#from langchain.chat_models import ChatOpenAI\n",
        "from langchain_openai import ChatOpenAI\n",
        "from langchain.prompts import ChatPromptTemplate\n",
        "from langchain.schema.runnable import RunnablePassthrough\n",
        "from langchain.schema.output_parser import StrOutputParser\n",
        "\n",
        "# Define LLM\n",
        "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
        "\n",
        "# Define prompt template\n",
        "template = \"\"\"You are an assistant for question-answering tasks.\n",
        "Use the following pieces of retrieved context to answer the question.\n",
        "If you don't know the answer, just say that you don't know.\n",
        "Use two sentences maximum and keep the answer concise.\n",
        "Question: {question}\n",
        "Context: {context}\n",
        "Answer:\n",
        "\"\"\"\n",
        "\n",
        "prompt = ChatPromptTemplate.from_template(template)\n",
        "\n",
        "# Setup RAG pipeline\n",
        "rag_chain = (\n",
        "    {\"context\": retriever,  \"question\": RunnablePassthrough()}\n",
        "    | prompt\n",
        "    | llm\n",
        "    | StrOutputParser()\n",
        ")"
      ],
      "metadata": {
        "id": "Bf9hn_ir4CZ0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Prepare Eval Data"
      ],
      "metadata": {
        "id": "vQmo-Dje4nDl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import Dataset\n",
        "\n",
        "questions = [\"What did the president say about Justice Breyer?\",\n",
        "             \"What did the president say about Intel's CEO?\",\n",
        "             \"What did the president say about gun violence?\",\n",
        "            ]\n",
        "ground_truths = [[\"The president said that Justice Breyer has dedicated his life to serve the country and thanked him for his service.\"],\n",
        "                [\"The president said that Pat Gelsinger is ready to increase Intel's investment to $100 billion.\"],\n",
        "                [\"The president asked Congress to pass proven measures to reduce gun violence.\"]]\n",
        "answers = []\n",
        "contexts = []\n",
        "\n",
        "# Inference\n",
        "for query in questions:\n",
        "  answers.append(rag_chain.invoke(query))\n",
        "  contexts.append([docs.page_content for docs in retriever.get_relevant_documents(query)])\n",
        "\n",
        "# To dict\n",
        "data = {\n",
        "    \"question\": questions,\n",
        "    \"answer\": answers,\n",
        "    \"contexts\": contexts,\n",
        "    \"ground_truths\": ground_truths\n",
        "}\n",
        "\n",
        "# Convert dict to dataset\n",
        "dataset = Dataset.from_dict(data)\n"
      ],
      "metadata": {
        "id": "ezyAe65f4o7f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluating the RAG application"
      ],
      "metadata": {
        "id": "UCgGHiIQ4xKe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from ragas import evaluate\n",
        "from ragas.metrics import (\n",
        "    faithfulness,\n",
        "    answer_relevancy,\n",
        "    context_recall,\n",
        "    context_precision,\n",
        ")\n",
        "\n",
        "result = evaluate(\n",
        "    dataset = dataset,\n",
        "    metrics=[\n",
        "        context_precision,\n",
        "        context_recall,\n",
        "        faithfulness,\n",
        "        answer_relevancy,\n",
        "    ],\n",
        ")\n",
        "\n",
        "df = result.to_pandas()\n"
      ],
      "metadata": {
        "id": "3YxvCaIp4ydD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "id": "KIElwswD49ky"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}