examples/run_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/akshat_g/athina/repos/athina-evals/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
      "  warnings.warn(\n",
      "/Users/akshat_g/athina/repos/athina-evals/.venv/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from athina.evals import ContextContainsEnoughInformation\n",
    "from athina.loaders import Loader\n",
    "from athina.keys import AthinaApiKey, OpenAiApiKey\n",
    "from athina.interfaces.athina import AthinaExperiment\n",
    "from athina.datasets import yc_query_mini"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configure your API keys\n",
    "\n",
    "Evals use OpenAI, so you need to configure your OpenAI API key.\n",
    "\n",
    "If you wish to view the results on Athina's UI, and maintain a historical record of experiments, then you also need an Athina API Key."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "OpenAiApiKey.set_key(os.getenv('OPENAI_API_KEY'))\n",
    "AthinaApiKey.set_key(os.getenv('ATHINA_API_KEY')) # Optional, recommended"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load your dataset\n",
    "\n",
    "You can use one of our `loaders` to load the data from a Dictionary, CSV or JSON file.\n",
    "\n",
    "Here's an example\n",
    "```\n",
    "from athina.loaders import Loader\n",
    "\n",
    "dataset = Loader().load_dict(raw_data)\n",
    "```\n",
    "\n",
    "Here is the complete [documentation](https://docs.athina.ai/evals/running_evals/loading_data) specifying the various ways you can load your dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>context</th>\n",
       "      <th>response</th>\n",
       "      <th>expected_response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What are some successful companies that went t...</td>\n",
       "      <td>[Y Combinator has invested in companies in var...</td>\n",
       "      <td>Airbnb, Dropbox, Stripe, Reddit, Coinbase, Ins...</td>\n",
       "      <td>Airbnb and Stripe are 2 of the successful comp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In which city is YC located?</td>\n",
       "      <td>[Y Combinator is located in Mountain View, Cal...</td>\n",
       "      <td>Y Combinator is located in San Francisco</td>\n",
       "      <td>YC is located in Mountain View, California.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How much equity does YC take?</td>\n",
       "      <td>[Y Combinator invests $500k in 200 startups tw...</td>\n",
       "      <td>YC invests $150k for 7%.</td>\n",
       "      <td>I cannot answer this question as I do not have...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How much equity does YC take?</td>\n",
       "      <td>[Y Combinator invests $500k in 200 startups tw...</td>\n",
       "      <td>I cannot answer this question as I do not have...</td>\n",
       "      <td>I cannot answer this question as I do not have...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Who founded YC and when was it founded?</td>\n",
       "      <td>[Y Combinator was founded in March 2005 by Pau...</td>\n",
       "      <td>Y Combinator was founded in 2005</td>\n",
       "      <td>Y Combinator was founded in March 2005 by Paul...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Does Y Combinator invest in startups outside t...</td>\n",
       "      <td>[Y Combinator invests in startups from all ove...</td>\n",
       "      <td>Yes, Y Combinator invests in international sta...</td>\n",
       "      <td>Yes, Y Combinator invests in startups from all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>How much does YC invest in startups?</td>\n",
       "      <td>[YC invests $150k for 7%.]</td>\n",
       "      <td>$150k</td>\n",
       "      <td>YC invests $150k for 7%.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>What is YC's motto?</td>\n",
       "      <td>[Y Combinator's motto is 'Make something peopl...</td>\n",
       "      <td>Make something people want</td>\n",
       "      <td>Make something people want</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               query  \\\n",
       "0  What are some successful companies that went t...   \n",
       "1                       In which city is YC located?   \n",
       "2                      How much equity does YC take?   \n",
       "3                      How much equity does YC take?   \n",
       "4            Who founded YC and when was it founded?   \n",
       "5  Does Y Combinator invest in startups outside t...   \n",
       "6               How much does YC invest in startups?   \n",
       "7                                What is YC's motto?   \n",
       "\n",
       "                                             context  \\\n",
       "0  [Y Combinator has invested in companies in var...   \n",
       "1  [Y Combinator is located in Mountain View, Cal...   \n",
       "2  [Y Combinator invests $500k in 200 startups tw...   \n",
       "3  [Y Combinator invests $500k in 200 startups tw...   \n",
       "4  [Y Combinator was founded in March 2005 by Pau...   \n",
       "5  [Y Combinator invests in startups from all ove...   \n",
       "6                         [YC invests $150k for 7%.]   \n",
       "7  [Y Combinator's motto is 'Make something peopl...   \n",
       "\n",
       "                                            response  \\\n",
       "0  Airbnb, Dropbox, Stripe, Reddit, Coinbase, Ins...   \n",
       "1           Y Combinator is located in San Francisco   \n",
       "2                           YC invests $150k for 7%.   \n",
       "3  I cannot answer this question as I do not have...   \n",
       "4                   Y Combinator was founded in 2005   \n",
       "5  Yes, Y Combinator invests in international sta...   \n",
       "6                                              $150k   \n",
       "7                         Make something people want   \n",
       "\n",
       "                                   expected_response  \n",
       "0  Airbnb and Stripe are 2 of the successful comp...  \n",
       "1        YC is located in Mountain View, California.  \n",
       "2  I cannot answer this question as I do not have...  \n",
       "3  I cannot answer this question as I do not have...  \n",
       "4  Y Combinator was founded in March 2005 by Paul...  \n",
       "5  Yes, Y Combinator invests in startups from all...  \n",
       "6                           YC invests $150k for 7%.  \n",
       "7                         Make something people want  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create or load batch dataset\n",
    "raw_data = yc_query_mini.data\n",
    "dataset = Loader().load_dict(raw_data)\n",
    "\n",
    "pd.DataFrame(dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Describe your experiment metadata fields (optional)\n",
    "These metadata fields are only used as identifiers when we save your experiment on Athina Develop.\n",
    "This helps you search, sort and filter through past experimentation runs.\n",
    "\n",
    "Currently, this includes your:\n",
    "- `experiment_name`: (string) The name of your experiment\n",
    "- `experiment_description`: (string) A description this iteration of your experiment\n",
    "- `language_model_provider`: (string) `openai`\n",
    "- `language_model_id`: (string) The language model used for the LLM inference (ex: `gpt-3.5-turbo`)\n",
    "- `prompt_template`: (object) A JS object representing the prompt you are sending to the LLM (for example, messages array in OpenAI)\n",
    "- `dataset_name`: (string) An identifier for the dataset you are using."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define your experiment parameters\n",
    "prompt_template = [\n",
    "    { \n",
    "        \"role\": \"system\",\n",
    "        \"content\": \"You are an expert at answering questions about Y Combinator. If you do not know the answer, say I don't know. Be direct and concise in your responses\" },\n",
    "    { \n",
    "        \"role\": \"user\", \n",
    "        \"content\": \"{query}\"\n",
    "    }\n",
    "]\n",
    "experiment = AthinaExperiment(\n",
    "    experiment_name=\"ContextRelevance\",\n",
    "    experiment_description=\"Checking retrieval scores for YC dataset with a simple zero-shot prompt\",\n",
    "    language_model_provider=\"openai\",\n",
    "    language_model_id=\"gpt-3.5-turbo\",\n",
    "    prompt_template=prompt_template,\n",
    "    dataset_name=\"yc_dataset_mini\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run your evaluation\n",
    "\n",
    "Simply instantiate the evaluator class you wish to use, and call `run_batch` to the eval\n",
    "\n",
    "##### Run evals in parallel (much faster)\n",
    "\n",
    "You may specify `max_parallel_evals` to run multiple LLM evaluation inferences in parallel.\n",
    "\n",
    "##### View as a dataframe\n",
    "Call `.to_df()` on the results to view as a dataframe\n",
    "\n",
    "\n",
    "##### Log results to Athina Develop (Dashboard UI)\n",
    "If you have specified an `AthinaApiKey`, then results will automatically logged to the dashboard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "201\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>context</th>\n",
       "      <th>response</th>\n",
       "      <th>expected_response</th>\n",
       "      <th>display_name</th>\n",
       "      <th>failed</th>\n",
       "      <th>grade_reason</th>\n",
       "      <th>runtime</th>\n",
       "      <th>model</th>\n",
       "      <th>passed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What are some successful companies that went through YC?</td>\n",
       "      <td>[Y Combinator has invested in companies in various fields like FinTech, Healthcare, AI, etc.]</td>\n",
       "      <td>Airbnb, Dropbox, Stripe, Reddit, Coinbase, Instacart.</td>\n",
       "      <td>Airbnb and Stripe are 2 of the successful companies that went through YC.</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>True</td>\n",
       "      <td>The context provided does not include specific examples of successful companies that went through Y Combinator (YC). It only mentions that YC has invested in companies across various fields such as FinTech, Healthcare, AI, etc. Without specific company names or examples, the chatbot cannot answer the user's query based solely on the context given.</td>\n",
       "      <td>4005</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In which city is YC located?</td>\n",
       "      <td>[Y Combinator is located in Mountain View, California.]</td>\n",
       "      <td>Y Combinator is located in San Francisco</td>\n",
       "      <td>YC is located in Mountain View, California.</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>False</td>\n",
       "      <td>The context clearly states that Y Combinator is located in Mountain View, California. Therefore, the chatbot can use this information to answer the user's query about the city in which YC is located, which is Mountain View.</td>\n",
       "      <td>2456</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How much equity does YC take?</td>\n",
       "      <td>[Y Combinator invests $500k in 200 startups twice a year.]</td>\n",
       "      <td>YC invests $150k for 7%.</td>\n",
       "      <td>I cannot answer this question as I do not have enough information.</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>True</td>\n",
       "      <td>The context provided states the amount Y Combinator invests in startups and how often, but it does not specify the percentage of equity YC takes in return for its investment. Therefore, the chatbot cannot answer the user's query about YC's equity stake based solely on the provided context.</td>\n",
       "      <td>3083</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How much equity does YC take?</td>\n",
       "      <td>[Y Combinator invests $500k in 200 startups twice a year.]</td>\n",
       "      <td>I cannot answer this question as I do not have enough information.</td>\n",
       "      <td>I cannot answer this question as I do not have enough information.</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>True</td>\n",
       "      <td>The context provided only states the amount Y Combinator invests in startups and how often, but it does not specify the percentage of equity YC takes in return for its investment. Therefore, the chatbot cannot answer the user's query about YC's equity take with the information given.</td>\n",
       "      <td>3361</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Who founded YC and when was it founded?</td>\n",
       "      <td>[Y Combinator was founded in March 2005 by Paul Graham, Jessica Livingston, Trevor Blackwell, and Robert Tappan Morris.]</td>\n",
       "      <td>Y Combinator was founded in 2005</td>\n",
       "      <td>Y Combinator was founded in March 2005 by Paul Graham, Jessica Livingston, Trevor Blackwell, and Robert Tappan Morris.</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>False</td>\n",
       "      <td>The context provides the names of the founders of Y Combinator (Paul Graham, Jessica Livingston, Trevor Blackwell, and Robert Tappan Morris) and the date it was founded (March 2005), which directly answers the user's query about who founded YC and when it was founded.</td>\n",
       "      <td>2833</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Does Y Combinator invest in startups outside the US?</td>\n",
       "      <td>[Y Combinator invests in startups from all over the world.]</td>\n",
       "      <td>Yes, Y Combinator invests in international startups as well as US startups.</td>\n",
       "      <td>Yes, Y Combinator invests in startups from all over the world.</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>False</td>\n",
       "      <td>The context clearly states that Y Combinator invests in startups from all over the world, which directly answers the user's query about whether Y Combinator invests in startups outside the US. Therefore, the chatbot can use the context information to affirm that Y Combinator does invest in startups outside the US.</td>\n",
       "      <td>4886</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>How much does YC invest in startups?</td>\n",
       "      <td>[YC invests $150k for 7%.]</td>\n",
       "      <td>$150k</td>\n",
       "      <td>YC invests $150k for 7%.</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>False</td>\n",
       "      <td>The context directly answers the user's query by stating that YC invests $150k for 7% equity in startups. This information is sufficient to inform the user about the amount YC invests.</td>\n",
       "      <td>1879</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>What is YC's motto?</td>\n",
       "      <td>[Y Combinator's motto is 'Make something people want'.]</td>\n",
       "      <td>Make something people want</td>\n",
       "      <td>Make something people want</td>\n",
       "      <td>Context Contains Enough Information</td>\n",
       "      <td>False</td>\n",
       "      <td>The context directly provides the answer to the user's query. The user asked for Y Combinator's motto, and the context states that Y Combinator's motto is 'Make something people want'. Therefore, the chatbot can use the context information to answer the user's query accurately.</td>\n",
       "      <td>3536</td>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      query  \\\n",
       "0  What are some successful companies that went through YC?   \n",
       "1                              In which city is YC located?   \n",
       "2                             How much equity does YC take?   \n",
       "3                             How much equity does YC take?   \n",
       "4                   Who founded YC and when was it founded?   \n",
       "5      Does Y Combinator invest in startups outside the US?   \n",
       "6                      How much does YC invest in startups?   \n",
       "7                                       What is YC's motto?   \n",
       "\n",
       "                                                                                                                    context  \\\n",
       "0                             [Y Combinator has invested in companies in various fields like FinTech, Healthcare, AI, etc.]   \n",
       "1                                                                   [Y Combinator is located in Mountain View, California.]   \n",
       "2                                                                [Y Combinator invests $500k in 200 startups twice a year.]   \n",
       "3                                                                [Y Combinator invests $500k in 200 startups twice a year.]   \n",
       "4  [Y Combinator was founded in March 2005 by Paul Graham, Jessica Livingston, Trevor Blackwell, and Robert Tappan Morris.]   \n",
       "5                                                               [Y Combinator invests in startups from all over the world.]   \n",
       "6                                                                                                [YC invests $150k for 7%.]   \n",
       "7                                                                   [Y Combinator's motto is 'Make something people want'.]   \n",
       "\n",
       "                                                                      response  \\\n",
       "0                        Airbnb, Dropbox, Stripe, Reddit, Coinbase, Instacart.   \n",
       "1                                     Y Combinator is located in San Francisco   \n",
       "2                                                     YC invests $150k for 7%.   \n",
       "3           I cannot answer this question as I do not have enough information.   \n",
       "4                                             Y Combinator was founded in 2005   \n",
       "5  Yes, Y Combinator invests in international startups as well as US startups.   \n",
       "6                                                                        $150k   \n",
       "7                                                   Make something people want   \n",
       "\n",
       "                                                                                                        expected_response  \\\n",
       "0                                               Airbnb and Stripe are 2 of the successful companies that went through YC.   \n",
       "1                                                                             YC is located in Mountain View, California.   \n",
       "2                                                      I cannot answer this question as I do not have enough information.   \n",
       "3                                                      I cannot answer this question as I do not have enough information.   \n",
       "4  Y Combinator was founded in March 2005 by Paul Graham, Jessica Livingston, Trevor Blackwell, and Robert Tappan Morris.   \n",
       "5                                                          Yes, Y Combinator invests in startups from all over the world.   \n",
       "6                                                                                                YC invests $150k for 7%.   \n",
       "7                                                                                              Make something people want   \n",
       "\n",
       "                          display_name  failed  \\\n",
       "0  Context Contains Enough Information    True   \n",
       "1  Context Contains Enough Information   False   \n",
       "2  Context Contains Enough Information    True   \n",
       "3  Context Contains Enough Information    True   \n",
       "4  Context Contains Enough Information   False   \n",
       "5  Context Contains Enough Information   False   \n",
       "6  Context Contains Enough Information   False   \n",
       "7  Context Contains Enough Information   False   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                    grade_reason  \\\n",
       "0  The context provided does not include specific examples of successful companies that went through Y Combinator (YC). It only mentions that YC has invested in companies across various fields such as FinTech, Healthcare, AI, etc. Without specific company names or examples, the chatbot cannot answer the user's query based solely on the context given.   \n",
       "1                                                                                                                                The context clearly states that Y Combinator is located in Mountain View, California. Therefore, the chatbot can use this information to answer the user's query about the city in which YC is located, which is Mountain View.   \n",
       "2                                                             The context provided states the amount Y Combinator invests in startups and how often, but it does not specify the percentage of equity YC takes in return for its investment. Therefore, the chatbot cannot answer the user's query about YC's equity stake based solely on the provided context.   \n",
       "3                                                                   The context provided only states the amount Y Combinator invests in startups and how often, but it does not specify the percentage of equity YC takes in return for its investment. Therefore, the chatbot cannot answer the user's query about YC's equity take with the information given.   \n",
       "4                                                                                   The context provides the names of the founders of Y Combinator (Paul Graham, Jessica Livingston, Trevor Blackwell, and Robert Tappan Morris) and the date it was founded (March 2005), which directly answers the user's query about who founded YC and when it was founded.   \n",
       "5                                    The context clearly states that Y Combinator invests in startups from all over the world, which directly answers the user's query about whether Y Combinator invests in startups outside the US. Therefore, the chatbot can use the context information to affirm that Y Combinator does invest in startups outside the US.   \n",
       "6                                                                                                                                                                       The context directly answers the user's query by stating that YC invests $150k for 7% equity in startups. This information is sufficient to inform the user about the amount YC invests.   \n",
       "7                                                                         The context directly provides the answer to the user's query. The user asked for Y Combinator's motto, and the context states that Y Combinator's motto is 'Make something people want'. Therefore, the chatbot can use the context information to answer the user's query accurately.   \n",
       "\n",
       "   runtime               model  passed  \n",
       "0     4005  gpt-4-1106-preview     0.0  \n",
       "1     2456  gpt-4-1106-preview     1.0  \n",
       "2     3083  gpt-4-1106-preview     0.0  \n",
       "3     3361  gpt-4-1106-preview     0.0  \n",
       "4     2833  gpt-4-1106-preview     1.0  \n",
       "5     4886  gpt-4-1106-preview     1.0  \n",
       "6     1879  gpt-4-1106-preview     1.0  \n",
       "7     3536  gpt-4-1106-preview     1.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checks if the LLM response answers the user query sufficiently\n",
    "results = ContextContainsEnoughInformation().configure_experiment(experiment).run_batch(\n",
    "    data=dataset,\n",
    "    max_parallel_evals=5 # Run up to 5 evals in parallel\n",
    ")\n",
    "\n",
    "results.to_df()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}