datasets/finrag/study/evaluation/evaluation_v1_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e7053c81-2d56-4894-bd53-616585195df2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from typing import *\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from run_evaluation import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2488f2e2-1899-4c87-8d99-be5853fbfc1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# GLOBAL settings for this run (v1)\n",
    "token_limit = 4000\n",
    "# Parsee.ai template ID, can be seen here (requires free log-in): https://app.parsee.ai/template/661bd24fb129ff2526f2af09\n",
    "template_id = \"661bd24fb129ff2526f2af09\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bc2bc59-0519-4fa4-8d95-27ef08f70d48",
   "metadata": {},
   "source": [
    "# RAG-text Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c12f248-7d24-4554-a7d7-6ee74eb92c27",
   "metadata": {},
   "source": [
    "For details about this dataset, check the [Readme.](../../README.md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a3765b3f-c5ed-4866-8951-7f76756c7b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The following models are being used (to re-run, replace 'n/a' with the respective API keys):\n",
    "models = [\n",
    "            anthropic_config(\"n/a\", \"claude-3-opus-20240229\", token_limit),\n",
    "            gpt_config(\"n/a\", token_limit, \"gpt-4-1106-preview\"),\n",
    "            replicate_config(\"n/a\", \"meta/meta-llama-3-70b-instruct\", token_limit),\n",
    "            replicate_config(\"n/a\", \"meta/meta-llama-3-8b-instruct\", token_limit),\n",
    "            together_config(\"n/a\", \"mistralai/Mixtral-8x22B-Instruct-v0.1\", token_limit),\n",
    "            together_config(\"n/a\", \"databricks/dbrx-instruct\", token_limit),\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "495ce52e-ef60-49ac-af14-e1e3ffc9c909",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>total</th>\n",
       "      <th>rag_text_50_rev23_meta_total</th>\n",
       "      <th>rag_text_50_rev23_thousands_no_hint_total</th>\n",
       "      <th>rag_text_50_rev23_thousands_hint_total</th>\n",
       "      <th>rag_text_50_rev23_millions_no_hint_total</th>\n",
       "      <th>rag_text_50_rev_meta_total</th>\n",
       "      <th>rag_text_50_rev22_millions_no_hint_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>claude-3-opus-20240229</td>\n",
       "      <td>0.888953</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.92</td>\n",
       "      <td>0.903716</td>\n",
       "      <td>0.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>0.741836</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.701014</td>\n",
       "      <td>0.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meta/meta-llama-3-70b-instruct</td>\n",
       "      <td>0.699459</td>\n",
       "      <td>0.885</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.631757</td>\n",
       "      <td>0.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mistralai/Mixtral-8x22B-Instruct-v0.1</td>\n",
       "      <td>0.540755</td>\n",
       "      <td>0.805</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.339527</td>\n",
       "      <td>0.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>databricks/dbrx-instruct</td>\n",
       "      <td>0.519144</td>\n",
       "      <td>0.585</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.489865</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>meta/meta-llama-3-8b-instruct</td>\n",
       "      <td>0.207500</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   model     total  \\\n",
       "0                 claude-3-opus-20240229  0.888953   \n",
       "1                     gpt-4-1106-preview  0.741836   \n",
       "2         meta/meta-llama-3-70b-instruct  0.699459   \n",
       "4  mistralai/Mixtral-8x22B-Instruct-v0.1  0.540755   \n",
       "5               databricks/dbrx-instruct  0.519144   \n",
       "3          meta/meta-llama-3-8b-instruct  0.207500   \n",
       "\n",
       "   rag_text_50_rev23_meta_total  rag_text_50_rev23_thousands_no_hint_total  \\\n",
       "0                         0.970                                       0.86   \n",
       "1                         0.910                                       0.66   \n",
       "2                         0.885                                       0.56   \n",
       "4                         0.805                                       0.72   \n",
       "5                         0.585                                       0.46   \n",
       "3                         0.365                                       0.24   \n",
       "\n",
       "   rag_text_50_rev23_thousands_hint_total  \\\n",
       "0                                    0.84   \n",
       "1                                    0.86   \n",
       "2                                    0.52   \n",
       "4                                    0.62   \n",
       "5                                    0.60   \n",
       "3                                    0.32   \n",
       "\n",
       "   rag_text_50_rev23_millions_no_hint_total  rag_text_50_rev_meta_total  \\\n",
       "0                                      0.92                    0.903716   \n",
       "1                                      0.76                    0.701014   \n",
       "2                                      0.82                    0.631757   \n",
       "4                                      0.48                    0.339527   \n",
       "5                                      0.60                    0.489865   \n",
       "3                                      0.24                         NaN   \n",
       "\n",
       "   rag_text_50_rev22_millions_no_hint_total  \n",
       "0                                      0.84  \n",
       "1                                      0.56  \n",
       "2                                      0.78  \n",
       "4                                      0.28  \n",
       "5                                      0.38  \n",
       "3                                      0.08  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df, all_dataset_names = make_df(template_id, \"../data/v1/by_sub_dataset_with_answers/rag-text\", models)\n",
    "df[['model', 'total']+[f\"{x}_total\" for x in all_dataset_names]].sort_values('total', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e13e723-cdc3-4865-8363-d17ea2b38f2b",
   "metadata": {},
   "source": [
    "# Selection-text Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b96912c-0e49-4025-a467-edbb5b43dc77",
   "metadata": {},
   "source": [
    "For details about this dataset, check the [Readme.](../../README.md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e0ffdad7-9b25-482b-86be-b40f70da17a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>total</th>\n",
       "      <th>selection_text_50_rev22_millions_no_hint_total</th>\n",
       "      <th>selection_text_50_rev23_meta_total</th>\n",
       "      <th>selection_text_50_rev23_millions_no_hint_total</th>\n",
       "      <th>selection_text_50_rev23_thousands_hint_total</th>\n",
       "      <th>selection_text_50_rev23_thousands_no_hint_total</th>\n",
       "      <th>selection_text_50_rev_meta_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>claude-3-opus-20240229</td>\n",
       "      <td>0.900236</td>\n",
       "      <td>0.90</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.981419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4-1106-preview</td>\n",
       "      <td>0.852849</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.970</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.907095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meta/meta-llama-3-70b-instruct</td>\n",
       "      <td>0.761745</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.910</td>\n",
       "      <td>0.96</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.660473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mistralai/Mixtral-8x22B-Instruct-v0.1</td>\n",
       "      <td>0.685946</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.925</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.550676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>databricks/dbrx-instruct</td>\n",
       "      <td>0.561227</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.655</td>\n",
       "      <td>0.62</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.552365</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   model     total  \\\n",
       "0                 claude-3-opus-20240229  0.900236   \n",
       "1                     gpt-4-1106-preview  0.852849   \n",
       "2         meta/meta-llama-3-70b-instruct  0.761745   \n",
       "4  mistralai/Mixtral-8x22B-Instruct-v0.1  0.685946   \n",
       "3               databricks/dbrx-instruct  0.561227   \n",
       "\n",
       "   selection_text_50_rev22_millions_no_hint_total  \\\n",
       "0                                            0.90   \n",
       "1                                            0.62   \n",
       "2                                            0.96   \n",
       "4                                            0.60   \n",
       "3                                            0.44   \n",
       "\n",
       "   selection_text_50_rev23_meta_total  \\\n",
       "0                               1.000   \n",
       "1                               0.970   \n",
       "2                               0.910   \n",
       "4                               0.925   \n",
       "3                               0.655   \n",
       "\n",
       "   selection_text_50_rev23_millions_no_hint_total  \\\n",
       "0                                            0.86   \n",
       "1                                            0.86   \n",
       "2                                            0.96   \n",
       "4                                            0.62   \n",
       "3                                            0.62   \n",
       "\n",
       "   selection_text_50_rev23_thousands_hint_total  \\\n",
       "0                                          0.90   \n",
       "1                                          0.96   \n",
       "2                                          0.54   \n",
       "4                                          0.60   \n",
       "3                                          0.58   \n",
       "\n",
       "   selection_text_50_rev23_thousands_no_hint_total  \\\n",
       "0                                             0.76   \n",
       "1                                             0.80   \n",
       "2                                             0.54   \n",
       "4                                             0.82   \n",
       "3                                             0.52   \n",
       "\n",
       "   selection_text_50_rev_meta_total  \n",
       "0                          0.981419  \n",
       "1                          0.907095  \n",
       "2                          0.660473  \n",
       "4                          0.550676  \n",
       "3                          0.552365  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df, all_dataset_names = make_df(template_id, \"../data/v1/by_sub_dataset_with_answers/selection-text\", models)\n",
    "df[['model', 'total']+[f\"{x}_total\" for x in all_dataset_names]].sort_values('total', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb3b01f8-9467-4896-ba23-1856271cef61",
   "metadata": {},
   "source": [
    "# Selection-images Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "632f89f1-cd59-4523-9c9f-47c17bb55432",
   "metadata": {},
   "source": [
    "For details about this dataset, check the [Readme.](../../README.md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7c4f8569-c3ab-447c-900b-b97dfc523a4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We are using the following models:\n",
    "models = [\n",
    "            anthropic_config(\"n/a\", \"claude-3-opus-20240229\", token_limit, True, 1),\n",
    "            gpt_config(\"n/a\", token_limit, \"gpt-4-1106-vision-preview\", True, 1),\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "86926f0a-7ba4-4584-9b79-acb7de16f4d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>total</th>\n",
       "      <th>selection_image_50_rev23_thousands_no_hint_total</th>\n",
       "      <th>selection_image_50_rev_meta_total</th>\n",
       "      <th>selection_image_50_rev23_millions_no_hint_total</th>\n",
       "      <th>selection_image_50_rev23_thousands_hint_total</th>\n",
       "      <th>selection_image_50_rev22_millions_no_hint_total</th>\n",
       "      <th>selection_image_50_rev23_meta_total</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt-4-1106-vision-preview</td>\n",
       "      <td>0.540968</td>\n",
       "      <td>0.52</td>\n",
       "      <td>0.435811</td>\n",
       "      <td>0.68</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>claude-3-opus-20240229</td>\n",
       "      <td>0.446926</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.741554</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.42</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.68</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       model     total  \\\n",
       "1  gpt-4-1106-vision-preview  0.540968   \n",
       "0     claude-3-opus-20240229  0.446926   \n",
       "\n",
       "   selection_image_50_rev23_thousands_no_hint_total  \\\n",
       "1                                              0.52   \n",
       "0                                              0.42   \n",
       "\n",
       "   selection_image_50_rev_meta_total  \\\n",
       "1                           0.435811   \n",
       "0                           0.741554   \n",
       "\n",
       "   selection_image_50_rev23_millions_no_hint_total  \\\n",
       "1                                             0.68   \n",
       "0                                             0.30   \n",
       "\n",
       "   selection_image_50_rev23_thousands_hint_total  \\\n",
       "1                                           0.56   \n",
       "0                                           0.42   \n",
       "\n",
       "   selection_image_50_rev22_millions_no_hint_total  \\\n",
       "1                                             0.20   \n",
       "0                                             0.12   \n",
       "\n",
       "   selection_image_50_rev23_meta_total  \n",
       "1                                 0.85  \n",
       "0                                 0.68  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df, all_dataset_names = make_df(template_id, \"../data/v1/by_sub_dataset_with_answers/selection-image\", models)\n",
    "df[['model', 'total']+[f\"{x}_total\" for x in all_dataset_names]].sort_values('total', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8da1a814-980d-4e25-8466-dd8301b8e1c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c43699ff-c552-43bf-8693-3ab278e6eb5d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}