Chatbot-Arena.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e50c43c3-2d53-4e9f-a5a7-9877b3e89e58",
   "metadata": {},
   "source": [
    "# Chatbot Arena with Evalica"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90e1f336-7aa4-4e2b-aaa7-101264e6c38c",
   "metadata": {},
   "source": [
    "[![Open in Colab][colab_badge]][colab_link] [![Binder][binder_badge]][binder_link]\n",
    "\n",
    "[colab_badge]: https://colab.research.google.com/assets/colab-badge.svg\n",
    "[colab_link]: https://colab.research.google.com/github/dustalov/evalica/blob/master/Chatbot-Arena.ipynb\n",
    "[binder_badge]: https://mybinder.org/badge_logo.svg\n",
    "[binder_link]: https://mybinder.org/v2/gh/dustalov/evalica/HEAD?labpath=Chatbot-Arena.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e289f1c-b2ac-42ae-8584-74680a745da2",
   "metadata": {},
   "source": [
    "We follow the LMSYS' [Chatbot Arena: MLE Elo Rating](https://colab.research.google.com/drive/1KdwokPjirkTmpO_P1WByFNFiqxWQquwH) notebook and implement a similar leaderboard with the [Evalica](https://github.com/dustalov/evalica) library that efficiently implements pairwise comparison aggregation routines in Rust."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "095fd582-04ad-4641-b4a5-bc6bbbb30030",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c01b1c9c-67a7-4615-92cf-4dd9afc1aaec",
   "metadata": {},
   "outputs": [],
   "source": [
    "!curl -LOC - 'https://storage.googleapis.com/arena_external_data/public/clean_battle_20240814_public.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a5b3bc7-5505-4859-81cd-edb7c0b98726",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import annotations  # noqa: F404\n",
    "\n",
    "from typing import TYPE_CHECKING\n",
    "\n",
    "import evalica\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "from tqdm.auto import trange\n",
    "\n",
    "if TYPE_CHECKING:\n",
    "    from plotly.graph_objects import Figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2f20ba6-f62e-4c1c-80ad-bedef88b6c3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "evalica.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb17ce44-c001-42f0-9b3b-1b7394e2e886",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_arena = pd.read_json(\"clean_battle_20240629_public.json\")\n",
    "df_arena = df_arena[df_arena[\"anony\"]]\n",
    "df_arena = df_arena[df_arena[\"dedup_tag\"].apply(lambda x: x.get(\"sampled\", False))]\n",
    "df_arena[\"winner\"] = df_arena[\"winner\"].map({\n",
    "    \"model_a\": evalica.Winner.X,\n",
    "    \"model_b\": evalica.Winner.Y,\n",
    "    \"tie\": evalica.Winner.Draw,\n",
    "    \"tie (bothbad)\": evalica.Winner.Draw,\n",
    "})\n",
    "df_arena = df_arena[~df_arena[\"winner\"].isna()]\n",
    "df_arena"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab6c5d2e-dc1f-46d9-859c-1f285e5af28d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_arena_no_ties = df_arena[df_arena[\"winner\"] != evalica.Winner.Draw]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfde8c25-562a-4f5d-b6ce-08850b5e4e8e",
   "metadata": {},
   "source": [
    "## Pairwise Win Fractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48a14783-1df7-49bc-aec0-1c5bee15ecd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "average_win_rates = evalica.average_win_rate(\n",
    "    df_arena[\"model_a\"],\n",
    "    df_arena[\"model_b\"],\n",
    "    df_arena[\"winner\"],\n",
    ")\n",
    "\n",
    "average_win_rates.scores.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2289b947-f462-45fc-ac51-270b50d0389b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "average_win_rates_no_ties = evalica.average_win_rate(\n",
    "    df_arena[\"model_a\"],\n",
    "    df_arena[\"model_b\"],\n",
    "    df_arena[\"winner\"],\n",
    "    tie_weight=0,  # LMSYS' leaderboard excludes ties\n",
    ")\n",
    "\n",
    "average_win_scores_no_ties = average_win_rates_no_ties.scores\n",
    "average_win_scores_no_ties.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42bdb614-0791-4db5-be97-c9efe71100bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize(df_pairwise: pd.DataFrame, title: str | None = None) -> Figure:\n",
    "    fig = px.imshow(df_pairwise, color_continuous_scale=\"RdBu\", text_auto=\".2f\")\n",
    "\n",
    "    fig.update_layout(\n",
    "        title=title,\n",
    "        title_x=0.5,\n",
    "        title_y=0.075,\n",
    "        xaxis_title=\"Loser\",\n",
    "        yaxis_title=\"Winner\",\n",
    "        xaxis_side=\"top\",\n",
    "        width=800,\n",
    "        height=640,\n",
    "    )\n",
    "\n",
    "    fig.update_traces(hovertemplate=\"Winner: %{y}<br>Loser: %{x}<br>Fraction of Wins: %{z}\")\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f7584fa-f1b6-4ef4-918c-9760a4cfbed2",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "xs_indexed, ys_indexes, index = evalica.indexing(df_arena[\"model_a\"], df_arena[\"model_b\"])\n",
    "\n",
    "matrices = evalica.matrices(\n",
    "    xs_indexed,\n",
    "    ys_indexes,\n",
    "    df_arena[\"winner\"],\n",
    "    index,\n",
    ")\n",
    "\n",
    "df_matrix = pd.DataFrame(\n",
    "    matrices.win_matrix,\n",
    "    index=index,\n",
    "    columns=index,\n",
    ")\n",
    "\n",
    "visualize(df_matrix.loc[\n",
    "          average_win_scores_no_ties.index[:15],\n",
    "          average_win_scores_no_ties.index[:15],  # type: ignore[index]\n",
    "], title=\"Win Counts\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20dc534a-a578-437f-bb1d-234a9dbe542d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_matrix_proba = (df_matrix / (df_matrix + df_matrix.T))\n",
    "df_matrix_proba = df_matrix_proba.loc[\n",
    "    average_win_scores_no_ties.index[:15],\n",
    "    average_win_scores_no_ties.index[:15],  # type: ignore[index]\n",
    "]\n",
    "\n",
    "visualize(df_matrix_proba, title=\"Win Fractions\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea01e9c5-e15b-4f67-aa6c-aef0844d69d8",
   "metadata": {},
   "source": [
    "## Elo Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5723e41c-0a68-41fe-aa8e-a7d344f3e69e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "elo = evalica.elo(\n",
    "    df_arena[\"model_a\"],\n",
    "    df_arena[\"model_b\"],\n",
    "    df_arena[\"winner\"],\n",
    ")\n",
    "\n",
    "elo.scores.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "045df4f5-b58b-43db-b67d-227f89d8e2f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_elo = evalica.pairwise_frame(elo.scores[:15])\n",
    "\n",
    "visualize(df_elo, title=\"Elo Win Probabilities\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfc3e2e8-5c6e-430b-a735-1126eca8181c",
   "metadata": {},
   "source": [
    "## Bradley&ndash;Terry Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fe410b8-c361-4647-b70f-512da8b53923",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "bt = evalica.bradley_terry(\n",
    "    df_arena[\"model_a\"],\n",
    "    df_arena[\"model_b\"],\n",
    "    df_arena[\"winner\"],\n",
    ")\n",
    "\n",
    "bt.scores.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17581ea4-2a76-4980-982c-29a40d39e0b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bt = evalica.pairwise_frame(bt.scores[:15])\n",
    "\n",
    "visualize(df_bt, title=\"Bradley–Terry Win Probabilities\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89c7da06-a276-4aa3-b5b5-aee4c891c8df",
   "metadata": {},
   "source": [
    "## Bradley&ndash;Terry Bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41976e8d-8809-48fd-bcc0-15de056d63a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "BOOTSTRAP_ROUNDS = 10\n",
    "\n",
    "bt_bootstrap = []\n",
    "\n",
    "for seed in trange(BOOTSTRAP_ROUNDS, desc=\"Bootstrap\"):\n",
    "    df_sample = df_arena.sample(frac=1.0, replace=True, random_state=seed)\n",
    "\n",
    "    result = evalica.bradley_terry(\n",
    "        df_sample[\"model_a\"],\n",
    "        df_sample[\"model_b\"],\n",
    "        df_sample[\"winner\"],\n",
    "        index=index,  # we safely save some time by not reindexing the elements\n",
    "    )\n",
    "\n",
    "    bt_bootstrap.append(result.scores)\n",
    "\n",
    "df_bootstrap = pd.DataFrame(bt_bootstrap)\n",
    "df_bootstrap = df_bootstrap[df_bootstrap.median().index]\n",
    "\n",
    "df_bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1829b9ef-6f2e-42c2-beec-209fc3d7fa12",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bootstrap.median().to_frame(name=\"bradley_terry\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c93db286-fbe5-443e-8508-2faf75963d0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bootstrap_ci = pd.DataFrame({\n",
    "    \"lower\": df_bootstrap.quantile(.025),\n",
    "    \"rating\": df_bootstrap.quantile(.5),\n",
    "    \"upper\": df_bootstrap.quantile(.975),\n",
    "}).reset_index(names=\"model\").sort_values(\"rating\", ascending=False)\n",
    "\n",
    "df_bootstrap_ci[\"error_y\"] = df_bootstrap_ci[\"upper\"] - df_bootstrap_ci[\"rating\"]\n",
    "df_bootstrap_ci[\"error_y_minus\"] = df_bootstrap_ci[\"rating\"] - df_bootstrap_ci[\"lower\"]\n",
    "df_bootstrap_ci[\"rating_rounded\"] = np.round(df_bootstrap_ci[\"rating\"], 2)\n",
    "\n",
    "df_bootstrap_ci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f30a3110-b143-4d05-b66c-66fdb2791a8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_ci(df_ci: pd.DataFrame, title: str | None = None) -> Figure:\n",
    "    fig = px.scatter(df_ci, x=\"model\", y=\"rating\", error_y=\"error_y\", error_y_minus=\"error_y_minus\", title=title)\n",
    "\n",
    "    fig.update_layout(xaxis_title=\"Model\", yaxis_title=\"Score\", width=800, height=640, title_x=.5)\n",
    "\n",
    "    fig.update_traces(hovertemplate=\"Model: %{x}<br>Score: %{y}\")\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "070f2811-4a55-4f2b-b5d0-8793e0cea97b",
   "metadata": {},
   "outputs": [],
   "source": [
    "visualize_ci(df_bootstrap_ci.head(30), \"Bootstrapped Confidence Intervals for Bradley–Terry Scores\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25574423-3831-4dbc-a167-8ded59a7769a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}