Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_37258_row0_col0 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row0_col1, #T_37258_row0_col2 {\n",
       "  background-color: #e2edf8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row0_col3 {\n",
       "  background-color: #dceaf6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row0_col4 {\n",
       "  background-color: #d5e5f4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row0_col5, #T_37258_row3_col3, #T_37258_row5_col3 {\n",
       "  background-color: #ccdff1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row0_col6 {\n",
       "  background-color: #000000;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_37258_row1_col0, #T_37258_row4_col0 {\n",
       "  background-color: #d9e8f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row1_col1, #T_37258_row4_col5, #T_37258_row9_col2 {\n",
       "  background-color: #b0d2e7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row1_col2, #T_37258_row1_col6 {\n",
       "  background-color: #bad6eb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row1_col3 {\n",
       "  background-color: #b7d4ea;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row1_col4, #T_37258_row8_col3 {\n",
       "  background-color: #aacfe5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row1_col5, #T_37258_row6_col3 {\n",
       "  background-color: #95c5df;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row2_col0 {\n",
       "  background-color: #dae8f6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row2_col1, #T_37258_row2_col3, #T_37258_row5_col5 {\n",
       "  background-color: #bed8ec;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row2_col2, #T_37258_row5_col1, #T_37258_row7_col0 {\n",
       "  background-color: #cbdef1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row2_col4, #T_37258_row4_col3 {\n",
       "  background-color: #bcd7eb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row2_col5, #T_37258_row8_col1 {\n",
       "  background-color: #a9cfe5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row2_col6, #T_37258_row4_col1 {\n",
       "  background-color: #c8dcf0;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row3_col0 {\n",
       "  background-color: #dfecf7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row3_col1 {\n",
       "  background-color: #cfe1f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row3_col2 {\n",
       "  background-color: #d2e3f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row3_col4 {\n",
       "  background-color: #cee0f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row3_col5 {\n",
       "  background-color: #c1d9ed;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row3_col6 {\n",
       "  background-color: #d9e7f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row4_col2, #T_37258_row5_col4 {\n",
       "  background-color: #c6dbef;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row4_col4 {\n",
       "  background-color: #b9d6ea;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row4_col6 {\n",
       "  background-color: #c4daee;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row5_col0 {\n",
       "  background-color: #e6f0f9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row5_col2, #T_37258_row9_col0 {\n",
       "  background-color: #d3e3f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row5_col6 {\n",
       "  background-color: #e5eff9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row6_col0 {\n",
       "  background-color: #cddff1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row6_col1 {\n",
       "  background-color: #9cc9e1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row6_col2, #T_37258_row7_col2 {\n",
       "  background-color: #a0cbe2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row6_col4 {\n",
       "  background-color: #91c3de;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row6_col5 {\n",
       "  background-color: #7cb7da;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row6_col6 {\n",
       "  background-color: #8fc2de;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row7_col1, #T_37258_row7_col3 {\n",
       "  background-color: #94c4df;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row7_col4, #T_37258_row8_col5 {\n",
       "  background-color: #81badb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row7_col5 {\n",
       "  background-color: #6aaed6;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_37258_row7_col6 {\n",
       "  background-color: #82bbdb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row8_col0 {\n",
       "  background-color: #d0e2f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row8_col2 {\n",
       "  background-color: #a6cee4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row8_col4 {\n",
       "  background-color: #9dcae1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row8_col6 {\n",
       "  background-color: #add0e6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row9_col1, #T_37258_row9_col3 {\n",
       "  background-color: #b5d4e9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row9_col4 {\n",
       "  background-color: #a4cce3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row9_col5 {\n",
       "  background-color: #a8cee4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_37258_row9_col6 {\n",
       "  background-color: #9fcae1;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_37258\">\n",
       "  <caption>coverage_score</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_37258_level0_col0\" class=\"col_heading level0 col0\" >random</th>\n",
       "      <th id=\"T_37258_level0_col1\" class=\"col_heading level0 col1\" >longembed</th>\n",
       "      <th id=\"T_37258_level0_col2\" class=\"col_heading level0 col2\" >vector</th>\n",
       "      <th id=\"T_37258_level0_col3\" class=\"col_heading level0 col3\" >keywords</th>\n",
       "      <th id=\"T_37258_level0_col4\" class=\"col_heading level0 col4\" >rerank3</th>\n",
       "      <th id=\"T_37258_level0_col5\" class=\"col_heading level0 col5\" >oracle</th>\n",
       "      <th id=\"T_37258_level0_col6\" class=\"col_heading level0 col6\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_37258_row0_col0\" class=\"data row0 col0\" >0.36</td>\n",
       "      <td id=\"T_37258_row0_col1\" class=\"data row0 col1\" >0.46</td>\n",
       "      <td id=\"T_37258_row0_col2\" class=\"data row0 col2\" >0.46</td>\n",
       "      <td id=\"T_37258_row0_col3\" class=\"data row0 col3\" >0.48</td>\n",
       "      <td id=\"T_37258_row0_col4\" class=\"data row0 col4\" >0.52</td>\n",
       "      <td id=\"T_37258_row0_col5\" class=\"data row0 col5\" >0.56</td>\n",
       "      <td id=\"T_37258_row0_col6\" class=\"data row0 col6\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_37258_row1_col0\" class=\"data row1 col0\" >0.50</td>\n",
       "      <td id=\"T_37258_row1_col1\" class=\"data row1 col1\" >0.65</td>\n",
       "      <td id=\"T_37258_row1_col2\" class=\"data row1 col2\" >0.62</td>\n",
       "      <td id=\"T_37258_row1_col3\" class=\"data row1 col3\" >0.63</td>\n",
       "      <td id=\"T_37258_row1_col4\" class=\"data row1 col4\" >0.67</td>\n",
       "      <td id=\"T_37258_row1_col5\" class=\"data row1 col5\" >0.72</td>\n",
       "      <td id=\"T_37258_row1_col6\" class=\"data row1 col6\" >0.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_37258_row2_col0\" class=\"data row2 col0\" >0.49</td>\n",
       "      <td id=\"T_37258_row2_col1\" class=\"data row2 col1\" >0.61</td>\n",
       "      <td id=\"T_37258_row2_col2\" class=\"data row2 col2\" >0.57</td>\n",
       "      <td id=\"T_37258_row2_col3\" class=\"data row2 col3\" >0.61</td>\n",
       "      <td id=\"T_37258_row2_col4\" class=\"data row2 col4\" >0.62</td>\n",
       "      <td id=\"T_37258_row2_col5\" class=\"data row2 col5\" >0.67</td>\n",
       "      <td id=\"T_37258_row2_col6\" class=\"data row2 col6\" >0.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row3\" class=\"row_heading level0 row3\" >command-r</th>\n",
       "      <td id=\"T_37258_row3_col0\" class=\"data row3 col0\" >0.47</td>\n",
       "      <td id=\"T_37258_row3_col1\" class=\"data row3 col1\" >0.55</td>\n",
       "      <td id=\"T_37258_row3_col2\" class=\"data row3 col2\" >0.53</td>\n",
       "      <td id=\"T_37258_row3_col3\" class=\"data row3 col3\" >0.56</td>\n",
       "      <td id=\"T_37258_row3_col4\" class=\"data row3 col4\" >0.55</td>\n",
       "      <td id=\"T_37258_row3_col5\" class=\"data row3 col5\" >0.60</td>\n",
       "      <td id=\"T_37258_row3_col6\" class=\"data row3 col6\" >0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_37258_row4_col0\" class=\"data row4 col0\" >0.50</td>\n",
       "      <td id=\"T_37258_row4_col1\" class=\"data row4 col1\" >0.58</td>\n",
       "      <td id=\"T_37258_row4_col2\" class=\"data row4 col2\" >0.59</td>\n",
       "      <td id=\"T_37258_row4_col3\" class=\"data row4 col3\" >0.62</td>\n",
       "      <td id=\"T_37258_row4_col4\" class=\"data row4 col4\" >0.63</td>\n",
       "      <td id=\"T_37258_row4_col5\" class=\"data row4 col5\" >0.65</td>\n",
       "      <td id=\"T_37258_row4_col6\" class=\"data row4 col6\" >0.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row5\" class=\"row_heading level0 row5\" >command-r-plus</th>\n",
       "      <td id=\"T_37258_row5_col0\" class=\"data row5 col0\" >0.44</td>\n",
       "      <td id=\"T_37258_row5_col1\" class=\"data row5 col1\" >0.56</td>\n",
       "      <td id=\"T_37258_row5_col2\" class=\"data row5 col2\" >0.53</td>\n",
       "      <td id=\"T_37258_row5_col3\" class=\"data row5 col3\" >0.56</td>\n",
       "      <td id=\"T_37258_row5_col4\" class=\"data row5 col4\" >0.59</td>\n",
       "      <td id=\"T_37258_row5_col5\" class=\"data row5 col5\" >0.61</td>\n",
       "      <td id=\"T_37258_row5_col6\" class=\"data row5 col6\" >0.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row6\" class=\"row_heading level0 row6\" >claude3-sonnet</th>\n",
       "      <td id=\"T_37258_row6_col0\" class=\"data row6 col0\" >0.56</td>\n",
       "      <td id=\"T_37258_row6_col1\" class=\"data row6 col1\" >0.71</td>\n",
       "      <td id=\"T_37258_row6_col2\" class=\"data row6 col2\" >0.70</td>\n",
       "      <td id=\"T_37258_row6_col3\" class=\"data row6 col3\" >0.72</td>\n",
       "      <td id=\"T_37258_row6_col4\" class=\"data row6 col4\" >0.73</td>\n",
       "      <td id=\"T_37258_row6_col5\" class=\"data row6 col5\" >0.78</td>\n",
       "      <td id=\"T_37258_row6_col6\" class=\"data row6 col6\" >0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row7\" class=\"row_heading level0 row7\" >claude3-opus</th>\n",
       "      <td id=\"T_37258_row7_col0\" class=\"data row7 col0\" >0.57</td>\n",
       "      <td id=\"T_37258_row7_col1\" class=\"data row7 col1\" >0.72</td>\n",
       "      <td id=\"T_37258_row7_col2\" class=\"data row7 col2\" >0.70</td>\n",
       "      <td id=\"T_37258_row7_col3\" class=\"data row7 col3\" >0.73</td>\n",
       "      <td id=\"T_37258_row7_col4\" class=\"data row7 col4\" >0.76</td>\n",
       "      <td id=\"T_37258_row7_col5\" class=\"data row7 col5\" >0.81</td>\n",
       "      <td id=\"T_37258_row7_col6\" class=\"data row7 col6\" >0.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row8\" class=\"row_heading level0 row8\" >gpt-4o</th>\n",
       "      <td id=\"T_37258_row8_col0\" class=\"data row8 col0\" >0.54</td>\n",
       "      <td id=\"T_37258_row8_col1\" class=\"data row8 col1\" >0.67</td>\n",
       "      <td id=\"T_37258_row8_col2\" class=\"data row8 col2\" >0.68</td>\n",
       "      <td id=\"T_37258_row8_col3\" class=\"data row8 col3\" >0.67</td>\n",
       "      <td id=\"T_37258_row8_col4\" class=\"data row8 col4\" >0.70</td>\n",
       "      <td id=\"T_37258_row8_col5\" class=\"data row8 col5\" >0.77</td>\n",
       "      <td id=\"T_37258_row8_col6\" class=\"data row8 col6\" >0.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_37258_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_37258_row9_col0\" class=\"data row9 col0\" >0.53</td>\n",
       "      <td id=\"T_37258_row9_col1\" class=\"data row9 col1\" >0.64</td>\n",
       "      <td id=\"T_37258_row9_col2\" class=\"data row9 col2\" >0.65</td>\n",
       "      <td id=\"T_37258_row9_col3\" class=\"data row9 col3\" >0.64</td>\n",
       "      <td id=\"T_37258_row9_col4\" class=\"data row9 col4\" >0.68</td>\n",
       "      <td id=\"T_37258_row9_col5\" class=\"data row9 col5\" >0.68</td>\n",
       "      <td id=\"T_37258_row9_col6\" class=\"data row9 col6\" >0.70</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n",
       "<style type=\"text/css\">\n",
       "#T_5907e_row0_col0 {\n",
       "  background-color: #f1f7fd;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row0_col1 {\n",
       "  background-color: #e7f0fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row0_col2 {\n",
       "  background-color: #e7f1fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row0_col3 {\n",
       "  background-color: #e6f0f9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row0_col4 {\n",
       "  background-color: #e4eff9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row0_col5 {\n",
       "  background-color: #d9e8f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row0_col6 {\n",
       "  background-color: #000000;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_5907e_row1_col0 {\n",
       "  background-color: #eaf2fb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row1_col1, #T_5907e_row1_col2 {\n",
       "  background-color: #d6e5f4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row1_col3 {\n",
       "  background-color: #d3e4f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row1_col4, #T_5907e_row2_col1 {\n",
       "  background-color: #d0e2f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row1_col5 {\n",
       "  background-color: #c4daee;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row1_col6 {\n",
       "  background-color: #e9f2fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row2_col0, #T_5907e_row6_col0 {\n",
       "  background-color: #e2edf8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row2_col2 {\n",
       "  background-color: #d1e2f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row2_col3, #T_5907e_row3_col6 {\n",
       "  background-color: #cddff1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row2_col4, #T_5907e_row4_col1, #T_5907e_row4_col2 {\n",
       "  background-color: #cbdef1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row2_col5, #T_5907e_row8_col4 {\n",
       "  background-color: #b4d3e9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row2_col6 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row3_col0, #T_5907e_row4_col0, #T_5907e_row7_col0 {\n",
       "  background-color: #e3eef8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row3_col1, #T_5907e_row7_col1 {\n",
       "  background-color: #c7dbef;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row3_col2, #T_5907e_row6_col1 {\n",
       "  background-color: #c6dbef;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row3_col3, #T_5907e_row6_col3, #T_5907e_row7_col3 {\n",
       "  background-color: #bfd8ed;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row3_col4 {\n",
       "  background-color: #b7d4ea;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row3_col5 {\n",
       "  background-color: #8fc2de;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row4_col3 {\n",
       "  background-color: #c7dcef;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row4_col4 {\n",
       "  background-color: #aed1e7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row4_col5 {\n",
       "  background-color: #97c6df;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row4_col6 {\n",
       "  background-color: #caddf0;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row5_col0 {\n",
       "  background-color: #deebf7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row5_col1, #T_5907e_row5_col2 {\n",
       "  background-color: #b3d3e8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row5_col3, #T_5907e_row9_col1 {\n",
       "  background-color: #afd1e7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row5_col4 {\n",
       "  background-color: #a5cde3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row5_col5 {\n",
       "  background-color: #79b5d9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row5_col6 {\n",
       "  background-color: #dfebf7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row6_col2 {\n",
       "  background-color: #c1d9ed;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row6_col4 {\n",
       "  background-color: #b5d4e9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row6_col5 {\n",
       "  background-color: #95c5df;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row6_col6 {\n",
       "  background-color: #d9e7f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row7_col2 {\n",
       "  background-color: #c3daee;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row7_col4 {\n",
       "  background-color: #b9d6ea;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row7_col5 {\n",
       "  background-color: #9ac8e0;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row7_col6 {\n",
       "  background-color: #dbe9f6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row8_col0, #T_5907e_row9_col0 {\n",
       "  background-color: #dce9f6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row8_col1, #T_5907e_row8_col3 {\n",
       "  background-color: #bcd7eb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row8_col2 {\n",
       "  background-color: #bdd7ec;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row8_col5 {\n",
       "  background-color: #8cc0dd;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row8_col6 {\n",
       "  background-color: #e5eff9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row9_col2 {\n",
       "  background-color: #abd0e6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row9_col3 {\n",
       "  background-color: #a6cee4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row9_col4 {\n",
       "  background-color: #9dcae1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_5907e_row9_col5 {\n",
       "  background-color: #6aaed6;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_5907e_row9_col6 {\n",
       "  background-color: #99c7e0;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_5907e\">\n",
       "  <caption>citation_score</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_5907e_level0_col0\" class=\"col_heading level0 col0\" >random</th>\n",
       "      <th id=\"T_5907e_level0_col1\" class=\"col_heading level0 col1\" >longembed</th>\n",
       "      <th id=\"T_5907e_level0_col2\" class=\"col_heading level0 col2\" >vector</th>\n",
       "      <th id=\"T_5907e_level0_col3\" class=\"col_heading level0 col3\" >keywords</th>\n",
       "      <th id=\"T_5907e_level0_col4\" class=\"col_heading level0 col4\" >rerank3</th>\n",
       "      <th id=\"T_5907e_level0_col5\" class=\"col_heading level0 col5\" >oracle</th>\n",
       "      <th id=\"T_5907e_level0_col6\" class=\"col_heading level0 col6\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_5907e_row0_col0\" class=\"data row0 col0\" >0.09</td>\n",
       "      <td id=\"T_5907e_row0_col1\" class=\"data row0 col1\" >0.15</td>\n",
       "      <td id=\"T_5907e_row0_col2\" class=\"data row0 col2\" >0.15</td>\n",
       "      <td id=\"T_5907e_row0_col3\" class=\"data row0 col3\" >0.16</td>\n",
       "      <td id=\"T_5907e_row0_col4\" class=\"data row0 col4\" >0.17</td>\n",
       "      <td id=\"T_5907e_row0_col5\" class=\"data row0 col5\" >0.23</td>\n",
       "      <td id=\"T_5907e_row0_col6\" class=\"data row0 col6\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_5907e_row1_col0\" class=\"data row1 col0\" >0.13</td>\n",
       "      <td id=\"T_5907e_row1_col1\" class=\"data row1 col1\" >0.25</td>\n",
       "      <td id=\"T_5907e_row1_col2\" class=\"data row1 col2\" >0.26</td>\n",
       "      <td id=\"T_5907e_row1_col3\" class=\"data row1 col3\" >0.27</td>\n",
       "      <td id=\"T_5907e_row1_col4\" class=\"data row1 col4\" >0.29</td>\n",
       "      <td id=\"T_5907e_row1_col5\" class=\"data row1 col5\" >0.36</td>\n",
       "      <td id=\"T_5907e_row1_col6\" class=\"data row1 col6\" >0.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_5907e_row2_col0\" class=\"data row2 col0\" >0.18</td>\n",
       "      <td id=\"T_5907e_row2_col1\" class=\"data row2 col1\" >0.29</td>\n",
       "      <td id=\"T_5907e_row2_col2\" class=\"data row2 col2\" >0.28</td>\n",
       "      <td id=\"T_5907e_row2_col3\" class=\"data row2 col3\" >0.31</td>\n",
       "      <td id=\"T_5907e_row2_col4\" class=\"data row2 col4\" >0.32</td>\n",
       "      <td id=\"T_5907e_row2_col5\" class=\"data row2 col5\" >0.41</td>\n",
       "      <td id=\"T_5907e_row2_col6\" class=\"data row2 col6\" >0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row3\" class=\"row_heading level0 row3\" >command-r</th>\n",
       "      <td id=\"T_5907e_row3_col0\" class=\"data row3 col0\" >0.18</td>\n",
       "      <td id=\"T_5907e_row3_col1\" class=\"data row3 col1\" >0.35</td>\n",
       "      <td id=\"T_5907e_row3_col2\" class=\"data row3 col2\" >0.35</td>\n",
       "      <td id=\"T_5907e_row3_col3\" class=\"data row3 col3\" >0.37</td>\n",
       "      <td id=\"T_5907e_row3_col4\" class=\"data row3 col4\" >0.40</td>\n",
       "      <td id=\"T_5907e_row3_col5\" class=\"data row3 col5\" >0.54</td>\n",
       "      <td id=\"T_5907e_row3_col6\" class=\"data row3 col6\" >0.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_5907e_row4_col0\" class=\"data row4 col0\" >0.17</td>\n",
       "      <td id=\"T_5907e_row4_col1\" class=\"data row4 col1\" >0.32</td>\n",
       "      <td id=\"T_5907e_row4_col2\" class=\"data row4 col2\" >0.32</td>\n",
       "      <td id=\"T_5907e_row4_col3\" class=\"data row4 col3\" >0.34</td>\n",
       "      <td id=\"T_5907e_row4_col4\" class=\"data row4 col4\" >0.44</td>\n",
       "      <td id=\"T_5907e_row4_col5\" class=\"data row4 col5\" >0.52</td>\n",
       "      <td id=\"T_5907e_row4_col6\" class=\"data row4 col6\" >0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row5\" class=\"row_heading level0 row5\" >command-r-plus</th>\n",
       "      <td id=\"T_5907e_row5_col0\" class=\"data row5 col0\" >0.20</td>\n",
       "      <td id=\"T_5907e_row5_col1\" class=\"data row5 col1\" >0.42</td>\n",
       "      <td id=\"T_5907e_row5_col2\" class=\"data row5 col2\" >0.42</td>\n",
       "      <td id=\"T_5907e_row5_col3\" class=\"data row5 col3\" >0.43</td>\n",
       "      <td id=\"T_5907e_row5_col4\" class=\"data row5 col4\" >0.47</td>\n",
       "      <td id=\"T_5907e_row5_col5\" class=\"data row5 col5\" >0.60</td>\n",
       "      <td id=\"T_5907e_row5_col6\" class=\"data row5 col6\" >0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row6\" class=\"row_heading level0 row6\" >claude3-sonnet</th>\n",
       "      <td id=\"T_5907e_row6_col0\" class=\"data row6 col0\" >0.18</td>\n",
       "      <td id=\"T_5907e_row6_col1\" class=\"data row6 col1\" >0.35</td>\n",
       "      <td id=\"T_5907e_row6_col2\" class=\"data row6 col2\" >0.37</td>\n",
       "      <td id=\"T_5907e_row6_col3\" class=\"data row6 col3\" >0.37</td>\n",
       "      <td id=\"T_5907e_row6_col4\" class=\"data row6 col4\" >0.41</td>\n",
       "      <td id=\"T_5907e_row6_col5\" class=\"data row6 col5\" >0.52</td>\n",
       "      <td id=\"T_5907e_row6_col6\" class=\"data row6 col6\" >0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row7\" class=\"row_heading level0 row7\" >claude3-opus</th>\n",
       "      <td id=\"T_5907e_row7_col0\" class=\"data row7 col0\" >0.18</td>\n",
       "      <td id=\"T_5907e_row7_col1\" class=\"data row7 col1\" >0.34</td>\n",
       "      <td id=\"T_5907e_row7_col2\" class=\"data row7 col2\" >0.36</td>\n",
       "      <td id=\"T_5907e_row7_col3\" class=\"data row7 col3\" >0.37</td>\n",
       "      <td id=\"T_5907e_row7_col4\" class=\"data row7 col4\" >0.39</td>\n",
       "      <td id=\"T_5907e_row7_col5\" class=\"data row7 col5\" >0.51</td>\n",
       "      <td id=\"T_5907e_row7_col6\" class=\"data row7 col6\" >0.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row8\" class=\"row_heading level0 row8\" >gpt-4o</th>\n",
       "      <td id=\"T_5907e_row8_col0\" class=\"data row8 col0\" >0.22</td>\n",
       "      <td id=\"T_5907e_row8_col1\" class=\"data row8 col1\" >0.38</td>\n",
       "      <td id=\"T_5907e_row8_col2\" class=\"data row8 col2\" >0.38</td>\n",
       "      <td id=\"T_5907e_row8_col3\" class=\"data row8 col3\" >0.39</td>\n",
       "      <td id=\"T_5907e_row8_col4\" class=\"data row8 col4\" >0.41</td>\n",
       "      <td id=\"T_5907e_row8_col5\" class=\"data row8 col5\" >0.55</td>\n",
       "      <td id=\"T_5907e_row8_col6\" class=\"data row8 col6\" >0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_5907e_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_5907e_row9_col0\" class=\"data row9 col0\" >0.22</td>\n",
       "      <td id=\"T_5907e_row9_col1\" class=\"data row9 col1\" >0.43</td>\n",
       "      <td id=\"T_5907e_row9_col2\" class=\"data row9 col2\" >0.45</td>\n",
       "      <td id=\"T_5907e_row9_col3\" class=\"data row9 col3\" >0.47</td>\n",
       "      <td id=\"T_5907e_row9_col4\" class=\"data row9 col4\" >0.50</td>\n",
       "      <td id=\"T_5907e_row9_col5\" class=\"data row9 col5\" >0.64</td>\n",
       "      <td id=\"T_5907e_row9_col6\" class=\"data row9 col6\" >0.51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n",
       "<style type=\"text/css\">\n",
       "#T_4cfe3_row0_col0 {\n",
       "  background-color: #f6faff;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row0_col1, #T_4cfe3_row0_col2, #T_4cfe3_row1_col0 {\n",
       "  background-color: #eef5fc;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row0_col3 {\n",
       "  background-color: #ecf4fb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row0_col4, #T_4cfe3_row3_col0 {\n",
       "  background-color: #eaf2fb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row0_col5 {\n",
       "  background-color: #dfecf7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row0_col6 {\n",
       "  background-color: #000000;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_4cfe3_row1_col1, #T_4cfe3_row1_col2 {\n",
       "  background-color: #d6e5f4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row1_col3 {\n",
       "  background-color: #d5e5f4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row1_col4, #T_4cfe3_row2_col3 {\n",
       "  background-color: #cfe1f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row1_col5, #T_4cfe3_row7_col2 {\n",
       "  background-color: #bcd7eb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row1_col6, #T_4cfe3_row4_col0 {\n",
       "  background-color: #e9f2fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row2_col0, #T_4cfe3_row5_col0, #T_4cfe3_row5_col6 {\n",
       "  background-color: #e8f1fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row2_col1 {\n",
       "  background-color: #d3e3f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row2_col2 {\n",
       "  background-color: #d6e6f4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row2_col4 {\n",
       "  background-color: #cee0f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row2_col5 {\n",
       "  background-color: #b3d3e8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row2_col6 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row3_col1, #T_4cfe3_row3_col2 {\n",
       "  background-color: #d0e2f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row3_col3, #T_4cfe3_row4_col3 {\n",
       "  background-color: #cbdef1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row3_col4 {\n",
       "  background-color: #c7dcef;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row3_col5 {\n",
       "  background-color: #a0cbe2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row3_col6 {\n",
       "  background-color: #d9e7f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row4_col1 {\n",
       "  background-color: #d1e2f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row4_col2 {\n",
       "  background-color: #d0e1f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row4_col4, #T_4cfe3_row6_col3, #T_4cfe3_row7_col3, #T_4cfe3_row9_col1 {\n",
       "  background-color: #b4d3e9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row4_col5 {\n",
       "  background-color: #9cc9e1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row4_col6 {\n",
       "  background-color: #cddff1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row5_col1 {\n",
       "  background-color: #c3daee;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row5_col2 {\n",
       "  background-color: #c6dbef;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row5_col3 {\n",
       "  background-color: #bfd8ed;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row5_col4 {\n",
       "  background-color: #b2d2e8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row5_col5 {\n",
       "  background-color: #8abfdd;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row6_col0 {\n",
       "  background-color: #e5eff9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row6_col1 {\n",
       "  background-color: #bed8ec;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row6_col2, #T_4cfe3_row8_col1, #T_4cfe3_row8_col3 {\n",
       "  background-color: #b9d6ea;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row6_col4 {\n",
       "  background-color: #a9cfe5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row6_col5 {\n",
       "  background-color: #7cb7da;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row6_col6 {\n",
       "  background-color: #d3e4f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row7_col0 {\n",
       "  background-color: #e4eff9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row7_col1 {\n",
       "  background-color: #bdd7ec;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row7_col4 {\n",
       "  background-color: #a8cee4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row7_col5 {\n",
       "  background-color: #75b4d8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row7_col6 {\n",
       "  background-color: #d4e4f4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row8_col0, #T_4cfe3_row9_col0 {\n",
       "  background-color: #e1edf8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row8_col2 {\n",
       "  background-color: #b8d5ea;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row8_col4, #T_4cfe3_row9_col2, #T_4cfe3_row9_col3 {\n",
       "  background-color: #abd0e6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row8_col5 {\n",
       "  background-color: #71b1d7;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_4cfe3_row8_col6 {\n",
       "  background-color: #e3eef9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row9_col4 {\n",
       "  background-color: #95c5df;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4cfe3_row9_col5 {\n",
       "  background-color: #6aaed6;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_4cfe3_row9_col6 {\n",
       "  background-color: #8cc0dd;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_4cfe3\">\n",
       "  <caption>joint_score</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_4cfe3_level0_col0\" class=\"col_heading level0 col0\" >random</th>\n",
       "      <th id=\"T_4cfe3_level0_col1\" class=\"col_heading level0 col1\" >longembed</th>\n",
       "      <th id=\"T_4cfe3_level0_col2\" class=\"col_heading level0 col2\" >vector</th>\n",
       "      <th id=\"T_4cfe3_level0_col3\" class=\"col_heading level0 col3\" >keywords</th>\n",
       "      <th id=\"T_4cfe3_level0_col4\" class=\"col_heading level0 col4\" >rerank3</th>\n",
       "      <th id=\"T_4cfe3_level0_col5\" class=\"col_heading level0 col5\" >oracle</th>\n",
       "      <th id=\"T_4cfe3_level0_col6\" class=\"col_heading level0 col6\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_4cfe3_row0_col0\" class=\"data row0 col0\" >0.04</td>\n",
       "      <td id=\"T_4cfe3_row0_col1\" class=\"data row0 col1\" >0.07</td>\n",
       "      <td id=\"T_4cfe3_row0_col2\" class=\"data row0 col2\" >0.07</td>\n",
       "      <td id=\"T_4cfe3_row0_col3\" class=\"data row0 col3\" >0.08</td>\n",
       "      <td id=\"T_4cfe3_row0_col4\" class=\"data row0 col4\" >0.09</td>\n",
       "      <td id=\"T_4cfe3_row0_col5\" class=\"data row0 col5\" >0.13</td>\n",
       "      <td id=\"T_4cfe3_row0_col6\" class=\"data row0 col6\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_4cfe3_row1_col0\" class=\"data row1 col0\" >0.07</td>\n",
       "      <td id=\"T_4cfe3_row1_col1\" class=\"data row1 col1\" >0.17</td>\n",
       "      <td id=\"T_4cfe3_row1_col2\" class=\"data row1 col2\" >0.17</td>\n",
       "      <td id=\"T_4cfe3_row1_col3\" class=\"data row1 col3\" >0.18</td>\n",
       "      <td id=\"T_4cfe3_row1_col4\" class=\"data row1 col4\" >0.20</td>\n",
       "      <td id=\"T_4cfe3_row1_col5\" class=\"data row1 col5\" >0.27</td>\n",
       "      <td id=\"T_4cfe3_row1_col6\" class=\"data row1 col6\" >0.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_4cfe3_row2_col0\" class=\"data row2 col0\" >0.10</td>\n",
       "      <td id=\"T_4cfe3_row2_col1\" class=\"data row2 col1\" >0.19</td>\n",
       "      <td id=\"T_4cfe3_row2_col2\" class=\"data row2 col2\" >0.17</td>\n",
       "      <td id=\"T_4cfe3_row2_col3\" class=\"data row2 col3\" >0.20</td>\n",
       "      <td id=\"T_4cfe3_row2_col4\" class=\"data row2 col4\" >0.21</td>\n",
       "      <td id=\"T_4cfe3_row2_col5\" class=\"data row2 col5\" >0.29</td>\n",
       "      <td id=\"T_4cfe3_row2_col6\" class=\"data row2 col6\" >0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row3\" class=\"row_heading level0 row3\" >command-r</th>\n",
       "      <td id=\"T_4cfe3_row3_col0\" class=\"data row3 col0\" >0.09</td>\n",
       "      <td id=\"T_4cfe3_row3_col1\" class=\"data row3 col1\" >0.20</td>\n",
       "      <td id=\"T_4cfe3_row3_col2\" class=\"data row3 col2\" >0.20</td>\n",
       "      <td id=\"T_4cfe3_row3_col3\" class=\"data row3 col3\" >0.22</td>\n",
       "      <td id=\"T_4cfe3_row3_col4\" class=\"data row3 col4\" >0.24</td>\n",
       "      <td id=\"T_4cfe3_row3_col5\" class=\"data row3 col5\" >0.34</td>\n",
       "      <td id=\"T_4cfe3_row3_col6\" class=\"data row3 col6\" >0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_4cfe3_row4_col0\" class=\"data row4 col0\" >0.09</td>\n",
       "      <td id=\"T_4cfe3_row4_col1\" class=\"data row4 col1\" >0.19</td>\n",
       "      <td id=\"T_4cfe3_row4_col2\" class=\"data row4 col2\" >0.20</td>\n",
       "      <td id=\"T_4cfe3_row4_col3\" class=\"data row4 col3\" >0.22</td>\n",
       "      <td id=\"T_4cfe3_row4_col4\" class=\"data row4 col4\" >0.29</td>\n",
       "      <td id=\"T_4cfe3_row4_col5\" class=\"data row4 col5\" >0.35</td>\n",
       "      <td id=\"T_4cfe3_row4_col6\" class=\"data row4 col6\" >0.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row5\" class=\"row_heading level0 row5\" >command-r-plus</th>\n",
       "      <td id=\"T_4cfe3_row5_col0\" class=\"data row5 col0\" >0.10</td>\n",
       "      <td id=\"T_4cfe3_row5_col1\" class=\"data row5 col1\" >0.25</td>\n",
       "      <td id=\"T_4cfe3_row5_col2\" class=\"data row5 col2\" >0.24</td>\n",
       "      <td id=\"T_4cfe3_row5_col3\" class=\"data row5 col3\" >0.26</td>\n",
       "      <td id=\"T_4cfe3_row5_col4\" class=\"data row5 col4\" >0.29</td>\n",
       "      <td id=\"T_4cfe3_row5_col5\" class=\"data row5 col5\" >0.38</td>\n",
       "      <td id=\"T_4cfe3_row5_col6\" class=\"data row5 col6\" >0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row6\" class=\"row_heading level0 row6\" >claude3-sonnet</th>\n",
       "      <td id=\"T_4cfe3_row6_col0\" class=\"data row6 col0\" >0.11</td>\n",
       "      <td id=\"T_4cfe3_row6_col1\" class=\"data row6 col1\" >0.26</td>\n",
       "      <td id=\"T_4cfe3_row6_col2\" class=\"data row6 col2\" >0.27</td>\n",
       "      <td id=\"T_4cfe3_row6_col3\" class=\"data row6 col3\" >0.28</td>\n",
       "      <td id=\"T_4cfe3_row6_col4\" class=\"data row6 col4\" >0.31</td>\n",
       "      <td id=\"T_4cfe3_row6_col5\" class=\"data row6 col5\" >0.41</td>\n",
       "      <td id=\"T_4cfe3_row6_col6\" class=\"data row6 col6\" >0.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row7\" class=\"row_heading level0 row7\" >claude3-opus</th>\n",
       "      <td id=\"T_4cfe3_row7_col0\" class=\"data row7 col0\" >0.11</td>\n",
       "      <td id=\"T_4cfe3_row7_col1\" class=\"data row7 col1\" >0.27</td>\n",
       "      <td id=\"T_4cfe3_row7_col2\" class=\"data row7 col2\" >0.27</td>\n",
       "      <td id=\"T_4cfe3_row7_col3\" class=\"data row7 col3\" >0.29</td>\n",
       "      <td id=\"T_4cfe3_row7_col4\" class=\"data row7 col4\" >0.32</td>\n",
       "      <td id=\"T_4cfe3_row7_col5\" class=\"data row7 col5\" >0.43</td>\n",
       "      <td id=\"T_4cfe3_row7_col6\" class=\"data row7 col6\" >0.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row8\" class=\"row_heading level0 row8\" >gpt-4o</th>\n",
       "      <td id=\"T_4cfe3_row8_col0\" class=\"data row8 col0\" >0.13</td>\n",
       "      <td id=\"T_4cfe3_row8_col1\" class=\"data row8 col1\" >0.27</td>\n",
       "      <td id=\"T_4cfe3_row8_col2\" class=\"data row8 col2\" >0.28</td>\n",
       "      <td id=\"T_4cfe3_row8_col3\" class=\"data row8 col3\" >0.27</td>\n",
       "      <td id=\"T_4cfe3_row8_col4\" class=\"data row8 col4\" >0.31</td>\n",
       "      <td id=\"T_4cfe3_row8_col5\" class=\"data row8 col5\" >0.43</td>\n",
       "      <td id=\"T_4cfe3_row8_col6\" class=\"data row8 col6\" >0.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4cfe3_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_4cfe3_row9_col0\" class=\"data row9 col0\" >0.12</td>\n",
       "      <td id=\"T_4cfe3_row9_col1\" class=\"data row9 col1\" >0.29</td>\n",
       "      <td id=\"T_4cfe3_row9_col2\" class=\"data row9 col2\" >0.31</td>\n",
       "      <td id=\"T_4cfe3_row9_col3\" class=\"data row9 col3\" >0.31</td>\n",
       "      <td id=\"T_4cfe3_row9_col4\" class=\"data row9 col4\" >0.36</td>\n",
       "      <td id=\"T_4cfe3_row9_col5\" class=\"data row9 col5\" >0.45</td>\n",
       "      <td id=\"T_4cfe3_row9_col6\" class=\"data row9 col6\" >0.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n",
       "<style type=\"text/css\">\n",
       "#T_99ad2_row0_col0, #T_99ad2_row0_col1, #T_99ad2_row0_col2, #T_99ad2_row0_col3, #T_99ad2_row0_col4, #T_99ad2_row0_col5, #T_99ad2_row0_col6, #T_99ad2_row1_col0, #T_99ad2_row1_col1, #T_99ad2_row1_col2, #T_99ad2_row1_col3, #T_99ad2_row1_col4, #T_99ad2_row1_col5, #T_99ad2_row1_col6, #T_99ad2_row2_col0, #T_99ad2_row2_col1, #T_99ad2_row2_col2, #T_99ad2_row2_col3, #T_99ad2_row2_col4, #T_99ad2_row2_col5, #T_99ad2_row2_col6, #T_99ad2_row3_col0, #T_99ad2_row3_col1, #T_99ad2_row3_col2, #T_99ad2_row3_col3, #T_99ad2_row3_col4, #T_99ad2_row3_col5, #T_99ad2_row3_col6, #T_99ad2_row4_col0, #T_99ad2_row4_col1, #T_99ad2_row4_col2, #T_99ad2_row4_col3, #T_99ad2_row4_col4, #T_99ad2_row4_col5, #T_99ad2_row4_col6, #T_99ad2_row5_col0, #T_99ad2_row5_col1, #T_99ad2_row5_col2, #T_99ad2_row5_col3, #T_99ad2_row5_col4, #T_99ad2_row5_col5, #T_99ad2_row5_col6, #T_99ad2_row6_col0, #T_99ad2_row6_col1, #T_99ad2_row6_col2, #T_99ad2_row6_col3, #T_99ad2_row6_col4, #T_99ad2_row6_col5, #T_99ad2_row6_col6, #T_99ad2_row7_col0, #T_99ad2_row7_col1, #T_99ad2_row7_col2, #T_99ad2_row7_col3, #T_99ad2_row7_col4, #T_99ad2_row7_col5, #T_99ad2_row7_col6, #T_99ad2_row8_col0, #T_99ad2_row8_col1, #T_99ad2_row8_col2, #T_99ad2_row8_col3, #T_99ad2_row8_col4, #T_99ad2_row8_col5, #T_99ad2_row8_col6, #T_99ad2_row9_col0, #T_99ad2_row9_col1, #T_99ad2_row9_col2, #T_99ad2_row9_col3, #T_99ad2_row9_col4, #T_99ad2_row9_col5, #T_99ad2_row9_col6 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_99ad2\">\n",
       "  <caption>count</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_99ad2_level0_col0\" class=\"col_heading level0 col0\" >random</th>\n",
       "      <th id=\"T_99ad2_level0_col1\" class=\"col_heading level0 col1\" >longembed</th>\n",
       "      <th id=\"T_99ad2_level0_col2\" class=\"col_heading level0 col2\" >vector</th>\n",
       "      <th id=\"T_99ad2_level0_col3\" class=\"col_heading level0 col3\" >keywords</th>\n",
       "      <th id=\"T_99ad2_level0_col4\" class=\"col_heading level0 col4\" >rerank3</th>\n",
       "      <th id=\"T_99ad2_level0_col5\" class=\"col_heading level0 col5\" >oracle</th>\n",
       "      <th id=\"T_99ad2_level0_col6\" class=\"col_heading level0 col6\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_99ad2_row0_col0\" class=\"data row0 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row0_col1\" class=\"data row0 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row0_col2\" class=\"data row0 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row0_col3\" class=\"data row0 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row0_col4\" class=\"data row0 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row0_col5\" class=\"data row0 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row0_col6\" class=\"data row0 col6\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_99ad2_row1_col0\" class=\"data row1 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row1_col1\" class=\"data row1 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row1_col2\" class=\"data row1 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row1_col3\" class=\"data row1 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row1_col4\" class=\"data row1 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row1_col5\" class=\"data row1 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row1_col6\" class=\"data row1 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_99ad2_row2_col0\" class=\"data row2 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row2_col1\" class=\"data row2 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row2_col2\" class=\"data row2 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row2_col3\" class=\"data row2 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row2_col4\" class=\"data row2 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row2_col5\" class=\"data row2 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row2_col6\" class=\"data row2 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row3\" class=\"row_heading level0 row3\" >command-r</th>\n",
       "      <td id=\"T_99ad2_row3_col0\" class=\"data row3 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row3_col1\" class=\"data row3 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row3_col2\" class=\"data row3 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row3_col3\" class=\"data row3 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row3_col4\" class=\"data row3 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row3_col5\" class=\"data row3 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row3_col6\" class=\"data row3 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_99ad2_row4_col0\" class=\"data row4 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row4_col1\" class=\"data row4 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row4_col2\" class=\"data row4 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row4_col3\" class=\"data row4 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row4_col4\" class=\"data row4 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row4_col5\" class=\"data row4 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row4_col6\" class=\"data row4 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row5\" class=\"row_heading level0 row5\" >command-r-plus</th>\n",
       "      <td id=\"T_99ad2_row5_col0\" class=\"data row5 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row5_col1\" class=\"data row5 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row5_col2\" class=\"data row5 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row5_col3\" class=\"data row5 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row5_col4\" class=\"data row5 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row5_col5\" class=\"data row5 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row5_col6\" class=\"data row5 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row6\" class=\"row_heading level0 row6\" >claude3-sonnet</th>\n",
       "      <td id=\"T_99ad2_row6_col0\" class=\"data row6 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row6_col1\" class=\"data row6 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row6_col2\" class=\"data row6 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row6_col3\" class=\"data row6 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row6_col4\" class=\"data row6 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row6_col5\" class=\"data row6 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row6_col6\" class=\"data row6 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row7\" class=\"row_heading level0 row7\" >claude3-opus</th>\n",
       "      <td id=\"T_99ad2_row7_col0\" class=\"data row7 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row7_col1\" class=\"data row7 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row7_col2\" class=\"data row7 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row7_col3\" class=\"data row7 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row7_col4\" class=\"data row7 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row7_col5\" class=\"data row7 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row7_col6\" class=\"data row7 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row8\" class=\"row_heading level0 row8\" >gpt-4o</th>\n",
       "      <td id=\"T_99ad2_row8_col0\" class=\"data row8 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row8_col1\" class=\"data row8 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row8_col2\" class=\"data row8 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row8_col3\" class=\"data row8 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row8_col4\" class=\"data row8 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row8_col5\" class=\"data row8 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row8_col6\" class=\"data row8 col6\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_99ad2_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_99ad2_row9_col0\" class=\"data row9 col0\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row9_col1\" class=\"data row9 col1\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row9_col2\" class=\"data row9 col2\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row9_col3\" class=\"data row9 col3\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row9_col4\" class=\"data row9 col4\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row9_col5\" class=\"data row9 col5\" >621.00</td>\n",
       "      <td id=\"T_99ad2_row9_col6\" class=\"data row9 col6\" >621.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from utils_eval import compute_2d_rag_results\n",
    "from utils_misc import display_2d_results\n",
    "\n",
    "topic_fns = [f\"data/topic_conv{i}.json\" for i in range(1, 6)]\n",
    "topic_fns += [f\"data/topic_news{i}.json\" for i in range(1, 6)]\n",
    "display_2d_results(compute_2d_rag_results(topic_fns, sort_key=\"joint_score\", sort_retriever_key=\"oracle\", retrivers_skip=[\"fl-ctxt-sort\", \"fl-ctxt-rev-sort\", \"fl-ctxt-middle\"]))\n",
    "\n",
    "## Note: you can also use the following code to load the data from the huggingface datasets library and pass in the data directly\n",
    "# from datasets import load_dataset\n",
    "# data = load_dataset(\"Salesforce/sumemary-of-a-haystack\")['train']\n",
    "# display_2d_results(compute_2d_rag_results(data, sort_key=\"joint_score\", sort_retriever_key=\"oracle\", retrivers_skip=[\"fl-ctxt-sort\", \"fl-ctxt-rev-sort\", \"fl-ctxt-middle\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Precision & Recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_4ad5b_row0_col0 {\n",
       "  background-color: #c2d9ee;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row0_col1 {\n",
       "  background-color: #000000;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_4ad5b_row1_col0 {\n",
       "  background-color: #bcd7eb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row1_col1 {\n",
       "  background-color: #e7f0fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row2_col0 {\n",
       "  background-color: #99c7e0;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row2_col1 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row3_col0 {\n",
       "  background-color: #8cc0dd;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row3_col1 {\n",
       "  background-color: #dceaf6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row4_col0 {\n",
       "  background-color: #a5cde3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row4_col1 {\n",
       "  background-color: #d1e2f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row5_col0 {\n",
       "  background-color: #89bedc;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row5_col1 {\n",
       "  background-color: #cadef0;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row6_col0, #T_4ad5b_row9_col1 {\n",
       "  background-color: #a3cce3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row6_col1 {\n",
       "  background-color: #d0e1f2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row7_col0 {\n",
       "  background-color: #8dc1dd;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row7_col1 {\n",
       "  background-color: #e1edf8;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row8_col0 {\n",
       "  background-color: #87bddc;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row8_col1 {\n",
       "  background-color: #e7f1fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_4ad5b_row9_col0 {\n",
       "  background-color: #6aaed6;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_4ad5b\">\n",
       "  <caption>citation_precision</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_4ad5b_level0_col0\" class=\"col_heading level0 col0\" >oracle</th>\n",
       "      <th id=\"T_4ad5b_level0_col1\" class=\"col_heading level0 col1\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_4ad5b_row0_col0\" class=\"data row0 col0\" >0.47</td>\n",
       "      <td id=\"T_4ad5b_row0_col1\" class=\"data row0 col1\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_4ad5b_row1_col0\" class=\"data row1 col0\" >0.49</td>\n",
       "      <td id=\"T_4ad5b_row1_col1\" class=\"data row1 col1\" >0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_4ad5b_row2_col0\" class=\"data row2 col0\" >0.62</td>\n",
       "      <td id=\"T_4ad5b_row2_col1\" class=\"data row2 col1\" >0.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row3\" class=\"row_heading level0 row3\" >claude3-opus</th>\n",
       "      <td id=\"T_4ad5b_row3_col0\" class=\"data row3 col0\" >0.66</td>\n",
       "      <td id=\"T_4ad5b_row3_col1\" class=\"data row3 col1\" >0.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_4ad5b_row4_col0\" class=\"data row4 col0\" >0.58</td>\n",
       "      <td id=\"T_4ad5b_row4_col1\" class=\"data row4 col1\" >0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row5\" class=\"row_heading level0 row5\" >claude3-sonnet</th>\n",
       "      <td id=\"T_4ad5b_row5_col0\" class=\"data row5 col0\" >0.67</td>\n",
       "      <td id=\"T_4ad5b_row5_col1\" class=\"data row5 col1\" >0.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row6\" class=\"row_heading level0 row6\" >command-r</th>\n",
       "      <td id=\"T_4ad5b_row6_col0\" class=\"data row6 col0\" >0.59</td>\n",
       "      <td id=\"T_4ad5b_row6_col1\" class=\"data row6 col1\" >0.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row7\" class=\"row_heading level0 row7\" >gpt-4o</th>\n",
       "      <td id=\"T_4ad5b_row7_col0\" class=\"data row7 col0\" >0.66</td>\n",
       "      <td id=\"T_4ad5b_row7_col1\" class=\"data row7 col1\" >0.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row8\" class=\"row_heading level0 row8\" >command-r-plus</th>\n",
       "      <td id=\"T_4ad5b_row8_col0\" class=\"data row8 col0\" >0.68</td>\n",
       "      <td id=\"T_4ad5b_row8_col1\" class=\"data row8 col1\" >0.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_4ad5b_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_4ad5b_row9_col0\" class=\"data row9 col0\" >0.76</td>\n",
       "      <td id=\"T_4ad5b_row9_col1\" class=\"data row9 col1\" >0.59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n",
       "<style type=\"text/css\">\n",
       "#T_fbb74_row0_col0 {\n",
       "  background-color: #dfebf7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row0_col1 {\n",
       "  background-color: #000000;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_fbb74_row1_col0, #T_fbb74_row6_col1 {\n",
       "  background-color: #c7dcef;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row1_col1, #T_fbb74_row3_col1 {\n",
       "  background-color: #d4e4f4;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row2_col0 {\n",
       "  background-color: #bdd7ec;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row2_col1 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row3_col0 {\n",
       "  background-color: #a0cbe2;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row4_col0 {\n",
       "  background-color: #82bbdb;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row4_col1 {\n",
       "  background-color: #a5cde3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row5_col0 {\n",
       "  background-color: #9cc9e1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row5_col1 {\n",
       "  background-color: #dbe9f6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row6_col0 {\n",
       "  background-color: #7db8da;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row7_col0 {\n",
       "  background-color: #8fc2de;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row7_col1 {\n",
       "  background-color: #e7f1fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row8_col0 {\n",
       "  background-color: #6caed6;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_fbb74_row8_col1 {\n",
       "  background-color: #d9e7f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_fbb74_row9_col0 {\n",
       "  background-color: #6aaed6;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_fbb74_row9_col1 {\n",
       "  background-color: #89bedc;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_fbb74\">\n",
       "  <caption>citation_recall</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_fbb74_level0_col0\" class=\"col_heading level0 col0\" >oracle</th>\n",
       "      <th id=\"T_fbb74_level0_col1\" class=\"col_heading level0 col1\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_fbb74_row0_col0\" class=\"data row0 col0\" >0.18</td>\n",
       "      <td id=\"T_fbb74_row0_col1\" class=\"data row0 col1\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_fbb74_row1_col0\" class=\"data row1 col0\" >0.32</td>\n",
       "      <td id=\"T_fbb74_row1_col1\" class=\"data row1 col1\" >0.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_fbb74_row2_col0\" class=\"data row2 col0\" >0.36</td>\n",
       "      <td id=\"T_fbb74_row2_col1\" class=\"data row2 col1\" >0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row3\" class=\"row_heading level0 row3\" >claude3-opus</th>\n",
       "      <td id=\"T_fbb74_row3_col0\" class=\"data row3 col0\" >0.46</td>\n",
       "      <td id=\"T_fbb74_row3_col1\" class=\"data row3 col1\" >0.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_fbb74_row4_col0\" class=\"data row4 col0\" >0.54</td>\n",
       "      <td id=\"T_fbb74_row4_col1\" class=\"data row4 col1\" >0.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row5\" class=\"row_heading level0 row5\" >claude3-sonnet</th>\n",
       "      <td id=\"T_fbb74_row5_col0\" class=\"data row5 col0\" >0.47</td>\n",
       "      <td id=\"T_fbb74_row5_col1\" class=\"data row5 col1\" >0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row6\" class=\"row_heading level0 row6\" >command-r</th>\n",
       "      <td id=\"T_fbb74_row6_col0\" class=\"data row6 col0\" >0.56</td>\n",
       "      <td id=\"T_fbb74_row6_col1\" class=\"data row6 col1\" >0.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row7\" class=\"row_heading level0 row7\" >gpt-4o</th>\n",
       "      <td id=\"T_fbb74_row7_col0\" class=\"data row7 col0\" >0.51</td>\n",
       "      <td id=\"T_fbb74_row7_col1\" class=\"data row7 col1\" >0.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row8\" class=\"row_heading level0 row8\" >command-r-plus</th>\n",
       "      <td id=\"T_fbb74_row8_col0\" class=\"data row8 col0\" >0.61</td>\n",
       "      <td id=\"T_fbb74_row8_col1\" class=\"data row8 col1\" >0.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_fbb74_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_fbb74_row9_col0\" class=\"data row9 col0\" >0.61</td>\n",
       "      <td id=\"T_fbb74_row9_col1\" class=\"data row9 col1\" >0.52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n",
       "<style type=\"text/css\">\n",
       "#T_9caab_row0_col0 {\n",
       "  background-color: #d9e8f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row0_col1 {\n",
       "  background-color: #000000;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_9caab_row1_col0 {\n",
       "  background-color: #c4daee;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row1_col1 {\n",
       "  background-color: #e9f2fa;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row2_col0 {\n",
       "  background-color: #b4d3e9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row2_col1 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row3_col0 {\n",
       "  background-color: #9ac8e0;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row3_col1 {\n",
       "  background-color: #dbe9f6;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row4_col0 {\n",
       "  background-color: #97c6df;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row4_col1 {\n",
       "  background-color: #caddf0;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row5_col0 {\n",
       "  background-color: #95c5df;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row5_col1 {\n",
       "  background-color: #d9e7f5;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row6_col0 {\n",
       "  background-color: #8fc2de;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row6_col1 {\n",
       "  background-color: #cddff1;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row7_col0 {\n",
       "  background-color: #8cc0dd;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row7_col1 {\n",
       "  background-color: #e5eff9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row8_col0 {\n",
       "  background-color: #79b5d9;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row8_col1 {\n",
       "  background-color: #dfebf7;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_9caab_row9_col0 {\n",
       "  background-color: #6aaed6;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_9caab_row9_col1 {\n",
       "  background-color: #99c7e0;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_9caab\">\n",
       "  <caption>citation_score</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_9caab_level0_col0\" class=\"col_heading level0 col0\" >oracle</th>\n",
       "      <th id=\"T_9caab_level0_col1\" class=\"col_heading level0 col1\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_9caab_row0_col0\" class=\"data row0 col0\" >0.23</td>\n",
       "      <td id=\"T_9caab_row0_col1\" class=\"data row0 col1\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_9caab_row1_col0\" class=\"data row1 col0\" >0.36</td>\n",
       "      <td id=\"T_9caab_row1_col1\" class=\"data row1 col1\" >0.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_9caab_row2_col0\" class=\"data row2 col0\" >0.41</td>\n",
       "      <td id=\"T_9caab_row2_col1\" class=\"data row2 col1\" >0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row3\" class=\"row_heading level0 row3\" >claude3-opus</th>\n",
       "      <td id=\"T_9caab_row3_col0\" class=\"data row3 col0\" >0.51</td>\n",
       "      <td id=\"T_9caab_row3_col1\" class=\"data row3 col1\" >0.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_9caab_row4_col0\" class=\"data row4 col0\" >0.52</td>\n",
       "      <td id=\"T_9caab_row4_col1\" class=\"data row4 col1\" >0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row5\" class=\"row_heading level0 row5\" >claude3-sonnet</th>\n",
       "      <td id=\"T_9caab_row5_col0\" class=\"data row5 col0\" >0.52</td>\n",
       "      <td id=\"T_9caab_row5_col1\" class=\"data row5 col1\" >0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row6\" class=\"row_heading level0 row6\" >command-r</th>\n",
       "      <td id=\"T_9caab_row6_col0\" class=\"data row6 col0\" >0.54</td>\n",
       "      <td id=\"T_9caab_row6_col1\" class=\"data row6 col1\" >0.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row7\" class=\"row_heading level0 row7\" >gpt-4o</th>\n",
       "      <td id=\"T_9caab_row7_col0\" class=\"data row7 col0\" >0.55</td>\n",
       "      <td id=\"T_9caab_row7_col1\" class=\"data row7 col1\" >0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row8\" class=\"row_heading level0 row8\" >command-r-plus</th>\n",
       "      <td id=\"T_9caab_row8_col0\" class=\"data row8 col0\" >0.60</td>\n",
       "      <td id=\"T_9caab_row8_col1\" class=\"data row8 col1\" >0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9caab_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_9caab_row9_col0\" class=\"data row9 col0\" >0.64</td>\n",
       "      <td id=\"T_9caab_row9_col1\" class=\"data row9 col1\" >0.51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n",
       "<style type=\"text/css\">\n",
       "#T_653cc_row0_col0, #T_653cc_row0_col1, #T_653cc_row1_col0, #T_653cc_row1_col1, #T_653cc_row2_col0, #T_653cc_row2_col1, #T_653cc_row3_col0, #T_653cc_row3_col1, #T_653cc_row4_col0, #T_653cc_row4_col1, #T_653cc_row5_col0, #T_653cc_row5_col1, #T_653cc_row6_col0, #T_653cc_row6_col1, #T_653cc_row7_col0, #T_653cc_row7_col1, #T_653cc_row8_col0, #T_653cc_row8_col1, #T_653cc_row9_col0, #T_653cc_row9_col1 {\n",
       "  background-color: #f7fbff;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table style=\"display:inline\" id=\"T_653cc\">\n",
       "  <caption>count</caption>\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_653cc_level0_col0\" class=\"col_heading level0 col0\" >oracle</th>\n",
       "      <th id=\"T_653cc_level0_col1\" class=\"col_heading level0 col1\" >full-ctxt</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >Summarizer</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row0\" class=\"row_heading level0 row0\" >gpt3.5</th>\n",
       "      <td id=\"T_653cc_row0_col0\" class=\"data row0 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row0_col1\" class=\"data row0 col1\" >nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row1\" class=\"row_heading level0 row1\" >claude3-haiku</th>\n",
       "      <td id=\"T_653cc_row1_col0\" class=\"data row1 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row1_col1\" class=\"data row1 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row2\" class=\"row_heading level0 row2\" >gpt4-turbo</th>\n",
       "      <td id=\"T_653cc_row2_col0\" class=\"data row2 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row2_col1\" class=\"data row2 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row3\" class=\"row_heading level0 row3\" >claude3-opus</th>\n",
       "      <td id=\"T_653cc_row3_col0\" class=\"data row3 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row3_col1\" class=\"data row3 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row4\" class=\"row_heading level0 row4\" >gemini-1.5-flash</th>\n",
       "      <td id=\"T_653cc_row4_col0\" class=\"data row4 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row4_col1\" class=\"data row4 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row5\" class=\"row_heading level0 row5\" >claude3-sonnet</th>\n",
       "      <td id=\"T_653cc_row5_col0\" class=\"data row5 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row5_col1\" class=\"data row5 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row6\" class=\"row_heading level0 row6\" >command-r</th>\n",
       "      <td id=\"T_653cc_row6_col0\" class=\"data row6 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row6_col1\" class=\"data row6 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row7\" class=\"row_heading level0 row7\" >gpt-4o</th>\n",
       "      <td id=\"T_653cc_row7_col0\" class=\"data row7 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row7_col1\" class=\"data row7 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row8\" class=\"row_heading level0 row8\" >command-r-plus</th>\n",
       "      <td id=\"T_653cc_row8_col0\" class=\"data row8 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row8_col1\" class=\"data row8 col1\" >621.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_653cc_level0_row9\" class=\"row_heading level0 row9\" >gemini-1.5-pro</th>\n",
       "      <td id=\"T_653cc_row9_col0\" class=\"data row9 col0\" >621.00</td>\n",
       "      <td id=\"T_653cc_row9_col1\" class=\"data row9 col1\" >621.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table style=\"display:inline\">\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from utils_eval import compute_2d_rag_results\n",
    "from utils_misc import display_2d_results\n",
    "from IPython.display import display\n",
    "\n",
    "topic_fns = [f\"data/topic_conv{i}.json\" for i in range(1, 6)]\n",
    "topic_fns += [f\"data/topic_news{i}.json\" for i in range(1, 6)]\n",
    "\n",
    "display_2d_results(compute_2d_rag_results(topic_fns, score_keys=[\"citation_precision\", \"citation_recall\", \"citation_score\"], sort_key=\"citation_score\", retrivers_skip=[\"fl-ctxt-sort\", \"random\", \"fl-ctxt-rev-sort\", \"vector\", \"longembed\", \"keywords\", \"rerank3\", \"fl-ctxt-middle\"], sort_retriever_key=\"oracle\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Computing Full-Context Sorting Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>txt-middle</th>\n",
       "      <th>-ctxt-sort</th>\n",
       "      <th>t-rev-sort</th>\n",
       "      <th>oracle</th>\n",
       "      <th>full-ctxt</th>\n",
       "      <th>Sort Sensitivity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Summarizer</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>claude3-opus</th>\n",
       "      <td>0.234648</td>\n",
       "      <td>0.204072</td>\n",
       "      <td>0.279877</td>\n",
       "      <td>0.425347</td>\n",
       "      <td>0.179861</td>\n",
       "      <td>0.100016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-4o</th>\n",
       "      <td>0.113129</td>\n",
       "      <td>0.137765</td>\n",
       "      <td>0.240761</td>\n",
       "      <td>0.433662</td>\n",
       "      <td>0.114111</td>\n",
       "      <td>0.126650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gemini-1.5-pro</th>\n",
       "      <td>0.388409</td>\n",
       "      <td>0.470490</td>\n",
       "      <td>0.389118</td>\n",
       "      <td>0.446387</td>\n",
       "      <td>0.378486</td>\n",
       "      <td>0.092004</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                txt-middle  -ctxt-sort  t-rev-sort    oracle  full-ctxt  \\\n",
       "Summarizer                                                                \n",
       "claude3-opus      0.234648    0.204072    0.279877  0.425347   0.179861   \n",
       "gpt-4o            0.113129    0.137765    0.240761  0.433662   0.114111   \n",
       "gemini-1.5-pro    0.388409    0.470490    0.389118  0.446387   0.378486   \n",
       "\n",
       "                Sort Sensitivity  \n",
       "Summarizer                        \n",
       "claude3-opus            0.100016  \n",
       "gpt-4o                  0.126650  \n",
       "gemini-1.5-pro          0.092004  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from utils_eval import compute_2d_rag_results\n",
    "from utils_misc import display_2d_results\n",
    "import pandas as pd\n",
    "\n",
    "topic_fns = [f\"data/topic_conv{i}.json\" for i in range(1, 6)]\n",
    "topic_fns += [f\"data/topic_news{i}.json\" for i in range(1, 6)]\n",
    "\n",
    "results_sorting = compute_2d_rag_results(topic_fns, sort_key=\"joint_score\", sort_retriever_key=\"oracle\", retrivers_skip=[\"random\", \"longembed\", \"vector\", \"keywords\", \"rerank3\"], summarizer_skip=[\"gpt3.5\", \"gemini-1.5-flash\", \"claude3-haiku\", \"gpt4-turbo\", \"command-r\", \"command-r-plus\", \"claude3-sonnet\"])\n",
    "results_sorting_join = results_sorting[\"joint_score\"]\n",
    "for row in results_sorting_join:\n",
    "    row[\"Sort Sensitivity\"] = max(abs(row[\"-ctxt-sort\"] - row[\"full-ctxt\"]), abs(row[\"t-rev-sort\"] - row[\"full-ctxt\"]), abs(row[\"txt-middle\"] - row[\"full-ctxt\"]))\n",
    "\n",
    "display(pd.DataFrame(results_sorting_join).set_index(\"Summarizer\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}