notebook/StackFAQ/09.Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required libraries\n",
    "import sys\n",
    "sys.path.insert(0, '../../../BERT-FAQ/')\n",
    "\n",
    "from evaluation import Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_results_filepath=\"../../../BERT-FAQ/data/StackFAQ/rank_results\"\n",
    "\n",
    "ev = Evaluation()\n",
    "df = ev.get_eval_df(rank_results_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Matching Field</th>\n",
       "      <th>Ranking Field</th>\n",
       "      <th>Loss</th>\n",
       "      <th>Training Data</th>\n",
       "      <th>Negative Sampling</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "      <th>P@3</th>\n",
       "      <th>P@5</th>\n",
       "      <th>P@10</th>\n",
       "      <th>MAP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>answer</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>question</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6410</td>\n",
       "      <td>0.3990</td>\n",
       "      <td>0.6634</td>\n",
       "      <td>0.3183</td>\n",
       "      <td>0.6732</td>\n",
       "      <td>0.2175</td>\n",
       "      <td>0.4459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6073</td>\n",
       "      <td>0.5890</td>\n",
       "      <td>0.6216</td>\n",
       "      <td>0.5267</td>\n",
       "      <td>0.6498</td>\n",
       "      <td>0.3606</td>\n",
       "      <td>0.6607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7190</td>\n",
       "      <td>0.5548</td>\n",
       "      <td>0.7421</td>\n",
       "      <td>0.4884</td>\n",
       "      <td>0.7540</td>\n",
       "      <td>0.3430</td>\n",
       "      <td>0.6119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7873</td>\n",
       "      <td>0.6408</td>\n",
       "      <td>0.8057</td>\n",
       "      <td>0.5544</td>\n",
       "      <td>0.8086</td>\n",
       "      <td>0.3818</td>\n",
       "      <td>0.6703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6527</td>\n",
       "      <td>0.4073</td>\n",
       "      <td>0.6733</td>\n",
       "      <td>0.3267</td>\n",
       "      <td>0.6801</td>\n",
       "      <td>0.2204</td>\n",
       "      <td>0.4545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6056</td>\n",
       "      <td>0.5874</td>\n",
       "      <td>0.6192</td>\n",
       "      <td>0.5267</td>\n",
       "      <td>0.6482</td>\n",
       "      <td>0.3610</td>\n",
       "      <td>0.6590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7233</td>\n",
       "      <td>0.5599</td>\n",
       "      <td>0.7457</td>\n",
       "      <td>0.4969</td>\n",
       "      <td>0.7568</td>\n",
       "      <td>0.3440</td>\n",
       "      <td>0.6164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7907</td>\n",
       "      <td>0.6442</td>\n",
       "      <td>0.8068</td>\n",
       "      <td>0.5608</td>\n",
       "      <td>0.8128</td>\n",
       "      <td>0.3846</td>\n",
       "      <td>0.6761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6730</td>\n",
       "      <td>0.4177</td>\n",
       "      <td>0.6906</td>\n",
       "      <td>0.3364</td>\n",
       "      <td>0.6972</td>\n",
       "      <td>0.2286</td>\n",
       "      <td>0.4727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6175</td>\n",
       "      <td>0.5997</td>\n",
       "      <td>0.6312</td>\n",
       "      <td>0.5377</td>\n",
       "      <td>0.6623</td>\n",
       "      <td>0.3682</td>\n",
       "      <td>0.6732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7473</td>\n",
       "      <td>0.5735</td>\n",
       "      <td>0.7647</td>\n",
       "      <td>0.5087</td>\n",
       "      <td>0.7737</td>\n",
       "      <td>0.3552</td>\n",
       "      <td>0.6364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.8075</td>\n",
       "      <td>0.6613</td>\n",
       "      <td>0.8236</td>\n",
       "      <td>0.5749</td>\n",
       "      <td>0.8261</td>\n",
       "      <td>0.3925</td>\n",
       "      <td>0.6937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6947</td>\n",
       "      <td>0.4369</td>\n",
       "      <td>0.7117</td>\n",
       "      <td>0.3502</td>\n",
       "      <td>0.7130</td>\n",
       "      <td>0.2369</td>\n",
       "      <td>0.4941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6241</td>\n",
       "      <td>0.6037</td>\n",
       "      <td>0.6386</td>\n",
       "      <td>0.5425</td>\n",
       "      <td>0.6695</td>\n",
       "      <td>0.3728</td>\n",
       "      <td>0.6795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7661</td>\n",
       "      <td>0.5903</td>\n",
       "      <td>0.7826</td>\n",
       "      <td>0.5228</td>\n",
       "      <td>0.7875</td>\n",
       "      <td>0.3619</td>\n",
       "      <td>0.6558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.8325</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.8420</td>\n",
       "      <td>0.5931</td>\n",
       "      <td>0.8420</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.7157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6304</td>\n",
       "      <td>0.3883</td>\n",
       "      <td>0.6522</td>\n",
       "      <td>0.3122</td>\n",
       "      <td>0.6662</td>\n",
       "      <td>0.2119</td>\n",
       "      <td>0.4324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6033</td>\n",
       "      <td>0.5805</td>\n",
       "      <td>0.6165</td>\n",
       "      <td>0.5172</td>\n",
       "      <td>0.6463</td>\n",
       "      <td>0.3568</td>\n",
       "      <td>0.6488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7086</td>\n",
       "      <td>0.5426</td>\n",
       "      <td>0.7288</td>\n",
       "      <td>0.4725</td>\n",
       "      <td>0.7442</td>\n",
       "      <td>0.3317</td>\n",
       "      <td>0.5914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7793</td>\n",
       "      <td>0.6293</td>\n",
       "      <td>0.7955</td>\n",
       "      <td>0.5364</td>\n",
       "      <td>0.8016</td>\n",
       "      <td>0.3699</td>\n",
       "      <td>0.6507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6013</td>\n",
       "      <td>0.3611</td>\n",
       "      <td>0.6270</td>\n",
       "      <td>0.2873</td>\n",
       "      <td>0.6435</td>\n",
       "      <td>0.1952</td>\n",
       "      <td>0.3982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5807</td>\n",
       "      <td>0.5647</td>\n",
       "      <td>0.5921</td>\n",
       "      <td>0.5036</td>\n",
       "      <td>0.6221</td>\n",
       "      <td>0.3464</td>\n",
       "      <td>0.6379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6802</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7070</td>\n",
       "      <td>0.4528</td>\n",
       "      <td>0.7252</td>\n",
       "      <td>0.3199</td>\n",
       "      <td>0.5694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7617</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7828</td>\n",
       "      <td>0.5287</td>\n",
       "      <td>0.7892</td>\n",
       "      <td>0.3639</td>\n",
       "      <td>0.6367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6772</td>\n",
       "      <td>0.4297</td>\n",
       "      <td>0.6982</td>\n",
       "      <td>0.3459</td>\n",
       "      <td>0.7071</td>\n",
       "      <td>0.2352</td>\n",
       "      <td>0.4865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6239</td>\n",
       "      <td>0.6050</td>\n",
       "      <td>0.6372</td>\n",
       "      <td>0.5430</td>\n",
       "      <td>0.6690</td>\n",
       "      <td>0.3713</td>\n",
       "      <td>0.6768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7543</td>\n",
       "      <td>0.5874</td>\n",
       "      <td>0.7728</td>\n",
       "      <td>0.5199</td>\n",
       "      <td>0.7832</td>\n",
       "      <td>0.3595</td>\n",
       "      <td>0.6473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.8173</td>\n",
       "      <td>0.6717</td>\n",
       "      <td>0.8323</td>\n",
       "      <td>0.5829</td>\n",
       "      <td>0.8318</td>\n",
       "      <td>0.3969</td>\n",
       "      <td>0.7020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6818</td>\n",
       "      <td>0.4342</td>\n",
       "      <td>0.6985</td>\n",
       "      <td>0.3512</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.2352</td>\n",
       "      <td>0.4886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6267</td>\n",
       "      <td>0.6048</td>\n",
       "      <td>0.6400</td>\n",
       "      <td>0.5395</td>\n",
       "      <td>0.6696</td>\n",
       "      <td>0.3677</td>\n",
       "      <td>0.6721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7535</td>\n",
       "      <td>0.5898</td>\n",
       "      <td>0.7703</td>\n",
       "      <td>0.5220</td>\n",
       "      <td>0.7811</td>\n",
       "      <td>0.3583</td>\n",
       "      <td>0.6458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.8211</td>\n",
       "      <td>0.6739</td>\n",
       "      <td>0.8347</td>\n",
       "      <td>0.5838</td>\n",
       "      <td>0.8346</td>\n",
       "      <td>0.3964</td>\n",
       "      <td>0.7037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6012</td>\n",
       "      <td>0.3608</td>\n",
       "      <td>0.6269</td>\n",
       "      <td>0.2865</td>\n",
       "      <td>0.6427</td>\n",
       "      <td>0.1946</td>\n",
       "      <td>0.3972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5803</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>0.5923</td>\n",
       "      <td>0.5039</td>\n",
       "      <td>0.6222</td>\n",
       "      <td>0.3468</td>\n",
       "      <td>0.6648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.6803</td>\n",
       "      <td>0.5161</td>\n",
       "      <td>0.7067</td>\n",
       "      <td>0.4524</td>\n",
       "      <td>0.7253</td>\n",
       "      <td>0.3196</td>\n",
       "      <td>0.5893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.7615</td>\n",
       "      <td>0.6138</td>\n",
       "      <td>0.7831</td>\n",
       "      <td>0.5284</td>\n",
       "      <td>0.7893</td>\n",
       "      <td>0.3637</td>\n",
       "      <td>0.6366</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Method          Matching Field Ranking Field     Loss Training Data  \\\n",
       "0   Unsupervised                  answer                                        \n",
       "1   Unsupervised                question                                        \n",
       "2   Unsupervised         question_answer                                        \n",
       "3   Unsupervised  question_answer_concat                                        \n",
       "4     Supervised                  answer      BERT-Q-a  triplet           faq   \n",
       "5     Supervised                question      BERT-Q-a  triplet           faq   \n",
       "6     Supervised         question_answer      BERT-Q-a  triplet           faq   \n",
       "7     Supervised  question_answer_concat      BERT-Q-a  triplet           faq   \n",
       "8     Supervised                  answer      BERT-Q-a  triplet           faq   \n",
       "9     Supervised                question      BERT-Q-a  triplet           faq   \n",
       "10    Supervised         question_answer      BERT-Q-a  triplet           faq   \n",
       "11    Supervised  question_answer_concat      BERT-Q-a  triplet           faq   \n",
       "12    Supervised                  answer      BERT-Q-a  triplet    user_query   \n",
       "13    Supervised                question      BERT-Q-a  triplet    user_query   \n",
       "14    Supervised         question_answer      BERT-Q-a  triplet    user_query   \n",
       "15    Supervised  question_answer_concat      BERT-Q-a  triplet    user_query   \n",
       "16    Supervised                  answer      BERT-Q-a  triplet    user_query   \n",
       "17    Supervised                question      BERT-Q-a  triplet    user_query   \n",
       "18    Supervised         question_answer      BERT-Q-a  triplet    user_query   \n",
       "19    Supervised  question_answer_concat      BERT-Q-a  triplet    user_query   \n",
       "20    Supervised                  answer      BERT-Q-a  softmax           faq   \n",
       "21    Supervised                question      BERT-Q-a  softmax           faq   \n",
       "22    Supervised         question_answer      BERT-Q-a  softmax           faq   \n",
       "23    Supervised  question_answer_concat      BERT-Q-a  softmax           faq   \n",
       "24    Supervised                  answer      BERT-Q-a  softmax           faq   \n",
       "25    Supervised                question      BERT-Q-a  softmax           faq   \n",
       "26    Supervised         question_answer      BERT-Q-a  softmax           faq   \n",
       "27    Supervised  question_answer_concat      BERT-Q-a  softmax           faq   \n",
       "28    Supervised                  answer      BERT-Q-a  softmax    user_query   \n",
       "29    Supervised                question      BERT-Q-a  softmax    user_query   \n",
       "30    Supervised         question_answer      BERT-Q-a  softmax    user_query   \n",
       "31    Supervised  question_answer_concat      BERT-Q-a  softmax    user_query   \n",
       "32    Supervised                  answer      BERT-Q-a  softmax    user_query   \n",
       "33    Supervised                question      BERT-Q-a  softmax    user_query   \n",
       "34    Supervised         question_answer      BERT-Q-a  softmax    user_query   \n",
       "35    Supervised  question_answer_concat      BERT-Q-a  softmax    user_query   \n",
       "36    Supervised                  answer      BERT-Q-q  triplet           faq   \n",
       "37    Supervised                question      BERT-Q-q  triplet           faq   \n",
       "38    Supervised         question_answer      BERT-Q-q  triplet           faq   \n",
       "39    Supervised  question_answer_concat      BERT-Q-q  triplet           faq   \n",
       "40    Supervised                  answer      BERT-Q-q  triplet           faq   \n",
       "41    Supervised                question      BERT-Q-q  triplet           faq   \n",
       "42    Supervised         question_answer      BERT-Q-q  triplet           faq   \n",
       "43    Supervised  question_answer_concat      BERT-Q-q  triplet           faq   \n",
       "44    Supervised                  answer      BERT-Q-q  triplet    user_query   \n",
       "45    Supervised                question      BERT-Q-q  triplet    user_query   \n",
       "46    Supervised         question_answer      BERT-Q-q  triplet    user_query   \n",
       "47    Supervised  question_answer_concat      BERT-Q-q  triplet    user_query   \n",
       "48    Supervised                  answer      BERT-Q-q  triplet    user_query   \n",
       "49    Supervised                question      BERT-Q-q  triplet    user_query   \n",
       "50    Supervised         question_answer      BERT-Q-q  triplet    user_query   \n",
       "51    Supervised  question_answer_concat      BERT-Q-q  triplet    user_query   \n",
       "52    Supervised                  answer      BERT-Q-q  softmax           faq   \n",
       "53    Supervised                question      BERT-Q-q  softmax           faq   \n",
       "54    Supervised         question_answer      BERT-Q-q  softmax           faq   \n",
       "55    Supervised  question_answer_concat      BERT-Q-q  softmax           faq   \n",
       "56    Supervised                  answer      BERT-Q-q  softmax           faq   \n",
       "57    Supervised                question      BERT-Q-q  softmax           faq   \n",
       "58    Supervised         question_answer      BERT-Q-q  softmax           faq   \n",
       "59    Supervised  question_answer_concat      BERT-Q-q  softmax           faq   \n",
       "60    Supervised                  answer      BERT-Q-q  softmax    user_query   \n",
       "61    Supervised                question      BERT-Q-q  softmax    user_query   \n",
       "62    Supervised         question_answer      BERT-Q-q  softmax    user_query   \n",
       "63    Supervised  question_answer_concat      BERT-Q-q  softmax    user_query   \n",
       "64    Supervised                  answer      BERT-Q-q  softmax    user_query   \n",
       "65    Supervised                question      BERT-Q-q  softmax    user_query   \n",
       "66    Supervised         question_answer      BERT-Q-q  softmax    user_query   \n",
       "67    Supervised  question_answer_concat      BERT-Q-q  softmax    user_query   \n",
       "\n",
       "   Negative Sampling  NDCG@3  NDCG@5 NDCG@10     P@3     P@5    P@10     MAP  \n",
       "0                     0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "1                     0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "2                     0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "3                     0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "4             simple  0.6410  0.3990  0.6634  0.3183  0.6732  0.2175  0.4459  \n",
       "5             simple  0.6073  0.5890  0.6216  0.5267  0.6498  0.3606  0.6607  \n",
       "6             simple  0.7190  0.5548  0.7421  0.4884  0.7540  0.3430  0.6119  \n",
       "7             simple  0.7873  0.6408  0.8057  0.5544  0.8086  0.3818  0.6703  \n",
       "8               hard  0.6527  0.4073  0.6733  0.3267  0.6801  0.2204  0.4545  \n",
       "9               hard  0.6056  0.5874  0.6192  0.5267  0.6482  0.3610  0.6590  \n",
       "10              hard  0.7233  0.5599  0.7457  0.4969  0.7568  0.3440  0.6164  \n",
       "11              hard  0.7907  0.6442  0.8068  0.5608  0.8128  0.3846  0.6761  \n",
       "12            simple  0.6730  0.4177  0.6906  0.3364  0.6972  0.2286  0.4727  \n",
       "13            simple  0.6175  0.5997  0.6312  0.5377  0.6623  0.3682  0.6732  \n",
       "14            simple  0.7473  0.5735  0.7647  0.5087  0.7737  0.3552  0.6364  \n",
       "15            simple  0.8075  0.6613  0.8236  0.5749  0.8261  0.3925  0.6937  \n",
       "16              hard  0.6947  0.4369  0.7117  0.3502  0.7130  0.2369  0.4941  \n",
       "17              hard  0.6241  0.6037  0.6386  0.5425  0.6695  0.3728  0.6795  \n",
       "18              hard  0.7661  0.5903  0.7826  0.5228  0.7875  0.3619  0.6558  \n",
       "19              hard  0.8325  0.6869  0.8420  0.5931  0.8420  0.4008  0.7157  \n",
       "20            simple  0.6304  0.3883  0.6522  0.3122  0.6662  0.2119  0.4324  \n",
       "21            simple  0.6033  0.5805  0.6165  0.5172  0.6463  0.3568  0.6488  \n",
       "22            simple  0.7086  0.5426  0.7288  0.4725  0.7442  0.3317  0.5914  \n",
       "23            simple  0.7793  0.6293  0.7955  0.5364  0.8016  0.3699  0.6507  \n",
       "24              hard  0.6013  0.3611  0.6270  0.2873  0.6435  0.1952  0.3982  \n",
       "25              hard  0.5807  0.5647  0.5921  0.5036  0.6221  0.3464  0.6379  \n",
       "26              hard  0.6802  0.5161  0.7070  0.4528  0.7252  0.3199  0.5694  \n",
       "27              hard  0.7617  0.6138  0.7828  0.5287  0.7892  0.3639  0.6367  \n",
       "28            simple  0.6772  0.4297  0.6982  0.3459  0.7071  0.2352  0.4865  \n",
       "29            simple  0.6239  0.6050  0.6372  0.5430  0.6690  0.3713  0.6768  \n",
       "30            simple  0.7543  0.5874  0.7728  0.5199  0.7832  0.3595  0.6473  \n",
       "31            simple  0.8173  0.6717  0.8323  0.5829  0.8318  0.3969  0.7020  \n",
       "32              hard  0.6818  0.4342  0.6985  0.3512  0.7067  0.2352  0.4886  \n",
       "33              hard  0.6267  0.6048  0.6400  0.5395  0.6696  0.3677  0.6721  \n",
       "34              hard  0.7535  0.5898  0.7703  0.5220  0.7811  0.3583  0.6458  \n",
       "35              hard  0.8211  0.6739  0.8347  0.5838  0.8346  0.3964  0.7037  \n",
       "36            simple  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "37            simple  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "38            simple  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "39            simple  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "40              hard  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "41              hard  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "42              hard  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "43              hard  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "44            simple  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "45            simple  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "46            simple  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "47            simple  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "48              hard  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "49              hard  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "50              hard  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "51              hard  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "52            simple  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "53            simple  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "54            simple  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "55            simple  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "56              hard  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "57              hard  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "58              hard  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "59              hard  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "60            simple  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "61            simple  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "62            simple  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "63            simple  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  \n",
       "64              hard  0.6012  0.3608  0.6269  0.2865  0.6427  0.1946  0.3972  \n",
       "65              hard  0.5803  0.5645  0.5923  0.5039  0.6222  0.3468  0.6648  \n",
       "66              hard  0.6803  0.5161  0.7067  0.4524  0.7253  0.3196  0.5893  \n",
       "67              hard  0.7615  0.6138  0.7831  0.5284  0.7893  0.3637  0.6366  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}