notebooks/ann_analysis_simulation_from_files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import chart_studio.plotly as py\n",
    "import plotly.tools as tls\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_auc_score, accuracy_score, log_loss, mean_squared_error, f1_score, matthews_corrcoef, classification_report, roc_curve, auc\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model from file\n",
    "#xnn_dir = '/home/kimm/article-information-2019/data/xnn_output/simulation_results/'\n",
    "#xnn_dir = '/Users/kmontgomery/Documents/git/article-information-2019/data/xnn_output/simulation_results/'\n",
    "xnn_dir = '/Users/phall/workspace/article-information-2019/data/xnn_output/simulation_results/'\n",
    "\n",
    "filename = 'simulated_ann_results_with_Shapley.csv'\n",
    "\n",
    "TEST = pd.read_csv(xnn_dir + filename)\n",
    "\n",
    "Feature_names = ['binary1', 'binary2', 'cat1_0', 'cat1_1', 'cat1_2', 'cat1_3', 'cat1_4', \n",
    "                 'fried1_std', 'fried2_std', 'fried3_std', 'fried4_std', 'fried5_std']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'binary1', 'binary2', 'ctrl_class1', 'ctrl_class2',\n",
       "       'fold', 'intercept', 'latent_no_noise', 'latent_with_noise',\n",
       "       'logistic_noise', 'outcome', 'outcome_no_noise', 'prot_class1',\n",
       "       'prot_class2', 'cat1_0', 'cat1_1', 'cat1_2', 'cat1_3', 'cat1_4',\n",
       "       'fried1_std', 'fried2_std', 'fried3_std', 'fried4_std', 'fried5_std',\n",
       "       'probability', 'binary1_Shapley_score', 'binary2_Shapley_score',\n",
       "       'cat1_0_Shapley_score', 'cat1_1_Shapley_score', 'cat1_2_Shapley_score',\n",
       "       'cat1_3_Shapley_score', 'cat1_4_Shapley_score',\n",
       "       'fried1_std_Shapley_score', 'fried2_std_Shapley_score',\n",
       "       'fried3_std_Shapley_score', 'fried4_std_Shapley_score',\n",
       "       'fried5_std_Shapley_score', 'Intercept_Shapley_score'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEST.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style>  \n",
       "<table id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7b\" ><caption>Recall and Precision</caption> \n",
       "<thead>    <tr> \n",
       "        <th class=\"blank level0\" ></th> \n",
       "        <th class=\"col_heading level0 col0\" >cutoff</th> \n",
       "        <th class=\"col_heading level0 col1\" >recall</th> \n",
       "        <th class=\"col_heading level0 col2\" >precision</th> \n",
       "        <th class=\"col_heading level0 col3\" >f1</th> \n",
       "    </tr></thead> \n",
       "<tbody>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row0\" class=\"row_heading level0 row0\" >0</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow0_col0\" class=\"data row0 col0\" >0</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow0_col1\" class=\"data row0 col1\" >1</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow0_col2\" class=\"data row0 col2\" >0.49935</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow0_col3\" class=\"data row0 col3\" >0.666089</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row1\" class=\"row_heading level0 row1\" >1</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow1_col0\" class=\"data row1 col0\" >0.01</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow1_col1\" class=\"data row1 col1\" >1</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow1_col2\" class=\"data row1 col2\" >0.5</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow1_col3\" class=\"data row1 col3\" >0.666667</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row2\" class=\"row_heading level0 row2\" >2</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow2_col0\" class=\"data row2 col0\" >0.02</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow2_col1\" class=\"data row2 col1\" >0.9998</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow2_col2\" class=\"data row2 col2\" >0.504752</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow2_col3\" class=\"data row2 col3\" >0.670832</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row3\" class=\"row_heading level0 row3\" >3</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow3_col0\" class=\"data row3 col0\" >0.03</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow3_col1\" class=\"data row3 col1\" >0.999199</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow3_col2\" class=\"data row3 col2\" >0.512401</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow3_col3\" class=\"data row3 col3\" >0.677415</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row4\" class=\"row_heading level0 row4\" >4</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow4_col0\" class=\"data row4 col0\" >0.04</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow4_col1\" class=\"data row4 col1\" >0.997597</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow4_col2\" class=\"data row4 col2\" >0.519583</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow4_col3\" class=\"data row4 col3\" >0.683286</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row5\" class=\"row_heading level0 row5\" >5</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow5_col0\" class=\"data row5 col0\" >0.05</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow5_col1\" class=\"data row5 col1\" >0.995995</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow5_col2\" class=\"data row5 col2\" >0.526436</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow5_col3\" class=\"data row5 col3\" >0.688803</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row6\" class=\"row_heading level0 row6\" >6</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow6_col0\" class=\"data row6 col0\" >0.06</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow6_col1\" class=\"data row6 col1\" >0.993992</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow6_col2\" class=\"data row6 col2\" >0.533652</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow6_col3\" class=\"data row6 col3\" >0.694463</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row7\" class=\"row_heading level0 row7\" >7</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow7_col0\" class=\"data row7 col0\" >0.07</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow7_col1\" class=\"data row7 col1\" >0.99219</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow7_col2\" class=\"data row7 col2\" >0.540619</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow7_col3\" class=\"data row7 col3\" >0.699887</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row8\" class=\"row_heading level0 row8\" >8</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow8_col0\" class=\"data row8 col0\" >0.08</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow8_col1\" class=\"data row8 col1\" >0.989787</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow8_col2\" class=\"data row8 col2\" >0.548223</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow8_col3\" class=\"data row8 col3\" >0.705618</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row9\" class=\"row_heading level0 row9\" >9</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow9_col0\" class=\"data row9 col0\" >0.09</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow9_col1\" class=\"data row9 col1\" >0.987484</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow9_col2\" class=\"data row9 col2\" >0.555637</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow9_col3\" class=\"data row9 col3\" >0.711134</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row10\" class=\"row_heading level0 row10\" >10</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow10_col0\" class=\"data row10 col0\" >0.1</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow10_col1\" class=\"data row10 col1\" >0.98438</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow10_col2\" class=\"data row10 col2\" >0.562543</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow10_col3\" class=\"data row10 col3\" >0.715945</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row11\" class=\"row_heading level0 row11\" >11</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow11_col0\" class=\"data row11 col0\" >0.11</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow11_col1\" class=\"data row11 col1\" >0.981977</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow11_col2\" class=\"data row11 col2\" >0.569215</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow11_col3\" class=\"data row11 col3\" >0.720679</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row12\" class=\"row_heading level0 row12\" >12</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow12_col0\" class=\"data row12 col0\" >0.12</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow12_col1\" class=\"data row12 col1\" >0.978973</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow12_col2\" class=\"data row12 col2\" >0.575355</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow12_col3\" class=\"data row12 col3\" >0.724759</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row13\" class=\"row_heading level0 row13\" >13</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow13_col0\" class=\"data row13 col0\" >0.13</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow13_col1\" class=\"data row13 col1\" >0.976169</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow13_col2\" class=\"data row13 col2\" >0.581509</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow13_col3\" class=\"data row13 col3\" >0.728843</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row14\" class=\"row_heading level0 row14\" >14</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow14_col0\" class=\"data row14 col0\" >0.14</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow14_col1\" class=\"data row14 col1\" >0.972965</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow14_col2\" class=\"data row14 col2\" >0.588339</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow14_col3\" class=\"data row14 col3\" >0.733275</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row15\" class=\"row_heading level0 row15\" >15</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow15_col0\" class=\"data row15 col0\" >0.15</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow15_col1\" class=\"data row15 col1\" >0.970562</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow15_col2\" class=\"data row15 col2\" >0.594663</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow15_col3\" class=\"data row15 col3\" >0.737475</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row16\" class=\"row_heading level0 row16\" >16</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow16_col0\" class=\"data row16 col0\" >0.16</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow16_col1\" class=\"data row16 col1\" >0.967057</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow16_col2\" class=\"data row16 col2\" >0.600996</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow16_col3\" class=\"data row16 col3\" >0.741298</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row17\" class=\"row_heading level0 row17\" >17</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow17_col0\" class=\"data row17 col0\" >0.17</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow17_col1\" class=\"data row17 col1\" >0.963252</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow17_col2\" class=\"data row17 col2\" >0.606137</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow17_col3\" class=\"data row17 col3\" >0.744064</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row18\" class=\"row_heading level0 row18\" >18</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow18_col0\" class=\"data row18 col0\" >0.18</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow18_col1\" class=\"data row18 col1\" >0.958947</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow18_col2\" class=\"data row18 col2\" >0.61148</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow18_col3\" class=\"data row18 col3\" >0.746774</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row19\" class=\"row_heading level0 row19\" >19</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow19_col0\" class=\"data row19 col0\" >0.19</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow19_col1\" class=\"data row19 col1\" >0.954541</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow19_col2\" class=\"data row19 col2\" >0.616823</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow19_col3\" class=\"data row19 col3\" >0.749391</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row20\" class=\"row_heading level0 row20\" >20</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow20_col0\" class=\"data row20 col0\" >0.2</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow20_col1\" class=\"data row20 col1\" >0.950335</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow20_col2\" class=\"data row20 col2\" >0.62326</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow20_col3\" class=\"data row20 col3\" >0.752806</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row21\" class=\"row_heading level0 row21\" >21</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow21_col0\" class=\"data row21 col0\" >0.21</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow21_col1\" class=\"data row21 col1\" >0.945629</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow21_col2\" class=\"data row21 col2\" >0.629139</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow21_col3\" class=\"data row21 col3\" >0.75558</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row22\" class=\"row_heading level0 row22\" >22</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow22_col0\" class=\"data row22 col0\" >0.22</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow22_col1\" class=\"data row22 col1\" >0.941524</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow22_col2\" class=\"data row22 col2\" >0.634523</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow22_col3\" class=\"data row22 col3\" >0.758123</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row23\" class=\"row_heading level0 row23\" >23</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow23_col0\" class=\"data row23 col0\" >0.23</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow23_col1\" class=\"data row23 col1\" >0.937919</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow23_col2\" class=\"data row23 col2\" >0.639604</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow23_col3\" class=\"data row23 col3\" >0.760555</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row24\" class=\"row_heading level0 row24\" >24</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow24_col0\" class=\"data row24 col0\" >0.24</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow24_col1\" class=\"data row24 col1\" >0.932913</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow24_col2\" class=\"data row24 col2\" >0.645043</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow24_col3\" class=\"data row24 col3\" >0.762719</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row25\" class=\"row_heading level0 row25\" >25</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow25_col0\" class=\"data row25 col0\" >0.25</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow25_col1\" class=\"data row25 col1\" >0.927906</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow25_col2\" class=\"data row25 col2\" >0.648813</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow25_col3\" class=\"data row25 col3\" >0.763659</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row26\" class=\"row_heading level0 row26\" >26</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow26_col0\" class=\"data row26 col0\" >0.26</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow26_col1\" class=\"data row26 col1\" >0.923</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow26_col2\" class=\"data row26 col2\" >0.653991</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow26_col3\" class=\"data row26 col3\" >0.765551</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row27\" class=\"row_heading level0 row27\" >27</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow27_col0\" class=\"data row27 col0\" >0.27</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow27_col1\" class=\"data row27 col1\" >0.917493</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow27_col2\" class=\"data row27 col2\" >0.658687</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow27_col3\" class=\"data row27 col3\" >0.766842</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row28\" class=\"row_heading level0 row28\" >28</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow28_col0\" class=\"data row28 col0\" >0.28</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow28_col1\" class=\"data row28 col1\" >0.911485</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow28_col2\" class=\"data row28 col2\" >0.662904</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow28_col3\" class=\"data row28 col3\" >0.76757</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row29\" class=\"row_heading level0 row29\" >29</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow29_col0\" class=\"data row29 col0\" >0.29</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow29_col1\" class=\"data row29 col1\" >0.906779</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow29_col2\" class=\"data row29 col2\" >0.66908</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow29_col3\" class=\"data row29 col3\" >0.770003</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row30\" class=\"row_heading level0 row30\" >30</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow30_col0\" class=\"data row30 col0\" >0.3</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow30_col1\" class=\"data row30 col1\" >0.901772</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow30_col2\" class=\"data row30 col2\" >0.673749</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow30_col3\" class=\"data row30 col3\" >0.77126</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row31\" class=\"row_heading level0 row31\" >31</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow31_col0\" class=\"data row31 col0\" >0.31</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow31_col1\" class=\"data row31 col1\" >0.897266</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow31_col2\" class=\"data row31 col2\" >0.678967</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow31_col3\" class=\"data row31 col3\" >0.773</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row32\" class=\"row_heading level0 row32\" >32</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow32_col0\" class=\"data row32 col0\" >0.32</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow32_col1\" class=\"data row32 col1\" >0.89226</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow32_col2\" class=\"data row32 col2\" >0.684777</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow32_col3\" class=\"data row32 col3\" >0.77487</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row33\" class=\"row_heading level0 row33\" >33</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow33_col0\" class=\"data row33 col0\" >0.33</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow33_col1\" class=\"data row33 col1\" >0.887354</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow33_col2\" class=\"data row33 col2\" >0.690079</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow33_col3\" class=\"data row33 col3\" >0.776381</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row34\" class=\"row_heading level0 row34\" >34</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow34_col0\" class=\"data row34 col0\" >0.34</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow34_col1\" class=\"data row34 col1\" >0.881045</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow34_col2\" class=\"data row34 col2\" >0.69442</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow34_col3\" class=\"data row34 col3\" >0.776679</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row35\" class=\"row_heading level0 row35\" >35</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow35_col0\" class=\"data row35 col0\" >0.35</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow35_col1\" class=\"data row35 col1\" >0.875638</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow35_col2\" class=\"data row35 col2\" >0.69932</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow35_col3\" class=\"data row35 col3\" >0.77761</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row36\" class=\"row_heading level0 row36\" >36</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow36_col0\" class=\"data row36 col0\" >0.36</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow36_col1\" class=\"data row36 col1\" >0.870031</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow36_col2\" class=\"data row36 col2\" >0.703962</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow36_col3\" class=\"data row36 col3\" >0.778236</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row37\" class=\"row_heading level0 row37\" >37</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow37_col0\" class=\"data row37 col0\" >0.37</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow37_col1\" class=\"data row37 col1\" >0.864624</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow37_col2\" class=\"data row37 col2\" >0.708716</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow37_col3\" class=\"data row37 col3\" >0.778945</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row38\" class=\"row_heading level0 row38\" >38</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow38_col0\" class=\"data row38 col0\" >0.38</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow38_col1\" class=\"data row38 col1\" >0.858015</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow38_col2\" class=\"data row38 col2\" >0.713667</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow38_col3\" class=\"data row38 col3\" >0.779213</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row39\" class=\"row_heading level0 row39\" >39</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow39_col0\" class=\"data row39 col0\" >0.39</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow39_col1\" class=\"data row39 col1\" >0.849604</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow39_col2\" class=\"data row39 col2\" >0.717669</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow39_col3\" class=\"data row39 col3\" >0.778083</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row40\" class=\"row_heading level0 row40\" >40</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow40_col0\" class=\"data row40 col0\" >0.4</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow40_col1\" class=\"data row40 col1\" >0.843497</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow40_col2\" class=\"data row40 col2\" >0.722842</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow40_col3\" class=\"data row40 col3\" >0.778522</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row41\" class=\"row_heading level0 row41\" >41</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow41_col0\" class=\"data row41 col0\" >0.41</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow41_col1\" class=\"data row41 col1\" >0.835887</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow41_col2\" class=\"data row41 col2\" >0.728256</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow41_col3\" class=\"data row41 col3\" >0.778368</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row42\" class=\"row_heading level0 row42\" >42</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow42_col0\" class=\"data row42 col0\" >0.42</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow42_col1\" class=\"data row42 col1\" >0.829779</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow42_col2\" class=\"data row42 col2\" >0.732779</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow42_col3\" class=\"data row42 col3\" >0.778268</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row43\" class=\"row_heading level0 row43\" >43</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow43_col0\" class=\"data row43 col0\" >0.43</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow43_col1\" class=\"data row43 col1\" >0.820967</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow43_col2\" class=\"data row43 col2\" >0.737652</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow43_col3\" class=\"data row43 col3\" >0.777083</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row44\" class=\"row_heading level0 row44\" >44</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow44_col0\" class=\"data row44 col0\" >0.44</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow44_col1\" class=\"data row44 col1\" >0.813257</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow44_col2\" class=\"data row44 col2\" >0.742889</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow44_col3\" class=\"data row44 col3\" >0.776482</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row45\" class=\"row_heading level0 row45\" >45</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow45_col0\" class=\"data row45 col0\" >0.45</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow45_col1\" class=\"data row45 col1\" >0.803845</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow45_col2\" class=\"data row45 col2\" >0.747138</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow45_col3\" class=\"data row45 col3\" >0.774455</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row46\" class=\"row_heading level0 row46\" >46</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow46_col0\" class=\"data row46 col0\" >0.46</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow46_col1\" class=\"data row46 col1\" >0.795534</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow46_col2\" class=\"data row46 col2\" >0.75194</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow46_col3\" class=\"data row46 col3\" >0.773123</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row47\" class=\"row_heading level0 row47\" >47</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow47_col0\" class=\"data row47 col0\" >0.47</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow47_col1\" class=\"data row47 col1\" >0.787223</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow47_col2\" class=\"data row47 col2\" >0.75618</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow47_col3\" class=\"data row47 col3\" >0.771389</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row48\" class=\"row_heading level0 row48\" >48</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow48_col0\" class=\"data row48 col0\" >0.48</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow48_col1\" class=\"data row48 col1\" >0.778913</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow48_col2\" class=\"data row48 col2\" >0.759816</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow48_col3\" class=\"data row48 col3\" >0.769246</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row49\" class=\"row_heading level0 row49\" >49</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow49_col0\" class=\"data row49 col0\" >0.49</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow49_col1\" class=\"data row49 col1\" >0.771503</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow49_col2\" class=\"data row49 col2\" >0.764764</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow49_col3\" class=\"data row49 col3\" >0.768119</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row50\" class=\"row_heading level0 row50\" >50</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow50_col0\" class=\"data row50 col0\" >0.5</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow50_col1\" class=\"data row50 col1\" >0.763793</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow50_col2\" class=\"data row50 col2\" >0.768797</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow50_col3\" class=\"data row50 col3\" >0.766287</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row51\" class=\"row_heading level0 row51\" >51</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow51_col0\" class=\"data row51 col0\" >0.51</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow51_col1\" class=\"data row51 col1\" >0.756083</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow51_col2\" class=\"data row51 col2\" >0.775018</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow51_col3\" class=\"data row51 col3\" >0.765433</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row52\" class=\"row_heading level0 row52\" >52</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow52_col0\" class=\"data row52 col0\" >0.52</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow52_col1\" class=\"data row52 col1\" >0.74647</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow52_col2\" class=\"data row52 col2\" >0.778427</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow52_col3\" class=\"data row52 col3\" >0.762114</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row53\" class=\"row_heading level0 row53\" >53</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow53_col0\" class=\"data row53 col0\" >0.53</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow53_col1\" class=\"data row53 col1\" >0.73836</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow53_col2\" class=\"data row53 col2\" >0.782387</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow53_col3\" class=\"data row53 col3\" >0.759736</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row54\" class=\"row_heading level0 row54\" >54</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow54_col0\" class=\"data row54 col0\" >0.54</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow54_col1\" class=\"data row54 col1\" >0.727946</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow54_col2\" class=\"data row54 col2\" >0.786286</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow54_col3\" class=\"data row54 col3\" >0.755992</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row55\" class=\"row_heading level0 row55\" >55</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow55_col0\" class=\"data row55 col0\" >0.55</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow55_col1\" class=\"data row55 col1\" >0.719636</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow55_col2\" class=\"data row55 col2\" >0.790041</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow55_col3\" class=\"data row55 col3\" >0.753196</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row56\" class=\"row_heading level0 row56\" >56</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow56_col0\" class=\"data row56 col0\" >0.56</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow56_col1\" class=\"data row56 col1\" >0.710023</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow56_col2\" class=\"data row56 col2\" >0.793887</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow56_col3\" class=\"data row56 col3\" >0.749617</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row57\" class=\"row_heading level0 row57\" >57</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow57_col0\" class=\"data row57 col0\" >0.57</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow57_col1\" class=\"data row57 col1\" >0.701812</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow57_col2\" class=\"data row57 col2\" >0.797474</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow57_col3\" class=\"data row57 col3\" >0.746591</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row58\" class=\"row_heading level0 row58\" >58</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow58_col0\" class=\"data row58 col0\" >0.58</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow58_col1\" class=\"data row58 col1\" >0.691699</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow58_col2\" class=\"data row58 col2\" >0.801857</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow58_col3\" class=\"data row58 col3\" >0.742716</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row59\" class=\"row_heading level0 row59\" >59</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow59_col0\" class=\"data row59 col0\" >0.59</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow59_col1\" class=\"data row59 col1\" >0.681786</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow59_col2\" class=\"data row59 col2\" >0.806658</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow59_col3\" class=\"data row59 col3\" >0.738984</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row60\" class=\"row_heading level0 row60\" >60</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow60_col0\" class=\"data row60 col0\" >0.6</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow60_col1\" class=\"data row60 col1\" >0.671773</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow60_col2\" class=\"data row60 col2\" >0.81007</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow60_col3\" class=\"data row60 col3\" >0.734468</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row61\" class=\"row_heading level0 row61\" >61</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow61_col0\" class=\"data row61 col0\" >0.61</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow61_col1\" class=\"data row61 col1\" >0.660358</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow61_col2\" class=\"data row61 col2\" >0.814097</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow61_col3\" class=\"data row61 col3\" >0.729213</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row62\" class=\"row_heading level0 row62\" >62</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow62_col0\" class=\"data row62 col0\" >0.62</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow62_col1\" class=\"data row62 col1\" >0.649344</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow62_col2\" class=\"data row62 col2\" >0.820056</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow62_col3\" class=\"data row62 col3\" >0.724783</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row63\" class=\"row_heading level0 row63\" >63</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow63_col0\" class=\"data row63 col0\" >0.63</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow63_col1\" class=\"data row63 col1\" >0.63843</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow63_col2\" class=\"data row63 col2\" >0.824199</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow63_col3\" class=\"data row63 col3\" >0.719517</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row64\" class=\"row_heading level0 row64\" >64</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow64_col0\" class=\"data row64 col0\" >0.64</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow64_col1\" class=\"data row64 col1\" >0.626815</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow64_col2\" class=\"data row64 col2\" >0.82859</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow64_col3\" class=\"data row64 col3\" >0.713716</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row65\" class=\"row_heading level0 row65\" >65</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow65_col0\" class=\"data row65 col0\" >0.65</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow65_col1\" class=\"data row65 col1\" >0.616702</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow65_col2\" class=\"data row65 col2\" >0.833762</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow65_col3\" class=\"data row65 col3\" >0.70899</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row66\" class=\"row_heading level0 row66\" >66</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow66_col0\" class=\"data row66 col0\" >0.66</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow66_col1\" class=\"data row66 col1\" >0.606088</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow66_col2\" class=\"data row66 col2\" >0.838947</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow66_col3\" class=\"data row66 col3\" >0.703755</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row67\" class=\"row_heading level0 row67\" >67</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow67_col0\" class=\"data row67 col0\" >0.67</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow67_col1\" class=\"data row67 col1\" >0.594773</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow67_col2\" class=\"data row67 col2\" >0.84351</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow67_col3\" class=\"data row67 col3\" >0.697633</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row68\" class=\"row_heading level0 row68\" >68</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow68_col0\" class=\"data row68 col0\" >0.68</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow68_col1\" class=\"data row68 col1\" >0.582357</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow68_col2\" class=\"data row68 col2\" >0.848432</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow68_col3\" class=\"data row68 col3\" >0.690654</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row69\" class=\"row_heading level0 row69\" >69</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow69_col0\" class=\"data row69 col0\" >0.69</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow69_col1\" class=\"data row69 col1\" >0.572644</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow69_col2\" class=\"data row69 col2\" >0.853073</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow69_col3\" class=\"data row69 col3\" >0.685279</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row70\" class=\"row_heading level0 row70\" >70</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow70_col0\" class=\"data row70 col0\" >0.7</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow70_col1\" class=\"data row70 col1\" >0.559527</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow70_col2\" class=\"data row70 col2\" >0.85824</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow70_col3\" class=\"data row70 col3\" >0.677415</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row71\" class=\"row_heading level0 row71\" >71</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow71_col0\" class=\"data row71 col0\" >0.71</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow71_col1\" class=\"data row71 col1\" >0.545109</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow71_col2\" class=\"data row71 col2\" >0.862211</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow71_col3\" class=\"data row71 col3\" >0.667934</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row72\" class=\"row_heading level0 row72\" >72</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow72_col0\" class=\"data row72 col0\" >0.72</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow72_col1\" class=\"data row72 col1\" >0.53039</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow72_col2\" class=\"data row72 col2\" >0.867934</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow72_col3\" class=\"data row72 col3\" >0.658421</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row73\" class=\"row_heading level0 row73\" >73</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow73_col0\" class=\"data row73 col0\" >0.73</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow73_col1\" class=\"data row73 col1\" >0.516572</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow73_col2\" class=\"data row73 col2\" >0.870864</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow73_col3\" class=\"data row73 col3\" >0.648482</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row74\" class=\"row_heading level0 row74\" >74</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow74_col0\" class=\"data row74 col0\" >0.74</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow74_col1\" class=\"data row74 col1\" >0.501252</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow74_col2\" class=\"data row74 col2\" >0.874258</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow74_col3\" class=\"data row74 col3\" >0.637179</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row75\" class=\"row_heading level0 row75\" >75</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow75_col0\" class=\"data row75 col0\" >0.75</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow75_col1\" class=\"data row75 col1\" >0.486633</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow75_col2\" class=\"data row75 col2\" >0.878049</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow75_col3\" class=\"data row75 col3\" >0.626208</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row76\" class=\"row_heading level0 row76\" >76</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow76_col0\" class=\"data row76 col0\" >0.76</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow76_col1\" class=\"data row76 col1\" >0.472414</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow76_col2\" class=\"data row76 col2\" >0.882529</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow76_col3\" class=\"data row76 col3\" >0.615405</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row77\" class=\"row_heading level0 row77\" >77</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow77_col0\" class=\"data row77 col0\" >0.77</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow77_col1\" class=\"data row77 col1\" >0.456493</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow77_col2\" class=\"data row77 col2\" >0.887138</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow77_col3\" class=\"data row77 col3\" >0.602803</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row78\" class=\"row_heading level0 row78\" >78</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow78_col0\" class=\"data row78 col0\" >0.78</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow78_col1\" class=\"data row78 col1\" >0.441474</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow78_col2\" class=\"data row78 col2\" >0.892329</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow78_col3\" class=\"data row78 col3\" >0.590702</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row79\" class=\"row_heading level0 row79\" >79</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow79_col0\" class=\"data row79 col0\" >0.79</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow79_col1\" class=\"data row79 col1\" >0.426755</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow79_col2\" class=\"data row79 col2\" >0.897074</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow79_col3\" class=\"data row79 col3\" >0.578369</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row80\" class=\"row_heading level0 row80\" >80</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow80_col0\" class=\"data row80 col0\" >0.8</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow80_col1\" class=\"data row80 col1\" >0.411835</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow80_col2\" class=\"data row80 col2\" >0.900986</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow80_col3\" class=\"data row80 col3\" >0.565283</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row81\" class=\"row_heading level0 row81\" >81</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow81_col0\" class=\"data row81 col0\" >0.81</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow81_col1\" class=\"data row81 col1\" >0.395214</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow81_col2\" class=\"data row81 col2\" >0.905275</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow81_col3\" class=\"data row81 col3\" >0.55022</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row82\" class=\"row_heading level0 row82\" >82</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow82_col0\" class=\"data row82 col0\" >0.82</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow82_col1\" class=\"data row82 col1\" >0.378592</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow82_col2\" class=\"data row82 col2\" >0.911743</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow82_col3\" class=\"data row82 col3\" >0.535022</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row83\" class=\"row_heading level0 row83\" >83</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow83_col0\" class=\"data row83 col0\" >0.83</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow83_col1\" class=\"data row83 col1\" >0.361971</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow83_col2\" class=\"data row83 col2\" >0.915653</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow83_col3\" class=\"data row83 col3\" >0.518837</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row84\" class=\"row_heading level0 row84\" >84</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow84_col0\" class=\"data row84 col0\" >0.84</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow84_col1\" class=\"data row84 col1\" >0.343847</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow84_col2\" class=\"data row84 col2\" >0.919904</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow84_col3\" class=\"data row84 col3\" >0.500583</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row85\" class=\"row_heading level0 row85\" >85</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow85_col0\" class=\"data row85 col0\" >0.85</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow85_col1\" class=\"data row85 col1\" >0.324922</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow85_col2\" class=\"data row85 col2\" >0.923975</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow85_col3\" class=\"data row85 col3\" >0.480776</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row86\" class=\"row_heading level0 row86\" >86</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow86_col0\" class=\"data row86 col0\" >0.86</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow86_col1\" class=\"data row86 col1\" >0.308101</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow86_col2\" class=\"data row86 col2\" >0.929326</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow86_col3\" class=\"data row86 col3\" >0.462776</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row87\" class=\"row_heading level0 row87\" >87</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow87_col0\" class=\"data row87 col0\" >0.87</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow87_col1\" class=\"data row87 col1\" >0.286172</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow87_col2\" class=\"data row87 col2\" >0.933987</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow87_col3\" class=\"data row87 col3\" >0.438108</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row88\" class=\"row_heading level0 row88\" >88</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow88_col0\" class=\"data row88 col0\" >0.88</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow88_col1\" class=\"data row88 col1\" >0.264944</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow88_col2\" class=\"data row88 col2\" >0.937633</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow88_col3\" class=\"data row88 col3\" >0.413147</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row89\" class=\"row_heading level0 row89\" >89</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow89_col0\" class=\"data row89 col0\" >0.89</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow89_col1\" class=\"data row89 col1\" >0.243216</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow89_col2\" class=\"data row89 col2\" >0.940744</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow89_col3\" class=\"data row89 col3\" >0.386506</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row90\" class=\"row_heading level0 row90\" >90</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow90_col0\" class=\"data row90 col0\" >0.9</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow90_col1\" class=\"data row90 col1\" >0.223591</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow90_col2\" class=\"data row90 col2\" >0.947391</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow90_col3\" class=\"data row90 col3\" >0.361795</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row91\" class=\"row_heading level0 row91\" >91</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow91_col0\" class=\"data row91 col0\" >0.91</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow91_col1\" class=\"data row91 col1\" >0.197757</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow91_col2\" class=\"data row91 col2\" >0.952266</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow91_col3\" class=\"data row91 col3\" >0.327502</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row92\" class=\"row_heading level0 row92\" >92</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow92_col0\" class=\"data row92 col0\" >0.92</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow92_col1\" class=\"data row92 col1\" >0.175328</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow92_col2\" class=\"data row92 col2\" >0.955265</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow92_col3\" class=\"data row92 col3\" >0.296277</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row93\" class=\"row_heading level0 row93\" >93</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow93_col0\" class=\"data row93 col0\" >0.93</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow93_col1\" class=\"data row93 col1\" >0.152398</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow93_col2\" class=\"data row93 col2\" >0.959042</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow93_col3\" class=\"data row93 col3\" >0.263003</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row94\" class=\"row_heading level0 row94\" >94</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow94_col0\" class=\"data row94 col0\" >0.94</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow94_col1\" class=\"data row94 col1\" >0.128968</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow94_col2\" class=\"data row94 col2\" >0.962631</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow94_col3\" class=\"data row94 col3\" >0.227461</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row95\" class=\"row_heading level0 row95\" >95</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow95_col0\" class=\"data row95 col0\" >0.95</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow95_col1\" class=\"data row95 col1\" >0.104736</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow95_col2\" class=\"data row95 col2\" >0.974837</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow95_col3\" class=\"data row95 col3\" >0.18915</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row96\" class=\"row_heading level0 row96\" >96</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow96_col0\" class=\"data row96 col0\" >0.96</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow96_col1\" class=\"data row96 col1\" >0.0790027</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow96_col2\" class=\"data row96 col2\" >0.980124</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow96_col3\" class=\"data row96 col3\" >0.146219</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row97\" class=\"row_heading level0 row97\" >97</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow97_col0\" class=\"data row97 col0\" >0.97</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow97_col1\" class=\"data row97 col1\" >0.0512666</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow97_col2\" class=\"data row97 col2\" >0.980843</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow97_col3\" class=\"data row97 col3\" >0.0974403</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row98\" class=\"row_heading level0 row98\" >98</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow98_col0\" class=\"data row98 col0\" >0.98</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow98_col1\" class=\"data row98 col1\" >0.0271353</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow98_col2\" class=\"data row98 col2\" >0.985455</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow98_col3\" class=\"data row98 col3\" >0.0528162</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row99\" class=\"row_heading level0 row99\" >99</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow99_col0\" class=\"data row99 col0\" >0.99</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow99_col1\" class=\"data row99 col1\" >0.00841093</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow99_col2\" class=\"data row99 col2\" >1</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow99_col3\" class=\"data row99 col3\" >0.0166816</td> \n",
       "    </tr>    <tr> \n",
       "        <th id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7blevel0_row100\" class=\"row_heading level0 row100\" >100</th> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow100_col0\" class=\"data row100 col0\" >1</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow100_col1\" class=\"data row100 col1\" >1.0013e-24</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow100_col2\" class=\"data row100 col2\" >1</td> \n",
       "        <td id=\"T_7e1d1bde_4769_11ea_96d8_c4b301c40d7brow100_col3\" class=\"data row100 col3\" >2.0002e-20</td> \n",
       "    </tr></tbody> \n",
       "</table> "
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x101a31550>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_prauc(frame, y, yhat, pos=1, neg=0, res=0.01):\n",
    "    \n",
    "    \"\"\" Calculates precision, recall, and f1 for a pandas dataframe of y \n",
    "        and yhat values.\n",
    "    \n",
    "    Args:\n",
    "        frame: Pandas dataframe of actual (y) and predicted (yhat) values.\n",
    "        y: Name of actual value column.\n",
    "        yhat: Name of predicted value column.\n",
    "        pos: Primary target value, default 1.\n",
    "        neg: Secondary target value, default 0.\n",
    "        res: Resolution by which to loop through cutoffs, default 0.01.\n",
    "    \n",
    "    Returns:\n",
    "        Pandas dataframe of precision, recall, and f1 values.\n",
    "        \n",
    "    \"\"\"\n",
    "    \n",
    "    frame_ = frame.copy(deep=True) # don't destroy original data\n",
    "    dname = 'd_' + str(y)          # column for predicted decisions\n",
    "    eps = 1e-20                    # for safe numerical operations\n",
    "    \n",
    "    # init p-r roc frame\n",
    "    prroc_frame = pd.DataFrame(columns=['cutoff', 'recall', 'precision', 'f1'])\n",
    "    \n",
    "    # loop through cutoffs to create p-r roc frame\n",
    "    for cutoff in np.arange(0, 1 + res, res):\n",
    "\n",
    "        # binarize decision to create confusion matrix values\n",
    "        frame_[dname] = np.where(frame_[yhat] > cutoff , 1, 0)\n",
    "        \n",
    "        # calculate confusion matrix values\n",
    "        tp = frame_[(frame_[dname] == pos) & (frame_[y] == pos)].shape[0]\n",
    "        fp = frame_[(frame_[dname] == pos) & (frame_[y] == neg)].shape[0]\n",
    "        tn = frame_[(frame_[dname] == neg) & (frame_[y] == neg)].shape[0]\n",
    "        fn = frame_[(frame_[dname] == neg) & (frame_[y] == pos)].shape[0]\n",
    "\n",
    "        # calculate precision, recall, and f1\n",
    "        recall = (tp + eps)/((tp + fn) + eps)\n",
    "        precision = (tp + eps)/((tp + fp) + eps)\n",
    "        f1 = 2/((1/(recall + eps)) + (1/(precision + eps)))\n",
    "        \n",
    "        # add new values to frame\n",
    "        prroc_frame = prroc_frame.append({'cutoff': cutoff,\n",
    "                                          'recall': recall,\n",
    "                                          'precision': precision,\n",
    "                                          'f1': f1}, \n",
    "                                          ignore_index=True)\n",
    "    \n",
    "    # housekeeping\n",
    "    del frame_\n",
    "    \n",
    "    return prroc_frame\n",
    "\n",
    "# calculate and display recall and precision\n",
    "#prauc_frame = get_prauc(test_yhat, y, yhat)\n",
    "prauc_frame = get_prauc(TEST, 'outcome', 'probability')\n",
    "prauc_frame.style.set_caption('Recall and Precision')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best F1 threshold: 0.38\n"
     ]
    }
   ],
   "source": [
    "# Find the maximum F1 cutoff for the accuracy calculation\n",
    "xnn_cut = prauc_frame.loc[prauc_frame['f1'].idxmax(), 'cutoff'] # value associated w/ index of max. F1\n",
    "print('Best F1 threshold: %.2f' % xnn_cut)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'AUC': 0.8497210910286438, 'accuracy_score': 0.7572, 'log_loss': 0.48037561622732866, 'rmse': 0.39754676883124707, 'mcc': 0.5253507888843505, 'F1': 0.7792125125034102}\n"
     ]
    }
   ],
   "source": [
    "# Calculate test statistics\n",
    "Prediction = list(TEST['probability'])\n",
    "Classification = list(TEST['probability'].apply(lambda x: int(x >= 0.38)))\n",
    "Actual = list(TEST['outcome'].apply(int))\n",
    "\n",
    "test_statistics = {}\n",
    "\n",
    "#classification_rep = classification_report(Actual, Classification, output_dict=True)\n",
    "classification_rep = classification_report(Actual, Classification, digits=3)\n",
    "\n",
    "test_statistics['AUC'] = roc_auc_score(Actual, Prediction)\n",
    "test_statistics['accuracy_score'] = accuracy_score(Actual, Classification)\n",
    "test_statistics['log_loss'] = log_loss(Actual, Prediction)\n",
    "test_statistics['rmse'] = math.sqrt(mean_squared_error(Actual, Prediction))\n",
    "test_statistics['mcc'] = matthews_corrcoef(Actual, Classification)\n",
    "#test_statistics['precision'] = classification_rep[1]['precision']\n",
    "#test_statistics['sensitivity'] = classification_rep[1]['recall']\n",
    "test_statistics['F1'] = f1_score(Actual, Classification)\n",
    "\n",
    "print(test_statistics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0      0.823     0.657     0.730     10013\n",
      "          1      0.714     0.858     0.779      9987\n",
      "\n",
      "avg / total      0.768     0.757     0.755     20000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# recall of the positive class is also known as “sensitivity”\n",
    "# recall of the negative class is “specificity”\n",
    "print(classification_rep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGX2wPHvoXdQsIIICoqgghAp9g4iiCsKWFBUxN7XVXf9rWXddVfXujYEFSsWVETFsqsgoiBFihQRBIQgIL0IAZKc3x/nxgwxZRIyc+dOzud58ky7M3NyM7ln7lvOK6qKc845V5RKYQfgnHMutXmicM45VyxPFM4554rlicI551yxPFE455wrlicK55xzxfJE4eImIheIyKdhx5FKRGSziBwQwvs2ExEVkSrJfu9EEJHZInJCGZ7nn8kk8EQRUSKyWES2BgeqFSIyTETqJPI9VfVVVT0tke8RS0SOEpHPRWSTiGwQkfdFpHWy3r+QeMaKyMDY+1S1jqouTND7HSQib4nI6uD3nykiN4tI5US8X1kFCavFrryGqrZR1bElvM/vkmOyP5MVlSeKaOupqnWAdsARwB0hx1MmhX0rFpEuwKfAe8C+QHNgBvBVIr7Bp9o3cxE5EPgGWAocpqr1gXOBDKBuOb9XaL97qu13VwRV9Z8I/gCLgVNibj8AfBhzuzrwb2AJsBJ4BqgZ83gvYDqwEfgR6BbcXx94DlgOLAPuAyoHjw0AxgfXnwb+XSCm94Cbg+v7Am8Dq4BFwPUx290NjABeCd5/YCG/35fAU4Xc/xHwUnD9BCAT+DOwOtgnF8SzD2KeexuwAngZ2A34IIh5XXC9SbD934EcIAvYDDwR3K9Ai+D6MOBJ4ENgE3agPzAmntOAecAG4Cngi8J+92DbV2L/noU83ix474uD32818JeYxzsCE4D1wd/yCaBazOMKXAPMBxYF9z2GJaaNwFTg2JjtKwf7+cfgd5sK7AeMC17r12C/9A2274F9vtYDXwOHF/js3gbMBLYBVYj5PAexTwniWAk8HNy/JHivzcFPF2I+k8E2bYD/AmuD5/457P/VdPgJPQD/KeMfbud/rCbAd8BjMY8/AowCdse+gb4P3B881jE4WJ2KnVU2BloFj70LDAZqA3sCk4Argsd++6cEjgsOKhLc3g3YiiWISsGB5K9ANeAAYCHQNdj2bmAHcFawbc0Cv1st7KB8YiG/9yXA8uD6CUA28DCWFI4PDlgHx7EP8p77r+C5NYGGQO/g/esCbwEjY957LAUO7Pw+UawJ9m8V4FXg9eCxRsGB7+zgsRuCfVBUolgBXFLM379Z8N5DgtjbYgfdQ4LHOwCdg/dqBswFbiwQ93+DfZOXPC8M9kEV4JYghhrBY7din7GDAQner2HBfRDcPgL4BeiEJZiLsc9r9ZjP7nQs0dSMuS/v8zwB6B9crwN0LvA7V4l5rwHkfybrYknxFqBGcLtT2P+r6fATegD+U8Y/nP1jbca+3SnwGdAgeEywA2bst9ku5H9zHAw8Ushr7hUcbGLPPM4DxgTXY/8pBfuGd1xw+3Lg8+B6J2BJgde+A3ghuH43MK6Y361J8Du1KuSxbsCO4PoJ2MG+dszjbwL/F8c+OAHYnncgLCKOdsC6mNtjKTlRDI15rDvwfXD9ImBCzGOCJdqiEsUOgrO8Ih7PO2g2iblvEtCviO1vBN4tEPdJJXzG1gFtg+vzgF5FbFcwUTwN/K3ANvOA42M+u5cW8nnOSxTjgHuARkX8zkUlivOAaYn8v6uoP94+GG1nqer/ROR44DXsW+t6YA/sW/FUEcnbVrBvd2Df5EYX8nr7A1WB5THPq4Qd0Haiqioir2P/nOOA87HmkrzX2VdE1sc8pTLWnJTnd68ZYx2QC+wDfF/gsX2wZpbftlXVX2Nu/4Sd1ZS0DwBWqWrWbw+K1MLOQrphZ0gAdUWksqrmFBNvrBUx17dg34gJYvrtdw72X2Yxr7MG+13L9H4ichB2ppWB7Ycq2FlerJ3+BiLyR+CyIFYF6mGfKbDPzI9xxAP2979YRK6Lua9a8LqFvncBlwH3At+LyCLgHlX9II73LU2MrhS8MzsNqOoX2LfZfwd3rcaagdqoaoPgp75axzfYP+mBhbzUUuyMolHM8+qpapsi3no4cI6I7I+dRbwd8zqLYl6jgarWVdXusWEX8/v8ijU/nFvIw32ws6c8u4lI7ZjbTYGf49gHhcVwC9a00klV62HNa2AJptiY47AcO1OyF7Ts1aTozfkf1gxWVk9jSbZl8Lv8mfzfI89vv4+IHAv8Cdu/u6lqA6x5Mu85RX1mCrMU+HuBv38tVR1e2HsXpKrzVfU8rOnzX8CI4G9c0v5fijVzunLmiSJ9PAqcKiJtVTUXa7t+RET2BBCRxiLSNdj2OeASETlZRCoFj7VS1eXYSKOHRKRe8NiBwRnL76jqNOyAPBT4RFXzziAmAZtE5DYRqSkilUXkUBE5shS/z+3Yt9LrRaSuiOwmIvdhzUf3FNj2HhGpFhzsegBvxbEPClMXSy7rRWR34K4Cj6+k7AeiD4HDROSsYKTPNcDexWx/F3CUiDwoInsH8bcQkVdEpEEc71cX6xPZLCKtgKvi2D4b68ivIiJ/xc4o8gwF/iYiLcUcLiINg8cK7pchwJUi0inYtraInCEicY3WEpELRWSP4G+Y95nKDWLLpei/wQfAPiJyo4hUDz43neJ5T1c8TxRpQlVXAS9hHchgo0oWABNFZCP2DfXgYNtJWKfwI9i3xi+w5gKwtvRqwBysCWgExTeBvAacElzmxZKDHbDbYSOe8pJJ/VL8PuOBrljn73KsSekI4BhVnR+z6Yogzp+xzuMrVTWvuarIfVCER7GO4dXARODjAo8/hp1BrRORx+P9XYLfZzV2hvQA1qzUGhvZs62I7X/EkmIzYLaIbMDO2KZg/VIl+SPWHLgJO3C/UcL2n2C/7w/Yvs5i5+ahh7H+n0+xBPQctq/A+pxeFJH1ItJHVadgfVZPYH+bBVhfQry6Yb/zZmyf91PVraq6BRt99lXwXp1jn6Sqm7ABGj2xz8V84MRSvK8rQt6IFeciJ5jJ+4qqFteEk5JEpBI2PPcCVR0TdjzOFcfPKJxLEhHpKiINRKQ6+X0GE0MOy7kSJSxRiMjzIvKLiMwq4nERkcdFZEFQmqB9omJxLkV0wUblrMaaR85S1a3hhuRcyRLW9CQix2Hj/F9S1UMLebw7cB021rwTNlnMO56ccy7FJOyMQlXHYdPoi9ILSyKqqhOBBiISz7hx55xzSRTmhLvG7DyqIjO4b3nBDUVkEDAIoHbt2h1atWqVlACdcy6WKuTmQnY2bN9ut7dtAxG7rmqP5ebafbm59nilSvZYTo795L1O3mN5r52IBp6m/EQD1jOT7NWqukdZXiMSM7NV9VngWYCMjAydMmVKyBE556Ji2zbYvBl++QW2brUD/LZtsHIlbNwIq1bBjh2wZo1tM38+1KsHWVmwdKkd0H/+ufTvKwJ160LVqrB2LRx6KNSsCfXrQ+3aUL26/axZAwcdlH9740Zo2dK2zc6GZs2gcmVo0MCSioj9FHsdteuVhbqvPE3lNb/Q4JG7fyrrPgwzUSzDptznaRLc55xzv5ObC5mZdtD+4Qf46Sc7mK9dCytWwLp1dsBXtfs3bICFpVgppGZN2GsvOyj/+is0aQLt2tnr9uhhSaZVK9tuxw444ABo1Ah2390SQs2adqCvVs0uq4R1dF22DK66Cvr2hQsugL8Ecy0fubvMLxlmohgFXBvUC+oEbAhmBjvnKoDsbDvIr1sH334Ls2ZZIgCYPdsOvllZ9g37hx9s+4KqVLFv7Dk5dtAWsW/njRpBixZw+un2WOvW9lijRvaNPu+AXrs2NG4Mu+1mrxNpqjB0KPzxj5bJzjij3F46YYlCRIZjFTobBcXP7sIKzqGqz2BF6bpjsza3YDOFnXMRt3YtLF5sTTrLlsGMGdbMk5lpB/6VK2HTJvvJzS38NQ491JqBDjkE9tzTmmLq1rXbhx8O++8PBx9sB3sH/PgjXH45jBkDJ54IQ4bAgfGW5ipZwhJFUNSruMcVq3fjnEthGzfmN+1s2mQH/PnzremnalVr/5871w7sq1YV/hp53/R37LADfW6ufctv1sy+4deqBV26wL772raulL77DqZOhWefhYEDy30nRqIz2zmXGNnZ9q0/MxMmTIB58+y++fNh2jRrzdhawpTAVq3sYL/33tamr2oH/b32svuaNLH2e1fOZs2yNruLLoKzzrIOmYYNS35eGXiicK4C2LzZksDkyTBlio20WbwYpk8vfPsOHeC44+xg37q1HX+qVrUmn+rV7XLvvf3bfyi2b4d//MN+9toL+vSBGjUSliTAE4VzaWX7dpg0yc4SZs2CcePsp6Dmze3nmmvsgL///tC+vfUFeLt/CvvmG7jsMuvtv/BCeOQRSxIJ5onCuYhRtQ7hBQvsDGHTJutD+PRT69OMtc8+0LWrnRVkZFhS6NLFm4IiadkyOPZYO4v44INyHdVUEk8UzqWw1autn/LTT+1yyxaYOdOajmLVq2dnCL17w2mnwZFHWkfxbrsV+rIuSn74wUYCNG4Mb7wBJ59sf/Ak8kThXIpYsQLGjoUvvoA5cyxJzJmz8zYNGkD37jZyqE0bSw4HHOBnCGlp/Xr4059sbsTYsdZp9Ic/hBKKJwrnkmz1amsyWrjQRhuNG2edylu27Lxdy5Y2oKVnT5s/cNBBaTApzMVn1CibXb1iBdx6q50ihsgThXMJtHq1/c9//DF89pnNEt6wYedtWreGk06yM4STT4bOnW1ymaugBg6E556Dww6D996zzqWQeaJwrhxt2AAjR9qglI8+spFHedq2tablvKaj5s1hv/2gTp3w4nUpIq9srIglhv33h9tuS5khaJ4onNsF339vzcfz58P778OiRfk1idq2hTvvtJpDZ5/tZwmuCEuXwpVXQr9+0L+/XU8xniicKwVVO1sYMcL6GJfF1Dtu3NhmJv/tb9bvWKtWeHG6CMjNhcGD7cwhJye0jup4eKJwrgRr18LLL1tz8Zgxdp8IHHWUDUpp0waOPjop855cupg/3/oixo2DU06xGk3Nm4cdVZE8UThXiHXr4NVXrd7Rq6/aQjf77GOtA02awPXXW/+Cc2UyZ45NiHn+eRgwIOVroXiicBXeihXW6fzTTzbh9csv7Swir3+xa1e44w5rTkrx/2eXymbMsHHQF18MvXrZ+OiIzIj0ROEqnF9/hXvvtf/bTz7Z+bFKlawp6ZxzrNbakUd6J7TbRdu2wX33wT//aaelfftaO2VEkgR4onAVwNq1NmT1jTdsXkNe+WywlSJr1oTjj7f5DIcd5pPaXDmaMMGK+M2da7MnH344kp1Znihc2lm92ia3ffCBnTV8913+Y7VqwdVX23DVE0/0piSXQMuW2TeQvfeG0aNtXdaI8kThIi9vktv//mdnDTt25D92wAH2ha5fPzjmmEh+mXNRM3eu1Vxp3BjefNOm20e8/dIThYuk9eut2fehh3a+v2tXWyq4Vy/7Mle9ejjxuQpo3Tq45RZ44QUb9nrssbbyXBrwROEi5Ztv4MYbYeJEu92smVU8OOMM63z2SW4uFO++a22aq1bZELmQi/iVN08ULqVt22bNSu+/byMLZ8+2kUlnnAE332zF9JwL1aWX2llEu3bw4Ye2VGCa8UThUs727bZWy2uvwf33599/4IHW1NS/P+yxR3jxObdTEb/Ona0m/B//mLZD5jxRuJSQkwOffw5PP20DRLZts/tr1cpPDrVrhxujc4DNzLziCjj/fBvyOmhQ2BElnCcKF6oxY+CVV+zMPe9L2kknWWI44AAbqVSpUrgxOgdYEb+nn4bbb7cP67nnhh1R0niicEm3fTs89hjcdRds3Wr3de9uCeKSS2D33cONz7nfmTfPiviNH2+Lkg8ebCMpKghPFC6p7rrLymcA7LmnzXG4995IVTNwFdG8eTaSYtgwa26qYDM1PVG4hMvKgv/8B+65x+osHXkkXHedNfFWrhx2dM4VYdo0G2p3ySVw5plWxK9Bg7CjCoUnCpcwM2bAI4/YbOmsLBsQMmCALQfs/Q4uZWVl2WnuAw/Y7OrzzrMp/RU0SQD4v6srd2PH2szodu3g9det/yEvWbzwgicJl8K++so+uPffb01M06d73Rf8jMKVk/Hj4e9/t6bcRYvsvl69rM9vr73Cjc25uCxbZpUiGze2+vOnnRZ2RCnDE4XbJbNnQ48esHix3a5RA/76Vyun0aZNqKE5F585c6zGfOPG8Pbblizq1Ak7qpTijQCuTNauhb/8Bdq2tS9i3brBjz/acNd77vEk4SJg7VrrNGvTxor4AfTs6UmiEH5G4Upl1SrroH76aavg2r69fQmrQEPKXTp4+2245hpYs8a+8XTsGHZEKc0ThYvL8uXw1FNW2hvgiCPgH/+wMwnnImXAAHjxRfuW8/HH1nntiuWJwhVr5UqbJDd4sN0++WQr892jR7hxOVcqsUX8jjrKFha65Rao4ofAeCS0j0JEuonIPBFZICK3F/J4UxEZIyLTRGSmiHRPZDyudJ56ylZxHDzYOqcnTLBV5DxJuEhZtMhGML30kt0eNAhuu82TRCkkLFGISGXgSeB0oDVwnoi0LrDZncCbqnoE0A94KlHxuPj9979w9NHWhNuggZ2dv/GGVVN2LjJycuDxx+HQQ22lq7yzCldqiTyj6AgsUNWFqrodeB3oVWAbBeoF1+sDPycwHleCNWtsKdHTTrMRg7162byIrl3Djsy5Upo715YiveEGWxN39mzrm3BlkshE0RhYGnM7M7gv1t3AhSKSCYwGrivshURkkIhMEZEpq1atSkSsFdrmzbbmyp57wqef2plEZqatLLfnnmFH51wZLFhg33JeftlWnWvaNOyIIi3seRTnAcNUtQnQHXhZRH4Xk6o+q6oZqpqxhy9tVq4mTrShrQ89ZIt0jR4NTzzhiwS5CJo6FZ5/3q737Gl9ExdeWOEqvSZCIhPFMmC/mNtNgvtiXQa8CaCqE4AaQKMExuRijBwJXbrYfIj334fvv4fTTw87KudKaetWW0yoUyf429+sqBhAvXrFP8/FLZGJYjLQUkSai0g1rLN6VIFtlgAnA4jIIVii8LalBNuwwb5o/eEP0KSJ9Uf4SCYXSePGWXmAf/3L+iCmTfMifgmQsEShqtnAtcAnwFxsdNNsEblXRM4MNrsFuFxEZgDDgQGqPjQhUZYtszUgGjSAV1+Ffv3sbP2gg8KOzLkyWLbMJvZkZ9u47aFDK3Qp8ESSqB2XMzIydMqUKWGHETkffAAXXGAd1z162Nrw3X3Wioui776Dww6z6x98YEX8vFOtRCIyVVUzyvLcsDuzXYJlZlpC6NkT9tjDyu2/954nCRdBq1dD//5w+OH5Rfx69PAkkQQ+NTFNrVkD115rCwcBtGoFkyd7YUwXQarw1lv2gV63zmrKdOoUdlQVip9RpJmtW22OUaNGliSOOcb6IebO9SThIurii6FvX9h/f/j2W7j7bqhePeyoKhQ/o0gjX3wBl11m60KcdhrccQeccELYUTlXBrFF/I4/3pqbbrzR6zOFxPd6mnjjDRvFBHaWfs454cbjXJktXAiXX25juC+5xL79uFB501MaePbZ/CTx9deeJFxE5eTAo4/aiKbJk6GSH55ShZ9RRNjWrfZla/hwW83xww+tGde5yJkzBy69FL75Bs44A555xmaDupTgiSKicnNtiOvYsVaG44MPYPfdw47KuTJatMg61157zU6PvT5TSvFzuwhavdpmU48da8v9fv21JwkXQZMnw5Ahdv2MM6xv4rzzPEmkIE8UETNjhq3DsmgRXH+91UBzLlK2bLG69p07w/335xfxq1s33LhckTxRRMiIETbPaMMGK23z2GP+5ctFzNixNtT1oYdsZJMX8YsETxQRkJNjk1LPPdeanKZNs/I2zkVKZiaceqpd//xz67CuXz/cmFxcPFFEwGWXwZNPWuf1V19ZOQ7nImPGDLts0sQKjc2c6d90IsYTRQpTtfWqX3wR+vSxkU3ejOsiY9Uqq2vfrp2VDQD7tlOrVrhxuVLzRJGiVq60Mhyffgq9e9tcCe+PcJGgah/Y1q2tY+2ee2wMt4usuOZRBCvUNVXVBQmOxwFLl1qn9fLlcOuttniXJwkXGf3728pYnTrBc8/ZbFAXaSUmChE5A3gYqAY0F5F2wF2q+odEB1cRLV5shfzWrLGziby+P+dSWm6ufZsRsf6HDh1s/HblymFH5spBPE1P9wKdgPUAqjodaJHIoCqql1+G5s3tjOL55z1JuIhYsMCWJH3hBbt92WVw002eJNJIPIlih6quL3BftNZPjYARI6xQZp06MGaMLVvqXErLzoZ//9uK+E2bBtWqhR2RS5B4EsVcEekDVBKR5iLyCDAxwXFVKP/4h82RaNkSliyB444LOyLnSjBrlnVQ33qrDc2bM8fKgru0FE+iuBboAOQC7wDbgBsSGVRF8vLLVq9p991h4kTYbbewI3IuDkuWwE8/2TKK774L++4bdkQugeIZ9dRVVW8Dbsu7Q0TOxpKG2wUzZ8JFF8ERR9hEVZ+k6lLaN9/Y5LlBg2w+xMKFvr5uBRHPGcWdhdz3l/IOpKJZuBCOPdauv/ceNGgQbjzOFenXX+Hmm62p6YEHYNs2u9+TRIVR5BmFiHQFugGNReThmIfqYc1QroyWLLEh5llZMGoU7Ldf2BE5V4TPP7fifQsXwlVXwT//CdWrhx2VS7Limp5+AWYBWcDsmPs3AbcnMqh0Nns2nH02rF0Lo0dbP6BzKSkz0z6gzZtbCQ4fZVFhFZkoVHUaME1EXlXVrCTGlLY++sjKcVSpYsuWepJwKWnaNOs4a9IE3n8fjj8eatYMOyoXonj6KBqLyOsiMlNEfsj7SXhkaeapp6z/Lzsbxo2Dbt3Cjsi5AlauhL59oX37/CJ+3bp5knBxJYphwAuAAKcDbwJvJDCmtPP223DNNXDAATaisF27sCNyLoYqvPKKFfEbORLuuw+OOirsqFwKiSdR1FLVTwBU9UdVvRNLGC4OixbZMsBgQ8732SfceJz7nfPPt0J+Bx8M06fbxJ6qVcOOyqWQeOZRbBORSsCPInIlsAzwVRHisGqVrUiXnW1D0I88MuyInAvEFvE77TQb+nrNNV6fyRUqnjOKm4DawPXA0cDlwKWJDCodbN9u/YHZ2VbZoGPHsCNyLvDDD1bh9fnn7fYll3ilV1esEhOFqn6jqptUdYmq9lfVM4HFiQ8turZvh4wMWLbMmntffjnsiJzDvrU88AC0bWtlAbyT2sWp2EQhIkeKyFki0ii43UZEXgK+SUp0EaQKJ50E330HV19tzb3OhW7mTOjcGW67DU4/3Yr4nX9+2FG5iCgyUYjI/cCrwAXAxyJyNzAGmAEclJToIiYz0yrAfvWV9Q0++WTYETkXyMy0hU7eesuG4fmoClcKxXVm9wLaqupWEdkdWAocpqoL431xEekGPAZUBoaq6j8L2aYPcDe2xsUMVY3k15yVK23Fx40b4a9/9TMJlwK+/trOJK68Mr+IX+3aYUflIqi4RJGlqlsBVHWtiPxQyiRRGXgSOBXIBCaLyChVnROzTUvgDuBoVV0nInuW6bcI2dq1thrdxo0wbBhcfHHYEbkKbfNm+6byn//AgQdaZ3X16p4kXJkVlygOEJG8UuKCrZf9W2lxVT27hNfuCCzISy4i8jp2ljInZpvLgSdVdV3wmr+UMv7QbdxoE1l/+gnuvtuThAvZp59aGfAlS2y46z/+4UX83C4rLlH0LnD7iVK+dmOsuSpPJrb2dqyDAETkK6x56m5V/bjgC4nIIGAQQNOmTUsZRmJdfLEliRdegAEDwo7GVWhLl8IZZ9hZxLhxcMwxYUfk0kRxRQE/S9L7twROAJoA40TksIJrdKvqs8CzABkZGSmzXvfs2VbxoGlTTxIuRFOnQocOVq9+9Ghb6KRGjbCjcmkkngl3ZbUMiF1poUlwX6xMYJSq7lDVRcAPWOJIeevWwaGH2oJDw4aFHY2rkFassMXWMzLyi/ideqonCVfuEpkoJgMtRaS5iFQD+gGjCmwzEjubIJircRAQd4d5mPIK+z31lE1ydS5pVOHFF62I3/vvWz+EF/FzCRRPrScARKS6qm6Ld3tVzRaRa4FPsP6H51V1tojcC0xR1VHBY6eJyBwgB7hVVdeU7ldIvssus77Co4/OL/jnXNL06wdvvmkfwKFDoVWrsCNyaU5Ui2/yF5GOwHNAfVVtKiJtgYGqel0yAiwoIyNDp0yZEsZbA1aNuX9/m+T65Ze2CJFzCRdbxO/FF2HTJpv6XymRjQIunYjIVFXNKMtz4/mUPQ70ANYAqOoMoEI2tuTNuD7wQFutzpOES4rvv7dlSJ97zm5ffDFce60nCZc08XzSKqnqTwXuy0lEMKls+3Yblg7w2WfWie1cQu3YYf0PbdtabaY6dcKOyFVQ8XwnXho0P2kw2/o6bHRShXLttTBjBjz8MOy/f9jRuLQ3fbrNqJ4+Hc45x2ZZ77132FG5CiqeRHEV1vzUFFgJ/C+4r8J45x0YMsSanW66KexoXIWwYoX9vP02nF1SEQTnEiuezuzdVXVtkuIpUbI7s1VtQt3y5VbTqV69pL21q2jGj7cifldfbbe3bIFatcKNyaWNRHdmTxaR0SJysYhUuCVQ33rLKjRff70nCZcgmzZZ2+axx8Kjj8K2YBS6JwmXIuJZ4e5A4D6gA/CdiIwUkX4JjywFzJ8PfftaZYS//z3saFxa+uQTm+L/1FNwww3w7bdexM+lnLjG16nq16p6PdAe2IgtaJT2/vEPu3zlFV810iXA0qXQo4edOYwfb2cTPrLJpaASE4WI1BGRC0TkfWASsApI+3oBY8daDacTT7Qh7M6VC1WYNMmu77efTciZNs1LcLiUFs8ZxSygM/CAqrZQ1VtUNe3XzP7zn+3ypZfCjcOlkeXLoXdv6NQpv4jfKad4ET+X8uIZHnuAquYmPJIUMmsWTJhg6780aRJ2NC7yVO309OabISsL/vUvq9PkXEQUmShE5CEXj0EwAAAeMUlEQVRVvQV4W0R+N4Y2jhXuIuv8YNXuO+4INw6XJvr0gREjbFTT0KFw0EFhR+RcqRR3RvFGcFnale0i7aef4LvvbLRTs2ZhR+MiKyfHCvhVqgQ9e8JJJ8EVV3h9JhdJRX5qVTXoceMQVf0s9gc4JDnhJd+gQXb5xz+GG4eLsLlz7ewhr4jfRRfBVVd5knCRFc8n99JC7rusvANJBRs32tr0Bx9si4Y5Vyo7dsB999mqVvPmQf36YUfkXLkoro+iL7YqXXMReSfmobrA+sKfFW15lRMefTTcOFwETZtmC6fPnGntlo8/DnvuGXZUzpWL4vooJmFrUDQBnoy5fxMwLZFBhWHRInj1VTjySOjWLexoXOSsXAmrV8PIkdCrV9jROFeuikwUqroIWIRVi017jz1mly++GG4cLkLGjbORD9dcY98uFizwKfwuLRXZRyEiXwSX60RkbczPOhFJmWqy5WHFCksUZ5wBh6RtN70rNxs3Wjvl8cdbE1NeET9PEi5NFdeZnbfcaSNgj5ifvNtp4/LL7fL//i/cOFwEjB4NbdrA4ME2gc6L+LkKoLjhsXmzsfcDKqtqDtAFuAKonYTYkmLhQvjgA+ja1SorOFekpUut/6F+ffj6a3joIaidNv8KzhUpnuGxI7FlUA8EXgBaAq8lNKok+s9/7DJv/oRzO1GFiRPt+n772fjpb7/1bxWuQoknUeSq6g7gbOA/qnoT0DixYSVHTo4NhW3VCv7wh7CjcSnn55/hrLOgS5f8In4nngjVqoUbl3NJFk+iyBaRc4H+wAfBfVUTF1LyjBhhl9ddZ9UWnAPsLGLoUGjd2s4g/v1vL+LnKrR4qsdeClyNlRlfKCLNgeGJDSvxcnKs9E7jxnbp3G/OOQfeecdGNQ0dCi1ahB2Rc6EqMVGo6iwRuR5oISKtgAWqGvmFQR94ADZssJpOlSuHHY0LXWwRv7POgtNOs+FwXp/JOUT1dxXEd95A5FjgZWAZIMDeQH9V/Srx4f1eRkaGTpkyZZdeQ9X+/xs3hiVL/FhQ4c2aBQMHwmWX5Y+Vdi7NiMhUVS1TFbt4DpGPAN1V9WhVPQo4A3isLG+WKl5/3S7PP9+TRIW2fTvccw+0bw8//gi77RZ2RM6lpHj6KKqp6py8G6o6V0QiPezjjjtsEu1dd4UdiQvN1KlWxG/WLPvG8OijsEdazSN1rtzEkyi+FZFngFeC2xcQ4aKAP/5oixMNHOhzpSq0NWtg/Xp4/33o0SPsaJxLafEkiiuB64E/Bbe/BP6TsIgS7MIL7dIXJqqAxoyxIn7XX2+d1fPnQ40aYUflXMorNlGIyGHAgcC7qvpAckJKnNmzbZLt2Wfb4kSugtiwAf70J3j2WZtdecUVVp/Jk4RzcSmueuyfsfIdFwD/FZHCVrqLlBdesEvvm6hA3n/fJs4NHWqnkVOnehE/50qpuDOKC4DDVfVXEdkDGA08n5ywEuOJJ6BePTjssLAjcUmxdCn07m1nESNH2qpUzrlSK25w6DZV/RVAVVeVsG3K++QTWzbg2GO9XEdaU7XKrpBfxG/KFE8Szu2C4g7+B4jIO8HPu8CBMbffKeZ5vxGRbiIyT0QWiMjtxWzXW0RURMo0GSQe99xjQ2Lzmp9cGsrMhDPPtLpMeUX8TjjBi/g5t4uKa3rqXeD2E6V5YRGpjK21fSqQCUwWkVGxczKC7eoCNwDflOb1S2PVKpgwAW64wYfKp6XcXBgyBG69FbKz4eGH4Zhjwo7KubRR3JrZn+3ia3fE6kItBBCR14FewJwC2/0N+Bdw6y6+X5FeC1bPOPnkRL2DC1Xv3tYHcdJJljAOOCDsiJxLK4nsd2gMLI25nUmBdSxEpD2wn6p+WNwLicggEZkiIlNWrVpV6kCeeAJatrQ1sV2ayM62MwmwRDFkCPzvf54knEuA0DqoRaQS8DBwS0nbquqzqpqhqhl7lLLtaPx4WLDAKkd7Xac0MXOmLSY0ZIjdvvBCm2rvoxScS4i4D50iUtrB58uw9bbzNAnuy1MXOBQYKyKLgc7AqPLu0P73v+3y5pvL81VdKLZts0kwHTpYHRbvcHIuKUpMFCLSUUS+A+YHt9uKSDwlPCYDLUWkeVBEsB8wKu9BVd2gqo1UtZmqNgMmAmeq6q7VEI8xeza8955VbGjUqLxe1YVi8mSr8nrvvXDeeTB3rk2xd84lXDxnFI8DPYA1AKo6AzixpCepajZwLfAJMBd4U1Vni8i9InJm2UOO36BBdjlwYDLezSXUunWweTOMHg0vvQQNG4YdkXMVRjwLF01S1Y4iMk1Vjwjum6GqbZMSYQGlWbioUiWbf1XCr+hS1eefWxG/G26w29u2efkN58oo0QsXLRWRjoCKSGURuRH4oSxvlkzLl1uCuOaasCNxpbZ+va00d/LJMHiwJQjwJOFcSOJJFFcBNwNNgZVYp/NViQyqPIwcaZdnnRVuHK6U3nvPivg9/7xVfPUifs6FrsT1KFT1F6wjOlKefNKasX2SXYQsWQLnnguHHAKjRkFGwiq6OOdKocREISJDgN+18qvqoIREVA5yc23E0zHH+ND6lKdqk12OPRaaNrVJc507e30m51JIPE1P/wM+C36+AvYEtiUyqF31/vt22bVruHG4EixZYtPljzsuv4jfccd5knAuxcTT9PRG7G0ReRkYn7CIysF/glkel18ebhyuCLm58MwzcNttdkbx+ONexM+5FBbPmtkFNQf2Ku9AytMXX9ik3b1SOsoK7OyzrdP61FNtedJmzcKOyDlXjHj6KNaR30dRCVgLFLm2RNiWLLF6cd6JnWKys21iS6VK0Lcv9OoFAwZ4J5JzEVBsohARAdqSX6MpV0uaoReyTz6xS292SiEzZsCll9of5corrQSHcy4yiu3MDpLCaFXNCX5SOkmATeQFqxvnQpaVBXfeacNcMzNh773Djsg5VwbxjHqaLiJHJDyScpCVZR3ZZ54J9euHHU0FN2kSHHEE/P3vcMEFVsTPZz86F0lFNj2JSJWgsN8R2DKmPwK/AoKdbLRPUoxxe/11uzyxxJKFLuE2boStW+Hjj32csnMRV1wfxSSgPZCUSq/l4Y1gIG+/yM0jTxOffmozHW+6CU45BebN8/IbzqWB4hKFAKjqj0mKZZetXAmNG3tTeNKtW2crQw0bBm3awNVXW4LwJOFcWiguUewhIkWuC6eqDycgnjLbvt0G11x9ddiRVDDvvGMleletgjvugL/+1ROEc2mmuERRGahDcGaR6kaMsAm/Rx8ddiQVyJIl1s536KG2oNARkRjz4JwrpeISxXJVvTdpkeyi2bPtskePcONIe6owbhwcf7wV8fv8c+jUCapWDTsy51yCFDc8NhJnEnneegtatIA6dcKOJI399BOcfjqccEJ+Eb9jjvEk4VyaKy5RRKYIRnY2LFsGBxwQdiRpKjcXnnjCOqrHj7fJKsceG3ZUzrkkKbLpSVXXJjOQXTF/PmzZAj17hh1JmjrrLKvd3rWrLU26//5hR+ScS6KyVI9NOZ9+apfNm4cbR1rZsQMqV7YifuedB+ecA/37exE/5yqgeEp4pLypU+3SZ2SXk2+/hY4dbc0IsERx0UWeJJyroNIiUXz7Ley+O9SqFXYkEbd1q82F6NgRVqyA/fYLOyLnXAqIfKJQtaGxrVuHHUnETZwI7drBP/8JF18Mc+Z4p49zDkiDPoolS+zSm5120a+/Wr/Ef/9rdZqccy4Q+USxeLFd+voTZfDxx3Y6dssttiTg999DtWphR+WcSzGRb3r6+mu7bNgw3DgiZc0aa146/XR48UUrlAWeJJxzhYp8opgzxy6PPDLcOCJB1YpitW4Nr71mq89NnuwJwjlXrMg3PY0ZAzVresHSuCxZAuefD4cfbpNP2rYNOyLnXARE+oxC1Up3HHJI2JGkMFUr3Ac2o3rsWBvh5EnCORenSCeKCRPs0pdiLsKiRXDaadZRnVfE76ijoErkTySdc0kU6UTx9tt22b9/uHGknJwceOwxWyfim2/g6ae9iJ9zrswi/dVyxgy79Bp1BfTqBR9+CN27WxkOn2HtnNsFkU4UkydbkvASROxcxK9/f6vPdP75vnOcc7ssoU1PItJNROaJyAIRub2Qx28WkTkiMlNEPhORuM8NVGHjRth33/KNOZKmTIGMDGtiAujbFy64wJOEc65cJCxRiEhl4EngdKA1cJ6IFKzINA3IUNXDgRHAA/G+/tpgtYyTI7O8UgJs3Qq33WZLka5a5W1wzrmESOQZRUdggaouVNXtwOtAr9gNVHWMqm4Jbk4EmsT74lOm2OVRR5VLrNEzYYINcX3gAbj0Upt56AuGO+cSIJF9FI2BpTG3M4FOxWx/GfBRYQ+IyCBgEEDTpk0Bq4INtk52hbR1qy1R+r//VfDTKudcoqVEZ7aIXAhkAMcX9riqPgs8C5CRkaGQPy1gn32SEmJqGD3aivjdeiucdBLMnQtVq4YdlXMuzSWy6WkZEDsus0lw305E5BTgL8CZqrot3hfPyrLLOnV2KcZoWL0aLrwQzjgDXn01v4ifJwnnXBIkMlFMBlqKSHMRqQb0A0bFbiAiRwCDsSTxS2le/KefoE2bcos1NanC669bjZI334S77oJJk7yIn3MuqRLW9KSq2SJyLfAJUBl4XlVni8i9wBRVHQU8CNQB3hIbyrlEVc+M5/WXLKkAX6iXLLFy4G3bwnPPwWGHhR2Rc64CSmgfhaqOBkYXuO+vMdfLvJTajh3QJO4xUhGiCp99ZqvM7b+/dcYceaRNpnPOuRBEstZTbi6sXJmGa1D8+KONYDr11Pze+s6dPUk450IVyUTx8892udtu4cZRbnJy4OGHrWlp6lQYPNiL+DnnUkZKDI8trbzlT9NmDkXPnvDRRzZh7umn07RNzTkXVZFMFHlnFMccE24cu2T7dlsXolIlGDDACvn16+f1mZxzKSeSTU+zZtllZL94T5oEHTrAU0/Z7T59rNqrJwnnXAqKZKLIO55Gbnjsli1wyy3QpQusWwcHHhh2RM45V6JINj3Nmwd77x12FKU0frzNiVi4EK64Av71L6hfP+yonHOuRJFMFBs2wF57hR1FKeUtLDRmDJxwQtjROOdc3CLZ9DRzJhx9dNhRxOH9960MOMCJJ1opcE8SzrmIiVyiyM0NO4I4rFply5CeeSYMH55fxK9KJE/gnHMVXOQSRXa2XbYuuFZeKlCF116zIn4jRsC998I333gRP+dcpEXuK+66dXaZkpVjlyyBSy6BI46wIn4pGaRzzpVO5M4o8tahOO64cOP4TW4ufPKJXd9/f/jyS/jqK08Szrm0EblEkddHUSkVIp8/31aa69YNxo2z+zp29CJ+zrm0kgqH21LZssW6AEKVnQ0PPgiHHw7Tp1szkxfxc86lqcj1UQDUrBlyAD16WHNTr15WhmPffUMOyLnUtGPHDjIzM8nKazN2CVejRg2aNGlC1XIsXRG5RLFjh7XuJN22bVYzpFIlGDgQLr0Uzj3X6zM5V4zMzEzq1q1Ls2bNEP9fSThVZc2aNWRmZtK8efNye93INT3l5EDDhkl+04kToX17ePJJu33OOVbIzz/4zhUrKyuLhg0bepJIEhGhYcOG5X4GF7lEAfblPil+/RVuugmOOgo2bYKWLZP0xs6lD08SyZWI/R25pidI0oJFX35pRfwWLYKrr4b774d69ZLwxs45l1oieUZRvXoS3iQ72/okvvjCmpw8STgXWSNHjkRE+P7773+7b+zYsfTo0WOn7QYMGMCIESMA64i//fbbadmyJe3bt6dLly589NFHuxzL/fffT4sWLTj44IP5JG8OVgGfffYZ7du3p127dhxzzDEsWLAAgGHDhrHHHnvQrl072rVrx9ChQ3c5nnhEMlHsuWeCXnjkSDtzACviN3t2Cs3sc86V1fDhwznmmGMYPnx43M/5v//7P5YvX86sWbP49ttvGTlyJJs2bdqlOObMmcPrr7/O7Nmz+fjjj7n66qvJycn53XZXXXUVr776KtOnT+f888/nvvvu++2xvn37Mn36dKZPn87AgQN3KZ54RbLpab/9yvkFV66E666Dt96yTutbbrH6TF7Ez7lyc+ONNu2oPLVrB48+Wvw2mzdvZvz48YwZM4aePXtyzz33lPi6W7ZsYciQISxatIjqQRPGXnvtRZ8+fXYp3vfee49+/fpRvXp1mjdvTosWLZg0aRJdunTZaTsRYePGjQBs2LCBfUMegh/JI2G5Hb9V4ZVX7BO8eTP8/e9w660RXDrPOVeU9957j27dunHQQQfRsGFDpk6dSocOHYp9zoIFC2jatCn14mhyvummmxgzZszv7u/Xrx+33377TvctW7aMzp07/3a7SZMmLFu27HfPHTp0KN27d6dmzZrUq1ePiRMn/vbY22+/zbhx4zjooIN45JFH2K/cvzn/XiQTRblVyFiyxOZEZGTY7OpWrcrphZ1zBZX0zT9Rhg8fzg033ADYwXv48OF06NChyNFBpR019Mgjj+xyjIW95ujRo+nUqRMPPvggN998M0OHDqVnz56cd955VK9encGDB3PxxRfz+eefl/v7FxTJRLFLZxR5RfxOP92K+H31lVV79fpMzqWdtWvX8vnnn/Pdd98hIuTk5CAiPPjggzRs2JB1eeWoY7Zv1KgRLVq0YMmSJWzcuLHEs4rSnFE0btyYpUuX/nY7MzOTxo0b77TNqlWrmDFjBp06dQKsT6Jbt24ANIyZRDZw4ED+9Kc/xbEXyoGqRuoHOujixVo28+apHnusKqiOHVvGF3HOxWvOnDmhvv/gwYN10KBBO9133HHH6RdffKFZWVnarFmz32JcvHixNm3aVNevX6+qqrfeeqsOGDBAt23bpqqqv/zyi7755pu7FM+sWbP08MMP16ysLF24cKE2b95cs7Ozd9pmx44d2rBhQ503b56qqg4dOlTPPvtsVVX9+eeff9vunXfe0U6dOhX6PoXtd2CKlvG4WzHOKLKz4aGH4K67rFDUCy/4aCbnKoDhw4dz22237XRf7969GT58OMcddxyvvPIKl1xyCVlZWVStWpWhQ4dSv359AO677z7uvPNOWrduTY0aNahduzb33nvvLsXTpk0b+vTpQ+vWralSpQpPPvkklYPWjO7duzN06FD23XdfhgwZQu/evalUqRK77bYbzz//PACPP/44o0aNokqVKuy+++4MGzZsl+KJl1iiiQ6RDF25ckrphsh27Qqffgpnn21zIvbeO2HxOefyzZ07l0NCL/dc8RS230VkqqpmlOX1InlGUadOHBtlZdnopcqVYdAg++ndO+GxOedcuonkhLsSR69+9ZUNsM4r4te7tycJ55wro0gmiiL7KDZvhuuvt0WEsrJSYIUj51zUmrejLhH7O5KJotBhzl98AYceCk88AddeC7NmwamnJj0251y+GjVqsGbNGk8WSaLBehQ1atQo19eNZB9FkWrVsqqvRx8ddiTOOWzmcWZmJqtWrQo7lAojb4W78hS5UU+VKmVobu4Uu/HOO/D99/DnP9vtnByfOOecc4XYlVFPCW16EpFuIjJPRBaIyO2FPF5dRN4IHv9GRJqV9JpVqwIrVtgqc717w7vvwvbt9qAnCeecK3cJSxQiUhl4EjgdaA2cJyKtC2x2GbBOVVsAjwD/Kul1d8tdY53UH3xgJcG//toqvTrnnEuIRJ5RdAQWqOpCVd0OvA70KrBNL+DF4PoI4GQpoSJX4+yfrNN6xgy4/Xav9OqccwmWyM7sxsDSmNuZQKeitlHVbBHZADQEVsduJCKDgEHBzW0yfvwsr/QKQCMK7KsKzPdFPt8X+Xxf5Du4rE+MxKgnVX0WeBZARKaUtUMm3fi+yOf7Ip/vi3y+L/KJyJSyPjeRTU/LgNgVNZoE9xW6jYhUAeoDaxIYk3POuVJKZKKYDLQUkeYiUg3oB4wqsM0o4OLg+jnA5xq18brOOZfmEtb0FPQ5XAt8AlQGnlfV2SJyL1YXfRTwHPCyiCwA1mLJpCTPJirmCPJ9kc/3RT7fF/l8X+Qr876I3IQ755xzyRXJWk/OOeeSxxOFc865YqVsokhE+Y+oimNf3Cwic0Rkpoh8JiL7hxFnMpS0L2K26y0iKiJpOzQynn0hIn2Cz8ZsEXkt2TEmSxz/I01FZIyITAv+T7qHEWeiicjzIvKLiMwq4nERkceD/TRTRNrH9cJlXWw7kT9Y5/ePwAFANWAG0LrANlcDzwTX+wFvhB13iPviRKBWcP2qirwvgu3qAuOAiUBG2HGH+LloCUwDdgtu7xl23CHui2eBq4LrrYHFYcedoH1xHNAemFXE492BjwABOgPfxPO6qXpGkZDyHxFV4r5Q1TGquiW4ORGbs5KO4vlcAPwNqxuWlczgkiyefXE58KSqrgNQ1V+SHGOyxLMvFKgXXK8P/JzE+JJGVcdhI0iL0gt4Sc1EoIGI7FPS66Zqoiis/EfjorZR1Wwgr/xHuolnX8S6DPvGkI5K3BfBqfR+qvphMgMLQTyfi4OAg0TkKxGZKCLdkhZdcsWzL+4GLhSRTGA0cF1yQks5pT2eABEp4eHiIyIXAhnA8WHHEgYRqQQ8DAwIOZRUUQVrfjoBO8scJyKHqer6UKMKx3nAMFV9SES6YPO3DlXV3LADi4JUPaPw8h/54tkXiMgpwF+AM1V1W5JiS7aS9kVd4FBgrIgsxtpgR6Vph3Y8n4tMYJSq7lDVRcAPWOJIN/Hsi8uANwFUdQJQAysYWNHEdTwpKFUThZf/yFfivhCRI4DBWJJI13ZoKGFfqOoGVW2kqs1UtRnWX3Omqpa5GFoKi+d/ZCR2NoGINMKaohYmM8gkiWdfLAFOBhCRQ7BEURHXZx0FXBSMfuoMbFDV5SU9KSWbnjRx5T8iJ8598SBQB3gr6M9foqpnhhZ0gsS5LyqEOPfFJ8BpIjIHyAFuVdW0O+uOc1/cAgwRkZuwju0B6fjFUkSGY18OGgX9MXcBVQFU9Rmsf6Y7sADYAlwS1+um4b5yzjlXjlK16ck551yK8EThnHOuWJ4onHPOFcsThXPOuWJ5onDOOVcsTxQu5YhIjohMj/lpVsy2zYqqlFnK9xwbVB+dEZS8OLgMr3GliFwUXB8gIvvGPDZURFqXc5yTRaRdHM+5UURq7ep7u4rLE4VLRVtVtV3Mz+Ikve8FqtoWKzb5YGmfrKrPqOpLwc0BwL4xjw1U1TnlEmV+nE8RX5w3Ap4oXJl5onCREJw5fCki3wY/RxWyTRsRmRSchcwUkZbB/RfG3D9YRCqX8HbjgBbBc08O1jD4Lqj1Xz24/5+SvwbIv4P77haRP4rIOVjNrVeD96wZnAlkBGcdvx3cgzOPJ8oY5wRiCrqJyNMiMkVs7Yl7gvuuxxLWGBEZE9x3mohMCPbjWyJSp4T3cRWcJwqXimrGNDu9G9z3C3CqqrYH+gKPF/K8K4HHVLUddqDODMo19AWODu7PAS4o4f17At+JSA1gGNBXVQ/DKhlcJSINgT8AbVT1cOC+2Cer6ghgCvbNv52qbo15+O3guXn6Aq+XMc5uWJmOPH9R1QzgcOB4ETlcVR/HSmqfqKonBqU87gROCfblFODmEt7HVXApWcLDVXhbg4NlrKrAE0GbfA5Wt6igCcBfRKQJ8I6qzheRk4EOwOSgvElNLOkU5lUR2QosxspQHwwsUtUfgsdfBK4BnsDWunhORD4APoj3F1PVVSKyMKizMx9oBXwVvG5p4qyGlW2J3U99RGQQ9n+9D7ZAz8wCz+0c3P9V8D7VsP3mXJE8UbiouAlYCbTFzoR/tyiRqr4mIt8AZwCjReQKbCWvF1X1jjje44LYAoIisnthGwW1hTpiRebOAa4FTirF7/I60Af4HnhXVVXsqB13nMBUrH/iP8DZItIc+CNwpKquE5FhWOG7ggT4r6qeV4p4XQXnTU8uKuoDy4P1A/pjxd92IiIHAAuD5pb3sCaYz4BzRGTPYJvdJf41xecBzUSkRXC7P/BF0KZfX1VHYwmsbSHP3YSVPS/Mu9hKY+dhSYPSxhkUtPs/oLOItMJWb/sV2CAiewGnFxHLRODovN9JRGqLSGFnZ879xhOFi4qngItFZAbWXPNrIdv0AWaJyHRsXYqXgpFGdwKfishM4L9Ys0yJVDULq675loh8B+QCz2AH3Q+C1xtP4W38w4Bn8jqzC7zuOmAusL+qTgruK3WcQd/HQ1hV2BnY+tjfA69hzVl5ngU+FpExqroKG5E1PHifCdj+dK5IXj3WOedcsfyMwjnnXLE8UTjnnCuWJwrnnHPF8kThnHOuWJ4onHPOFcsThXPOuWJ5onDOOVes/wd9Foi0P51WjwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, threshold = roc_curve(Actual, Prediction)\n",
    "roc_auc = auc(fpr, tpr)\n",
    "\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "plt.legend(loc = 'lower right')\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}