analysis-jupyter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import analysis_relabel_funcs\n",
    "from analysis import read_dfs\n",
    "import torch\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from scipy.stats import chi2_contingency\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "01:05:10-conv_ai_3-train-eval_sbc.out\r\n",
      "01:05:10-conv_ai_3-train-partial-predictions-eval_sbc.out\r\n",
      "01:26:28-conv_ai_3-validation-eval_sbc.out\r\n",
      "01:26:28-conv_ai_3-validation-partial-predictions-eval_sbc.out\r\n",
      "02:05:13-conv_ai_3-train-eval_rtgender.out\r\n",
      "02:05:13-conv_ai_3-train-partial-predictions-eval_rtgender.out\r\n",
      "02:27:05-conv_ai_3-validation-eval_rtgender.out\r\n",
      "02:27:05-conv_ai_3-validation-partial-predictions-eval_rtgender.out\r\n",
      "02:32:58-ted_talks_iwslt-train-eval_rtgender.out\r\n",
      "02:32:58-ted_talks_iwslt-train-partial-predictions-eval_rtgender.out\r\n",
      "02:39:43-ted_talks_iwslt-train-eval_rtgender.out\r\n",
      "02:39:43-ted_talks_iwslt-train-partial-predictions-eval_rtgender.out\r\n",
      "02:47:59-ted_talks_iwslt-train-eval_sbc.out\r\n",
      "02:47:59-ted_talks_iwslt-train-partial-predictions-eval_sbc.out\r\n",
      "02:54:48-ted_talks_iwslt-train-eval_sbc.out\r\n",
      "02:54:48-ted_talks_iwslt-train-partial-predictions-eval_sbc.out\r\n"
     ]
    }
   ],
   "source": [
    "!ls results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for using sbc and rt_gender for conv_ai\n",
    "sbc_conv, rt_gender_conv, sbc_rt_conv = read_dfs(\"results/01:05:10-conv_ai_3-train-eval_sbc.out\", \n",
    "                                                \"results/02:05:13-conv_ai_3-train-eval_rtgender.out\",\n",
    "                                                suffixes=('_sbf', '_rt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>predictions_sbf</th>\n",
       "      <th>predictions_rt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yes am interested in obamas family</td>\n",
       "      <td>[tensor(1.0437), tensor(-0.6024), tensor(0.1484)]</td>\n",
       "      <td>[tensor(-0.6332), tensor(0.6919), tensor(-0.84...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yes i want to know who made up his family</td>\n",
       "      <td>[tensor(1.8043), tensor(-0.6631), tensor(-0.11...</td>\n",
       "      <td>[tensor(0.0839), tensor(0.8344), tensor(-0.699...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yes this is what am looking for</td>\n",
       "      <td>[tensor(2.2968), tensor(-0.6913), tensor(-0.45...</td>\n",
       "      <td>[tensor(0.1760), tensor(1.0104), tensor(-0.106...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>no find the barack obamas family tree time mag...</td>\n",
       "      <td>[tensor(1.2287), tensor(-0.5359), tensor(0.0472)]</td>\n",
       "      <td>[tensor(-0.6308), tensor(0.5328), tensor(-0.92...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>no i need the specific time magazine article t...</td>\n",
       "      <td>[tensor(1.1137), tensor(-0.4908), tensor(0.1666)]</td>\n",
       "      <td>[tensor(-0.0311), tensor(1.0237), tensor(-0.65...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  \\\n",
       "0                yes am interested in obamas family    \n",
       "1         yes i want to know who made up his family    \n",
       "2                   yes this is what am looking for    \n",
       "3  no find the barack obamas family tree time mag...   \n",
       "4  no i need the specific time magazine article t...   \n",
       "\n",
       "                                     predictions_sbf  \\\n",
       "0  [tensor(1.0437), tensor(-0.6024), tensor(0.1484)]   \n",
       "1  [tensor(1.8043), tensor(-0.6631), tensor(-0.11...   \n",
       "2  [tensor(2.2968), tensor(-0.6913), tensor(-0.45...   \n",
       "3  [tensor(1.2287), tensor(-0.5359), tensor(0.0472)]   \n",
       "4  [tensor(1.1137), tensor(-0.4908), tensor(0.1666)]   \n",
       "\n",
       "                                      predictions_rt  \n",
       "0  [tensor(-0.6332), tensor(0.6919), tensor(-0.84...  \n",
       "1  [tensor(0.0839), tensor(0.8344), tensor(-0.699...  \n",
       "2  [tensor(0.1760), tensor(1.0104), tensor(-0.106...  \n",
       "3  [tensor(-0.6308), tensor(0.5328), tensor(-0.92...  \n",
       "4  [tensor(-0.0311), tensor(1.0237), tensor(-0.65...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sbc_rt_conv[[\"sentence\", \"predictions_sbf\", \"predictions_rt\"]].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for each prediction in predictions_sbc and predictions_rt, \n",
    "# return the categorical variable and the score using the relabel_funcs\n",
    "\n",
    "s = sbc_rt_conv[\"predictions_sbf\"].map(analysis_relabel_funcs.return_social_bias_frames)\n",
    "s = pd.DataFrame(s.tolist(), columns=[\"scores_sbf\", \"category_sbf\"])\n",
    "s[\"scores_sbf\"] = s[\"scores_sbf\"].map(lambda x: x.item()) # comes back as a tensor, change it to float\n",
    "sbc_rt_conv = sbc_rt_conv.join(s)\n",
    "\n",
    "# do the same thing, but for rt_gender instead \n",
    "s = sbc_rt_conv[\"predictions_rt\"].map(analysis_relabel_funcs.return_rt_gender)\n",
    "s = pd.DataFrame(s.tolist(), columns=[\"scores_rt\", \"category_rt\"])\n",
    "s[\"scores_rt\"] = s[\"scores_rt\"].map(lambda x: x.item()) # comes back as a tensor, change it to float\n",
    "sbc_rt_conv = sbc_rt_conv.join(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>predictions_sbf</th>\n",
       "      <th>predictions_rt</th>\n",
       "      <th>scores_sbf</th>\n",
       "      <th>category_sbf</th>\n",
       "      <th>scores_rt</th>\n",
       "      <th>category_rt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yes am interested in obamas family</td>\n",
       "      <td>[tensor(1.0437), tensor(-0.6024), tensor(0.1484)]</td>\n",
       "      <td>[tensor(-0.6332), tensor(0.6919), tensor(-0.84...</td>\n",
       "      <td>-0.369400</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.016630</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yes i want to know who made up his family</td>\n",
       "      <td>[tensor(1.8043), tensor(-0.6631), tensor(-0.11...</td>\n",
       "      <td>[tensor(0.0839), tensor(0.8344), tensor(-0.699...</td>\n",
       "      <td>-0.691929</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.106754</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yes this is what am looking for</td>\n",
       "      <td>[tensor(2.2968), tensor(-0.6913), tensor(-0.45...</td>\n",
       "      <td>[tensor(0.1760), tensor(1.0104), tensor(-0.106...</td>\n",
       "      <td>-0.840425</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.050733</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>no find the barack obamas family tree time mag...</td>\n",
       "      <td>[tensor(1.2287), tensor(-0.5359), tensor(0.0472)]</td>\n",
       "      <td>[tensor(-0.6308), tensor(0.5328), tensor(-0.92...</td>\n",
       "      <td>-0.468977</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.022241</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>no i need the specific time magazine article t...</td>\n",
       "      <td>[tensor(1.1137), tensor(-0.4908), tensor(0.1666)]</td>\n",
       "      <td>[tensor(-0.0311), tensor(1.0237), tensor(-0.65...</td>\n",
       "      <td>-0.385269</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.074436</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  \\\n",
       "0                yes am interested in obamas family    \n",
       "1         yes i want to know who made up his family    \n",
       "2                   yes this is what am looking for    \n",
       "3  no find the barack obamas family tree time mag...   \n",
       "4  no i need the specific time magazine article t...   \n",
       "\n",
       "                                     predictions_sbf  \\\n",
       "0  [tensor(1.0437), tensor(-0.6024), tensor(0.1484)]   \n",
       "1  [tensor(1.8043), tensor(-0.6631), tensor(-0.11...   \n",
       "2  [tensor(2.2968), tensor(-0.6913), tensor(-0.45...   \n",
       "3  [tensor(1.2287), tensor(-0.5359), tensor(0.0472)]   \n",
       "4  [tensor(1.1137), tensor(-0.4908), tensor(0.1666)]   \n",
       "\n",
       "                                      predictions_rt  scores_sbf category_sbf  \\\n",
       "0  [tensor(-0.6332), tensor(0.6919), tensor(-0.84...   -0.369400          Not   \n",
       "1  [tensor(0.0839), tensor(0.8344), tensor(-0.699...   -0.691929          Not   \n",
       "2  [tensor(0.1760), tensor(1.0104), tensor(-0.106...   -0.840425          Not   \n",
       "3  [tensor(-0.6308), tensor(0.5328), tensor(-0.92...   -0.468977          Not   \n",
       "4  [tensor(-0.0311), tensor(1.0237), tensor(-0.65...   -0.385269          Not   \n",
       "\n",
       "   scores_rt category_rt  \n",
       "0  -0.016630    Positive  \n",
       "1  -0.106754     Neutral  \n",
       "2  -0.050733     Neutral  \n",
       "3  -0.022241    Positive  \n",
       "4  -0.074436     Neutral  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sbc_rt_conv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>predictions_sbf</th>\n",
       "      <th>predictions_rt</th>\n",
       "      <th>scores_sbf</th>\n",
       "      <th>category_sbf</th>\n",
       "      <th>scores_rt</th>\n",
       "      <th>category_rt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>936577</th>\n",
       "      <td>possibly specifically i need information on he...</td>\n",
       "      <td>[tensor(1.3039), tensor(-0.5613), tensor(0.1189)]</td>\n",
       "      <td>[tensor(0.6306), tensor(0.4677), tensor(-0.486...</td>\n",
       "      <td>-0.475317</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.226670</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>937993</th>\n",
       "      <td>no i want general information about the califo...</td>\n",
       "      <td>[tensor(2.0380), tensor(-0.6950), tensor(-0.31...</td>\n",
       "      <td>[tensor(0.8506), tensor(0.8291), tensor(-0.163...</td>\n",
       "      <td>-0.780087</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.232890</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>937995</th>\n",
       "      <td>no i want to know any information about the ca...</td>\n",
       "      <td>[tensor(1.9638), tensor(-0.6826), tensor(-0.27...</td>\n",
       "      <td>[tensor(0.8496), tensor(0.7869), tensor(-0.123...</td>\n",
       "      <td>-0.758711</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.229701</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940655</th>\n",
       "      <td>no i am trying to figure out who can be burrie...</td>\n",
       "      <td>[tensor(2.0287), tensor(-0.6655), tensor(-0.31...</td>\n",
       "      <td>[tensor(0.9473), tensor(0.7669), tensor(-0.060...</td>\n",
       "      <td>-0.777120</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.252575</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940667</th>\n",
       "      <td>i would like to know who can be buried at culp...</td>\n",
       "      <td>[tensor(2.0564), tensor(-0.7147), tensor(-0.34...</td>\n",
       "      <td>[tensor(0.7529), tensor(0.4704), tensor(-0.369...</td>\n",
       "      <td>-0.787959</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.250786</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947120</th>\n",
       "      <td>i am interested in the civil rights movement</td>\n",
       "      <td>[tensor(1.1974), tensor(-0.5314), tensor(0.2618)]</td>\n",
       "      <td>[tensor(0.5889), tensor(0.3434), tensor(-0.464...</td>\n",
       "      <td>-0.387076</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.227977</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947258</th>\n",
       "      <td>yes and the homepage for the supporters of the...</td>\n",
       "      <td>[tensor(2.3197), tensor(-0.7113), tensor(-0.48...</td>\n",
       "      <td>[tensor(0.5934), tensor(0.4272), tensor(-0.416...</td>\n",
       "      <td>-0.846790</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.207520</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947350</th>\n",
       "      <td>yes i would like information on the restrictio...</td>\n",
       "      <td>[tensor(2.4708), tensor(-0.7096), tensor(-0.48...</td>\n",
       "      <td>[tensor(0.7698), tensor(0.7047), tensor(-0.215...</td>\n",
       "      <td>-0.867169</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.224905</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947376</th>\n",
       "      <td>i need information about the transportation se...</td>\n",
       "      <td>[tensor(2.0462), tensor(-0.7196), tensor(-0.30...</td>\n",
       "      <td>[tensor(0.9966), tensor(0.6159), tensor(-0.254...</td>\n",
       "      <td>-0.781017</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.305009</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947783</th>\n",
       "      <td>yes as well as the full names of all our presi...</td>\n",
       "      <td>[tensor(2.0165), tensor(-0.7443), tensor(-0.29...</td>\n",
       "      <td>[tensor(0.6838), tensor(0.4934), tensor(-0.361...</td>\n",
       "      <td>-0.775599</td>\n",
       "      <td>Not</td>\n",
       "      <td>-0.228233</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>143 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 sentence  \\\n",
       "936577  possibly specifically i need information on he...   \n",
       "937993  no i want general information about the califo...   \n",
       "937995  no i want to know any information about the ca...   \n",
       "940655  no i am trying to figure out who can be burrie...   \n",
       "940667  i would like to know who can be buried at culp...   \n",
       "...                                                   ...   \n",
       "947120      i am interested in the civil rights movement    \n",
       "947258  yes and the homepage for the supporters of the...   \n",
       "947350  yes i would like information on the restrictio...   \n",
       "947376  i need information about the transportation se...   \n",
       "947783  yes as well as the full names of all our presi...   \n",
       "\n",
       "                                          predictions_sbf  \\\n",
       "936577  [tensor(1.3039), tensor(-0.5613), tensor(0.1189)]   \n",
       "937993  [tensor(2.0380), tensor(-0.6950), tensor(-0.31...   \n",
       "937995  [tensor(1.9638), tensor(-0.6826), tensor(-0.27...   \n",
       "940655  [tensor(2.0287), tensor(-0.6655), tensor(-0.31...   \n",
       "940667  [tensor(2.0564), tensor(-0.7147), tensor(-0.34...   \n",
       "...                                                   ...   \n",
       "947120  [tensor(1.1974), tensor(-0.5314), tensor(0.2618)]   \n",
       "947258  [tensor(2.3197), tensor(-0.7113), tensor(-0.48...   \n",
       "947350  [tensor(2.4708), tensor(-0.7096), tensor(-0.48...   \n",
       "947376  [tensor(2.0462), tensor(-0.7196), tensor(-0.30...   \n",
       "947783  [tensor(2.0165), tensor(-0.7443), tensor(-0.29...   \n",
       "\n",
       "                                           predictions_rt  scores_sbf  \\\n",
       "936577  [tensor(0.6306), tensor(0.4677), tensor(-0.486...   -0.475317   \n",
       "937993  [tensor(0.8506), tensor(0.8291), tensor(-0.163...   -0.780087   \n",
       "937995  [tensor(0.8496), tensor(0.7869), tensor(-0.123...   -0.758711   \n",
       "940655  [tensor(0.9473), tensor(0.7669), tensor(-0.060...   -0.777120   \n",
       "940667  [tensor(0.7529), tensor(0.4704), tensor(-0.369...   -0.787959   \n",
       "...                                                   ...         ...   \n",
       "947120  [tensor(0.5889), tensor(0.3434), tensor(-0.464...   -0.387076   \n",
       "947258  [tensor(0.5934), tensor(0.4272), tensor(-0.416...   -0.846790   \n",
       "947350  [tensor(0.7698), tensor(0.7047), tensor(-0.215...   -0.867169   \n",
       "947376  [tensor(0.9966), tensor(0.6159), tensor(-0.254...   -0.781017   \n",
       "947783  [tensor(0.6838), tensor(0.4934), tensor(-0.361...   -0.775599   \n",
       "\n",
       "       category_sbf  scores_rt category_rt  \n",
       "936577          Not  -0.226670    Negative  \n",
       "937993          Not  -0.232890    Negative  \n",
       "937995          Not  -0.229701    Negative  \n",
       "940655          Not  -0.252575    Negative  \n",
       "940667          Not  -0.250786    Negative  \n",
       "...             ...        ...         ...  \n",
       "947120          Not  -0.227977    Negative  \n",
       "947258          Not  -0.207520    Negative  \n",
       "947350          Not  -0.224905    Negative  \n",
       "947376          Not  -0.305009    Negative  \n",
       "947783          Not  -0.228233    Negative  \n",
       "\n",
       "[143 rows x 7 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sbc_rt_conv[(sbc_rt_conv[\"category_sbf\"] == \"Not\") & \n",
    "            (sbc_rt_conv[\"category_rt\"] == \"Negative\") ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>scores_sbf</th>\n",
       "      <th>scores_rt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>scores_sbf</th>\n",
       "      <td>1.000001</td>\n",
       "      <td>-0.787093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scores_rt</th>\n",
       "      <td>-0.787093</td>\n",
       "      <td>1.000001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            scores_sbf  scores_rt\n",
       "scores_sbf    1.000001  -0.787093\n",
       "scores_rt    -0.787093   1.000001"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# normalize, take the scores of sbf and rt and do a covariance\n",
    "std_scaler = StandardScaler()\n",
    "scores_df = sbc_rt_conv[[\"scores_sbf\", \"scores_rt\"]]\n",
    "scores_df = pd.DataFrame(std_scaler.fit_transform(scores_df), columns=scores_df.columns)\n",
    "scores_df.cov()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value is 1.520037777915489e-211\n"
     ]
    }
   ],
   "source": [
    "# take the categories and do a chi2 test\n",
    "# first get the contingency table\n",
    "contingency = pd.crosstab(sbc_rt_conv[\"category_sbf\"], sbc_rt_conv[\"category_rt\"])\n",
    "c, p, dof, expected = chi2_contingency(contingency)\n",
    "print(f\"p-value is {p}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>category_rt</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Neutral</th>\n",
       "      <th>Positive</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_sbf</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Not</th>\n",
       "      <td>143</td>\n",
       "      <td>573686</td>\n",
       "      <td>1530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yes</th>\n",
       "      <td>7</td>\n",
       "      <td>372406</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "category_rt   Negative  Neutral  Positive\n",
       "category_sbf                             \n",
       "Not                143   573686      1530\n",
       "Yes                  7   372406        38"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f972abc0730>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAFzCAYAAABilkCnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZhcVZ3/8fe3sxGWhASIQNghyKYgIIZFhkUgLE4QRRQRZHAiDorI6E+QmQFRZHFBcUGiIIswEEENCAIRBFQQCEjCKkS2BMKWRAIkhKTz/f1RN5m6sbvTnVuVTjfv1/Pcp6rOXc7pSqX60+eee09kJpIkSY3Q0t0NkCRJvYfBQpIkNYzBQpIkNYzBQpIkNYzBQpIkNUzf7m5Axx73khX9k4EbnNrdTdAKZu6zX+vuJmiFtXk0u4aBG3y80u+quc/+b9PbuDzZYyFJkhpmBe+xkCRpxRbh3+j1DBaSJFUQdv6X+G5IkqSGscdCkqQKPBVSZrCQJKkCg0WZwUKSpAoietXVopUZLCRJqsQei3q+G5IkqWHssZAkqQLHWJQZLCRJqsBgUWawkCSpAm+QVWawkCSpAnssynw3JElSw9hjIUlSBfZYlBksJEmqwGBRZrCQJKmCwDtv1jNmSZKkhrHHQpKkCjwVUmawkCSpAoNFmcFCkqQKDBZlBgtJkioxWNTz3ZAkSQ1jj4UkSRV4KqTMYCFJUgUGizKDhSRJFTi7aZnBQpKkCuyxKPPdkCRJDWOPhSRJFUQ4V0g9g4UkSRV4KqTMYCFJUgUO3izz3ZAkSQ1jj4UkSRV4KqTMYCFJUgUGizKDhSRJFTjGosxgIUlSFfZYlPhuSJKkhrHHQpKkChxjUea7IUlSBRFRaelkHU9HxIMR8UBETCzKhkbEhIh4ongcUpRHRJwXEVMiYnJEbF93nKOK7Z+IiKPqyncojj+l2Dc6qqMjBgtJkioIWiotXbBnZm6XmTsWr08CbsnMEcAtxWuA/YERxTIGOB9qIQE4FXgfsBNwal1QOL/YdtF+o5ZSR7sMFpIk9UyjgUuK55cAB9eVX5o1fwFWj4h1gP2ACZk5MzNnAROAUcW6QZl5V2YmcOkSx2qrjnYZLCRJqiCipeISYyJiYt0ypo1qErg5Iu6rW/+OzJwOUDwOK8qHA1Pr9p1WlHVUPq2N8o7qaJeDNyVJqqLi7KaZORYYu5TNds3M5yNiGDAhIh7rqEVtVbMM5cvEHgtJkqpoqbh0QmY+Xzy+BPya2hiJF4vTGBSPLxWbTwPWr9t9PeD5pZSv10Y5HdTRLoOFJElVRFRblnr4WCUiVlv0HNgXeAi4Flh0ZcdRwPji+bXAkcXVISOBV4vTGDcB+0bEkGLQ5r7ATcW61yJiZHE1yJFLHKutOtrlqRBJklZs7wB+XVwB2he4IjNvjIh7gXERcQzwLHBosf0NwAHAFGAOcDRAZs6MiK8D9xbbnZ6ZM4vnnwUuBgYCvysWgLPaqaNdBgtJkqqoOMZiaTLzSWDbNspnAHu3UZ7Ace0c6yLgojbKJwLbdLaOjhgsJEmqwkEFJQYLSZIqyCb3WPQ0BgtJkqowV5TYgSNJkhrGHgtJkqposcuinsFCkqQqHGNRYrCQJKkKc0WJYywkSVLD2GMhSVIVjrEoMVhIklSFYyxKDBaSJFVhrigxWEiSVIWnQkocvClJkhrGHgtJkqqww6LEYCFJUgVOQlZmsJAkqQrHWJQYLCRJqsJcUeLgTUmS1DD2WEiSVIVjLEoMFpIkVeEYixKDhSRJVZgrShxjIUmSGsYeC0mSqnCMRYnBQpKkKgwWJQYLSZKqcFBBicFCkqQq7LEoMWdJkqSGscdCkqQq7LAoMVhIklRBeoOsEoOFJElVOMaixGCxgjj55O9z2233ssYag/ntb39UWnfhhb/inHN+zl13/YKhQwfz+9//he9//3JaWoI+ffrw1a9+mh133LqbWq5l9difz+O1N+bS2rqQBa0L2e2gU7jsR8czYpN1AFh90Cr8Y/YbjNz/ZHbcdlN+eNanAYgIzjj3aq69aSIAgwetzPnnjGGrzdcjE4798gXcff8TvHurDfnBN49hwIB+LGhdyAmnXMTESX8H4P0jt+Rbpx5Jv359mTHzNfb96Ond8yaoU9r6fvjBD65g3LibGDp0MAAnnngk//IvOzJ58uP893//EIDM5POfP5x99tkZgDvuuI8zzvgpCxcu5NBD92HMmEO75wfqbcwVJQaLFcQhh+zNEUccyFe+cm6pfPr0l7nzzgdYd921FpftvPO27L33+4gIHnvsKU444WxuvPEny7vJaoBRh32DGbNeW/z6k8edt/j5Wf91BK++NgeAh/82lV0POoXW1oWsPWx17r7xLK7//f20ti7k26cdxc23TeLwY79Hv359WHngAADO+OrhnPG9a7j5tknst+d2nPHVw9nvsK8zeNDKfP+Mf2P0J89i6vMzWGuNQcv3h1aXtff98KlPjeaYYw4plY0YsQHXXHMuffv24aWXZjJ69PHsuedORMDpp/+En//867zjHWvwkY+cyF57vY/NNttgef4oehto+lUhEbFxZ8re7t773m0YPHi1fyo/88yf8eUvH03UdbWtssrAxa/nzp1XWqfe48MHjWTc+DsBmPvmW7S2LgRgwIB+ZNa2WW3Vgey20xZcfOUfAJg/v5VXZ9fCSGYyaLWBAAxebWWmvzgLgMNG78r4393L1OdnAPDyjNnL7WfSsmnv+6EtAweuRN++fQCYN++txd8Pkyc/wYYbrsP6669N//79OPDA3bnllrub1ua3lZaotvQyy6PH4hpg+yXKrgZ2WA5192i33HI3w4atwRZb/HMOmzDhLr7znUuYOfNVLrjg1G5onarKTK77xckkyYWX38JFV9y6eN2uO23Bi6+8yt+ffmFx2Xu325SffPtYNhi+Jsec8CNaWxey8QbDeGXmbMZ+51jeteWG/PXBJ/nSaZcyZ+48vvy1S7nuspM585QjaGkJ9vxQ7XMyYpN16Nu3Dzdd9d+suupK/OiiG7nimj8u959f1V1++fX85jd/YJttNuOkk45h8OBVAZg06W989avf5/nnX+acc06kb98+vPjiDNZee83F+77jHWswefLj3dX03sU/7kqa1mMREVtExIeBwRFxSN3yKWClDvYbExETI2Li2LFXNat5K7y5c9/kJz8Zxxe+8Ik21++zz87ceONP+NGPTuH73//Fcm6dGmGvD5/GLgd+lYOPPJvPHLkvu+60xeJ1Hx29C78seisWufeBv7PDB77Mbh88hS8fN5oBA/rRt28ftttmY3562QR2PuBk5sydx5f+418BGPPJffh/p1/GiJGf4/+dfhnnf2sMAH37tLD9uzbmQ586h3894ixOPv5DbLbx2svvB1dDfPzj+zNhwljGj/8+w4YN4ayzLly8bttt38n11/+Yq6/+Lhdc8EvmzXuLXNTNVcfezgaJiksv08xTIe8EDgJWBz5Yt2wP/Ht7O2Xm2MzcMTN3HDPmsCY2b8X27LMvMG3ai4wefTx77XUML7zwCocccgIvvzyrtN1737sNzz47nZkzX+2mlmpZLTo18fKM2Vx70728d7tNAejTp4XRo3bi6uvuanO/v015njfmzGPrd67Pc9Nn8Nz0mdz7QG1Q5q9vuJvttqn1cH3iw7vzm9/dA8A1v/0LO25bO/5zL8zk5tsnMWfuPGbMeo0/3f0Y795qw6b+rGq8NdccQp8+fWhpaeHQQ/fjwQf/ufdh003XZ+DAlXj88WdYe+01eeGFVxave/HFGQwbNnR5NllvE00LFpk5PjOPBg7KzKPrluMz886lHuBt7p3v3Ii77voFt956IbfeeiFrr70mv/rV91hrrSE888zzi//6ePjhKcyfv4AhQxyA15OsPHAAq66y0uLnH3j/u3n4b9MA2Gu3d/H435/nuRdmLt5+w/XXok+f2n/XDYavyeabrsszU1/mxZdfZdr0GYuvJNlj12147Inacaa/OIv3j9yyKN+aKcVpletunsiuO21Bnz4tDFypP+99z2Y89sRzy+cHV8O89NL/fT5+//u7GDGiFg6nTn2BBQtaAXjuuZd46qnnGD58GO961wiefvp5pk59gbfems/119/BXnvt1C1t73UcY1GyPMZYTI2IXwO7Agn8CfhCZk5bDnX3GCee+C3uuedBZs2aze67f4rPf/5wDj103za3vemmOxk//lb69u3LSiv159xz/59dmj3MsLUGc9XYEwHo27cPV/3mz0y4fRIAh/7rzoy7tpy9d3nvO/nSf4xm/vwFLFyYfOGUixZfTXLi/1zMz8/7HP379eXpZ19kzJcuAOC4k37Kt047kr59+jBv3nw+d9LPgFqPx4TbJnHvzWezcGFy8ZV/4JHH/e+4Imvr++Geex7ksceeAoLhw4dx+unHAXDffY/w059eTd++fWlpCU477djFl6T+z/8cy6c/fSqtrQv58Ic/sDiMqKJeGA6qiLbOuzW0gogJwBXAZUXREcAnMnOfpe/9eHMbpx5p4AYOVlXZ3Ge/1t1N0Apr86b/1t/k07+s9LvqyZ8d2quSyfKYhGxYZv48MxcUy8XAWkvbSZKkHsFTISXLI1i8HBFHRESfYjkCmLEc6pUkScvZ8ggW/wZ8FHgBmA58pCiTJKnni6i29DJNH7yZmc8C/9rseiRJ6ha98HRGFU0LFhHxPx2szsz8erPqliRpuVkeff89SDN7LN5oo2wV4BhgDcBgIUnq+Xrh6YwqmhYsMvM7i55HxGrAF4CjgSuB77S3nyRJ6rmaOsYiIoYCJwKfAC4Bts/MWR3vJUlSD+IYi5JmjrH4FnAIMBZ4V2a+3qy6JEnqLumpkJJmDjn5T2Bd4L+A5yNidrG8FhGzm1ivJEnLT0vFpZOKe0H9NSJ+W7zeOCLujognIuKqiOhflA8oXk8p1m9Ud4yTi/K/RcR+deWjirIpEXFSXXmbdSzt7WiKzGzJzIGZuVpmDqpbVstMZ8ySJKlrvgA8Wvf6bODczBwBzKJ2cQTF46zM3Aw4t9iOiNgK+BiwNTAK+PGim1cCPwL2B7YCPl5s21Ed7fIiGUmSqlgOt/SOiPWAA4GfFa8D2Au4utjkEuDg4vno4jXF+r2L7UcDV2bmvMx8CpgC7FQsUzLzycx8i9pFFqOXUkf7b0enfiJJktS2infejIgxETGxbhnTRi3fA/4fsLB4vQbwj8xcULyeBgwvng8HpgIU618ttl9cvsQ+7ZV3VEe7lse06ZIk9V4VrwrJzLHULnRoU0QcBLyUmfdFxB6Lits61FLWtVfeVidDR9t3yGAhSVIVzb8oZFfgXyPiAGAlYBC1HozVI6Jv0aOwHvB8sf00YH1gWkT0BQYDM+vKF6nfp63yVzqoo12eCpEkaQWWmSdn5nqZuRG1wZe3ZuYngD9Qm9gT4ChgfPH82uI1xfpbMzOL8o8VV41sDIwA7gHuBUYUV4D0L+q4ttinvTraZY+FJEkVZPfdIOsrwJUR8Q3gr8CFRfmFwGURMYVaT8XHADLz4YgYBzwCLACOy8xWgIj4HHAT0Ae4KDMfXkod7TJYSJJUxXIMFpl5G3Bb8fxJald0LLnNm8Ch7ex/BnBGG+U3ADe0Ud5mHR0xWEiSVIV33iwxWEiSVIWjFUt8OyRJUsPYYyFJUhWeCikxWEiSVIXTppcYLCRJqsJgUeIYC0mS1DD2WEiSVEE6xqLEYCFJUhX2/ZcYLCRJqsIeixKDhSRJVTh4s8QOHEmS1DD2WEiSVIU9FiUGC0mSqjBXlBgsJEmqIO2xKDFYSJJUhVeFlDh4U5IkNYw9FpIkVeGpkBKDhSRJVZgrSgwWkiRV0OKgghLfDkmS1DD2WEiSVIEXhZQZLCRJqsBgUWawkCSpgjBZlBgsJEmqwFxR5uBNSZLUMPZYSJJUgT0WZQYLSZIqCPv+S5b6dkTErsXjgOY3R5KkniWi2tLbdCZnnVc83tXMhkiSpJ6vM6dC5kfEz4HhEXHekisz8/jGN0uSpJ7BOcjKOhMsDgI+AOwF3Nfc5kiS1LP0xtMZVSw1WGTmK8CVEfFoZk5aDm2SJKnHMFiUdWUs62sRcV1EvBwRL0XE+IjYpGktkySpB4iISktv05VgcQUwDlgHWBf4JfC/zWiUJEnqmboSLCIzL8vMBcXyCyCb1TBJknqCaKm29DZLHWMREUOLp3+IiJOAK6kFisOA65vYNkmSVni98GxGJZ25KuQ+akFi0Vv3mbp1CXy90Y2SJKmnMFiUdeaqkI2XR0MkSeqJDBZlnT67ExGHRsRqxfP/iohfRcR7mtc0SZLU03Rl2Mh/Z+ZrEbEbsB9wCfCT5jRLkqSeoSWqLb1NV4JFa/F4IHB+Zo4H+je+SZIk9RxOQlbWlWnTn4uIC6jd3vvsYrbTXnihjCRJndcbw0EVXQkGHwVuAkZl5j+AocCXF62MiCENbpskSephOt1jkZlzgF/VvZ4OTK/b5BZg+8Y1TZKkFV/0xoESFXTlVMjS+M5Kkt52PBVS1shg4e29JUlvOwaLskYGC0mS3nYMFmWNvKrDt1aSpAaLiJUi4p6ImBQRD0fE14ryjSPi7oh4IiKuioj+RfmA4vWUYv1Gdcc6uSj/W0TsV1c+qiibUswLRkd1dKQrd978dkRs3cEme3f2WJIk9RbL4QZZ84C9MnNbYDtgVESMBM4Gzs3MEcAs4Jhi+2OAWZm5GXBusR0RsRXwMWBrYBTw44joExF9gB8B+wNbAR8vtqWDOtp/Pzr1I9U8BowtksuxETG4fmVmzuzCsSRJ6hWafYOsrHm9eNmvWBLYC7i6KL8EOLh4Prp4TbF+74iIovzKzJyXmU8BU4CdimVKZj6ZmW9Rm8V8dLFPe3W0q9PBIjN/lpm7AkcCGwGTI+KKiNizs8eQJKm3iZaKS8SYiJhYt4z5pzpqPQsPAC8BE4C/A//IzAXFJtOA4cXz4cBUgGL9q8Aa9eVL7NNe+Rod1NGuLg3eLLpLtiiWV4BJwIkR8ZnM/FhXjiVJUm9QdfBmZo4Fxi5lm1Zgu4hYHfg1sGVbmy1qUjvr2itvq5Oho+071OlgERHfBT4I3Ap8MzPvKVadHRF/6+xxJEnSssnMf0TEbcBIYPWI6Fv0KKwHPF9sNg1YH5gWEX2BwcDMuvJF6vdpq/yVDupoV6dOhRTnWWYB22bmZ+pCxSI7deY4kiT1NhFRaenE8dcqeiqIiIHU5ux6FPgD8JFis6OA8cXza4vXFOtvzcwsyj9WXDWyMTACuAe4FxhRXAHSn9oAz2uLfdqro12dChbFwQ8ubuvd1vpXO3McSZJ6m+Uwu+k6wB8iYjK1EDAhM38LfIXacIQp1MZDXFhsfyGwRlF+InASQGY+DIwDHgFuBI7LzNaiN+Jz1OYDexQYV2xLB3W0qytjLP4SEe/NzHu7sI8kSb1as2+QlZmTgfe0Uf4kbZwxyMw3gUPbOdYZwBltlN8A3NDZOjrSlWCxJ/CZiHgGeIPaoI7MzHd3pUJJktR7dSVY7N+0VkhdMOyzS70/iyQtN97Su6wr06Y/ExHbAu8viv6YmZOa0yxJknoGZ00v68otvb8AXA4MK5ZfRMTnm9UwSZJ6guVwS+8epSunQo4B3peZbwBExNnAXcAPmtEwSZJ6gpZY6j2j3la6MldIAK11r1txRlNJklSnKz0WPwfujohfF68PphPXs0qS1Jv1xtMZVXRl8OZ3i9uI7katp+LozPxrsxomSVJP0JWu/7eDrswVMhR4ulgWlfXLzPmNb5YkST2DYyzKunIq5H5qk5TMotZjsTowPSJeAv49M+9rQvskSVqheSqkrCs9ODcCB2Tmmpm5BrUbZo0D/gP4cTMaJ0mSepauBIsdM/OmRS8y82Zg98z8CzCg4S2TJKkHaKm49DZdORUyMyK+AlxZvD4MmBURfYCFDW+ZJEk9gKdCyroSlg4H1gN+UyzrF2V9gI82vmmSJK34IrLS0tt05XLTV4DPR8Sqmfn6EqunNLZZkiSpJ+rKXCG7RMQjwCPF620jwkGbkqS3NecKKevKqZBzgf2AGQDFzKa7N6NRkiT1FA7eLOvK4E0yc2qUJ55vbW9bSZLeDrxBVllXgsXUiNgFyIjoDxwPPNqcZkmS1DP0xtMZVXSlF+ZY4DhgODAN2I7azbEkSZKArvVYvDMzP1FfEBG7An9ubJMkSeo5euM4iSq68n78oJNlkiS9bXhVSNlSeywiYmdgF2CtiDixbtUgajfHkiTpbcvBm2WdORXSH1i12Ha1uvLZwEea0ShJknqK3tjrUMVSg0Vm3g7cHhEXZ+Yzy6FNkiSph+rK4M05EfEtYGtgpUWFmblXw1slSVIP4eDNsq68H5cDjwEbA18DngbubUKbJEnqMVoiKy29TVeCxRqZeSEwPzNvz8x/A0Y2qV2SJPUIXhVS1pVTIfOLx+kRcSDwPLVp1CVJkoCuBYtvRMRg4D+p3b9iEHBCU1olSVIP0Rt7HaroyqmQQ4HIzIcyc09gH+BDzWmWJEk9g7OblnWlx+LdmfmPRS8yc2ZEvKcJbZIkqcfojQMwq+hKsGiJiCGZOQsgIoZ2cX9JknodT4WUdSUYfAe4MyKuBhL4KHBGU1olSZJ6pE4Hi8y8NCImAnsBARySmY80rWWSJPUAvXGcRBVdOpVRBAnDhCRJBU+FlDlGQpKkCsLBmyUGC0mSKrDHosxTQ5IkqWHssZAkqQL/Qi8zWEiSVIE3yCozWEiSVIFjLMrswZEkSQ1jj4UkSRXYY1FmsJAkqYI+3d2AFYzBQpKkChy8WWawkCSpAk+FlDl4U5IkNYzBQpKkClqi2rI0EbF+RPwhIh6NiIcj4gtF+dCImBARTxSPQ4ryiIjzImJKREyOiO3rjnVUsf0TEXFUXfkOEfFgsc95EREd1dHh+9H1t1CSJC3SJ6otnbAA+M/M3BIYCRwXEVsBJwG3ZOYI4JbiNcD+wIhiGQOcD7WQAJwKvA/YCTi1LiicX2y7aL9RRXl7dbTLYCFJUgXN7rHIzOmZeX/x/DXgUWA4MBq4pNjsEuDg4vlo4NKs+QuwekSsA+wHTMjMmZk5C5gAjCrWDcrMuzIzgUuXOFZbdbT/fiz9R5IkSc0SEWMiYmLdMqaDbTcC3gPcDbwjM6dDLXwAw4rNhgNT63abVpR1VD6tjXI6qKNdXhUiSVIFVS83zcyxwNilbRcRqwLXACdk5uxiGESbm7ZVzTKULxN7LCRJqqDZp0IAIqIftVBxeWb+qih+sTiNQfH4UlE+DVi/bvf1gOeXUr5eG+Ud1dH++9G5H0mSJLWlT8VlaYorNC4EHs3M79atuhZYdGXHUcD4uvIji6tDRgKvFqcxbgL2jYghxaDNfYGbinWvRcTIoq4jlzhWW3W0y1MhkiRVsBxukLUr8EngwYh4oCj7KnAWMC4ijgGeBQ4t1t0AHABMAeYARwNk5syI+Dpwb7Hd6Zk5s3j+WeBiYCDwu2KhgzraZbCQJGkFlpl/ou1xEAB7t7F9Ase1c6yLgIvaKJ8IbNNG+Yy26uiIwUKSpAqcK6TMYCFJUgWdvMnV24bBQpKkCpyErMxgIUlSBQaLMi83lSRJDWOPhSRJFdhjUWawkCSpgj5eFVJisJAkqQLHFJT5fkiSpIaxx0KSpAocY1FmsJAkqQKDRZnBQpKkChy8WWawkCSpAnssyhy8KUmSGsYeC0mSKrDHosxgIUlSBQaLMoOFJEkVOG16mcFCkqQKWrwqpMTBm5IkqWHssZAkqQL/Qi8zWEiSVIGDN8sMFpIkVeDgzTKDxQruySen8cUvnrP49dSpL3D88Z/gU58a3Y2tUlUD+rQw7ogd6N+nhb4twQ1/e4lz//gkvzxiB1bpX/tvuebK/Xlg+quMuWYyB2+9NseO3BCAOW+1cspNj/HoS68vPl5LwG8/9T5eeP1N/u2XkwBYf/BK/ODgd7H6Sv146IXZfPG6h5m/sDbI7MAthvHF929CJjz60uscf+1Dy/kdUCPMm/cWn/jESbz11nxaW1vZb79dOf74T3DXXZM455yLWLgwWXnllTjrrBPYcMN1u7u5epswWKzgNtlkPcaPPw+A1tZWdt/9U+yzz87d3CpVNa91IR+/4n7mzG+lb0tw9Sd35La/v8Khv7hv8TY/+dC7ufmJlwGY+o+5fPTy+5j95gL22GQNztx/Sw6+5N7F2/7bjhswZcYbrDqgz+Kyk/YcwYX3PMt1j77IGfttwWHbrssv/vocGw0ZyHE7b8whl01k9psLWGPlfsvvB1dD9e/fj0suOYNVVhnI/PkLOPzwr7D77jtw2mk/5sc//i823XR9Lr/8es4//yrOOuuL3d3cXsurQsqaOuYkIj4XEYOK5xdExD0RsXcz6+zN7rprEuuvvw7Dhw/r7qaoAebMbwWgb0vQryWo/2papX8fdtlwCDc//hIA9z33KrPfXADA/c+/yjqrDVi87dqrDWCvzdbkyknPlY6/y4ZDuOGx2v7XPDSdfTevfW4+vt1wLr1/6uLjzZgzvyk/n5ovIlhllYEALFiwgAULFhARQPD663MAeP31OQwbtkY3trL3a4lqS2/T7B6LMZn5w4jYFxgOfBYYC+zQ5Hp7peuv/yMHHbR7dzdDDdIS8Nuj38dGQwZy6X3TeOD52YvX7bf5MP78zExef6v1n/b72LvX5ba/z1j8+tQPbM43//AEq/b/v//OQwb2Y/a8BbRmLa5Mn/0maxdhZOOhKwNwzSd3pCWC7/3pSW5/cgbqmVpbWznkkC/y7LPTOfzwA9l223dyxhmfZ8yYrzFgQH9WXXVlxo37dnc3s1frjeGgimZfJbPoj7D9gZ9n5n1LqzMixkTExIiYOHbsVU1uXs/x1lvzufXWuxk1atfubooaZGHCARfdzcgf/ont1h3E5muusnjd6K3ewbWPvPhP++y8wRAO23Y4Z942BYC9NluTGXPe4qEXXitt19b3XJEx6NsSbDRkZQ67/D6OH/8gZ++/JYMGeFa0p+rTpw/jx5/H7bf/nMmTH+fxx5/h4ovHM3bsqdxxx8UccsgHOPPMn3V3M3u1lopLb9Psb5NJEXEDsDlwSkSsCnR4Miozx1Lr1QAe98RV4Y477rashvkAAA2vSURBVGPrrTdlzTWHdHdT1GCz5y3grmdnsccma/D4K2+w+sB+bLvuIMZcM7m03RZrrcrZB2zJUeMe4B9za6cvdhw+mA9sthZ7bLImA/q2sNqAvnzvg1tzwnUPM2hAX/pE0JrJOoNW4sXX5wEwffY8/vr8qyxYmEx99U2enDmHjYauzOTps/+pbeo5Bg1alfe9713cccd9PPbYU2y77TsBOOCA3fj0p0/r1rbp7aXZYelo4DRgp8ycA6wEHNPkOnul66+/gwMP/JfuboYaZOjAfot7CQb0bWG3jYYyZWbtnPiBWwzjlimvMK914eLt1x00gAs+/G6+eN3DPFVsB3DO7X9n5I/+xG7n/5nPj3+IO5+ZyQnXPQzAXc/M4oAtauMqPrzNOkwoBoLe/MTL7LxhLaAOGdiPjYeuzLP/mNv8H1oNN3Pmq8yeXbs66M0353HnnQ+w6abr8dprb/DUU7UxN3/+c61MzRNRbeltmtpjkZmtEbEJsA9wBjCQ3tnz01Rz577JnXc+wOmnH9fdTVGDDFt1AN89aGtaWqAlgt8++iK3TnkFgA9uuTbn/+Xp0vZf2HUThqzUj6/vtwUArQuTD158T4d1nHnbFH44ehu+9C+b8vALr3FVMbjz9idnsPvGQ/n9v4+kdSF889YnFveAqGd56aWZnHTS92htXUjmQkaN2o0999yJb3zj8xx//JlEBIMHr8o3v/mF7m5qr9YLs0Elkdm8sw0R8UOgH7B7Zm4ZEUOBmzLzvZ07gqdC9M82PPPZ7m6CVjDPnLxBdzdBK6zNm/57f+Ir11f6XbXjmgf2qmzS7DEWu2Tm9hHxV4DMnBkR/ZtcpyRJy43d8GXNfj/mR0QLxYDNiFgDWNjxLpIkqadqdo/Fj4BrgLUi4mvAR4GvNblOSZKWm/DOmyVNCRbFJab/kZmXRsR9wAeojW85NDOdlECS1Gv0qgESDdCsHouLgZsj4hLgnMx8uEn1SJLUrXrjJaNVNCVYZOa4iLge+B9gYkRcRt3Yisz8bjPqlSRJ3auZYyzmA28AA4DVcNCmJKkXssOirFljLEYB3wWuBbYv7ropSVKv4yRkZc3qsTiF2kBNx1ZIkno1c0VZs8ZYvL8Zx5UkaUXj4M0ybxgmSZIaptk3yJIkqVezw6LMYCFJUgUGizKDhSRJFXhVSJnBQpKkCswVZQ7elCRJDWOPhSRJFTi7aZk9FpIkVRAVl6UeP+KiiHgpIh6qKxsaERMi4onicUhRHhFxXkRMiYjJEbF93T5HFds/ERFH1ZXvEBEPFvucF1G7M0d7dSyNwUKSpAoiqi2dcDEwaomyk4BbMnMEcEvxGmB/YESxjAHOr7UxhgKnAu8DdgJOrQsK5xfbLtpv1FLq6JDBQpKkFVhm3gHMXKJ4NHBJ8fwS4OC68kuz5i/A6hGxDrAfMCEzZ2bmLGACMKpYNygz78rMBC5d4lht1dEhg4UkSRW0VFwiYkxETKxbxnSi2ndk5nSA4nFYUT4cmFq33bSirKPyaW2Ud1RHhxy8KUlSBVXnCsnMscDYhjSm7WEbuQzly8weC0mSKmj24M12vFicxqB4fKkonwasX7fdesDzSylfr43yjurokMFCkqQKlsPgzbZcCyy6suMoYHxd+ZHF1SEjgVeL0xg3AftGxJBi0Oa+wE3FutciYmRxNciRSxyrrTo65KkQSZJWYBHxv8AewJoRMY3a1R1nAeMi4hjgWeDQYvMbgAOAKcAc4GiAzJwZEV8H7i22Oz0zFw0I/Sy1K08GAr8rFjqoo0MGC0mSKmj2Lb0z8+PtrNq7jW0TOK6d41wEXNRG+URgmzbKZ7RVx9IYLCRJqsBJyMoMFpIkVWCuKDNYSJJUgXOFlHlViCRJahh7LCRJqsBTIWUGC0mSKqh6583exmAhSVIF5ooyx1hIkqSGscdCkqQK/Au9zGAhSVIFjrEoM1hIklSJyaKewUKSpArCYFHiqSFJktQw9lhIklRBhH+j1zNYSJJUiadC6hksJEmqwDEWZQYLSZIqMVjU88SQJElqGHssJEmqwMGbZQYLSZIq8VRIPYOFJEkVOHizzP4bSZLUMPZYSJJUgT0WZQYLSZIqsfO/nsFCkqQKwnnTSwwWkiRVYrCoZ/+NJElqGHssJEmqwMGbZQYLSZIqsfO/nsFCkqQK7LEoM2ZJkqSGscdCkqQKvNy0zGAhSVIlBot6BgtJkioIRxWUGCwkSarEHot6xixJktQw9lhIklSBgzfLDBaSJFVisKhnsJAkqQIHb5YZLCRJqsQei3rGLEmS1DD2WEiSVIFzhZQZLCRJqsCrQsoMFpIkVeKognq+G5IkqWHssZAkqQLHWJQZLCRJqsRgUc9gIUlSBQ7eLDNYSJJUicMV6/luSJKkhonM7O42qBMiYkxmju3udmjF4udCbfFzoe5kj0XPMaa7G6AVkp8LtcXPhbqNwUKSJDWMwUKSJDWMwaLn8Hyp2uLnQm3xc6Fu4+BNSZLUMPZYSJKkhjFYSJKkhjFYNEFEZER8p+71lyLitCbU89UlXt/Z6DrUHI38jETE6hHxH8u479MRseay7KvGiojWiHggIh6KiF9GxMrLcIyfRcRWxXO/H9QtDBbNMQ84ZDl8YZe+ODJzlybXp8Zp5GdkdaDNYBERfRpwfC0fczNzu8zcBngLOLarB8jMT2fmI8VLvx/ULQwWzbGA2qjsLy65IiLWiohrIuLeYtm1rnxCRNwfERdExDOLfulExG8i4r6IeDgixhRlZwEDi79wLi/KXi8er4qIA+rqvDgiPhwRfSLiW0W9kyPiM01/J9SeZfmMnBYRX6rb7qGI2Ag4C9i0+Cx8KyL2iIg/RMQVwIPFtv/0GdIK7Y/AZgARcWLxb/1QRJxQlK0SEddHxKSi/LCi/LaI2NHvB3WrzHRp8AK8DgwCngYGA18CTivWXQHsVjzfAHi0eP5D4OTi+SgggTWL10OLx4HAQ8Aai+pZst7i8UPAJcXz/sDUYt8xwH8V5QOAicDG3f1+vR2XZfyMnAZ8qe4YDwEbFctDdeV7AG/U/9t28Bl6etHnzKX7PxPFY19gPPBZYAdq4XAVYFXgYeA9wIeBn9btO7h4vA3Ysf54bRzf7weXpi7ObtokmTk7Ii4Fjgfm1q36ALBV3TS7gyJiNWA3av/hycwbI2JW3T7HR8SHiufrAyOAGR1U/zvgvIgYQC2k3JGZcyNiX+DdEfGRYrvBxbGeWtafU8tuGT4jXXFPZtb/u3b1M6Tlb2BEPFA8/yNwIbVw8evMfAMgIn4FvB+4Efh2RJwN/DYz/9iFevx+UFMZLJrre8D9wM/rylqAnTOz/hcJUfdbZInyPaj9otk5M+dExG3ASh1VmplvFtvtBxwG/O+iwwGfz8ybuvyTqFm68hlZQPn0ZUefgzfq9tuDLn6G1C3mZuZ29QXtfS9k5uMRsQNwAHBmRNycmad3phK/H9RsjrFoosycCYwDjqkrvhn43KIXEbHoi+RPwEeLsn2BIUX5YGBW8QthC2Bk3bHmR0S/dqq/Ejia2l83i74obgI+u2ifiNg8IlZZxh9PDdDFz8jTwPZF2fbAxkX5a0BHPRodfYa0YrsDODgiVi7+r34I+GNErAvMycxfAN+m+Fwswe8HdQuDRfN9B6gf+X88sGMxOOoR/m/k99eAfSPifmB/YDq1Xxg3An0jYjLwdeAvdccaC0xeNDhrCTcDuwO/z8y3irKfAY8A90fEQ8AF2Gu1IujsZ+QaYGjRXf5Z4HGAzJwB/LkYxPetNo7f0WdIK7DMvB+4GLgHuBv4WWb+FXgXcE/xWTgF+EYbu/v9oG7hLb1XEMX5ztbMXBAROwPnL9ktKknSis40uuLYABgXES3UrmH/925ujyRJXWaPhSRJahjHWEiSpIYxWEiSpIYxWEiSpIYxWEjdpJjTY4WfGCqWmCVTkjpisJC6zx5AU4NF1CzT//O6fQ0WkjrNYCE1WEQcWdzcalJEXBYRH4yIuyPirxHx+4h4RzEr6bHAF4sZKN8fyzbzbVszX24UEY9GxI+p3S78vyPi3Lr2/XtEfLedti+574UsMUumJHXEy02lBoqIrYFfAbtm5isRMZTaTLX/yMyMiE8DW2bmf0bEadRmnPx2se8VwI8z808RsQFwU2ZuGRE/BJ7LzDMjYhS1SaTWAjakdlfGkdTmebgbOAKYBTwJ7JKZfyluyzwZ2CIz50fEncBnMvPBNtq/Uf2+RdnrmblqE94uSb2QN8iSGmsv4OrMfAVqc4FExLuAqyJiHWrTVLc3W2RXZ77djbZnvrwWeGZRMMjMNyLiVuCgiHgU6NdWqKizeF9J6iqDhdRYQa2Hot4PgO9m5rXFTKOntbNvl2a+LepqzxtLvP4ZtbESj1GeSbUz+0pSpznGQmqsW4CPRsQaAMWpkMHAc8X6o+q2XXJW0q7OfNvmzJdtNSoz7wbWBw7n/6bJ7qyOZsmUpBKDhdRAmfkwcAZwe0RMAr5LrYfilxHxR+CVus2vAz60aPAmXZz5toOZL9szDvhzZs7qYJu2dDRLpiSVOHhTWsE1aubbiPgtcG5m3tLwRkpSwTEW0oqv0sy3EbE6tV6NSYYKSc1mj4X0NlSMAWkrZOydmTOWd3sk9R4GC0mS1DAO3pQkSQ1jsJAkSQ1jsJAkSQ1jsJAkSQ1jsJAkSQ3z/wG6SYFe8ZkrmAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 648x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, ax = plt.subplots(figsize=(9, 6))\n",
    "sns.heatmap(contingency, annot=True, fmt=\"d\", cmap=\"YlGnBu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}