Binary Classification/OnionOrNot/DunnBC22-distilbert-base-uncased-OnionOrNot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2007d2da",
   "metadata": {},
   "source": [
    "## Binary Text Classification - OnionOrNot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saved as 'distilbert-base-uncased-OnionOrNot' on my HuggingFace profile\n",
    "\n",
    "Dataset Source: https://www.kaggle.com/datasets/chrisfilo/onion-or-not\n",
    "\n",
    "@article{Sanh2019DistilBERTAD,\n",
    "  title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter},\n",
    "  author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf},\n",
    "  journal={ArXiv},\n",
    "  year={2019},\n",
    "  volume={abs/1910.01108}\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf8a06e9",
   "metadata": {},
   "source": [
    "#### Import Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8c88f3bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "NOTE: Redirects are currently not supported in Windows or MacOs.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch.nn.functional import cross_entropy\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification\n",
    "from transformers import Trainer, TrainingArguments, AdamW, get_scheduler, set_seed\n",
    "\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, ConfusionMatrixDisplay\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.dummy import DummyClassifier\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c12ebf4",
   "metadata": {},
   "source": [
    "#### Versions of Many Libraries Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "402dc4cf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pandas:            1.4.3\n",
      "NumPy:             1.22.4\n",
      "Torch:             1.12.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Pandas:\".ljust(18), pd.__version__)\n",
    "print(\"NumPy:\".ljust(18), np.__version__)\n",
    "print(\"Torch:\".ljust(18), torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0062b60",
   "metadata": {},
   "source": [
    "#### Basic Constants/Values Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dd373a12",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ckpt = \"distilbert-base-uncased\"\n",
    "device = \"cpu\"\n",
    "max_input_len = 50\n",
    "\n",
    "learning_rate=2e-5\n",
    "batch_size = 64\n",
    "\n",
    "num_of_epochs = 2\n",
    "model_name = f\"{model_ckpt}-OnionOrNot\"\n",
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef3b0e13",
   "metadata": {},
   "source": [
    "#### Ingestion Data into Pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c1a1ab45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Entire Facebook Staff Laughs As Man Tightens P...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Muslim Woman Denied Soda Can for Fear She Coul...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bold Move: Hulu Has Announced That They’re Gon...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Despondent Jeff Bezos Realizes He’ll Have To W...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>For men looking for great single women, online...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23995</th>\n",
       "      <td>Teen Pregnancy Rate Prompting More High School...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23996</th>\n",
       "      <td>Connecticut TV station under fire after using ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23997</th>\n",
       "      <td>Jurisprudence Fetishist Gets Off On Technicality</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23998</th>\n",
       "      <td>Employees From Other Department Announce Plan ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23999</th>\n",
       "      <td>Police drone crashes into police SWAT team</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  label\n",
       "0      Entire Facebook Staff Laughs As Man Tightens P...      1\n",
       "1      Muslim Woman Denied Soda Can for Fear She Coul...      0\n",
       "2      Bold Move: Hulu Has Announced That They’re Gon...      1\n",
       "3      Despondent Jeff Bezos Realizes He’ll Have To W...      1\n",
       "4      For men looking for great single women, online...      1\n",
       "...                                                  ...    ...\n",
       "23995  Teen Pregnancy Rate Prompting More High School...      1\n",
       "23996  Connecticut TV station under fire after using ...      0\n",
       "23997   Jurisprudence Fetishist Gets Off On Technicality      1\n",
       "23998  Employees From Other Department Announce Plan ...      1\n",
       "23999        Police drone crashes into police SWAT team       0\n",
       "\n",
       "[24000 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('~/Documents/nlpnn/Datasets/OnionOrNot/OnionOrNot.csv')\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4556dabf",
   "metadata": {},
   "source": [
    "#### Visualize Title Length (0: 'Ham', 1: 'Spam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "120564e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEHCAYAAABbZ7oVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAT9UlEQVR4nO3de5RlZX3m8e/DxRv3TneAIE0jQbMgEzFTkmAyGY0kAZKImhUzBBEUQ+LoKBOTKM6sSGaMMUY0MyvR2EYCCV7xMpo1eAEGZCDE2DCo3ByFwAA20AgEDCwV+M0fe5eelPWeOtVd59Jd389ae9XZl7Pf3zldvZ/a+92XVBWSJC1mp2kXIEmaXYaEJKnJkJAkNRkSkqQmQ0KS1GRISJKaDAltlSTXJXn2kPmXJnn55CpaWUlOSXL5Mpa/JcnR46xJmgZDQotK8s2B4bEkDw+Mn1hVh1fVpf2yZyY5b4Xa3ZCkkuyyEuub1TaHSfKGJP/Yf9e3J/nQFGv5oSS396+/LwiXG6ba/szEfwrNnqraff51kluAl1fVRdOraHVIcjJwEnB0Vd2UZD/geVMs6Tjg01NsX1PmnoS2yvxflUmOAd4A/Fr/l+8XG8u/LMkNSe5L8pkkB21Fm3sleW+SzUnuSPKmJDv3805JcnmSt/Vt/GOSYwfee3CSy5I8mOSiJH8+sPdzWf/z/v4zHDXwvkXX1/DMJNf3y/9Vkif067g2yS8PrHPXJPckecZi6wA+U1U3AVTVnVW1ceC9lyb5oyT/kOSBJJ9IsmZg/vlJ7kzyT/3nPXxg3jlJ3pnkU/3nvCLJfkn+tK/5xkVqOg64YInP/V1JXp/kpv57vj7JCwbmndK3+Y4k9ye5Ocmz+um3Jbm7D0nNEENC26SqPg28GfhQVe1eVU9fuEyS4+mC5IXAOuB/Ax/YiubOAR4Bfhh4BvDzwGC/x08AXwHWAm8F3psk/bz3A/8A/ABwJt1f6/N+pv+5d/8ZrhxhfYs5EfgF4BDgqcB/7qf/NfDigeWOAzZX1f9ZZB1/D7wkye8mmZsPwQVeArwM2J/u+/jvA/M+BRwK/CBwNfC+Be99UV/XWuBbwJX9cmuBjwBvn18wya50382FQz7zQjcB/wbYC/gD4Lwk+w/M/wngS3T/Du8HPkgXjD9M9x39WZLd0eyoKgeHoQNwC93hj0Wn0W10z1sw/1K6Q1TQbbhOHZi3E/AQcNAibW0ACthlwfR96TZqTxyYdgJwSf/6FOBrA/Oe1K9nP2A93cb0SQPzz5uvebE2h61vyHf0WwPjxwE39a9/CHgQ2LMf/wjwe0O+7xOBi4B/Br4BvG7B9/qWgfHDgG8DOy+ynr37mvfqx88B3jMw/z8ANwyM/yvg/oHx5wIXL/iM3wTuHxgeAi4f8lmuAY4f+E6/uqC9AvYdmPYN4Ihp/847fG9wT0KTcBDw3/pDDPcD9wIBDljmOnYFNg+s5910fzHPu3P+RVU91L/cnW4jfe/ANIDbRmiztb6WwXXe2rdLVX0duAL4lSR7A8fy/X/hf1dVva+qjqbbyP8W8F+T/MKQdnYF1ibZOclb+sM9D9Bt1KHbS5h318DrhxcZH/x8ix1qen5V7T0/AP9+cGaSlyS5ZuDf6EeXaJ+qGlaDpsyOa62EpW4lfBvwh1XV3DCO4Da6PYm1VfXIMt+7GViT5EkDG/sDB+av1K2QB9e5Hvj6wPi5dIfGdgGurKo7llpZVX0HOD/J6+g2tp9ptPMd4B7g14HjgaPpAmIv4D66QN4ax9EdIhxJ38/0Hro9kCur6tEk12xD+5oB7kloJdwFbEjS+n36C+CM+U7UvgP6V5dY5+OTPGF+6Nv4LHBWkj2T7JTkkCT/dqniqupWYBNwZpLH9R3TvzywyBbgMeApS61rCa9M8uS+I/k/AYOnrv4P4MeB19D1USyq78T9xSR79J/xWOBw4PMDi704yWFJngT8F+AjVfUosAddkH6D7vDYm7f2gyQ5GHh8Vd2wjLftRhe4W/p1vJQu3LQdMyS0Es7vf34jydULZ1bVx4E/Bj7YHwa5lu6QyzDfpDv0MD/8LF2H7eOA6+n+Qv4IXeftKE4EjqLbgL6JbgP+rb6+h4A/BK7oD5P85IjrXOj9dEF2M10H7pvmZ1TVw8BHgYOBjw1ZxwN0nfz/j+6Y/1uBV1TV4LUIf0PXv3An8ATg1f30v6Y7/HQH3Xf091v5OQB+kWWc1QRQVdcDZ9F1ht9F1+dwxTbUoBmQvrNIWlXSXaB2Y1W9cYJt/j7w1Kp68ZILt9dxKV2H+1+uWGGLt3MB8GdVtayg0I7HPQmtCkme2R+e2indtR3H0x0CmlT7a4BTgY1LLTsjLgUumXYRmj5DQqvFfnQbvm/SXVfwilr8OoUVl+Q36DreP1VVly21/Cyoqrf2h8i0ynm4SZLU5J6EJKnJkJAkNW0XF9OtXbu2NmzYMO0yJGmHddVVV91TVesWTt8uQmLDhg1s2rRp2mVI0g4rya2LTfdwkySpyZCQJDUZEpKkJkNCktRkSEiSmgwJSVKTISFJajIkJElN28XFdJJWl2zlA0+9X+nKMyQkzZzWxj4xCCbNw02SpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTYaEJKnJkJAkNRkSkqSmsYVEkgOTXJLk+iTXJXlNP/3MJHckuaYfjhtXDZKkbTPO23I8Ary2qq5OsgdwVZIL+3nvqKq3jbFtSdIKGFtIVNVmYHP/+sEkNwAHjKs9SdLKm0ifRJINwDOAz/eTXpXkS0nOTrLPJGqQJC3f2EMiye7AR4HTq+oB4F3AIcARdHsaZzXed1qSTUk2bdmyZdxlSpIWMdaQSLIrXUC8r6o+BlBVd1XVo1X1GPAe4MjF3ltVG6tqrqrm1q1bN84yJUkN4zy7KcB7gRuq6u0D0/cfWOwFwLXjqkGStG3GeXbTTwEnAV9Ock0/7Q3ACUmOAAq4BfjNMdYgSdoG4zy76XJgsYcQXjCuNiVJK8srriVJTYaEJKnJkJAkNRkSkqQmQ0KS1GRISJKaDAlJUpMhIUlqMiQkSU2GhCSpyZCQJDUZEpKkJkNCktRkSEiSmgwJSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTbtMuwCNX7J176ta2TokbX8MiVWgtbFPDAJJw3m4SZLUZEhIkprGFhJJDkxySZLrk1yX5DX99DVJLkzy1f7nPuOqQZK0bca5J/EI8NqqOgz4SeCVSQ4DXg9cXFWHAhf345KkGTS2kKiqzVV1df/6QeAG4ADgeODcfrFzgeePqwZJ0raZSJ9Ekg3AM4DPA/tW1eZ+1p3Avo33nJZkU5JNW7ZsmUSZkqQFxh4SSXYHPgqcXlUPDM6rqgIWPQmzqjZW1VxVza1bt27cZUqSFjHWkEiyK11AvK+qPtZPvivJ/v38/YG7x1mDJGnrjfPspgDvBW6oqrcPzPokcHL/+mTgE+OqQZK0bcZ5xfVPAScBX05yTT/tDcBbgA8nORW4FXjRGGuQJG2DsYVEVV0OtO4a9NxxtStJWjlecS1JajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTSOFRJKfTvLS/vW6JAePtyxJ0ixYMiSSvBF4HXBGP2lX4LxxFiVJmg2j7Em8AHge8M8AVfV1YI9xFiVJmg2jhMS3B58gl2S38ZYkSZoVo4TEh5O8G9g7yW8AFwHvGW9ZkqRZsOTzJKrqbUl+DngAeBrw+1V14dgrkyRN3UgPHepDwWCQpFWmGRJJHqTrh0j/87uzgKqqPcdcm6Qd2Jo1cN99y39fWs+7bNhnH7j33uW3o04zJKrKM5gkjc1990HV0sttq+WGiv6lUa6T+JtRpkmSdjyjnN10+OBIkl2Afz2eciRJs6QZEknO6PslfizJA/3wIHAX8ImJVShJmppmSFTVH/X9En9SVXv2wx5V9QNVdUbrfZKkHcews5t+pKpuBM5P8uML51fV1WOtTJI0dcOuk/ht4DTgrEXmFfCzY6lIkjQzhoXElwCq6jkTqkWSNGOGnd30solVIUmaST6ZTpLUNOxw048leWCR6d6WQ5JWiWF7El8eOPV1zwWnwS4ZEEnOTnJ3kmsHpp2Z5I4k1/TDcSvyKSRJYzHOw03nAMcsMv0dVXVEP1wwxvYlSdtoWEicvy0rrqrLAO+9KEnbsWFXXL95TG2+KsmX+sNR+7QWSnJakk1JNm3ZsmVMpUiShpn02U3vAg4BjgA2s/iFegBU1caqmququXXr1k2oPEnSoGE3+DsqWdk7sVfVXVX1aFU9Rvec7CNXcv2SpJU1bE/iJcBVST6Y5JQk+21rY0n2Hxh9AXBta1lJ0vQNezLdK6C70R9wLHBOkr2AS4BPA1dU1aOt9yf5APBsYG2S24E3As9OcgTdvZ9uAX5zRT6FJGksUst4fmCSJwLPoQuNo6pqblyFDZqbm6tNmzZNoqlVJZnM4yOlxUzq98/f89EkuWqxbfqwK66/T1U9DFzQD5KkHZz3bpIkNRkSO4g1a7rd6uUMsPz3rFkz3c8pabKWPNyUZDfg4ap6LMlTgR8BPlVV3xl7dRrZffdN7viupNVjlD2Jy4AnJDkA+CxwEt19mSRJO7hRQiJV9RDwQuCdVfWrwOHjLUuSNAtGCokkRwEnAv+zn7bz+EqSJM2KUULidOAM4ONVdV2Sp9BdUCdJ2sEt2XFdVZ8DPjcwfjPw6nEWJUmaDc2QSPK3dLfPWFRVPW8sFUlaNTxbbvYN25N4W//zhcB+wHn9+AnAXeMsStLq4Gnbs2/YDf4+B5DkrAX38/jbJN5ISZJWgVE6rnfrO6sBSHIwsNv4SpIkzYpRbvB3OnBpkpuBAAcBp42zKEnSbBgaEkl2AvYCDqW7HQfAjVX1rXEXJkmavqGHm/rHjP5eVX2rqr7YDwaEJK0So/RJXJTkd5IcmGTN/DD2yiRJUzdKn8Sv9T9fOTCtgKcssqwkaQcyyhXXB0+iEEnS7BnleRK7Aq8AfqafdCnwbp8nIUk7vlEON70L2BV4Zz9+Uj/t5eMqSpI0G0YJiWdW1dMHxv9Xki+OqyBJ0uwY5eymR5McMj/SX3396PhKkiTNilH2JH4XuGTBFdcvHWtVkqSZMOxW4acDf0f3LIlDgaf1s77iBXWzybtdSlppw/Ykngz8Kd3tOL4MXEEXGrcDhsQM8rbLklbasFuF/w5AkscBc8Cz6A4zbUxyf1UdNpkSJUnTMkqfxBOBPelu9LcX8HW6PQtJ0g5uWJ/ERuBw4EHg83SHmt5eVfeNsuIkZwO/BNxdVT/aT1sDfAjYANwCvGjU9UmSJm/YKbDrgccDdwJ30PVF3L+MdZ8DHLNg2uuBi6vqUODiflySNKOaIVFVxwDP5HvPun4t8IUkn03yB0utuKouA+5dMPl44Nz+9bnA85dbsCRpcob2SVRVAdcmuR/4p374JeBI4I1b0d6+VbW5f30nsO9WrEOSNCHD+iReTXdG07OA79D1SfwdcDYr0HFdVZWkedJmktPoH5O6fv36bW1O0gyaxCnV++wz/jZ2ZMP2JDYA5wP/ceCv/211V5L9q2pzkv2Bu1sLVtVGYCPA3NzcBK4AkDRJW3NdTzKZ64H0PcP6JH67qj66ggEB8Eng5P71ycAnVnDdkqQVNsoN/rZKkg8AVwJPS3J7klOBtwA/l+SrwNH9uCRpRo1yMd1WqaoTGrOeO642JUkra2x7EpKk7Z8hIUlqMiQkSU1j65PQ5HnOuaSVZkjsIDznXNI4eLhJktRkSEiSmgwJSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTYaEJKnJkJAkNRkSkqQmQ0KS1GRISJKaDAlJUpMhIUlqMiQkSU2GhCSpyZCQJDUZEpKkpl2m0WiSW4AHgUeBR6pqbhp1SJKGm0pI9J5TVfdMsX1J0hI83CRJappWSBTw2SRXJTltSjVIkpYwrcNNP11VdyT5QeDCJDdW1WWDC/ThcRrA+vXrp1GjJK16U9mTqKo7+p93Ax8HjlxkmY1VNVdVc+vWrZt0iZIkphASSXZLssf8a+DngWsnXYckaWnTONy0L/DxJPPtv7+qPj2FOiRJS5h4SFTVzcDTJ92uJGn5PAVWktRkSEiSmgwJSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIkpoMCUlSkyEhSWoyJCRJTYaEJKnJkJAkNRkSkqQmQ0KS1DSNx5dqwronxS5/XtXK1yJp+2JIrAJu7CVtLQ83SZKaDAlJUpMhIUlqMiQkSU2GhCSpyZCQJDV5CqykmeO1PbPDkJA0c9zYzw4PN0mSmgwJSVLTVEIiyTFJvpLka0leP40aJElLm3hIJNkZ+HPgWOAw4IQkh026DknS0qaxJ3Ek8LWqurmqvg18EDh+CnVIkpYwjZA4ALhtYPz2fpokacbMbMd1ktOSbEqyacuWLdMuR5JWpWmExB3AgQPjT+6n/QtVtbGq5qpqbt26dRMrTpL0PakJX7WSZBfg/wLPpQuHLwC/XlXXDXnPFuDWyVS4qqwF7pl2EdIy+Ds7PgdV1ff9RT7xK66r6pEkrwI+A+wMnD0sIPr3uCsxBkk2VdXctOuQRuXv7ORN5bYcVXUBcME02pYkjW5mO64lSdNnSKxuG6ddgLRM/s5O2MQ7riVJ2w/3JCRJTYbEKuVNFrU9SXJ2kruTXDvtWlYbQ2IV8iaL2g6dAxwz7SJWI0NidfImi9quVNVlwL3TrmM1MiRWJ2+yKGkkhoQkqcmQWJ1GusmiJBkSq9MXgEOTHJzkccC/Az455ZokzSBDYhWqqkeA+Zss3gB8eKmbLErTlOQDwJXA05LcnuTUade0WnjFtSSpyT0JSVKTISFJajIkJElNhoQkqcmQkCQ1GRKSpCZDQpLUZEhIkpr+Px6cy6y4HeERAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['text_len'] = data['text'].str.split().apply(len)\n",
    "\n",
    "data = data[data['text_len'] < 40]\n",
    "\n",
    "data.boxplot(\"text_len\", by=\"label\", grid=False, showfliers=False, color=\"Blue\")\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Title Length by Spam/Ham')\n",
    "plt.xlabel('')\n",
    "plt.ylabel('Words / Title')\n",
    "plt.show()\n",
    "\n",
    "data = data.drop(columns=['text_len'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fd47f52",
   "metadata": {},
   "source": [
    "#### Actual Project"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "275e923a",
   "metadata": {},
   "source": [
    "#### Convert Pandas DataFrame to Dataset; Then Split into Train/Test/Valid Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4771c6b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data shape: (19180, 3)\n",
      "Testing data shape: (2398, 3)\n",
      "Validation data shape: (2397, 3)\n"
     ]
    }
   ],
   "source": [
    "# Converts the pandas DataFrame to a Dataset\n",
    "dataset = Dataset.from_pandas(data)\n",
    "dataset.shuffle(seed=42)\n",
    "\n",
    "# Splitting the Dataset into the train, test, and valid datasets and then packing all of them into a Dataset Dictionary\n",
    "train_testvalid = dataset.train_test_split(test_size=0.20)\n",
    "\n",
    "test_valid = train_testvalid['test'].train_test_split(test_size=0.5)\n",
    "\n",
    "dataset = DatasetDict({\n",
    "    'train': train_testvalid['train'], \n",
    "    'test': test_valid['test'], \n",
    "    'valid': test_valid['train']})\n",
    "\n",
    "print('Training data shape:', dataset['train'].shape)\n",
    "print('Testing data shape:', dataset['test'].shape)\n",
    "print('Validation data shape:', dataset['valid'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "377245e8",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "92dee09d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74aa909b82a340ca8cce0588dae53cbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7102fd8d33d24bfc9b69ab43583d4537",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f7e620013c6469a885f3c597b681861",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "def tokenize(batch):\n",
    "    return tokenizer(batch['text'], truncation=True, padding=True, max_length=max_input_len)\n",
    "\n",
    "encoded_data = dataset.map(tokenize, batched=True, batch_size=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d577b98",
   "metadata": {},
   "source": [
    "#### Print Example to Ensure Edits Work as Expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5ac061d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask']\n",
      "{'text': 'A True War: EA Reveals The Next Battlefield Game Will Take Place In A Divorcing Family', 'label': 1, '__index_level_0__': 11077, 'input_ids': [101, 1037, 2995, 2162, 1024, 19413, 7657, 1996, 2279, 11686, 2208, 2097, 2202, 2173, 1999, 1037, 4487, 14550, 6129, 2155, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}\n"
     ]
    }
   ],
   "source": [
    "print(encoded_data['train'].column_names)\n",
    "print(encoded_data['train'][240])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20e46ae7",
   "metadata": {},
   "source": [
    "#### Ensure That git lfs is Installed; Remember to Enter Access Token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "63b11285",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "# HuggingFace Access tokens...\n",
    "\n",
    "# Used this the first time to install the git lfs; otherwise, it lets me know that git lfs has been initialized\n",
    "!git lfs install"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0ccad14",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dc56358c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.bias', 'vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_transform.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "num_labels=2\n",
    "\n",
    "model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to(device))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20fe31fc",
   "metadata": {},
   "source": [
    "#### Create Compute Performance Metrics Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "38d8169b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    f1 = f1_score(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\"accuracy\": acc, \"f1\": f1}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e634a3ac",
   "metadata": {},
   "source": [
    "#### Instantiate Optimizer (& Learning Rate Scheduler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2518be36",
   "metadata": {},
   "outputs": [],
   "source": [
    "logging_steps = round(len(encoded_data[\"train\"]) // batch_size, 0) + 1\n",
    "optimizer_name = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "\n",
    "num_training_steps = num_of_epochs * len(dataset['train'])\n",
    "\n",
    "lr_scheduler = get_scheduler(\"linear\",\n",
    "                             optimizer=optimizer_name,\n",
    "                             num_warmup_steps=0,\n",
    "                             num_training_steps=num_training_steps)\n",
    "\n",
    "optimizer = [optimizer_name, lr_scheduler]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff1297fc",
   "metadata": {},
   "source": [
    "#### Define TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "780ab103",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(output_dir=model_name,\n",
    "                                 num_train_epochs=num_of_epochs,\n",
    "                                 learning_rate=learning_rate,\n",
    "                                 per_device_train_batch_size=batch_size,\n",
    "                                 per_device_eval_batch_size=batch_size,\n",
    "                                 weight_decay=0.01,\n",
    "                                 evaluation_strategy=\"epoch\",\n",
    "                                 disable_tqdm=False,\n",
    "                                 logging_steps=logging_steps,\n",
    "                                 push_to_hub=True,\n",
    "                                 log_level=\"error\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0650e163",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dcb845a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/distilbert-base-uncased-OnionOrNot into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(model=model, args=training_args, \n",
    "                  compute_metrics=compute_metrics, \n",
    "                  train_dataset=encoded_data['train'],\n",
    "                  eval_dataset=encoded_data['valid'],\n",
    "                  tokenizer=tokenizer, \n",
    "                  optimizers=optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "384fe433",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3b9ffa8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b86e005e58cd4c8085e6c1abd0755577",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/600 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3334, 'learning_rate': 1.984358706986444e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32bac40475f543369f82629eb5ad09e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/38 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.23822952806949615, 'eval_accuracy': 0.9023779724655819, 'eval_f1': 0.9010689642400332, 'eval_runtime': 251.1887, 'eval_samples_per_second': 9.543, 'eval_steps_per_second': 0.151, 'epoch': 1.0}\n",
      "{'loss': 0.1822, 'learning_rate': 1.9687174139728885e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db7dac6fec1d4b0b9d288a0440af9357",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/38 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.20392799377441406, 'eval_accuracy': 0.9224030037546934, 'eval_f1': 0.921847601870649, 'eval_runtime': 255.6312, 'eval_samples_per_second': 9.377, 'eval_steps_per_second': 0.149, 'epoch': 2.0}\n",
      "{'train_runtime': 12803.6661, 'train_samples_per_second': 2.996, 'train_steps_per_second': 0.047, 'train_loss': 0.2578143946329753, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=600, training_loss=0.2578143946329753, metrics={'train_runtime': 12803.6661, 'train_samples_per_second': 2.996, 'train_steps_per_second': 0.047, 'train_loss': 0.2578143946329753, 'epoch': 2.0})"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23dc7163",
   "metadata": {},
   "source": [
    "#### Predict Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d3e96801",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1479951badc641969246e9ce5a2d0f1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/38 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.20392799377441406,\n",
       " 'test_accuracy': 0.9224030037546934,\n",
       " 'test_f1': 0.921847601870649,\n",
       " 'test_runtime': 257.7233,\n",
       " 'test_samples_per_second': 9.301,\n",
       " 'test_steps_per_second': 0.147}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_predictions = trainer.predict(encoded_data['valid'])\n",
    "\n",
    "output_predictions.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbd97974",
   "metadata": {},
   "source": [
    "#### Push Fine-Tuned Model to HuggingFace Hub (My Profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c4a6e14e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d706284f738c4c839f1ef2d8183b4b3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 32.0k/255M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ee73b12a3814dc1ae0e255e9cfb8bba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Sep04_20-59-28_Brians-Mac-mini/events.out.tfevents.1662343176.Brians-Mac-mini.8652.0: 100%|##…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remote: Scanning LFS files for validity, may be slow...        \n",
      "remote: LFS file scan complete.        \n",
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-OnionOrNot\n",
      "   e980428..baf8c73  main -> main\n",
      "\n",
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-OnionOrNot\n",
      "   baf8c73..3af1419  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/distilbert-base-uncased-OnionOrNot/commit/baf8c73ed2ebaad98a1cb7ec54b3cd9fe5b03226'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub(commit_message=\"All Dunn!!!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8e1df35",
   "metadata": {},
   "source": [
    "#### Save Exact Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6df488dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataset['train'].to_csv('OnionOrNot-training_data.csv', index=None)\n",
    "#dataset['test'].to_csv('OnionOrNot-testing_data.csv', index=None)\n",
    "#dataset['valid'].to_csv('OnionOrNot-validation_data.csv', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "730943d5",
   "metadata": {},
   "source": [
    "#### Run Model with Tokenizer Returning 'np' Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c51d5ef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93306e9254a747ec91b7ad1ff2cdeaa7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4efabfba83a74e6c9e27176bd691852f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ff117b5a6f14191b3ad7741147987e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7616a5a1cb284ecb86008fbb37261efd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d45415ac3b3945d198511147579930a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf228071a72947728768af77a3ee503f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "((19180, 768), (2397, 768))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels=2\n",
    "tokenizer2 = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "def tokenize2(batch):\n",
    "    return tokenizer2(batch['text'], truncation=True, padding=True, max_length=max_input_len, return_tensors=\"np\")\n",
    "\n",
    "encoded_data2 = dataset.map(tokenize2, batched=True, batch_size=None)\n",
    "\n",
    "model2 = (AutoModel.from_pretrained(model_ckpt, num_labels=num_labels, output_hidden_states=True).to(device))\n",
    "\n",
    "def extract_hidden_states(batch=True):\n",
    "    inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer2.model_input_names}\n",
    "    with torch.no_grad():\n",
    "        last_hidden_state = model2(**inputs)[0]\n",
    "    return {\"hidden_state\": last_hidden_state[:,0].cpu().numpy()}\n",
    "\n",
    "encoded_data2.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "\n",
    "hidden_data = encoded_data2.map(extract_hidden_states, batched=True)\n",
    "\n",
    "X_train = np.array(hidden_data[\"train\"][\"hidden_state\"])\n",
    "X_valid = np.array(hidden_data[\"valid\"][\"hidden_state\"])\n",
    "y_train = np.array(hidden_data[\"train\"][\"label\"])\n",
    "y_valid = np.array(hidden_data[\"valid\"][\"label\"])\n",
    "X_train.shape, X_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01732132",
   "metadata": {},
   "source": [
    "#### Create Function to Calculate & Display Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0b971419",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_preds, y_true, labels):\n",
    "    cm = confusion_matrix(y_true, y_preds, normalize=\"true\")\n",
    "    fig, ax = plt.subplots(figsize=(6,6))\n",
    "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\n",
    "    disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax)\n",
    "    plt.title('Normalized Confusion Matrix')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97435b9f",
   "metadata": {},
   "source": [
    "#### Plot Confusion Matrix for Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c748c743",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAFWCAYAAAC7G+CvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAoeElEQVR4nO3dd5hV1fn28e89IAoiiICKAqIGsRck9oqaqDHWRI2mqIkl1sTuG6Nokl8iRmOJmmii2A1GVIxYEg3RiIUiFrAGCwKWQUFRRIHn/WPvwTPDlDMDe2bWcH+8zsU5u6z97DPjPOdZa+91FBGYmZkVoaKlAzAzs7bLScbMzArjJGNmZoVxkjEzs8I4yZiZWWGcZMzMrDDtWzoAMzNrWLsua0XMn9vk/WPuBw9FxJ5LMaSyOMmYmSUg5n/O8usf2uT9P3/2yh5LMZyyOcmYmaVAgNTSUTSax2TMzKwwrmTMzFKh9OoCJxkzs1Qk2F3mJGNmlgS5kjEzswIlWMmklxbNzCwZrmTMzFIg3F1mZmZFUZLdZU4yZmapSLCSSS9iMzNLhpOMtUqSRkv6Sf78cEkPL+X2+0kKSc1ezUvqKOk+SbMl3bkE7Sz196UlSHpA0o9aOo4kSE1/tBAnmWWUpDclvS9pxZJlP5E0ugXDqlVE3BoR32ju40o6TNI4SXMkzcj/GO6wFJr+DrAa0D0ivtvURop6XyTtkifgu2ss3yxfPrrMdoZIuqWh7SJir4i4sYnhLkPy+2Sa+mghTjLLtnbAKUvaiDJt6ndJ0qnAZcD/kSWEvsDVwH5Lofm1gFcjYv5SaKsoHwDbSupesuxHwKtL6wBt8femUFUTZLqSsYRcDJwuaeXaVkraTtLYvFtnrKTtStaNlvQbSU8AnwHr5J9yj5f0mqRPJP1K0rqSxkj6WNJwSR3y/btJ+oekDyR9lD/vXUccR0j6b/78zLyyqHp8KWlYvq6rpL/mVcc0Sb+W1C5f107S7yVVSpoCfKuuN0VSV+BC4ISIGBERn0bElxFxX0SckW+zvKTLJE3PH5dJWj5ft4ukdySdlleLMyQdma+7ADgPOCSP/8c1P/HX7MrLz39K/p6+Ienwmu9LmT+vX0l6Im/nYUn1Tf3+BXAPcGjV+wccAtxa4726XNLU/Oc7XtKO+fI9gf9Xcp7PlcRR8/emtGv0Gkl3lbR/kaRHpAQvqyqCKxlLzDhgNHB6zRWSVgHuB64AugOXAvfX+GT7A+AYYCXgrXzZN4EtgW2AM4Frge8DfYCNge/l21UAN5B9qu8LzAX+2FDAETE0IjpHRGdgA7JP3H/LVw8D5gNfA7YAvgH8JF93NLBPvnwQWZdVXbYFVgDurmebX+TnuDmwGbAVcG7J+tWBrsCawI+BqyR1i4jzyaqjv+Xn8df6zldZd+YVwF4RsRKwHTCxlu3K+XkdBhwJrAp0oJafew03AT/Mn38TeBGYXmObsWTvwSrAbcCdklaIiAdrnOdmJfvU9ntT5TRgkzyB7kj23v0oIqKBWK2VcpKx84CTJPWssfxbwGsRcXNEzI+I24GXgW+XbDMsIibl67/Mlw2NiI8jYhLZH6WHI2JKRMwGHiD7I09EzIyIuyLis4j4BPgNsHO5QUvqSPZJ+/KIeEDSasDewM/yyuN94A/kn8SBg4HLImJqRHwI/Lae5rsDlQ10Zx0OXBgR70fEB8AFZH88q3yZr/8yIkYBc4AB5Z5fDQuBjSV1jIgZ+XtbUzk/rxsi4tWImAsMJ0sOdYqIMcAqkgaQJZubatnmlvxnOT8iLgGWp+HzrO33pqq9z8jex0uBW4CTIuKdBtpbRnhMxhIUES8C/wDOrrFqDRb/lPkW2SfzKlNrafK9kudza3ndGUBSJ0l/lvSWpI+Bx4CVq7q3yvBX4JWIuCh/vRawHDBD0ixJs4A/k31qrzqf0nhrnlupmUAP1X/lWc3356182aI2aiSpz8jPvTEi4lOybqrjyM7tfknrlxFPVUylP693mxDPzcCJwK7UUtlJOl3SS3kX3Syy6q2hb2Cs7fdmkYh4GphCNgoxvIwYlx0VavqjpUJusSNba3I+WXdS6R+k6WR/uEv1BaaVvF6SLozTyD7xbh0RXYCd8uUN/t8g6WxgPbKulCpTgXlAj4hYOX90iYiN8vUzyLrsqvSt5xBP5m3tX882Nd+fvizelVSuT4FOJa9XL10ZEQ9FxB5AL7Lq5Loy4qmKaVot2zbGzcDxwKi8ylgk7846k6xK7BYRKwOz+epnWNfvR72/N5JOIKuIpuftG7BoWhlXMpaaiHidbFzj5JLFo4D1lF3G217SIcCGZFXP0rASWWUzKx9POL+cnSTtlcd5QN7tU3UOM4CHgUskdZFUoeyig6ouuOHAyZJ6S+rG4pUbJW3NJutGvErS/nnVtZykvSQNzTe7HThXUs98AP08su6dppgI7CSpb37RwTkl57uapP3ysZl5ZN1uC2tpo5CfV0S8QdaN+YtaVq9ENgb2AdBe0nlAl5L17wH91IgryCStB/yabBzvB8CZkjZvWvRtkK8us4RdCCy6ZyYiZpINlJ9G1n10JrBPRFQupeNdBnQEKoGngAfL3O8QoCfwkr66wuxP+bofkg1oTwY+Av5O9ukfsk//DwHPAROAEfUdJB9fOJVsMP8DskrpRLJxIMj+EI4DngdeyNv8dZnnUPNY/yRL8s8D46meGCryOKYDH5L9wf9pLW0U9vOKiP9GRG1V2kNkP7dXybrmPqd6V1jVjaYzJU1o6Dh59+QtwEUR8VxEvEZ2hdrNVVfuWXrkizbMzFq/ii69Y/mtTmzy/p8/cs74iBi0FEMqiyfINDNLRYK3CznJmJmlIsEJEtKL2MzMkuFKxswsBS18lVhTOcmYmaUiwe4yJ5kmUvuOoQ4rtXQY1spssUF993jasmrChPGVEVFz6qbGcyWz7FCHlVh+wMEtHYa1Mk883eAcn7YM6ric6pvGqExKspJJL2IzM0uGKxkzs1S4u8zMzApRNUFmYpxkzMySkOaYjJOMmVkqEuwuSy8tmplZMlzJmJmlwt1lZmZWmAS7y5xkzMxSoDQH/tOL2MzMkuFKxswsFe4uMzOzoshJxszMiiCcZMzMrCjKH4nxwL+ZmRXGlYyZWRLk7jIzMyuOk4yZmRXGScbMzAqTYpLxwL+ZmRXGlYyZWQp8CbOZmRVF+dVlTX2UdQxpT0mvSHpd0tm1rO8r6d+SnpX0vKS9G2rTlYyZWSKKHJOR1A64CtgDeAcYK2lkREwu2excYHhEXCNpQ2AU0K++dl3JmJkZwFbA6xExJSK+AO4A9quxTQBd8uddgekNNepKxswsEQVfXbYmMLXk9TvA1jW2GQI8LOkkYEVg94YadSVjZpaIJRyT6SFpXMnjmCaE8D1gWET0BvYGbpbq/yY1VzJmZilY8qvLKiNiUD3rpwF9Sl73zpeV+jGwJ0BEPClpBaAH8H5djbqSMTNLRMFXl40F+ktaW1IH4FBgZI1t3gZ2y2PZAFgB+KC+Rp1kzMyMiJgPnAg8BLxEdhXZJEkXSto33+w04GhJzwG3A0dERNTXrrvLzMwSoGaYhTkiRpFdlly67LyS55OB7RvTppOMmVkiUpy7zEnGzCwV6eUYj8mYmVlxXMmYmaVA7i4zM7MCOcmYmVlhnGTMzKwQzXEJcxE88G9mZoVxJWNmlor0ChknGTOzJPjqMjMzK5KTjJmZFSbFJOOBfzMzK4wrGTOzVKRXyDjJmJmlIsXuMicZM7MENOIbLlsVj8mYmVlhXMmYmSUixUrGScbMLBFOMmZmVpz0coyTjJlZKlKsZDzwb2ZmhXElY2aWAk+QaWZmRRGQYI5xkjEzS0OaN2M6yZiZJSLBHOOBfzMzK44rGTOzRLi7zMzMiqE0u8ucZMzMEiCgoiK9LOMxGavXbttuwDN//yXjR5zPz360x2Lr+6zejXuuPon/3nYO9/3pFNZYdeVF63qv1o27rjyBp4afy5N/+wV9eq3SjJFbkf41ZjJfP+hCBh4whD8Me3ix9fO++JKjzrmegQcMYfcjLubt6TOrrZ/67of03ulUrrz5X80VsrWQwpKMpJB0Scnr0yUNaWCf/SVtWM/6YyS9nD+ekbRDGXHsK+nsRgVvQPap6eIzD+a7p1zNNgf/moO+sSUD1l692jYXnnIAd9z/DDsc9luG/uUBzjth30Xrrrngh1x58yNsc/Cv2f2Ii6n88JPmPgUrwIIFCzlj6HDuvPx4nhp+Lnc9PJ6Xp8yots3N9z5J1y4dmXD3EH562K4MufLeauvP/cMIdt9uo+YMu02Qmv5oKUVWMvOAAyX1aMQ++wO1JhlJ+wDHAjtExPrAccBtklavbfsqETEyIn7XiBgst+VG/ZgytZK3ps3ky/kLGPHPCey986bVthmwTi8eH/cKAI+Pe5W9dtokW7726rRvV8HoZ14G4NO5XzB33pfNewJWiPGT3mSdPj3o17sHHZZrz4F7DGTUf56vts0Djz3P9761NQD7Dd6C/4x9hYgA4P7Rz9F3je6sv069/+taLaq+uKwpj5ZSZJKZD1wL/LzmCkn9JD0q6XlJj0jqK2k7YF/gYkkTJa1bY7ezgDMiohIgIiYANwIn5G2+KekCSRMkvSBp/Xz5EZL+WNdx8+XDJF0haYykKZK+U8xbkpZePbsy7b2PFr2e/t5H9OrZtdo2k16dxj67bg7APrtuRpfOHenWdUXW7bsqsz+Zy01Df8J/bjmLC0/eP8n+ZFvcjA9ms+Zq3Ra9XmO1bsz4YHa1baa//9U27du3o0vnjnw4+1PmfDaPy2/6J2cdvXezxtwmLEEV01YrGYCrgMMlda2x/ErgxojYFLgVuCIixgAjyRLJ5hHxvxr7bASMr7FsXL68SmVEDASuAU6vJZ7FjluyrhewA7AP4MqnTL+8/G62H/g1/nPLWWw/8GtMe+8jFixYSPt2FWy7xbr88vK7Gfyji1lrzR4cts82LR2utbCLrr2fn35vMJ07Ld/SoSQnm1YmvUqm0KvLIuJjSTcBJwNzS1ZtCxyYP78ZGLqUDjki/3d8Sful6jvuPRGxEJgsabXaGpd0DHAMAMt1XhrxtmrlfGJ9t3I2PzzzLwCs2LED3951cz6eM5fp78/ihVff4a1p2YDvqNHPMWiTtbll5JPNdwJWiHIq3DVWzbZZc7VuzJ+/gI/nzGWVrisybtJb3PvoRM6/8h5mfzKXigqx/PLLcczBOzf3aVgzaY5LmC8DJgA3LGE7k4EtgUdLlm0JTCp5PS//dwGNP7d5Jc9rTfsRcS1ZFyAVnVaNRrafnAmT32Ldvj3pu0Z3Zrw/iwP3GMjRvxxWbZtVuq7IRx9/RkTw8yO+ya33PbVo366dO9J95c7MnDWHHb8+gIkvvd0CZ2FL28AN1+J/b3/AW9Mq6bXqyoz45wSu+9UR1bbZc8dNuP3+p9lq03W499Fn2enr6yGJB677qvf8d9fez4odl3eCKZvnLqtVRHwoaTjwY+D6fPEY4FCyauJw4PF8+SfASnU0NRS4SNKeETFT0ubAEcDWjQinruNaLRYsWMiZQ4dz1xUn0K6duHXkU7w85V3OOfZbTHzpbR547AV22LI/552wLxEw5tnXOWPocAAWLgx+efk93Hv1SUhi4stvc+PdT7TwGdnS0L59O4aeeTAHnXwVCxYEh++7DRus24v/+9M/2HyDvuy986b8YL/tOO78mxh4wBC6dVmRv/7myJYOu01IMMegqis+lnrD0pyI6Jw/Xw14AxgaEUMkrUVW2fQAPgCOjIi3JW0PXEdWVXyn5riMpJ8CPwOCLCGdFhGP5eveBAZFRKWkQcDvI2IXSUfky0+s57jDgH9ExN9rxl6Xik6rxvIDDl7Cd8namo/G/rGlQ7BWqONyGh8Rg5akjU5rDIgBx17T5P0nDtltiWNoisIqmdI/0hHxHtCp5PVbwOBa9nmCOi5hztdfQzaoX9u6fiXPxwG75M+HAcMaOO4RdcVuZtYqJDqtjO/4NzOzwnjuMjOzBFRdwpwaJxkzs0QkmGOcZMzMUpFiJeMxGTMzK4wrGTOzRCRYyDjJmJklQWl2lznJmJklILu6rKWjaDwnGTOzJKQ5d5kH/s3MrDCuZMzMEpFgIeMkY2aWihS7y5xkzMxSkOgEmU4yZmYJSHXuMg/8m5lZYVzJmJklIsVKxknGzCwRCeYYJxkzs1SkWMl4TMbMzArjSsbMLAW+hNnMzIqiROcuc5IxM0tEgjnGScbMLBUVCWYZD/ybmRkAkvaU9Iqk1yWdXcc2B0uaLGmSpNsaatOVjJlZIoosZCS1A64C9gDeAcZKGhkRk0u26Q+cA2wfER9JWrWhdp1kzMwSoOK/fnkr4PWImJIdT3cA+wGTS7Y5GrgqIj4CiIj3G2rU3WVmZomoUNMfQA9J40oex9Rofk1gasnrd/JlpdYD1pP0hKSnJO3ZUMyuZMzMErGElUxlRAxawhDaA/2BXYDewGOSNomIWXXt4ErGzMwApgF9Sl73zpeVegcYGRFfRsQbwKtkSadOTjJmZomQmv4ow1igv6S1JXUADgVG1tjmHrIqBkk9yLrPptTXqLvLzMwSILK7/osSEfMlnQg8BLQDro+ISZIuBMZFxMh83TckTQYWAGdExMz62nWSMTNLREXB92JGxChgVI1l55U8D+DU/FEWd5eZmVlhXMmYmaVAniDTzMwKlGCOcZIxM0uBSHOCTCcZM7NEJJhjPPBvZmbFcSVjZpYID/ybmVkhGnHnfqtSZ5KRdCUQda2PiJMLicjMzGrV1gb+xzVbFGZm1qD0Ukw9SSYibix9LalTRHxWfEhmZtZWNHh1maRt88nQXs5fbybp6sIjMzOzapTf9d+UR0sp5xLmy4BvAjMBIuI5YKcCYzIzsxqymzGX6JsxW0RZV5dFxNQamXBBMeGYmVmt2vDcZVMlbQeEpOWAU4CXig3LzMzagnKSzHHA5cCawHSyL605ocigzMxscQkWMg0nmYioBA5vhljMzKweKXaXlXN12TqS7pP0gaT3Jd0raZ3mCM7MzDKpDvyXc3XZbcBwoBewBnAncHuRQZmZ2eLa6iXMnSLi5oiYnz9uAVYoOjAzM0tffXOXrZI/fUDS2cAdZHOZHQKMaobYzMysRHojMvUP/I8nSypV53VsyboAzikqKDMzq05qYxNkRsTazRmImZnVL8EcU94d/5I2BjakZCwmIm4qKigzM2sbGkwyks4HdiFLMqOAvYD/Ak4yZmbNqE3eJwN8B9gNeDcijgQ2A7oWGpWZmS2m6tsxm/JoKeV0l82NiIWS5kvqArwP9Ck4LjMzKyHUtgb+S4yTtDJwHdkVZ3OAJ4sMyszMamjhiqSpypm77Pj86Z8kPQh0iYjniw3LzMzagvpuxhxY37qImFBMSGZmVpsUB/7rq2QuqWddAIOXcixJ2XhAH0Y9Wt9bZMuibvte0dIhWBtWzpVarU19N2Pu2pyBmJlZ3UTbq2TMzKwVackp+5sqxerLzMwS4UrGzCwRbbKSUeb7ks7LX/eVtFXxoZmZWZXszv22+aVlVwPbAt/LX38CXFVYRGZmVqsUv365nO6yrSNioKRnASLiI0kdCo7LzMzagHKSzJeS2pHdG4OknsDCQqMyM7PFJHgFc1lJ5grgbmBVSb8hm5X53EKjMjOzakQb+2bMKhFxq6TxZNP9C9g/Il4qPDIzM6smxXtOyvnSsr7AZ8B9pcsi4u0iAzMzs+oSLGTK6i67n2w8RmRfv7w28AqwUYFxmZlZG1BOd9kmpa/z2ZmPr2NzMzMrgNR2v7SsmoiYIGnrIoIxM7O6JZhjyhqTObXkZQUwEJheWERmZlarFKeVKaeSWank+XyyMZq7ignHzMxq0yYvYc5vwlwpIk5vpnjMzKwNqe/rl9tHxHxJ2zdnQGZmVrsEC5l6K5lnyMZfJkoaCdwJfFq1MiJGFBybmZlVaeGJLpuqnDGZFYCZwGC+ul8mACcZM7NmJNLLMvUlmVXzK8te5KvkUiUKjcrMzNqE+pJMO6Az1Jo6nWTMzJpRdnVZS0fRePUlmRkRcWGzRWJmZvVqa0kmwdMxM2u7WvJrlJuqviSzW7NFYWZm9Uq1u6zOryeIiA+bMxAzM2t7Gj1BppmZtQC1vZsxzcysFWlzc5eZmVnrkOqYjJOMmVkiEixk6h74NzMzW1JOMmZmSRAVS/Ao6wjSnpJekfS6pLPr2e4gSSFpUENturvMzCwBotjusvz7w64C9gDeAcZKGhkRk2tstxJwCvB0Oe26kjEzS0E+1X9TH2XYCng9IqZExBfAHcB+tWz3K+Ai4PNyGnWSMTMzgDWBqSWv38mXLSJpINAnIu4vt1F3l5mZJWIJ75PpIWlcyetrI+LacneWVAFcChzRmIM6yZiZJWApjMlURkR9A/XTgD4lr3vny6qsBGwMjM4n6lwdGClp34goTV7VOMmYmSWi4Dv+xwL9Ja1NllwOBQ6rWhkRs4EeVa8ljQZOry/BgMdkzMySITX90ZCImA+cCDwEvAQMj4hJki6UtG9TY3YlY2ZmAETEKGBUjWXn1bHtLuW06SRjZpYAkWbXk5OMmVkK1Pa+GdPMzFqR9FKMk4yZWRKyqf7TSzMpdvGZmVkiXMmYmSUivTrGScbMLBkJ9pY5yZiZpUFJXl3mMRkzMyuMKxkzswT4ZkwzMytUit1lTjJmZolIL8U4yZiZpSHRaWVS7OIzM7NEuJIxM0uAB/7NzKxQKXaXOcmYmSUivRTjJGNmlowEC5kku/jMzCwRrmTMzBKQDfynV8o4yZiZJSLF7jInGavXY8+8zK//eA8LFi7k4L235tjDdqu2/pnn/sdvrrqXV6bM4A+//D577bxZtfWffPo5ex05lD2235jzTzmwOUO3Au02cC1++5OdaNdO3PzwJC67a3y19b17dObqn32Drp2Xp12FuODGJ/jn+LcA2Khfdy49fjArdepALAwGn/Y35n25oCVOIzFCrmSKJak3cBWwIdl40j+AMyLii3r2GQUcFhGzmiXINmTBgoUMuXwEwy4+ltV7duWgn17G4O02on+/1Rdts8Zq3bjorEP56/DRtbZx2Q0P8vVN12mmiK05VFSIi4/dhQPOu5vpM+fw6CWH8MAzb/DK1A8XbXPaIVtxzxOvcf0DLzCgzyoMP29fNjt6GO0qxJ9P/SbHXfowL75ZSbeVVuDLBQtb8GysaMkM/Cu7QHwEcE9E9AfWAzoDv6lvv4jY2wmmaZ5/+W3WWrM7fdfoTofl2vOtwVvwyJhJ1bbpvfoqrL/uGqhi8U9YL746lZkffcIOg9ZrrpCtGWzZfzWmzJjFW+99zJfzFzLi8dfYe+saHyQiWKljBwC6dOrAux9+CsDgLfoy6c1KXnyzEoCPPvmchQujWeNPmdT0R0tJJskAg4HPI+IGgIhYAPwcOErS8ZJGSHpQ0muShlbtJOlNST3y56dKejF//Cxf1k/SS5KukzRJ0sOSOjb/6bU+71bOpteqKy96vXqPrrz3weyy9l24cCG/veY+zjru2wVFZy2lV/fOTKucs+j19Mo59Oq+YrVtfnf70xy8ywBevP4ohp+/L2deOxqAddfsRgT8fch+jP7DoZx84MDmDD1pVQP/TX20lJSSzEZAtY7fiPgYeJus229z4BBgE+AQSX1Kt5W0JXAksDWwDXC0pC3y1f2BqyJiI2AWcFBhZ7GMuPXeMey89fr06rlyS4diLeCgnQZw26MvsfFR13PwBSP508+/iQTtK8Q2G/bimEseYq+z/s63tlmXnTbt3dLhpmEJqpiWrGSSGpNpwCMRMRtA0mRgLWBqyfodgLsj4tN8mxHAjsBI4I2ImJhvNx7oV9sBJB0DHAOwZu8+tW3Spqzeoysz3p+16PW7lbNZrWfXsvZ9dvKbjHvhDW67dwyfzZ3HF/MX0KljB844Zp+CorXmMmPmHNbs0XnR6zV6dGbGzE+rbfP9PTbku0PuBWDsK++yQod2dO/Skekz5zBm0nQ+/ORzAP45/k02W3dVHnv+neY7gYSleHVZSpXMZGDL0gWSugB9gfnAvJJVC2hcAi1r34i4NiIGRcSgVXr0bETzadpk/T68Oa2SqTNm8sWX87n/0WfZbduNytr30l98n8fu+CWjbz+Xs477NgfsMcgJpo2Y8Np7rLvGyvRdrQvLta/gwB3788DTU6ptM+2DT9hp0+yD2Hq9u7H8cu2onD2XRya8zYZrdadjh/a0qxDbb7RmtQsGrO1JqZJ5BPidpB9GxE2S2gGXAMOAz8rY/3FgmKTfkXVvHgD8oKhg24L27dpx/kkHctRZ17JgQfCdvbai/9qrc9kND7LJer3ZbfuNef7ltzn+vGF8PGcu/35yMlcMe4gHbjizpUO3Ai1YGJz559HcNWQ/2lVUcOu/JvHy1A8557Ctmfj6+zzwzBuce/1/ufzEwRy/3+ZEwAmX/wuA2Z/O4+p7n+WRSw+ByCqZh8e92bInlJAUL2FWRDpXduTjLFcD65NVYaOA04HvAYMi4sR8u38Av4+I0ZLezNdVSjoVOCpv7i8RcZmkfsA/ImLjfN/Tgc4RMaS+WDbdYssY9eiYpX2Klrj+h/+5pUOwVujzB04ZHxGDlqSNARtvHlf//V9N3n/3DXoucQxNkVIlQ0RMBWq7XGlY/qjabp+S5/1Knl8KXFqjzTeBjUte/34phWtmtlSlWMmkNCZjZmaJSaqSMTNblqV4dZmTjJlZIlLsLnOSMTNLgIBaZm9q9ZxkzMySkOYszB74NzOzwriSMTNLQQvPQdZUTjJmZolIMMc4yZiZpSAb+E8vzTjJmJklIr0U44F/MzMrkCsZM7NUJFjKOMmYmSUixftknGTMzBKR4Li/x2TMzKw4rmTMzBKRYCHjJGNmlowEs4yTjJlZAoQH/s3MrCiJzl3mgX8zMyuMKxkzs0QkWMg4yZiZJSPBLOMkY2aWhDS/GdNJxswsER74NzMzK+FKxswsASLJIRknGTOzZCSYZZxkzMwSkeLAv8dkzMysMK5kzMwSkeLVZU4yZmaJSDDHuLvMzCwJWsJHOYeQ9pT0iqTXJZ1dy/pTJU2W9LykRySt1VCbTjJmZonQEvzXYNtSO+AqYC9gQ+B7kjassdmzwKCI2BT4OzC0oXadZMzMDGAr4PWImBIRXwB3APuVbhAR/46Iz/KXTwG9G2rUScbMLAEiG/hv6qMMawJTS16/ky+ry4+BBxpq1AP/ZmaJWMKB/x6SxpW8vjYirm1SHNL3gUHAzg1t6yRjZpaKJcsylRExqJ7104A+Ja9758uqhyDtDvwC2Dki5jV0UCcZM7NEFHzH/1igv6S1yZLLocBh1Y4vbQH8GdgzIt4vp1GPyZiZGRExHzgReAh4CRgeEZMkXShp33yzi4HOwJ2SJkoa2VC7rmTMzBJR9B3/ETEKGFVj2Xklz3dvbJtOMmZmiUjxjn8nGTOzVCSYZTwmY2ZmhXElY2aWgGwKsvRKGScZM7MUlH/nfqviJGNmlogEc4yTjJlZMhLMMh74NzOzwriSMTNLQnnfC9PaOMmYmSXCA/9mZlaIRnyLcqviJGNmlooEs4wH/s3MrDCuZMzMEuGBfzMzK4wH/s3MrDAJ5hiPyZiZWXFcyZiZpcATZJqZWbHSyzJOMmZmCRCuZMzMrEAJ5hgP/JuZWXFcyTTRCxMnVPZZZYW3WjqOVqIHUNnSQVir49+Lr6y1NBpxd9kyJCJ6tnQMrYWkcRExqKXjsNbFvxdLn+/4NzOz4qSXY5xkzMxSkWCO8cC/LRXXtnQA1ir598JcydiSiwj/MbHF+Pdi6ZLv+DczsyKlOPDv7rJlgKSQdEnJ69MlDWlgn/0lbVjP+mMkvZw/npG0Qxlx7Cvp7EYFb62CpN6S7pX0mqT/SbpcUocG9hklaeVmCnHZoCV4tBAnmWXDPOBAST0asc/+QK1JRtI+wLHADhGxPnAccJuk1etrMCJGRsTvGhGDtQKSBIwA7omI/sB6QGfgN/XtFxF7R8Ss4iO01sxJZtkwn2wQ9uc1V0jqJ+lRSc9LekRSX0nbAfsCF0uaKGndGrudBZwREZUAETEBuBE4IW/zTUkXSJog6QVJ6+fLj5D0x7qOmy8fJukKSWMkTZH0nWLeEmuEwcDnEXEDQEQsIPtdOkrS8ZJGSHowr3KGVu2U/x70yJ+fKunF/PGzfFk/SS9Juk7SJEkPS+rY/KeXjgQLGSeZZchVwOGSutZYfiVwY0RsCtwKXBERY4CRZIlk84j4X419NgLG11g2Ll9epTIiBgLXAKfXEs9ixy1Z1wvYAdgHcOXT8hb7eUfEx8DbZOO6mwOHAJsAh0jqU7qtpC2BI4GtgW2AoyVtka/uD1wVERsBs4CDCjuLNqBq8L8pj5biJLOMyP8o3AScXGPVtsBt+fObyf64Lw0j8n/HA/1qWV/fce+JiIURMRlYbSnFY8V5JCJmR8TnwGQWn0JlB+DuiPg0IuaQ/W7smK97IyIm5s/r+l0xIBv2b/p/LcVJZtlyGfBjYMUlbGcysGWNZVsCk0pez8v/XUDjr2KcV/I8vctp2p7Fft6SugB9ybpiS39ejf15L8m+y5Sqqf5dyVirFREfAsPJEk2VMcCh+fPDgcfz558AK9XR1FDgIkndASRtDhwBXN2IcOo6rrU+jwCdJP0QQFI74BJgGPBZGfs/DuwvqZOkFYED8M97meEks+y5hGx23ConAUdKeh74AXBKvvwO4AxJz9Yc+I+IkcD1wBhJLwPXAd+PiBmNiKOu41orExFBlhi+K+k14FXgc+D/lbn/BLKE9AzwNPCXiHi2mGittVH2+2NmZq3ZFgMHxb+feLrJ+3fr1H58S8yK7f5PM7NEpHjHv5OMmVkKEp27zGMyZmZWGFcyZmYJaOk795vKScbMLBUJZhl3l1nSJC3I51d7UdKdkjotQVvDquZKk/SXBmah3iWf462xx1g0n1c5y2tsM6eRxxoiqbYpfSxRvuPfrPnNzedX2xj4gmxG6EUkNalaj4if5NPa1GUXoNFJxmxZ4yRjbcnjwNfyKuNxSSOByZLaSbpY0th81udjIZvCXtIfJb0i6V/AqlUNSRotaVD+fM98Runn8hmj+5Els5/nVdSOknpKuis/xlhJ2+f7ds9nF54k6S+U0eEh6R5J4/N9jqmx7g/58kck9cyXrZvPgjw+P+/1l8q7aa1OitPKeEzG2oS8YtkLeDBfNBDYOCLeyP9Qz46Ir0taHnhC0sPAFsAAsu/NWY1sjq7ra7Tbk2xGg53ytlaJiA8l/QmYExG/z7e7DfhDRPw3/9qCh4ANgPOB/0bEhZK+RfUpfepyVH6MjsBYSXdFxEyyOefGRcTPJZ2Xt30i2dc4HBcRr0nammx6n8FNeButlUtwSMZJxpLXUdLE/PnjwF/JurGeiYg38uXfADYt+W6armRTzO8E3J5/P8p0SY/W0v42wGNVbeXzv9Vmd2BDffWRsYukzvkxDsz3vV/SR2Wc08mSDsif98ljnQksBP6WL78FGJEfYzvgzpJjL1/GMSxFCWYZJxlL3dyI2Lx0Qf7H9tPSRcBJEfFQje32XopxVADb5NPd14ylbJJ2IUtY20bEZ5JGAyvUsXnkx51V8z2wtinFO/49JmPLgoeAn0paDkDSevlswI+RfclWO0m9gF1r2fcpYCdJa+f7rpIvrzlL9cNkk36Sb7d5/vQx4LB82V5AtwZi7Qp8lCeY9ckqqSoVQFU1dhhZN9zHwBuSvpsfQ5I2a+AYZs3GScaWBX8hG2+ZIOlF4M9kVfzdwGv5upuAJ2vuGBEfAMeQdU09x1fdVfcBB1QN/JN9Gdyg/MKCyXx1ldsFZElqElm32dsNxPog0F7SS2TfCvpUybpPga3ycxgMXJgvPxz4cR7fJGC/Mt4TS0yq3yfjWZjNzBIg6UGqf01HY1VGxJ5LK55yOcmYmVlh3F1mZmaFcZIxM7PCOMmYmVlhnGTMzKwwTjJmZlYYJxkzMyuMk4yZmRXm/wOPeBPk37W7jwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_preds = np.argmax(output_predictions.predictions, axis=1)\n",
    "labels = ['Not Onion', 'Onion']\n",
    "\n",
    "plot_confusion_matrix(y_preds, y_valid, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c19ed7a",
   "metadata": {},
   "source": [
    "#### Calculate Logistic Regression for Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "731d313d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8781810596579057"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg_clf = LogisticRegression(max_iter=3000)\n",
    "log_reg_clf.fit(X_train, y_train)\n",
    "log_reg_clf.score(X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d2f96599",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAFWCAYAAAC7G+CvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAst0lEQVR4nO3deZxWZf3/8dd7BlCQxQXcWEQNdxORXNBcW9DMtVyrr2apKVmamVY/Ur7ZotnXJcwvmqm45S4lLn0101xZRBMURVQWTRnXFFEYPr8/zhm652aWe2Y4M1zD++njPDjLda5znbnH+dzXcq6jiMDMzKwIVR1dADMz67wcZMzMrDAOMmZmVhgHGTMzK4yDjJmZFcZBxszMCtOlowtgZmbNq+69UcSSj1p9fny04N6IGLkCi1QRBxkzswTEkkWstsURrT5/0VOX9F2BxamYg4yZWQoESB1dihZzn4yZmRXGQcbMLBWqav1SSfbSSEkzJc2SdGYDxzeSdL+kZyQ9KGlAc3k6yJiZpUJq/dJs1qoGxgL7AlsBR0raqizZb4BrIuLTwBjgl83l6yBjZpYEFV2T2RGYFRGzI+IT4EbgwLI0WwEP5Ot/a+D4chxkzMxSUWBNBugPzC3ZnpfvK/U0cEi+fjDQS9I6TWXqIGNmtmroK2lyyXJ8K/I4HdhD0lPAHsB8oLapEzyE2cwsBaLiDvxG1ETE8CaOzwcGlmwPyPctExGvkddkJPUEDo2Id5u6qGsyZmZJaENTWWXNZZOAIZI2ltQNOAKYUK8EUl9pWaQ7C7iyuUwdZMzMUlFgx39ELAFGAfcCzwE3RcR0SWMkHZAn2xOYKekFYD3g3ObydXOZmZkBEBETgYll+0aXrN8C3NKSPF2TsZVS/qDXt/L1oyXdt4LzHywpJLX7Fy1J3SX9WdJ7km5uQz4r/OfSESTdLem/OrocSSi2uawQDjKrKEmvSHpT0hol+74l6cEOLFaDIuK6iPhCe19X0lH5KJwPJL2e/zHcbQVk/RWypoZ1IuKrrc2kqJ+LpD3zAHx72f7t8v0PVpjP2ZKubS5dROwbEVe3srirkMKfkymEg8yqrRr4XlszUaZT/S5JOg24EPgFWUAYBFxKBQ+fVWAj4IW8DXxltQDYpewZiP8CXlhRF+iMvzeFqpsg0zUZS8j5wOmS1mzooKQRkiblzTqTJI0oOfagpHMlPQIsBDbJv+WeJOlFSf+W9N+SNpX0qKT3Jd2Uj1pB0lqS/iJpgaR38vUG50GSdIykf+TrZ+Q1i7plsaSr8mN9JP0hr3XMl/TzfKoMJFVL+o2kGkmzgS819kOR1IdsyoyTI+K2iPgwIhZHxJ8j4od5mtUkXSjptXy5UNJq+bE9Jc2T9IO8tvi6pGPzY+cAo4HD8/IfV/6Nv7wpL7//2fnP9GVJR5f/XCr8vP5b0iN5PvdJamrq90+AO8hGGNVNOXI4cF3Zz+oiSXPzz3eKpM/m+0cCPy65z6dLylH+e1PaNPp7SbeW5P9rZXNlpTf9cBFck7HETAYeJHvAqh5JawN3ARcD6wC/Be4q+2b7deB4oBfwar7vi8AOwM7AGcA44Gtk4++3AY7M01UBfyT7Vj8I+Aj4XXMFjojzIqJnRPQEtiT7xv2n/PBVwBLgU8D2wBeAb+XHvg3sn+8fTtZk1ZhdgNWB25tI85P8HocC25FNyfHTkuPrA33Inpg+Dhgraa2I+BlZ7ehP+X38oan7VdaceTGwb0T0AkYA0xpIV8nndRRwLLAu0I0GPvcy1wDfyNe/CDwLvFaWZhLZz2Bt4HrgZkmrR8Q9Zfe5Xck5Df3e1PkBsG0eQD9L9rP7r4iIZspqKykHGRsNfFdSv7L9XwJejIjxEbEkIm4Ange+XJLmqoiYnh9fnO87LyLej4jpZH+U7svnQnoPuJvsjzwR8VZE3BoRCyPi32RDIfeotNCSupN9074oIu6WtB6wH/D9vObxJvA/5N/EgcOACyNibkS8TdMT+61D9uBaU81ZRwNjIuLNiFgAnEP2x7PO4vz44nzEzgfA5pXeX5mlwDaSukfE6/nPtlwln9cfI+KFiPgIuIksODQqIh4F1pa0OVmwuaaBNNfmn+WSiLgAWI3m77Oh35u6/BaS/Rx/C1wLfDci5jWT3yrCfTKWoIh4FvgLUD6t94Ys/y3zVerPZTSX5b1Rsv5RA9s9AST1kPS/kl6V9D7wELBmXfNWBf4AzIyIX+fbGwFdgdclvSvpXeB/yb61191PaXnL763UW2RTcDQ18qz85/Nqvm9ZHmVBaiH5vbdERHxI1kx1Itm93SVpiwrKU1em0s/rX60oz3iyZyf2ooGanaTTJT2XN9G9S1Z7a+4NjA393iwTEU8As8l6IW6qoIyrjiq1fumoInfYlW1l8jOy5qTSP0ivkf3hLjWI+tNMtKUJ4wdk33h3iojewO75/mb/b1D2novNyJpS6swFPgb6RsSa+dI7IrbOj79O/SkzBjVxicfyvA5qIk35z2cQyzclVepDoEfJ9vqlByPi3oj4PLABWe3k8grKU1em+Q2kbYnxwEnAxLyWsUzenHUGWS1xrYhYE3iP/3yGjf1+NPl7I+lkshrRa3n+BiybVsY1GUtNRMwi69c4pWT3RGAzZcN4u0g6nGya77+soMv2IqvZvJv3J/yskpMk7ZuX8+C82afuHl4H7gMukNRbUpWyQQd1TXA3AadIGiBpLZavuVGS13tkzYhjJR2U17q6StpX0nl5shuAn0rql3egjyZr3mmNacDukgblgw7OKrnf9SQdmPfNfEzW7La0gTwK+bwi4mWyZsyfNHC4F1kf2AKgi6TRQO+S428Ag9WCEWSSNgN+TtaP93XgDElDW1f6TsijyyxhY4Blz8xExFtkHeU/IGs+OgPYPyJqVtD1LgS6AzXA48A9FZ53ONAPeE7/GWF2WX7sG2Qd2jOAd8ieTN4gP3Y52XQZTwNTgduaukjev3AaWWf+ArKa0iiyfiDI/hBOBp4B/pnn+fMK76H8Wn8lC/LPAFOoHxiq8nK8BrxN9gf/Ow3kUdjnFRH/yCdGLHcv2ef2AlnT3CLqN4XVPWj6lqSpzV0nb568Fvh1RDwdES+SjVAbXzdyz9IjD9owM1v5VfUeEKvtOKrV5y+6/6wpzczCXAjPXWZmlooEHxdykDEzS0WCEySkV2IzM0uGazJmZino4FFireUgY2aWigSbyxxkWklduoe69eroYthKZvstm3rG01ZVU6dOqYmI8qmbWs41mVWHuvVitc0P6+hi2ErmkSeanePTVkHdu6qpaYwqpCRrMumV2MzMkuGajJlZKtxcZmZmhaibIDMxDjJmZklIs0/GQcbMLBUJNpelFxbNzCwZrsmYmaXCzWVmZlaYBJvLHGTMzFKgNDv+0yuxmZklwzUZM7NUuLnMzMyKogSDjJvLzMwSILIg09qlomtIIyXNlDRL0pkNHB8k6W+SnpL0jKT9msvTQcbMLAVq49Jc9lI1MBbYF9gKOFLSVmXJfgrcFBHbA0cAlzaXr4OMmZkB7AjMiojZEfEJcCNwYFmaAHrn632A15rL1H0yZmZJqLzZqxF9JU0u2R4XEeNKtvsDc0u25wE7leVxNnCfpO8CawCfa+6iDjJmZoloY5CpiYjhbSzCkcBVEXGBpF2A8ZK2iYiljZ3gIGNmloiCR5fNBwaWbA/I95U6DhgJEBGPSVod6Au82Vim7pMxM0tEwaPLJgFDJG0sqRtZx/6EsjRzgH3ysmwJrA4saCpTBxkzMyMilgCjgHuB58hGkU2XNEbSAXmyHwDflvQ0cANwTEREU/m6uczMLAUVDkVui4iYCEws2ze6ZH0GsGtL8nSQMTNLgNo+uqxDOMiYmSUixSDjPhkzMyuMazJmZolIsSbjIGNmlggHGTMzK0Y7jC4rgoOMmVkiUqzJuOPfzMwK45qMmVkC/JyMmZkVykHGzMyKk16McZ+MmZkVxzUZM7MUyM1lZmZWIAcZMzMrjIOMmZkVItUhzO74NzOzwrgmY2aWivQqMg4yZmZJ8OgyMzMrkoOMmZkVJsUg445/MzMrjGsyZmapSK8i4yBjZpaKFJvLHGTMzBIg+WFMMzOzelyTMTNLRIo1GQcZM7NEOMiYmVlx0osxDjJmZqlIsSbjjn8zMwNA0khJMyXNknRmA8f/R9K0fHlB0rvN5emajJlZCgqeIFNSNTAW+DwwD5gkaUJEzKhLExGnlqT/LrB9c/m6JmNmlgABUuuXCuwIzIqI2RHxCXAjcGAT6Y8EbmguU9dkzMyS0OaHMftKmlyyPS4ixpVs9wfmlmzPA3ZqsCTSRsDGwAPNXdRBxswsEW1sLauJiOErqChHALdERG1zCd1cZmZmAPOBgSXbA/J9DTmCCprKwDUZM7NkFDyEeRIwRNLGZMHlCOCoBsqwBbAW8FglmbomY2aWgjZ0+lcSmyJiCTAKuBd4DrgpIqZLGiPpgJKkRwA3RkRUUmzXZMzMEiCgqqrYhzEjYiIwsWzf6LLts1uSp4OMNWmfXbbklz/4CtVVVYy/81EuvPqv9Y4PXH8tLhn9Nfqu2ZN33l/ICaOv5rU332WbzfpzwY+OoFfP1Vlau5QL/ngvt/91agfdha1o//foDM664BZqly7l6weO4NRjvlDv+MefLOY7PxvPtOfnsHafNbjyF99k0IbrsHhJLaf8/Dqefn4utbVLOXy/HTnt2C920F1YeyisuUxSSLqgZPt0SWc3c85BkrZq4vjxkp7Plycl7VZBOQ5o6MlVa15VlTj/jMP46vcuZefDfs6hX9iBzTdev16aMd87mBvvepLdjvol511xN6NPzmrVHy1azHfOvoYRh5/LV065lF+cdii9e3bviNuwFay2dik/PO8mbr7oJB6/6afcet8Unp/9er004+98jD69uzP19rP5zlF7cfYldwJwx/9N5eNPlvDojT/hb+N/xFW3P8Kc197qiNtIUsHPyRSiyD6Zj4FDJPVtwTkHAQ0GGUn7AycAu0XEFsCJwPWS1m8ofZ2ImBARv2pBGSy3w9aDmT23hlfnv8XiJbXc9tep7LfHp+ul2XyTDXh48kwAHp78Avvuvi0AL815k9lzFwDwr5r3qHn73/Rdq2f73oAVYsr0V9hkYF8GD+hLt65dOOTzw5j492fqpbn7oWc48kvZIxYH7r09f580k4hAEgs/+oQlS2pZtOgTunWtptcaq3fEbSSp7sVlrVk6SpFBZgkwDji1/ICkwZIekPSMpPslDZI0AjgAOD+fF2fTstN+BPwwImoAImIqcDVwcp7nK5LOkTRV0j/zERBIOkbS7xq7br7/KkkXS3pU0mxJXynmR5KWDfr1Yf4b7yzbfu2Nd9igX596aaa/MJ/99xoKwP57bUfvnt1Zq88a9dIM22ojunbtwsvzagovsxXv9QXv0X+9tZZtb7jeWry+4L16aV578z9punSppnfP7rz93occuM/29OjejS32/Qnbfnk0o47eZ7nfF2tEwR3/RSl6dNlY4GhJfcr2XwJcHRGfBq4DLo6IR4EJZIFkaES8VHbO1sCUsn2T8/11aiJiGPB74PQGyrPcdUuObQDsBuwPuOZTof930e3sOuxT/P3aH7HrsE8x/413qK1duuz4euv05rIx32DUmGupcDCKdWJTpr9CdVUVz919LtPuPIex1z3AK/7yUZFsWpn0ajKFdvxHxPuSrgFOAT4qObQLcEi+Ph44bwVd8rb83ykl+Zdq6rp3RMRSYIak9RrKXNLxwPEAdO38TT+VfGP9V817fOOMKwBYo3s3vrzXUN7/IPuoe62xOn+68Dv8/NI/M/nZV9qt3FasSmq4G66bpem/3losWVLL+x98xNp91uCWeyazz4it6Nqlmn5r92Kn7TbhqefmMHhAS1rVLSXt8ZzMhcBxQFvrxDOAHcr27QBML9n+OP+3lpYH0I9L1hsM+xExLiKGR8Rwden8ndhTZ7zKpoP6MWjDdejapZpDPj+Mux+q3/a+dp81ln1LOvWYL3Ldnx8HoGuXasaf/21unPgEEx6Y1t5FtwIN22ojXpqzgFfn1/DJ4iXc9tep7Lt7/b66kZ/dlhvuegKAOx94it0/sxmSGLD+2jw8KevD+/Cjj5n87CsMGdzgdzpbTutrMZ22JgMQEW9Luoks0FyZ736U7IGe8cDRwMP5/n8DvRrJ6jzg15JGRsRbkoYCx9DIBG6NaOy61oDa2qWccd5N3HrxyVRXi+smPM7zs//FWSd8iWnPzeHuh/7JbjsMYfTJBxABjz41ix+edxMAB39+GCO2/xRr91mDo/bfGYCTzhnPsy80NkuFpaJLl2rOO+MwDj1lLLW1wdEH7MyWm27ALy77C0O3HMR+e3yarx84ghN/dg3DDj6btXqvwR/OPRaAb311d0aNuZZdDvs5ARz15Z3ZZkj/jr2hhCT4zjJUVDu5pA8iome+vh7wMnBeRJydz+D5R6AvsAA4NiLmSNoVuJysVvGV8n4ZSd8Bvg8EWUD6QUQ8lB97BRgeETWShgO/iYg9JR2T7x/VxHWvAv4SEbeUl70xVT3WjdU2P6yNPyXrbN6Z9LuOLoKthLp31ZS2Tk7ZY8PNY/MTft/q86edvU+by9AahdVkSv9IR8QbQI+S7VeBvRs45xEaGcKcH/89Wad+Q8cGl6xPBvbM168Crmrmusc0VnYzs5VCB48Say3PXWZmZoXxtDJmZgmoG8KcGgcZM7NEJBhjHGTMzFKRYk3GfTJmZlYY12TMzBKRYEXGQcbMLAlKs7nMQcbMLAHZ6LKOLkXLOciYmSWhY+cgay13/JuZWWFckzEzS0SCFRkHGTOzVKTYXOYgY2aWgkQnyHSQMTNLQKpzl7nj38zMCuOajJlZIlKsyTjImJklIsEY4yBjZpaKFGsy7pMxM7PCOMiYmaUgH8Lc2qWiS0gjJc2UNEvSmY2kOUzSDEnTJV3fXJ5uLjMzS4AKnrtMUjUwFvg8MA+YJGlCRMwoSTMEOAvYNSLekbRuc/m6JmNmloiCazI7ArMiYnZEfALcCBxYlubbwNiIeAcgIt5sLlMHGTOzRFRJrV4q0B+YW7I9L99XajNgM0mPSHpc0sjmMnVzmZnZqqGvpMkl2+MiYlwL8+gCDAH2BAYAD0naNiLebeoEMzNLQBu7ZGoiYngTx+cDA0u2B+T7Ss0DnoiIxcDLkl4gCzqTGsvUzWVmZglQ/vrl1i4VmAQMkbSxpG7AEcCEsjR3kNVikNSXrPlsdlOZuiZjZpaIqgKfxYyIJZJGAfcC1cCVETFd0hhgckRMyI99QdIMoBb4YUS81VS+DjJmZoko+on/iJgITCzbN7pkPYDT8qUibi4zM7PCuCZjZpaIBKcuc5AxM0uByJ76T42DjJlZIors+C+K+2TMzKwwrsmYmaWg8uddVioOMmZmiUgwxjjImJmlQFDpRJcrFQcZM7NEJBhj3PFvZmbFcU3GzCwR7vg3M7NCtOANlyuVRoOMpEuAaOx4RJxSSInMzKxBna3jf3ITx8zMrJ2lF2KaCDIRcXXptqQeEbGw+CKZmVln0ezoMkm75C+oeT7f3k7SpYWXzMzM6in4zZiFqGQI84XAF4G3ACLiaWD3AstkZmZlsocxW790lIpGl0XE3LJIWFtMcczMrEGdeO6yuZJGACGpK/A94Llii2VmZp1BJUHmROAioD/wGnAvcHKRhTIzs+UlWJFpPshERA1wdDuUxczMmpBic1klo8s2kfRnSQskvSnpTkmbtEfhzMwsk2rHfyWjy64HbgI2ADYEbgZuKLJQZma2vM46hLlHRIyPiCX5ci2wetEFMzOz9DU1d9na+erdks4EbiSby+xwYGI7lM3MzEqk1yPTdMf/FLKgUndfJ5QcC+CsogplZmb1SZ1sgsyI2Lg9C2JmZk1LMMZU9sS/pG2ArSjpi4mIa4oqlJmZdQ7NBhlJPwP2JAsyE4F9gX8ADjJmZu2oUz4nA3wF2Af4V0QcC2wH9Cm0VGZmtpy6t2O2ZukolQSZjyJiKbBEUm/gTWBgscUyM7NSQlSp9UtF15BGSpopaVY+qrj8+DH5g/nT8uVbzeVZSZ/MZElrApeTjTj7AHisohKbmdmKUXCNRFI1MBb4PDAPmCRpQkTMKEv6p4gYVWm+lcxddlK+epmke4DeEfFMpRcwM7Mk7AjMiojZAJJuBA4EyoNMizT1MOawpo5FxNS2XNjMzFqm4I7//sDcku15wE4NpDtU0u7AC8CpETG3gTTLNFWTuaCJYwHs3VTGnd3Wmw3gzvvO7+hi2EpmrcP/0NFFsE6skk70JvSVNLlke1xEjGthHn8GboiIjyWdAFxNM7GgqYcx92rhxc3MrCCizTWZmogY3sTx+dQf1DUg37dMRLxVsnkFcF5zF21jYDQzs/ZS8FT/k4AhkjaW1A04AphQmkDSBiWbB1DBW5IreuLfzMw6t4hYImkU2duPq4ErI2K6pDHA5IiYAJwi6QBgCfA2cExz+TrImJklouiXj0XERMpm2Y+I0SXrZ9HCyZEreTOmJH1N0uh8e5CkHVtyETMza5vsyf3O+dKyS4FdgCPz7X+TPbBjZmbtKMXXL1fSXLZTRAyT9BRARLyTdwqZmZk1qZIgszifbiAAJPUDlhZaKjMzW06CkzBXFGQuBm4H1pV0LtmszD8ttFRmZlaP6GRvxqwTEddJmkI23b+AgyKi2bHRZma2YqX4YGMlLy0bBCwkm05g2b6ImFNkwczMrL4EKzIVNZfdRdYfI7LXL28MzAS2LrBcZmbWCVTSXLZt6XY+O/NJjSQ3M7MCqAUvH1uZtPiJ/4iYKqmh6Z/NzKxACcaYivpkTivZrAKGAa8VViIzM2tQRz5U2VqV1GR6lawvIeujubWY4piZWUM65RDm/CHMXhFxejuVx8zMOpGmXr/cJZ/6edf2LJCZmTUswYpMkzWZJ8n6X6ZJmgDcDHxYdzAibiu4bGZmVqeDJ7psrUr6ZFYH3iJ7j3Pd8zIBOMiYmbUjkV6UaSrIrJuPLHuW/wSXOlFoqczMrFNoKshUAz2hwdDpIGNm1o6y0WUdXYqWayrIvB4RY9qtJGZm1qTOFmQSvB0zs86rI1+j3FpNBZl92q0UZmbWpFSbyxp9PUFEvN2eBTEzs86nxRNkmplZB1DnexjTzMxWIp1u7jIzM1s5pNon4yBjZpaIBCsyjXf8m5mZtZVrMmZmSRBVCT6+6CBjZpYAkWZzmYOMmVkKEp3q330yZmYGgKSRkmZKmiXpzCbSHSopJA1vLk/XZMzMElHkczKSqoGxwOeBecAkSRMiYkZZul7A94AnKsnXNRkzswTU9cm0dqnAjsCsiJgdEZ8ANwIHNpDuv4FfA4sqydRBxswsEVVSq5cK9AfmlmzPy/ctI2kYMDAi7qq0zG4uMzNLRBtby/pKmlyyPS4ixlV+bVUBvwWOaclFHWTMzFYNNRHRVEf9fGBgyfaAfF+dXsA2wIP5e23WByZIOiAiSoNXPQ4yZmYJEIX3b0wChkjamCy4HAEcVXcwIt4D+i4rj/QgcHpTAQYcZMzM0qBi34wZEUskjQLuBaqBKyNiuqQxwOSImNCafB1kzMwSUfSzmBExEZhYtm90I2n3rCRPBxkzswRkU/2n98i/hzCbmVlhXJMxM0tEevUYBxkzs2Qk2FrmIGNmlgYVOrqsKO6TMTOzwrgmY2aWgHZ4GLMQDjJmZolIsbnMQcbMLBHphRgHGTOzNBQ8rUxRUmziMzOzRLgmY2aWAHf8m5lZoVJsLnOQMTNLRHohxkHGzCwZCVZkkmziMzOzRLgmY2aWgKzjP72qjIOMmVkiUmwuc5CxJj086Xl+9fsJ1C5dyqEjd+TbR+xd7/jkZ2bzq8sm8MLs1zn/x0fzxd0/vezYBVfcxUNPPAfAiUd/jn33HNqeRbcC7bNdf355zM5UV1Ux/oGZXHjnM/WOD1hnDS49eXf69FiN6ipxzvWT+Ou0eazVczWuPm1vtt+0Hzc8+CJn/PGxDrqDFAklWJNJqk9G0gBJd0p6UdJLki6S1K2ZcyZKWrOditip1NYu5dzf3c5l5x7HhMtPZ+KD05j16hv10myw7pqce/phfGnvofX2//2J53juxfncetmp3HDxKfzxlr/zwYeL2rH0VpQqifO/OYKv/vI+dj7tVg7ddRM2779mvTQ/OGQodzz2MnuceQfHXfQ3fnPcCAA+XlzLL/40ldHjn+yAkltHSCbIKBsgfhtwR0QMATYDegLnNnVeROwXEe8WX8LO558z5zBww74M3GAdunXtwn57DOVvj06vl6b/+muz+SYbLjd+/6VX32CHbTemS3U1Pbp3Y/ONN+Afk2e2Z/GtIDt8qh+z33ifV9/8N4trl3Lbo7PZ7zODlkvXq3v2/a93j278652FACz8eAmPz3yDRYtr27XMnYXU+qWjJBNkgL2BRRHxR4CIqAVOBb4p6SRJt0m6J6/lnFd3kqRXJPXN10+T9Gy+fD/fN1jSc5IulzRd0n2Surf/7a183qh5nw36rblse71+fXjjrfcqOnfzTbKg8tGiT3jnvQ958umX+NeCd4spqLWrDdbuwfy3Ply2/dpbC9lgrTXqpfnVzVM57LOb8uylR3DTmV9ws9gKUNfx39qlo6TUJ7M1MKV0R0S8L2kO2X0MBbYHPgZmSrokIubWpZW0A3AssBPZ5/WEpL8D7wBDgCMj4tuSbgIOBa4t/pY6r12Hb86zL8zj6O//jrX79GS7LTeiqiql7zTWFofuuinX//1Fxv7lWT4zZF0uG7UHI06/jYiOLlnCOrhG0lqd6f/6+yPivYhYBMwANio7vhtwe0R8GBEfkDW9fTY/9nJETMvXpwCDG7qApOMlTZY0+e23alb4Daxs1uvbm9dLah9vLHiP9dbpU/H5Jxy1D7dddhpX/Pp4gmDwgL4FlNLa2+tvL6T/Ov+puWy4Tg9ef+fDemm+ttdm3PHYywBMevFNVu9azTq9Vm/XcnZGbi4r1gxgh9IdknoDg4AlZDWYOrW0rJZW0bkRMS4ihkfE8LXX6fx/MLfZfCBz5tcw7/W3+WTxEib+fRp77bJVRefW1i7l3fezPzwzZ7/GC7NfZ8QOmxVZXGsnU19awKbr92ZQv550ra7ikBGbcPfkOfXSzK/5gN232RCAzfr3YbWu1dS874Efq6KUmsvuB34l6RsRcY2kauAC4CpgYQXnPwxcJelXZM1lBwNfL6qwnUGX6mp+Muogjv/x5SxdupSDv7gjnxq8PpdcfS9bbzaAvXfZmn/OnMv3zrma9/+9kAcff46x4+9jwuWns6S2lq+fdikAPXuszq/OPJIu1dUdfEe2ItQuDc648jFu/fFIqqvEdQ++wPPz3uWsrw5j2uwa7p4yh5+Of5KLTtiNk760NRFw8u8fXnb+05ccRq8e3ejapYr9PrMRh557DzPnv9txN5SQFIcwKxJqJJU0ELgU2IKsFjYROB04EhgeEaPydH8BfhMRD0p6JT9WI+k04Jt5dldExIWSBgN/iYht8nNPB3pGxNlNlWXbocPizr8+sqJv0RK39YnXd3QRbCW06LZvTYmI4W3JY/Nthsalt/xfq8//3Jb92lyG1kipJkPekf/lBg5dlS916fYvWR9csv5b4Ldleb4CbFOy/ZsVVFwzsxUqxZpMSn0yZmaWmKRqMmZmqzIPYTYzs8KoDf9VlL80UtJMSbMkndnA8RMl/VPSNEn/kNTscFMHGTOzBAioUuuXZvPPRuyOBfYFtgKObCCIXB8R20bEUOA8yvq4G+IgY2aWhLbUYyqqyewIzIqI2RHxCXAjcGBpgoh4v2RzDaDZ4cnukzEzM4D+wNyS7Xlk03DVI+lk4DSgG9mckk1yTcbMLAVtmFImHzDQt25arHw5vjXFiIixEbEp8CPgp82ld03GzCwRbRxcVtPMw5jzgYEl2wPyfY25Efh9cxd1TcbMLAFZx79avVRgEjBE0sb5yyCPACbUK4M0pGTzS8CLzWXqmoyZWSKKfEwmIpZIGgXcC1QDV0bEdEljgMkRMQEYJelzwGKy16T8V3P5OsiYmRkAETGRbE7I0n2jS9a/19I8HWTMzFKR4BP/DjJmZolIcYJMBxkzs0R47jIzM7MSrsmYmSUiwYqMg4yZWTISjDIOMmZmCRDu+Dczs6LIHf9mZmb1uCZjZpaIBCsyDjJmZslIMMo4yJiZJaHiN1yuVBxkzMwS4Y5/MzOzEq7JmJklQCTZJeMgY2aWjASjjIOMmVkiUuz4d5+MmZkVxjUZM7NEpDi6zEHGzCwRCcYYBxkzsyQkOrzMQcbMLBHu+DczMyvhmoyZWQKEO/7NzKxACcYYBxkzs2QkGGUcZMzMEuGOfzMzsxKuyZiZJcId/2ZmVpgEY4yby8zMkqE2LJVkL42UNFPSLElnNnD8NEkzJD0j6X5JGzWXp4OMmZkhqRoYC+wLbAUcKWmrsmRPAcMj4tPALcB5zeXrIGNmloCsQtL6/yqwIzArImZHxCfAjcCBpQki4m8RsTDffBwY0Fym7pMxM0uB2tzx31fS5JLtcRExrmS7PzC3ZHsesFMT+R0H3N3cRR1kzMwS0caO/5qIGL5CyiF9DRgO7NFcWgcZM7NUFDu8bD4wsGR7QL6vfhGkzwE/AfaIiI+by9R9MmZmBjAJGCJpY0ndgCOACaUJJG0P/C9wQES8WUmmrsmYmSWh4g78VomIJZJGAfcC1cCVETFd0hhgckRMAM4HegI3K+sgmhMRBzSVr4OMmVkiin7iPyImAhPL9o0uWf9cS/N0kDEzS0Cib192kDEzS0aCUcYd/2ZmVhjXZMzMEpHi+2QcZMzMEuGp/s3MrDAJxhj3yZiZWXFckzEzS0HbJ8jsEA4yZmbJSC/KOMiYmSVAuCZjZmYFSjDGuOPfzMyK45pMKz379FM1m67b49WOLsdKoi9Q09GFsJWOfy/+Y6MVkYmby1YhEdGvo8uwspA0eUW9cc86D/9erHh+4t/MzIqTXoxxkDEzS0WCMcYd/7ZCjOvoAthKyb8X5pqMtV1E+I+JLce/FyuW/MS/mZkVKcWOfzeXrQIkhaQLSrZPl3R2M+ccJGmrJo4fL+n5fHlS0m4VlOMASWe2qPC2UpA0QNKdkl6U9JKkiyR1a+aciZLWbKcirhrUhqWDOMisGj4GDpHUtwXnHAQ0GGQk7Q+cAOwWEVsAJwLXS1q/qQwjYkJE/KoFZbCVgCQBtwF3RMQQYDOgJ3BuU+dFxH4R8W7xJbSVmYPMqmEJWSfsqeUHJA2W9ICkZyTdL2mQpBHAAcD5kqZJ2rTstB8BP4yIGoCImApcDZyc5/mKpHMkTZX0T0lb5PuPkfS7xq6b779K0sWSHpU0W9JXivmRWAvsDSyKiD8CREQt2e/SNyWdJOk2SffktZzz6k7Kfw/65uunSXo2X76f7xss6TlJl0uaLuk+Sd3b//bSkWBFxkFmFTIWOFpSn7L9lwBXR8SngeuAiyPiUWACWSAZGhEvlZ2zNTClbN/kfH+dmogYBvweOL2B8ix33ZJjGwC7AfsDrvl0vOU+74h4H5hD1q87FDgc2BY4XNLA0rSSdgCOBXYCdga+LWn7/PAQYGxEbA28Cxxa2F10AnWd/61ZOoqDzCoi/6NwDXBK2aFdgOvz9fFkf9xXhNvyf6cAgxs43tR174iIpRExA1hvBZXHinN/RLwXEYuAGSw/hcpuwO0R8WFEfED2u/HZ/NjLETEtX2/sd8WArNu/9f91FAeZVcuFwHHAGm3MZwawQ9m+HYDpJdsf5//W0vJRjB+XrKc3nKbzWe7zltQbGETWFFv6ebX0827LuauUuqn+XZOxlVZEvA3cRBZo6jwKHJGvHw08nK//G+jVSFbnAb+WtA6ApKHAMcClLShOY9e1lc/9QA9J3wCQVA1cAFwFLKzg/IeBgyT1kLQGcDD+vFcZDjKrngvIZset813gWEnPAF8HvpfvvxH4oaSnyjv+I2ICcCXwqKTngcuBr0XE6y0oR2PXtZVMRARZYPiqpBeBF4BFwI8rPH8qWUB6EngCuCIiniqmtLayUfb7Y2ZmK7Pthw2Pvz3yRKvPX6tHlykdMSu22z/NzBKR4hP/DjJmZilIdO4y98mYmRkAkkZKmilpVkNTQEnaPX/IekmlD0o7yJiZJaAtT/tXUgHKRw2OBfYlm1LqyAbmL5xDNpL0eirk5jIzs1QU21y2IzArImYDSLoROJDsOSkAIuKV/NjSSjN1TcaSJqk2n1/tWUk3S+rRhryuqmsCkHRFM7NQ75nP8dbSayybz6uS/WVpPmjhtc6W1NCUPpaoNj7x31fS5JLl+LLs+wNzS7bn5fvaxDUZS91HETEUQNJ1ZDNC/7buoKQuEbGkpZlGxLeaSbIn8AHZQ6VmKajpiCHMrslYZ/Iw8Km8lvGwpAnADEnVks6XNCmf9fkEyKawl/S7vKPz/4B16zKS9KCk4fn6yLyz8+l8xujBZMHs1LwW9VlJ/STdml9jkqRd83PXyWcXni7pCipo8JB0h6Qp+TnHlx37n3z//ZL65fs2zWdBnpLf9xYr5KdpK52Cp5WZD5RObjog39cmrslYpyCpC1mH5T35rmHANhHxcv6H+r2I+Iyk1YBHJN0HbA9sTtbJuR5Z2/OVZfn2I5vRYPc8r7Uj4m1JlwEfRMRv8nTXA/8TEf9Q9tqCe4EtgZ8B/4iIMZK+RP0pfRrzzfwa3YFJkm6NiLfI5pybHBGnShqd5z2K7DUOJ0bEi5J2IpveZ+9W/BhtJVfwCOZJwBBJG5MFlyOAo9qaqYOMpa67pGn5+sPAH4ARwJMR8XK+/wvAp0uGXPYhm2J+d+CG/P0or0l6oIH8dwYeqssrn/+tIZ8DttJ/vjL2ltQzv8Yh+bl3SXqngns6RdLB+frAvKxvAUuBP+X7rwVuy68xAri55NqrVXANS1GBUSYilkgaRfYFqRq4MiKmSxpD9uVmgqTPALcDawFflnRO/pqGRjnIWOqW9cnUyf/Yfli6C/huRNxblm6/FViOKmDnfLr78rJUTNKeZAFrl4hYKOlBYPVGkkd+3XfLfwbWORX9xH9ETAQmlu0bXbI+iawZrWLuk7FVwb3AdyR1BZC0mbLZgB8ie8lWtaQNgL0aOPdxYPe8CQFJa+f7y2epvo9s0k/ydEPz1YfImxwk7Uv2DbApfYB38gCzBVlNqk4VUFcbO4qsGe594GVJX82vIUnbNXMNs3bjIGOrgivI+lumSnoW+F+yWvztwIv5sWuAx8pPjIgFwPFkTVNP85/mqj8DB9d1/JO9DG54PrBgBtnAAIBzyILUdLJmsznNlPUeoIuk58jeCvp4ybEPgR3ze9gbGJPvPxo4Li/fdLJnG6yTSfV9Mp6F2cwsAZLuof5rOlqqJiJGrqjyVMpBxszMCuPmMjMzK4yDjJmZFcZBxszMCuMgY2ZmhXGQMTOzwjjImJlZYRxkzMysMP8fPdS3eq/MZZoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_y_preds = log_reg_clf.predict(X_valid)\n",
    "plot_confusion_matrix(lr_y_preds, y_valid, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08b0c484",
   "metadata": {},
   "source": [
    "#### Calculate Using Dummy Classifier for Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c44729ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6203587818105966"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dummy = DummyClassifier(strategy=\"most_frequent\")\n",
    "dummy.fit(X_train, y_train)\n",
    "dummy.score(X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "18dfc852",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZkAAAFeCAYAAABXSGLCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAov0lEQVR4nO3debxVdb3/8dcbcEBlUAEjETGvs5kCiVPmeC9o1/kqagNmWqlZqZlWPyXKrmKWaTRYKg6pYWKSolCYmaLGoJHgjBPDlUGGFFTAz++PtQ5sNufss8+wzuF7eD997Ad7fdda3/Xdg+ezP9/vWt+liMDMzKwI7Vq7AWZm1nY5yJiZWWEcZMzMrDAOMmZmVhgHGTMzK4yDjJmZFcZBxszMkHSTpHmSnq1jvSRdJ+llSdMk9a2mXgcZMzMDGAkMrLB+ELBT/jgb+GU1lTrImJkZEfEo8HaFTY4Fbo3Mk0BXST3rq9dBxszMqrEt8GbJ8qy8rKIOhTXHzMyaTfvO20esXN7o/WP5/OnAeyVFN0TEDU1uWD0cZMzMEhAr32OTXQc3ev/3nr7+vYjo34QmzAa2K1nulZdV5O4yM7MUCJAa/2i6McDn87PM9gOWRMTc+nZyJmNmlgoVlxdIuhM4BOgmaRZwObARQET8ChgLHAW8DCwDzqimXgcZMzMjIk6tZ30A5za0XgcZM7NUNE+3V4tykDEzS4IK7S4rioOMmVkqEsxk0guLZmaWDGcyZmYpEO4uMzOzojTb9S4tykHGzCwVzmTMzKwwCWYy6YVF2yBIekTSl/Lnp0sa38z195EUklr8h5akjpL+JGmJpLubUE+zvy+tQdKDkr7Q2u2wYjjIbKAkvZbfBW/zkrIvSXqkFZtVq4j4XUT8Z0sfV9JpkiZLekfS3PyP4UHNUPVJwDbA1hHxP42tpKj3RdIheQC+t6z8E3n5I1XWM1TS7fVtFxGDIuKWRjZ3A5JfJ9PYRytxkNmwtQe+3tRK8gnz2tR3SdIFwLXAj8gCQm/gF2Q3bmqq7YEXI2JlM9RVlPnA/pK2Lin7AvBicx2gLX5vCtX6E2Q2ij/gDdvVwEWSuta2UtIBkibl3TqTJB1Qsu4RSVdIepxssryP5b9yz5H0kqR/S/qBpB0lTZS0VNIoSRvn+28p6X5J8yUtyp/3qqMdQyQ9lj+/OM8sah4rJI3M13WRdGOedcyW9ENJ7fN17SX9WNICSTOBo+t6UyR1AYYB50bE6Ih4NyJWRMSfIuJb+TabSLpW0pz8ca2kTfJ1h0iaJenCPFucK+mMfN33gcuAU/L2n1n+i7+8Ky9//TPz9/RVSaeXvy9Vfl4/kPR4Xs94Sd3qeg+AD4A/AoNr3j/gFOB3Ze/VzyS9mX++UyR9Ki8fCHyn5HX+s6Qd5d+b0q7RX0q6p6T+qyRNkBIcjCiCMxlLzGTgEeCi8hWStgIeAK4DtgZ+AjxQ9sv2c2T3+u4EvJ6X/RfQD9gPuBi4Afgs2X0o9gRqJuFrB9xM9qu+N7Ac+Hl9DY6I4RGxRURsAexG9ov79/nqkcBK4D+AfYD/BL6UrzsL+Exe3p+sy6ou+wObAvdW2Oa7+WvcG/gEsC/wvZL1HwG6kN058ExghKQtI+Jysuzo9/nruLHS61XWnXkdMCgiOgEHAM/Usl01n9dpZDPn9gA2ppbPvcytwOfz5/8FPAvMKdtmEtl7sBVwB3C3pE0j4qGy1/mJkn1q+97UuBD4eB5AP0X23n0hn5xxA+fuMkvTZcDXJHUvKz8aeCkibouIlRFxJ/A88N8l24yMiOn5+hV52fCIWBoR08n+KI2PiJkRsQR4kOyPPBGxMCLuiYhlEfFv4Arg09U2WlJHsl/aP4uIByVtQzYN+TfyzGMe8FPyX+LAycC1EfFmRLwN/G+F6rcGFtTTnXU6MCwi5kXEfOD7ZH88a6zI16+IiLHAO8Au1b6+Mh8Ce0rqGBFz8/e2XDWf180R8WJELAdGkQWHOkXERGArSbuQBZtba9nm9vyzXBkR1wCbUP/rrO17U1PfMrL38SfA7cDXImJWPfXZesxBZgMXEc8C9wOXlK36KOv+ynydte/p/Sbreqvk+fJalrcAkLSZpF9Lel3SUuBRoGtN91YVbgReiIir8uXtye59MVfSYkmLgV+T/WqveT2l7S1/baUWkt1To9KZZ+Xvz+t52eo6yoLUMvLX3hAR8S5ZN9VXyF7bA5J2raI9NW0q/bz+rxHtuQ04DziUWjI7SRdJei7voltMlr1V6oaD2r83q0XEU8BMslGIUVW0ccPRTo1/tFaTW+3Itj65nKw7qfQP0hyyP9ylerP27Vab0oVxIdkv3gER0Rk4OC+v9/8GSZcAO5N1pdR4E3gf6BYRXfNH54jYI18/l7VvHdu7wiGeyOs6rsI25e9Pb9btSqrWu8BmJcsfKV0ZEeMi4kigJ1l28psq2lPTpnpvj1uP24BzgLF5lrFa3p11MVmWuGVEdAWWsOYzrOv7UfF7I+lcsoxoTl6/wZppZdxdZqmJiJfJxjXOLykeC+ys7DTeDpJOAXYny3qaQyeyzGZxPp5weTU7SRqUt/P4vNun5jXMBcYD10jqLKmdspMOarrgRgHnS+olaUvWzdwoqWsJWTfiCEnH5VnXRpIGSRqeb3Yn8D1J3fMB9MvIunca4xngYEm985MOLi15vdtIOjYfm3mfrNvtw1rqKOTziohXyboxv1vL6k5kY2DzgQ6SLgM6l6x/C+ijBpxBJmln4Idk43ifAy6WtHfjWt8G+ewyS9gwYPU1MxGxkGyg/EKy7qOLgc9ExIJmOt61QEdgAfAk8FCV+50CdAee05ozzH6Vr/s82YD2DGAR8AeyX/+Q/fofB/wTmAqMrnSQfHzhArLB/PlkmdJ5ZONAkP0hnAxMA/6V1/nDKl9D+bH+TBbkpwFTWDswtMvbMQd4m+wP/ldrqaOwzysiHouI2rK0cWSf24tkXXPvsXZXWM2FpgslTa3vOHn35O3AVRHxz4h4iewMtdtqztyz9MgnbZiZrf/ade4Vm+x7XqP3f2/CpVMion8zNqkqnrvMzCwVCV4u5CBjZpaKBCdIcJAxM0tBKw/gN1Z6YdHMzJLhTMbMLBXuLttwqEPH0MadWrsZtp7ZZ7dK13jahmrq1CkLIqJ86qaGS7C7zEGmkbRxJzbZ5eTWboatZx5/qt45Pm0D1HEjVZrGqEpyJmNmZgVKMJNJLyyamVkynMmYmaWgZoLMxDjImJklwWMyZmZWJI/JmJmZreFMxswsFe4uMzOzwiTYXeYgY2aWAnng38zMipRgJpNeWDQzs2Q4kzEzS4QSzGQcZMzMEiAcZMzMrCjKH4nxmIyZmRXGmYyZWRLk7jIzMyuOg4yZmRXGQcbMzAqTYpDxwL+ZmRXGmYyZWQoSPYXZQcbMLAHy2WVmZlYkBxkzMytMikHGA/9mZlYYZzJmZolIMZNxkDEzS4HPLjMzsyKlmMl4TMbMzArjTMbMLAGpXifjTMbMLBGSGv2osv6Bkl6Q9LKkS2pZ31vSXyU9LWmapKPqq9NBxswsFWrCo76qpfbACGAQsDtwqqTdyzb7HjAqIvYBBgO/qK9ed5eZmaVAhQ/87wu8HBEzASTdBRwLzCjZJoDO+fMuwJz6KnWQMTMzgG2BN0uWZwEDyrYZCoyX9DVgc+CI+ip1d5mZWSKaOCbTTdLkksfZjWjCqcDIiOgFHAXcJqliHHEmY2aWiCZ2ly2IiP4V1s8GtitZ7pWXlToTGAgQEU9I2hToBsyrq1JnMmZmCag5hbnAs8smATtJ2kHSxmQD+2PKtnkDOBxA0m7ApsD8SpU6kzEzS0WB4/4RsVLSecA4oD1wU0RMlzQMmBwRY4ALgd9I+ibZSQBDIiIq1esgY2ZmAETEWGBsWdllJc9nAAc2pE4HGTOzFBR/CnMhHGTMzBLhIGNmZoVJMcj47DIzMyuMMxkzs1Skl8g4yJiZpSLF7jIHGTOzBDRkyv71iYOMmVkiUgwyHvg3M7PCOJMxM0tEipmMg4yZWSrSizEOMmZmqXAmY2ZmxUh07jIP/JuZWWGcyZiZJUBAgomMg4yZWRp8MaaZmRUowRjjMRkzMyuOMxkzs0S4u8zMzIqhNLvLHGTMzBIgoF279KKMx2SsTtf/v9N5cdz/MvGu79S5zZUXnsSU0Zfz2B2XstcuvVaXDz56AJPvuYzJ91zG4KMHtERzrQX9ZeIMPnniMPoeP5Sfjhy/zvr3P1jBFy+9ib7HD+WIIVfzxpyFq9f95OZx9D1+KJ88cRgTnpjRks1OntT4R2spLMhICknXlCxfJGloPfscJ2n3CuvPlvR8/viHpIOqaMcxki5pUOMNgDvvf5KTzh9R5/ojD9idHXt3p98J3+cbP7qTay4ZDEDXzpvx7bMGccQZP+bwIVfz7bMG0aVTx5ZqthVs1aoP+dbwUdz9s3N4ctT3uGf8FJ6fOXetbW677wm6dO7I1HuH8tXTDmXo9fcB8PzMuYz+81Se+P13+cN153DRVaNYterD1ngZ1kKKzGTeB06Q1K0B+xwH1BpkJH0G+DJwUETsCnwFuEPSRypVGBFjIuLKBrTBchOffoVFS5fVuf6oT+/FXQ/8A4DJz75Gl04d2Wbrzhy+32488tTzLF66jCX/Xs4jTz3PEfvX+dvBEjNl+mt8bLtu9OnVjY036sAJR/Zl7N+mrbXNg49O49Q8gz32sH3426QXiAjG/m0aJxzZl0023ojtt+3Gx7brxpTpr7XCq0hTzY3LGvNoLUUGmZXADcA3y1dI6iPpYUnTJE2Q1FvSAcAxwNWSnpG0Y9lu3wa+FRELACJiKnALcG5e52uSvi9pqqR/Sdo1Lx8i6ed1HTcvHynpOkkTJc2UdFIxb0nb0rN7V2a/tWj18px5i+nZoys9e3RlVkn57Lzc2oa585ew7TZbrl7+6DZbMnf+krW2mTNvzTYdOrSn8xYdeXvJu+vu22Pdfa0OTegqa5PdZbkRwOmSupSVXw/cEhF7Ab8DrouIicAYskCyd0S8UrbPHsCUsrLJeXmNBRHRF/glcFEt7VnnuCXregIHAZ8BnPmY2Xolm1bGmcxaImIpcCtwftmq/YE78ue3kf1xbw6j83+nAH1qWV/puH+MiA8jYgawTW2V52NCkyVNjpXLm6nJ6Zo7f3HZr9KuzJ23mLnzFtOrpHzbvNzahp7du6ydwb61iJ7d1/4d+dEea7ZZuXIVS99ZzlZdNl9333nr7mt1aXyAabNBJnctcCaweRPrmQH0KyvrB0wvWX4//3cVDT89+/2S57V+IhFxQ0T0j4j+6uCB7Acf/ReDj94XgP579mHpO8t5a+FSJjz5HIcO2JUunTrSpVNHDh2wKxOefK6VW2vNpe/u2/PKG/N5ffYCPlixktF/nsqgg/daa5uBn/o4dz7wFAD3Pfw0B39yZyQx6OC9GP3nqbz/wQpen72AV96YT789+rTCq7CWUvh1MhHxtqRRZIHmprx4IjCYLJs4Hfh7Xv5voFMdVQ0HrpI0MCIWStobGAI05PzYuo5rtfjtD4dwYL+d2LrrFjx7/w+48oaxbNShPQA3j36M8Y9P58gD92DqvZez/L0VnDvsdgAWL13G1Tc+xMO3XAzA8BsfYnGFEwgsLR06tGf4xSdz4vkjWLUqOP2Y/dhtx5786Ff3s/duvTnq03vxuWMP4CuX30rf44eyZefNufGKMwDYbceeHHfEPux38hV0aN+Oqy8+mfbtfSVFtVK8GFMRUUzF0jsRsUX+fBvgVWB4RAyVtD1wM9ANmA+cERFvSDoQ+A1ZVnFS+biMpK8C3wCCLCBdGBGP5uteA/pHxAJJ/YEfR8Qhkobk5edVOO5I4P6I+EN52+vSbrMesckuJzfxXbK2ZtGkn7d2E2w91HEjTYmI/k2pY7OP7hK7fPmXjd7/maGHN7kNjVFYJlP6Rzoi3gI2K1l+HTisln0ep45TmPP1vyQb1K9tXZ+S55OBQ/LnI4GR9Rx3SF1tNzNbLyQ6rYzzVDMzK4znLjMzS0DNKcypcZAxM0tEgjHGQcbMLBXOZMzMrDAJxhgP/JuZWXGcyZiZpUDuLjMzs4JkZ5e1disazkHGzCwJrTvRZWM5yJiZJSLBGOOBfzMzK44zGTOzRLi7zMzMipHoBJkOMmZmCUh17jKPyZiZWWGcyZiZJSLFTMZBxswsEQnGGAcZM7NUOJMxM7NiJHp2mQf+zcysMM5kzMwSIM9dZmZmRUowxjjImJmlol2CUcZBxswsEQnGGA/8m5lZcZzJmJklQIneftmZjJlZItqp8Y9qSBoo6QVJL0u6pI5tTpY0Q9J0SXfUV6czGTOzRBSZyUhqD4wAjgRmAZMkjYmIGSXb7ARcChwYEYsk9aivXmcyZmYGsC/wckTMjIgPgLuAY8u2OQsYERGLACJiXn2VOsiYmSVCavwD6CZpcsnj7LLqtwXeLFmelZeV2hnYWdLjkp6UNLC+Nru7zMwsASK76r8JFkRE/yY2owOwE3AI0At4VNLHI2JxpR3MzCwB1Q7gN9JsYLuS5V55WalZwFMRsQJ4VdKLZEFnUl2VurvMzCwFyuYua+yjCpOAnSTtIGljYDAwpmybP5JlMUjqRtZ9NrNSpQ4yZmZGRKwEzgPGAc8BoyJiuqRhko7JNxsHLJQ0A/gr8K2IWFipXneXmZklouhrMSNiLDC2rOyykucBXJA/quIgY2aWAOEJMs3MrEAJxhiPyZiZWXGcyZiZJSLFCTIdZMzMElBy5X5S6gwykq4Hoq71EXF+IS0yM7NatbWB/8kt1gozM6tXeiGmQpCJiFtKlyVtFhHLim+SmZm1FfWeXSZp//zqzufz5U9I+kXhLTMzs7UUPK1MIao5hfla4L+AhQAR8U/g4ALbZGZmZbKLMYu9M2YRqjq7LCLeLIuEq4ppjpmZ1aqVM5LGqibIvCnpACAkbQR8nWzyNDMza0EJxpiqusu+ApxLdoe0OcDe+bKZmVlF9WYyEbEAOL0F2mJmZhWk2F1WzdllH5P0J0nzJc2TdJ+kj7VE48zMLJPqwH813WV3AKOAnsBHgbuBO4tslJmZrautnsK8WUTcFhEr88ftwKZFN8zMzNJXae6yrfKnD0q6BLiLbC6zUyi7c5qZmRUvvRGZygP/U8iCSs3r+nLJugAuLapRZma2NqmNTZAZETu0ZEPMzKyyBGNMdVf8S9oT2J2SsZiIuLWoRpmZ2bpSPIW53iAj6XLgELIgMxYYBDwGOMiYmVlF1ZxddhJwOPB/EXEG8AmgS6GtMjOzddTcHbMxj9ZSTXfZ8oj4UNJKSZ2BecB2BbfLzMxKCLWtgf8SkyV1BX5DdsbZO8ATRTbKzMzKtHJG0ljVzF12Tv70V5IeAjpHxLRim2VmZuXa1MC/pL6V1kXE1GKaZGZmbUWlTOaaCusCOKyZ22JmZhVUc6bW+qbSxZiHtmRDzMysbqKNdZeZmdn6pTWn7G+sFLMvMzNLhDMZM7NEtMlMRpnPSrosX+4tad/im2ZmZjWyK/fb5k3LfgHsD5yaL/8bGFFYi8zMrFYp3n65mu6yARHRV9LTABGxSNLGBbfLzMzKJHhyWVWZzApJ7cmujUFSd+DDQltlZmZtQjWZzHXAvUAPSVeQzcr8vUJbZWZmaxFt7M6YNSLid5KmkE33L+C4iHiu8JaZmdlaUrzmpJqblvUGlgF/Ki2LiDeKbJiZma0twUSmqu6yB8jGY0R2++UdgBeAPQpsl5mZlZDa6P1kIuLjpcv57Mzn1LG5mZnZag2+4j8ipkoaUERjzMysbgkmMlWNyVxQstgO6AvMKaxFZmZWqxSnlakmk+lU8nwl2RjNPcU0x8zMatMmT2HOL8LsFBEXtVB7zMysDal0++UOEbFS0oEt2SAzM6tdgolMxUzmH2TjL89IGgPcDbxbszIiRhfcNjMzq9HKE102VjVjMpsCC4HDWHO9TAAOMmZmLUikF2UqBZke+Zllz7ImuNSIQltlZmZryQb+W7sVDVcpyLQHtoBaQ6eDjJmZ1atSkJkbEcNarCVmZlZRW8tkEnw5ZmZtV2veRrmxKgWZw1usFWZmVlGbG5OJiLdbsiFmZlaB0rxOJsV74JiZWSIaPAuzmZm1jhTnLnMmY2aWgJoxmcY+qjqGNFDSC5JelnRJhe1OlBSS+tdXpzMZM7NEFJnI5BMijwCOBGYBkySNiYgZZdt1Ar4OPFVNvc5kzMwMYF/g5YiYGREfAHcBx9ay3Q+Aq4D3qqnUQcbMLAmiXRMeQDdJk0seZ5cdYFvgzZLlWXnZmhZIfYHtIuKBalvt7jIzswSIJneXLYiIesdQ6jy+1A74CTCkIfs5yJiZpaD4qf5nA9uVLPfKy2p0AvYEHslnHvgIMEbSMRExua5KHWTMzBJR8CnMk4CdJO1AFlwGA6fVrIyIJUC3mmVJjwAXVQow4DEZMzMDImIlcB4wDngOGBUR0yUNk3RMY+t1JmNmloBmGJOpV0SMBcaWlV1Wx7aHVFOng4yZWSJSvOLfQcbMLBEJxhgHGTOzFIg0B9FTbLOZmSXCmYyZWQrU9u6MaWZm65H0QoyDjJlZErKp/tMLMx6TMTOzwjiTMTNLRHp5jIOMmVkyEuwtc5AxM0uDfHaZmZkVwxdjmpmZlXEmY2aWCHeXmZlZYdILMQ4yZmZpSHRaGY/JmJlZYZzJmJklINWzyxxkzMwSkWJ3mYOMmVki0gsxDjJmZslIMJFJsovPzMwS4UzGzCwB2cB/eqmMg4yZWSLcXWZtyvX/73ReHPe/TLzrO3Vuc+WFJzFl9OU8dsel7LVLr9Xlg48ewOR7LmPyPZcx+OgBLdFca0F/mTiDT544jL7HD+WnI8evs/79D1bwxUtvou/xQzliyNW8MWfh6nU/uXkcfY8fyidPHMaEJ2a0ZLMTpyb911qSCjKSekm6T9JLkl6R9DNJG9ezz1hJXVuoiW3Knfc/yUnnj6hz/ZEH7M6OvbvT74Tv840f3ck1lwwGoGvnzfj2WYM44owfc/iQq/n2WYPo0qljSzXbCrZq1Yd8a/go7v7ZOTw56nvcM34Kz8+cu9Y2t933BF06d2TqvUP56mmHMvT6+wB4fuZcRv95Kk/8/rv84bpzuOiqUaxa9WFrvIwkSY1/tJZkgoyyE8RHA3+MiJ2AnYEtgCsq7RcRR0XE4uJb2PZMfPoVFi1dVuf6oz69F3c98A8AJj/7Gl06dWSbrTtz+H678chTz7N46TKW/Hs5jzz1PEfsv3tLNdsKNmX6a3xsu2706dWNjTfqwAlH9mXs36attc2Dj07j1DyDPfawffjbpBeICMb+bRonHNmXTTbeiO237cbHtuvGlOmvtcKrsJaSTJABDgPei4ibASJiFfBN4IuSzpE0WtJDeZYzvGYnSa9J6pY/v0DSs/njG3lZH0nPSfqNpOmSxkvyz+4q9OzeldlvLVq9PGfeYnr26ErPHl2ZVVI+Oy+3tmHu/CVsu82Wq5c/us2WzJ2/ZK1t5sxbs02HDu3pvEVH3l7y7rr79lh3X6tdzcB/Yx+tJaUgswcwpbQgIpYCb5CdwLA3cArwceAUSduVbiupH3AGMADYDzhL0j756p2AERGxB7AYOLGwV2Fm1hhN6Cpzd1nzmBARSyLiPWAGsH3Z+oOAeyPi3Yh4h6zr7VP5ulcj4pn8+RSgT20HkHS2pMmSJsfK5c3+AlIzd/7isl+lXZk7bzFz5y2mV0n5tnm5tQ09u3dZO4N9axE9u3dZa5uP9lizzcqVq1j6znK26rL5uvvOW3dfq5uDTLFmAP1KCyR1BnoDK4H3S1atomGnZ1e1b0TcEBH9I6K/OrhH7cFH/8Xgo/cFoP+efVj6znLeWriUCU8+x6EDdqVLp4506dSRQwfsyoQnn2vl1lpz6bv79rzyxnxen72AD1asZPSfpzLo4L3W2mbgpz7OnQ88BcB9Dz/NwZ/cGUkMOngvRv95Ku9/sILXZy/glTfm02+PPq3wKqylpHSdzATgSkmfj4hbJbUHrgFGAnWPTq/xd2CkpCvJujePBz5XVGPbgt/+cAgH9tuJrbtuwbP3/4ArbxjLRh3aA3Dz6McY//h0jjxwD6beeznL31vBucNuB2Dx0mVcfeNDPHzLxQAMv/EhFlc4gcDS0qFDe4ZffDInnj+CVauC04/Zj9127MmPfnU/e+/Wm6M+vRefO/YAvnL5rfQ9fihbdt6cG684A4DdduzJcUfsw34nX0GH9u24+uKTad8+pd+6ras1T0VuLEVEa7ehavk4yy+AXcmysLHARcCpQP+IOC/f7n7gxxHxiKTX8nULJF0AfDGv7rcRca2kPsD9EbFnvu9FwBYRMbRSW9pt1iM22eXk5n6JlrhFk37e2k2w9VDHjTQlIvo3pY5d9tw7fvGHvzR6/yN2697kNjRGSpkMEfEm8N+1rBqZP2q2+0zJ8z4lz38C/KSszteAPUuWf9xMzTUza1YpZjJJBRkzsw2Zp5UxMzMr4UzGzCwR7i4zM7NCCGiXXoxxkDEzS0PrzqbcWA4yZmYpaOUr9xvLA/9mZlYYZzJmZolIMJFxkDEzS0E28J9emHGQMTNLRHohxmMyZmZWIGcyZmapSDCVcZAxM0uEr5MxM7PCJDju7yBjZpaKBGOMB/7NzKw4zmTMzFKRYCrjIGNmlgDhgX8zMytKohNkOsiYmSUiwRjjgX8zMyuOMxkzs1QkmMo4kzEzS4Ka9F9VR5AGSnpB0suSLqll/QWSZkiaJmmCpO3rq9NBxswsEVLjH/XXrfbACGAQsDtwqqTdyzZ7GugfEXsBfwCG11evg4yZmQHsC7wcETMj4gPgLuDY0g0i4q8RsSxffBLoVV+lDjJmZglQEx9V2BZ4s2R5Vl5WlzOBB+ur1AP/ZmapaNrAfzdJk0uWb4iIGxrVDOmzQH/g0/Vt6yBjZpaIJl7xvyAi+ldYPxvYrmS5V162dhukI4DvAp+OiPfrO6iDjJlZIgq+4n8SsJOkHciCy2DgtLWPr32AXwMDI2JeNZV6TMbMzIiIlcB5wDjgOWBUREyXNEzSMflmVwNbAHdLekbSmPrqdSZjZpaIoq/FjIixwNiysstKnh/R0DodZMzMUtCA08TWJw4yZmaJ8FT/ZmZWCJHmVP8e+Dczs8I4kzEzS0SCiYyDjJlZMhKMMg4yZmaJSHHg32MyZmZWGGcyZmaJSPHsMgcZM7NEJBhjHGTMzJKRYJRxkDEzS0A2q0x6UcYD/2ZmVhhnMmZmKZAH/s3MrEAJxhgHGTOzZCQYZRxkzMySIA/8m5mZlXImY2aWCA/8m5lZIRK9+7KDjJlZMhKMMh6TMTOzwjiTMTNLRIpnlznImJklwgP/ZmZWmARjjIOMmVkSEp27zAP/ZmZWGGcyZmbJSC+VcZAxM0uASLO7zEHGzCwRCcYYBxkzs1Q4k9mAxPL5C957ZsTrrd2O9UQ3YEFrN2J90HGjEa3dhPWJvxdrbN/aDWgtDjKNFBHdW7sN6wtJkyOif2u3w9Yv/l40P1/xb2ZmxUkvxjjImJmlIsEY44sxrVnc0NoNsPWSvxfmTMaaLiL8x8TW4e9F81Ki08o4yJiZJSLFgX93l20AJIWka0qWL5I0tJ59jpO0e4X1Z0t6Pn/8Q9JBVbTjGEmXNKjxtl6Q1EvSfZJekvSKpJ9J2riefcZK6tpCTdwwqAmPVuIgs2F4HzhBUrcG7HMcUGuQkfQZ4MvAQRGxK/AV4A5JH6lUYUSMiYgrG9AGWw9IEjAa+GNE7ATsDGwBXFFpv4g4KiIWF9/CDUeCMcZBZgOxkmwQ9pvlKyT1kfSwpGmSJkjqLekA4BjgaknPSNqxbLdvA9+KiAUAETEVuAU4N6/zNUnflzRV0r8k7ZqXD5H087qOm5ePlHSdpImSZko6qZi3xBrgMOC9iLgZICJWkX2XvijpHEmjJT2UZznDa3bKvwfd8ucXSHo2f3wjL+sj6TlJv5E0XdJ4SR1b/uVZkRxkNhwjgNMldSkrvx64JSL2An4HXBcRE4ExZIFk74h4pWyfPYApZWWT8/IaCyKiL/BL4KJa2rPOcUvW9QQOAj4DOPNpfet83hGxFHiDbFx3b+AU4OPAKZK2K91WUj/gDGAAsB9wlqR98tU7ASMiYg9gMXBiYa+iDagZ/G/Mo7U4yGwg8j8KtwLnl63aH7gjf34b2R/35jA6/3cK0KeW9ZWO+8eI+DAiZgDbNFN7rDgTImJJRLwHzGDdKVQOAu6NiHcj4h2y78an8nWvRsQz+fO6visGZMP+jf+vtTjIbFiuBc4ENm9iPTOAfmVl/YDpJcvv5/+uouFnMb5f8jy902nannU+b0mdgd5kXbGln1dDP++m7LtBqZnq35mMrbci4m1gFFmgqTERGJw/Px34e/7830CnOqoaDlwlaWsASXsDQ4BfNKA5dR3X1j8TgM0kfR5AUnvgGmAksKyK/f8OHCdpM0mbA8fjz3uD4SCz4bmGbHbcGl8DzpA0Dfgc8PW8/C7gW5KeLh/4j4gxwE3AREnPA78BPhsRcxvQjrqOa+uZiAiywPA/kl4CXgTeA75T5f5TyQLSP4CngN9GxNPFtNbWN8q+P2Zmtj7bp2//+OvjTzV6/y036zClNWbFdv+nmVkiUrzi30HGzCwFic5d5jEZMzMrjDMZM7MEtPb0MI3lIGNmlooEo4y7yyxpklbl86s9K+luSZs1oa6RNXOlSfptPbNQH5LP8dbQY6yez6ua8rJt3mngsYZKqm1KH0uUr/g3a3nL8/nV9gQ+IJsRejVJjcrWI+JL+bQ2dTkEaHCQMWsKX/Fv1rr+DvxHnmX8XdIYYIak9pKuljQpn/X5y5BNYS/p55JekPQXoEdNRZIekdQ/fz4wn1H6n/mM0X3Igtk38yzqU5K6S7onP8YkSQfm+26dzy48XdJvqaLDQ9IfJU3J9zm7bN1P8/IJkrrnZTvmsyBPyV/3rs3ybpo1A4/JWJuQZyyDgIfyor7AnhHxav6HeklEfFLSJsDjksYD+wC7kN03ZxuyObpuKqu3O9mMBgfndW0VEW9L+hXwTkT8ON/uDuCnEfFYftuCccBuwOXAYxExTNLRrD2lT12+mB+jIzBJ0j0RsZBszrnJEfFNSZfldZ9HdhuHr0TES5IGkE3vc1gj3kZbzyU4JOMgY8nrKOmZ/PnfgRvJurH+ERGv5uX/CexVcm+aLmRTzB8M3JnfH2WOpIdrqX8/4NGauvL532pzBLC71vRLdJa0RX6ME/J9H5C0qIrXdL6k4/Pn2+VtXQh8CPw+L78dGJ0f4wDg7pJjb1LFMSxFBUcZSQOBnwHtyab/ubJs/SZks7n3I/tOnhIRr1Wq00HGUrc8IvYuLcj/2L5bWgR8LSLGlW13VDO2ox2wXz7dfXlbqibpELKAtX9ELJP0CLBpHZtHftzF5e+BtU1FDuDnE5+OAI4EZpFl0WPKxibPBBZFxH9IGgxcRXYvoTp5TMY2BOOAr0raCEDSzvlswI+S3WSrvaSewKG17PskcLCkHfJ9t8rLy2epHk826Sf5dnvnTx8FTsvLBgFb1tPWLmT/Ey/Lx1b2K1nXDqjJxk4j64ZbCrwq6X/yY0jSJ+o5hllt9gVejoiZEfEB2SS5x5ZtcyzZXXAB/gAcrnp+STnI2Ibgt2TjLVMlPQv8miyLvxd4KV93K/BE+Y4RMR84m6xr6p+s6a76E3B8zcA/2c3g+ucnFsxgzVlu3ycLUtPJus3eqKetDwEdJD1HdlfQJ0vWvQvsm7+Gw4BhefnpwJl5+6az7h8GawNa4H4y2wJvlizPystq3SYiVgJLgK0rttuzMJuZrf8kPcTat+loqE3JbtFQ44aIuKGk/pOAgRHxpXz5c8CAiDivZJtn821m5cuv5NssqOugHpMxM0tARAws+BCzyU40qdErL6ttm1n5GZ1dyE4AqJO7y8zMDGASsJOkHSRtTHbn2jFl24wBvpA/Pwl4OOrpDnMmY2ZmRMRKSeeRnSjTHrgpIqZLGkZ2fdYYsksEbpP0MvA2a26hXiePyZiZWWHcXWZmZoVxkDEzs8I4yJiZWWEcZMzMrDAOMmZmVhgHGTMzK4yDjJmZFcZBxszMCvP/AZuHVvHzDilhAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dc_y_preds = dummy.predict(X_valid)\n",
    "plot_confusion_matrix(dc_y_preds, y_valid, labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('nlp_ch4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "vscode": {
   "interpreter": {
    "hash": "a52fe47989fdc78fafbb981021cec52a6b82df6453830b9ffbd04250493e6cab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}