Sentiment Analysis/Emotions Sentiment Analysis/Emotions Sentiment Analysis Project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentiment Analysis of Emotions Dataset (Multiclass)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saved as 'distilbert-base-uncased-Emotions_Detection' on my HuggingFace profile\n",
    "\n",
    "Source for the Dataset: https://www.kaggle.com/datasets/praveengovi/emotions-dataset-for-nlp?select=test.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploratory Data Analysis & Preprocessing in Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "NOTE: Redirects are currently not supported in Windows or MacOs.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset, DatasetDict, load_metric\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "from transformers import TrainingArguments, Trainer, set_seed, DataCollatorWithPadding\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score, f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Versions of Many Libraries Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pandas:            1.4.3\n",
      "NumPy:             1.22.4\n",
      "Torch:             1.12.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Pandas:\".ljust(18), pd.__version__)\n",
    "print(\"NumPy:\".ljust(18), np.__version__)\n",
    "print(\"Torch:\".ljust(18), torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest & Preprocess Training Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i didnt feel humiliated</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i can go from feeling so hopeless to so damned...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>im grabbing a minute to post i feel greedy wrong</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i am ever feeling nostalgic about the fireplac...</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i am feeling grouchy</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text   labels\n",
       "0                            i didnt feel humiliated  sadness\n",
       "1  i can go from feeling so hopeless to so damned...  sadness\n",
       "2   im grabbing a minute to post i feel greedy wrong    anger\n",
       "3  i am ever feeling nostalgic about the fireplac...     love\n",
       "4                               i am feeling grouchy    anger"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('~/Desktop/Spark Datasets/Emotions Dataset for NLP/train.txt', delimiter=';', header=None, engine=\"c\")\n",
    "\n",
    "train_df.rename(columns={1 : 'labels', 0 : 'text'}, inplace=True)\n",
    "train_df = train_df.apply(lambda x: x.str.strip())\n",
    "\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize: Histogram of Text Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBQAAAHwCAYAAAAW8jyvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmy0lEQVR4nO3de7hlZ10n+O+PFDeBJkDKTCYXiksEuUiANA2oNBK1gUJD94M0NEqk052hDQ5g+2jJ9GjrDE4xo8QgGicCEhSBdFokUrRKB2gVSJoEYi4EhhIrJjE3LgECCh34zR97Hdk5VKXOe+qc2lXnfD7Ps5+91rvetddv71rPOXW++13vqu4OAAAAwIi7LboAAAAA4PAjUAAAAACGCRQAAACAYQIFAAAAYJhAAQAAABgmUAAAAACGCRQAYA1V1dVV9fRF17FIVfXPq+q6qrq9qh6/6HoOFVXVVfXwRdcBAGtFoAAAK1RVe6rq+5e1/XhV/cXSenc/urs/sJ/X2Tb9cbllnUpdtF9J8rLuvm93f2ypsapOmEKGpUdX1Zfn1r939EDLP/999PlAVf2bVbyPVVvEMQHgYNuo/5EBgE2rqrZ09x0LLOHBSa5e3tjdf5PkvkvrVdVJHtfduw9ibQDAGjFCAQDW0Pwohqp6UlVdWlVfrKqbq+q1U7c/m55vm76Zf0pV3a2q/kNVXVtVt1TVW6rq/nOv++Jp22er6n9fdpz/WFUXVNXvVdUXk/z4dOwPV9VtVXVjVb2+qu4x93pdVT9RVZ+qqi9V1f9RVQ+rqg9N9Z4/33/Ze9xrrVV1z6q6PckRSf6yqv5q4HO7Z1X9SlX9zfRZ/VZV3Xva9p6q+tW5vm+vqjdV1Xcm+a0kT5k+x9tWery51/rXVXVNVX2+qv6kqh687DN66fQZ3VZVv1FVNW07oqp+tao+U1V/XVUvWxp1UlWvTvK9SV4/1fX6uUN+/95eDwAORwIFAFg/Zyc5u7v/UZKHJTl/an/a9HzkdFnAh5P8+PT4viQPzeyb/NcnSVU9KslvJnlRkmOS3D/JscuOdWqSC5IcmeStSb6e5JVJjkrylCSnJPmJZfv8syRPTPLkJD+T5NwkP5rk+CSPSfLCfbyvvdba3V/t7qURCI/r7oft85P5VjuTfEeSk5I8fHp/Pz9t+9dJfqyqnlFVL0rypCQv7+5rkrw0yYenz/HIgeOlqk5N8qok/yLJ1iR/nuRty7o9J8k/TvJdSZ6f2WeWJP82ybOmep+Q5LlLO3T3/za91tJlHy9bwesBwGFHoAAAY/5w+nb5tukb8d+8i77/I8nDq+qo7r69uy++i74vSvLa7v50d9+e5OeSvGCaZ+F5Sf6ou/+iu7+W2R/avWz/D3f3H3b3N7r777r7su6+uLvv6O49Sf7fJP902T7/d3d/sbuvTnJVkj+djv+FJP8lyb4mVLyrWodN39KfkeSV3f257v5Skl9O8oIk6e6bkvy7JOdlFtK8eOpzoF6a5P/q7mumS0R+OclJ86MUkuzs7tumyzXen1mAkMzCgLO7+/ru/nxmgchK7Ov1AOCwI1AAgDHP7e4jlx751m/9552e2bfun6iqj1TVc+6i7/+c5Nq59Wszm+vo6GnbdUsbuvsrST67bP/r5leq6juq6t1VddN0GcQvZzZaYd7Nc8t/t5f1+2bv7qrW1dia5NuSXDYX1Pzx1L7kjzK7lOKT3X2XkzAOeHCSs+eO+bkklTuP/rhpbvkr+eZncqd/k2XLd2VfrwcAhx2BAgCsk+7+VHe/MMm3J3lNkguq6j751tEFSfK3mf2Bu+SEJHdk9kf+jUmOW9owzS3woOWHW7Z+TpJPJDlxuuTiVZn9sbwW7qrW1fhMZgHGo+fCmvvPXT6RJK9Ock2SY6pq/lKMvX2WK3Vdkv9lPiDq7nt394dWsO+d/k0yu0xk3oHUBQCHBYECAKyTqvrRqtra3d9IctvU/I0kt07PD53r/rYkr6yqh1TVfTMbUfCOaSj+BUl+qKqeOk2U+B+z/3Dgfkm+mOT2qnpkZpcMrJW7qnXY9Pn8dpKzqurbk6Sqjq2qfzYtPy3JS5K8OMlpSX69qpZGEdyc5Lh9TSA5Z0tV3WvucffMJnT8uap69HSc+1fVj6yw7POTvHyq88gkP7ts+825878vAGw4AgUAWD/PTHL1dOeDs5O8YJrf4CuZfeP+wWm4/ZOTvCnJ72Z2B4i/TvL3SX4ySaY5Dn4yydsz+2b89iS3JPnqXRz7p5P8qyRfyuyP9Xes4fvaZ60H4GeT7E5y8XSJxn9N8oiq+kdJ3pLZBIc3dPefJ3ljkt+Z5l54X2a3qLypqj5zF69/TmajIJYev9Pd78xs5Mjbp2NeldlEiyvx20n+NMkVST6W5D2ZjdL4+rT97CTPm+4e8boVviYAHFaq24g8ADicTKMCbsvscoa/XnA5JKmqZyX5re5+8H47A8AGYYQCABwGquqHqurbpjkYfiXJlUn2LLaqzauq7l1Vz66qLdPlF7+Q5J2LrgsADiaBAgAcHk7NbDLEv01yYmaXTxhmuDiV5BeTfD6zSx6uyex2ngCwabjkAQAAABhmhAIAAAAwTKAAAAAADNuy6AKS5Kijjupt27YtugwAAABgzmWXXfaZ7t66t22HRKCwbdu2XHrppYsuAwAAAJhTVdfua5tLHgAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGLZl0QUAK7Ntx66Dcpw9O7cflOMAAACHNyMUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYtmXRBcBmtG3HrkWXAAAAcECMUAAAAACGCRQAAACAYQIFAAAAYJg5FGDOauY22LNz+zpUAgAAcGgzQgEAAAAYZoQCG5Y7KQAAAKwfgQKHBeEAAADAoUWgABwwc08AAMDmYw4FAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYQIFAAAAYJi7PMABcktLAABgMzJCAQAAABgmUAAAAACGueQBuJODdQnHao6zZ+f2dagEAABYDSMUAAAAgGErChSq6siquqCqPlFV11TVU6rqgVX13qr61PT8gKlvVdXrqmp3VV1RVU9Y37cAAAAAHGwrHaFwdpI/7u5HJnlckmuS7EhyUXefmOSiaT1JnpXkxOlxRpJz1rRiAAAAYOH2O4dCVd0/ydOS/HiSdPfXknytqk5N8vSp23lJPpDkZ5OcmuQt3d1JLp5GNxzT3TeuefXAprLa+R3MvQAAAGtvJSMUHpLk1iS/U1Ufq6o3VNV9khw9FxLclOToafnYJNfN7X/91HYnVXVGVV1aVZfeeuutq38HAAAAwEG3kkBhS5InJDmnux+f5Mv55uUNSZJpNEKPHLi7z+3uk7v75K1bt47sCgAAACzYSgKF65Nc392XTOsXZBYw3FxVxyTJ9HzLtP2GJMfP7X/c1AYAAABsEPsNFLr7piTXVdUjpqZTknw8yYVJTpvaTkvyrmn5wiQvnu728OQkXzB/AgAAAGws+52UcfKTSd5aVfdI8ukkL8ksjDi/qk5Pcm2S509935Pk2Ul2J/nK1BcAAADYQFYUKHT35UlO3sumU/bSt5OceWBlAQAAAIeylcyhAAAAAHAnAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIYJFAAAAIBhAgUAAABgmEABAAAAGCZQAAAAAIZtWXQBAOtt245dw/vs2bl9HSoBAICNwwgFAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYQIFAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYQIFAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYVsWXQCbz7YduxZdAgAAAAfICAUAAABgmEABAAAAGOaSB4C9WM2lOXt2bl+HSgAA4NBkhAIAAAAwTKAAAAAADHPJA8AacZkEAACbiREKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAM27LoAgA2s207dg3vs2fn9nWoBAAAxhihAAAAAAwTKAAAAADDVhQoVNWeqrqyqi6vqkuntgdW1Xur6lPT8wOm9qqq11XV7qq6oqqesJ5vAAAAADj4RkYofF93n9TdJ0/rO5Jc1N0nJrloWk+SZyU5cXqckeSctSoWAAAAODQcyCUPpyY5b1o+L8lz59rf0jMXJzmyqo45gOMAAAAAh5iVBgqd5E+r6rKqOmNqO7q7b5yWb0py9LR8bJLr5va9fmoDAAAANoiV3jbye7r7hqr69iTvrapPzG/s7q6qHjnwFEyckSQnnHDCyK4AAADAgq1ohEJ33zA935LknUmelOTmpUsZpudbpu43JDl+bvfjprblr3lud5/c3Sdv3bp19e8AAAAAOOj2GyhU1X2q6n5Ly0l+MMlVSS5MctrU7bQk75qWL0zy4uluD09O8oW5SyMAAACADWAllzwcneSdVbXU//e7+4+r6iNJzq+q05Ncm+T5U//3JHl2kt1JvpLkJWteNQAAALBQ+w0UuvvTSR63l/bPJjllL+2d5Mw1qQ4AAAA4JB3IbSMBAACATUqgAAAAAAxb6W0jYa+27di16BIAAABYACMUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGDYlkUXAMCYbTt2De+zZ+f2dagEAIDNzAgFAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYQIFAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYQIFAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYQIFAAAAYNiWRRcAwPrbtmPX8D57dm5fh0oAANgojFAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGErDhSq6oiq+lhVvXtaf0hVXVJVu6vqHVV1j6n9ntP67mn7tnWqHQAAAFiQkREKL09yzdz6a5Kc1d0PT/L5JKdP7acn+fzUftbUDwAAANhAVhQoVNVxSbYnecO0XkmekeSCqct5SZ47LZ86rWfafsrUHwAAANggVjpC4deS/EySb0zrD0pyW3ffMa1fn+TYafnYJNclybT9C1N/AAAAYIPYb6BQVc9Jckt3X7aWB66qM6rq0qq69NZbb13LlwYAAADW2UpGKHx3kh+uqj1J3p7ZpQ5nJzmyqrZMfY5LcsO0fEOS45Nk2n7/JJ9d/qLdfW53n9zdJ2/duvWA3gQAAABwcO03UOjun+vu47p7W5IXJHlfd78oyfuTPG/qdlqSd03LF07rmba/r7t7TasGAAAAFmrkLg/L/WySn6qq3ZnNkfDGqf2NSR40tf9Ukh0HViIAAABwqNmy/y7f1N0fSPKBafnTSZ60lz5/n+RH1qA2AAAA4BB1ICMUAAAAgE1KoAAAAAAMEygAAAAAwwQKAAAAwLChSRkB4K5s27FreJ89O7evQyUAAKw3IxQAAACAYUYoALBXqxltAADA5mGEAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwLAtiy4AAA6GbTt2De+zZ+f2dagEAGBjMEIBAAAAGCZQAAAAAIYJFAAAAIBh5lAAYKHMbQAAcHgyQgEAAAAYZoQCAIed1YxqAABgbRmhAAAAAAwTKAAAAADDBAoAAADAMIECAAAAMEygAAAAAAxzlwcA2IfV3E1iz87t61AJAMChxwgFAAAAYJhAAQAAABgmUAAAAACGCRQAAACAYSZl5B+sZvIxAAAANicjFAAAAIBh+w0UqupeVfXfq+ovq+rqqvrFqf0hVXVJVe2uqndU1T2m9ntO67un7dvW+T0AAAAAB9lKRih8NckzuvtxSU5K8syqenKS1yQ5q7sfnuTzSU6f+p+e5PNT+1lTPwAAAGAD2W+g0DO3T6t3nx6d5BlJLpjaz0vy3Gn51Gk90/ZTqqrWqmAAAABg8VY0h0JVHVFVlye5Jcl7k/xVktu6+46py/VJjp2Wj01yXZJM27+Q5EFrWDMAAACwYCsKFLr76919UpLjkjwpySMP9MBVdUZVXVpVl956660H+nIAAADAQTR0l4fuvi3J+5M8JcmRVbV028njktwwLd+Q5PgkmbbfP8ln9/Ja53b3yd198tatW1dXPQAAALAQK7nLw9aqOnJavneSH0hyTWbBwvOmbqclede0fOG0nmn7+7q717BmAAAAYMG27L9LjklyXlUdkVkAcX53v7uqPp7k7VX1fyb5WJI3Tv3fmOR3q2p3ks8lecE61A0AAAAs0H4Dhe6+Isnj99L+6czmU1je/vdJfmRNqgMAAAAOSUNzKAAAAAAkAgUAAABgFQQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAw7YsugAA2Ei27dg1vM+endvXoRIAgPVlhAIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADNuy6AIAYLPbtmPX8D57dm5fh0oAAFbOCAUAAABg2H4Dhao6vqreX1Ufr6qrq+rlU/sDq+q9VfWp6fkBU3tV1euqandVXVFVT1jvNwEAAAAcXCsZoXBHkn/f3Y9K8uQkZ1bVo5LsSHJRd5+Y5KJpPUmeleTE6XFGknPWvGoAAABgofYbKHT3jd390Wn5S0muSXJsklOTnDd1Oy/Jc6flU5O8pWcuTnJkVR2z1oUDAAAAizM0KWNVbUvy+CSXJDm6u2+cNt2U5Ohp+dgk183tdv3UduNcW6rqjMxGMOSEE04YrRsANrXVTOS4GiZ/BAD2ZcWTMlbVfZP85ySv6O4vzm/r7k7SIwfu7nO7++TuPnnr1q0juwIAAAALtqJAoarunlmY8Nbu/oOp+ealSxmm51um9huSHD+3+3FTGwAAALBBrOQuD5XkjUmu6e7Xzm26MMlp0/JpSd411/7i6W4PT07yhblLIwAAAIANYCVzKHx3kh9LcmVVXT61vSrJziTnV9XpSa5N8vxp23uSPDvJ7iRfSfKStSwYAAAAWLz9Bgrd/RdJah+bT9lL/05y5gHWBQAAABzCVjwpIwAAAMASgQIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADNuy6AIAgEPXth27hvfZs3P7OlQCABxqjFAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGFuGwkArCm3mgSAzcEIBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhm1ZdAEAAKuxbceu4X327Ny+DpUAwOZkhAIAAAAwzAgFAGDhVjPaAABYLCMUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhgkUAAAAgGECBQAAAGCYQAEAAAAYJlAAAAAAhm1ZdAEAAAfLth27hvfZs3P7OlQCAIc/IxQAAACAYUYoAADchdWMalgtoyEAOJwYoQAAAAAMEygAAAAAwwQKAAAAwDCBAgAAADBMoAAAAAAMEygAAAAAw/YbKFTVm6rqlqq6aq7tgVX13qr61PT8gKm9qup1VbW7qq6oqiesZ/EAAADAYqxkhMKbkzxzWduOJBd194lJLprWk+RZSU6cHmckOWdtygQAAAAOJfsNFLr7z5J8blnzqUnOm5bPS/Lcufa39MzFSY6sqmPWqFYAAADgELHaORSO7u4bp+Wbkhw9LR+b5Lq5ftdPbQAAAMAGcsCTMnZ3J+nR/arqjKq6tKouvfXWWw+0DAAAAOAgWm2gcPPSpQzT8y1T+w1Jjp/rd9zU9i26+9zuPrm7T966desqywAAAAAWYcsq97swyWlJdk7P75prf1lVvT3JP0nyhblLIwAAWGPbduwa3mfPzu3rUAkAm81+A4WqeluSpyc5qqquT/ILmQUJ51fV6UmuTfL8qft7kjw7ye4kX0nyknWoGQAAAFiw/QYK3f3CfWw6ZS99O8mZB1oUAAAAcGg74EkZAQAAgM1HoAAAAAAMW+2kjAAArLHVTLAIAItihAIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADBMoAAAAAMMECgAAAMAwgQIAAAAwbMuiCwAA4ODatmPX8D57dm5fh0oAOJwZoQAAAAAMEygAAAAAwwQKAAAAwDBzKAAAsF+rmXdhNczVAHD4ECgAAHDIMGEkwOHDJQ8AAADAMIECAAAAMEygAAAAAAwTKAAAAADDBAoAAADAMIECAAAAMEygAAAAAAzbsugCAADgQGzbsWt4nz07t69DJQCbixEKAAAAwDCBAgAAADBMoAAAAAAMM4fCBrWaawkBAABgpYxQAAAAAIYZoQAAACvgbhIAd2aEAgAAADBMoAAAAAAMc8kDAACbjgmsAQ6cEQoAAADAMIECAAAAMEygAAAAAAwzhwIAABxCDtb8Dm5pCRwogQIAAKyTjTb542rej+ACNi6XPAAAAADDjFAAAIBNaKONngAOPiMUAAAAgGFGKAAAAIcUczXA4cEIBQAAAGCYEQoAAMC6MVcDbFxGKAAAAADDBAoAAADAMIECAAAAMMwcCgAAwGHPnSHg4DNCAQAAABhmhAIAAMAKGQkB3yRQAAAANiW3tIQDI1A4DPhBBwAAwKFmXeZQqKpnVtUnq2p3Ve1Yj2MAAAAAi7PmIxSq6ogkv5HkB5Jcn+QjVXVhd398rY8FAABwqDvURxyvZo4Hc0mQrM8IhScl2d3dn+7uryV5e5JT1+E4AAAAwIKsxxwKxya5bm79+iT/ZB2OAwAAwAE6lEdQGAlxaFvYpIxVdUaSM6bV26vqk4uq5QAcleQziy6ChXMesMS5QOI8YMZ5wBLnAonzIElSr9lYx1mlw/FcePC+NqxHoHBDkuPn1o+b2u6ku89Ncu46HP+gqapLu/vkRdfBYjkPWOJcIHEeMOM8YIlzgcR5wDdttHNhPeZQ+EiSE6vqIVV1jyQvSHLhOhwHAAAAWJA1H6HQ3XdU1cuS/EmSI5K8qbuvXuvjAAAAAIuzLnModPd7krxnPV77EHNYX7LBmnEesMS5QOI8YMZ5wBLnAonzgG/aUOdCdfeiawAAAAAOM+sxhwIAAACwwQkUVqGqnllVn6yq3VW1Y9H1cPBU1Zuq6paqumqu7YFV9d6q+tT0/IBF1sj6q6rjq+r9VfXxqrq6ql4+tTsXNpmquldV/feq+svpXPjFqf0hVXXJ9HviHdMkxWxwVXVEVX2sqt49rTsPNpmq2lNVV1bV5VV16dTmd8MmVFVHVtUFVfWJqrqmqp7iXNhcquoR08+CpccXq+oVG+08ECgMqqojkvxGkmcleVSSF1bVoxZbFQfRm5M8c1nbjiQXdfeJSS6a1tnY7kjy77v7UUmenOTM6eeAc2Hz+WqSZ3T345KclOSZVfXkJK9JclZ3PzzJ55OcvrgSOYhenuSauXXnweb0fd190txt4fxu2JzOTvLH3f3IJI/L7GeDc2ET6e5PTj8LTkryxCRfSfLObLDzQKAw7klJdnf3p7v7a0nenuTUBdfEQdLdf5bkc8uaT01y3rR8XpLnHsyaOPi6+8bu/ui0/KXM/pNwbJwLm07P3D6t3n16dJJnJLlgancubAJVdVyS7UneMK1XnAfM+N2wyVTV/ZM8Lckbk6S7v9bdt8W5sJmdkuSvuvvabLDzQKAw7tgk182tXz+1sXkd3d03Tss3JTl6kcVwcFXVtiSPT3JJnAub0jTM/fIktyR5b5K/SnJbd98xdfF7YnP4tSQ/k+Qb0/qD4jzYjDrJn1bVZVV1xtTmd8Pm85Aktyb5nekyqDdU1X3iXNjMXpDkbdPyhjoPBAqwhnp22xS3Ttkkquq+Sf5zkld09xfntzkXNo/u/vo0nPG4zEaxPXKxFXGwVdVzktzS3ZctuhYW7nu6+wmZXRp7ZlU9bX6j3w2bxpYkT0hyTnc/PsmXs2xYu3Nh85jmz/nhJP9p+baNcB4IFMbdkOT4ufXjpjY2r5ur6pgkmZ5vWXA9HARVdffMwoS3dvcfTM3OhU1sGs76/iRPSXJkVW2ZNvk9sfF9d5Ifrqo9mV0K+YzMrp92Hmwy3X3D9HxLZtdKPyl+N2xG1ye5vrsvmdYvyCxgcC5sTs9K8tHuvnla31DngUBh3EeSnDjN3HyPzIavXLjgmlisC5OcNi2fluRdC6yFg2C6NvqNSa7p7tfObXIubDJVtbWqjpyW753kBzKbU+P9SZ43dXMubHDd/XPdfVx3b8vs/wXv6+4XxXmwqVTVfarqfkvLSX4wyVXxu2HT6e6bklxXVY+Ymk5J8vE4FzarF+ablzskG+w8qNkoC0ZU1bMzu1byiCRv6u5XL7YiDpaqeluSpyc5KsnNSX4hyR8mOT/JCUmuTfL87l4+cSMbSFV9T5I/T3Jlvnm99Ksym0fBubCJVNV3ZTah0hGZhfTnd/cvVdVDM/um+oFJPpbkR7v7q4urlIOlqp6e5Ke7+znOg81l+vd+57S6Jcnvd/erq+pB8bth06mqkzKbpPUeST6d5CWZfk/EubBpTOHi3yR5aHd/YWrbUD8TBAoAAADAMJc8AAAAAMMECgAAAMAwgQIAAAAwTKAAAAAADBMoAAAAAMMECgCwYFV1VlW9Ym79T6rqDXPrv1pVP7XK1356Vb17Dcpclap6VlVdWlUfr6qPVdWvrvPx3lxVz9tPn1+qqu/fT5+nV9VT17Y6ANhYBAoAsHgfTPLUJKmquyU5Ksmj57Y/NcmHVvJCVXXEmle3QlW1Zdn6Y5K8PsmPdvejkpycZPciapvX3T/f3f91P92enunfBADYO4ECACzeh5I8ZVp+dJKrknypqh5QVfdM8p1JPlpVp0zf8l9ZVW+atqWq9lTVa6rqo0l+pKqeWVWfmNb/xdJBquqfVtXl0+NjVXW/+SKqatu031ur6pqquqCqvm3a9sSq+m9Vddk0guKYqf0DVfVrVXVpkpcve18/k+TV3f2JJOnur3f3OXPHel9VXVFVF1XVCVP7m6vqnKq6uKo+PY0UeNNUz5vnar19Gtlx9bT/1uUfalX9fFV9pKquqqpzq6rmjvG8uc/uF6vqo9Pn+siq2pbkpUleOX1W3zv47wkAm4JAAQAWrLv/Nskd0x/VT03y4SSXZBYynJzkysx+Z785yb/s7scm2ZLk3829zGe7+wlJ/jDJbyf5oSRPTPI/zfX56SRndvdJSb43yd/tpZxHJPnN7v7OJF9M8hNVdfckv57ked39xCRvSvLquX3u0d0nd/fyyxkek+SyfbztX09yXnd/V5K3Jnnd3LYHTO/9lUkuTHJWZkHLY6vqpKnPfZJc2t2PTvLfkvzCXo7x+u7+x939mCT3TvKcfdTymemzOyfJT3f3niS/leSs7j6pu/98H/sBwKYmUACAQ8OHMgsTlgKFD8+tfzCzP/T/urv/v6n/eUmeNrf/O6bnR079PtXdneT35vp8MMlrq+p/TXJkd9+xlzqu6+4PTsu/l+R7pmM/Jsl7q+ryJP8hyXF7OfaIpyT5/Wn5d6fjLPmjqfYrk9zc3Vd29zeSXJ1k29TnG3PH/b1l+y/5vqq6pKquTPKM3Pkyknl/MD1fNvf6AMB+CBQA4NCwNI/CYzO75OHizP7oXun8CV/eX4fu3pnk32T2bf0Hq+qRe+u2l/VKcvX0bf1J3f3Y7v7BFRz76sxGSYz66vT8jbnlpfUt39r9H+r8B1V1ryS/mdmoisdmNmrjXvs53tfv4vUBgGUECgBwaPhQZkPyPzfNNfC5JEdmFip8KMknk2yrqodP/X8ss6H+y31i6vewaf2FSxuq6mHTt/2vSfKRzEYzLHdCVS3N5/CvkvzFdOytS+1Vdfeq2te3/fP+nySvqqrvmPa7W1W9dO79vmBaflGS0csK7pZk6W4OS3XOWwoPPlNV953ru1JfSnK//fYCgE1MoAAAh4YrM7u7w8XL2r7Q3Z/p7r9P8pIk/2kawv+NzK7zv5Op3xlJdk2TMt4yt/kV0wSFVyT5H0n+y17q+GSSM6vqmszmMjinu7+W2R/kr6mqv0xyeVZwB4TuviLJK5K8bXq9q5I8dNr8k0leMtXyY/nWCR3358tJnlRVV2V2OcMvLTv2bZmNSrgqyZ9kFqCM+KMk/9ykjACwbzW7RBEA2Oymuxu8e5rE8JBWVbd3930XXQcAbGZGKAAAAADDjFAAAAAAhhmhAAAAAAwTKAAAAADDBAoAAADAMIECAAAAMEygAAAAAAwTKAAAAADD/n/o8O7V7VDLzgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1296x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df['Text Length'] = train_df['text'].str.split().str.len()\n",
    "train_df.hist('Text Length', grid=False, range=(0,70), bins=70, figsize=(18,8))\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Histogram of Text Length')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('Words per Complaint')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize Description Lengths (By Output Class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABDwAAAHfCAYAAAC8gshoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAroElEQVR4nO3debgsZ1kv7N+TBAgGhGyIMSSEzRDhBASEDRJERRBEZRICIQcxDBLxeAREUXBEwe+g+KmIcjDKPIOABJDJQMIggwGNhEkwJjIkEDJgGAQSnvNH1ZbFzhp6r7V69V617/u6+trdVdVVT/d606n+9fu+Vd0dAAAAgCk5YNEFAAAAAGw2gQcAAAAwOQIPAAAAYHIEHgAAAMDkCDwAAACAyRF4AAAAAJMj8ACAfUxVPbiq3rKB5z+rqn5rM2val1TV6VX1szNue+eq+vS8a9rOqup5VfWURdcBAJtN4AEAo6o6t6q+WlWXVdWlVfUPVfWoqtrS/19294u7++6zbFtVD62qd+3x/Ed195M3u66qelJVvWiz97uvHXM1VXXHqnrb2Ea+WFWvq6pj9+L5mxouzLK/Gjy6qs6uqi9X1aer6pVV9b2bVQcA7IsEHgDw7e7V3ddMcoMkT03ya0mevVUHr6qDtupY7J2qOi7JW5K8Nsn1ktwwyVlJ3l1VN1pkbWt4epLHJHl0kh1JvifJ3yb5yQXWBABzJ/AAgGV09xe7+9QkJyQ5qapukSRVdbWq+qOq+o+q+tw4fOTq47rrVtXrx94hF1fVO3f3Dqmq61fVq6vqwqq6qKr+fFz+0Kp6d1X9SVVdlORJe/baqKoef6E/p6q+UFVPq6oDqup/JHlWkuOq6ktVdem4/bf96l9Vj6yqT441nVpV19tj34+qqk+Mdf9FVdXevl9VdYexR8ylVXVWVd15ybrTq+rJ4+u8rKreUlXXXbL+Z6rqvPF9+a2xp82PVtU9kvx6khPG13fWkkPeYKX9rVDfr4/v3blV9eBx2e3Gv+GBS7a73x7HWeoPk7ygu5/e3Zd198Xd/ZtJ3pvkSePzr9TjZnyPb1JVJyd5cJJfHV/P68b151bVE6vqI1V1SVU9t6oOXu/+9tj2mCS/kOTE7n5bd3+tu78y9iJ66jLbHzq24QvHWl5fVUctWf/QsR1eVlX/vuS9vElVnVFDr5cvVNXLV/t7AMBWEHgAwCq6+/1JPp3kB8dFT83wC/mtk9wkyZFJfntc98vjtoclOTzDl/Uev1C/Psl5SXaOz3nZksN8f5Jzxuf8/gql/FSSXUluk+Q+SR7e3R9N8qgk7+nua3T3tfd8UlXdJcn/SfLAJEeMNbxsj83umeR2SW45bvdjK74hy6iqI5O8IclTMvQg+JUkr6qqw5Zs9j+TPCzJdyW56rhNahgO8swMX9yPSHKtDO9PuvtNSf6/JC8fX9+t1trfCr47yXXH/Z6U5JSquml3/2OSi5IsHT70kCQvWOY1fkeSOyZ55TL7f0WSu61y/Iyv55QkL07yh+PrudeS1Q/O8L7fOEP7+s0N7m+3uyb59NiOZ3FAkudm6OF0dJKvJtkdzh2S5M+S/PjYC+qOSf55fN6TM/R+OTTJUUmeMePxAGBuBB4AsLbPJtkx9nw4Ockvjb/uX5bhC/mDxu2+keFL+w26+xvd/c7u7iS3zzAE4vHd/eXu/q/uXvqr/We7+xndfXl3f3WFGv5gPOZ/JPnTJCfOWPuDkzynuz/Y3V9L8sQMPUJ2Ltnmqd196bjvt2cIc/bGTyf5u+7+u+7+Zne/NcmZSX5iyTbP7e5/HV/fK5Yc4/gkr+vud3X31zOERz3DMVfa30p+a+zdcEaGcOaB4/Lnj/WnqnZkCB1esszzd2Q4bzp/mXXnZwhUNuLPu/tT3X1xhtBr1r/vWq6T5WteVndf1N2vGnuBXDbW8sNLNvlmkltU1dW7+/zu/vC4/BsZQpLrLdO+AWAhBB4AsLYjk1ycoefGdyT5wDh049IkbxqXJ8nTknwyyVvGbv9PGJdfP8l53X35Cvv/1Aw1LN3mvAwByiyuN26fJOnuL2Xo1XDkkm0uWHL/K0muMeO+d7tBkgfsfk/G9+VOGcKftY5xvSx5bd39lbG+texNzZd095eXPF76/r0oyb3G3gsPTPLO7l4uILgkw5f9I5ZZd0SSL8xQ82rW+/ddy0VZvuZlVdV3VNVfjkOM/jPJO5Jcu6oOHN/DEzL0Kjq/qt5QVTcbn/qrSSrJ+6vqw1X18E2qHwDWTeABAKuoqttlCAfeleFL7VeT3Ly7rz3ertXd10iScV6HX+7uGyW5d5LHVdVdM3yZPbpWnpB0lh4N119y/+gMvU5mee5nMwQSu1/PIRl+9f/MDMec1aeSvHDJe3Lt7j5kuTkilnF+hiEQu+u7+ljfbrO8N2s5dHzdu/33+9fdn0nyniT3yzCc5YXL7WD8sv+eJA9YZvUDk5w23v9yhlAsSVJV373nrlaocaW/73r3t9tpSY6qql1rbLfbLye5aZLv7+7vTPJDuw+dJN395u6+W4YQ5WNJ/mpcfkF3P7K7r5fk55I8s6puMuMxAWAuBB4AsIyq+s6qumeG+S5e1N0f6u5vZviC9ydV9V3jdkdW1Y+N9+85Tt5YSb6Y5IoMvQLen+GL/VOr6pCqOriqfmAvS3r8OKHk9TNccWP3pJCfy/CF9qorPO+lSR5WVbeuqqtlGILzvu4+dy+Pv9sBY/27b1fLt3pJ/FhVHTguv/PSyS5X8Tfjc+84voYnZfxyveT17ayNXxr4d6vqqlX1gxnmLFk6F8cLMvRQ+N4kr15lH0/IMIHto6vqmuPf4ylJjkvyu+M2ZyW5+fh+Hzy+nqU+l2S5K7r8QlUdNQ6r+Y186++73v0lSbr7ExnmSHnp+De56vj3edCSHkhLXTNDqHfpWMvv7F5RVYdX1X3G8OhrSb6UoX2nqh6w5O99SYYg5psr1QUAW0HgAQDf7nVVdVmGXgu/keSPM0yOuduvZRi28t6xy//fZ/hFPEmOGR9/KUNvgGd299u7+4ok98owyel/ZJjY9IS9rOu1ST6QYZLIN+Rbl8p9W5IPJ7mgqq40rKK7/z7JbyV5VYbQ5cb51pwj63Fihi/Eu2//1t2fyjCR6q8nuTDDe/f4zHCeMc4B8YsZgqXzM7x3n8/whTr5VjBxUVV9cJ01X5DhS/hnM0zy+aju/tiS9a/J0AvmNeOQmpVqfVeGOT7uN9Z6XpLvS3KnMVhId/9rkt/L0A4+kaFn0FLPTnLsOPTnb5csf0mGST/PSfJvGSaA3cj+lnp0holH/yLJpeP+fyrJla7qkmF+mKtn6M303gxDtnY7IMnjMryPF2eY2+Pnx3W3S/K+qvpSklOTPKa7z1mhHgDYEjXMpQYA7KuqqpMc092fXHQt81ZV18jwpfyY7v73LTzuvyX5uTEg2lJVdW6Sn13EsQFgyvTwAAAWqqruNU6WeUiSP0ryoSTnbuHx759hCMbbtuqYAMD8rTR5GgDAVrlPhslCK8PlbB/UW9QFtapOT3JskoeMc7QAABNhSAsAAAAwOYa0AAAAAJMj8AAAAAAmZ1vM4XHd6163d+7cuegyAAAAgH3MBz7wgS9092F7Lt8WgcfOnTtz5plnLroMAAAAYB9TVectt9yQFgAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEzOQfPceVWdm+SyJFckuby7d1XVjiQvT7IzyblJHtjdl8yzDgAAAGD/shU9PH6ku2/d3bvGx09Iclp3H5PktPExAAAAwKZZxJCW+yR5/nj/+Unuu4AaAAAAgAmb65CWJJ3kLVXVSf6yu09Jcnh3nz+uvyDJ4cs9sapOTnJykhx99NFzLnPfV7XoCvZO96IrAAAAYH8278DjTt39mar6riRvraqPLV3Z3T2GIVcyhiOnJMmuXbv2+6/P8wgQqgQTAAAATNNch7R092fGfz+f5DVJbp/kc1V1RJKM/35+njUAAAAA+5+5BR5VdUhVXXP3/SR3T3J2klOTnDRudlKS186rBgAAAGD/NM8hLYcneU0Nk08clOQl3f2mqvrHJK+oqkckOS/JA+dYAwAAALAfmlvg0d3nJLnVMssvSnLXeR0XAAAAYBGXpQUAAACYK4EHAAAAMDkCDwAAAGBy5jlpKQAAczDMCb99dC+6AgD2RwIPAIBtZl4BQpVwAoDpMKQFAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAk+OytADs96oWXcHecdlQAIC1CTwA2O/NK0CoEk4AACyKIS0AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHJelBTZd1aIr2HsuHQoAwG7b7XzWuezyBB7AppvXB26VD3MAAOZvHueczmW3niEtAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmJyDFl0AAAAwbVWLrmDvdC+6AmAzCDwAAIC5mkeAUCWYAFZnSAsAAAAwOQIPAAAAYHIEHgAAAMDkCDwAAACAyRF4AAAAAJMj8AAAAAAmR+ABAAAATI7AAwAAAJgcgQcAAAAwOQIPAAAAYHIEHgAAAMDkCDwAAACAyRF4AAAAAJMj8AAAAAAmR+ABAAAATI7AAwAAAJgcgQcAAAAwOQIPAAAAYHIEHgAAAMDkCDwAAACAyRF4AAAAAJMj8AAAAAAmR+ABAAAATI7AAwAAAJgcgQcAAAAwOQIPAAAAYHIEHgAAAMDkCDwAAACAyRF4AAAAAJMj8AAAAAAmR+ABAAAATI7AAwAAAJgcgQcAAAAwOQIPAAAAYHIEHgAAAMDkCDwAAACAyZl74FFVB1bVP1XV68fHN6yq91XVJ6vq5VV11XnXAAAAAOxftqKHx2OSfHTJ4z9I8ifdfZMklyR5xBbUAAAAAOxH5hp4VNVRSX4yyV+PjyvJXZL8zbjJ85Pcd541AAAAAPufeffw+NMkv5rkm+Pj6yS5tLsvHx9/OsmRyz2xqk6uqjOr6swLL7xwzmUCAAAAUzK3wKOq7pnk8939gfU8v7tP6e5d3b3rsMMO2+TqAAAAgCk7aI77/oEk966qn0hycJLvTPL0JNeuqoPGXh5HJfnMHGsAAAAA9kNz6+HR3U/s7qO6e2eSByV5W3c/OMnbkxw/bnZSktfOqwYAAABg/7QVV2nZ068leVxVfTLDnB7PXkANAAAAwITNc0jLf+vu05OcPt4/J8ntt+K4AAAAwP5pET08AAAAAOZK4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZnzcCjqn5glmUAAAAA+4pZeng8Y8ZlAAAAAPuEg1ZaUVXHJbljksOq6nFLVn1nkgPnXRgAAADAeq0YeCS5apJrjNtcc8ny/0xy/DyLAgAAANiIFQOP7j4jyRlV9bzuPm8LawIAAADYkNV6eOx2tao6JcnOpdt3913mVRQAAADARswSeLwyybOS/HWSK+ZbDgAAAMDGzRJ4XN7d/3fulQAAAABsklkuS/u6qvpfVXVEVe3YfZt7ZQAAAADrNEsPj5PGfx+/ZFknudHmlwMAAACwcWsGHt19w60oBAAAAGCzrBh4VNVduvttVXW/5dZ396vnVxYAAADA+q3Ww+OHk7wtyb2WWddJBB4AAADAPmnFwKO7f2f892FbVw4AAADAxs0yaWmq6ieT3DzJwbuXdffvzasoAAAAgI1Y87K0VfWsJCck+cUkleQBSW4ww/MOrqr3V9VZVfXhqvrdcfkNq+p9VfXJqnp5VV11g68BAAAA4NusGXgkuWN3/0ySS7r7d5Mcl+R7Znje15LcpbtvleTWSe5RVXdI8gdJ/qS7b5LkkiSPWFflAAAAACuYJfD46vjvV6rqekm+keSItZ7Ugy+ND68y3jrJXZL8zbj8+UnuuzcFAwAAAKxllsDj9VV17SRPS/LBJOcmeeksO6+qA6vqn5N8Pslbk/xbkku7+/Jxk08nOXLvSgYAAABY3ZqTlnb3k8e7r6qq1yc5uLu/OMvOu/uKJLceA5PXJLnZrIVV1clJTk6So48+etanAQAAAKwceFTV/VZZl+5+9awH6e5Lq+rtGeb/uHZVHTT28jgqyWdWeM4pSU5Jkl27dvWsxwIAAABYrYfHvVZZ10lWDTyq6rAk3xjDjqsnuVuGCUvfnuT4JC9LclKS1+5VxQAAAABrWDHw6O6HbXDfRyR5flUdmGGukFd09+ur6iNJXlZVT0nyT0mevcHjAAAAAHybNefwqKrrJPmdJHfK0LPjXUl+r7svWu153f0vSb5vmeXnJLn9uqoFAAAAmMEsV2l5WZILk9w/w1CUC5O8fJ5FAQAAAGzEmj08khyx5EotSfKUqjphXgUBAAAAbNQsPTzeUlUPqqoDxtsDk7x53oUBAAAArNcsgccjk7wkydfH28uS/FxVXVZV/znP4gAAAADWY80hLd19za0oBAAAAGCzzDKHR6rqlkl2Lt2+u189p5oAAAAANmSWy9I+J8ktk3w4yTfHxZ1E4AEAAADsk2bp4XGH7j527pUAAAAAbJJZJi19T1UJPAAAAIBtY5YeHi/IEHpckORrSSpJd/ct51oZAAAAwDrNEng8O8lDknwo35rDAwAAAGCfNUvgcWF3nzr3SgAAAAA2ySyBxz9V1UuSvC7DkJYkLksLAAAA7LtmCTyuniHouPuSZS5LCwAAAOyz1gw8uvthW1EIAAAAwGZZ87K0VXVUVb2mqj4/3l5VVUdtRXEAAAAA67Fm4JHkuUlOTXK98fa6cRkAAADAPmmWwOOw7n5ud18+3p6X5LA51wUAAACwbrMEHhdV1U9X1YHj7aeTXDTvwgAAAADWa5bA4+FJHpjkgiTnJzk+iYlMAQAAgH3WLFdpOS/JvbegFgCASdmxI7nkkkVXsXeqFl3BbA49NLn44kVXAcC+bMXAo6qeluST3f2Xeyz/uSQ37O4nzLs4AIDt7JJLku5FVzFN2yWYAWBxVhvScpckpyyz/K+S3HM+5QAAAABs3GqBx9W6r/ybRHd/M4lMHQAAANhnrRZ4fLWqjtlz4bjsq/MrCQAAAGBjVpu09LeTvLGqnpLkA+OyXUmemOSxc64LAAAAYN1WDDy6+41Vdd8kj0/yi+Pis5Pcv7s/tAW1AQAAAKzLqpel7e6zk5y0RbUAAAAAbIpVAw/23o4dwyXotovtckm3Qw9NLr540VUA+wKfs/PhcxaA7ch5wXxM5bxA4LHJLrkkufK1bdio7fLBAMyfz9n58DkLwHbkvGA+pnJesNpVWlJVB1bVL21VMQAAAACbYdXAo7uvSHLiFtUCAAAAsClmGdLy7qr68yQvT/Ll3Qu7+4NzqwoAAABgA2YJPG49/vt7S5Z1krtsejUAAAAAm2DNwKO7f2QrCgEAAADYLKvO4ZEkVXV4VT27qt44Pj62qh4x/9IAAAAA1mfNwCPJ85K8Ocn1xsf/muSxc6oH2GI7dgyXndoOt2TxNcx627FjsX9XAADY380yh8d1u/sVVfXEJOnuy6vqijnXBWwR1y6fj6lcuxwAALarWXp4fLmqrpNhotJU1R2SfHGuVQEAAABswCw9PB6X5NQkN66qdyc5LMnxc60KAAAAYANmuUrLB6vqh5PcNEkl+Xh3f2PulQEAAACs05qBR1UdnOR/JblThmEt76yqZ3X3f827OAAAAID1mGVIywuSXJbkGePj/5nkhUkeMK+iAAAAADZilsDjFt197JLHb6+qj8yrIAAAAICNmiXw+GBV3aG735skVfX9Sc6cb1kAAMBW27FjuGT9drGdLgN/6KHJxRcvugrYv8wSeNw2yT9U1X+Mj49O8vGq+lCS7u5bzq06AABgy1xySdK96CqmaTuFMzAVswQe95h7FQAAAACbaJbL0p63FYUAAAAAbJYDFl0AAAAAwGYTeAAAAACTs2bgUVWHVNUB4/3vqap7V9VV5l8aAAAAwPrM0sPjHUkOrqojk7wlyUOSPG+eRQEAAABsxCyBR3X3V5LcL8kzu/sBSW4+37IAAAAA1m+mwKOqjkvy4CRvGJcdOL+SAAAAADZmlsDjsUmemOQ13f3hqrpRkrfPtSoAAACADThorQ26+4wkZyx5fE6SR8+zKAAAAICNWDHwqKrXJemV1nf3vedSEQAAAMAGrdbD44/Gf++X5LuTvGh8fGKSz82zKAAAAICNWDHwGIeypKr+/+7etWTV66rqzLlXBgAAALBOs0xaesg4UWmSpKpumOSQ+ZUEAAAAsDFrTlqa4Sotp1fVOUkqyQ2SnDzPogAAAAA2YtXAo6oOSHKtJMckudm4+GPd/bV5FwYAAACwXqsOaenubyb51e7+WnefNd6EHQAAAMA+bZY5PP6+qn6lqq5fVTt23+ZeGQAAAMA6zTKHxwnjv7+wZFknudEy2wIAAAAs3JqBR3ffcCsKAQAAANgsawYeVXWVJD+f5IfGRacn+cvu/sYc6wIAAABYt1mGtPzfJFdJ8szx8UPGZT87r6IAAAAANmKWwON23X2rJY/fVlVnzasgAAAAgI2a5SotV1TVjXc/qKobJblifiUBAAAAbMwsPTwen+TtVXVOkkpygyQPm2tVAAAAABuwYuBRVY9N8g9JzkhyTJKbjqs+3t1fm39pAAAAAOuz2pCWo5L8aZLPJ3lLkgclOTrJIfMvCwAAAGD9Vgw8uvtXuvuOSb47yROTXJxhKMvZVfWRtXZcVdevqrdX1Ueq6sNV9Zhx+Y6qemtVfWL899BNei0AAAAASWabtPTqSb4zybXG22eTvG+G512e5Je7+9gkd0jyC1V1bJInJDmtu49Jctr4GAAAAGDTrDaHxylJbp7ksgwBxz8k+ePuvmSWHXf3+UnOH+9fVlUfTXJkkvskufO42fOTnJ7k19ZXPgAAAMCVrdbD4+gkV0tyQZLPJPl0kkvXc5Cq2pnk+zIEJ4ePYUjGfR++nn0CAAAArGTFHh7dfY+qqgy9PO6Y5JeT3KKqLk7ynu7+nVkOUFXXSPKqJI/t7v8cdvnfx+iq6hWed3KSk5Pk6KOPnvHlAAAAAKwxh0cPzk7yd0nemOTdSW6c5DGz7LyqrpIh7Hhxd796XPy5qjpiXH9EhqvALHfsU7p7V3fvOuyww2Z6MQAAAADJKoFHVT26ql5WVf+R5Iwk90zysST3S7JjrR2PvUOeneSj3f3HS1admuSk8f5JSV67ztoBAAAAlrXikJYkO5O8MskvLZlzY2/8QJKHJPlQVf3zuOzXkzw1ySuq6hFJzkvywHXsGwAAAGBFq83h8biN7Li735WkVlh9143sGwAAAGA1q87hAQAAALAdCTwAAACAyRF4AAAAAJOz2qSlAABsUK00oxkAMFcCDwCAOepedAXTJEgCYC2GtAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByXJYWgG3H5SgBgN2cF7ASgQcA2073oiuYHieLAGxXzgs231TOCwxpAQAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOS4LO0cTOUSPuw/tFkAAGBqBB5z4DrQm88X8vnSZjefNgsAAItlSAsAAAAwOQIPAAAAYHIEHgAAAMDkCDwAAACAyRF4AAAAAJMj8AAAAAAmR+ABAAAATM5Biy4AAADYd1QtugKAzSHwAAAA/lv3oiuYJkESbD1DWgAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5By26AACAKatadAXTdOihi64A2Ff4nN18U/mMFXgAAMxJ96Ir2DtV269mYP+2nT6zfMZuPUNaAAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByDlp0AQCwt6oWXcH0HHrooisAANhcAg8AtpXuRVcwu6rtVS8AwJQY0gIAAABMjsADAAAAmJy5BR5V9Zyq+nxVnb1k2Y6qemtVfWL814hhAAAAYNPNs4fH85LcY49lT0hyWncfk+S08TEAAADApppb4NHd70hy8R6L75Pk+eP95ye577yODwAAAOy/tnoOj8O7+/zx/gVJDt/i4wMAAAD7gYVdlra7u6pWvFhfVZ2c5OQkOfroo7esrs1QtegKpudQs73MlTa7+bRZALYr5wXz4dwAtt5WBx6fq6ojuvv8qjoiyedX2rC7T0lySpLs2rVrxWBkX9PbptLhf2bbqV7mYzu1AW0WAOZrO/1/1nkBsJatHtJyapKTxvsnJXntFh8fAAAA2A/M87K0L03yniQ3rapPV9Ujkjw1yd2q6hNJfnR8DAAAALCp5jakpbtPXGHVXed1TAAAAIBk64e0AAAAAMydwAMAAACYHIEHAAAAMDkCDwAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByBB4AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHIEHAAAAMDkCDwAAAGByDlp0AQAA7J2q7bXv7s3fJ8A8zetzdl779Tm7PIEHAMA248QWYL58zk6DIS0AAADA5Ag8AAAAgMkReAAAAACTI/AAAAAAJkfgAQAAAEyOwAMAAACYHJel3SZcB5rtZF7tap771mb3b9utzWqvwHbjXBZYBIHHNuFDl+1Ee2W70WYB5svnLLAIhrQAAAAAkyPwAAAAACZH4AEAAABMjsADAAAAmByBBwAAADA5Ag8AAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMzkICj6q6R1V9vKo+WVVPWEQNAAAAwHRteeBRVQcm+YskP57k2CQnVtWxW10HAAAAMF2L6OFx+ySf7O5zuvvrSV6W5D4LqAMAAACYqEUEHkcm+dSSx58elwEAAABsin120tKqOrmqzqyqMy+88MJFlwMAAABsI4sIPD6T5PpLHh81Lvs23X1Kd+/q7l2HHXbYlhUHAAAAbH+LCDz+MckxVXXDqrpqkgclOXUBdQAAAAATVd299Qet+okkf5rkwCTP6e7fX2P7C5OctwWl7W+um+QLiy4C9oI2y3ajzbLdaLNsJ9or2402Oz836O4rDQ1ZSODBvqGqzuzuXYuuA2alzbLdaLNsN9os24n2ynajzW69fXbSUgAAAID1EngAAAAAkyPw2L+dsugCYC9ps2w32izbjTbLdqK9st1os1vMHB4AAADA5OjhAQAAAEyOwAPYJ1TVo6vqo1X14kXXAnurqv5h0TXAaqrqS4uuATaiqnZW1dmLrgPmpar+rqquveg6psaQFvZKVVWGdvPNRdfCtFTVx5L8aHd/egP7OKi7L9/EsgAmoaq+1N3XWHQdsF5VtTPJ67v7FouuBWYx63mp71fzpYfHRFTV31bVB6rqw1V18rjsS1X1+1V1VlW9t6oOH5ffeHz8oap6ytJffarq8VX1j1X1L1X1u+OynVX18ap6QZKzk1x/Ea+R6aqqZyW5UZI3VtVvVNVzqur9VfVPVXWfcZudVfXOqvrgeLvjuPzO4/JTk3xkgS+D/dj4eVtV9bSqOnv8fD1hXPeCqrrvkm1fvLtdw1ZbpZ2+rKp+csl2z6uq46vqwHH73ecGP7e46pmCqjqkqt4wnp+eXVUnVNVvj23s7Ko6ZfwCmKq67bjdWUl+Yck+HlpVr66qN1XVJ6rqD5esu3tVvWc8V3hlVV1jXP7UqvrI2I7/aFz2gPGYZ1XVO7b4rWCbWKHNnltV1x3X76qq08f7T6qqF1bVu5O8cGyrr62q08e2+jvjdlf6frV7n8sdb3zObavqjBq+8725qo5YzDuyvQg8puPh3X3bJLuSPLqqrpPkkCTv7e5bJXlHkkeO2z49ydO7+3uT/Pev6VV19yTHJLl9klsnuW1V/dC4+pgkz+zum3f3eVvxgth/dPejknw2yY9kaLdv6+7bj4+fVlWHJPl8krt1922SnJDkz5bs4jZJHtPd37O1lcO3uV+Gz85bJfnRDG33iCTPTvLQJKmqayW5Y5I3LKZEWLGdvjzJA5Okqq6a5K4Z2ukjknyxu2+X5HZJHllVN1xA3UzHPZJ8trtvNfbWeFOSP+/u242Pr57knuO2z03yi+O57J5uneF84HuTnFBV1x+/gP5mhh6jt0lyZpLHjefFP5Xk5t19yyRPGffx20l+bNz/vefxYpmE5drsao7N0AZPHB/fPsn9k9wyyQOqate4fKXvV1c6XlVdJckzkhw/fud7TpLf35RXN3ECj+l49Jh+vzdDD4xjknw9yevH9R9IsnO8f1ySV473X7JkH3cfb/+U5INJbjbuJ0nO6+73zqt4WOLuSZ5QVf+c5PQkByc5OslVkvxVVX0oQ/s9dslz3t/d/77FdcKe7pTkpd19RXd/LskZSW7X3WckOaaqDktyYpJXGXrFAi3bTpO8McmPVNXVkvx4knd091czfCb/zPiZ/L4k18m3zg1gPT6U5G5V9QdV9YPd/cUMbe994//j75Lk5jXMZXDt7t7d8+KFe+zntO7+Ynf/V4YenjdIcocM5wfvHtvsSePyLyb5ryTPrqr7JfnKuI93J3leVT0yyYFzer1sf8u12dWcOn5+7vbW7r5oXPbqDJ/Dycrfr5Y73k2T3CLJW8e2/ZtJjtrIi9pfHLToAti4qrpzhl9pjuvur4xdqg5O8o3+1iQtV2Ttv3cl+T/d/Zd77H9nki9vYsmwmkpy/+7++LctrHpSks9l+FXygAwnLrtpn+zrXpDkp5M8KMnDFlwLXEl3/9d4/vBjGX41f9m4qjL8wv7mRdXGtHT3v1bVbZL8RJKnVNVpGYar7OruT43/vz94hl19bcn93ee5leHL5Yl7blxVt8/Qc+n4JP87yV26+1FV9f1JfjLJB6rqtt190QZeHhO0Qpu9PN/qPLBne93zvHTPSTN7he1WO95rkny4u49b58vYb+nhMQ3XSnLJGHbcLEO6vZr3ZuhWlQwn37u9OcnDl4x1PLKqvmvTq4XVvTnJLy4Zv/t94/JrJTl/nNDpIfFLDPued2boVn3g2Jvjh5K8f1z3vCSPTZLuNtcMi7RaO315hkDuB/OtLttvTvLzY3fqVNX3jMMMYV2q6npJvtLdL0rytAzDUpPkC+M56PFJ0t2XJrm0qnb/Gv7gGXb/3iQ/UFU3GY91yNhmr5HkWt39d0l+KcOPJ6mqG3f3+7r7t5NcGPPUsYwV2uy5SW47bnL/FZ66292qakdVXT3JfTP0LNrb4308yWFVddy4zVWq6ubre0X7Fz08puFNSR5VVR/N8B/DWkNPHpvkRVX1G+Nzv5gk3f2WqvofSd4zftf8UoZfJK+YU92wnCcn+dMk/1JVByT59wxjeZ+Z5FVV9TMZ2q1eHexLOsOvL8clOWt8/KvdfUGSdPfnxs/ov11YhTBYsZ0meUuGYQOv7e6vj8v+OsOQ2A+OQfSFGU7YYb2+N8PcMd9M8o0kP5+hTZ2d5IIk/7hk24cleU5VdYb2uaruvrCqHprkpePwrGTo+n9ZktdW1cEZeoE8blz3tKo6Zlx2Wob/LmBPy7XZq2cYIvXkDEOwV/P+JK/KMATlRd195tiDfubjdffXq+r4JH82zgd2UIbz5Q+v+1XtJ1yWdj9UVd+R5Kvd3VX1oCQndrcrBgCswzgZ3ge7+warbPMdGcbk3maGsb8AwASMAdyu7v7fi65lf6WHx/7ptkn+fPyl5tIkD19sOQDb09jt9PQkf7TKNj+a4UotfyLsAADYOnp4AAAAAJNj0lIAAABgcgQeAAAAwOQIPAAAAIDJEXgAAAAAkyPwAAAAACZH4AEAAABMzv8D7aYApVgEY2cAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1296x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df.boxplot(\"Text Length\", by=\"labels\", grid=False, showfliers=False, color=\"Blue\", figsize=(18, 8))\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Description Length by Output Class')\n",
    "plt.xlabel('')\n",
    "plt.ylabel('Words per Complaint')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Distribution of Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "joy         5362\n",
       "sadness     4666\n",
       "anger       2159\n",
       "fear        1937\n",
       "love        1304\n",
       "surprise     572\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['labels'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Unique Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sadness' 'anger' 'love' 'surprise' 'fear' 'joy'] \n",
      " 6\n"
     ]
    }
   ],
   "source": [
    "labels_names = train_df['labels'].unique()\n",
    "num_of_labels = len(labels_names)\n",
    "print(labels_names, '\\n', num_of_labels)\n",
    "\n",
    "cols_convert2_int = {'sadness' : 0, 'anger' : 1, 'love' : 2, 'surprise' : 3, 'fear' : 4, 'joy' : 5}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest & Preprocess Testing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>im feeling rather rotten so im not very ambiti...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>im updating my blog because i feel shitty</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i never make her separate from me because i do...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i left with my bouquet of red and yellow tulip...</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i was feeling a little vain when i did this one</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text   labels\n",
       "0  im feeling rather rotten so im not very ambiti...  sadness\n",
       "1          im updating my blog because i feel shitty  sadness\n",
       "2  i never make her separate from me because i do...  sadness\n",
       "3  i left with my bouquet of red and yellow tulip...      joy\n",
       "4    i was feeling a little vain when i did this one  sadness"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('~/Desktop/Spark Datasets/Emotions Dataset for NLP/test.txt', delimiter=';', header=None, engine=\"c\")\n",
    "\n",
    "test_df.rename(columns={1 : 'labels', 0 : 'text'}, inplace=True)\n",
    "test_df = test_df.apply(lambda x: x.str.strip())\n",
    "\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest & Preprocess Validation Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>im feeling quite sad and sorry for myself but ...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i feel like i am still looking at a blank canv...</td>\n",
       "      <td>sadness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i feel like a faithful servant</td>\n",
       "      <td>love</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i am just feeling cranky and blue</td>\n",
       "      <td>anger</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i can have for a treat or if i am feeling festive</td>\n",
       "      <td>joy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text   labels\n",
       "0  im feeling quite sad and sorry for myself but ...  sadness\n",
       "1  i feel like i am still looking at a blank canv...  sadness\n",
       "2                     i feel like a faithful servant     love\n",
       "3                  i am just feeling cranky and blue    anger\n",
       "4  i can have for a treat or if i am feeling festive      joy"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_df = pd.read_csv('~/Desktop/Spark Datasets/Emotions Dataset for NLP/val.txt', delimiter=';', header=None, engine=\"c\")\n",
    "\n",
    "eval_df.rename(columns={1 : 'labels', 0 : 'text'}, inplace=True)\n",
    "eval_df = eval_df.apply(lambda x: x.str.strip())\n",
    "\n",
    "eval_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 16000 entries, 0 to 15999\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    16000 non-null  object\n",
      " 1   labels  16000 non-null  int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 375.0+ KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2000 entries, 0 to 1999\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    2000 non-null   object\n",
      " 1   labels  2000 non-null   int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 31.4+ KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2000 entries, 0 to 1999\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    2000 non-null   object\n",
      " 1   labels  2000 non-null   int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 31.4+ KB\n"
     ]
    }
   ],
   "source": [
    "train_df = train_df[train_df['Text Length'] < 70]\n",
    "\n",
    "train_df = train_df.drop(columns=['Text Length'])\n",
    "\n",
    "train_df['labels'].replace(cols_convert2_int, inplace=True)\n",
    "test_df['labels'].replace(cols_convert2_int, inplace=True)\n",
    "eval_df['labels'].replace(cols_convert2_int, inplace=True)\n",
    "\n",
    "train_df.info()\n",
    "test_df.info()\n",
    "eval_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regular Project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Constants/Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ckpt = 'distilbert-base-uncased'\n",
    "device=\"cpu\"\n",
    "max_input_length = 91\n",
    "set_seed(42)\n",
    "batch_size = 64\n",
    "num_of_epochs = 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Pandas DataFrame to Dataset; Then Split into Train/Test/Valid Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data shape: (16000, 3)\n",
      "testing data shape: (2000, 2)\n",
      "validation data shape: (2000, 2)\n",
      "{'text': Value(dtype='string', id=None), 'labels': Value(dtype='int64', id=None), '__index_level_0__': Value(dtype='int64', id=None)}\n"
     ]
    }
   ],
   "source": [
    "# Convert pandas dataframe to a dataset\n",
    "ds_train = Dataset.from_pandas(train_df)\n",
    "ds_test = Dataset.from_pandas(test_df)\n",
    "ds_eval = Dataset.from_pandas(eval_df)\n",
    "\n",
    "# Combine the train/test/valid into one datasetdict\n",
    "ds = DatasetDict({\n",
    "    'train' : ds_train,\n",
    "    'test' : ds_test,\n",
    "    'valid' : ds_eval\n",
    "})\n",
    "\n",
    "print('training data shape:', ds['train'].shape)\n",
    "print('testing data shape:', ds['test'].shape)\n",
    "print('validation data shape:', ds['valid'].shape)\n",
    "print(ds['train'].features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a9463a0854743a6a5031e0c6dcdbb30",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a8adc1cfbe74bbbaf8d4c8af356bdce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "692a41d584894d75bb0319ac9e4e78df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data shape: (16000, 5)\n",
      "testing data shape: (2000, 4)\n",
      "validation data shape: (2000, 4)\n"
     ]
    }
   ],
   "source": [
    "# Define the Tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "# Method to Tokenize each Sample in the Dataset\n",
    "def tokenize(batch):\n",
    "    return tokenizer(batch['text'], padding=True, truncation=True, max_length=91, return_tensors=\"np\")\n",
    "\n",
    "# Map the 'Tokenize' Method to All Samples in the Entire Dataset\n",
    "encoded_data = ds.map(tokenize, batched=True, batch_size=None)\n",
    "\n",
    "print('training data shape:', encoded_data['train'].shape)\n",
    "print('testing data shape:', encoded_data['test'].shape)\n",
    "print('validation data shape:', encoded_data['valid'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print Example to Ensure Edits Worked as Expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': Value(dtype='string', id=None), 'labels': Value(dtype='int64', id=None), '__index_level_0__': Value(dtype='int64', id=None), 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}\n",
      "Dataset({\n",
      "    features: ['text', 'labels', '__index_level_0__', 'input_ids', 'attention_mask'],\n",
      "    num_rows: 16000\n",
      "})\n",
      "{'text': Value(dtype='string', id=None), 'labels': Value(dtype='int64', id=None), 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}\n",
      "Dataset({\n",
      "    features: ['text', 'labels', 'input_ids', 'attention_mask'],\n",
      "    num_rows: 2000\n",
      "})\n",
      "{'text': Value(dtype='string', id=None), 'labels': Value(dtype='int64', id=None), 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}\n",
      "Dataset({\n",
      "    features: ['text', 'labels', 'input_ids', 'attention_mask'],\n",
      "    num_rows: 2000\n",
      "})\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "print(encoded_data['train'].features)\n",
    "print(encoded_data['train'])\n",
    "\n",
    "print(encoded_data['valid'].features)\n",
    "print(encoded_data['valid'])\n",
    "\n",
    "print(encoded_data['test'].features)\n",
    "print(encoded_data['test'])\n",
    "\n",
    "print((encoded_data['test']['labels'][9]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'pre_classifier.weight', 'pre_classifier.bias', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_of_labels).to(device))\n",
    "model.config.num_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ensure git lfs is Installed; Remember to Enter Access Token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "!git lfs install\n",
    "\n",
    "# HuggingFace Access Token ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Function to Compute Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    f1 = f1_score(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\"accuracy\": acc, \"f1_score\": f1}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate=2e-5\n",
    "\n",
    "model_name = f\"{model_ckpt}-Emotions_Detection\"\n",
    "\n",
    "training_args = TrainingArguments(output_dir=model_name,\n",
    "                        num_train_epochs=num_of_epochs,\n",
    "                        learning_rate=learning_rate,\n",
    "                        per_device_train_batch_size=batch_size, \n",
    "                        per_device_eval_batch_size=batch_size, \n",
    "                        weight_decay=0.01,\n",
    "                        log_level=\"error\",\n",
    "                        logging_strategy='epoch', \n",
    "                        hub_strategy=\"checkpoint\",\n",
    "                        evaluation_strategy=\"epoch\",\n",
    "                        save_strategy=\"epoch\",\n",
    "                        save_total_limit=5,\n",
    "                        load_best_model_at_end=True,\n",
    "                        metric_for_best_model=\"f1_score\",\n",
    "                        greater_is_better=True,\n",
    "                        logging_first_step=True,\n",
    "                        disable_tqdm=False,\n",
    "                        push_to_hub=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorWithPadding(tokenizer, padding=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Subclass Trainer to Correctl Compute Loss for Imbalanced Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.get(\"labels\")\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.get(\"logits\")\n",
    "        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0, 6.0,]))\n",
    "        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))\n",
    "        return (loss, outputs) if return_outputs else loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/distilbert-base-uncased-Emotions_Detection into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = CustomTrainer(model=model, \n",
    "                  args=training_args, \n",
    "                  compute_metrics=compute_metrics,\n",
    "                  train_dataset=encoded_data['train'], \n",
    "                  eval_dataset=encoded_data['valid'], \n",
    "                  data_collator=data_collator,\n",
    "                  tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/nlpnn/nlp_ch4/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75e9d6259295429295a20581a7e0f012",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 1.7336, 'learning_rate': 1.9980000000000002e-05, 'epoch': 0.0}\n",
      "{'loss': 0.7287, 'learning_rate': 1.5000000000000002e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a69c80b783294a13bc7793d6196fed4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.25972381234169006, 'eval_accuracy': 0.8955, 'eval_f1_score': 0.8948022836666599, 'eval_runtime': 285.7294, 'eval_samples_per_second': 7.0, 'eval_steps_per_second': 0.112, 'epoch': 1.0}\n",
      "{'loss': 0.2054, 'learning_rate': 1e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb7d0bbba581484ca2efa87a766f50d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.16384392976760864, 'eval_accuracy': 0.9325, 'eval_f1_score': 0.9326052321267978, 'eval_runtime': 287.0329, 'eval_samples_per_second': 6.968, 'eval_steps_per_second': 0.111, 'epoch': 2.0}\n",
      "{'loss': 0.1338, 'learning_rate': 5e-06, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ef257b8cc854f17858fc2089f4632ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.14151246845722198, 'eval_accuracy': 0.935, 'eval_f1_score': 0.9349669876769724, 'eval_runtime': 285.0049, 'eval_samples_per_second': 7.017, 'eval_steps_per_second': 0.112, 'epoch': 3.0}\n",
      "{'loss': 0.1067, 'learning_rate': 0.0, 'epoch': 4.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b877418e01c4cf0a05dc3b9519dbd6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.14396128058433533, 'eval_accuracy': 0.9345, 'eval_f1_score': 0.9347098524383352, 'eval_runtime': 285.0021, 'eval_samples_per_second': 7.017, 'eval_steps_per_second': 0.112, 'epoch': 4.0}\n",
      "{'train_runtime': 36266.4063, 'train_samples_per_second': 1.765, 'train_steps_per_second': 0.028, 'train_loss': 0.2946525176763535, 'epoch': 4.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1000, training_loss=0.2946525176763535, metrics={'train_runtime': 36266.4063, 'train_samples_per_second': 1.765, 'train_steps_per_second': 0.028, 'train_loss': 0.2946525176763535, 'epoch': 4.0})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assessing the Results of the Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate Predictions for Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2e44a29ea26464fb504078c73adbbf7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/32 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.14151246845722198,\n",
       " 'test_accuracy': 0.935,\n",
       " 'test_f1_score': 0.9349669876769724,\n",
       " 'test_runtime': 288.5569,\n",
       " 'test_samples_per_second': 6.931,\n",
       " 'test_steps_per_second': 0.111}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_predictions = trainer.predict(encoded_data['valid'])\n",
    "\n",
    "output_predictions.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract Last Hidden States"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cc001d8c401446788cb015bfac531ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/16 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d27f10a52b44ee29b0923533eb65884",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff182c15796d4036894935f990fc9f32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['text', 'labels', '__index_level_0__', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 16000\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['text', 'labels', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 2000\n",
      "    })\n",
      "    valid: Dataset({\n",
      "        features: ['text', 'labels', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 2000\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "def extract_hidden_states(batch):\n",
    "    inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}   \n",
    "    with torch.no_grad():\n",
    "        last_hidden_state = model(**inputs)[0]\n",
    "    return {\"hidden_state\" : last_hidden_state[:,0].cpu().numpy()}\n",
    "\n",
    "encoded_data.set_format(\"torch\", columns=['input_ids', 'attention_mask', 'labels']) \n",
    "\n",
    "hidden_data = encoded_data.map(extract_hidden_states, batched=True)\n",
    "print(hidden_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Feature Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((16000,), (2000,))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = np.array(hidden_data['train']['hidden_state'])\n",
    "X_valid = np.array(hidden_data['valid']['hidden_state'])\n",
    "y_train = np.array(hidden_data['train']['labels'])\n",
    "y_valid = np.array(hidden_data['valid']['labels'])\n",
    "X_train.shape, X_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Function to Display Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_preds, y_true, labels):\n",
    "    cm = confusion_matrix(y_true, y_preds, normalize=\"true\")\n",
    "    fig, ax = plt.subplots(figsize=(11, 11))\n",
    "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\n",
    "    disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax)\n",
    "    plt.title('Normalized Confusion Matrix')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Logits to Predictions; Display Confusion Matrix for Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAooAAAJqCAYAAACsMGEOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABkGklEQVR4nO3deXwV1fn48c+TBBBQdtCwifh1QUURUEGFuhfUuotr69bWpWpbq1Rrq7grauv+q7ZaK1p3rFRxtwoqKouigksVRQSUoIJ1BZLz++NeMIErcGuSG7ift6/7MjNzZuY5h5mbk+fMEiklJEmSpKWVFDoASZIkNUx2FCVJkpSTHUVJkiTlZEdRkiRJOdlRlCRJUk52FCVJkpRTWaEDkCRJWpWUtlg3pUVf1ft+01cVj6SUBtXnPu0oSpIk5SEt+oomGw2p9/1+/fK17ep7n3YUJUmS8hIQxXH1XnHUUpIkSXmzoyhJkqScHHqWJEnKRwARhY6iXphRlCRJUk5mFCVJkvLlzSySJEkqZmYUJUmS8uU1ipIkSSpmdhQlSZKUk0PPkiRJefHNLJIkSSpyZhQlSZLy5c0skiRJKmZmFCVJkvIReI2iJEmSipsdRUmSJOXk0LMkSVJewptZJEmSVNzMKEqSJOXLm1kkSZJUzMwoSpIk5ctrFCVJklTM7ChKkiQpJ4eeJUmS8hLezCJJkqTiZkZRkiQpH4E3s0iSJKm4mVGUJEnKl9coSpIkqZjZUZQkSVJODj1LkiTlxcfjSJIkqciZUZQkScpXiY/HkSRJUhEzoyhJkpSPwGsUJSmXiHgqIn6a/fmwiHi0lrffLSJSRNT7H7IR0TQi/hUR8yPi7u+xnVpvl0KIiIci4ohCxyGpcOwoSg1MRLwXEXMionm1eT+NiKcKGFZOKaXbUkq71fd+I+LQiJgQEZ9HxOxsh2b7Wtj0AcDaQNuU0oH/60bqql0iYodsJ/q+peZvkZ3/1EpuZ1hE3LqicimlwSmlv/+P4UpaDdhRlBqmUuCX33cjkbFanecRcQpwBXAhmU5dV+A6YO9a2Py6wFsppUW1sK26UgH0j4i21eYdAbxVWztYHY8bqdZF1P+nAPwikBqmS4FTI6JVroURsW1EjM8OkY6PiG2rLXsqIi6IiGeBL4Hu2WzTCRHxn4j4b0ScFxHrR8RzEfFZRNwVEY2z67eOiAcioiIiPs3+3Pk74jgyIp7J/jw0m+Fb/FkYETdnl7WMiBuz2b+ZEXF+RJRml5VGxGURMTcipgF7fFejRERL4FzgFymlkSmlL1JKC1NK/0opnZYt0yQiroiIWdnPFRHRJLtsh4j4ICJ+k83azo6Io7LLzgHOAg7Kxn/M0pm3pYfFs/Wflm3TdyPisKXbZSX/vc6LiGez23k0Itp9VxsAC4B/Agcvbj/gIOC2pdrqyoiYkf33nRgRA7LzBwG/q1bPydXiWPq4qX6Zwf+LiHurbf+SiHgiokC/vSTVCzuKUsM0AXgKOHXpBRHRBngQuApoC/wReHCpDNOPgZ8DawHTs/N+CPQB+gFDgRuAw4EuwGbAIdlyJcDfyGTXugJfAdesKOCU0vCU0poppTWBHmQyX3dmF98MLAL+D9gS2A34aXbZz4A9s/P7khn+/S79gTWA+5ZT5sxsHXsBWwBbA7+vtnwdoCXQCTgGuDYiWqeUziaTpbwzW48bl1ffyFwacBUwOKW0FrAt8HKOcivz73UocBTQAWhMjn/3pdwC/CT78w+B14BZS5UZT6YN2gD/AO6OiDVSSg8vVc8tqq2T67hZ7DdAz2wneACZtjsipZRWEKu0Gso+cLu+PwVgR1FquM4CToqI9kvN3wP4T0ppREppUUrpduAN4EfVytycUpqSXb4wO294SumzlNIUMh2LR1NK01JK84GHyHTUSCl9nFK6N6X0ZUrpv8AFwA9WNuiIaEom43VlSumhiFgb2B34VTYDOAf4E9mMGDAEuCKlNCOl9Alw0XI23xaYu4Kh4cOAc1NKc1JKFcA5ZDpAiy3MLl+YUhoNfA5stLL1W0oVsFlENE0pzc627dJW5t/rbymlt1JKXwF3kengfaeU0nNAm4jYiEyH8ZYcZW7N/lsuSildDjRhxfXMddws3t6XZNrxj8CtwEkppQ9WsD1Jqzg7ilIDlVJ6DXgAOH2pRR1ZNtsznUyGbLEZOTb5UbWfv8oxvSZARDSLiOsjYnpEfAaMAVotHipeCTcCb6aULslOrws0AmZHxLyImAdcTyZ7trg+1eNdum7VfQy0i+XfEb10+0zPzluyjaU6ml+SrXs+UkpfkBnyPY5M3R6MiI1XIp7FMVX/9/rwf4hnBHAisCM5MqwRcWpEvJ4d7p5HJou6vCFtyH3cLJFSegGYRubhIHetRIzS6strFCU1AGeTGZqt3qmYRabzVV1XYGa16e8zHPgbMpmnbVJKLYCB2fkr/JaKiNOBDckMSy42A/gGaJdSapX9tEgpbZpdPpvM8PdiXZezi3HZbe2znDJLt09Xlh2WXVlfAM2qTa9TfWFK6ZGU0q5AOZks4V9WIp7FMc3MUTYfI4ATgNHZbN8S2aHhoWSyta1TSq2A+Xz7b/hdx8dyj5uI+AWZzOSs7PYlrebsKEoNWErpbTLX+Z1cbfZoYMPIPCKmLCIOAjYhk32sDWuRyTDOy15fd/bKrBQRg7Nx7psdQl1ch9nAo8DlEdEiIkoicyPN4uHsu4CTI6JzRLRm2Qwq1bY1n8yQ/LURsU82+9koIgZHxPBssduB30dE++xNIWeRGSr9X7wMDIyIrtkbac6oVt+1I2Lv7LWK35AZwq7KsY06+fdKKb1L5pKAM3MsXovMNaEVQFlEnAW0qLb8I6Bb5HFnc0RsCJxP5rrWHwNDI6LX/xa9pFWFHUWp4TsXWPJMxZTSx2Ru/vgNmaHYocCeKaW5tbS/K4CmwFzgeeDhlVzvIKA98Hp8e+fzn7PLfkLmJo2pwKfAPWSycJDJwj0CTAYmASOXt5Ps9XankLlBpYJMxvJEMtdFQqYzMwF4BXg1u83zV7IOS+/rMTId9VeAidTs3JVk45gFfEKm03Z8jm3U2b9XSumZlFKubOkjZP7d3iIzzP01NYeVFz9M/OOImLSi/WSH+m8FLkkpTU4p/YfMndMjFt9RLhWdIrmZJbxhTZIkaeWVtOicmvT73o+6zdvXjw2dmFLqW5/79F3PkiRJ+SjgzSX1zaFnSZIk5WRGUZIkKV9F8pbL4qilJEmS8mZGMYdo1CxFk5aFDmOV12ujTisupBUqjqtgtKqo8v7HWlEkl7fVqfenv8fcuXNtyTpmRzGHaNKSJj2PKHQYq7wxT11Y6BBWC2WlJv7VcHyzsLLQIawWGpd5Xn9f2/XbqrABFElv3yNVkiRJOZlRlCRJykt4M4skSZKKmxlFSZKkfHmNoiRJkoqZHUVJkiTl5NCzJElSPgJvZpEkSVJxM6MoSZKUFx+PI0mSpCJnRlGSJClfPh5HkiRJxcyOoiRJknJy6FmSJClf3swiSZKkYmZGUZIkKV/ezCJJkqRiZkZRkiQpH+EDtyVJklTk7ChKkiQpJ4eeJUmS8uXNLJIkSSpmZhQlSZLyFGYUJUmSVMzMKEqSJOUhMKMoSZKkImdHUZIkSTk59CxJkpSPyH6KgBlFSZIk5WRGUZIkKS/hzSySJEkqbmYUJUmS8mRGUZIkSUXNjqIkSZJycuhZkiQpT8Uy9GxHsYHZeesNueiXP6K0JBjxwHiuuO3pGsu7rN2Kq884gHatmvPpZ19x7Hl3MKvisyXL12rWhHEjTmH02CkMvWJUfYffYDwxbipn/mkklVVVHL5Xf375k11rLP9mwUJ+cc6tTH5zBm1aNOcv5x9J145teeqFNzjvulEsXFRJo7JShp20DwP6bligWhTe489N5YzL76Gyqoof770tvz5ytxrLv1mwkOPPHsHLb7xPm5bNuenCo+nasS0Af/zbI9w6ahylJSVcfOoB7Nx/k0JUoUGwHWvHk8+/zh+uGEllZRWH/agfJy1zXi/ipPNu5ZU3ZtC6ZXOuP+8Iupa3ZdLU6Zx2yZ0ApJQ49ZhB7P6DLQpRhQbh8XFT+d3l92aPx/786ogcx+OwEUzOtuNNFxxF145t+WTeFxx5xo28NHU6h+y5DcNPG1KgGqg+Nbih54joFhGvFTqOQigpCS49ZW8OPPVv9Pvxn9h/l15s1K1DjTLn/mJ37nh4EtsfeSXDb36Cs44dVGP57366G+Mmv1ufYTc4lZVVnH7Z3dzxp+N49vbfcd+jE3nz3dk1ytw26nlatWjG+HvO4rhDduDcazOd6jatmnPbZccy5rYzuOaswznhnBGFqEKDUFlZxWnD7+LuK0/g+bt+z72PTuSNaTXbccT942jZoimT7hvG8YfuyLCr7wfgjWmzGfnYJMbdeSb3XHUCp15yF5WVVYWoRsHZjrWjsrKKMy67m39cfixj/nEG9z0+iTff/bBGmX/8axyt1mrK83f/gWMP2oHzr/sXABt3L+eRG3/DE38fyu1/PI7TLrmLRYsqC1GNgqusrGLo8Lu568rjGXfnmdz7yLLH462jxtFqrWZMHHk2xx+yI8OuyRyPTZqU8btj9+Dck/ctROgNTkTU+6cQGlxHsZj16dGFaTM/ZvrsT1i4qJKRT0xm9+1rZg826rY2Yye9A8DYSe8wuNryLTbsRIc2a/Lk+P/Ua9wNzaSp0+nWuT3dOrWjcaMy9tm1Nw+NebVGmYfGvspBu28NwI927MXYCW+RUmLzjbqwTvuWQOaXy9ffLOSbBQvrvQ4NwcQp79G9Szu6dc6043679mb006/UKPPQmFc4ZI9tANh7py15evybpJQY/fQr7Ldrb5o0bsS6ndrRvUs7Jk55rwC1KDzbsXa8NHU663Vuz7qLz+tdevPI2Jrn9SNjX2PI4Mx5veeOW/BM9rxutkZjyspKAfh6wSKKZMQwp4lTprNe53ZLvh/3263PMt+Po59+lYOXHI+9GDM+047NmzahX6/1adLEwchiUmcdxYhoHhEPRsTkiHgtIg6KiLMiYnx2+obIdo8jok+23GTgF9W2cWREjIyIhyPiPxExvNqy3SJiXERMioi7I2LN7PyLI2JqRLwSEZdl5x2Y3efkiBhTV3X+vsrbt2DmnPlLpmdVzKe8XYsaZaa8PZs9B24GwJ4DN6VF8zVo3aIZEcH5J+7BH659sF5jbohmV8yjU4dWS6Y7dmjF7Ir5Ncp8WDGfTmtnypSVldJizTX4ZP4XNcr8698vs/mGnWnSuFFdh9wgza6YT6e1Wy+Z7rh262Xacdacb8tk2rEpn8z/Ytl1Oyy7brGwHWvH7Ir5dMyeswDl7Zc9r2dXzKNjtXZcq/m35/WkKe8x8LCL2PHHFzN86JAlHcdiM7ti3lLHVCtmV8xbqszS349Nl/l+LHpRoE8B1GVGcRAwK6W0RUppM+Bh4JqU0lbZ6abAntmyfwNOSinlumikF3AQ0BM4KCK6REQ74PfALiml3sAE4JSIaAvsC2yaUtocOD+7jbOAH2a3v1euYCPi5xExISImpIVffv/a15E/XPsg2/Vaj6dvPJntenVn5pz5VFZV8dN9+/HY82/UuF5R/7s3ps3mvGtHcdnpBxU6FEm1oPem3Rhz2xk8fONvuOqWx/n6m+IcKZDyVZf541eByyPiEuCBlNLYiNg/IoYCzYA2wJSIGAu0SiktzvSNAAZX284TKaX5ABExFVgXaAVsAjybTUo2BsYB84GvgRsj4gHggew2ngVujoi7gJG5gk0p3QDcAFCyZnmqhfrnbXbFZ3Tq0HLJdMf2LZk9t2bH78OP/8tPfn8rAM2bNuZHP9iMzz7/mq027Ur/LdbjmH3607xpYxo1KuWLrxZwzvUP12sdGoLy9q2YOWfekulZc+ZR3r5ljTLrtG/JzI/m0bFDaxYtquSzz7+mTcvm2fKfcsRv/8o1Z/2Y9Tq3r8/QG5Ty9i2Z+dGnS6ZnffTpMu3YsUOmTKe1F7fjV7Rp2XzZdecsu26xsB1rR3n7lsz6aN6S6dkVy57X5e1bMeujT+nYoRWLFlXy3y++Pa8X27DbOjRv2oQ3ps2mV4+u9RF6g1LevtVSx9Q8ytu3WqpM5vtx6eNRxanOMooppbeA3mQ6jOdHxFnAdcABKaWewF+ANVZiU99U+7mSTOc2gMdSSr2yn01SSseklBYBWwP3kMlWPpyN5TgyGcguwMRs5rHBmfTGB6zfuS1dy1vTqKyU/XbegoeemVqjTJuWzZZc0Prrw3fgttETAPj5eXfS84CL2WLIJfzhutHc+fCkouwkAmzZoyvvzqhg+qyPWbBwEf98bBKDBvSsUWbQgM24c/SLQGaIefu+GxARzP/vlxx6yvX84YS92GaL7oUIv8Hovcm6vPN+BdNnzmXBwkWMfGwSgwduXqPMoAE9uf3BFwC4/8mXGLjVhkQEgwduzsjHJvHNgoVMnzmXd96voM+m3QpQi8KzHWtHrx5dmfZBtfP68Unstv1mNcrsNmAz7nooc14/8O/JbNcnc15Pn/XxkptXZsz+hLff/4gu5W3qvQ4NQe9NujJtRrXj8dGJy3w/Dh7YkzuWHI8vM6DvhkXzKJiVFdT/jSyF+jeos4xiRHQEPkkp3RoR84CfZhfNzV5PeABwT0ppXkTMi4jtU0rPAIetxOafB66NiP9LKb0dEc2BTsAsoFlKaXREPAtMy8ayfkrpBeCFiBhMpsP4cW3WtzZUVlYx9E+juPfyoyktKeG2ByfwxntzOOOYXXn5jQ946NnX2X7L7pz180EkEs9Nfo/T/vjPQofd4JSVlXLRqQcw5JfXUVVVxSF79mPj7uVcfMOD9Nq4K4MG9uSwH/XnhHNGsNUB59K6RTNuOO9IAP5691je/WAul930MJfdlOlo333lCbRvs1YBa1QYZWWlDB86hP1PvpbKysRhe/Wjx/rlXPjnB+jVoyu7/2Bzfrz3thx39i303ncYrVs058YLjgKgx/rl7LPLlvQbcgFlpSVcOnQIpaXFee+c7Vg7yspKufCU/Tnk1/+Pyspvz+tL/jKaXht34YcDenLonv048dxb6XfgebRq0Yzrzz0CgBcnT+PqWx+nUVkpJRFc/JsDadtqzQLXqDDKykoZftqBHHDydVRWJQ77UfZ4vP5BtuzRlcEDe3L4Xv057uxb6LPfObRu0Yy/Zo9HgC32Ppv/fvE1Cxcu4sGnX+Xeq05g4+7lBayR6lqkVDejrBHxQ+BSoApYCBwP7AMcAnwIvAVMTykNi4g+wE1AAh4Fdk8pbRYRRwJ9U0onZrf5AHBZSumpiNgJuARokt3l74HxwP1kMpWRLfv3iBgJbJCd9wTwq7ScipesWZ6a9Dyi1tqiWFU8dWGhQ1gtlBVpx0AN0zcLi/OxMrWtcZnn9fe1Xb+tmDRxQkHSbGVtu6e1Bp9X7/udd9vhE1NKfetzn3WWUUwpPQI8stTsCWQ6dEuXnQhUv5FlaHb+zcDN1crtWe3nJ4Gtcux66xzb32/lI5ckSRL4HEVJkqS8NcRrFCNiUES8GRFvR8TpOZZ3jYh/R8RL2ccI7r6ibdpRlCRJWsVFRClwLZknx2wCHBIRS7/z8/fAXSmlLYGDydxkvFx2FCVJklZ9WwNvp5SmpZQWAHcAey9VJgGL3+TRksxNwMvle3gkSZLy1AAfGdQJmFFt+gNgm6XKDAMejYiTgObALivaqBlFSZKkVUO7xW+Ry35+nuf6hwA3p5Q6A7sDIyJiuX1BM4qSJEn5KNy7l+cu5/E4M8k8J3qxztl51R1D5hXLpJTGRcQaQDtgznft0IyiJEnSqm88sEFErBcRjcncrDJqqTLvAzsDREQPMs+drljeRs0oSpIk5amhXaOYUloUESeSeYZ1KXBTSmlKRJwLTEgpjQJ+A/wlIn5N5saWI5f3AhKwoyhJkrRaSCmNBkYvNe+saj9PBbbLZ5sOPUuSJCknM4qSJEl5CFbuTSmrAzOKkiRJysmMoiRJUp7MKEqSJKmomVGUJEnKV3EkFM0oSpIkKTc7ipIkScrJoWdJkqR8hDezSJIkqciZUZQkScqTGUVJkiQVNTOKkiRJeTKjKEmSpKJmR1GSJEk5OfQsSZKUhyAcepYkSVJxM6MoSZKUr+JIKJpRlCRJUm5mFCVJkvLhK/wkSZJU7OwoSpIkKSeHniVJkvLk0LMkSZKKmhlFSZKkPBVLRtGOYg69NurEM09fVOgwVnltd/hdoUNYLXw6xmNRDUdKhY5g9WA7alVhR1GSJClfxZFQ9BpFSZIk5WZHUZIkSTk59CxJkpSnYrmZxYyiJEmScjKjKEmSlIeIMKMoSZKk4mZGUZIkKU9mFCVJklTU7ChKkiQpJ4eeJUmS8uTQsyRJkoqaGUVJkqR8FUdC0YyiJEmScjOjKEmSlCevUZQkSVJRs6MoSZKknBx6liRJykc49CxJkqQiZ0ZRkiQpDwEUSULRjKIkSZJyM6MoSZKUl/AaRUmSJBU3O4qSJEnKyaFnSZKkPBXJyLMZRUmSJOVmRlGSJClP3swiSZKkomZGUZIkKR/hNYqSJEkqcnYUJUmSlJNDz5IkSXkIoKSkOMaezShKkiQpJzOKkiRJefJmFkmSJBU1M4qSJEl58oHbKognxk1l6wPPo+/+53DF3x9dZvk3CxZyzJk30Xf/c9j16Mt4f9bHAHwy/wv2Pv4quu7wG4Zeeld9h93g7Lz1hrx46ylM/Mep/OqwHyyzvMvarfjnn47hmb+dzL+u/Bkd27dYMv+pv57ImBtP4rm//4qj9tq6vkNvUB5/bipb7X8uvfcdxp9uzn08Hn3GTfTedxi7HHnpkuMR4I9/e4Te+w5jq/3P5YlxU+sz7AbHdqwdTz7/OtsffAH9h5zH1SMeW2b5NwsWcewfbqb/kPPY/Wd/ZMbsj2ss/+DDT1h/l9P4f/94sr5CbpD8PaN82FFsQCorqxh66d3cdcXxPHfHmYx8dCJvTJtdo8yto8bRaq1mTLj3bI4/eEfOufZ+AJo0LuOMY/fgnJP3LUToDUpJSXDpr/fiwNP+Rr+f/In9d96CjdbtUKPMuSfszh2PvMT2R13F8L8/wVk/HwTAhx//l92O/38MPOZqdj3uOn512A6s03atQlSj4Corqzht+F3cfeUJPH/X77k3x/E44v5xtGzRlEn3DeP4Q3dk2NWZ4/GNabMZ+dgkxt15JvdcdQKnXnIXlZVVhahGwdmOtaOysorfXX43t11+LE/fdgb/fHwSb777YY0ytz8wjpZrNWXcXX/g5wftwPnX/avG8mFX/5Od+m1Sn2E3OP6eUb6KrqMYGQ2y3pOmTme9zu3o1qkdjRuVse+ufXhozKs1yjw05lUO3mMbAPbaqRdjxr9FSonmTZvQr9f6NGns1QR9enRh2syPmT77UxYuqmTkE5PZffseNcps1K0DYye9A8DYSdMYnF2+cFElCxZWAtC4UVnRPP4gl4lT3qN7l3Z065w5HvfbtTejn36lRpmHxrzCIdnjce+dtuTp8W+SUmL006+w3669adK4Eet2akf3Lu2YOOW9AtSi8GzH2vHS69Pp1rk962a/H/feuTePjK35/fjw2NcYsntmFGDPHbZg7MTM9yNk2rhreVs2Wm+deo+9IfH3TC3Jvpmlvj+F0GA6TBHxz4iYGBFTIuLn2XmfR8QFETE5Ip6PiLWz89fPTr8aEedHxOfVtnNaRIyPiFci4pzsvG4R8WZE3AK8BnQpRB1XZPaceXRau/WS6Y4dWjG7Yl7NMhXz6dihFQBlZaW0WLMpn8z/oh6jbPjK27Vg5pz5S6ZnVXxGefuWNcpMeXs2ew7cFIA9B25Ki+Zr0LpFMwA6dWjJM387mdfu+S1X/uNpPvz4v/UXfAMyu2J+zeNx7dbMrphfo8ysOd+WqX48LrNuh2XXLRa2Y+34sGI+nbLffQDlHVrx4VJt8WHFPDp2qNaOzdfgk/lf8MWX33DtrU/wm6MH1WfIDZK/Z5SvBtNRBI5OKfUB+gInR0RboDnwfEppC2AM8LNs2SuBK1NKPYEPFm8gInYDNgC2BnoBfSJiYHbxBsB1KaVNU0rT66NCarj+cN1otuu1Hk//9SS267UeM+fMp7IqM6Q3c858tj/qKvocchkHD+pN+9ZrFjhaSd/HZTc9xM8P2oHmzZoUOhStJoLMzSz1/SmEhpQ/PjkiFl/40IVMx24B8EB23kRg1+zP/YF9sj//A7gs+/Nu2c9L2ek1s9t5H5ieUnr+u3aezWL+HKBL167fsyr/m/IOrZj50adLpmfNmUd5+1Y1y7RvyazsX4SLFlXy2edf0aZl83qOtGGbPfczOnX4NoPYsX2LZbIwH378X37y+9sAaN60MT8auBmfff71MmVen/YR/TfvxqinX6v7wBuY8vYtax6PH326TGa2Y4dMmaWPx2XWnbPsusXCdqwd67Rvycw585ZMz54zj3WWaot12rdi1pxP6dihVaYdv/iaNi2bM2nKdB7492TOu24Un33+FSURNGlcxtEHDKTY+HtG+WoQGcWI2AHYBeifzR6+BKwBLEyLLzCBSlbcsQ3gopRSr+zn/1JKN2aXLTdvnlK6IaXUN6XUt1279v9rVb6XLXt0ZdqMCqbPmsuChYu477GJDB7Ys0aZQQN6cseDLwAw6smXGdB3w6K5RX9lTXrjA9bv3I6u5a1pVFbKfjtvwUPPvl6jTJuWzZa0268P24HbRk8AMp3KNbLX37Rccw36bd6Nt2dU1G8FGojem6zLO+9XMH1m5ngc+dgkBg/cvEaZQQN6cnv2eLz/yZcYuFXmeBw8cHNGPjaJbxYsZPrMubzzfgV9Nu1WgFoUnu1YO3pt3JV3P6jg/Vkfs2DhIu5/YhI/3H6zGmV+uP1m3DX6RQAeeGoy2/fZgIjg/v/3S8bfezbj7z2bnw35ASf/ZNei7CSCv2dqT/1nE4s9o9gS+DSl9GVEbAz0W0H554H9gTuBg6vNfwQ4LyJuSyl9HhGdgIV1EnEdKCsr5ZJTD+TAk6+jsipx6I/6sXH3ci66/kF69ejK4IE9OXyv/hw/7Bb67n8OrVo046/nH7Vk/V77nM1/v/iahQsXMfrpV7nnqhPYuHt5AWtUGJWVVQy9YhT3XnY0pSXBbaMn8MZ7czjj6F14+c2ZPPTs62zfqztnHftDUoLnJr/LaX/K3NW34bodOP8Xu5NS5sLha+4Yw9RpHxW4RoVRVlbK8KFD2P/ka6msTBy2Vz96rF/OhX9+gF49urL7Dzbnx3tvy3Fn30LvfYfRukVzbrwgczz2WL+cfXbZkn5DLqCstIRLhw6htLRB/F1a72zH2lFWVsqFv96fQ075f1RWVnHwnv3YqHs5w/8ymi027sIPB/TkkD37cdJ5t9J/yHm0atGMP59zRKHDbnD8PaN8xbcJuwIGEdEE+CfQDXgTaAUMAx5IKa2ZLXMAsGdK6ciI2AC4FWgKPAwcllLqlC33S+Cn2U1/DhxOJhv5QEqp5p+f36F3n77pmXHja6NqRa3tDr8rdAirhU/HXFToEKQlvl5QWegQVguNy4qzw1+btu+/FZMmTihImq1Zx43SBj+7rt73+8q5u0xMKfWtz302iIxiSukbYHCORWtWK3MPcE92cibQL6WUIuJgYKNq5a4kc7PL0laqkyhJkrQixTIa3yA6iv+DPsA1kRmwnwccXdhwJEmSVj+rZEcxpTQW2KLQcUiSpOJULDf4eJGEJEmSclolM4qSJEkFU8BX6tU3M4qSJEnKyY6iJEmScnLoWZIkKQ+L3/VcDMwoSpIkKSczipIkSXkqkoSiGUVJkiTlZkZRkiQpT16jKEmSpKJmR1GSJEk5OfQsSZKUpyIZeTajKEmSpNzMKEqSJOUjvJlFkiRJRc6MoiRJUh4yr/ArdBT1w4yiJEmScrKjKEmSpJwcepYkScpLeDOLJEmSipsZRUmSpDwVSULRjKIkSZJyM6MoSZKUJ69RlCRJUlGzoyhJkqScHHqWJEnKR3gziyRJkoqcGUVJkqQ8ZN71XBwpRTOKkiRJysmMoiRJUp7MKEqSJKmo2VGUJElSTg49S5Ik5alIRp7tKH6XqpQKHcIq79MxFxU6hNVC+8P+XugQVgsVtx1R6BBWC2s0Li10CKuF9yq+KHQIq7wFi6oKHUJRsKMoSZKUJ29mkSRJUlEzoyhJkpQPX+EnSZKkYmdHUZIkSTk59CxJkpSHILyZRZIkScXNjqIkSVKeIur/s+KYYlBEvBkRb0fE6d9RZkhETI2IKRHxjxVt06FnSZKkVVxElALXArsCHwDjI2JUSmlqtTIbAGcA26WUPo2IDivarh1FSZKkPJU0vGsUtwbeTilNA4iIO4C9ganVyvwMuDal9ClASmnOijbq0LMkSdKqoV1ETKj2+Xm1ZZ2AGdWmP8jOq25DYMOIeDYino+IQSvaoRlFSZKkVcPclFLf77F+GbABsAPQGRgTET1TSvOWt4IkSZLy0PBGnpkJdKk23Tk7r7oPgBdSSguBdyPiLTIdx/HftVGHniVJklZ944ENImK9iGgMHAyMWqrMP8lkE4mIdmSGoqctb6NmFCVJkvKQeVxNw0opppQWRcSJwCNAKXBTSmlKRJwLTEgpjcou2y0ipgKVwGkppY+Xt107ipIkSauBlNJoYPRS886q9nMCTsl+VoodRUmSpDyVNKyEYp3xGkVJkiTlZEdRkiRJOTn0LEmSlKeGdjNLXTGjKEmSpJzMKEqSJOWpSBKKZhQlSZKUmxlFSZKkPAQQFEdK0YyiJEmScrKjKEmSpJwcepYkScqTb2aRJElSUTOjKEmSlI8IH7gtSZKk4mZGUZIkKU9FklA0oyhJkqTc7ChKkiQpJ4eeJUmS8hBASZGMPZtRlCRJUk5mFCVJkvJUJAlFO4oNzRPjpnLmn0ZSWVXF4Xv155c/2bXG8m8WLOQX59zK5Ddn0KZFc/5y/pF07diWp154g/OuG8XCRZU0Kitl2En7MKDvhgWqReE9/txUzrj8Hiqrqvjx3tvy6yN3q7H8mwULOf7sEbz8xvu0admcmy48mq4d2wLwx789wq2jxlFaUsLFpx7Azv03KUQVGoSdNu/IBT/emtKS4Nan/sNV/3qtxvJObZtzzXHb0bJZY0pKgvPvmMTjk2eyZfd2/PGn/ZeUu3TkZEZPeL++w28wPB5rh+1YO56d8CaXXj+KqqrEPj/ciqOH7Fhj+cRXp3HZDf/iP+9+yEWnH8Ku22++ZNnsOZ9y7pX38tHceUBwzblH0XHtNvVbAdWrVWboOSI+L3QMda2ysorTL7ubO/50HM/e/jvue3Qib747u0aZ20Y9T6sWzRh/z1kcd8gOnHvtKADatGrObZcdy5jbzuCasw7nhHNGFKIKDUJlZRWnDb+Lu688gefv+j33PjqRN6bVbMcR94+jZYumTLpvGMcfuiPDrr4fgDemzWbkY5MYd+eZ3HPVCZx6yV1UVlYVohoFVxLBxUf24+Dhj7Pd0PvZt/96bNipZY0yp+yzOfc/P52dznyAn18zhkuO6gfAGx98yi6/f4Adf/cvDh7+OJcd3Y/SYnnf1VI8HmuH7Vg7KiuruPi6f3LNuUdz759P4eGnJ/PO+x/VKFPeoRXnnDKEQTv0Wmb9P1x+F0fsP5CR15/KrVecSOuWa9ZT5A1PZB+6XZ+fQlhlOorFYNLU6XTr3J5undrRuFEZ++zam4fGvFqjzENjX+Wg3bcG4Ec79mLshLdIKbH5Rl1Yp33ml/jG3cv5+puFfLNgYb3XoSGYOOU9undpR7fOmXbcb9fejH76lRplHhrzCofssQ0Ae++0JU+Pf5OUEqOffoX9du1Nk8aNWLdTO7p3acfEKe8VoBaF13v9drz30WdMr/ichZVV/PP5dxncp0uNMikl1mraCIAWTRvz4adfAvDVgkoqqxIATRqVkuo39AbF47F22I6147W3ZtClY1s6l7elUaMyfjhwC54aN7VGmY5rt2HD9copWeqPu3fe/4jKyir69c6MVjVr2oSmazSut9hVGKtcRzEyLo2I1yLi1Yg4KDv/jojYo1q5myPigIgozZYfHxGvRMSxhYt++WZXzKNTh1ZLpjt2aMXsivk1ynxYMZ9Oa2fKlJWV0mLNNfhk/hc1yvzr3y+z+YadadK4UV2H3CDNrphPp7VbL5nuuHbrZdpx1pxvy2TasSmfzP9i2XU7LLtusShv04yZH397bM365EvKWzevUebSkZM5YPvuTL76AG4fujNn/P2FJct6r9+OsZfszZiL9+K0m55f0nEsNh6PtcN2rB1zPp7P2u1aLZleu11LKj5eubZ4/4O5rNV8DX5z/i0cfOKV/OnGB4s2M1tMVrmOIrAf0AvYAtgFuDQiyoE7gSEAEdEY2Bl4EDgGmJ9S2grYCvhZRKxXgLjrxRvTZnPetaO47PSDCh2KisC+/dfjjjFvs8VJ93DI8Ce47oQBSy7wnvTOXAb89n52/cOD/HKvnjRptCp+3UhabFFVJS9NeZdfH7MHt155Ih/M/oRRj08odFgFEVGYTyGsit/c2wO3p5QqU0ofAU+T6QA+BOwYEU2AwcCYlNJXwG7ATyLiZeAFoC2wwdIbjYifR8SEiJgwt6KinqpSU3n7VsycM2/J9Kw58yhvX/OasHXat2TmR5kyixZV8tnnX9OmZfNs+U854rd/5Zqzfsx6ndvXV9gNTnn7lsz86NMl07M++nSZduzY4dsymXb8ijYtmy+77pxl1y0Wsz/5kk5tv80gdmzTjNmf1sxeH7bDBtz//HsATHi7giaNSmm71ho1yvxn1ny++HohG3duTTHyeKwdtmPt6NC2ZfZGlIyP5s6nfduVa4u127Vkw+4d6VzelrLSUnbsvylvvD2rjiJVQ7EqdhRzSil9DTwF/BA4iEyGETLPxTwppdQr+1kvpfRojvVvSCn1TSn1bde+MJ2sLXt05d0ZFUyf9TELFi7in49NYtCAnjXKDBqwGXeOfhHIDDFv33cDIoL5//2SQ0+5nj+csBfbbNG9EOE3GL03WZd33q9g+sy5LFi4iJGPTWLwwM1rlBk0oCe3P5gZJr3/yZcYuNWGRASDB27OyMcm8c2ChUyfOZd33q+gz6bdClCLwntp2lzWW6cFXduvSaPSEvbptx4PT/ygRpmZH3/OwM3KAdigY0vWaFTK3M++pmv7NZfcvNK5XXM26NiSGRWr/f1oOXk81g7bsXZsumFn3p/1MTM//ISFCxfxyJjJ7NCvx8qtu0EX/vvF13wyP3Muj5/8Nt27dqjLcBu0koh6/xTCqvh4nLHAsRHxd6ANMBA4LbvsTuCnQF/gyOy8R4DjI+LJlNLCiNgQmJlSqpkaaQDKykq56NQDGPLL66iqquKQPfuxcfdyLr7hQXpt3JVBA3ty2I/6c8I5I9jqgHNp3aIZN5x3JAB/vXss734wl8tuepjLbnoYgLuvPIH2bdYqYI0Ko6yslOFDh7D/yddSWZk4bK9+9Fi/nAv//AC9enRl9x9szo/33pbjzr6F3vsOo3WL5tx4wVEA9Fi/nH122ZJ+Qy6grLSES4cOobR0tfl7Ki+VVYkzbn6Bu367CyUlJdz+9H94c+Y8frt/L15+92MemTSDs26bwJ9+ui3HDso8auSk658FYJuNOnDyj3qyqLKKqqrE0L89zyeff1PI6hSMx2PtsB1rR1lpKb89fm9O+P2NVFVVsfduW7H+uutw3YhH2WSDzuzQbxOmvDWDU867hc8+/4oxL7zOn299jHv//BtKS0s45ZjdOe6Mv5AS9NigE/sN2rrQVVIdi5RWjQvMI+LzlNKakbk/fDiZ4eUEnJ9SujNbphHwEXB/Sumo7LwS4HzgR2SyixXAPiml77x6t3efvmnMcy/WaX2KQVmRfhHXtvaH/b3QIawWKm47otAhSEu8V9HgchWrnAMGDeC1yZMKkmZrs94mabdht9X7fu88svfElFLf+tznKpNRTCmtmf1/IpNBPC1HmYVksozV51UBv8t+JEmStJJM+UiSJCmnVSajKEmS1FAU6k0p9c2MoiRJknIyoyhJkpSHAIrl9fVmFCVJkpSTGUVJkqR8RHiNoiRJkoqbHUVJkiTl5NCzJElSnopk5NmMoiRJknIzoyhJkpQnb2aRJElSUTOjKEmSlIdieuD2d3YUI+JqIH3X8pTSyXUSkSRJkhqE5WUUJ9RbFJIkSWpwvrOjmFL6e/XpiGiWUvqy7kOSJElq2LyZJSsi+kfEVOCN7PQWEXFdnUcmSZKkglqZu56vAH4IfAyQUpoMDKzDmCRJkhq0KMCnEFbq8TgppRlLzaqsg1gkSZLUgKzM43FmRMS2QIqIRsAvgdfrNixJkqSGKQJKvEZxieOAXwCdgFlAr+y0JEmSVmMrzCimlOYCh9VDLJIkSWpAVuau5+4R8a+IqIiIORFxf0R0r4/gJEmSGqKI+v8UwsoMPf8DuAsoBzoCdwO312VQkiRJKryV6Sg2SymNSCktyn5uBdao68AkSZIaqoio908hLO9dz22yPz4UEacDd5B59/NBwOh6iE2SJEkFtLybWSaS6Rgu7sIeW21ZAs6oq6AkSZIasiJ5Os5y3/W8Xn0GIkmSpIZlZR64TURsBmxCtWsTU0q31FVQkiRJKrwVdhQj4mxgBzIdxdHAYOAZwI6iJEkqOkH4ZpZqDgB2Bj5MKR0FbAG0rNOoJEmSVHArM/T8VUqpKiIWRUQLYA7QpY7jkiRJapgK+ADs+rYyHcUJEdEK+AuZO6E/B8bVZVCSJEkqvJV51/MJ2R//HBEPAy1SSq/UbViSJEkNV6EegF3flvfA7d7LW5ZSmlQ3IUmSJKkhWF5G8fLlLEvATrUcS4NSLH8pqOH7cMRPCh3CamHtn4wodAirhVeuPrDQIawW2rdoUugQVnllpf6erg/Le+D2jvUZiCRJ0qpiZR4bszoolnpKkiQpTyv1ZhZJkiRlBMVziZoZRUmSJOW0Mq/wC+AwoHtK6dyI6Aqsk1J6sc6jkyRJaoBKiiOhuFIZxeuA/sAh2en/AtfWWUSSJElqEFbmGsVtUkq9I+IlgJTSpxHRuI7jkiRJUoGtTEdxYUSUknl2IhHRHqiq06gkSZIaMIeev3UVcB/QISIuAJ4BLqzTqCRJklRwK/Ou59siYiKwM5k7wvdJKb1e55FJkiQ1QBHF83iclbnruSvwJfCv6vNSSu/XZWCSJEkqrJW5RvFBMtcnBrAGsB7wJrBpHcYlSZLUYBXLNYorM/Tcs/p0RPQGTqiziCRJktQg5P1mlpTSJGCbOohFkiRJDcjKXKN4SrXJEqA3MKvOIpIkSWrgiuRelpW6RnGtaj8vInPN4r11E44kSZIaiuV2FLMP2l4rpXRqPcUjSZLUoAVQUiQpxe+8RjEiylJKlcB29RiPJEmSGojlZRRfJHM94ssRMQq4G/hi8cKU0sg6jk2SJEkFtDLXKK4BfAzsxLfPU0yAHUVJklSU8n5szCpqeR3FDtk7nl/j2w7iYqlOo5IkSVLBLa+jWAqsSc0O4mJ2FCVJUtEqkntZlttRnJ1SOrfeIpEkSVKDsryOYpH0lSVJklZeRPh4HGDneotCkiRJDc53dhRTSp/UZyCSJElqWFbm8TiSJEmqpkhGnovmMUCSJEnKkxlFSZKkPJWYUZQkSVIxM6MoSZKUhwAfjyNJkqTiZkdRkiRJOTn0LEmSlKciGXk2oyhJkqTczChKkiTlI4rn8Th2FBuYJ8ZN5Xd/vJeqqioO36s/vzxitxrLv1mwkBPOGcErb8ygdcvm/PX8o+jasS2fzP+Co06/kZdfn87Be2zDJacNKVANGobHn5vKGZffQ2VVFT/ee1t+feSy7Xj82SN4+Y33adOyOTddeDRdO7YF4I9/e4RbR42jtKSEi089gJ37b1KIKjQIHo+1Y8eeHTn/8L6UlgS3Pf02Vz8wpcbyTm2bcfXPtqNF88aURnD+XZN44pVZtF6zMTee+AN6dW/LHWPf4XcjxheoBg3D2Bff4ILr7qeqqooDBm/Dzw/Zqcby8a+8w0XXjeLNabO5/PeHMWjgFkuW/fT0vzD59en03mw9rr/gmPoOvUH59/Ovc/aVI6msShyyZz9O/PEuNZZ/s2ARvzr/Vl558wNat2jG/zv3CLqUt2XG7I/Z4bCLWb9rewB6b9qNi4v83C4Gq/zQc0SMjohWhY6jNlRWVvHbS+/mziuO59k7zmTkoxN5c9rsGmVuGzWOVms1Y/y9Z3PcwTtyzrX3A9CkcRlnHLsHw07etxChNyiVlVWcNvwu7r7yBJ6/6/fc++hE3liqHUfcP46WLZoy6b5hHH/ojgy7OtOOb0ybzcjHJjHuzjO556oTOPWSu6isrCpENQrO47F2lERw8U+25tDLnmTA6f9i337d2LBjyxplfr3X5tz/4nR2+cODHHvdWC4+YhsAvllQxcUjX2bY7RMLEXqDUllZxblX38dfLvwpD9x4Gg/++yXenv5hjTLlHVpz0dCD2HOnLZdZ/5ghO3DJ6YfUV7gNVmVlFb//4z2MuOxY/n3r6dz/+CTeerdmO97xwPO0XKsZz975e3520A5c+P/+tWRZt05tefTmoTx689Ci7yRGAf4rhAbXUYyIlcpyRkZJSmn3lNK8Og6rXkyaOp31OrejW6d2NG5Uxr679uGhMa/WKPPQmFc5eI/ML5G9durF2PFvkVKiedMm9Ou1Pms0Nkk8ccp7dO/Sjm6dM+243669Gf30KzXKPDTmFQ7JtuPeO23J0+PfJKXE6KdfYb9de9OkcSPW7dSO7l3aMXHKewWoReF5PNaO3uu35d05/2V6xecsrKzin89PZ1DvLjXKpJRYq2kjAFo0a8RH874E4MsFi3jxrQq+WVhZ73E3NK+8+T5dO7alS8e2NG5Uxu479OKJZ2tmZjuv04aNunckcowJ9u+9Ac2bNqmvcBusl1+fTrfO7Vg3e17vvcuWPPpMzfP60Wde5cDBWwGwxw5b8MzE/5BSKkS4agDqrKMYEc0j4sGImBwRr0XEQRHxXkS0yy7vGxFPZX8eFhEjIuJZYEREHBkR90fEUxHxn4g4O1uuW0S8GRG3AK8BXRZvM9f+suv0iYinI2JiRDwSEeV1Vefva/aceXRcu/WS6Y4dWjG7Yl7NMhXz6dShFQBlZaW0WLMpn8z/oh6jbPhmV8ynU/V2XLs1syvm1ygza863Zaq34zLrdlh23WLh8Vg71mndjFkff9smsz75gnVaN61R5tL7XuGAbdfjpSv247bf7FT0Q8y5fDR3PuXZYw1gnfat+Ojj4jw3v4/ZFfMp7/Dteb1O+1bLfMd9WK1MWVkpLZqvwafZ8/r92Z/ww6MuZf8Tr+aFye/UX+AqmLr8c38QMCultAdARLQELllO+U2A7VNKX0XEkcDWwGbAl8D4iHgQmAtsAByRUno+u93v3F9ENAKuBvZOKVVkO48XAEfXak0l6XvYt3837hj7Dn9++HX6/l87rjl2O37wu39hEkcNSYe2LXnx3rNp3bI5r7wxg2N+dyNPjjidtZqvUejQ6l3mzSyFjqJ+1OXQ86vArhFxSUQMSCmt6E+/USmlr6pNP5ZS+jg7bySwfXb+9MWdxJXY30ZkOpuPRcTLwO+Bzrl2HhE/j4gJETFh7tyKla9lLSrv0IpZH326ZHrWnHmUt29Vs0z7lsycMw+ARYsq+ezzr2jTsnk9Rtnwlbdvyczq7fjRp5S3r3lNWMcO35ap3o7LrDtn2XWLhcdj7fjw0y/p2PbbNunYpjkffvpVjTKHDvw/Rr04HYAJb89ljUaltF2z+H75Ls/a7VoyO3usAXxYMY+12xbnufl9lLdvyew5357XH1bMW+Y7bp1qZRYtquSzL76mdcvmNGlcRuvs+b35xl1Yt2Nbps2YU3/BqyDqrKOYUnoL6E2mA3d+RJwFLKq2z6W/BZcer1r6b+n0HeWWt78ApqSUemU/PVNKu33H+jeklPqmlPq2a9d+JWpY+7bs0ZVpMyqYPmsuCxYu4r7HJjJoYM8aZQYN6MkdD74AwKgnX2ZA3w2rZ1UF9N5kXd55v4LpMzPtOPKxSQweuHmNMoMG9OT2bDve/+RLDNwq046DB27OyMcm8c2ChUyfOZd33q+gz6bdClCLwvN4rB0vTfuY7muvRdd2a9KotIR9+q3LIy/NqFFm5sdfMGCTdQDYoGMLmjQqZe5/vy5EuA1Wz426MH3mXD6Y/TELFi5i9FMvs9O2mxY6rFXOFht35d0Zc3l/VqYd73/8JXbdbrMaZXbdbjPufihz+cODT01mu94bEBF8/OnnS27umz5zLu9+MHfJ0yKKUUnU/6cQ6mzoOSI6Ap+klG6NiHnAT4H3gD7AQ8D+K9jErhHRBvgK2IcVDBd/x/4uBtpHRP+U0rjsUPSGKaUpy9tWoZSVlXLxqQdy4MnXUVWVOPRH/di4ezkXXf8gvXp0ZfDAnhy2V39OGHYLW+1/Dq1aNOMv5x+1ZP0t9zmb/37xNQsXLmL0069yz1UnsFH3BntJZp0pKytl+NAh7H/ytVRWJg7bqx891i/nwj8/QK8eXdn9B5vz47235bizb6H3vsNo3aI5N16Qacce65ezzy5b0m/IBZSVlnDp0CGUlja4e77qhcdj7aisSpxxy4vcMXRnSiO4fczbvDlzPkP324LJ737MIy99wLDbJ3L50f04dlAPUoKT//LckvXHX74vazVtROOyEgb36cJBw5/grVnFd21eWWkpfzhpX445/S9UVSX2H7QVG3Rbh6tufpjNNuzCTttuyqtvvM+Jw/7OZ59/yb/HTeWavz/KAzeeBsBhv7qWaTPm8OVX3/CDg8/j/N8MYcBWGxW4VvWvrKyU807Zn8NO+TNVVVUctMc2bNS9nEv/OpotNu7KbttvxsF79uOX593KdgedT6sWzbhu2E8AeH7yO1z+14coKyuhpKSEi089kNYtHEFY3UVd3ckUET8ELgWqgIXA8UBT4EbgM+ApoG9KaYeIGAZ8nlK6LLvukWQ6hy3JDBXfmlI6JyK6AQ+klDartp/3gL5kOqA19pdSmhARvYCrstsqA65IKf1lebH37tM3jR3nxeTfV2mxXMBRxyqrvFCtNnQ88tZCh7BaeOXqAwsdwmphzTV8IsD3tfOAbXh50sSC/KLpvFHP9Mvr76/3/Q7dcf2JKaW+9bnPOjtSU0qPAI/kWLRhjrLDcpT7IKW0z1Ll3iNzzWH1ed2yP+bcX0rpZWDgiiOWJElSdcU5piZJkqQVapC575TSzcDNBQ5DkiRpGT4eR5IkSauUiBiUfTHJ2xFx+nLK7R8RKSJWeL1jg8woSpIkNVgBDe1JYBFRClwL7Ap8QOZlJaNSSlOXKrcW8EvghZXZrhlFSZKkVd/WwNsppWkppQXAHcDeOcqdR+ZNeSv1sFY7ipIkSXkqiaj3zwp0Aqo/zf+D7LwlIqI30CWl9ODK1tOhZ0mSpFVDu4iYUG36hpTSDSuzYkSUAH8Ejsxnh3YUJUmSVg1zl/PA7ZlAl2rTnbPzFluLzLOon8q+anUdYFRE7JVSqt75rMGOoiRJUh4a6ONxxgMbRMR6ZDqIBwOHLl6YUpoPtFs8HRFPAacur5MIXqMoSZK0ykspLQJOJPOWuteBu1JKUyLi3IjY63/drhlFSZKkPDW0x+MApJRGA6OXmnfWd5TdYWW2aUZRkiRJOZlRlCRJyktQQgNMKdYBM4qSJEnKyY6iJEmScnLoWZIkKQ9Bw7yZpS6YUZQkSVJOZhQlSZLyEQ3ygdt1woyiJEmScjKjKEmSlKeSIrlI0YyiJEmScrKjKEmSpJwcepYkScqDj8eRJElS0TOjKEmSlCdvZpEkSVJRM6MoSZKUpyJJKJpRlCRJUm52FCVJkpSTQ8+SJEl5CIon01Ys9ZQkSVKezChKkiTlIyCK5G4WM4qSJEnKyYyiJElSnoojn2hGUZIkSd/BjOJ3KJa/FOpSSqnQIawWSks8GmvDjJsOK3QIq4W1B5xa6BBWC588d3mhQ1jlFcsr9ArNjqIkSVIeguLpqDr0LEmSpJzMKEqSJOWpOPKJZhQlSZL0HcwoSpIk5alILlE0oyhJkqTc7ChKkiQpJ4eeJUmS8hK+61mSJEnFzYyiJElSHoLiybQVSz0lSZKUJzOKkiRJefIaRUmSJBU1O4qSJEnKyaFnSZKkPBXHwLMZRUmSJH0HM4qSJEn5CG9mkSRJUpEzoyhJkpQHH7gtSZKkomdHUZIkSTk59CxJkpQnb2aRJElSUTOjKEmSlKfiyCeaUZQkSdJ3MKMoSZKUpyK5RNGMoiRJknKzoyhJkqScHHqWJEnKQ+bNLMUx9mxGUZIkSTmZUZQkScqTN7NIkiSpqJlRlCRJyksQXqMoSZKkYmZHUZIkSTk59CxJkpSnYrmZxY5iA/PEuKmc8cd7qaqq4vC9+vOrI3arsfybBQs54ZwRTH5jBq1bNufG84+ia8e2fDL/C446/UZeen06B++xDcNPG1KgGjQMj4+byu8uv5fKqip+vHfudjx+2LfteNMFmXYE+NPNj3LrqHGUlpRw0W8OYOf+PQpRhQbh8eemcsbl92TbcVt+fWSOdjx7BC+/8T5tWjbnpguPXtKOf/zbI0va8eJTD2Dn/psUogoNwpPjpvL7K0ZSWVnFYXv15+Sf7Fpj+TcLFnLiubfySvZ4vOH8I+la3pZJU6Zz6iV3AJBS4rRjBrP7DlsUogoNws79NuaiX+1DaWkJI0Y9zxUjnqyxvMs6rbn6zINo12pNPv3sS44ddhuzKuYDMPeZy5j6zmwAPvjoUw4delO9x99Q+P2ofKxSQ88RcXJEvB4RtxU6lrpQWVnF0Evv5q4rjue5O85k5KMTeWPa7Bplbh01jlZrNWPCvWdz/ME7cs619wPQpHEZZxy7B+ecvG8hQm9QKiurGDr8bu668njG3Xkm9z7y3e04ceTZHH/Ijgy7JtOOb0ybzchHJ/LcHb/j7iuP57Thd1FZWVWIahRcZWUVpw2/i7uvPIHn7/o99+Y4HkfcP46WLZoy6b5hHH/ojgy7ulo7PjaJcXeeyT1XncCplxR3O55++d3844/HMfb233HfYxN5892a7fiPfz1Pq7Wa8cI9Z3HswTtw3rWjANh4/XIevelUnrzlt9zxp+M5dfidLFpUWYhqFFxJSXDpb/bjwFNuoN8hl7D/rr3ZqNvaNcqce9KPuOOhCWz/48sYftOjnHX8HkuWffXNQgYecTkDj7i8qDuJfj/WjsUP3K7vTyGsUh1F4ARg15TSYf/rBiKiwWZRJ02dznqd29GtUzsaNypj31378NCYV2uUeWjMqxy8xzYA7LVTL8aMf4uUEs2bNqFfr/Vp0rjBVq/eTJxSsx33223Zdhz99LftuHe1dnxozKvst1sfmjRuxLqd2rFe53ZMnDK9ENUouIlT3qN7l3Z065xtx117M/rpV2qUeWjMKxyypB235Onxb5JSYvTTr7Dfrr2XtGP3Lu2YOOW9AtSi8DLndfslx+M+u/Tm4aWOx4fHvsqQ3bcG4Ec79uKZCZnjsdkajSkrKwXg6wWLiuYuy1z6bNKVaR/MZfqsT1i4qJKRj7/E7gM3q1Fmo27rMHbC2wCMnfg2g5daLr8flb9VpqMYEX8GugMPRcSZEXFTRLwYES9FxN7ZMt0iYmxETMp+ts3O3yE7fxQwtYDVWK7Zc+bRae3WS6Y7dmjF7Ip5NctUzKdjh1YAlJWV0mLNpnwy/4t6jLLhm12xcu3Yae1WQM12XJl1i0Wmjaq1xdqtmZ0dxlts1pxvy9Rsx6XW7bDsusXiw4p5S85ZyBxTHy7VFksfj2utucaS83rilPcYeOiF7HD4RVw6dMiSjmOxKW/fkplz5i2ZnjVnHuXtW9YoM+XtWey5Q08A9vxBT1o0X4PWLZoBsEbjMp686dc8+pdfLtPBLCZ+P9aSyFyjWN+fQlhl0k8ppeMiYhCwI3AK8GRK6eiIaAW8GBGPA3PIZBy/jogNgNuBvtlN9AY2Sym9W4DwJel/0mfTboz5x+94670POencW9mp/yas0aRRocNqkP5w9SiG/2Y/Dt1jK557aRoz58yjsiozNLr5fuczu2I+63Zsw6hrTmDqO7N5b+bHBY5YavhWmYziUnYDTo+Il4GngDWArkAj4C8R8SpwN1D96vkXl9dJjIifR8SEiJgwd25FnQW+POUdWjHzo0+XTGf+Ym5Vs0z7lszK/lW9aFEln33+FW1aNq/HKBu+8vYr144zP5oH1GzHlVm3WGTaqFpbfPTpMhmcjh2+LVOzHZdad86y6xaLddq3WnLOQuaYWmeptlj6ePzv518vc15v2G0dmjdrssz1ZMVidsV8Oi2VmV06S/3h3M/4yRk384Mj/sj5148G4LPPv16yPsD0WZ/wzKS32XzDTvUTeAPj96Pytap2FAPYP6XUK/vpmlJ6Hfg18BGwBZlMYuNq6yx3fDaldENKqW9KqW+7du3rLPDl2bJHV6bNqGD6rLksWLiI+x6byOCBPWuUGTSgJ3c8+AIAo558mQF9NySK5R79ldR7k2w7zsy048hHJzJoQM12HDzw23a8v1o7DhrQk5GPTuSbBQuZPnMu02ZU0GfTdQtRjYLrvcm6vPN+tXZ8bBKDB25eo8ygAT25fUk7vsTArTLtOHjg5ox8bNKSdnzn/Qr6bNqtALUovG/P649ZsHAR/3x8Ej9c6nj84fabcdfoFwH4179fZvs+GxARTJ/18ZKbV2bM/oS3p39El/I29V6HhmDS6zNYv0t7upa3oVFZKfvtsiUPjX2tRpk2LZsv+T789U925rYHMm3acq2mNG5UuqTMNpuvx5vvflS/FWgg/H6sPQ49N2yPACdFxEkppRQRW6aUXgJaAh+klKoi4ghglbqYp6yslEtOPZADT76OyqrEoT/qx8bdy7no+gfp1aMrgwf25PC9+nP8sFvou/85tGrRjL+ef9SS9Xvtczb//eJrFi5cxOinX+Weq05g4+7lBaxRYZSVlTL8tAM5INuOh/2oHz3WL+fC6x9ky2rteNzZt9Bnv3No3aIZf70g04491i9nn1160/+gCykrLWH40AMpLV1V/576fsrKShk+dAj7n3wtlZWJw/bKtuOfH6BXj67s/oPN+fHe23Lc2bfQe99htG7RnBtrtOOW9BtyAWWlJVw6dEhRt+NFvzmAg391HZVVVRyyZ+a8vuSGB9miR1cGDejJoT/qz4nnjGCbA86lVYtmXH/ekQC8OPkdrh7xOGVlpZREcPGpQ2jbas3CVqhAKiurGHr5SO694ueUlpRw2wMv8sa7H3HGzwbx8uszeOiZKWzfe33OOn4PUko89/I0TrvsXgA26rY2f/rtgVRVJUpKgitGPMmb7xVnR9HvR+UrUkqFjmGlRcR7ZDKFXwBXANuSyYq+m1LaM3td4r1AAh4GfpFSWjMidgBOTSntuTL76d2nb3pm3Phaj7/YmOisHWaMa8eCRcX5GI/atvaAUwsdwmrhk+cuL3QIq7zt+m3FpIkTCvIFueFmvdK1dz9e7/vdbZP2E1NKfVdcsvasUhnFlFK3apPH5lj+H6D62Nhvs/OfInMtoyRJklbSKtVRlCRJKrQASopksMeLCyRJkpSTHUVJkiTl5NCzJElSnorllZpmFCVJkpSTGUVJkqQ8FcuTy8woSpIkKSczipIkSXnyGkVJkiQVNTuKkiRJysmhZ0mSpDz4ZhZJkiQVPTOKkiRJeQlvZpEkSVJxM6MoSZKUj/CB25IkSSpydhQlSZKUk0PPkiRJeSqSkWczipIkScrNjKIkSVIeMg/cLo6cohlFSZIk5WRGUZIkKU/FkU80oyhJkqTvYEdRkiRJOTn0LEmSlK8iGXs2oyhJkqSczChKkiTlKYokpWhGUZIkSTmZUZQkScpTkTxv24yiJEmScrOjKEmSpJwcepYkScpTkYw8m1GUJElSbmYUJUmS8lUkKUUzipIkScrJjOJ3KJbb3utS2Ii1YlFlVaFDWC00KvV4rA2fjvtjoUNYLbTe6sRCh7DK++bN9wu278AHbkuSJKnI2VGUJElSTg49S5Ik5SOK5xI1M4qSJEnKyYyiJElSnookoWhGUZIkSbmZUZQkScpXkaQUzShKkiQpJzuKkiRJq4GIGBQRb0bE2xFxeo7lp0TE1Ih4JSKeiIh1V7RNO4qSJEl5iYL8t9yIIkqBa4HBwCbAIRGxyVLFXgL6ppQ2B+4Bhq+opnYUJUmSVn1bA2+nlKallBYAdwB7Vy+QUvp3SunL7OTzQOcVbdSbWSRJkvJUoAdut4uICdWmb0gp3ZD9uRMwo9qyD4BtlrOtY4CHVrRDO4qSJEmrhrkppb7fdyMRcTjQF/jBisraUZQkScpD0CCfjjMT6FJtunN2Xg0RsQtwJvCDlNI3K9qo1yhKkiSt+sYDG0TEehHRGDgYGFW9QERsCVwP7JVSmrMyG7WjKEmStIpLKS0CTgQeAV4H7kopTYmIcyNir2yxS4E1gbsj4uWIGPUdm1vCoWdJkqR8NcCx55TSaGD0UvPOqvbzLvlu04yiJEmScjKjKEmSlKcVPQB7dWFGUZIkSTmZUZQkScpTgR64Xe/MKEqSJCknO4qSJEnKyaFnSZKkPBXJyLMZRUmSJOVmRlGSJCkfDfRlz3XBjKIkSZJyMqMoSZKUJx+4LUmSpKJmR1GSJEk5OfQsSZKUh8A3s0iSJKnImVGUJEnKU5EkFM0oSpIkKTczipIkSfkqkpSiGUVJkiTlZEaxgXl83FR+d/m9VFZV8eO9+/OrI3arsfybBQs5ftgIJr8xg9Ytm3PTBUfRtWNbAP5086PcOmocpSUlXPSbA9i5f49CVKFBePy5qZxx+T3ZdtyWXx+Zox3PHsHLb7xPm5bNuenCo5e04x//9siSdrz41APYuf8mhahCg/DEuKmc+aeRVFZVcfhe/fnlT3atsfybBQv5xTm3MvnNGbRp0Zy/nH8kXTu25akX3uC860axcFEljcpKGXbSPgzou2GBalF4nte1w/O6duzcvwcX/eYASktKGHH/c1zx98dqLO+yTmuuPutw2rVak08/+5Jjz/o7s+bMY/s+G3DhKfsvKbfBumtzzJl/Y/TTr9R3FVSPVouMYkQ8V+gYakNlZRVDh9/NXVcez7g7z+TeRybyxrTZNcrcOmocrdZqxsSRZ3P8ITsy7Jr7AXhj2mxGPjqR5+74HXdfeTynDb+LysqqQlSj4Corqzht+F3cfeUJPH/X77n30WXbccT942jZoimT7hvG8YfuyLCrq7XjY5MYd+eZ3HPVCZx6SXG34+mX3c0dfzqOZ2//Hfc9OpE3363ZjreNep5WLZox/p6zOO6QHTj32lEAtGnVnNsuO5Yxt53BNWcdzgnnjChEFRoEz+va4XldO0pKgkuHDuHAX15HvyHns/9ufdhovXVqlDn3l/tyx4Mvsv2hFzH8rw9x1i/2AuCZif9h4GEXM/Cwi9nr+Kv46usF/Pv51wtRjQYhCvBfIawWHcWU0raFjqE2TJwynfU6t6Nbp3Y0blTGfrv14aExr9YoM/rpVzl4j20A2HunXowZ/xYpJR4a8yr77daHJo0bsW6ndqzXuR0Tp0wvRDUKbuKU9+jepR3dOmfbcdfey/zF+9CYVzhkSTtuydPj3ySlxOinX2G/XXsvacfuXdoxccp7BahF4U2aOp1undsvOR732bX3MsfjQ2Nf5aDdtwbgRzv2YuyEzPG4+UZdWKd9SwA27l7O198s5JsFC+u9Dg2B53Xt8LyuHX027ca0GXOZPvNjFi6qZORjk9j9B5vXKLNR93LGTngTgLET3mLwwJ7LbGfvnbfk8XFT+eqb4jyvi8lq0VGMiM8j49KIeC0iXo2Ig7LLbomIfaqVvS0i9i5YsMsxu2IendZuvWS6Y4dWzK6Yt1SZ+XRauxUAZWWltFizKZ/M/2Kl1i0WmTaq1hZrt2Z2xfwaZWbN+bZMzXZcat0Oy65bLGZXzKNTh1ZLpjPHVM22+HCZ43ENPpn/RY0y//r3y2y+YWeaNG5U1yE3SJ7XtcPzunaUt2/JzI8+XTI966NPKc/+UbfYlLdmsueOvQDYc8ctaLFmU1q3bF6jzH679ubeRybWebwNWUT9fwphtegoZu0H9AK2AHYBLo2IcuBG4EiAiGgJbAs8uPTKEfHziJgQERPmzq2or5il1dob02Zz3rWjuOz0gwodiqSV9Icr72O73v/H07f+lu16/x8zP/q0xlD92m1bsMn/deSJcVMLGKXqy+rUUdweuD2lVJlS+gh4GtgqpfQ0sEFEtAcOAe5NKS1aeuWU0g0ppb4ppb7t2rWv38izytu3qvmX3px5lLdvtVSZlsz8aB4AixZV8tnnX9GmZfOVWrdYrMxfzB07fFumZjsute6cZdctFuXtWzFzzrwl05ljqmZbrLPM8fg1bbKZh1lzPuWI3/6Va876Met1Lsw51RB4XtcOz+vasTKZ2Q/nzucnQ//KDw6/hPOv+xcAn33+1ZLl++zamweeeoVFRXqd52JRgE8hrE4dxeW5BTgcOAq4qcCxfKfem3Rl2owKps+cy4KFixj56EQGDah5bcjggT2548EXALj/yZcZ0HdDIoJBA3oy8tGJfLNgIdNnzmXajAr6bLpuIapRcL03WZd33q/Wjo9NYvDAmtfgDBrQk9uXtONLDNwq046DB27OyMcmLWnHd96voM+m3QpQi8LbskdX3p1RwfRZH7Ng4SL++dikZY7HQQM2487RLwKZIebt+25ARDD/v19y6CnX84cT9mKbLboXIvwGw/O6dnhe145JU6ezftf2dO3YlkZlpey3a28eGlPzWs82LZsT2XHOXx/5Q2771/M1lu+/Wx/ufWRCvcWswlqdHo8zFjg2Iv4OtAEGAqdll90MvAh8mFJqsLnysrJShp92IAecfB2VVYnDftSPHuuXc+H1D7Jlj64MHtiTw/fqz3Fn30Kf/c6hdYtm/PWCowDosX45++zSm/4HXUhZaQnDhx5IaWmx/B1QU1lZKcOHDmH/k6+lsjJx2F7ZdvzzA/Tq0ZXdf7A5P957W447+xZ67zuM1i2ac2ONdtySfkMuoKy0hEuHDinqdrzo1AMY8svrqKqq4pA9+7Fx93IuvuFBem3clUEDe3LYj/pzwjkj2OqAc2ndohk3nHckAH+9eyzvfjCXy256mMtuehiAu688gfZt1ipgjQrD87p2eF7Xjsxd+Hdx71W/oLQ0uG3U87wx7UPOOHYPXn79fR4a8yrb99mAs36xFynBcy+9zWnD71qyfpfyNnRauzXPTnq7gLVQfYqUUqFj+N4i4r9AC2A4MBhIwPkppTurlXkY+GdK6c8r2l7vPn3Ts8+Pr6twi0YU6srb1UyxD+/UltISj8fa4HldO1pvdWKhQ1jlffPmXVR9OacgB+RmW/ROIx99pt73u9E6zSemlPrW5z5X+YxiRLQFPkmZHu9pfJtFrF6mGbABcHs9hydJkrTKWqVz7xHRERgHXLacMrsArwNXp5SK83kIkiSp1mRuLimOB26v0hnFlNIsYLnvBUspPQ4U59XfkiRJ38Mq3VGUJEmqdwV8AHZ9W6WHniVJklR37ChKkiQpJ4eeJUmS8lQkI89mFCVJkpSbGUVJkqR8FUlK0YyiJEmScjKjKEmSlJfCPQC7vplRlCRJUk52FCVJkpSTQ8+SJEl58s0skiRJKmpmFCVJkvIQFM3TccwoSpIkKTczipIkSfkqkpSiGUVJkiTlZEdRkiRJOTn0LEmSlCffzCJJkqSiZkZRkiQpTz5wW5IkSUXNjKIkSVKeiiShaEZRkiRJudlRlCRJUk4OPUuSJOUjvJlFkiRJRc6MoiRJUt6KI6VoRlGSJEk5mVGUJEnKQ+A1ipIkSSpydhQlSZKUk0PPkiRJeSqSkWczipIkScrNjGIOL02aOLdZ45LphY5jBdoBcwsdxGrAdqwdtuP3ZxvWDtuxdqwK7bhuIXdeLDez2FHMIaXUvtAxrEhETEgp9S10HKs627F22I7fn21YO2zH2mE7ajE7ipIkSXmKIrlK0WsUJUmSlJMdxVXXDYUOYDVhO9YO2/H7sw1rh+1YO2xHARAppULHIEmStMrYYss+6ZGnn6/3/Za3bDyxvq8dNaMoSZKknLyZRZIkKU/FcSuLGcUGLyK6RcRrhY5DxSkiPi90DIKIGB0RrQodR6FFxMkR8XpE3FboWFYnEfFcoWNQw2VGUcpTRASZ63urCh2LVk0RUZZSWrQS5RYfa7vXQ1irghOAXVJKH/yvG1jZti8mKaVtCx3DqiaieB64bUaxnkRE84h4MCImR8RrEXFQRJwVEeOz0zdkfykQEX2y5SYDv6i2jSMjYmREPBwR/4mI4dWW7RYR4yJiUkTcHRFrZudfHBFTI+KViLgsO+/A7D4nR8SYem6KOhMR/4yIiRExJSJ+np33eURckK3r8xGxdnb++tnpVyPi/OqZs4g4Lfvv8kpEnJOd1y0i3oyIW4DXgC6FqGOhRMal2ePm1Yg4KDv/jojYo1q5myPigIgozZZf3I7HFi76uvMd5/V7EdEuu7xvRDyV/XlYRIyIiGeBEdnz+f6IeCp7Pp+dLbfMsbZ4m7n2l12nT0Q8nT3+H4mI8sK0SN2JiD8D3YGHIuLMiLgpIl6MiJciYu9smW4RMTb7PTgpIrbNzt8hO38UMLWA1WiQst+T33WO3xIR+1Qre9vi9lZxsKNYfwYBs1JKW6SUNgMeBq5JKW2VnW4K7Jkt+zfgpJTSFjm20ws4COgJHBQRXbK/lH5P5i/t3sAE4JSIaAvsC2yaUtocOD+7jbOAH2a3v1ddVLZAjk4p9QH6Aidn698ceD5b1zHAz7JlrwSuTCn1BJZkJyJiN2ADYGsybd0nIgZmF28AXJdS2jSl1NBf8Vjb9iPTHlsAuwCXZjsjdwJDACKiMbAz8CBwDDA/pbQVsBXws4hYrwBx17Vc5/XybELmPD0kO701sD+wOXBgRCy+m/G7jrVl9hcRjYCrgQOyx/9NwAW1UrsGJKV0HDAL2JHMef1kSmnr7PSlEdEcmAPsmv0ePAi4qtomegO/TCltWL+RrzK+6xy/ETgSICJaAtuSOcdVJOwo1p9XgV0j4pKIGJBSmg/sGBEvRMSrwE7AppG5DqlVSmlxpm/EUtt5IqU0P6X0NZm/jNcF+pH5BfRsRLwMHJGdPx/4GrgxIvYDvsxu41ng5oj4GVBaR/UthJMjk4V9nkzGbwNgAfBAdvlEoFv25/7A3dmf/1FtG7tlPy8Bk4CNs9sBmJ5Sqv/nITQM2wO3p5QqU0ofAU+T6QA+ROY4bgIMBsaklL4i04Y/yR6PLwBt+bYdVye5zuvlGZVtn8UeSyl9nJ03kkw7w3cfa7n2txGwGfBYtr1/D3T+PpVaBewGnJ6t71PAGkBXoBHwl+x36t1kvhcXezGl9G49x7kqyXmOp5SeBjaIiPbAIcC9Dt1nRAH+KwSvUawnKaW3IqI3sDtwfkQ8QWZYuW9KaUZEDCPzZbci31T7uZLMv2GQ+YVzyNKFI2JrMlmeA4ATgZ1SSsdFxDbAHsDEiOiTUvr4e1Sv4CJiBzJ/BfdPKX2ZHe5bA1iYvn1Y6OL2Wu6mgItSStcvtf1uwBe1GPJqIaX0dbatf0gmg3NHdlGQyYo/UqjY6sN3nNeL+PaP8KXP6aWPoaUfZJu+o9zy9ncfMCWl1P9/rMaqKID9U0pv1piZ+R79iExWrITMH8qLef7+724BDgcOBo4qcCyqZ2YU60lEdAS+TCndClxKZhgEYG5kric8ACClNA+YFxGLMwuHrcTmnwe2i4j/y+6reURsmN1uy5TSaODXZL48iYj1U0ovpJTOAipYPa63awl8mu0kbkwmy7o8z5MZ8oPMl99ijwBHx7fXeHaKiA61Hu2qZyyZSx1Ks5mFgcCL2WV3kvnlMYBvh14fAY7PDouSPR6b13PMde47zuv3gD7ZIvt/x6qL7RoRbSKiKbAPmWx/vvt7E2gfEf2zZRpFxKb/W41WGY8AJ0Usua57y+z8lsDs7I1mP2b1GjGpa8s7x28GfgWQUvIaz8WiAJ8CMKNYf3qSueajClgIHE/mF8NrwIfA+GpljwJuiogEPLqiDaeUKiLiSOD27BAgZIaf/gvcHxFrkDnETskuuzQiNsjOewKY/P2q1iA8DBwXEa+T+cW5oiHiXwG3RsSZ2XXnA6SUHo2IHsC47O+gz8n8JV1ZR3GvKu4jM1w/mUzWa2hK6cPsskfJXCJxf0ppQXbeX8kM80/K/jKvIHO8r25ynddNyVzucR6ZYdHleRG4l8xQ8a0ppQnZ7PVK7y+ltCAiDgCuyl5DVgZcAUz5n2vV8J1Hpo6vREQJ8C6Za7yvA+6NiJ+QOa/NIq6cxHLO8ZTSR9nv1n8WLEIVjK/wU1GKiGbAVymlFBEHA4eklLyTT/Um+8dd35TSiYWORcUre9PfpJTSussp04zM9bG9V+I63KLQq3ef9PiYF+p9v+3XalTvr/Azo6hi1Qe4JpvtmgccXdhwJKl+ZS9leAq4bDlldiFz5/Of7CQWJzuKKkoppbFkr9mUCiGldDOZa7+kgkgpzQKW+7iglNLjZJ6ioSJlR1GSJClPvplFkiRJRc2MoiRJUl4K9wDs+mZGUVKti4jKiHg5+97Yu7N3Tf6v27o5+/gXIuKvEbHJcsruENn3++a5jyXvZ16Z+UuV+Xx5y3OUHxYRp+YboyQVgh1FSXXhq5RSr+z7iBcAx1VfGBH/02hGSumnK3jg7w5k3kUrSXUmyFyjWN+fQrCjKKmujQX+L5vtGxsRo4Cp2TdAXBoR4yPilYg4FiAyromINyPicWDJm3Ei4qmI6Jv9eVBETIqIyRHxRPZB1ccBv85mMwdERPuIuDe7j/ERsV123bYR8WhETImIv7IS7zyIiH9GxMTsOj9fatmfsvOfyL7VgohYPyIezq4zNvvGIElapXiNoqQ6k80cDubbV/v1BjZLKb2b7WzNTyltlX2j0LMR8SiwJbARsAmwNjAVuGmp7bYH/gIMzG6rTUrpk4j4M/B5SumybLl/kHn+2zMR0ZXMq996AGcDz6SUzo2IPYBjVqI6R2f30RQYHxH3Zt+R3hyYkFL6dUScld32icANwHEppf9E5t3q1wE7/Q/NKEkFY0dRUl1oGhEvZ38eS+aBvdsCL6aU3s3O3w3YfPH1h2Te07sBmXfM3p5SqgRmRcSTObbfDxizeFsppU++I45dgE3i2zGbFpF5j/dAYL/sug9GxKcrUaeTI2Lf7M9dsrF+DFSRed81wK3AyOw+tgXurrbvJkjSKsaOoqS68FVKqVf1GdkOU/V37wZwUkrpkaXK7V6LcZQA/VJKX+eIZaVFxA5kOp39U0pfRsRTwBrfUTxl9ztv6TaQpFWN1yhKKpRHgOMjohFARGwYEc2BMcBB2WsYy4Edc6z7PDAwItbLrtsmO/+/wFrVyj0KnLR4IiJ6ZX8cAxyanTcYaL2CWFsCn2Y7iRuTyWguVgIszooeSmZI+zPg3Yg4MLuPiAjfBCStRryZRZLq1l/JXH84KSJeA64nM8pxH/Cf7LJbgHFLr5hSqgB+TmaYdzLfDv3+C9h38c0swMlA3+zNMlP59u7rc8h0NKeQGYJ+fwWxPgyURcTrwMVkOqqLfQFsna3DTsC52fmHAcdk45sC7L0SbSJJDUqklAodgyRJ0ipjy95901PPvljv+23VrHRiSqlvfe7TjKIkSZJysqMoSZKknLzrWZIkKR8FvLmkvplRlCRJUk5mFCVJkvIQrMR7P1cTZhQlSZKUkxlFSZKkfBVJStGMoiRJknKyoyhJkqScHHqWJEnKUxTJ2LMZRUmSJOVkRlGSJClPPnBbkiRJRc2MoiRJUp6KJKFoRlGSJEm52VGUJElSTg49S5Ik5atIxp7NKEqSJCknM4qSJEl58oHbkiRJWmVExKCIeDMi3o6I03MsbxIRd2aXvxAR3Va0TTuKkiRJeQgyD9yu789yY4ooBa4FBgObAIdExCZLFTsG+DSl9H/An4BLVlRXO4qSJEmrvq2Bt1NK01JKC4A7gL2XKrM38Pfsz/cAO0csvwtqR1GSJGnV1wmYUW36g+y8nGVSSouA+UDb5W3Um1kkSZLyMGnSxEeaNop2Bdj1GhExodr0DSmlG+pyh3YUJUmS8pBSGlToGHKYCXSpNt05Oy9XmQ8iogxoCXy8vI069CxJkrTqGw9sEBHrRURj4GBg1FJlRgFHZH8+AHgypZSWt1EzipIkSau4lNKiiDgReAQoBW5KKU2JiHOBCSmlUcCNwIiIeBv4hExncrliBR1JSZIkFSmHniVJkpSTHUVJkiTlZEdRkiRJOdlRlCRJUk52FCVJkpSTHUVJkiTlZEdRkiRJOdlRlCRJUk7/H+avYB71Vf2SAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 792x792 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_preds = np.argmax(output_predictions.predictions, axis=1)\n",
    "\n",
    "names_of_labels = [((x+1)*1.0) for x in range(0, 6)]\n",
    "names_of_labels = ['sadness', 'anger', 'love', 'surprise', 'fear', 'joy'] \n",
    "\n",
    "plot_confusion_matrix(y_preds, y_valid, labels=names_of_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push Fine-Tuned Model to HuggingFace Hub (My Profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71239e1d60c942bdbc8be61eef0d9a52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Oct01_19-24-58_Brians-Mac-mini/events.out.tfevents.1664670305.Brians-Mac-mini.1184.0: 100%|##…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remote: Scanning LFS files for validity, may be slow...        \n",
      "remote: LFS file scan complete.        \n",
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-Emotions_Detection\n",
      "   62d1266..8b1d057  main -> main\n",
      "\n",
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-Emotions_Detection\n",
      "   8b1d057..cae6215  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/distilbert-base-uncased-Emotions_Detection/commit/8b1d05712067a8fc818a6b03b4d61f5bee415f8f'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub(commit_message=\"All Dunn!!!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('nlp_ch4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a52fe47989fdc78fafbb981021cec52a6b82df6453830b9ffbd04250493e6cab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}