Sentiment Analysis/Financial Sentiment Analysis/Financial Sentiment Analysis-Original Version.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentiment Analysis of Statements about Finances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saved as 'distilbert-base-uncased-Finanacial_Sentiment_Analysis' on my HuggingFace profile\n",
    "\n",
    "Source for the Datasets: \n",
    "\n",
    "*https://www.kaggle.com/datasets/sbhatti/financial-sentiment-analysis\n",
    "\n",
    "*https://www.kaggle.com/datasets/ankurzing/sentiment-analysis-for-financial-news"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploratory Data Analysis & Preprocessing in Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Importing the Necessary Libraries for the Entire Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "NOTE: Redirects are currently not supported in Windows or MacOs.\n"
     ]
    }
   ],
   "source": [
    "import os, sys\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import AdamW\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset, DatasetDict, load_metric\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoModelForSequenceClassification, AutoConfig, AutoTokenizer, AutoModel\n",
    "from transformers import TrainingArguments, Trainer, AdamW, get_scheduler, set_seed, DataCollatorWithPadding\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score, f1_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.dummy import DummyClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Versions of Libraries Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pandas:            1.4.3\n",
      "NumPy:             1.22.4\n",
      "Torch:             1.12.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Pandas:\".ljust(18), pd.__version__)\n",
    "print(\"NumPy:\".ljust(18), np.__version__)\n",
    "print(\"Torch:\".ljust(18), torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest First Dataset & Preprocessing it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 5836 entries, 0 to 5841\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    5836 non-null   object\n",
      " 1   label   5836 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 136.8+ KB\n"
     ]
    }
   ],
   "source": [
    "data1 = pd.read_csv('~/Documents/nlpnn/Datasets/Finanacial Sentiment Analysis/data.csv', engine=\"c\")\n",
    "\n",
    "data1.dropna(inplace=True)\n",
    "\n",
    "data1['Sentence'] = data1['Sentence'].str.strip()\n",
    "data1['Sentiment'] = data1['Sentiment'].str.strip()\n",
    "\n",
    "data1.rename(columns={'Sentence' : 'text', 'Sentiment' : 'label'}, inplace=True)\n",
    "data1.drop_duplicates(inplace=True)\n",
    "data1.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest Second Dataset & Preprocessing it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 4840 entries, 0 to 4845\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   label   4840 non-null   object\n",
      " 1   text    4840 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 113.4+ KB\n"
     ]
    }
   ],
   "source": [
    "data2 = pd.read_csv('~/Documents/nlpnn/Datasets/Finanacial Sentiment Analysis/all-data.csv', \n",
    "                    encoding_errors='ignore', \n",
    "                    engine=\"c\", \n",
    "                    header=None, \n",
    "                    names=['label', 'text'])\n",
    "\n",
    "data2['label'] = data2['label'].str.strip()\n",
    "data2['text'] = data2['text'].str.strip()\n",
    "\n",
    "data2.drop_duplicates(inplace=True)\n",
    "data2.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Concatenate Both Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10676 entries, 0 to 10675\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    10676 non-null  object\n",
      " 1   label   10676 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 166.9+ KB\n"
     ]
    }
   ],
   "source": [
    "data = pd.concat([data1, data2]).reset_index(drop=True)\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Histogram of Text Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEWCAYAAACdaNcBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ4klEQVR4nO3dfZRcVZnv8e/PAOElQAiJmZhEmpcIA4yAZkEABzPgKG8jzCxFcUYCkzU4I86FK16B8d65OOqd5C4Bucs7IAIKwkAwyoDRETDAeHmnw3sISJRgEgMJkPAiyhh87h97l54U1V2V7uqu6t2/z1pn9Tl7n9rnqVPdT+3a59RuRQRmZlaWt3Q6ADMzaz8ndzOzAjm5m5kVyMndzKxATu5mZgVycjczK5CTe8EkLZU0u9NxdJKkP5e0UtKrkg7odDwGkmZLWtXpOErn5D5CSVoh6X11ZSdLuqO2HRH7RMTtTdrpkRSSthiiUDvty8CnImJcRDxYXynpOEkPSXpZ0vOSbpW062APKulcSVcNtp12y6/1HqUf06DUP2jrEpK2iIiNHQxhF2Bpo4qccK4E/gK4FRgHvB94Y9iiMxsi7rkXrNq7l3SgpN7cQ31O0vl5tx/nnxvy0MXBkt4i6b9LekbSWklXStqx0u5Jue4FSf+j7jjnSloo6SpJLwMn52PfLWmDpDWSvippq0p7IemTkp6S9IqkL0jaXdJdOd7rqvvXPceGsUoaK+lVYAzwsKSfNnj4/sDTEbE4klci4jsR8fNK22dL+ml+rtdJmpDrap945kj6ee71fy7XHQn8A/CRfE4fzuU7Srosn4PVkr4oaUyuO1nSHZK+LGm9pKclHVV5nhMkfUPSL3L9v1Xqjs2fPjbkc/bOFn9FqudxbD72z/Pvx8WStsl1syWtknRmPsdrJJ1SeezOkr6XX6v78/O6I9fVfr8ezufiI5XHNWzP2iQivIzABVgBvK+u7GTgjkb7AHcDH8/r44BZeb0HCGCLyuP+GlgO7Jb3/S7wrVy3N/Aq8B5gK9Kwx28qxzk3bx9P6jxsA7wbmEX6pNgDLAPOqBwvgBuAHYB9gNeBxfn4OwKPA3P6OA99xlppe48+Hrsb8GvgAuBPgHF19acD9wDTgLHA14Br6s7b1/Nz3C/H/YeV83BVXXvX5za2A94K3Ad8ovLa/Qb4G9Ib0t8BvwCU678PLAB2ArYE3pvLDwDWAgflx83Jr/vYPp5zw/ORz8GNwARge+B7wD/nutnARuCf8rGPBl4Ddsr11+Zl2/z7sZJNfw83OWaz9ry0KUd0OgAvA3zh0h/wq8CGyvIafSf3HwOfBybWtVNLUtXkvhj4ZGV7z5x4tgD+sZbgct22wH+yaXL/cZPYzwCur2wHcGhlewlwVmX7POArfbTVZ6yVthsm91w/C7gOWEdK9N8kJ3nSm9ARlX2nVM5D7bxNq9TfB3y0ch6uqtRNJiX/bSplJwK35fWTgeV15zWAP8jH/W2j5AdcBHyhruxJcvJvsP+bzgcg4JfA7pWyg0mfaiAl41/V/Y6szeduTD4ne1bqvkjz5N6wvU7/XZW0eFhmZDs+IsbXFuCT/ew7F3gH8ET+6HxsP/u+DXimsv0MKaFNznUraxUR8RrwQt3jV1Y3JL1D0iJJz+ahmv8FTKx7zHOV9V812B43gFibioh7IuKEiJgE/DFwGPC5XL0LcH0e7thASvZv1LX9bGX9tX7i3IXUS11Tae9rpB78m9rK55Xc3nTgxYhY30e7Z9bazO1OJ52XVk0ivZksqbTxw1xe80Jseu2k9lwnkc539TXf5PXvQ1/tWZs4uY8SEfFURJxISibzgYWStiP1qur9gpQ0at5O+hj9HLCGNEwBQB6X3bn+cHXbFwFPADMiYgfSeLQG/mxajnWzRMT9pGGdfXPRSuCo6htoRGwdEatbaa5ueyWp5z6x0tYOEbFPC22tBCZIGt9H3ZfqYtw2Iq5pod2a50lvoPtU2tgxIlpJtutI53tapWz6ZhzbhoiT+ygh6a8kTYqI35KGcCB91F+Xf+5W2f0a4L9K2lXSOFJPe0HuaS0E/kzSIfki57k0T9TbAy8Dr0raizSe3C79xdovSe+R9DeS3pq39wI+SBpnB7gY+JKkXXL9JEnHtRjXc0CPpLcARMQa4GbgPEk75Iu1u0t6b7OG8mP/HfgXSTtJ2lLSYbn668DfSjpIyXaSjpG0fT9NbiVp69pCev2+DlxQORdTJX2ghdjeIL0hnitp23wOT2pwLnZ704NtSDm5jx5HAkuV7iC5kDQ2/Kv88f9LwJ35I/ks4HLgW6Rx+qdJY9F/DxARS/P6taRe/Kuk8dLX+zn2Z4CPAa+QksiCNj6vPmNtwQZSMn80n5cfki56/u9cfyHpIuPNkl4hJf2DWmz72/nnC5IeyOsnkS5CPw6sJ71RTmmxvY+TxrafIJ3vMwAiopd0Efaruc3lpPH7/iwl9dRryynAWfmx9+Shsx+Rrl+04lOkC9/Pkl6La9j09+Fc4Ir8+3VCi23aINWuxJsNSO4tbyANuTzd4XCsC0iaD/xBRMzpdCyjmXvuttkk/Vn+CL4d6VbIR0l35tgoJGkvSe/Mw0IHki7eX9/puEY7J3cbiONIFzJ/AcwgDfH4I+DotT1p3P2XpCG380jfW7AO8rCMmVmB3HM3MytQV0wcNnHixOjp6el0GGZmI8qSJUuez1/Ae5OuSO49PT309vZ2OgwzsxFF0jN91XlYxsysQE7uZmYFcnI3MyuQk7uZWYGc3M3MCuTkbmZWICd3M7MCObmbmRXIyd3MrEBd8Q1Va6+es7+/yfaKecd0KBIz6xT33M3MCuSeewHqe+pmZu65m5kVyMndzKxATu5mZgVycjczK5CTu5lZgXy3zCjk++DNyueeu5lZgZzczcwK5GEZa8hDN2Yjm3vuZmYFcnI3MyuQh2VGgMHOHeO5Z8xGH/fczcwK5ORuZlYgJ3czswI5uZuZFcgXVM0XXM0K5J67mVmBnNzNzArk5G5mVqCWk7ukMZIelLQob+8q6V5JyyUtkLRVLh+bt5fn+p4hit3MzPqwOT3304Flle35wAURsQewHpiby+cC63P5BXk/MzMbRi0ld0nTgGOAS/O2gMOBhXmXK4Dj8/pxeZtcf0Te38zMhkmrPfevAJ8Ffpu3dwY2RMTGvL0KmJrXpwIrAXL9S3n/TUg6VVKvpN5169YNLHozM2uoaXKXdCywNiKWtPPAEXFJRMyMiJmTJk1qZ9NmZqNeK19iOhT4oKSjga2BHYALgfGStsi982nA6rz/amA6sErSFsCOwAttj9zMzPrUNLlHxDnAOQCSZgOfiYi/lPRt4EPAtcAc4Ib8kBvz9t25/taIiLZHXjB/Y9TMBmsw97mfBXxa0nLSmPplufwyYOdc/mng7MGFaGZmm2uz5paJiNuB2/P6z4ADG+zza+DDbYjNzMwGyN9QNTMrkJO7mVmBnNzNzArk+dy7wEi4O6Y+xhXzjulQJGbWCvfczcwK5ORuZlYgJ3czswJ5zH0YlDheXeJzMiuJe+5mZgVycjczK5CTu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYF8jdUO2AkzAJpZiObe+5mZgVyz93awnPNmHUX99zNzArk5G5mViAndzOzAnnM3YaEx+DNOss9dzOzAjm5m5kVyMndzKxATu5mZgVycjczK5CTu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYFaprcJW0t6T5JD0taKunzuXxXSfdKWi5pgaStcvnYvL081/cM8XMwM7M6rfTcXwcOj4j9gP2BIyXNAuYDF0TEHsB6YG7efy6wPpdfkPczM7Nh1DS5R/Jq3twyLwEcDizM5VcAx+f14/I2uf4ISWpXwGZm1lxLY+6Sxkh6CFgL3AL8FNgQERvzLquAqXl9KrASINe/BOzcoM1TJfVK6l23bt2gnoSZmW2qpX/WERFvAPtLGg9cD+w12ANHxCXAJQAzZ86MwbbXTer/UYWZ2XDbrLtlImIDcBtwMDBeUu3NYRqwOq+vBqYD5PodgRfaEayZmbWmlbtlJuUeO5K2Af4UWEZK8h/Ku80BbsjrN+Ztcv2tEVFUz9zMrNu1MiwzBbhC0hjSm8F1EbFI0uPAtZK+CDwIXJb3vwz4lqTlwIvAR4cgbjMz60fT5B4RjwAHNCj/GXBgg/JfAx9uS3RmZjYg/oaqmVmBnNzNzArU0q2QZoNVf3voinnHdCgSs9HByd06wsnebGh5WMbMrEDuuVtXcE/erL3cczczK5CTu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYFcnI3MyuQk7uZWYGc3M3MCuTkbmZWICd3M7MCObmbmRXIU/62Qf10tWZmneaeu5lZgZzczcwK5ORuZlYgJ3czswL5gqp1pWYXqf0/Vs365567mVmBnNzNzArk5G5mViAndzOzAjm5m5kVyHfL2IhUfzeN754x25R77mZmBXJyNzMrkJO7mVmBnNzNzArUNLlLmi7pNkmPS1oq6fRcPkHSLZKeyj93yuWS9H8kLZf0iKR3DfWTMDOzTbXSc98InBkRewOzgNMk7Q2cDSyOiBnA4rwNcBQwIy+nAhe1PWozM+tX01shI2INsCavvyJpGTAVOA6YnXe7ArgdOCuXXxkRAdwjabykKbkdsyHhWyPNNrVZY+6SeoADgHuByZWE/SwwOa9PBVZWHrYql5mZ2TBpOblLGgd8BzgjIl6u1uVeemzOgSWdKqlXUu+6des256FmZtZES8ld0pakxH51RHw3Fz8naUqunwKszeWrgemVh0/LZZuIiEsiYmZEzJw0adJA4zczswZauVtGwGXAsog4v1J1IzAnr88BbqiUn5TvmpkFvOTxdjOz4dXK3DKHAh8HHpX0UC77B2AecJ2kucAzwAm57gfA0cBy4DXglHYGbGZmzbVyt8wdgPqoPqLB/gGcNsi4zMxsEPwNVTOzAjm5m5kVyMndzKxATu5mZgVycjczK5CTu5lZgZzczcwK5H+QbaOCZ4200cY9dzOzAjm5m5kVyMndzKxAHnO3Uclj8FY699zNzArk5G5mViAndzOzAjm5m5kVyMndzKxATu5mZgXyrZBWpPpbHc1GG/fczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYFcnI3MyuQ73MfAN9DbWbdzj13M7MCObmbmRXIyd3MrEBO7mZmBXJyNzMrkJO7mVmBfCukWYua3QK7Yt4xwxSJWXNO7ma8OXE7UdtI52EZM7MCObmbmRWo6bCMpMuBY4G1EbFvLpsALAB6gBXACRGxXpKAC4GjgdeAkyPigaEJ3ay7eGjHukkrY+7fBL4KXFkpOxtYHBHzJJ2dt88CjgJm5OUg4KL802zUc/K34dR0WCYifgy8WFd8HHBFXr8COL5SfmUk9wDjJU1pU6xmZtaigY65T46INXn9WWByXp8KrKzstyqXvYmkUyX1Supdt27dAMMwM7NGBn0rZESEpBjA4y4BLgGYOXPmZj/ebCh5Wmcb6Qbac3+uNtySf67N5auB6ZX9puUyMzMbRgNN7jcCc/L6HOCGSvlJSmYBL1WGb8zMbJi0civkNcBsYKKkVcD/BOYB10maCzwDnJB3/wHpNsjlpFshTxmCmM2K4LtnbCg1Te4RcWIfVUc02DeA0wYblJmZDY6/oWpmViAndzOzAjm5m5kVyFP+tsD3PJvZSOOeu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFch3y5gNEd9lZZ3knruZWYGc3M3MCuRhGbMu4VkirZ2c3M1GCCd/2xweljEzK5CTu5lZgZzczcwK5DH3Bnx/spmNdE7uZl3KnQwbDA/LmJkVyD13sxHKt0Zaf5zczQrhZG9VTu5mo4ST/+ji5I4vXJlZeXxB1cysQE7uZmYFcnI3MyuQx9zNCuVrSaObk7vZKOW7Z8o26pK7ezNmNhp4zN3MrEBO7mZmBRp1wzJm1pjH4MvinruZWYGc3M3MClT8sIzvjjGz0WhIkrukI4ELgTHApRExbyiOY2ZDp9kYfKOOU7N9PI4/fNqe3CWNAf4v8KfAKuB+STdGxOPtPpaZDZ9WPgX7k3L3GIqe+4HA8oj4GYCka4HjACd3M+vX5n5a8CeBvg1Fcp8KrKxsrwIOqt9J0qnAqXnzVUlPDvB4E4HnB/jY4dLtMXZ7fOAY26Hj8Wl+0102ibHZ/i20124dP4d1dumromMXVCPiEuCSwbYjqTciZrYhpCHT7TF2e3zgGNuh2+OD7o+x2+OrGopbIVcD0yvb03KZmZkNk6FI7vcDMyTtKmkr4KPAjUNwHDMz60Pbh2UiYqOkTwE3kW6FvDwilrb7OBWDHtoZBt0eY7fHB46xHbo9Puj+GLs9vt9RRHQ6BjMzazNPP2BmViAndzOzAo3o5C7pSElPSlou6exOxwMg6XJJayU9VimbIOkWSU/lnzt1ML7pkm6T9LikpZJO78IYt5Z0n6SHc4yfz+W7Sro3v94L8gX7jpE0RtKDkhZ1aXwrJD0q6SFJvbmsm17n8ZIWSnpC0jJJB3dZfHvmc1dbXpZ0RjfF2J8Rm9wr0xwcBewNnChp785GBcA3gSPrys4GFkfEDGBx3u6UjcCZEbE3MAs4LZ+3borxdeDwiNgP2B84UtIsYD5wQUTsAawH5nYuRABOB5ZVtrstPoA/iYj9K/dmd9PrfCHww4jYC9iPdC67Jr6IeDKfu/2BdwOvAdd3U4z9iogRuQAHAzdVts8Bzul0XDmWHuCxyvaTwJS8PgV4stMxVmK7gTQPUFfGCGwLPED6lvPzwBaNXv8OxDWN9Id9OLAIUDfFl2NYAUysK+uK1xnYEXiafFNHt8XXIN73A3d2c4z1y4jtudN4moOpHYqlmckRsSavPwtM7mQwNZJ6gAOAe+myGPOQx0PAWuAW4KfAhojYmHfp9Ov9FeCzwG/z9s50V3wAAdwsaUme7gO653XeFVgHfCMPbV0qabsuiq/eR4Fr8nq3xriJkZzcR6RIb/cdv/9U0jjgO8AZEfFyta4bYoyINyJ9HJ5Gmoxur07GUyXpWGBtRCzpdCxNvCci3kUaujxN0mHVyg6/zlsA7wIuiogDgF9SN7zRDb+HAPnayQeBb9fXdUuMjYzk5D6Spjl4TtIUgPxzbSeDkbQlKbFfHRHfzcVdFWNNRGwAbiMNc4yXVPviXSdf70OBD0paAVxLGpq5kO6JD4CIWJ1/riWNFR9I97zOq4BVEXFv3l5ISvbdEl/VUcADEfFc3u7GGN9kJCf3kTTNwY3AnLw+hzTO3RGSBFwGLIuI8ytV3RTjJEnj8/o2pGsCy0hJ/kN5t47FGBHnRMS0iOgh/d7dGhF/2S3xAUjaTtL2tXXSmPFjdMnrHBHPAisl7ZmLjiBNC94V8dU5kd8PyUB3xvhmnR70H+RFjqOBn5DGYz/X6XhyTNcAa4DfkHonc0njsYuBp4AfARM6GN97SB8jHwEeysvRXRbjO4EHc4yPAf+Yy3cD7gOWkz4ij+2C13s2sKjb4suxPJyXpbW/jy57nfcHevPr/G/ATt0UX45xO+AFYMdKWVfF2Nfi6QfMzAo0kodlzMysD07uZmYFcnI3MyuQk7uZWYGc3M3MCuTkbkNC0gWSzqhs3yTp0sr2eZI+PcC2Z9dmYhxukiZLWpRnrHxc0g8G0dbJkt7WzvjMapzcbajcCRwCIOktwERgn0r9IcBdrTSUZwDtiMo3Tmv+CbglIvaLNLPmYGYEPBlwcrch4eRuQ+Uu0pQBkJL6Y8ArknaSNBb4Q+ABSUfkiaMeVZoLfyz8bi7y+ZIeAD6sNHf/E3n7L2oHkfTeynzbD9a+lVmp78mPuzrPGb5Q0ra57t2S/iNPrHVT5Svlt0v6Sp4D/fS65zWF9OU0ACLikcqx/puk+yU9ot/PQd+Tj/t1pbnpb5a0jaQPATOBq3Ps2zSJZ77SHPc/kfTHuXyMpC9Leiwf8+/7e142ynT6W1Reyl1IU7q+HfgE8LfAF0jfhj0U+H/A1qSZPd+R97+SNJEZpOlqP5vXa/vNIE2tex2//1bo94BD8/o48pS7lRh6SN/Ire1zOfAZYEvSG9CkXP4R0j9zB7gd+Jc+ntMHgA2kqQY+B7wtl7+f9M+TReo0LQIOy8ffCOyf97sO+KvKcWbm9WbxnJfXjwZ+lNf/jjQnS22a4Qn9teNldC31HznN2uku0vDLIcD5pClwDwFeIg3b7Ak8HRE/yftfAZxGmk4XYEH+uVfe7ykASVcBtSls7wTOl3Q18N2I+F2vumJlRNyZ168C/gvwQ2Bf4JY03Q5jSNNG1CyggYi4SdJupH/IchTwoKR9Scn9/aRpEyC90cwAfp5jfyiXLyEl/Hp7NomnNsFb9fHvAy6OPM1wRLyYY+mvHRslnNxtKNXG3f+INCyzEjgTeBn4RguP/2WzHSJinqTvk3q0d0r6QEQ8Ub9bg20BSyPiYBrr89gR8SLwr8C/5gu7h+X2/jkivlbdV2nO/NcrRW8A2zRotlk8tTbeoP+/22bt2CjhMXcbSncBxwIvRpqf/UVgPGks/i7Sf7TpkbRH3v/jwH80aOeJvN/uefvEWoWk3SPi0YiYT5optNG872+XVEt2HwPuyMeeVCuXtKWkfRo8dhOSDq+M2W8P7E7qnd8E/LXSPPlImirprU2aewWoXSMYSDy3AJ+oXfSVNGGgz8vK4+RuQ+lR0l0y99SVvRQRz0fEr4FTgG9LepT0X40urm8k73cq8P18QbU6f/YZtQuKpJk4/71BHE+S/lnFMtLMgxdFxH+SpuedL+lh0uyYh7TwnN4N9Obj3Q1cGhH3R8TNpN783fm5LOT3ibsv3wQuVvqPU2MGEM+lpDeWR/JjPjaI52WF8ayQVrQ8LLIoIvbtdCxmw8k9dzOzArnnbmZWIPfczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQP8fbVVsCL4ABSkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['Sentence Length'] = data['text'].str.split().apply(len)\n",
    "n=75\n",
    "data.hist('Sentence Length', grid=False, bins=n, range=(0,n))\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Histogram of Sentence Length')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('Words per Sentence')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Visualize: Description Lengths (By Output Class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAEHCAYAAAAu3HtHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAc6klEQVR4nO3debhkVXnv8e9PEFBQoKElDDatghr0Crm2KGiiwUTNDYpRZBAQlQS5GhPFIeqNiYnG4KOJ8WqUYCBgHAA1PAJOINDGAYeGKIOIEgQBGVqgFRSRhvf+sde5FM2pc+p0d1V1nfP9PM9+au+1h/VW1a4+b6+99tqpKiRJkjQ5HjDuACRJkjQ3JnCSJEkTxgROkiRpwpjASZIkTRgTOEmSpAljAidJkjRhTOAkDV2SQ5KctQ77H5vkreszpg1JkuVJ/njAbZ+R5NphxyRpw2YCJ81jSa5KckeS25KsSvL1JEclGelvv6o+VlXPGmTbJC9N8tU19j+qqt6+vuNK8rYkH13fx93Q6uwTx+eT3N6mu5L8umf52HHHJ2lmG487AElD99yq+lKSLYGnA+8Dngy8bBSVJ9m4qlaPoi4Nrqr+YGo+yYnAtVX1l+OLSNJc2AInLRBV9bOqOh04EDg8yeMBkmya5D1Jfpzkxna58kFt3bZJzmytd7ck+cpU612Shyf5jyQrk9yc5AOt/KVJvpbkvUluBt62ZqtakkryZ0muTPLTJO9O8oAkvwkcC+zVWoJWte1PTPKOnv3/JMkVLabTk+ywxrGPSvLDFvc/J8lcP68kT2ktlquSfDfJM3rWLU/y9vY+b0tyVpJte9a/JMnV7XN5a2sJ/b0kzwHeAhzY3t93e6rcud/x+sT3lvbZXZXkkFb2pPYdbtSz3QvWqGe29/3ZJK9eo+yiJH/U5qf97nq2fXmSy5LcmuSLSXYetG5JgzOBkxaYqvoWcC3w263oGODRwB7ALsCOwF+1da9r2y4GtqNLPqolCGcCVwNL2z4n91TzZODKts/f9Qnlj4BlwP8E9gNeXlWXAUcB51fVFlW11Zo7JdkH+HvgAGD7FsPJa2y2L/Ak4Altu2f3/UCmkWRH4LPAO4BFwOuBTydZ3LPZi+laMR8GbNK2IcluwAeBQ1p8W9J9PlTVF4B3Aqe097f7bMfr4zeAbdtxDweOS/KYqvo2cDPQe7n6MOAjc3j7JwGHTi0k2b3V89mebe733bVt96M7R15Ad858BfjEHOqWNCATOGlh+gmwqLVMHQm8tqpuqarb6BKMg9p2d9ElITtX1V1V9ZXqHqC8J7AD8Iaq+kVV/aqqevut/aSq3l9Vq6vqjj4xvKvV+WPgn4CDB4z9EOCEqrqwqu4E3kzXYre0Z5tjqmpVO/Z5dMnpXBwKfK6qPldV91TV2cAK4H/1bPNvVfWD9v5O7aljf+CMqvpqVf2aLhke5KHT/Y7Xz1ur6s6q+jJdcnVAK///CViSRXTJ68cHqH/K6cCjk+zalg+jSzh/3bNNv+/uKODvq+qydtn8ncAetsJJ658JnLQw7QjcQtdK8mDggnapcBXwhVYO8G7gCuCsdsnsTa384cDVM/Rtu2aAGHq3uZouIRzEDm17AKrqdrpWpx17trmhZ/6XwBYDHnvKzsCLpj6T9rk8jS6Zna2OHeh5b1X1yxbfbOYS861V9Yue5d7P76PAc5NsTpfUfaWqrh+g/ql4fwWcAhzaLo0eDPz7Gpv1++52Bt7X85ndAoT7fjeS1gNvYpAWmCRPovuD+lXgp8AdwOOq6ro1t20tcq8DXtf6zJ2b5Nt0f8CXzHCDwiAtTg8HLm3zS+haBQfZ9yd0icLU+9kc2Aa4X/zr4Brg36vqT9Zi3+uBx0wttP6E2/SsH+Szmc3WSTbvSeKWAJcAVNV1Sc6nu4x5GPChtTj+SXRJ21eBX1bV+Wus7/fdXQP8XVV9bC3qlDQHtsBJC0SShybZl66/2Eer6uKqugf4MPDeJA9r2+2Y5Nltft8ku7RLrT8D7gbuAb5Fl6gck2TzJJsleeocQ3pDkq2TPBz4c7pWH4AbgZ2SbNJnv08AL0uyR5JN6S7TfbOqrppj/VMe0OKfmjbl3lasZyfZqJU/I8lOAxzvU23fvdt7eBtdK9SUG4GlWfehXP4mySZJfpuuz98ne9Z9BHgj8D+A/5jrgVvCdg/wD9y/9Q36f3fHAm9O8jiAJFsmedFc65c0OxM4af47I8ltdK0j/wf4R+47hMhf0F0m/UaSnwNf4t4WpF3b8u3A+cAHq+q8qrobeC7dTQ8/prvR4cA5xvUZ4ALgO3R9uI5v5efSte7ckOSna+5UVV8C3gp8mi6JfBT39tlbGwfTtUJOTf9dVdfQdc5/C7CS7rN7AwP8m1lVlwKvpkuUr6f77G4C7mybTCVaNye5cC1jvgG4la7l62PAUVX1/Z71p9G1Up7WLuGujY/QJYDTjVk37XdXVacB7wJObufSJcAfTLO/pHWUrj+yJI1OkgJ2raorxh3LsCXZAlhF935/NMJ6/xt4RUt412b/lwBHVtXT1ihfMN+dtCGzBU6S1rMkz03y4NY/7z3AxcBVI6z/hXR97c5dy/0fDLwSOG59xiVp/TGBk6T1bz+6y5s/obsMfVCN6HJHkuV0Ny68qvVxnOv+z6a7bHwjcxt+RNIIeQlVkiRpwtgCJ0mSNGFM4CRJkibMRAzku+2229bSpUvHHYYkSdLIXHDBBT+tqsXTrZuIBG7p0qWsWLFi3GFIkiSNTJKr+63zEqokSdKEMYGTJEmaMCZwkiRJE8YETpIkacKYwEmSJE0YEzhJkqQJYwInSZI0YUzgJEmSJsxQB/JNchVwG3A3sLqqliVZBJwCLAWuAg6oqluHGYckSQtVMp56q8ZT70Ixiha4362qPapqWVt+E3BOVe0KnNOWJUnSEFSt/bQu+2u4xnEJdT/gpDZ/EvD8McQgSZI0sYadwBVwVpILkhzZyrarquvb/A3AdkOOQZIkaV4Z9sPsn1ZV1yV5GHB2ku/3rqyqSjJtQ2tL+I4EWLJkyZDDlCRJmhxDbYGrquva603AacCewI1Jtgdorzf12fe4qlpWVcsWL148zDAlSZImytASuCSbJ3nI1DzwLOAS4HTg8LbZ4cBnhhWDJEnSfDTMS6jbAaelu395Y+DjVfWFJN8GTk1yBHA1cMAQY5AkSZp3hpbAVdWVwO7TlN8MPHNY9UqSJM13PolBkiRpwpjASZIkTRgTOEmSpAkz7HHgtIHwWXiSJM0fJnALxLokUomJmCRJGxIvoUqSJE0YEzhJkqQJ4yVUSfOefUAlzTcmcJLmPfuASppvvIQqSZI0YUzgJEmSJowJnCRJ0oQxgZMkSZowJnCSJEkTxgROkiRpwpjASZIkTRgTOEmSpAljAidJkjRhTOAkSZImjAmcJEnShDGBkyRJmjAmcJIkSRPGBE6SJGnCmMBJkiRNGBM4SZKkCWMCJ0mSNGFM4CRJkiaMCZwkSdKEMYGTJEmaMCZwkiRJE8YETpIkacIMPYFLslGS/0pyZlt+RJJvJrkiySlJNhl2DJIkSfPJKFrg/hy4rGf5XcB7q2oX4FbgiBHEIEmSNG8MNYFLshPwh8C/tuUA+wCfapucBDx/mDFIkiTNN8Nugfsn4I3APW15G2BVVa1uy9cCOw45BkmSpHllaAlckn2Bm6rqgrXc/8gkK5KsWLly5XqOTpIkaXINswXuqcDzklwFnEx36fR9wFZJNm7b7ARcN93OVXVcVS2rqmWLFy8eYpiSJEmTZdYELslTBylbU1W9uap2qqqlwEHAuVV1CHAesH/b7HDgM3OKWJIkaYEbpAXu/QOWDeovgKOTXEHXJ+74dTiWJEnSgrNxvxVJ9gL2BhYnObpn1UOBjeZSSVUtB5a3+SuBPecaqCRJkjp9EzhgE2CLts1Desp/zr2XQCVJkjRifRO4qvoy8OUkJ1bV1SOMSZIkSTOYqQVuyqZJjgOW9m5fVfsMKyhJkiT1N0gC90ngWLqnKdw93HAkSZI0m0ESuNVV9aGhRyJJkqSBDDKMyBlJXplk+ySLpqahRyZJkqRpDdICd3h7fUNPWQGPXP/hSJIkaTazJnBV9YhRBCJJkqTBzDSQ7z5VdW6SF0y3vqr+Y3hhSZIkqZ+ZWuCeDpwLPHeadQWYwEmSJI3BTAP5/nV7fdnowpEkSdJsBrmJgSR/CDwO2GyqrKr+dlhBSZIkqb9ZhxFJcixwIPBqIMCLgJ2HHJckSZL6GGQcuL2r6iXArVX1N8BewKOHG5YkSZL6GSSBu6O9/jLJDsBdwPbDC0mSJEkzGaQP3JlJtgLeDVxIdwfqvw4zKEmSJPU3yEC+b2+zn05yJrBZVf1suGFJkiSpn5kG8p12AN+2zoF8JUmSxmSmFrjpBvCd4kC+kiRJYzLTQL4O4CtJkrQBGmQcuG2S/N8kFya5IMn7kmwziuAkSZJ0f4MMI3IysBJ4IbB/mz9lmEFJkiSpv0GGEdm+505UgHckOXBYAUmSJGlmg7TAnZXkoCQPaNMBwBeHHZgkSZKmN0gC9yfAx4Fft+lk4BVJbkvy82EGJ0mSpPsbZCDfh4wiEEmSJA1mkD5wJHkCsLR3ewfylSRJGo9ZE7gkJwBPAC4F7mnFDuQrSZI0JoO0wD2lqnYbeiSSJEkayCA3MZyfxAROkiRpAzFIC9xH6JK4G4A7gQBVVU8YamSSJEma1iAJ3PHAYcDF3NsHblZJNgP+E9i01fOpqvrrJI+gG4pkG+AC4LCq+vVcA5ckSVqoBrmEurKqTq+qH1XV1VPTAPvdCexTVbsDewDPSfIU4F3Ae6tqF+BW4Ii1DV6SJGkhGqQF7r+SfBw4gy4pA2YfRqSqCri9LT6wTQXsA7y4lZ8EvA340JyiliRJWsAGSeAeRJe4PaunbKBhRJJsRHeZdBfgn4H/BlZV1eq2ybXAjnMJWJIkaaEb5EkML1vbg1fV3cAeSbYCTgMeO+i+SY4EjgRYsmTJ2oYgSZI078zaBy7JTklOS3JTmz6dZKe5VFJVq4DzgL2ArZJMJY47Adf12ee4qlpWVcsWL148l+okSZLmtUFuYvg34HRghzad0cpmlGRxa3kjyYOA3wcuo0vk9m+bHQ58Zs5RS5IkLWCDJHCLq+rfqmp1m04EBmkS2x44L8lFwLeBs6vqTOAvgKOTXEE3lMjxaxm7JEnSgjTITQw3JzkU+ERbPhi4ebadquoi4LemKb8S2HMuQUqStNAtWgS33jr6epPR1rf11nDLLaOtcxINksC9HHg/8F66u0+/Dqz1jQ2SJGnubr0VqsYdxfCNOmGcVIPchXo18LwRxCJJkqQB9O0Dl+TdSV4xTfkrkhwz3LAkSZLUz0w3MewDHDdN+YeBfYcTjiT1t2hRd3lllBOMvs5Fi8b7OUva8M10CXXT9jis+6iqexKvUEsaPfsASVJnpha4O5LsumZhK7tjeCFJkiRpJjO1wP0V8Pkk76B7ninAMuDNwGuGHJckSZL66JvAVdXnkzwfeAPw6lZ8CfDCqrp4BLFpGo4DJEmSZhxGpKouoXvclTYQ9gGSJEmDPEpLkiRJGxATOEmSpAkzYwKXZKMkrx1VMJIkSZrdjAlcVd1N9/B6SZIkbSAGeZj915J8ADgF+MVUYVVdOLSoJEmS1NcgCdwe7fVve8qK7lFbkiRJGrFZE7iq+t1RBCJJkqTBzHoXapLtkhyf5PNtebckRww/NEmSJE1nkGFETgS+COzQln+Aj9KSJEkam0ESuG2r6lTgHoCqWg3cPdSoJEmS1NcgCdwvkmxDd+MCSZ4C/GyoUUmSJKmvQe5CPRo4HXhUkq8Bi4H9hxqVJEmS+hrkLtQLkzwdeAwQ4PKqumvokUmSJGlasyZwSTYDXgk8je4y6leSHFtVvxp2cJIkSbq/QS6hfgS4DXh/W34x8O/Ai4YVlCRJkvobJIF7fFXt1rN8XpLvDSsgSZIkzWyQu1AvbHeeApDkycCK4YUkSZKkmQzSAvdE4OtJftyWlwCXJ7kYqKp6wtCikyRJ0v0MksA9Z+hRSJIkaWCDDCNy9SgCkSRJ0mAG6QMnSZKkDYgJnCRJ0oSZNYFLsnmSB7T5Ryd5XpIHDrDfw5Ocl+R7SS5N8uetfFGSs5P8sL1uve5vQ9JCkcz/SZJmM0gL3H8CmyXZETgLOAw4cYD9VgOva2PIPQV4VZLdgDcB51TVrsA5bVmSBlI1/ydJms0gCVyq6pfAC4APVtWLgMfNtlNVXV9VF7b524DLgB2B/YCT2mYnAc9fi7glSZIWrIESuCR7AYcAn21lG82lkiRLgd8CvglsV1XXt1U3ANvN5ViSJEkL3SDjwL0GeDNwWlVdmuSRwHmDVpBkC+DTwGuq6ufp6eBRVZVk2gsGSY4EjgRYsmTJoNVJkjRv2UdSU1JD7HDRbnY4E/hiVf1jK7sceEZVXZ9ke2B5VT1mpuMsW7asVqzw6V3Q/XgXQh+ZhfI+NTcL5bxYKO9Tc7NQzouF8j4HkeSCqlo23bq+LXBJzgD6foRV9bxZKg1wPHDZVPLWnA4cDhzTXj8z03EkSZJ0XzNdQn1Pe30B8BvAR9vywcCNAxz7qXR3rF6c5Dut7C10idupSY4ArgYOmGPMkiRJC1rfBK6qvgyQ5B/WaL47I8ms1zOr6qtAv6v1z5xTlLoP+0BIkrSwDXITw+ZJHllVVwIkeQSw+XDD0kwWQt8Ak1RJkvob9C7U5UmupGtR25l2d6gkSZJGb8YErj1Ca0tgV+Cxrfj7VXXnsAOTJEnS9GYcyLeq7gHeWFV3VtV322TyJkmSNEaDPInhS0le3x5Ov2hqGnpkkiRJmtYgfeAObK+v6ikr4JHrPxxJkiTNZtYErqoeMYpAJEmSNJhZE7j2OKz/DfxOK1oO/EtV3TXEuCRJktTHIJdQPwQ8EPhgWz6slf3xsIKSJElSf4MkcE+qqt17ls9N8t1hBSRJkqSZDXIX6t1JHjW1kOSRwN3DC0mSJEkzGaQF7g3AeWs8ieFlQ41KkiRJffVN4JK8Bvg68GW6JzE8pq263MF8JUmSxmemS6g7Af8E3AScBRwELMEH2UuSJI1V3xa4qno9QJJNgGXA3nSXTo9LsqqqdhtNiJIkSeo1SB+4BwEPpXuo/ZbAT4CLhxmUJPWTjDuC4dt663FHIGlDN1MfuOOAxwG3Ad+k6w/3j1V164hik6T7qBp9ncl46pWkmczUB24JsClwA3AdcC2wagQxSZIkaQYz9YF7TpLQtcLtDbwOeHySW4Dzq+qvRxSjJEmSeszYB66qCrgkySrgZ23aF9gTMIGTJGmE7AOqKTP1gfszupa3vYG76PrAfR04AW9ikCRppOwDql4ztcAtBT4JvLaqrh9NOJIkSZrNTH3gjh5lIJIkSRrMIOPAaQNjHwhJkhY2E7gJYx8ISZI00zhwkiRJ2gCZwEmSJE0YEzhJkqQJYwInSZI0YUzgJEmSJowJnCRJ0oQZWgKX5IQkNyW5pKdsUZKzk/ywvTralyRJ0hwNswXuROA5a5S9CTinqnYFzmnLkiRJmoOhJXBV9Z/ALWsU7wec1OZPAp4/rPolSZLmq1H3gduuqq5v8zcA2424fkmSpIk3tpsYqqqAvg9oSnJkkhVJVqxcuXKEkUmSJG3YRp3A3Zhke4D2elO/DavquKpaVlXLFi9ePLIAJUmSNnSjTuBOBw5v84cDnxlx/ZIkSRNvmMOIfAI4H3hMkmuTHAEcA/x+kh8Cv9eWJUmSNAcbD+vAVXVwn1XPHFadkiRJC4FPYpAkSZowJnCSJEkTxgROkiRpwpjASZIkTRgTOEmSpAljAidJkjRhTOAkSZImjAmcJEnShDGBkyRJmjAmcJIkSRPGBE6SJGnCmMBJkiRNGBM4SZKkCWMCJ0mSNGFM4CRJkiaMCZwkSdKEMYGTJEmaMCZwkiRJE8YETpIkacKYwEmSJE0YEzhJkqQJYwInSZI0YUzgJEmSJowJnCRJ0oQxgZMkSZowJnCSJEkTxgROkiRpwmw87gA0Gsl49q9at3ql9cHzXwuZ5//8ZAK3QPhD0kLm+a+FzPN/fvISqiRJ0oQxgZMkSZowY0ngkjwnyeVJrkjypnHEIEmSNKlGnsAl2Qj4Z+APgN2Ag5PsNuo4JEmSJtU4WuD2BK6oqiur6tfAycB+Y4hDkiRpIo0jgdsRuKZn+dpWJkmSpAFssDcxJDkyyYokK1auXDnucCRJkjYY40jgrgMe3rO8Uyu7j6o6rqqWVdWyxYsXjyw4SZKkDV1qxCP8JdkY+AHwTLrE7dvAi6vq0hn2WQlcPZoINY1tgZ+OOwhpTDz/tZB5/o/XzlU1bSvWyJ/EUFWrk/wp8EVgI+CEmZK3to9NcGOUZEVVLRt3HNI4eP5rIfP833CN5VFaVfU54HPjqFuSJGnSbbA3MUiSJGl6JnAaxHHjDkAaI89/LWSe/xuokd/EIEmSpHVjC5wkSdKEMYHTnCTZKskre5Z3SPKpccYkDVuSpUlevJb73r6+45GGLclRSV7S5l+aZIeedf/qM8zHz0uompMkS4Ezq+rx445FGpUkzwBeX1X7TrNu46paPcO+t1fVFkMMTxqqJMvpzv8V445F97IFbp5pLQWXJflwkkuTnJXkQUkeleQLSS5I8pUkj23bPyrJN5JcnOQdU60FSbZIck6SC9u6/VoVxwCPSvKdJO9u9V3S9vlGksf1xLI8ybIkmyc5Icm3kvxXz7GkoVqL38OJSfbv2X+q9ewY4Lfbef/a1iJxepJzgXNm+L1II9fO++8n+Vg7/z+V5MFJntn+Db64/Zu8adv+mCTfS3JRkve0srcleX37PSwDPtbO/wf1/Nt+VJJ399T70iQfaPOHtn/zv5PkX5JsNI7PYl6rKqd5NAFLgdXAHm35VOBQ4Bxg11b2ZODcNn8mcHCbPwq4vc1vDDy0zW8LXAGkHf+SNeq7pM2/FvibNr89cHmbfydwaJvfiu5JHJuP+7Nymv/TWvweTgT279l/6vfwDLqW56nylwLXAova8rS/l95jODmNamrnfQFPbcsnAH8JXAM8upV9BHgNsA1wec/5ulV7fRtdqxvAcmBZz/GX0yV1i4Ereso/DzwN+E3gDOCBrfyDwEvG/bnMt8kWuPnpR1X1nTZ/Ad2PeW/gk0m+A/wLXYIFsBfwyTb/8Z5jBHhnkouALwE7AtvNUu+pwFTrxQHAVN+4ZwFvanUvBzYDlsztLUlrbS6/h7k4u6puafNr83uRhumaqvpam/8o3eMrf1RVP2hlJwG/A/wM+BVwfJIXAL8ctIKqWglcmeQpSbYBHgt8rdX1RODb7Tf2TOCR6/6W1GssT2LQ0N3ZM3833R+SVVW1xxyOcQjd/66eWFV3JbmKLvHqq6quS3JzkicAB9K16EH3x+2FVXX5HOqX1pe5/B5W07qWJHkAsMkMx/1Fz/ycfy/SkK3ZwX0VXWvbfTfqHm+5J12StT/wp8A+c6jnZLr/sH8fOK2qKkmAk6rqzWsTuAZjC9zC8HPgR0leBJDO7m3dN4AXtvmDevbZErip/TH6XWDnVn4b8JAZ6joFeCOwZVVd1Mq+CLy6/ahJ8lvr+oakdTDT7+EqupYDgOcBD2zzs533/X4v0rgsSbJXm38xsAJYmmSXVnYY8OUkW9D9e/05um4wu9//UDOe/6cB+wEH0yVz0HVR2D/JwwCSLErib2I9M4FbOA4BjkjyXeBSuh8cdH0gjm6Xfnaha04H+BiwLMnFwEvo/ndFVd0MfC3JJb2dV3t8ii4RPLWn7O10fwgvSnJpW5bGqd/v4cPA01v5XtzbynYRcHeS7yZ57TTHm/b3Io3R5cCrklwGbA28F3gZXdeBi4F7gGPpErMz29+ArwJHT3OsE4Fjp25i6F1RVbcClwE7V9W3Wtn36PrcndWOezZr101BM3AYkQUuyYOBO1qz90F0NzR4B50kTag43NOCYB84PRH4QLu8uQp4+XjDkSRJs7EFTpIkacLYB06SJGnCmMBJkiRNGBM4SZKkCWMCJ0mSNGFM4CRJkiaMCZwkSdKE+X8QQc1od1qzjAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.boxplot(\"Sentence Length\", by=\"label\", grid=False, showfliers=False, color=\"Blue\", figsize=(10,4))\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Description Length by Type')\n",
    "plt.xlabel('')\n",
    "plt.ylabel('Words per Complaint')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove overly Length Sentences & Remove Unnecessary Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 10670 entries, 0 to 10675\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    10670 non-null  object\n",
      " 1   label   10670 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 250.1+ KB\n"
     ]
    }
   ],
   "source": [
    "data = data[data['Sentence Length'] < 59]\n",
    "\n",
    "data = data.drop(columns=['Sentence Length'])\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Quick Look at Updated Class Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neutral     5991\n",
      "positive    3215\n",
      "negative    1464\n",
      "Name: label, dtype: int64 \n",
      "\n",
      "Total: 10670\n"
     ]
    }
   ],
   "source": [
    "results_data = pd.Series(dtype=int, name='result_count')\n",
    "results_data = data['label'].value_counts()\n",
    "\n",
    "print(results_data, '\\n\\nTotal:', results_data.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final Prepocessing Before loading it to Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'positive': 0, 'negative': 1, 'neutral': 2}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The GeoSolutions technology will leverage Bene...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$ESI on lows, down $1.50 to $2.50 BK a real po...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>For the last quarter of 2010 , Componenta 's n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>According to the Finnish-Russian Chamber of Co...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Swedish buyout firm has sold its remaining...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label\n",
       "0  The GeoSolutions technology will leverage Bene...      0\n",
       "1  $ESI on lows, down $1.50 to $2.50 BK a real po...      1\n",
       "2  For the last quarter of 2010 , Componenta 's n...      0\n",
       "3  According to the Finnish-Russian Chamber of Co...      2\n",
       "4  The Swedish buyout firm has sold its remaining...      2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_classes = data['label'].unique()\n",
    "\n",
    "unique_class_converter = {unique_classes[x] : x for x in range(len(unique_classes))}\n",
    "\n",
    "print(unique_class_converter)\n",
    "\n",
    "data['label'].replace(unique_class_converter, inplace=True)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regular Project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Constants/Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104\n"
     ]
    }
   ],
   "source": [
    "model_ckpt = 'distilbert-base-uncased'\n",
    "device=\"cpu\"\n",
    "max_input_length = int(round(80*1.3,0))\n",
    "set_seed(42)\n",
    "batch_size = 64\n",
    "print(max_input_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Pandas DataFrame to Dataset; Then Split into Train/Test/Valid Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data shape: (8536, 3)\n",
      "testing data shape: (1067, 3)\n",
      "validation data shape: (1067, 3)\n",
      "{'text': Value(dtype='string', id=None), 'label': Value(dtype='int64', id=None), '__index_level_0__': Value(dtype='int64', id=None)}\n"
     ]
    }
   ],
   "source": [
    "# Convert pandas dataframe to a dataset\n",
    "dataset = Dataset.from_pandas(data)\n",
    "\n",
    "# Split the datasetdict into train/test/valid subsets\n",
    "train_testvalid = dataset.train_test_split(test_size=0.20)\n",
    "test_valid = train_testvalid['test'].train_test_split(test_size=0.50)\n",
    "\n",
    "# Combine the train/test/valid into one datasetdict\n",
    "dataset = DatasetDict({\n",
    "    'train' : train_testvalid['train'],\n",
    "    'test' : test_valid['test'],\n",
    "    'valid' : test_valid['train']\n",
    "})\n",
    "\n",
    "print('training data shape:', dataset['train'].shape)\n",
    "print('testing data shape:', dataset['test'].shape)\n",
    "print('validation data shape:', dataset['valid'].shape)\n",
    "print(dataset['train'].features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a82ee5515d3e4b9a87a30ec4dfb4e82e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d16798180024564b94fb4f9274aaff9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c91e998db75642ddad26f3e2e7b51e2b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data shape: (8536, 5)\n",
      "testing data shape: (1067, 5)\n",
      "validation data shape: (1067, 5)\n"
     ]
    }
   ],
   "source": [
    "# Define the Tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "# Method to Tokenize each Sample in the Dataset\n",
    "def tokenize(batch):\n",
    "    return tokenizer(batch['text'], truncation=True, padding=True, max_length=max_input_length, return_tensors=\"np\")\n",
    "\n",
    "# Map the 'Tokenize' Method to All Samples in the Entire Dataset\n",
    "encoded_data = dataset.map(tokenize, batched=True, batch_size=None)\n",
    "\n",
    "print('training data shape:', encoded_data['train'].shape)\n",
    "print('testing data shape:', encoded_data['test'].shape)\n",
    "print('validation data shape:', encoded_data['valid'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check Results of Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': Value(dtype='string', id=None), 'label': Value(dtype='int64', id=None), '__index_level_0__': Value(dtype='int64', id=None), 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}\n",
      "Dataset({\n",
      "    features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask'],\n",
      "    num_rows: 8536\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(encoded_data['train'].features)\n",
    "print(encoded_data['train'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.bias', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_transform.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'pre_classifier.weight', 'classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels=len(unique_classes)\n",
    "\n",
    "model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to(device))\n",
    "model.config.num_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ensure that git lfs is Installed; Remember to Enter Access Token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "!git lfs install\n",
    "\n",
    "# HuggingFace Access Token ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Function to Compute Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    f1 = f1_score(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\"accuracy\": acc, \"f1_score\": f1}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_of_epochs = 3\n",
    "logging_steps = (len(encoded_data['train']) // batch_size)\n",
    "learning_rate=2e-5\n",
    "label_names = unique_classes\n",
    "steps = 100\n",
    "\n",
    "model_name = f\"{model_ckpt}-Finanacial_Sentiment_Analysis\"\n",
    "\n",
    "training_args = TrainingArguments(output_dir=model_name,\n",
    "                        num_train_epochs=num_of_epochs,\n",
    "                        learning_rate=learning_rate,\n",
    "                        per_device_train_batch_size=batch_size, \n",
    "                        per_device_eval_batch_size=batch_size, \n",
    "                        weight_decay=0.01,\n",
    "                        log_level=\"error\",\n",
    "                        logging_strategy='epoch', \n",
    "                        hub_strategy=\"checkpoint\",\n",
    "                        evaluation_strategy=\"epoch\",\n",
    "                        save_strategy=\"epoch\",\n",
    "                        save_total_limit=5,\n",
    "                        load_best_model_at_end=True,\n",
    "                        metric_for_best_model=\"f1_score\",\n",
    "                        greater_is_better=True,\n",
    "                        logging_first_step=True,\n",
    "                        group_by_length=True,\n",
    "                        disable_tqdm=False,\n",
    "                        push_to_hub=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Optimizer (& Learning Rate Scheduler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_name = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "\n",
    "num_training_steps = num_of_epochs * len(encoded_data['train'])\n",
    "\n",
    "lr_scheduler = get_scheduler(\n",
    "    name=\"linear\", optimizer=optimizer_name, num_warmup_steps=0, num_training_steps=num_training_steps\n",
    ")\n",
    "\n",
    "optimizer = [optimizer_name, lr_scheduler]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Subclass Trainer to Correctly Compute Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.get(\"labels\")\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.get(\"logits\")\n",
    "        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([1.0, 2.0, 3.0]))\n",
    "        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))\n",
    "        return (loss, outputs) if return_outputs else loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/distilbert-base-uncased-Finanacial_Sentiment_Analysis into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = CustomTrainer(model=model, \n",
    "                  args=training_args, \n",
    "                  compute_metrics=compute_metrics,\n",
    "                  train_dataset=encoded_data['train'], \n",
    "                  eval_dataset=encoded_data['valid'], \n",
    "                  data_collator=data_collator,\n",
    "                  optimizers=optimizer,\n",
    "                  tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67819541d960403a9d04ddd6bd47153f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/402 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 1.1124, 'learning_rate': 1.9999218994064354e-05, 'epoch': 0.01}\n",
      "{'loss': 0.5501, 'learning_rate': 1.989534520462356e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a6773dc92c64ed283af2af74be6a2d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3857389986515045, 'eval_accuracy': 0.7656982193064668, 'eval_f1_score': 0.7614242410331038, 'eval_runtime': 235.0767, 'eval_samples_per_second': 4.539, 'eval_steps_per_second': 0.072, 'epoch': 1.0}\n",
      "{'loss': 0.3062, 'learning_rate': 1.9790690409247112e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "494b9eebd17c45c5acdd691d18a73af4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.32900911569595337, 'eval_accuracy': 0.8322399250234301, 'eval_f1_score': 0.8349432757741744, 'eval_runtime': 231.5048, 'eval_samples_per_second': 4.609, 'eval_steps_per_second': 0.073, 'epoch': 2.0}\n",
      "{'loss': 0.2347, 'learning_rate': 1.9686035613870666e-05, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "081ee1c70af048c09fa44101f8b98d3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3071199357509613, 'eval_accuracy': 0.8425492033739457, 'eval_f1_score': 0.8470211920321009, 'eval_runtime': 231.0708, 'eval_samples_per_second': 4.618, 'eval_steps_per_second': 0.074, 'epoch': 3.0}\n",
      "{'train_runtime': 17675.2787, 'train_samples_per_second': 1.449, 'train_steps_per_second': 0.023, 'train_loss': 0.3650743406803454, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=402, training_loss=0.3650743406803454, metrics={'train_runtime': 17675.2787, 'train_samples_per_second': 1.449, 'train_steps_per_second': 0.023, 'train_loss': 0.3650743406803454, 'epoch': 3.0})"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assessing the Results of the Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate Predictions for Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6364bbfc6d5648d39ccb0b60fd634192",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.3071199357509613,\n",
       " 'test_accuracy': 0.8425492033739457,\n",
       " 'test_f1_score': 0.8470211920321009,\n",
       " 'test_runtime': 233.8672,\n",
       " 'test_samples_per_second': 4.562,\n",
       " 'test_steps_per_second': 0.073}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_predictions = trainer.predict(encoded_data['valid'])\n",
    "\n",
    "output_predictions.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract Last Hidden States to Create Feature Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc97731ee5c14f3293abf931d5001e34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bf06bd952ea4c669d444e2d1fbbe6d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e57316fea13c4a029b1afe9550d27d6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 8536\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 1067\n",
      "    })\n",
      "    valid: Dataset({\n",
      "        features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 1067\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "def extract_hidden_states(batch):\n",
    "    inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}   \n",
    "    with torch.no_grad():\n",
    "        last_hidden_state = model(**inputs)[0]\n",
    "    return {\"hidden_state\" : last_hidden_state[:,0].cpu().numpy()}\n",
    "\n",
    "encoded_data.set_format(\"torch\", columns=['input_ids', 'attention_mask', 'label']) \n",
    "\n",
    "hidden_data = encoded_data.map(extract_hidden_states, batched=True)\n",
    "print(hidden_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Feature Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8536,), (1067,))"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = np.array(hidden_data['train']['hidden_state'])\n",
    "X_valid = np.array(hidden_data['valid']['hidden_state'])\n",
    "y_train = np.array(hidden_data['train']['label'])\n",
    "y_valid = np.array(hidden_data['valid']['label'])\n",
    "X_train.shape, X_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Function to Display Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_preds, y_true, labels):\n",
    "    cm = confusion_matrix(y_true, y_preds, normalize=\"true\")\n",
    "    fig, ax = plt.subplots(figsize=(11, 11))\n",
    "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\n",
    "    disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax)\n",
    "    plt.title('Normalized Confusion Matrix')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Logits to Integers & Display Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAJkCAYAAAB05hsHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABCDUlEQVR4nO3deZgcZdWw8ftMFnYSSEJICGEHWWRLRMCXXTb1BVF2VFARURZlkQ/UFxB3QQUFRAREQcSwKAHCpoggiiRBtiSAyJ4ESAIJO2Q53x9dM3SGZGoynZmeTN8/r77sqnrqqac6zcyZc6qeisxEkiRJaktTvQcgSZKk7s+gUZIkSaUMGiVJklTKoFGSJEmlDBolSZJUyqBRkiRJpXrXewCSJElLql4rrpE5580uP26+Oe2WzNyjK49p0ChJktRBOedNltpg/y4/7lv3nzewq49p0ChJktRhAdEYV/s1xllKkiSpJgaNkiRJKmV5WpIkqaMCiKj3KLqEmUZJkiSVMtMoSZJUC2+EkSRJkirMNEqSJNXCaxolSZKkCoNGSZIklbI8LUmS1GE+EUaSJElqYaZRkiSpFt4II0mSJFWYaZQkSeqowGsaJUmSpGYGjZIkSSpleVqSJKnDwhthJEmSpGZmGiVJkmrhjTCSJElShZlGSZKkWnhNoyRJklRh0ChJkqRSlqclSZI6LLwRRpIkSWpmplGSJKmjAm+EkSRJkpqZaZQkSaqF1zRKkiRJFQaNkiRJKmV5WpIkqcOcckeSJElqYaZRkiSpFk1OuSNJkiQBZholSZI6LvCaRkkqExF3RMThxftDIuLWxdz/mhGREdHlf+BGxDIRcX1EzIqIq2roZ7F/LvUQETdFxKH1Hoek+jFolLqxiHgqIl6MiOWq1h0eEXfUcVgLlJm/y8zduvq4EXFwRIyLiNciYmoR3PzPYuh6X2AwMCAz9+toJ531uUTEjkVA/cdW6zcr1t/Rzn5Oj4jLy9pl5p6Z+ZsODldSD2DQKHV/vYCv1NpJVPSo/+Yj4njgbOB7VAK84cD5wN6Lofs1gMcyc85i6KuzTAO2iYgBVesOBR5bXAfoid8babGL6PpXHfiDQOr+zgROjIj+C9oYEdtGxNiijDo2Irat2nZHRHw3Iu4G3gDWLrJQX46I/0TEqxHx7YhYJyL+ERGvRMSoiOhb7L9SRNwQEdMi4uXi/bCFjOOwiPh78f6kIvPX/JodEZcW2/pFxMVFVnByRHwnInoV23pFxFkRMT0ingA+urAPJSL6AWcAR2XmtZn5embOzszrM/NrRZulIuLsiJhSvM6OiKWKbTtGxHMRcUKRzZ0aEZ8ttn0LOBU4oBj/51tn5FqXzovzf6L4TJ+MiENafy7t/Pf6dkTcXfRza0QMXNhnALwD/Ak4sPnzAw4AftfqszonIp4t/n3HR8R2xfo9gK9XnecDVeNo/b2pvhThFxFxTVX/P4yIv0TU6TeZpC5h0Ch1f+OAO4ATW2+IiJWBG4GfAQOAnwA3tso8fRo4AlgBeLpYtzswAtgaOAm4EPgUsDqwCXBQ0a4J+DWVrNtw4E3g3LIBZ+aPMnP5zFwe2JBKRuwPxeZLgTnAusAWwG7A4cW2LwAfK9aPpFIiXphtgKWBP7bR5hvFOW4ObAZsBXyzavuqQD9gNeDzwHkRsVJmnkYle/mH4jwubut8o3L5wM+APTNzBWBb4P4FtGvPv9fBwGeBVYC+LODfvZXfAp8p3u8OPAxMadVmLJXPYGXgCuCqiFg6M29udZ6bVe2zoO9NsxOA9xcB8XZUPrtDMzNLxir1QMXk3l39qgODRmnJcCpwTEQMarX+o8B/MvOyzJyTmb8HHgH+t6rNpZk5odg+u1j3o8x8JTMnUAkybs3MJzJzFnATlaCNzJyRmddk5huZ+SrwXWCH9g46Ipahkgk7JzNviojBwEeArxaZwReBn1JkyoD9gbMz89nMfAn4fhvdDwCml5SPDwHOyMwXM3Ma8C0qwVCz2cX22Zk5BngN2KC959fKPGCTiFgmM6cWn21r7fn3+nVmPpaZbwKjqAR7C5WZ/wBWjogNqASPv11Am8uLf8s5mfljYCnKz3NB35vm/t6g8jn+BLgcOCYznyvpT1IXiog9IuLRiHg8Ik5ewPbhEfHXiPh3RDwYER8p69OgUVoCZObDwA1A6//wh/LeLNDTVDJnzZ5dQJcvVL1/cwHLywNExLIR8cuIeDoiXgHuBPo3l5Pb4WLg0cz8YbG8BtAHmBoRMyNiJvBLKlm15vOpHm/rc6s2AxgYbd9Z3frzebpY19JHq6DzDYpzXxSZ+TqVsvCRVM7txoh4XzvG0zym6n+v5zswnsuAo4GdWEDmNSJOjIhJRUl8JpXsaltlb1jw96ZFZv4LeILKhCOj2jFGqefqZtc0Fj+jzwP2BDYCDoqIjVo1+yYwKjO3oPKH+/llp2nQKC05TqNSvq0OMKZQCcSqDQcmVy3XUjI8gUpG6oOZuSKwfbG+9Nq14i/b9amULps9C7wNDMzM/sVrxczcuNg+lUqJvNnwNg7xz6Kvj7fRpvXnM5z3lm7b63Vg2arlVas3ZuYtmbkrMIRK9vBX7RhP85gmL6DtorgM+DIwpsgCtijKxydRyeKulJn9gVm8+2+4sO9Hm9+biDiKSsZyStG/pO5jK+DxooL0DnAl771BMIEVi/f9aMfPRoNGaQmRmY9TuS7w2KrVY4D1ozLtTO+IOIDKX5U3LKbDrkAl8zizuB7vtPbsFBF7FuPcpyizNp/DVOBW4McRsWJENEXlJpzmkvco4NiIGBYRK/HezCpVfc2iUrY/LyI+XmRF+0TEnhHxo6LZ74FvRsSg4oaSU6mUUzvifmD7oqTTDzil6nwHR8TexbWNb1Mpc89bQB+d8u+VmU9SuWzgGwvYvAKVa0inAb0j4lTe/UUBlSzzmrEId0hHxPrAd6hcB/tp4KSI2Lxjo5fUCVZj/mrBc8yfcAA4HfhURDxH5WfTMWWdGjRKS5YzgJY5GzNzBpUbR06gUq49CfhYZk5fTMc7G1gGmA7cA9zczv0OAAYBk+LdO6gvKLZ9hsoNHhOBl4GrqWTnoJKduwV4ALgPuLatgxTX5x1PpcwyjcoPyaOpXEcJlcBmHPAg8FDR53faeQ6tj3UblaD9QWA88wd6TcU4pgAvUQngvrSAPjrt3ysz/56ZC8oU3ELl3+0xKqXwt5j/l0nzxOUzIuK+suMUlwNcDvwwMx/IzP9QuQP7sijuTJcaTn1uhBkYlTlqm19HLOKoD6Jy7fIwKteaX1b2x2N4s5skSVLHNK04LJfauuapdBfZW7edND4zRy5oW0RsA5yembsXy6cAZOb3q9pMAPbIzGeL5SeArYsbFBfITKMkSVJH1eMmmPIpUccC60XEWlGZd/dAYHSrNs8Au1ROITakMoXZtLY6NWiUJEnqQYpZIY6mcnnKJCp3SU+IiDMiYq+i2QnAF4pJ/X8PHFY212pbU1VIkiSpTDd80mYx9+yYVutOrXo/EfjQovTZ/c5SkiRJ3Y6ZxhLRd7mMpVeq9zAkADZbd9XyRlIXmT13QbMKSV1v8rPP8PJL0332eSczaCwRS6/EUluVTl0kdYk7ri97DLHUdaa8/Fa9hyABsN+e29V3AOU3pvQIlqclSZJUykyjJElSh0W3vBGmMzTGWUqSJKkmZholSZJq4TWNkiRJUoVBoyRJkkpZnpYkSeqowBthJEmSpGZmGiVJkjrMKXckSZKkFmYaJUmSauGUO5IkSVKFQaMkSZJKWZ6WJEmqhTfCSJIkSRVmGiVJkmrhjTCSJElShZlGSZKkjgon95YkSZJaGDRKkiSplOVpSZKkWngjjCRJklRhplGSJKkGYaZRkiRJqjDTKEmS1EGBmUZJkiSphUGjJEmSSlmeliRJ6qgoXg3ATKMkSZJKmWmUJEnqsPBGGEmSJKmZmUZJkqQamGmUJEmSCgaNkiRJKmV5WpIkqQaWpyVJkqSCmUZJkqQamGmUJEmSCmYaJUmSOsrHCEqSJEnvMmiUJElSKcvTkiRJHRQ+e1qSJEl6l5lGSZKkGphplCRJkgpmGiVJkmpgplGSJEkqGDRKkiSplOVpSZKkGlieliRJkgpmGiVJkjrKZ09LkiRJ7zLTKEmSVAOvaZQkSZIKBo2SJEkqZXlakiSpg4KwPC1JkiQ1M9MoSZJUAzONkiRJWiJFxB4R8WhEPB4RJy9g+08j4v7i9VhEzCzr00yjJElSLbpZojEiegHnAbsCzwFjI2J0Zk5sbpOZx1W1PwbYoqxfM42SJEk9y1bA45n5RGa+A1wJ7N1G+4OA35d1atAoSZLUs6wGPFu1/Fyx7j0iYg1gLeD2sk4tT0uSJHVU1O1GmIERMa5q+cLMvLAD/RwIXJ2Zc8saGjRKkiQteaZn5siFbJsMrF61PKxYtyAHAke154AGjZIkSTXohlPujAXWi4i1qASLBwIHt24UEe8DVgL+2Z5OvaZRkiSpB8nMOcDRwC3AJGBUZk6IiDMiYq+qpgcCV2ZmtqdfM42SJEk16IaZRjJzDDCm1bpTWy2fvih9mmmUJElSKYNGSZIklbI8LUmS1EFBdMvydGcw0yhJkqRSZholSZJq0RiJRjONkiRJKmemUZIkqaPq9xjBLmemUZIkSaUMGiVJklRqiQsaI+LIiPhM8f6wiBhate2iiNiofqNrPLuMWIt7Lzqc8ZccwVf3/+B7tg8btAKjf3ggfzv3MP7+i8+y6wfWbtl23AFbM/6SI7j3osPZecRaXTls9VC33zOJDx34Xbbe79v8/Le3vWf72+/M4Yj/u5St9/s2ex7+E56ZOmO+7c89/xJr7/I1zr/i9q4asnqwu8c9yj5HnMleh/+IX4/663u2j3/4CQ4+9hw+8L+n8Oe/PzjftqkvvsyXv3kRn/jiWXzyyB8z5YWXumrY6oCI6PJXPSxx1zRm5gVVi4cBDwNTim2H12NMjaqpKTjzqF3Z5+t/YMr0V7n9Z4dy0z2P8+gz7/4iPuGgbfnTnY9wyY33s8HwAYz69n5sdugFbDB8AJ/YYUO2+eLFrLry8vzp+wcw8vBfMW9eux5/Kb3H3LnzOOWsqxh1zpcZskp/9vj8j9ltu/ezwVqrtrS54vp/0n+FZbjnqv/jT7fdx3fOv54Lv31Yy/bTfvYndt7avztVu7lz5/HDX/yJ879zOIMH9uNTx53LDltvxNrDB7e0GTKoP6cftz+XXXvne/Y/9Sej+PwBO7H1FuvzxptvN8w1c+reujTTGBFrRsQjEfG7iJgUEVdHxLIRsUtE/DsiHoqISyJiqaL9DyJiYkQ8GBFnFetOj4gTI2JfYCTwu4i4PyKWiYg7ImJkkY08s+q4h0XEucX7T0XEvcU+v4yIXl35GfQkIzYYwhNTZ/L087OYPWce1/5tEh/ZZr33tFth2aUAWHG5pXh+xmsAfGSb9bj2b5N4Z/ZcnnlhFk9MncmIDYZ06fjVs/x74tOsNWwQa6w2kL59evPxD2/JLXc9NF+bW+56mP333AqAj+20GX8f9xiZlT9UbvrbgwwfOmC+IFPqqIcfe5ZhQwcwbMgA+vTpze7bb8Yd90ycr83QwSuz/lpDaGoVED7xzAvMnTuPrbdYH4Bll1mKZZbu22Vj16JrlExjPcrTGwDnZ+aGwCvA8cClwAGZ+X4q2c8vRcQAYB9g48zcFPhOdSeZeTUwDjgkMzfPzDerNl9T7NvsAODKiNiweP+hzNwcmAscsvhPsTEMGbACk6e90rI8ZfqrDBmw/HxtfnD539l/5415+LIvM+qM/Tjp/NuKfZdfwL4rdM3A1SNNnTaLoYP7tywPGdSfqdNmtWozk6GDVwKgd+9erLDc0rw063Vef+Ntzr38L5z4uT26csjqwabNmMWqA/u3LK8ysB8vzpi18B2qPD15OssvtzQnfOe3HHTMOfz04huZO3deJ41Uar96BI3PZubdxfvLgV2AJzPzsWLdb4DtgVnAW8DFEfEJ4I32HiAzpwFPRMTWRfD5PuDu4lgjgLERcX+xvHbr/SPiiIgYFxHjcvbrHTlHFT6540ZccdtDbPLp89n/1Ku44GsfwyqLupszL76JIw7ckeWKrLhUT3PnzuX+CU9y3Oc/ymVnH83k51/i+j+Pq/ew1Jaow6sO6nFNY+uL1mYCA97TKHNORGxFJbDbFzga2HkRjnMlsD/wCPDHzMyo5HN/k5mntDnAzAuBCwGaVhzmRXYLMXXGq6w2aMWW5aEDV2BqUX5u9qndN2W/b4wCYOykKSzdtzcDVlyWqTNeW8C+r3bNwNUjDRnUjykvzGxZnjptJkMG9WvVpj9TXniZoav0Z86cubz6+lus3G85/j3xaW746wN8+7zRvPLamzRFsFTf3nx+3+27+CzUUwwa0I/np89sWX5x+ixWGdBv4TtUWWVgP9ZfeyjDhlR+Ne64zcY89MgzfLwTxiktinpkGodHxDbF+4OplJjXjIh1i3WfBv4WEcsD/TJzDHAcsNkC+noVWFhN84/A3sBBVAJIgL8A+0bEKgARsXJErFHrCTWq+x6dyjpDV2L44H706d3EJ3bYkJvueXy+NpNffIXtt6h8xOuvPoCl+vZi+qw3uOmex/nEDhvSt08vhg/uxzpDV2L8o1PrcRrqITbfcDhPPDeNp6fM4J3Zc/jTn+9jt//ZZL42u223CaNuuheAG/76AB8asR4RwXW/+Arjrj2Ncdeexhf234FjD93VgFE12Xj9YTw7eQaTn3+J2bPncMudD7DDBzds377rrc6rr7/Fy7Mqf4SPfeBx1h6+SmcOV2qXemQaHwWOiohLgInAscA9wFUR0RsYC1wArAxcFxFLU0nEHr+Avi4FLoiIN4Ftqjdk5ssRMQnYKDPvLdZNjIhvArdGRBMwGzgKeHrxn2bPN3dectL5t3HNd/enV1Pwu1sf4pGnp3PKp/+H+//zPDfd8zjf/NXtnPOVPfjyPh8gMznqx2MAeOTp6fzpzke455efZ868eXztvNu8c1o16d27F987/pMcdNwvmDt3Hgd9bGvet/YQfvirMWz+vtXZfbv3c/DHtuboMy5n6/2+Tf8Vl+WXZxxa72Grh+rdqxf/70t7c9T/Xcy8efPYa9cPsM4aq/KLy25lo/WGscPWGzHhsWc54Tu/5ZXX3uTOeydxwe9u4+pfnECvXk0c9/mP8MWv/woSNlx3NT6x+1b1PiW1oVHubo/mOwe75GARawI3ZOYmZW27i6YVh+VSWx1T72FIADx//Yn1HoLUYsrLb9V7CBIA++25HQ8/cF9dIre+q6ybqx7wky4/7rPn7j0+M0d25TGXuHkaJUmSuot6ToHT1bo0aMzMp4AlJssoSZKkCjONkiRJNWiUTOMS9+xpSZIkdT2DRkmSJJWyPC1JklQDy9OSJElSwUyjJElSLRoj0WimUZIkSeXMNEqSJNXAaxolSZKkgkGjJEmSSlmeliRJ6qiwPC1JkiS1MNMoSZLUQQE0SKLRTKMkSZLKmWmUJEnqsPCaRkmSJKmZQaMkSZJKWZ6WJEmqQYNUp800SpIkqZyZRkmSpBp4I4wkSZJUMNMoSZLUUeE1jZIkSVILg0ZJkiSVsjwtSZLUQQE0NTVGfdpMoyRJkkqZaZQkSaqBN8JIkiRJBTONkiRJNXByb0mSJKlg0ChJkqRSlqclSZI6yifCSJIkSe8y0yhJktRBgTfCSJIkSS3MNEqSJHVYmGmUJEmSmhk0SpIkqZTlaUmSpBo0SHXaTKMkSZLKmWmUJEmqgTfCSJIkaYkUEXtExKMR8XhEnLyQNvtHxMSImBARV5T1aaZRkiSpo7rhYwQjohdwHrAr8BwwNiJGZ+bEqjbrAacAH8rMlyNilbJ+zTRKkiT1LFsBj2fmE5n5DnAlsHerNl8AzsvMlwEy88WyTg0aJUmSepbVgGerlp8r1lVbH1g/Iu6OiHsiYo+yTi1PS5IkdVAdnz09MCLGVS1fmJkXLsL+vYH1gB2BYcCdEfH+zJzZ1g6SJElaskzPzJEL2TYZWL1qeVixrtpzwL8yczbwZEQ8RiWIHLuwA1qeliRJqkFE179KjAXWi4i1IqIvcCAwulWbP1HJMhIRA6mUq59oq1ODRkmSpB4kM+cARwO3AJOAUZk5ISLOiIi9ima3ADMiYiLwV+BrmTmjrX4tT0uSJNWgO07unZljgDGt1p1a9T6B44tXu5hplCRJUimDRkmSJJWyPC1JklSDblid7hRmGiVJklTKTKMkSVJHRfe8EaYzmGmUJElSKTONkiRJHVR5jGC9R9E1zDRKkiSplEGjJEmSSlmeliRJ6rDwRhhJkiSpmZlGSZKkGjRIotFMoyRJksqZaZQkSaqB1zRKkiRJBYNGSZIklbI8LUmS1FHhjTCSJElSCzONkiRJHVR59nRjpBrNNEqSJKmUmUZJkqQamGmUJEmSCgaNkiRJKmV5WpIkqQYNUp02aCyz+XqrcvdNJ9V7GBIAK+/4zXoPQWrx76u/Xu8hSOpCBo2SJEk18EYYSZIkqWCmUZIkqaN8jKAkSZL0LoNGSZIklbI8LUmS1EFBeCOMJEmS1MxMoyRJUg0aJNFoplGSJEnlzDRKkiTVoKlBUo1mGiVJklTKoFGSJEmlLE9LkiTVoEGq02YaJUmSVM5MoyRJUgdF4OTekiRJUjMzjZIkSTVoaoxEo5lGSZIklTNolCRJUinL05IkSTXwRhhJkiSpYKZRkiSpBg2SaDTTKEmSpHJmGiVJkjoogKAxUo1mGiVJklTKoFGSJEmlLE9LkiTVwCfCSJIkSQUzjZIkSR0V4eTekiRJUjMzjZIkSTVokESjmUZJkiSVM2iUJElSKcvTkiRJHRRAU4PUp800SpIkqZSZRkmSpBo0SKLRTKMkSZLKGTRKkiTVIIoJvrvy1Y4x7RERj0bE4xFx8gK2HxYR0yLi/uJ1eFmflqclSZJ6kIjoBZwH7Ao8B4yNiNGZObFV0z9k5tHt7ddMoyRJUs+yFfB4Zj6Rme8AVwJ719qpQaMkSVIHRdTnBQyMiHFVryOqhrUa8GzV8nPFutY+GREPRsTVEbF62blanpYkSVryTM/MkTXsfz3w+8x8OyK+CPwG2LmtHQwaJUmSatANJ/eeDFRnDocV61pk5oyqxYuAH5V1anlakiSpZxkLrBcRa0VEX+BAYHR1g4gYUrW4FzCprFMzjZIkSTXobnnGzJwTEUcDtwC9gEsyc0JEnAGMy8zRwLERsRcwB3gJOKysX4NGSZKkHiYzxwBjWq07ter9KcApi9Kn5WlJkiSVMtMoSZJUg/Y8oaUnMNMoSZKkUmYaJUmSOiiApsZINJpplCRJUjkzjZIkSR0V4TWNkiRJUjODRkmSJJWyPC1JklSDBqlOm2mUJElSOTONkiRJNfBGGEmSJKlgplGSJKmDGmly74UGjRHxcyAXtj0zj+2UEUmSJKnbaSvTOK7LRiFJkqRubaFBY2b+pno5IpbNzDc6f0iSJElLDm+EKUTENhExEXikWN4sIs7v9JFJkiSp22jP3dNnA7sDMwAy8wFg+04ckyRJ0hIj6vCqh3ZNuZOZz7ZaNbcTxiJJkqRuqj1T7jwbEdsCGRF9gK8Akzp3WJIkSd1fBDR5TWOLI4GjgNWAKcDmxbIkSZIaRGmmMTOnA4d0wVgkSZLUTbXn7um1I+L6iJgWES9GxHURsXZXDE6SJKm7i+j6Vz20pzx9BTAKGAIMBa4Cft+Zg5IkSVL30p6gcdnMvCwz5xSvy4GlO3tgkiRJS4KI6PJXPbT17OmVi7c3RcTJwJVUnkV9ADCmC8YmSZKkbqKtG2HGUwkSm8PZL1ZtS+CUzhqUJEnSkqJBZtxp89nTa3XlQCRJktR9tWdybyJiE2Ajqq5lzMzfdtagJEmS1L2UBo0RcRqwI5WgcQywJ/B3wKBRkiQ1tCB8IkyVfYFdgOcz87PAZkC/Th2VJEmSupX2lKffzMx5ETEnIlYEXgRW7+RxaQnx539O5Os/voa58+bx6b234auH7jbf9rffmc2XTr+MBx55lpX6Lccl3/0sw4cO4K//eoQzzhvNO7Pn0LdPb751zN5s/4EN6nQW6il22Wo9vn/MR+nV1MRlN47j7CvunG/7sFX6cf7X96Xf8kvTq6mJb/3yFm7712Ns+b5hnH3ix4HKBe0/uPR2brxrYh3OQD3J3eMe5cxfjmbevOTju3+Az+2/03zbxz/0BGddeD3/efJ5vn/yQez6P5u2bBvxsZNZd81VAVh1UH/OOe2wrhy6FkUdJ9vuau0JGsdFRH/gV1TuqH4N+GdnDqo9ijEdnJnnF8tDgZ9l5r51HVgDmTt3Hif96CquPfcohq7Sn10OPZM9tns/71t7SEuby0f/k/4rLMv4a0/jmlvHc/q513HJ9z7HgP7LccWPv8iQQf2Y+N8p7Hfs+Uy48Tt1PBst6ZqagjO/+r/sc8KvmTLtFW7/5Ze46e5JPPr0tJY2J3xmJ/7014e45Lp72WCNQYz64aFsduBZTHryBXb64vnMnTuPwSuvwF2XHM3N/3iEuXPn1fGMtCSbO3cePzj/T/ziu4czeGA/Dvnqueyw9UasM3xwS5shq/TnW8fvz2+vufM9+y/Vtw9/OPerXThiqVxpeTozv5yZMzPzAmBX4NCiTF1v/YEvNy9k5hQDxq41fsLTrDVsIGuuNpC+fXrzid1GcNOdD83XZszfHuLAj34QgL133pw7xz5GZrLpBqszZFDlKocN1x7Cm2/P5u13Znf5OajnGLHhMJ6Y/BJPT32Z2XPmcu3tD/KR/9lw/kaZrLDsUgCsuPzSPD/jFQDefHt2S4C4VN/eZHbp0NUDPfzYs6w+dADDhgygT5/e7L79Ztzxz/mz10MHr8z6aw2hqalB0lQ9WKNM7r3QoDEitmz9AlYGehfv2xQRa0bEpIj4VURMiIhbI2KZiFgnIm6OiPERcVdEvK9ov05E3BMRD0XEdyLitWL98hHxl4i4r9i2d3GIHwDrRMT9EXFmcbyHi33uiYiNq8ZyR0SMjIjlIuKSiLg3Iv5d1Zc6YOq0maw2eKWW5aGr9GfqtJmt2sxitcH9AejduxcrLr8ML816fb42o2+/n802GMZSfft09pDVgw0ZuCKTX5zVsjxl2isMGTj/5dc/+PXt7L/b5jx81UmM+uGhnHTODS3bRmw4jH9ceix3//oYjv/JdWYZVZMXZ8xi8MD+LcuDB/Zj2oxZC9+hlXfemcPBx/6Mzxx3Ln/9x4ROGKG06NoqT/+4jW0J7NyO/tcDDsrML0TEKOCTwGeBIzPzPxHxQeD8oq9zgHMy8/cRcWRVH28B+2TmKxExELgnIkYDJwObZObmUAlSq/b5A7A/cFpEDAGGZOa4iPgecHtmfq4ob98bEX/OzPmjGHWZSf+dyrfOHc01P/9yeWOpRp/88KZccdN9nDfqbj6w8epc8I392Pawn5GZjJ/0HNse9jPWX2MQ55+yL3/+12O8/c6ceg9ZDWrMpSezysB+PDd1Bkec8ivWXWtVVh8yoN7DUoNra3LvnRa2bRE8mZn3F+/HA2sC2wJXVaVWlyr+fxvg48X7K4CzivcBfC8itgfmAasB714UsmCjgFuB06gEj1cX63cD9oqIE4vlpYHhwKTqnSPiCOAIgNWHDy89yUY1ZFB/Jr/wcsvylBdnMmRQ/1Zt+jH5hUpGcs6cubzy2pus3G85ACa/8DKfOelXnH/6p1lr2KCuHLp6oKnTX2G1Vd7NLA4dtCJTp8+f2fnUR0aw39d+A8DYCc+ydN/eDOi3LNNnvvt342NPT+P1N99mw7UGc/+jk7tm8OpxVhnQjxemz2xZfmH6LAYNaP/EI6sUWfJhQwYwctO1eeS/kw0au7H2TEXTE3T2eb5d9X4ulfL2zMzcvOq14UL2bXYIMAgYUWQVX6BqkvEFyczJwIyI2JTKs7L/UGwK4JNVxx6emZMWsP+FmTkyM0cOHGgwszBbbjScJ56dxtOTp/PO7Dlce+t49tju/fO12XP793Pljf8C4Lrb72e7kesTEcx69Q0OPO4CTj16L7bebO16DF89zH2PTGadYQMYvupK9Ondi0/svCk33f3IfG0mvziL7UdUvm/rrzGIpfr2ZvrM1xm+6kr06lX5cbj64P6sN3wQzzz/8nuOIbXXxusP45kpM5j8/EvMnj2HW+58gB23Lvt1V/HKq2/wzuxKlvvlWa9z/8SnWHt4Wa5E6nzteiLMYvQK8GRE7JeZV0Ul3bhpZj4A3EOlfP0H4MCqffoBL2bm7IjYCVijWP8qsEIbx/oDcBLQLzMfLNbdAhwTEcdkZkbEFpn578V3eo2ld+9e/Ohr+7Hvseczd15yyP9uzYbrDOF7v7yRLTYczp7bv59P7bUNR572W0Z84lustOKyXPTdyj1Uvxp1J08+N50zL7qZMy+6GYBrfn4Ug1Zu659UWri5c+dx0tnXc81Zh9GrKfjdmPt45KkXOeVzu3D/I5O56R+P8M3zxnDO1/bhy/t9iEw46vvXALDNpmvwlYO3Z86ceczL5MSfjualWW/U+Yy0JOvdqxf/70t78+VvXsy8efPYe7cPsM4aq3L+Zbey0XrD2HHrjZjw2LMc/+3f8sprb3LnvyZxweW3cc0FJ/DEsy/y3Z//kWgKcl7y2f12nO+ua3UvAXW7MaWrRXbSbYLFNYY3ZOYmxfKJwPLAb4BfAEOAPsCVmXlGRKwHXA4sA9wMHJKZqxXXMV5f7DsO2BrYMzOfiogrgE2Bm4DzWh1vMDAZ+HZmfqtYtwxwNpUSeROV8vnH2jqPLUeMzLvvGbt4PhSpRivv+M16D0Fq8e+rv17vIUgA7LvHdjz8wH11idwGr7tJHnDW1eUNF7Of77Ph+Mwc2ZXHbM9jBINKiXjtIrgbDqyamfe2tV9mPgVsUrV8VtXmPRawy2Rg6yIDeCCwQbHfdCrXOy7oGAe3WlV9vBdodX6Z+SbwxbbGLUmStCgaZdak9pSnz6dyA8rOwBlUysLXAB9YzGMZAZxbBKkzgc8t5v4lSZLUQe0JGj+YmVtGxL8BMvPliOi7uAeSmXdRea61JEmSupn2BI2zI6IXlbkZiYhBVDKPkiRJDa9RytPtmXLnZ8AfgVUi4rvA34HvdeqoJEmS1K2UZhoz83cRMR7Yhcqd5R9f0NyGkiRJjSaicabcac/d08OBN6hMe9OyLjOf6cyBSZIkqftozzWNN1K5njGoPIllLeBRYONOHJckSdISoVGuaWxPeXq+58JFxJbAlzttRJIkSep2FvnZ05l5H/DBThiLJEmSuqn2XNN4fNViE7AlMKXTRiRJkrQEaZD7YNp1TeMKVe/nULnG8ZrOGY4kSZK6ozaDxmJS7xUy88QuGo8kSdISI4CmBkk1LvSaxojonZlzgQ914XgkSZLUDbWVabyXyvWL90fEaOAq4PXmjZl5bSePTZIkSd1Ee65pXBqYAezMu/M1JmDQKEmSGt4iT0WzhGoraFyluHP6Yd4NFptlp45KkiRJ3UpbQWMvYHnmDxabGTRKkiThlDsAUzPzjC4biSRJkrqttoLGBombJUmSOiYinHIH2KXLRiFJkqRubaFBY2a+1JUDkSRJ0uIREXtExKMR8XhEnNxGu09GREbEyLI+2zPljiRJkhaiu1Wniyf6nQfsCjwHjI2I0Zk5sVW7FYCvAP9qT7+NMrWQJElSo9gKeDwzn8jMd4Argb0X0O7bwA+Bt9rTqUGjJElSDZqi61/AwIgYV/U6ompIqwHPVi0/V6xrERFbAqtn5o3tPU/L05IkSUue6ZlZeh3igkREE/AT4LBF2c+gUZIkqYMCuuOUO5OB1auWhxXrmq0AbALcEZWxrwqMjoi9MnPcwjq1PC1JktSzjAXWi4i1IqIvcCAwunljZs7KzIGZuWZmrgncA7QZMIJBoyRJUo+SmXOAo4FbgEnAqMycEBFnRMReHe3X8rQkSVINul91GjJzDDCm1bpTF9J2x/b0aaZRkiRJpcw0SpIkddS7U+D0eGYaJUmSVMpMoyRJUg2Cxkg1mmmUJElSKYNGSZIklbI8LUmS1EGVJ8LUexRdw0yjJEmSSplplCRJqoGZRkmSJKlgplGSJKkG0R2fI9gJzDRKkiSplEGjJEmSSlmeliRJ6iCn3JEkSZKqmGmUJEnqqIAGuQ/GTKMkSZLKmWmUJEmqQVODpBrNNEqSJKmUQaMkSZJKWZ6WJEnqIKfckSRJkqqYaZQkSapBg9wHY6ZRkiRJ5cw0SpIkdVjQRGOkGs00SpIkqZRBoyRJkkpZnpYkSeqgwBthJEmSpBZmGiVJkjoqnNxbkiRJamGmUZIkqQZNDXJRo5lGSZIklTJolCRJUinL05IkSR3klDuSJElSFTONkiRJNfBGGEmSJKlgplGSJKkGDZJoNNMoSZKkcgaNkiRJKmV5WpIkqYOCxsnANcp5SpIkqQZmGiVJkjoqIBrkThgzjZIkSSplplGSJKkGjZFnNNMoSZKkdjDTWKLyIPJG+RtC3d3E6/6v3kOQWmy064n1HoIEwNv/ea7eQ2gIBo2SJEkdFPjsaUmSJKmFmUZJkqQaNEae0UyjJEmS2sFMoyRJUg0a5JJGM42SJEkqZ9AoSZKkUpanJUmSOiwaZj5nM42SJEkqZaZRkiSpg4LGycA1ynlKkiSpBmYaJUmSauA1jZIkSVLBoFGSJKmHiYg9IuLRiHg8Ik5ewPYjI+KhiLg/Iv4eERuV9WnQKEmSVIOow6vN8UT0As4D9gQ2Ag5aQFB4RWa+PzM3B34E/KTsPA0aJUmSepatgMcz84nMfAe4Eti7ukFmvlK1uByQZZ16I4wkSVJHRbe8EWY14Nmq5eeAD7ZuFBFHAccDfYGdyzo10yhJkrTkGRgR46peRyxqB5l5XmauA/w/4Jtl7c00SpIkdVAdJ/eenpkjF7JtMrB61fKwYt3CXAn8ouyAZholSZJ6lrHAehGxVkT0BQ4ERlc3iIj1qhY/CvynrFMzjZIkST1IZs6JiKOBW4BewCWZOSEizgDGZeZo4OiI+DAwG3gZOLSsX4NGSZKkGnTDG2HIzDHAmFbrTq16/5VF7dPytCRJkkqZaZQkSapB98szdg4zjZIkSSplplGSJKkG3fCSxk5hplGSJEmlDBolSZJUyvK0JElSB1WeCNMY9WkzjZIkSSplplGSJKkG3ggjSZIkFcw0SpIkdVgQXtMoSZIkVRg0SpIkqZTlaUmSpBp4I4wkSZJUMNMoSZLUQU7uLUmSJFUx0yhJktRR4TWNkiRJUguDRkmSJJWyPC1JklQDy9OSJElSwUyjJElSDXz2tCRJklQw0yhJktRBATQ1RqLRTKMkSZLKGTRKkiSplOVpSZKkGngjjCRJklQw0yhJklQDJ/eWJEmSCmYaJUmSauA1jZIkSVLBoFGSJEmlLE9LkiR1kE+EkSRJkqqYaZQkSeqw8EYYSZIkqZmZRkmSpI4KJ/eWJEmSWhg0SpIkqZTlaUmSpBo0SHXaTKMkSZLKmWmUJEnqoMrk3o2RazTTKEmSpFJmGiVJkmrQGHlGM42SJElqB4NGSZIklbI8LUmSVIsGqU+baZQkSVIpM42SJEk1iAZJNZpplCRJUikzjZIkSTVokLm9zTRKkiSpnEGjJEmSSlmeVk3+/I+JnPLjq5k7bx6f3ntbjjtst/m2v/3ObL502mXc/8gzrNxvOS753ucYPnQAL818jUNPvph/T3yagz62NWeetH+dzkA9yV1jH+H751/H3Hnz2HfPD/KFA3eeb/u4B//L938xmseemMpZ3ziE3bffDIBJj0/mjJ9dy2tvvEWvpia+ePAu7Lnj5nU4A/Uku2yzId8/YV96NTVx2XX/4Ozf3Dbf9mGDV+L80z9NvxWWoVdTE9869zpu+8dE9ttjJMd8+sMt7TZedyg7fPqHPPzY5K4+BbVTg1Snl/ygMSLWBLbNzCs6sO9rmbn84h9VY5g7dx5f+9Eo/nju0Qwd3J+dDz2TPbd/P+9be0hLm8uu+yf9VlyG+/54OtfcOo7Tf34dl3z/cyy1VB++fuTHmPTfKUz679Q6noV6irlz5/Gdn/+Ri354BIMH9uOAo89hp202Yt01Vm1pM2SVlfje1w7g11f9bb59l1m6L98/6UDWHDaIF6fPYt+jzuZDIzdgxeWX6erTUA/R1BScedL+7HP0uUx5YSa3/+Zr3HTnQzz65PMtbU74/B786c/3cck1f2eDtVZl1NlfYrO9T+Oqm8dx1c3jANhonaFcftYXDBjVLfSE8vSawMEL2hARS3xQ3J2Nn/AUa68+kDWHDaRvn958YtctGfO3B+drc9OdD3LQRz8IwN47b8Hfxj5KZrLcMkuxzebrsHTfPvUYunqghx59huFDB7D6kAH07dObPXfcnNv/MWG+NqutujIbrD2UplZXra85bBBrDhsEwCoD+zGg//K8NPO1Lhu7ep4RG6/JE89O5+nJM5g9Zy7X3nYfH9lh0/kbZbLCcksDsOLyy/D89Fnv6eeTu4/g2lvv64ohqxZRh1cd1C1ojIg1I2JSRPwqIiZExK0RsUxErBMRN0fE+Ii4KyLeV7S/NCL2rdq/+Sf6D4DtIuL+iDguIg6LiNERcTvwl4hYPiL+EhH3RcRDEbF3HU63R5o6bRarDV6pZXno4JWYOm3+H3pTXny3Te/evVhx+WV4adbrXTpONYYXps9i1UH9W5ZXHdifFxfwS7jMg488w+zZcxk+dMBiHJ0azZBB/Zj8wssty1NeeJkhg/rN1+YHF45h/z234uEbvs2os7/ESWde9Z5+9tl1S665dVynj1dqj3pnGtcDzsvMjYGZwCeBC4FjMnMEcCJwfkkfJwN3ZebmmfnTYt2WwL6ZuQPwFrBPZm4J7AT8OKJRbo6XtCimzXiFk3/4e7574gE0NdX7x6N6uk/uPpIrbriHTT72f+z/1V9wwbc+Q/WvpxEbr8Gbb832Ep5urpL46/r/1UO9fyo+mZn3F+/HUyk1bwtcFRH3A78Ehixwz7bdlpkvFe8D+F5EPAj8GVgNGNzWzhFxRESMi4hx06ZP68DhG0N7/pIeusq7bebMmcsrr73Jyv2W69JxqjEMHtiP56fNbFl+fvpMVhnYb+E7tPLa629x5Dcv5iuf3YPNNlqjE0aoRtKeSsyn9t6GP/25Unoe+9CTLL1UHwb0f/fn4yd2G8E1t5hlVPdR76Dx7ar3c4GVgZlF1rD5tWGxfQ7FeCOiCejbRr/V9c9DgEHAiMzcHHgBWLqtQWXmhZk5MjNHDho4aJFOqJFsudEa/PeZaTw9eTrvzJ7Dtbfdx57bz3/Nzh7bvZ/f3/gvAK67/d9s/4H1MdGrzrDJBqvz9OTpPDd1Bu/MnsNNd9zPTtts3K5935k9h2NOv5S9dx3Rcke1VIv7Jj7NOsMHMXzoAPr07sUndt2Sm+6c/5rvyc+/xPYf2ACA9dcczFJ9+zD95cqVVxHBxz+8JdfcNr7Lx66eISL2iIhHI+LxiDh5AduPj4iJEfFgcRlf6V/L3e1GkVeAJyNiv8y8qigjb5qZDwBPASOAUcBeQPMdFK8CK7TRZz/gxcycHRE7AaYQFpPevXvxo5P255PHnsfcuckhe23NhusM4XsX3MDmGw7nIztsyqf33pYjT/stW+5zOiutuBwXf/ezLftvutepvPr6W8yePYcxf3uQa35+1Hx3XkuLonevXnzj6H34wim/Yt68ZJ/dP8B6a67Kzy+9mY3XX52dt92Yhx59hmNP/w2vvPYGf71nIuf+9lauv+hr3Py3Bxj/0BPMfOUN/lhkdr73tQPYcN3V6nxWWlLNnTuPk340imt+dhS9egW/G30PjzzxPKd88aPcP+kZbrrzIb559h855xsH8eWDdiKBo751Wcv+226xLpNfeJmnJ8+o30mofaL7PREmInoB5wG7As8BYyNidGZOrGr2b2BkZr4REV8CfgQc0Ga/mdlZY25TMVXODZm5SbF8IrA88BvgF1TK0n2AKzPzjIgYDFwHLAPcDByVmctHRB/gFmAAcCnwMpUP4eii34HA9UXf44CtgT0z86n2TLkzYsTIvPtflgfUPUyd+Va9hyC12GjXE+s9BAmAtx8dxbw3XqxL6LbRplvkZaP/Vt5wMRu5Vr/xmTlyQdsiYhvg9MzcvVg+BSAzv7+Q9lsA52bmh9o6Zt0yjZn5FLBJ1fJZVZv3WED7F6gEfM3+X7F+NrBzq+aXVu03HdhmIWNwjkZJklSTbpZohMr9G89WLT8HfLCN9p8HbirrtLuVpyVJklRuYERUl0IvzMwLF7WTiPgUMBLYoaytQaMkSVIt6pNqnL6w8jQwGVi9anlYsW4+EfFh4BvADpn5duvtrdX77mlJkiQtXmOB9SJirYjoCxwIjK5uUFzH+Etgr8x8sT2dGjRKkiT1IJk5Bziayo3Ck4BRmTkhIs6IiL2KZmdSuUn4quKpeqMX0l0Ly9OSJEkdVr8ntLQlM8cAY1qtO7Xq/YcXtU8zjZIkSSplplGSJKkG3W1y785iplGSJEmlzDRKkiR1UNAtJ/fuFGYaJUmSVMqgUZIkSaUsT0uSJNWiQerTZholSZJUykyjJElSDbrj5N6dwUyjJEmSSplplCRJqoGTe0uSJEkFg0ZJkiSVsjwtSZJUgwapTptplCRJUjkzjZIkSR3VQA+fNtMoSZKkUmYaJUmSauDk3pIkSVLBoFGSJEmlLE9LkiR1UOATYSRJkqQWZholSZJq0CCJRjONkiRJKmemUZIkqRYNkmo00yhJkqRSBo2SJEkqZXlakiSpBj4RRpIkSSqYaZQkSaqBk3tLkiRJBTONkiRJNWiQRKOZRkmSJJUzaJQkSVIpy9OSJEm1aJD6tJlGSZIklTLTKEmS1EGBk3tLkiRJLcw0SpIkdVQ4ubckSZLUwqBRkiRJpSxPS5Ik1aBBqtNmGiVJklTOTKMkSVItGiTVaKZRkiRJpcw0SpIkdVg4ubckSZLUzKBRkiRJpSxPS5Ik1cAnwkiSJEkFM42SJEkdFDTMjDtmGiVJklTOTKMkSVItGiTVaKZRkiRJpQwaJUmSVMrytCRJUg18IowkSZJUMNMoSZJUAyf3liRJkgpmGiVJkmrQIIlGM42SJEkqZ9AoSZLUw0TEHhHxaEQ8HhEnL2D79hFxX0TMiYh929On5WlJkqSOiu53I0xE9ALOA3YFngPGRsTozJxY1ewZ4DDgxPb2a9AoSZLUs2wFPJ6ZTwBExJXA3kBL0JiZTxXb5rW3U8vTkiRJNYk6vNq0GvBs1fJzxbqamGmUJEla8gyMiHFVyxdm5oWdeUCDRkmSpA4K6nZN4/TMHLmQbZOB1auWhxXramJ5WpIkqWcZC6wXEWtFRF/gQGB0rZ0aNEqSJPUgmTkHOBq4BZgEjMrMCRFxRkTsBRARH4iI54D9gF9GxISyfi1PS5Ik1aCbzbgDQGaOAca0Wndq1fuxVMrW7WamUZIkSaXMNJa4777x05fpE0/Xexw9wEBger0HIeF3Ud2L38fFY416Hry7Te7dWQwaS2TmoHqPoSeIiHFt3OUldRm/i+pO/D5qSWLQKEmSVIPollc1Ln5e0yhJkqRSBo3qKp06S720CPwuqjvx+6glhuVpdYnOfrSR1F5+F9Wd+H3sIRqjOm2mUZIkSeXMNEqSJNWgQRKNZhrVuSLiyIj4TPH+sIgYWrXtoojYqH6jU6OLiP4R8eWq5aERcXU9x6TGExFrRsTBHdz3tcU9HmlhDBrVqTLzgsz8bbF4GDC0atvhmTmxLgOTKvoDLUFjZk7JzH3rNxw1qDWBBQaNEWFFsJuLqM+rHgwatVDFX7+PRMTvImJSRFwdEctGxC4R8e+IeCgiLomIpYr2P4iIiRHxYEScVaw7PSJOjIh9gZHA7yLi/ohYJiLuiIiRRTbyzKrjHhYR5xbvPxUR9xb7/DIietXjs1B9FN/BSRHxq4iYEBG3Ft+ddSLi5ogYHxF3RcT7ivbrRMQ9xXfzO81ZmIhYPiL+EhH3Fdv2Lg7xA2Cd4vt1ZnG8h4t97omIjavG0vx9Xa743t9b/Hewd+txqzF04Pt5afGzsHn/5izhD4Dtiu/hccXPwNERcTvwlza+v1KXMmhUmQ2A8zNzQ+AV4HjgUuCAzHw/letivxQRA4B9gI0zc1PgO9WdZObVwDjgkMzcPDPfrNp8TbFvswOAKyNiw+L9hzJzc2AucMjiP0V1c+sB52XmxsBM4JNUpik5JjNHACcC5xdtzwHOKb6bz1X18RawT2ZuCewE/DgiAjgZ+G/xnfxaq+P+AdgfICKGAEMycxzwDeD2zNyq6OvMiFhucZ+0lhiL8v1cmJOBu4rv4U+LdVsC+2bmDiz8+yt1KdPeKvNsZt5dvL8c+D/gycx8rFj3G+Ao4FwqP9gujogbgBvae4DMnBYRT0TE1sB/gPcBdxf9jgDGFj8flwFerP2UtIR5MjPvL96Pp1LK2xa4qur35lLF/28DfLx4fwVwVvE+gO9FxPbAPGA1YHDJcUcBtwKnUQkem6913A3YKyJOLJaXBoYDkxbttNRDLMr3c1HclpkvFe8X9v19voNj1mLWKE+EMWhUmWy1PBMY8J5GmXMiYitgF2Bf4Ghg50U4zpVUfjE/AvwxM7P4S/o3mXlKRwauHuPtqvdzqfyynFlkn9vrEGAQMCIzZ0fEU1SCvYXKzMkRMSMiNqWS8T6y2BTAJzPz0UU4vnquRfl+zqGo8EVEE9C3jX5fr3q/yN9fqTNYnlaZ4RGxTfH+YCol5jUjYt1i3aeBv0XE8kC/zBwDHAdstoC+XgVWWMhx/gjsDRxEJYAE+Auwb0SsAhARK0fEGrWekJZ4rwBPRsR+AFHR/H27h0p5EODAqn36AS8Wv3B3Apq/R219J6FSoj6Jynf7wWLdLcAxzeXBiNii1hNSj9LW9/MpKtUTgL2APsX7su/hwr6/6i6iDq86MGhUmUeBoyJiErAS8FPgs1RKLw9RKZVcQOUH3g0R8SDwdyrXPrZ2KXBB840w1Rsy82Uq5b01MvPeYt1E4JvArUW/twFDFv8pagl0CPD5iHgAmEDlDw6ArwLHF9+XdYFZxfrfASOL7+xnqGS0ycwZwN0R8XD1zVhVrqYSfI6qWvdtKr/sH4yICcWyVG1h389fATsU67fh3Wzig8DciHggIo5bQH8L/P5KXS0yW1cfpYqIWBO4ITM3qfdYpPaIiGWBN4vLGw4EDspM7zSV1Gk233JE/vnOf3X5cQet0Gd8Zo7symN6TaOknmQEcG5ROp4JfK6+w5GknsOgUQuVmU8BZhm1xMjMu1jw9bSSpBoZNEqSJNWgUWbN9EYYSZIklTLTKEmS1GHRMJN7m2mU1KkiYm4xzdLDEXFVcYdzR/tqeXZvRFwUERu10XbHiNi2A8d4KiIGtnd9qzavtbV9Ae1Pr3qyjCR1awaNkjrbm8UzdTcB3uHdJ6sAEBEdqnhk5uHFXJ4LsyOVx7lJUqcJKtc0dvWrHgwaJXWlu4B1iyzgXRExGpgYEb0i4syIGBsRD0bEF6HlaRrnRsSjEfFnYJXmjiLijogYWbzfIyLuKyZH/ksxx+iRwHFFlnO7iBgUEdcUxxgbER8q9h0QEbdGxISIuIh2PGshIv4UEeOLfY5ote2nxfq/RMSgYt06EXFzsc9dEfG+xfJpSlIX8ppGSV2iyCjuCdxcrNoS2CQznywCr1mZ+YGIWIrKU1puBbYANgA2ovJM34nAJa36HUTlSRvbF32tnJkvRcQFwGuZeVbR7grgp5n594gYTuVxgBsCpwF/z8wzIuKjwOfbcTqfK46xDDA2Iq4pni6zHDAuM4+LiFOLvo8GLgSOzMz/RMQHgfNZtGezS1LdGTRK6mzLRMT9xfu7gIuplI3vzcwni/W7AZs2X69I5Vm76wHbA7/PzLnAlIi4fQH9bw3c2dxXZr60kHF8GNgo3q3rrBiVZ6ZvD3yi2PfGiHi5Hed0bETsU7xfvRjrDCqP1fxDsf5y4NriGNtSefRm8/5LteMYktStGDRK6mxvZubm1SuK4On16lXAMZl5S6t2H1mM42gCts7MtxYwlnaLiB2pBKDbZOYbEXEHsPRCmmdx3JmtPwNJWtJ4TaOk7uAW4EsR0QcgItaPiOWAO4EDimsehwA7LWDfe4DtI2KtYt+Vi/WvAitUtbsVOKZ5ISI2L97eCRxcrNsTWKlkrP2Al4uA8X1UMp3NmoDmbOnBVMrerwBPRsR+xTEiInxqjdSDeCOMJHWdi6hcr3hfRDwM/JJKJeSPwH+Kbb8F/tl6x8ycBhxBpRT8AO+Wh68H9mm+EQY4FhhZ3GgzkXfv4v4WlaBzApUy9TMlY70Z6B0Rk4AfUAlam70ObFWcw87AGcX6Q4DPF+ObAOzdjs9EkrqVyMx6j0GSJGmJtMWWI/OOu+/t8uP2X7bX+Mwc2ZXHNNMoSZKkUgaNkiRJKuXd05IkSR1VxxtTupqZRkmSJJUy0yhJktRBQTuePdpDmGmUJElSKTONkiRJtWiQVKOZRkmSJJUyaJQkSVIpy9OSJEk1iAapT5tplCRJUikzjZIkSTVwcm9JkiSpYKZRkiSpBg2SaDTTKEmSpHIGjZIkSSpleVqSJKkWDVKfNtMoSZKkUmYaJUmSauDk3pIkSVLBTKMkSVIHBU7uLUmSJLUwaJQkSVKpyMx6j0GSJGmJFBE3AwPrcOjpmblHVx7QoFGSJEmlLE9LkiSplEGjJEmSShk0SpIkqZRBoyRJkkoZNEqSJKnU/we+AfKtd4VTzgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 792x792 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_preds = np.argmax(output_predictions.predictions, axis=1)\n",
    "names_of_labels = label_names\n",
    "\n",
    "plot_confusion_matrix(y_preds, y_valid, names_of_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push Fine-Tuned Model to HuggingFace Hub (My Profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.push_to_hub(commit_message=\"All Dunn!!!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save Datasets to File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataset['train'].to_csv('~/Documents/nlpnn/Datasets/Finanacial Sentiment Analysis/train_ds.csv')\n",
    "#dataset['test'].to_csv('~/Documents/nlpnn/Datasets/Finanacial Sentiment Analysis/test_ds.csv')\n",
    "#dataset['valid'].to_csv('~/Documents/nlpnn/Datasets/Finanacial Sentiment Analysis/eval_ds.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('nlp_ch4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a52fe47989fdc78fafbb981021cec52a6b82df6453830b9ffbd04250493e6cab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}