Sentiment Analysis/Financial Sentiment Analysis/Financial Sentiment Analysis-Updated Version.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentiment Analysis of Statements about Finances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saved as 'distilbert-base-uncased-Finanacial_Sentiment_Analysis_5e' on my HuggingFace profile\n",
    "\n",
    "Source for the Datasets: \n",
    "\n",
    "*https://www.kaggle.com/datasets/sbhatti/financial-sentiment-analysis\n",
    "\n",
    "*https://www.kaggle.com/datasets/ankurzing/sentiment-analysis-for-financial-news"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploratory Data Analysis & Preprocessing in Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Importing the Necessary Libraries for the Entire Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "NOTE: Redirects are currently not supported in Windows or MacOs.\n"
     ]
    }
   ],
   "source": [
    "import os, sys\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import AdamW\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "import datasets\n",
    "from datasets import Dataset, DatasetDict, load_metric\n",
    "\n",
    "import transformers\n",
    "from transformers import AutoModelForSequenceClassification, AutoConfig, AutoTokenizer, AutoModel\n",
    "from transformers import TrainingArguments, Trainer, AdamW, get_scheduler, set_seed, DataCollatorWithPadding\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score, f1_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.dummy import DummyClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Versions of Libraries Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pandas:            1.4.3\n",
      "NumPy:             1.22.4\n",
      "Torch:             1.12.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Pandas:\".ljust(18), pd.__version__)\n",
    "print(\"NumPy:\".ljust(18), np.__version__)\n",
    "print(\"Torch:\".ljust(18), torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest First Dataset & Preprocess it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 5836 entries, 0 to 5841\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    5836 non-null   object\n",
      " 1   label   5836 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 136.8+ KB\n"
     ]
    }
   ],
   "source": [
    "data1 = pd.read_csv('~/Documents/nlpnn/Datasets/Finanacial Sentiment Analysis/data.csv', engine=\"c\")\n",
    "\n",
    "data1.dropna(inplace=True)\n",
    "\n",
    "data1['Sentence'] = data1['Sentence'].str.strip()\n",
    "data1['Sentiment'] = data1['Sentiment'].str.strip()\n",
    "\n",
    "data1.rename(columns={'Sentence' : 'text', 'Sentiment' : 'label'}, inplace=True)\n",
    "data1.drop_duplicates(inplace=True)\n",
    "data1.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest Second Dataset & Preprocess it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 4840 entries, 0 to 4845\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   label   4840 non-null   object\n",
      " 1   text    4840 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 113.4+ KB\n"
     ]
    }
   ],
   "source": [
    "data2 = pd.read_csv('~/Documents/nlpnn/Datasets/Finanacial Sentiment Analysis/all-data.csv', \n",
    "                    encoding_errors='ignore', \n",
    "                    engine=\"c\", \n",
    "                    header=None, \n",
    "                    names=['label', 'text'])\n",
    "\n",
    "data2['label'] = data2['label'].str.strip()\n",
    "data2['text'] = data2['text'].str.strip()\n",
    "\n",
    "data2.drop_duplicates(inplace=True)\n",
    "data2.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Combine Both Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10676 entries, 0 to 10675\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    10676 non-null  object\n",
      " 1   label   10676 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 166.9+ KB\n"
     ]
    }
   ],
   "source": [
    "data = pd.concat([data1, data2]).reset_index(drop=True)\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Histogram of Text Lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEWCAYAAACdaNcBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ4klEQVR4nO3dfZRcVZnv8e/PAOElQAiJmZhEmpcIA4yAZkEABzPgKG8jzCxFcUYCkzU4I86FK16B8d65OOqd5C4Bucs7IAIKwkAwyoDRETDAeHmnw3sISJRgEgMJkPAiyhh87h97l54U1V2V7uqu6t2/z1pn9Tl7n9rnqVPdT+3a59RuRQRmZlaWt3Q6ADMzaz8ndzOzAjm5m5kVyMndzKxATu5mZgVycjczK5CTe8EkLZU0u9NxdJKkP5e0UtKrkg7odDwGkmZLWtXpOErn5D5CSVoh6X11ZSdLuqO2HRH7RMTtTdrpkRSSthiiUDvty8CnImJcRDxYXynpOEkPSXpZ0vOSbpW062APKulcSVcNtp12y6/1HqUf06DUP2jrEpK2iIiNHQxhF2Bpo4qccK4E/gK4FRgHvB94Y9iiMxsi7rkXrNq7l3SgpN7cQ31O0vl5tx/nnxvy0MXBkt4i6b9LekbSWklXStqx0u5Jue4FSf+j7jjnSloo6SpJLwMn52PfLWmDpDWSvippq0p7IemTkp6S9IqkL0jaXdJdOd7rqvvXPceGsUoaK+lVYAzwsKSfNnj4/sDTEbE4klci4jsR8fNK22dL+ml+rtdJmpDrap945kj6ee71fy7XHQn8A/CRfE4fzuU7Srosn4PVkr4oaUyuO1nSHZK+LGm9pKclHVV5nhMkfUPSL3L9v1Xqjs2fPjbkc/bOFn9FqudxbD72z/Pvx8WStsl1syWtknRmPsdrJJ1SeezOkr6XX6v78/O6I9fVfr8ezufiI5XHNWzP2iQivIzABVgBvK+u7GTgjkb7AHcDH8/r44BZeb0HCGCLyuP+GlgO7Jb3/S7wrVy3N/Aq8B5gK9Kwx28qxzk3bx9P6jxsA7wbmEX6pNgDLAPOqBwvgBuAHYB9gNeBxfn4OwKPA3P6OA99xlppe48+Hrsb8GvgAuBPgHF19acD9wDTgLHA14Br6s7b1/Nz3C/H/YeV83BVXXvX5za2A94K3Ad8ovLa/Qb4G9Ib0t8BvwCU678PLAB2ArYE3pvLDwDWAgflx83Jr/vYPp5zw/ORz8GNwARge+B7wD/nutnARuCf8rGPBl4Ddsr11+Zl2/z7sZJNfw83OWaz9ry0KUd0OgAvA3zh0h/wq8CGyvIafSf3HwOfBybWtVNLUtXkvhj4ZGV7z5x4tgD+sZbgct22wH+yaXL/cZPYzwCur2wHcGhlewlwVmX7POArfbTVZ6yVthsm91w/C7gOWEdK9N8kJ3nSm9ARlX2nVM5D7bxNq9TfB3y0ch6uqtRNJiX/bSplJwK35fWTgeV15zWAP8jH/W2j5AdcBHyhruxJcvJvsP+bzgcg4JfA7pWyg0mfaiAl41/V/Y6szeduTD4ne1bqvkjz5N6wvU7/XZW0eFhmZDs+IsbXFuCT/ew7F3gH8ET+6HxsP/u+DXimsv0MKaFNznUraxUR8RrwQt3jV1Y3JL1D0iJJz+ahmv8FTKx7zHOV9V812B43gFibioh7IuKEiJgE/DFwGPC5XL0LcH0e7thASvZv1LX9bGX9tX7i3IXUS11Tae9rpB78m9rK55Xc3nTgxYhY30e7Z9bazO1OJ52XVk0ivZksqbTxw1xe80Jseu2k9lwnkc539TXf5PXvQ1/tWZs4uY8SEfFURJxISibzgYWStiP1qur9gpQ0at5O+hj9HLCGNEwBQB6X3bn+cHXbFwFPADMiYgfSeLQG/mxajnWzRMT9pGGdfXPRSuCo6htoRGwdEatbaa5ueyWp5z6x0tYOEbFPC22tBCZIGt9H3ZfqYtw2Iq5pod2a50lvoPtU2tgxIlpJtutI53tapWz6ZhzbhoiT+ygh6a8kTYqI35KGcCB91F+Xf+5W2f0a4L9K2lXSOFJPe0HuaS0E/kzSIfki57k0T9TbAy8Dr0raizSe3C79xdovSe+R9DeS3pq39wI+SBpnB7gY+JKkXXL9JEnHtRjXc0CPpLcARMQa4GbgPEk75Iu1u0t6b7OG8mP/HfgXSTtJ2lLSYbn668DfSjpIyXaSjpG0fT9NbiVp69pCev2+DlxQORdTJX2ghdjeIL0hnitp23wOT2pwLnZ704NtSDm5jx5HAkuV7iC5kDQ2/Kv88f9LwJ35I/ks4HLgW6Rx+qdJY9F/DxARS/P6taRe/Kuk8dLX+zn2Z4CPAa+QksiCNj6vPmNtwQZSMn80n5cfki56/u9cfyHpIuPNkl4hJf2DWmz72/nnC5IeyOsnkS5CPw6sJ71RTmmxvY+TxrafIJ3vMwAiopd0Efaruc3lpPH7/iwl9dRryynAWfmx9+Shsx+Rrl+04lOkC9/Pkl6La9j09+Fc4Ir8+3VCi23aINWuxJsNSO4tbyANuTzd4XCsC0iaD/xBRMzpdCyjmXvuttkk/Vn+CL4d6VbIR0l35tgoJGkvSe/Mw0IHki7eX9/puEY7J3cbiONIFzJ/AcwgDfH4I+DotT1p3P2XpCG380jfW7AO8rCMmVmB3HM3MytQV0wcNnHixOjp6el0GGZmI8qSJUuez1/Ae5OuSO49PT309vZ2OgwzsxFF0jN91XlYxsysQE7uZmYFcnI3MyuQk7uZWYGc3M3MCuTkbmZWICd3M7MCObmbmRXIyd3MrEBd8Q1Va6+es7+/yfaKecd0KBIz6xT33M3MCuSeewHqe+pmZu65m5kVyMndzKxATu5mZgVycjczK5CTu5lZgXy3zCjk++DNyueeu5lZgZzczcwK5GEZa8hDN2Yjm3vuZmYFcnI3MyuQh2VGgMHOHeO5Z8xGH/fczcwK5ORuZlYgJ3czswI5uZuZFcgXVM0XXM0K5J67mVmBnNzNzArk5G5mVqCWk7ukMZIelLQob+8q6V5JyyUtkLRVLh+bt5fn+p4hit3MzPqwOT3304Flle35wAURsQewHpiby+cC63P5BXk/MzMbRi0ld0nTgGOAS/O2gMOBhXmXK4Dj8/pxeZtcf0Te38zMhkmrPfevAJ8Ffpu3dwY2RMTGvL0KmJrXpwIrAXL9S3n/TUg6VVKvpN5169YNLHozM2uoaXKXdCywNiKWtPPAEXFJRMyMiJmTJk1qZ9NmZqNeK19iOhT4oKSjga2BHYALgfGStsi982nA6rz/amA6sErSFsCOwAttj9zMzPrUNLlHxDnAOQCSZgOfiYi/lPRt4EPAtcAc4Ib8kBvz9t25/taIiLZHXjB/Y9TMBmsw97mfBXxa0nLSmPplufwyYOdc/mng7MGFaGZmm2uz5paJiNuB2/P6z4ADG+zza+DDbYjNzMwGyN9QNTMrkJO7mVmBnNzNzArk+dy7wEi4O6Y+xhXzjulQJGbWCvfczcwK5ORuZlYgJ3czswJ5zH0YlDheXeJzMiuJe+5mZgVycjczK5CTu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYF8jdUO2AkzAJpZiObe+5mZgVyz93awnPNmHUX99zNzArk5G5mViAndzOzAnnM3YaEx+DNOss9dzOzAjm5m5kVyMndzKxATu5mZgVycjczK5CTu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYFaprcJW0t6T5JD0taKunzuXxXSfdKWi5pgaStcvnYvL081/cM8XMwM7M6rfTcXwcOj4j9gP2BIyXNAuYDF0TEHsB6YG7efy6wPpdfkPczM7Nh1DS5R/Jq3twyLwEcDizM5VcAx+f14/I2uf4ISWpXwGZm1lxLY+6Sxkh6CFgL3AL8FNgQERvzLquAqXl9KrASINe/BOzcoM1TJfVK6l23bt2gnoSZmW2qpX/WERFvAPtLGg9cD+w12ANHxCXAJQAzZ86MwbbXTer/UYWZ2XDbrLtlImIDcBtwMDBeUu3NYRqwOq+vBqYD5PodgRfaEayZmbWmlbtlJuUeO5K2Af4UWEZK8h/Ku80BbsjrN+Ztcv2tEVFUz9zMrNu1MiwzBbhC0hjSm8F1EbFI0uPAtZK+CDwIXJb3vwz4lqTlwIvAR4cgbjMz60fT5B4RjwAHNCj/GXBgg/JfAx9uS3RmZjYg/oaqmVmBnNzNzArU0q2QZoNVf3voinnHdCgSs9HByd06wsnebGh5WMbMrEDuuVtXcE/erL3cczczK5CTu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYFcnI3MyuQk7uZWYGc3M3MCuTkbmZWICd3M7MCObmbmRXIU/62Qf10tWZmneaeu5lZgZzczcwK5ORuZlYgJ3czswL5gqp1pWYXqf0/Vs365567mVmBnNzNzArk5G5mViAndzOzAjm5m5kVyHfL2IhUfzeN754x25R77mZmBXJyNzMrkJO7mVmBnNzNzArUNLlLmi7pNkmPS1oq6fRcPkHSLZKeyj93yuWS9H8kLZf0iKR3DfWTMDOzTbXSc98InBkRewOzgNMk7Q2cDSyOiBnA4rwNcBQwIy+nAhe1PWozM+tX01shI2INsCavvyJpGTAVOA6YnXe7ArgdOCuXXxkRAdwjabykKbkdsyHhWyPNNrVZY+6SeoADgHuByZWE/SwwOa9PBVZWHrYql5mZ2TBpOblLGgd8BzgjIl6u1uVeemzOgSWdKqlXUu+6des256FmZtZES8ld0pakxH51RHw3Fz8naUqunwKszeWrgemVh0/LZZuIiEsiYmZEzJw0adJA4zczswZauVtGwGXAsog4v1J1IzAnr88BbqiUn5TvmpkFvOTxdjOz4dXK3DKHAh8HHpX0UC77B2AecJ2kucAzwAm57gfA0cBy4DXglHYGbGZmzbVyt8wdgPqoPqLB/gGcNsi4zMxsEPwNVTOzAjm5m5kVyMndzKxATu5mZgVycjczK5CTu5lZgZzczcwK5H+QbaOCZ4200cY9dzOzAjm5m5kVyMndzKxAHnO3Uclj8FY699zNzArk5G5mViAndzOzAjm5m5kVyMndzKxATu5mZgXyrZBWpPpbHc1GG/fczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQE7uZmYFcnI3MyuQ73MfAN9DbWbdzj13M7MCObmbmRXIyd3MrEBO7mZmBXJyNzMrkJO7mVmBfCukWYua3QK7Yt4xwxSJWXNO7ma8OXE7UdtI52EZM7MCObmbmRWo6bCMpMuBY4G1EbFvLpsALAB6gBXACRGxXpKAC4GjgdeAkyPigaEJ3ay7eGjHukkrY+7fBL4KXFkpOxtYHBHzJJ2dt88CjgJm5OUg4KL802zUc/K34dR0WCYifgy8WFd8HHBFXr8COL5SfmUk9wDjJU1pU6xmZtaigY65T46INXn9WWByXp8KrKzstyqXvYmkUyX1Supdt27dAMMwM7NGBn0rZESEpBjA4y4BLgGYOXPmZj/ebCh5Wmcb6Qbac3+uNtySf67N5auB6ZX9puUyMzMbRgNN7jcCc/L6HOCGSvlJSmYBL1WGb8zMbJi0civkNcBsYKKkVcD/BOYB10maCzwDnJB3/wHpNsjlpFshTxmCmM2K4LtnbCg1Te4RcWIfVUc02DeA0wYblJmZDY6/oWpmViAndzOzAjm5m5kVyFP+tsD3PJvZSOOeu5lZgZzczcwK5ORuZlYgJ3czswI5uZuZFch3y5gNEd9lZZ3knruZWYGc3M3MCuRhGbMu4VkirZ2c3M1GCCd/2xweljEzK5CTu5lZgZzczcwK5DH3Bnx/spmNdE7uZl3KnQwbDA/LmJkVyD13sxHKt0Zaf5zczQrhZG9VTu5mo4ST/+ji5I4vXJlZeXxB1cysQE7uZmYFcnI3MyuQx9zNCuVrSaObk7vZKOW7Z8o26pK7ezNmNhp4zN3MrEBO7mZmBRp1wzJm1pjH4MvinruZWYGc3M3MClT8sIzvjjGz0WhIkrukI4ELgTHApRExbyiOY2ZDp9kYfKOOU7N9PI4/fNqe3CWNAf4v8KfAKuB+STdGxOPtPpaZDZ9WPgX7k3L3GIqe+4HA8oj4GYCka4HjACd3M+vX5n5a8CeBvg1Fcp8KrKxsrwIOqt9J0qnAqXnzVUlPDvB4E4HnB/jY4dLtMXZ7fOAY26Hj8Wl+0102ibHZ/i20124dP4d1dumromMXVCPiEuCSwbYjqTciZrYhpCHT7TF2e3zgGNuh2+OD7o+x2+OrGopbIVcD0yvb03KZmZkNk6FI7vcDMyTtKmkr4KPAjUNwHDMz60Pbh2UiYqOkTwE3kW6FvDwilrb7OBWDHtoZBt0eY7fHB46xHbo9Puj+GLs9vt9RRHQ6BjMzazNPP2BmViAndzOzAo3o5C7pSElPSlou6exOxwMg6XJJayU9VimbIOkWSU/lnzt1ML7pkm6T9LikpZJO78IYt5Z0n6SHc4yfz+W7Sro3v94L8gX7jpE0RtKDkhZ1aXwrJD0q6SFJvbmsm17n8ZIWSnpC0jJJB3dZfHvmc1dbXpZ0RjfF2J8Rm9wr0xwcBewNnChp785GBcA3gSPrys4GFkfEDGBx3u6UjcCZEbE3MAs4LZ+3borxdeDwiNgP2B84UtIsYD5wQUTsAawH5nYuRABOB5ZVtrstPoA/iYj9K/dmd9PrfCHww4jYC9iPdC67Jr6IeDKfu/2BdwOvAdd3U4z9iogRuQAHAzdVts8Bzul0XDmWHuCxyvaTwJS8PgV4stMxVmK7gTQPUFfGCGwLPED6lvPzwBaNXv8OxDWN9Id9OLAIUDfFl2NYAUysK+uK1xnYEXiafFNHt8XXIN73A3d2c4z1y4jtudN4moOpHYqlmckRsSavPwtM7mQwNZJ6gAOAe+myGPOQx0PAWuAW4KfAhojYmHfp9Ov9FeCzwG/z9s50V3wAAdwsaUme7gO653XeFVgHfCMPbV0qabsuiq/eR4Fr8nq3xriJkZzcR6RIb/cdv/9U0jjgO8AZEfFyta4bYoyINyJ9HJ5Gmoxur07GUyXpWGBtRCzpdCxNvCci3kUaujxN0mHVyg6/zlsA7wIuiogDgF9SN7zRDb+HAPnayQeBb9fXdUuMjYzk5D6Spjl4TtIUgPxzbSeDkbQlKbFfHRHfzcVdFWNNRGwAbiMNc4yXVPviXSdf70OBD0paAVxLGpq5kO6JD4CIWJ1/riWNFR9I97zOq4BVEXFv3l5ISvbdEl/VUcADEfFc3u7GGN9kJCf3kTTNwY3AnLw+hzTO3RGSBFwGLIuI8ytV3RTjJEnj8/o2pGsCy0hJ/kN5t47FGBHnRMS0iOgh/d7dGhF/2S3xAUjaTtL2tXXSmPFjdMnrHBHPAisl7ZmLjiBNC94V8dU5kd8PyUB3xvhmnR70H+RFjqOBn5DGYz/X6XhyTNcAa4DfkHonc0njsYuBp4AfARM6GN97SB8jHwEeysvRXRbjO4EHc4yPAf+Yy3cD7gOWkz4ij+2C13s2sKjb4suxPJyXpbW/jy57nfcHevPr/G/ATt0UX45xO+AFYMdKWVfF2Nfi6QfMzAo0kodlzMysD07uZmYFcnI3MyuQk7uZWYGc3M3MCuTkbkNC0gWSzqhs3yTp0sr2eZI+PcC2Z9dmYhxukiZLWpRnrHxc0g8G0dbJkt7WzvjMapzcbajcCRwCIOktwERgn0r9IcBdrTSUZwDtiMo3Tmv+CbglIvaLNLPmYGYEPBlwcrch4eRuQ+Uu0pQBkJL6Y8ArknaSNBb4Q+ABSUfkiaMeVZoLfyz8bi7y+ZIeAD6sNHf/E3n7L2oHkfTeynzbD9a+lVmp78mPuzrPGb5Q0ra57t2S/iNPrHVT5Svlt0v6Sp4D/fS65zWF9OU0ACLikcqx/puk+yU9ot/PQd+Tj/t1pbnpb5a0jaQPATOBq3Ps2zSJZ77SHPc/kfTHuXyMpC9Leiwf8+/7e142ynT6W1Reyl1IU7q+HfgE8LfAF0jfhj0U+H/A1qSZPd+R97+SNJEZpOlqP5vXa/vNIE2tex2//1bo94BD8/o48pS7lRh6SN/Ire1zOfAZYEvSG9CkXP4R0j9zB7gd+Jc+ntMHgA2kqQY+B7wtl7+f9M+TReo0LQIOy8ffCOyf97sO+KvKcWbm9WbxnJfXjwZ+lNf/jjQnS22a4Qn9teNldC31HznN2uku0vDLIcD5pClwDwFeIg3b7Ak8HRE/yftfAZxGmk4XYEH+uVfe7ykASVcBtSls7wTOl3Q18N2I+F2vumJlRNyZ168C/gvwQ2Bf4JY03Q5jSNNG1CyggYi4SdJupH/IchTwoKR9Scn9/aRpEyC90cwAfp5jfyiXLyEl/Hp7NomnNsFb9fHvAy6OPM1wRLyYY+mvHRslnNxtKNXG3f+INCyzEjgTeBn4RguP/2WzHSJinqTvk3q0d0r6QEQ8Ub9bg20BSyPiYBrr89gR8SLwr8C/5gu7h+X2/jkivlbdV2nO/NcrRW8A2zRotlk8tTbeoP+/22bt2CjhMXcbSncBxwIvRpqf/UVgPGks/i7Sf7TpkbRH3v/jwH80aOeJvN/uefvEWoWk3SPi0YiYT5optNG872+XVEt2HwPuyMeeVCuXtKWkfRo8dhOSDq+M2W8P7E7qnd8E/LXSPPlImirprU2aewWoXSMYSDy3AJ+oXfSVNGGgz8vK4+RuQ+lR0l0y99SVvRQRz0fEr4FTgG9LepT0X40urm8k73cq8P18QbU6f/YZtQuKpJk4/71BHE+S/lnFMtLMgxdFxH+SpuedL+lh0uyYh7TwnN4N9Obj3Q1cGhH3R8TNpN783fm5LOT3ibsv3wQuVvqPU2MGEM+lpDeWR/JjPjaI52WF8ayQVrQ8LLIoIvbtdCxmw8k9dzOzArnnbmZWIPfczcwK5ORuZlYgJ3czswI5uZuZFcjJ3cysQP8fbVVsCL4ABSkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['Sentence Length'] = data['text'].str.split().apply(len)\n",
    "n=75\n",
    "data.hist('Sentence Length', grid=False, bins=n, range=(0,n))\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Histogram of Sentence Length')\n",
    "plt.ylabel('')\n",
    "plt.xlabel('Words per Sentence')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize: Description Lengths By Output/Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAEHCAYAAAAu3HtHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAc6klEQVR4nO3debhkVXnv8e9PEFBQoKElDDatghr0Crm2KGiiwUTNDYpRZBAQlQS5GhPFIeqNiYnG4KOJ8WqUYCBgHAA1PAJOINDGAYeGKIOIEgQBGVqgFRSRhvf+sde5FM2pc+p0d1V1nfP9PM9+au+1h/VW1a4+b6+99tqpKiRJkjQ5HjDuACRJkjQ3JnCSJEkTxgROkiRpwpjASZIkTRgTOEmSpAljAidJkjRhTOAkDV2SQ5KctQ77H5vkreszpg1JkuVJ/njAbZ+R5NphxyRpw2YCJ81jSa5KckeS25KsSvL1JEclGelvv6o+VlXPGmTbJC9N8tU19j+qqt6+vuNK8rYkH13fx93Q6uwTx+eT3N6mu5L8umf52HHHJ2lmG487AElD99yq+lKSLYGnA+8Dngy8bBSVJ9m4qlaPoi4Nrqr+YGo+yYnAtVX1l+OLSNJc2AInLRBV9bOqOh04EDg8yeMBkmya5D1Jfpzkxna58kFt3bZJzmytd7ck+cpU612Shyf5jyQrk9yc5AOt/KVJvpbkvUluBt62ZqtakkryZ0muTPLTJO9O8oAkvwkcC+zVWoJWte1PTPKOnv3/JMkVLabTk+ywxrGPSvLDFvc/J8lcP68kT2ktlquSfDfJM3rWLU/y9vY+b0tyVpJte9a/JMnV7XN5a2sJ/b0kzwHeAhzY3t93e6rcud/x+sT3lvbZXZXkkFb2pPYdbtSz3QvWqGe29/3ZJK9eo+yiJH/U5qf97nq2fXmSy5LcmuSLSXYetG5JgzOBkxaYqvoWcC3w263oGODRwB7ALsCOwF+1da9r2y4GtqNLPqolCGcCVwNL2z4n91TzZODKts/f9Qnlj4BlwP8E9gNeXlWXAUcB51fVFlW11Zo7JdkH+HvgAGD7FsPJa2y2L/Ak4Altu2f3/UCmkWRH4LPAO4BFwOuBTydZ3LPZi+laMR8GbNK2IcluwAeBQ1p8W9J9PlTVF4B3Aqe097f7bMfr4zeAbdtxDweOS/KYqvo2cDPQe7n6MOAjc3j7JwGHTi0k2b3V89mebe733bVt96M7R15Ad858BfjEHOqWNCATOGlh+gmwqLVMHQm8tqpuqarb6BKMg9p2d9ElITtX1V1V9ZXqHqC8J7AD8Iaq+kVV/aqqevut/aSq3l9Vq6vqjj4xvKvV+WPgn4CDB4z9EOCEqrqwqu4E3kzXYre0Z5tjqmpVO/Z5dMnpXBwKfK6qPldV91TV2cAK4H/1bPNvVfWD9v5O7aljf+CMqvpqVf2aLhke5KHT/Y7Xz1ur6s6q+jJdcnVAK///CViSRXTJ68cHqH/K6cCjk+zalg+jSzh/3bNNv+/uKODvq+qydtn8ncAetsJJ658JnLQw7QjcQtdK8mDggnapcBXwhVYO8G7gCuCsdsnsTa384cDVM/Rtu2aAGHq3uZouIRzEDm17AKrqdrpWpx17trmhZ/6XwBYDHnvKzsCLpj6T9rk8jS6Zna2OHeh5b1X1yxbfbOYS861V9Yue5d7P76PAc5NsTpfUfaWqrh+g/ql4fwWcAhzaLo0eDPz7Gpv1++52Bt7X85ndAoT7fjeS1gNvYpAWmCRPovuD+lXgp8AdwOOq6ro1t20tcq8DXtf6zJ2b5Nt0f8CXzHCDwiAtTg8HLm3zS+haBQfZ9yd0icLU+9kc2Aa4X/zr4Brg36vqT9Zi3+uBx0wttP6E2/SsH+Szmc3WSTbvSeKWAJcAVNV1Sc6nu4x5GPChtTj+SXRJ21eBX1bV+Wus7/fdXQP8XVV9bC3qlDQHtsBJC0SShybZl66/2Eer6uKqugf4MPDeJA9r2+2Y5Nltft8ku7RLrT8D7gbuAb5Fl6gck2TzJJsleeocQ3pDkq2TPBz4c7pWH4AbgZ2SbNJnv08AL0uyR5JN6S7TfbOqrppj/VMe0OKfmjbl3lasZyfZqJU/I8lOAxzvU23fvdt7eBtdK9SUG4GlWfehXP4mySZJfpuuz98ne9Z9BHgj8D+A/5jrgVvCdg/wD9y/9Q36f3fHAm9O8jiAJFsmedFc65c0OxM4af47I8ltdK0j/wf4R+47hMhf0F0m/UaSnwNf4t4WpF3b8u3A+cAHq+q8qrobeC7dTQ8/prvR4cA5xvUZ4ALgO3R9uI5v5efSte7ckOSna+5UVV8C3gp8mi6JfBT39tlbGwfTtUJOTf9dVdfQdc5/C7CS7rN7AwP8m1lVlwKvpkuUr6f77G4C7mybTCVaNye5cC1jvgG4la7l62PAUVX1/Z71p9G1Up7WLuGujY/QJYDTjVk37XdXVacB7wJObufSJcAfTLO/pHWUrj+yJI1OkgJ2raorxh3LsCXZAlhF935/NMJ6/xt4RUt412b/lwBHVtXT1ihfMN+dtCGzBU6S1rMkz03y4NY/7z3AxcBVI6z/hXR97c5dy/0fDLwSOG59xiVp/TGBk6T1bz+6y5s/obsMfVCN6HJHkuV0Ny68qvVxnOv+z6a7bHwjcxt+RNIIeQlVkiRpwtgCJ0mSNGFM4CRJkibMRAzku+2229bSpUvHHYYkSdLIXHDBBT+tqsXTrZuIBG7p0qWsWLFi3GFIkiSNTJKr+63zEqokSdKEMYGTJEmaMCZwkiRJE8YETpIkacKYwEmSJE0YEzhJkqQJYwInSZI0YUzgJEmSJsxQB/JNchVwG3A3sLqqliVZBJwCLAWuAg6oqluHGYckSQtVMp56q8ZT70Ixiha4362qPapqWVt+E3BOVe0KnNOWJUnSEFSt/bQu+2u4xnEJdT/gpDZ/EvD8McQgSZI0sYadwBVwVpILkhzZyrarquvb/A3AdkOOQZIkaV4Z9sPsn1ZV1yV5GHB2ku/3rqyqSjJtQ2tL+I4EWLJkyZDDlCRJmhxDbYGrquva603AacCewI1Jtgdorzf12fe4qlpWVcsWL148zDAlSZImytASuCSbJ3nI1DzwLOAS4HTg8LbZ4cBnhhWDJEnSfDTMS6jbAaelu395Y+DjVfWFJN8GTk1yBHA1cMAQY5AkSZp3hpbAVdWVwO7TlN8MPHNY9UqSJM13PolBkiRpwpjASZIkTRgTOEmSpAkz7HHgtIHwWXiSJM0fJnALxLokUomJmCRJGxIvoUqSJE0YEzhJkqQJ4yVUSfOefUAlzTcmcJLmPfuASppvvIQqSZI0YUzgJEmSJowJnCRJ0oQxgZMkSZowJnCSJEkTxgROkiRpwpjASZIkTRgTOEmSpAljAidJkjRhTOAkSZImjAmcJEnShDGBkyRJmjAmcJIkSRPGBE6SJGnCmMBJkiRNGBM4SZKkCWMCJ0mSNGFM4CRJkiaMCZwkSdKEMYGTJEmaMCZwkiRJE8YETpIkacIMPYFLslGS/0pyZlt+RJJvJrkiySlJNhl2DJIkSfPJKFrg/hy4rGf5XcB7q2oX4FbgiBHEIEmSNG8MNYFLshPwh8C/tuUA+wCfapucBDx/mDFIkiTNN8Nugfsn4I3APW15G2BVVa1uy9cCOw45BkmSpHllaAlckn2Bm6rqgrXc/8gkK5KsWLly5XqOTpIkaXINswXuqcDzklwFnEx36fR9wFZJNm7b7ARcN93OVXVcVS2rqmWLFy8eYpiSJEmTZdYELslTBylbU1W9uap2qqqlwEHAuVV1CHAesH/b7HDgM3OKWJIkaYEbpAXu/QOWDeovgKOTXEHXJ+74dTiWJEnSgrNxvxVJ9gL2BhYnObpn1UOBjeZSSVUtB5a3+SuBPecaqCRJkjp9EzhgE2CLts1Desp/zr2XQCVJkjRifRO4qvoy8OUkJ1bV1SOMSZIkSTOYqQVuyqZJjgOW9m5fVfsMKyhJkiT1N0gC90ngWLqnKdw93HAkSZI0m0ESuNVV9aGhRyJJkqSBDDKMyBlJXplk+ySLpqahRyZJkqRpDdICd3h7fUNPWQGPXP/hSJIkaTazJnBV9YhRBCJJkqTBzDSQ7z5VdW6SF0y3vqr+Y3hhSZIkqZ+ZWuCeDpwLPHeadQWYwEmSJI3BTAP5/nV7fdnowpEkSdJsBrmJgSR/CDwO2GyqrKr+dlhBSZIkqb9ZhxFJcixwIPBqIMCLgJ2HHJckSZL6GGQcuL2r6iXArVX1N8BewKOHG5YkSZL6GSSBu6O9/jLJDsBdwPbDC0mSJEkzGaQP3JlJtgLeDVxIdwfqvw4zKEmSJPU3yEC+b2+zn05yJrBZVf1suGFJkiSpn5kG8p12AN+2zoF8JUmSxmSmFrjpBvCd4kC+kiRJYzLTQL4O4CtJkrQBGmQcuG2S/N8kFya5IMn7kmwziuAkSZJ0f4MMI3IysBJ4IbB/mz9lmEFJkiSpv0GGEdm+505UgHckOXBYAUmSJGlmg7TAnZXkoCQPaNMBwBeHHZgkSZKmN0gC9yfAx4Fft+lk4BVJbkvy82EGJ0mSpPsbZCDfh4wiEEmSJA1mkD5wJHkCsLR3ewfylSRJGo9ZE7gkJwBPAC4F7mnFDuQrSZI0JoO0wD2lqnYbeiSSJEkayCA3MZyfxAROkiRpAzFIC9xH6JK4G4A7gQBVVU8YamSSJEma1iAJ3PHAYcDF3NsHblZJNgP+E9i01fOpqvrrJI+gG4pkG+AC4LCq+vVcA5ckSVqoBrmEurKqTq+qH1XV1VPTAPvdCexTVbsDewDPSfIU4F3Ae6tqF+BW4Ii1DV6SJGkhGqQF7r+SfBw4gy4pA2YfRqSqCri9LT6wTQXsA7y4lZ8EvA340JyiliRJWsAGSeAeRJe4PaunbKBhRJJsRHeZdBfgn4H/BlZV1eq2ybXAjnMJWJIkaaEb5EkML1vbg1fV3cAeSbYCTgMeO+i+SY4EjgRYsmTJ2oYgSZI078zaBy7JTklOS3JTmz6dZKe5VFJVq4DzgL2ArZJMJY47Adf12ee4qlpWVcsWL148l+okSZLmtUFuYvg34HRghzad0cpmlGRxa3kjyYOA3wcuo0vk9m+bHQ58Zs5RS5IkLWCDJHCLq+rfqmp1m04EBmkS2x44L8lFwLeBs6vqTOAvgKOTXEE3lMjxaxm7JEnSgjTITQw3JzkU+ERbPhi4ebadquoi4LemKb8S2HMuQUqStNAtWgS33jr6epPR1rf11nDLLaOtcxINksC9HHg/8F66u0+/Dqz1jQ2SJGnubr0VqsYdxfCNOmGcVIPchXo18LwRxCJJkqQB9O0Dl+TdSV4xTfkrkhwz3LAkSZLUz0w3MewDHDdN+YeBfYcTjiT1t2hRd3lllBOMvs5Fi8b7OUva8M10CXXT9jis+6iqexKvUEsaPfsASVJnpha4O5LsumZhK7tjeCFJkiRpJjO1wP0V8Pkk76B7ninAMuDNwGuGHJckSZL66JvAVdXnkzwfeAPw6lZ8CfDCqrp4BLFpGo4DJEmSZhxGpKouoXvclTYQ9gGSJEmDPEpLkiRJGxATOEmSpAkzYwKXZKMkrx1VMJIkSZrdjAlcVd1N9/B6SZIkbSAGeZj915J8ADgF+MVUYVVdOLSoJEmS1NcgCdwe7fVve8qK7lFbkiRJGrFZE7iq+t1RBCJJkqTBzHoXapLtkhyf5PNtebckRww/NEmSJE1nkGFETgS+COzQln+Aj9KSJEkam0ESuG2r6lTgHoCqWg3cPdSoJEmS1NcgCdwvkmxDd+MCSZ4C/GyoUUmSJKmvQe5CPRo4HXhUkq8Bi4H9hxqVJEmS+hrkLtQLkzwdeAwQ4PKqumvokUmSJGlasyZwSTYDXgk8je4y6leSHFtVvxp2cJIkSbq/QS6hfgS4DXh/W34x8O/Ai4YVlCRJkvobJIF7fFXt1rN8XpLvDSsgSZIkzWyQu1AvbHeeApDkycCK4YUkSZKkmQzSAvdE4OtJftyWlwCXJ7kYqKp6wtCikyRJ0v0MksA9Z+hRSJIkaWCDDCNy9SgCkSRJ0mAG6QMnSZKkDYgJnCRJ0oSZNYFLsnmSB7T5Ryd5XpIHDrDfw5Ocl+R7SS5N8uetfFGSs5P8sL1uve5vQ9JCkcz/SZJmM0gL3H8CmyXZETgLOAw4cYD9VgOva2PIPQV4VZLdgDcB51TVrsA5bVmSBlI1/ydJms0gCVyq6pfAC4APVtWLgMfNtlNVXV9VF7b524DLgB2B/YCT2mYnAc9fi7glSZIWrIESuCR7AYcAn21lG82lkiRLgd8CvglsV1XXt1U3ANvN5ViSJEkL3SDjwL0GeDNwWlVdmuSRwHmDVpBkC+DTwGuq6ufp6eBRVZVk2gsGSY4EjgRYsmTJoNVJkjRv2UdSU1JD7HDRbnY4E/hiVf1jK7sceEZVXZ9ke2B5VT1mpuMsW7asVqzw6V3Q/XgXQh+ZhfI+NTcL5bxYKO9Tc7NQzouF8j4HkeSCqlo23bq+LXBJzgD6foRV9bxZKg1wPHDZVPLWnA4cDhzTXj8z03EkSZJ0XzNdQn1Pe30B8BvAR9vywcCNAxz7qXR3rF6c5Dut7C10idupSY4ArgYOmGPMkiRJC1rfBK6qvgyQ5B/WaL47I8ms1zOr6qtAv6v1z5xTlLoP+0BIkrSwDXITw+ZJHllVVwIkeQSw+XDD0kwWQt8Ak1RJkvob9C7U5UmupGtR25l2d6gkSZJGb8YErj1Ca0tgV+Cxrfj7VXXnsAOTJEnS9GYcyLeq7gHeWFV3VtV322TyJkmSNEaDPInhS0le3x5Ov2hqGnpkkiRJmtYgfeAObK+v6ikr4JHrPxxJkiTNZtYErqoeMYpAJEmSNJhZE7j2OKz/DfxOK1oO/EtV3TXEuCRJktTHIJdQPwQ8EPhgWz6slf3xsIKSJElSf4MkcE+qqt17ls9N8t1hBSRJkqSZDXIX6t1JHjW1kOSRwN3DC0mSJEkzGaQF7g3AeWs8ieFlQ41KkiRJffVN4JK8Bvg68GW6JzE8pq263MF8JUmSxmemS6g7Af8E3AScBRwELMEH2UuSJI1V3xa4qno9QJJNgGXA3nSXTo9LsqqqdhtNiJIkSeo1SB+4BwEPpXuo/ZbAT4CLhxmUJPWTjDuC4dt663FHIGlDN1MfuOOAxwG3Ad+k6w/3j1V164hik6T7qBp9ncl46pWkmczUB24JsClwA3AdcC2wagQxSZIkaQYz9YF7TpLQtcLtDbwOeHySW4Dzq+qvRxSjJEmSeszYB66qCrgkySrgZ23aF9gTMIGTJGmE7AOqKTP1gfszupa3vYG76PrAfR04AW9ikCRppOwDql4ztcAtBT4JvLaqrh9NOJIkSZrNTH3gjh5lIJIkSRrMIOPAaQNjHwhJkhY2E7gJYx8ISZI00zhwkiRJ2gCZwEmSJE0YEzhJkqQJYwInSZI0YUzgJEmSJowJnCRJ0oQZWgKX5IQkNyW5pKdsUZKzk/ywvTralyRJ0hwNswXuROA5a5S9CTinqnYFzmnLkiRJmoOhJXBV9Z/ALWsU7wec1OZPAp4/rPolSZLmq1H3gduuqq5v8zcA2424fkmSpIk3tpsYqqqAvg9oSnJkkhVJVqxcuXKEkUmSJG3YRp3A3Zhke4D2elO/DavquKpaVlXLFi9ePLIAJUmSNnSjTuBOBw5v84cDnxlx/ZIkSRNvmMOIfAI4H3hMkmuTHAEcA/x+kh8Cv9eWJUmSNAcbD+vAVXVwn1XPHFadkiRJC4FPYpAkSZowJnCSJEkTxgROkiRpwpjASZIkTRgTOEmSpAljAidJkjRhTOAkSZImjAmcJEnShDGBkyRJmjAmcJIkSRPGBE6SJGnCmMBJkiRNGBM4SZKkCWMCJ0mSNGFM4CRJkiaMCZwkSdKEMYGTJEmaMCZwkiRJE8YETpIkacKYwEmSJE0YEzhJkqQJYwInSZI0YUzgJEmSJowJnCRJ0oQxgZMkSZowJnCSJEkTxgROkiRpwmw87gA0Gsl49q9at3ql9cHzXwuZ5//8ZAK3QPhD0kLm+a+FzPN/fvISqiRJ0oQxgZMkSZowY0ngkjwnyeVJrkjypnHEIEmSNKlGnsAl2Qj4Z+APgN2Ag5PsNuo4JEmSJtU4WuD2BK6oqiur6tfAycB+Y4hDkiRpIo0jgdsRuKZn+dpWJkmSpAFssDcxJDkyyYokK1auXDnucCRJkjYY40jgrgMe3rO8Uyu7j6o6rqqWVdWyxYsXjyw4SZKkDV1qxCP8JdkY+AHwTLrE7dvAi6vq0hn2WQlcPZoINY1tgZ+OOwhpTDz/tZB5/o/XzlU1bSvWyJ/EUFWrk/wp8EVgI+CEmZK3to9NcGOUZEVVLRt3HNI4eP5rIfP833CN5VFaVfU54HPjqFuSJGnSbbA3MUiSJGl6JnAaxHHjDkAaI89/LWSe/xuokd/EIEmSpHVjC5wkSdKEMYHTnCTZKskre5Z3SPKpccYkDVuSpUlevJb73r6+45GGLclRSV7S5l+aZIeedf/qM8zHz0uompMkS4Ezq+rx445FGpUkzwBeX1X7TrNu46paPcO+t1fVFkMMTxqqJMvpzv8V445F97IFbp5pLQWXJflwkkuTnJXkQUkeleQLSS5I8pUkj23bPyrJN5JcnOQdU60FSbZIck6SC9u6/VoVxwCPSvKdJO9u9V3S9vlGksf1xLI8ybIkmyc5Icm3kvxXz7GkoVqL38OJSfbv2X+q9ewY4Lfbef/a1iJxepJzgXNm+L1II9fO++8n+Vg7/z+V5MFJntn+Db64/Zu8adv+mCTfS3JRkve0srcleX37PSwDPtbO/wf1/Nt+VJJ399T70iQfaPOHtn/zv5PkX5JsNI7PYl6rKqd5NAFLgdXAHm35VOBQ4Bxg11b2ZODcNn8mcHCbPwq4vc1vDDy0zW8LXAGkHf+SNeq7pM2/FvibNr89cHmbfydwaJvfiu5JHJuP+7Nymv/TWvweTgT279l/6vfwDLqW56nylwLXAova8rS/l95jODmNamrnfQFPbcsnAH8JXAM8upV9BHgNsA1wec/5ulV7fRtdqxvAcmBZz/GX0yV1i4Ereso/DzwN+E3gDOCBrfyDwEvG/bnMt8kWuPnpR1X1nTZ/Ad2PeW/gk0m+A/wLXYIFsBfwyTb/8Z5jBHhnkouALwE7AtvNUu+pwFTrxQHAVN+4ZwFvanUvBzYDlsztLUlrbS6/h7k4u6puafNr83uRhumaqvpam/8o3eMrf1RVP2hlJwG/A/wM+BVwfJIXAL8ctIKqWglcmeQpSbYBHgt8rdX1RODb7Tf2TOCR6/6W1GssT2LQ0N3ZM3833R+SVVW1xxyOcQjd/66eWFV3JbmKLvHqq6quS3JzkicAB9K16EH3x+2FVXX5HOqX1pe5/B5W07qWJHkAsMkMx/1Fz/ycfy/SkK3ZwX0VXWvbfTfqHm+5J12StT/wp8A+c6jnZLr/sH8fOK2qKkmAk6rqzWsTuAZjC9zC8HPgR0leBJDO7m3dN4AXtvmDevbZErip/TH6XWDnVn4b8JAZ6joFeCOwZVVd1Mq+CLy6/ahJ8lvr+oakdTDT7+EqupYDgOcBD2zzs533/X4v0rgsSbJXm38xsAJYmmSXVnYY8OUkW9D9e/05um4wu9//UDOe/6cB+wEH0yVz0HVR2D/JwwCSLErib2I9M4FbOA4BjkjyXeBSuh8cdH0gjm6Xfnaha04H+BiwLMnFwEvo/ndFVd0MfC3JJb2dV3t8ii4RPLWn7O10fwgvSnJpW5bGqd/v4cPA01v5XtzbynYRcHeS7yZ57TTHm/b3Io3R5cCrklwGbA28F3gZXdeBi4F7gGPpErMz29+ArwJHT3OsE4Fjp25i6F1RVbcClwE7V9W3Wtn36PrcndWOezZr101BM3AYkQUuyYOBO1qz90F0NzR4B50kTag43NOCYB84PRH4QLu8uQp4+XjDkSRJs7EFTpIkacLYB06SJGnCmMBJkiRNGBM4SZKkCWMCJ0mSNGFM4CRJkiaMCZwkSdKE+X8QQc1od1qzjAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 720x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.boxplot(\"Sentence Length\", by=\"label\", grid=False, showfliers=False, color=\"Blue\", figsize=(10,4))\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Description Length by Type')\n",
    "plt.xlabel('')\n",
    "plt.ylabel('Words per Complaint')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove Overly Lengthy Samples & Unnecessary Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 10670 entries, 0 to 10675\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    10670 non-null  object\n",
      " 1   label   10670 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 250.1+ KB\n"
     ]
    }
   ],
   "source": [
    "data = data[data['Sentence Length'] < 59]\n",
    "\n",
    "data = data.drop(columns=['Sentence Length'])\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Quick Look at Updated Class (Output) Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neutral     5991\n",
      "positive    3215\n",
      "negative    1464\n",
      "Name: label, dtype: int64 \n",
      "\n",
      "Total: 10670\n"
     ]
    }
   ],
   "source": [
    "results_data = pd.Series(dtype=int, name='result_count')\n",
    "results_data = data['label'].value_counts()\n",
    "\n",
    "print(results_data, '\\n\\nTotal:', results_data.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final Prepocessing Before Loading into Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'positive': 0, 'negative': 1, 'neutral': 2}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The GeoSolutions technology will leverage Bene...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$ESI on lows, down $1.50 to $2.50 BK a real po...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>For the last quarter of 2010 , Componenta 's n...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>According to the Finnish-Russian Chamber of Co...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Swedish buyout firm has sold its remaining...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label\n",
       "0  The GeoSolutions technology will leverage Bene...      0\n",
       "1  $ESI on lows, down $1.50 to $2.50 BK a real po...      1\n",
       "2  For the last quarter of 2010 , Componenta 's n...      0\n",
       "3  According to the Finnish-Russian Chamber of Co...      2\n",
       "4  The Swedish buyout firm has sold its remaining...      2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique_classes = data['label'].unique()\n",
    "\n",
    "unique_class_converter = {unique_classes[x] : x for x in range(len(unique_classes))}\n",
    "\n",
    "print(unique_class_converter)\n",
    "\n",
    "data['label'].replace(unique_class_converter, inplace=True)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Regular Project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Constants/Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "104\n"
     ]
    }
   ],
   "source": [
    "model_ckpt = 'distilbert-base-uncased'\n",
    "device=\"cpu\"\n",
    "max_input_length = int(round(80*1.3,0))\n",
    "set_seed(42)\n",
    "batch_size = 64\n",
    "print(max_input_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Pandas DataFrame to Dataset & Split into Train/Test/Valid Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data shape: (8536, 3)\n",
      "testing data shape: (1067, 3)\n",
      "validation data shape: (1067, 3)\n",
      "{'text': Value(dtype='string', id=None), 'label': Value(dtype='int64', id=None), '__index_level_0__': Value(dtype='int64', id=None)}\n"
     ]
    }
   ],
   "source": [
    "# Convert pandas dataframe to a dataset\n",
    "dataset = Dataset.from_pandas(data)\n",
    "\n",
    "# Split the datasetdict into train/test/valid subsets\n",
    "train_testvalid = dataset.train_test_split(test_size=0.20)\n",
    "test_valid = train_testvalid['test'].train_test_split(test_size=0.50)\n",
    "\n",
    "# Combine the train/test/valid into one datasetdict\n",
    "dataset = DatasetDict({\n",
    "    'train' : train_testvalid['train'],\n",
    "    'test' : test_valid['test'],\n",
    "    'valid' : test_valid['train']\n",
    "})\n",
    "\n",
    "print('training data shape:', dataset['train'].shape)\n",
    "print('testing data shape:', dataset['test'].shape)\n",
    "print('validation data shape:', dataset['valid'].shape)\n",
    "print(dataset['train'].features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8da378f063734b3ab54db807f75bfc12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96e8a1bc54df45b0939669f3fd060f8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb19758c2f384ab7826ceebee7a274bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data shape: (8536, 5)\n",
      "testing data shape: (1067, 5)\n",
      "validation data shape: (1067, 5)\n"
     ]
    }
   ],
   "source": [
    "# Define the Tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "# Method to Tokenize each Sample in the Dataset\n",
    "def tokenize(batch):\n",
    "    return tokenizer(batch['text'], truncation=True, padding=True, max_length=max_input_length, return_tensors=\"np\")\n",
    "\n",
    "# Map the 'Tokenize' Method to All Samples in the Entire Dataset\n",
    "encoded_data = dataset.map(tokenize, batched=True, batch_size=None)\n",
    "\n",
    "print('training data shape:', encoded_data['train'].shape)\n",
    "print('testing data shape:', encoded_data['test'].shape)\n",
    "print('validation data shape:', encoded_data['valid'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check Result of Tokenizing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': Value(dtype='string', id=None), 'label': Value(dtype='int64', id=None), '__index_level_0__': Value(dtype='int64', id=None), 'input_ids': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), 'attention_mask': Sequence(feature=Value(dtype='int8', id=None), length=-1, id=None)}\n",
      "Dataset({\n",
      "    features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask'],\n",
      "    num_rows: 8536\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "print(encoded_data['train'].features)\n",
    "print(encoded_data['train'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.bias', 'vocab_projector.weight', 'vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_layer_norm.weight']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'pre_classifier.weight', 'classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels=len(unique_classes)\n",
    "\n",
    "model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to(device))\n",
    "model.config.num_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ensure that git lfs is Installed; Remember to Enter Access Token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "!git lfs install\n",
    "\n",
    "# HuggingFace Access Token ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Function to Compute Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    f1 = f1_score(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\"accuracy\": acc, \"f1_score\": f1}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Data Collator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_of_epochs = 5\n",
    "logging_steps = (len(encoded_data['train']) // batch_size)\n",
    "learning_rate=2e-5\n",
    "label_names = unique_classes\n",
    "steps = 100\n",
    "\n",
    "model_name = f\"{model_ckpt}-Finanacial_Sentiment_Analysis_5e\"\n",
    "\n",
    "training_args = TrainingArguments(output_dir=model_name,\n",
    "                        num_train_epochs=num_of_epochs,\n",
    "                        learning_rate=learning_rate,\n",
    "                        per_device_train_batch_size=batch_size, \n",
    "                        per_device_eval_batch_size=batch_size, \n",
    "                        weight_decay=0.01,\n",
    "                        log_level=\"error\",\n",
    "                        logging_strategy='epoch', \n",
    "                        hub_strategy=\"checkpoint\",\n",
    "                        evaluation_strategy=\"epoch\",\n",
    "                        save_strategy=\"epoch\",\n",
    "                        save_total_limit=5,\n",
    "                        load_best_model_at_end=True,\n",
    "                        metric_for_best_model=\"f1_score\",\n",
    "                        greater_is_better=True,\n",
    "                        logging_first_step=True,\n",
    "                        group_by_length=True,\n",
    "                        disable_tqdm=False,\n",
    "                        push_to_hub=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Subclass Trainer to Correctly Compute Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.get(\"labels\")\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.get(\"logits\")\n",
    "        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([1.0, 2.0, 3.0]))\n",
    "        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))\n",
    "        return (loss, outputs) if return_outputs else loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning https://huggingface.co/DunnBC22/distilbert-base-uncased-Finanacial_Sentiment_Analysis_5e into local empty directory.\n"
     ]
    }
   ],
   "source": [
    "trainer = CustomTrainer(model=model, \n",
    "                  args=training_args, \n",
    "                  compute_metrics=compute_metrics,\n",
    "                  train_dataset=encoded_data['train'], \n",
    "                  eval_dataset=encoded_data['valid'], \n",
    "                  data_collator=data_collator,\n",
    "                  tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/nlpnn/nlp_ch4/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73fc57ff62eb4b449386e8944f253320",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/670 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 1.1124, 'learning_rate': 1.9970149253731346e-05, 'epoch': 0.01}\n",
      "{'loss': 0.5569, 'learning_rate': 1.6000000000000003e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9bea67d592a429d98bd03dc3fde73cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.39540889859199524, 'eval_accuracy': 0.7591377694470478, 'eval_f1_score': 0.7558879300287058, 'eval_runtime': 231.6275, 'eval_samples_per_second': 4.607, 'eval_steps_per_second': 0.073, 'epoch': 1.0}\n",
      "{'loss': 0.3177, 'learning_rate': 1.2e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df13b7fe84d64207b542b75e380495ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3391183614730835, 'eval_accuracy': 0.8134957825679475, 'eval_f1_score': 0.8151397951439218, 'eval_runtime': 231.1509, 'eval_samples_per_second': 4.616, 'eval_steps_per_second': 0.074, 'epoch': 2.0}\n",
      "{'loss': 0.2479, 'learning_rate': 8.000000000000001e-06, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bceaab7a68d44878a918674a88a8c8c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3210773766040802, 'eval_accuracy': 0.8322399250234301, 'eval_f1_score': 0.8353417132688705, 'eval_runtime': 231.1395, 'eval_samples_per_second': 4.616, 'eval_steps_per_second': 0.074, 'epoch': 3.0}\n",
      "{'loss': 0.2049, 'learning_rate': 4.000000000000001e-06, 'epoch': 4.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50580a11f64b4d12898509fe38e06494",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3065849840641022, 'eval_accuracy': 0.8462980318650422, 'eval_f1_score': 0.8506075560174677, 'eval_runtime': 231.1636, 'eval_samples_per_second': 4.616, 'eval_steps_per_second': 0.074, 'epoch': 4.0}\n",
      "{'loss': 0.1802, 'learning_rate': 0.0, 'epoch': 5.0}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f6fef925a254949ac920daf48c9d81c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3078767955303192, 'eval_accuracy': 0.8528584817244611, 'eval_f1_score': 0.8564085217556289, 'eval_runtime': 231.121, 'eval_samples_per_second': 4.617, 'eval_steps_per_second': 0.074, 'epoch': 5.0}\n",
      "{'train_runtime': 29299.5005, 'train_samples_per_second': 1.457, 'train_steps_per_second': 0.023, 'train_loss': 0.30236058751148964, 'epoch': 5.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=670, training_loss=0.30236058751148964, metrics={'train_runtime': 29299.5005, 'train_samples_per_second': 1.457, 'train_steps_per_second': 0.023, 'train_loss': 0.30236058751148964, 'epoch': 5.0})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assessing the Results of the Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate Predictions for Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40c81f05cc0d44db92fe81914bb5c39c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/17 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.3078767955303192,\n",
       " 'test_accuracy': 0.8528584817244611,\n",
       " 'test_f1_score': 0.8564085217556289,\n",
       " 'test_runtime': 233.3529,\n",
       " 'test_samples_per_second': 4.572,\n",
       " 'test_steps_per_second': 0.073}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_predictions = trainer.predict(encoded_data['valid'])\n",
    "\n",
    "output_predictions.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract Last Hidden States to Create Feature Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09167c34b7f4426099a9d56aa7841bd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f371c8d51f146ec9243090a8a18a194",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e498cfa98a04822bdf54c076b2ed52f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 8536\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 1067\n",
      "    })\n",
      "    valid: Dataset({\n",
      "        features: ['text', 'label', '__index_level_0__', 'input_ids', 'attention_mask', 'hidden_state'],\n",
      "        num_rows: 1067\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "def extract_hidden_states(batch):\n",
    "    inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}   \n",
    "    with torch.no_grad():\n",
    "        last_hidden_state = model(**inputs)[0]\n",
    "    return {\"hidden_state\" : last_hidden_state[:,0].cpu().numpy()}\n",
    "\n",
    "encoded_data.set_format(\"torch\", columns=['input_ids', 'attention_mask', 'label']) \n",
    "\n",
    "hidden_data = encoded_data.map(extract_hidden_states, batched=True)\n",
    "print(hidden_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Feature Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8536,), (1067,))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = np.array(hidden_data['train']['hidden_state'])\n",
    "X_valid = np.array(hidden_data['valid']['hidden_state'])\n",
    "y_train = np.array(hidden_data['train']['label'])\n",
    "y_valid = np.array(hidden_data['valid']['label'])\n",
    "X_train.shape, X_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define Function to Display Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_preds, y_true, labels):\n",
    "    cm = confusion_matrix(y_true, y_preds, normalize=\"true\")\n",
    "    fig, ax = plt.subplots(figsize=(11, 11))\n",
    "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\n",
    "    disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax)\n",
    "    plt.title('Normalized Confusion Matrix')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Logits to Integers & Display Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAJkCAYAAAB05hsHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAABCXUlEQVR4nO3dd5wdZfX48c/ZTUILJKQAKYQgvbeAFAEB8Rv0K4gg1YJiQcCGiPjDL01FFFRQQKUJIop0Ix1BpCiSEEJJKGJoKUACJFQh2ZzfH3d2uVmS3M292b2bvZ+3r/vyzswzzzyzOyQn58w8E5mJJEmStChN9R6AJEmSuj+DRkmSJFVk0ChJkqSKDBolSZJUkUGjJEmSKjJolCRJUkW96j0ASZKkpVXzSmtkzn2ry4+bb824OTNHd+UxDRolSZKqlHPfYpn19uvy4/53wtmDuvqYBo2SJElVC4jGuNuvMc5SkiRJNTFolCRJUkWWpyVJkqoVQES9R9ElzDRKkiSpIjONkiRJtfBBGEmSJKnETKMkSVItvKdRkiRJKjFolCRJUkWWpyVJkqrmG2EkSZKkNmYaJUmSauGDMJIkSVKJmUZJkqRqBd7TKEmSJLUyaJQkSVJFlqclSZKqFj4II0mSJLUy0yhJklQLH4SRJEmSSsw0SpIk1cJ7GiVJkqQSg0ZJkiRVZHlakiSpauGDMJIkSVIrM42SJEnVCnwQRpIkSWplplGSJKkW3tMoSZIklRg0SpIkqSLL05IkSVVzyh1JkiSpjZlGSZKkWjQ55Y4kSZIEmGmUJEmqXuA9jZJUSUTcERFfKL4fHBG3LOH+R0ZERkSX/wM3IpaLiL9ExOyIuKKGfpb4z6UeIuLGiPhsvcchqX4MGqVuLCKejogXI2KFsnVfiIg76jisBcrMSzPzw1193Ig4KCLGRcTrETG9CG4+sAS63hdYFRiYmZ+stpPO+rlExAeLgPqadus3K9bf0cF+ToyI31dql5l7ZObFVQ5XUg9g0Ch1f83A12vtJEp61H/zEXEUcAZwCqUAbwRwDrDXEuh+DeCJzJy7BPrqLDOA7SJiYNm6zwJPLKkD9MTrRlriIrr+Uwf+QSB1f6cBR0dE/wVtjIjtI2JsUUYdGxHbl227IyJ+GBH3AG8C7yuyUIdHxL8j4rWI+H5ErBUR/4iIVyPi8ojoU+y/ckRcFxEzIuKV4vvwhYzjkIi4u/h+TJH5a/3MiYiLim39IuKCIis4NSJ+EBHNxbbmiDg9ImZGxGTgowv7oUREP+Bk4IjMvDoz38jMOZn5l8z8dtFmmYg4IyKmFZ8zImKZYtsHI2JKRHyryOZOj4jPFdtOAo4H9i/Gf2j7jFz70nlx/pOLn+lTEXFw+59LB39f34+Ie4p+bomIQQv7GQDvANcCB7T+/ID9gUvb/azOjIjnit/v/RGxY7F+NPD/ys7zwbJxtL9uym9F+FVEXFXW/48j4raIOv1NJqlLGDRK3d844A7g6PYbImIAcD3wC2Ag8DPg+naZp08DXwJWBJ4p1v0PsBWwLXAMcC7wKWB1YGPgwKJdE/BbSlm3EcBbwFmVBpyZP8nMvpnZF9iAUkbsT8Xmi4C5wNrAFsCHgS8U274I/G+xfhSlEvHCbAcsC1yziDbHFee4ObAZsA3wvbLtqwH9gGHAocDZEbFyZp5AKXv5p+I8LljU+Ubp9oFfAHtk5orA9sCEBbTryO/rIOBzwCpAHxbwe2/nd8Bniu//AzwCTGvXZiyln8EA4A/AFRGxbGbe1O48NyvbZ0HXTatvAZsUAfGOlH52n83MrDBWqQcqJvfu6k8dGDRKS4fjga9GxOB26z8K/DszL8nMuZn5R+Ax4GNlbS7KzInF9jnFup9k5quZOZFSkHFLZk7OzNnAjZSCNjLzpcy8KjPfzMzXgB8CO3d00BGxHKVM2JmZeWNErAp8BPhGkRl8Efg5RaYM2A84IzOfy8yXgR8tovuBwMwK5eODgZMz88XMnAGcRCkYajWn2D4nM28AXgfW6+j5tTMP2DgilsvM6cXPtr2O/L5+m5lPZOZbwOWUgr2Fysx/AAMiYj1KwePvFtDm98Xvcm5m/hRYhsrnuaDrprW/Nyn9HH8G/B74amZOqdCfpKWcQaO0FMjMR4DrgGPbbRrKe7NAz1DKnLV6bgFdvlD2/a0FLPcFiIjlI+I3EfFMRLwK3An0by0nd8AFwOOZ+eNieQ2gNzA9ImZFxCzgN5Syaq3nUz7e9udW7iVgUCz6yer2P59ninVtfbQLOt+kOPfFkZlvUCoLH0bp3K6PiPU7MJ7WMZX/vp6vYjyXAEcCu7CAzGtEHB0RjxYl8VmUsquLKnvDgq+bNpn5L2AypQlHLu/AGKWey3saJXUzJ1Aq35YHGNMoBWLlRgBTy5ZrKRl+i1JG6v2ZuRKwU7G+4p9YEXEssC6l0mWr54C3gUGZ2b/4rJSZGxXbp1MqkbcasYhD/LPo6+OLaNP+5zOC95ZuO+oNYPmy5dXKN2bmzZm5OzCEUvbwvA6Mp3VMUxfQdnFcAhwO3FBkAdsU5eNjKGVxV87M/sBs3v0dLuz6WOR1ExFHUMpYTiv6l9TDGTRKS4nMfJLSfYFfK1t9A7BulKad6RUR+wMbUspKLgkrUso8ziruxzuhIztFxB7FOPcuyqyt5zAduAX4aUSsFBFNUXoIp7XkfTnwtYgYHhEr897MKmV9zaZUtj87Ij5eZEV7R8QeEfGTotkfge9FxODigZLjKZVTqzEB2CkiRhQP4Xy37HxXjYi9insb36ZU5p63gD465feVmU9Rum3guAVsXpHSPaQzgF4RcTywUtn2F4CRsRhPSEfEusAPKN0H+2ngmIjYvLrRS1paGDRKS5eTgbY5GzPzJUoPjnyLUrn2GOB/M3PmEjreGcBywEzgXuCmDu63PzAYeDTefYL618W2z1B6wGMS8ApwJaXsHJSyczcDDwLjgasXdZDi/ryjKD3cMoNSJvNISvdRQimwGQc8BDxc9PmDDp5D+2PdSilofwi4n/kDvaZiHNOAlykFcF9ZQB+d9vvKzLszc0FZ1Jsp/d6eoFQK/y/zl55bJy5/KSLGVzpOcTvA74EfZ+aDmflvSk9gX9L6ZLrUcBrkQZjwYTdJkqTqNK00PJfZtuapdBfbf2895v7MHNWVx/Td05IkSdWq44MpXc3ytCRJUg8TEaMj4vGIeLJ4MLH99hER8beIeCAiHoqIj1Tq00yjJElSLbrZmzaLadHOBnYHpgBjI2JMZk4qa/Y94PLM/FVEbEjpQb2Ri+q3e52lJEmSarUN8GTx0oZ3gMuAvdq1Sd6dSaEfHZiOzExjBdF7+Yxl+tV7GBIAm683rHIjqYvMbfFBSnUPU557hpdfmtkYNxa+a1BEjCtbPjczzy2+D2P+WRKmAO9vt/+JwC0R8VVKs3J8qNIBDRoriGX6scwmn633MCQA7rzjlHoPQWrz0uvv1HsIEgAf222H+g6gPg/CzKzx6ekDKb0u9KcRsR2labM2zswFzTELWJ6WJEnqaaYy/9u1hvPeN08dSvEK0Mz8J7AsFV4vatAoSZJUteiOk3uPBdaJiDUjog9wADCmXZtngd0AImIDSkHjjEV1atAoSZLUg2TmXEpvx7oZeJTSU9ITI+LkiNizaPYt4IsR8SClV64ekhXe+OI9jZIkSbXohpN7Z+YNlKbRKV93fNn3ScBi3QxqplGSJEkVGTRKkiSpIsvTkiRJ1Qq63RthOktjnKUkSZJqYqZRkiSpamGmUZIkSWplplGSJKkW3XDKnc5gplGSJEkVGTRKkiSpIsvTkiRJtfBBGEmSJKnETKMkSVItfBBGkiRJKjHTKEmSVK1wcm9JkiSpjUGjJEmSKrI8LUmSVAsfhJEkSZJKzDRKkiTVIMw0SpIkSSVmGiVJkqoUmGmUJEmS2hg0SpIkqSLL05IkSdWK4tMAzDRKkiSpIjONkiRJVQsfhJEkSZJamWmUJEmqgZlGSZIkqWDQKEmSpIosT0uSJNXA8rQkSZJUMNMoSZJUAzONkiRJUsFMoyRJUrV8jaAkSZL0LoNGSZIkVWR5WpIkqUrhu6clSZKkd5lplCRJqoGZRkmSJKlgplGSJKkGZholSZKkgkGjJEmSKrI8LUmSVAPL05IkSVLBTKMkSVK1fPe0JEmS9C4zjZIkSTXwnkZJkiSpYNAoSZKkiixPS5IkVSkIy9OSJElSKzONkiRJNTDTKEmSJBXMNEqSJNWiMRKNZholSZJUmUGjJEmSKrI8LUmSVK3wQRhJkiSpjUGjJElSDSKiyz8dGNPoiHg8Ip6MiGMXsP3nETGh+DwREbMq9Wl5WpIkqQeJiGbgbGB3YAowNiLGZOak1jaZ+c2y9l8FtqjUr0GjJElSDbrhPY3bAE9m5mSAiLgM2AuYtJD2BwInVOrU8rQkSVLPMgx4rmx5SrHuPSJiDWBN4PZKnZpplCRJWvoMiohxZcvnZua5VfRzAHBlZrZUamjQKEmSVKWgYw+mdIKZmTlqIdumAquXLQ8v1i3IAcARHTmg5WlJkqSeZSywTkSsGRF9KAWGY9o3ioj1gZWBf3akU4NGSZKkWkQdPouQmXOBI4GbgUeByzNzYkScHBF7ljU9ALgsM7Mjp2l5WpIkqYfJzBuAG9qtO77d8omL06dBoyRJUrV8jaAkSZL0LoNGSZIkVbTUlacj4jDgzcz8XUQcAtySmdOKbecDPyt/TY46127brMuPvv4xmpuCS64byxmX/n2+7cNX6cc5x+1Hv77L0dwcnPTrm7j13sdZfbWV+dfvj+LJZ2cAMG7isxz102vrcAbqSW775ySO+/nVtMybx6f23I6vf2b3+ba//c4cjjjp9zz4+HMMWGkFzvvBIYwYOrBt+5TnX2aHA0/hmC/swREH79bVw1cPc9d9j/HDc/7MvHnz2HeP9/OlA3edb/vYh/7Dj84Zw+OTp/PT7x3M6J02A+DRJ6dy4plX88ab/6WpqYnDDtqNj+yyeR3OQB3VKOXppS5ozMxfly0eAjwCTCu2faEeY2pUTU3BaUftxd7fvIBpM2Zz+3lHcuM9j/L40y+2tfnWZ3fl2r89xIXX/ov1Rq7C5T/5HJvt92MAnp76Ejt9/hf1Gr56mJaWeRx7+hVc8YsjGLpKfz78udMZvePGrLfmkLY2l465l/4rLc/YK4/nmlvv5+Szx3D+Dz/Xtv3/zryG3bbbsB7DVw/T0jKPk395DRf++EusOrgfnzziTHbdfkPWXmO1tjZDVlmZHx2zPxdePv8/tpddtg8//s4BjBw+mBdmzmbfw8/gA1uvx0p9l+vq05Dm06Xl6YgYGRGPRcSlEfFoRFwZEctHxG4R8UBEPBwRF0bEMkX7UyNiUkQ8FBGnF+tOjIijI2JfYBRwaURMiIjlIuKOiBgVEYdFxGllxz0kIs4qvn8qIu4r9vlN8VJvVWGrDVZn8tSXeGb6y8yZ28LVtz3IRz7Q7i/chBWXXxaAlVZYludnvlqHkaoRjJ/0DCOHD2bksEH06d2Lj+++JTfe+fB8bW6862H2/8g2AHxsl825a9wTtM40ccPfH2KNoQNZf83V3tO3tLgeevxZRgwdyOpDB9Kndy8+8sHNue2eifO1Gb7aANZ731Ciaf4s1ZrDBzNy+GAAVh3UjwH9+/LyrNe7bOxafBHR5Z96qMc9jesB52TmBsCrwFHARcD+mbkJpeznVyJiILA3sFFmbgr8oLyTzLwSGAccnJmbZ+ZbZZuvKvZttT9wWURsUHzfITM3B1qAg5f8KTaGIYNXYuqLs9uWp82YzZBBK83X5tTf/pX9PrwFj1z1XS4/7XMcc8a7c4uOGDKAv1/wNa775ZfYbtORXTVs9VDTZ8xi2Cr925aHrtKf6TNmz9fm+RmzGbZqqU2vXs2s1HdZXp79Bq+/+Ta/vOSvHH3oHl04YvVkL8yczZCy63G1wf154aXZC99hIR567FnmzG2Z7zYKqV7qETQ+l5n3FN9/D+wGPJWZTxTrLgZ2AmYD/wUuiIhPAG929ACZOQOYHBHbFsHn+sA9xbG2AsZGxIRi+X3t94+IL0XEuIgYl3M6fFgtwD4f2ow/3Hg/G+/zI/b79m/59f/tR0Twwkuvssm+p7Lzob/guF9ez3nHH8CKyy9T7+GqQZ12/o18+YAP0tdrUN3Iiy+9yjGn/pFTjt6fpiafW+3Wutnk3p2lHvc0tp91fBbwnn9CZebciNiGUmC3L6WZzXdt324RLgP2Ax4DrsnMjFI+9+LM/O4iB1h64fe5AE19h3RolvRGNH3GqwxbpV/b8tDB/Zjervz8qY9uzSePvhCAsROfZdk+vRjYb3lmznqDd4qA/MEnpvLUtJdZa/VBTHh8Ya/GlBZtyOD+TH1xVtvytBdnMWRwv/narDa4H1NfmMXQVVZm7twWXn39vwzotwL3T3yav9w+gZPPGsPs19+iqSlYpk9vvvDJnbr4LNRTrDqoH9PLrsfnZ8xi1YH9Fr5DO6+/8V8OO+4CvvH50Wy+4RqdMEJp8dXjny4jImK74vtBlErMIyNi7WLdp4G/R0RfoF8xo/k3gc0W0NdrwIoLOc41wF7AgZQCSIDbgH0jYhWAiBgQEf7XWKXxj01hreEDGTFkZXr3auYTu23GjXfP/+D61BdmsdNWpV/tumsMZpk+vZk56w0G9l+BpuI+njWGDOB9wwfy9LSXu/wc1HNsscEInnpuBs9Me4l35szl2lvHM3rHTeZrM3rHjfnTDfcB8Je/TeADo9YhIrjuN99g/LUnMv7aE/ny/jvzjc/ubsCommyy3uo8M3UmU6aXrscb7pjArttv1KF935kzlyNPvIi9dt+q7YlqqTuoR6bxceCIiLgQmAR8DbgXuCIielF6yfavgQHAnyNiWUqJ2KMW0NdFwK8j4i1gu/INmflKRDwKbJiZ9xXrJkXE94BbIqIJmAMcATyz5E+z52tpmccxPx/DVT/9PM1NTVx6/Tgee/pFvnvo7kx4bAo33vMo3zv7es485hMcvt8HyEyOOOUKALbfbE2+e+juzJ3bwrxMvnX6tcx67a0KR5QWrlevZn509L7s9/VzmDdvHgf+77as/74hnHru9Wy+/ghG77QJB39sOw4/6RK23vdkVl5pec79/iH1HrZ6qF7NzfzfV/fm0GPPY968ZJ/RW7POyNX4xUU3sfG6q7Pr9hvx8GPPcuSJF/Pq62/yt39O4qyLb+G6C77NTX9/kHEPTWbWq29yzS3jAPjRt/dng7WH1fmstDCNMuVOdPAd1UvmYBEjgesyc+MuO2iNmvoOyWU2+Wy9hyEBMOOOU+o9BKnNS6+/U+8hSAB8bLcdeGjC/XWJ3Pqssnautv/Puvy4z5211/2ZOaorj7nUzdMoSZLUXdRzCpyu1qVBY2Y+DSw1WUZJkiSVmGmUJEmqQaNkGp34SZIkSRUZNEqSJKkiy9OSJEk1sDwtSZIkFcw0SpIk1aIxEo1mGiVJklSZmUZJkqQaeE+jJEmSVDBolCRJUkWWpyVJkqoVlqclSZKkNmYaJUmSqhRAgyQazTRKkiSpMjONkiRJVQvvaZQkSZJaGTRKkiSpIsvTkiRJNWiQ6rSZRkmSJFVmplGSJKkGPggjSZIkFcw0SpIkVSu8p1GSJElqY9AoSZKkiixPS5IkVSmApqbGqE+baZQkSVJFZholSZJq4IMwkiRJUsFMoyRJUg2c3FuSJEkqGDRKkiSpIsvTkiRJ1fKNMJIkSdK7zDRKkiRVKfBBGEmSJKmNmUZJkqSqhZlGSZIkqZVBoyRJkiqyPC1JklSDBqlOm2mUJElSZWYaJUmSauCDMJIkSVLBTKMkSVK1fI2gJEmS9C6DRkmSJFVkeVqSJKlKvntakiRJKmPQKEmSVIOIrv9UHlOMjojHI+LJiDh2IW32i4hJETExIv5QqU/L05IkST1IRDQDZwO7A1OAsRExJjMnlbVZB/gusENmvhIRq1Tq16BRkiSpBt3wnsZtgCczczJARFwG7AVMKmvzReDszHwFIDNfrNSp5WlJkqSeZRjwXNnylGJduXWBdSPinoi4NyJGV+rUTKMkSdLSZ1BEjCtbPjczz12M/XsB6wAfBIYDd0bEJpk5a1E7SJIkqUp1qk7PzMxRC9k2FVi9bHl4sa7cFOBfmTkHeCoinqAURI5d2AEtT0uSJPUsY4F1ImLNiOgDHACMadfmWkpZRiJiEKVy9eRFdWqmUZIkqVrR/R6Eycy5EXEkcDPQDFyYmRMj4mRgXGaOKbZ9OCImAS3AtzPzpUX1a9AoSZLUw2TmDcAN7dYdX/Y9gaOKT4cYNEqSJFWp9BrBeo+ia3hPoyRJkioyaJQkSVJFlqclSZKqFt3uQZjOYqZRkiRJFZlplCRJqkGDJBrNNEqSJKkyM42SJEk18J5GSZIkqWDQKEmSpIosT0uSJFUrfBBGkiRJamOmUZIkqUqld083RqrRTKMkSZIqMtMoSZJUAzONkiRJUsGgUZIkSRVZnpYkSapBg1SnDRor2Xy9Ydz19x/VexgSAIP2O6/eQ5Da/OPMA+o9BAmAufPm1XsIDcGgUZIkqQY+CCNJkiQVzDRKkiRVy9cISpIkSe8yaJQkSVJFlqclSZKqFIQPwkiSJEmtzDRKkiTVoEESjWYaJUmSVJmZRkmSpBo0NUiq0UyjJEmSKjJolCRJUkWWpyVJkmrQINVpM42SJEmqzEyjJElSlSJwcm9JkiSplZlGSZKkGjQ1RqLRTKMkSZIqM2iUJElSRZanJUmSauCDMJIkSVLBTKMkSVINGiTRaKZRkiRJlZlplCRJqlIAQWOkGs00SpIkqSKDRkmSJFVkeVqSJKkGvhFGkiRJKphplCRJqlaEk3tLkiRJrcw0SpIk1aBBEo1mGiVJklSZQaMkSZIqsjwtSZJUpQCaGqQ+baZRkiRJFZlplCRJqkGDJBrNNEqSJKkyM42SJEk1cHJvSZIkqWDQKEmSpIosT0uSJFUpwgdhJEmSpDYGjZIkSTVoiujyTyURMToiHo+IJyPi2AVsPyQiZkTEhOLzhUp9Wp6WJEnqQSKiGTgb2B2YAoyNiDGZOald0z9l5pEd7ddMoyRJUg2iDp8KtgGezMzJmfkOcBmwV63nadAoSZK09BkUEePKPl8q2zYMeK5seUqxrr19IuKhiLgyIlavdEDL05IkSUufmZk5qob9/wL8MTPfjogvAxcDuy5qB4NGSZKkGnTDN8JMBcozh8OLdW0y86WyxfOBn1Tq1PK0JElSzzIWWCci1oyIPsABwJjyBhExpGxxT+DRSp2aaZQkSapSAE3dLNGYmXMj4kjgZqAZuDAzJ0bEycC4zBwDfC0i9gTmAi8Dh1Tq16BRkiSph8nMG4Ab2q07vuz7d4HvLk6fBo2SJEnViuiO9zR2Cu9plCRJUkUGjZIkSarI8rQkSVINGqQ6baZRkiRJlZlplCRJqoEPwkiSJEkFM42SJElV6o6Te3eWhQaNEfFLIBe2PTO/1ikjkiRJUrezqEzjuC4bhSRJkrq1hQaNmXlx+XJELJ+Zb3b+kCRJkpYePghTiIjtImIS8FixvFlEnNPpI5MkSVK30ZGnp88A/gd4CSAzHwR26sQxSZIkLTWiDp966NCUO5n5XLtVLZ0wFkmSJHVTHZly57mI2B7IiOgNfB14tHOHJUmS1P1FQJP3NLY5DDgCGAZMAzYvliVJktQgKmYaM3MmcHAXjEWSJEndVEeenn5fRPwlImZExIsR8eeIeF9XDE6SJKm7i+j6Tz10pDz9B+ByYAgwFLgC+GNnDkqSJEndS0eCxuUz85LMnFt8fg8s29kDkyRJWhpERJd/6mFR754eUHy9MSKOBS6j9C7q/YEbumBskiRJ6iYW9SDM/ZSCxNZw9stl2xL4bmcNSpIkaWnRIDPuLPLd02t25UAkSZLUfXVkcm8iYmNgQ8ruZczM33XWoCRJktS9VAwaI+IE4IOUgsYbgD2AuwGDRkmS1NCC8I0wZfYFdgOez8zPAZsB/Tp1VJIkSepWOlKefisz50XE3IhYCXgRWL2Tx6WlxG3/nMT/+9lVzJs3j0/tuR1f/+yH59v+9jtzOPykS3josedYud8KnP+DzzFi6EDGT3yao350GQCZyTFf/Agf/eBm9TgF9SC7bT6cH31ue5qbgktue4wzrn1wvu0/PGQ7dtxoCADLLdOLwf2WY+RnL+YDGw3hlEO2a2u3zrD+HPrz27hh7DNdOn71LPeOf4Izzr+Olnnz+NjuW/OZfXaeb/sDE5/izAuu5z9PP89JR+/Prttv0rbt7Itv4h/3PwbA5/bblQ99YNMuHbsWQx0n2+5qHQkax0VEf+A8Sk9Uvw78szMH1RHFmA7KzHOK5aHALzJz37oOrIG0tMzjO6ddwZW/PIKhq/Rn90NOY/SOm7De+4a0tbl0zD/pv+LyjL3qBK6+5X5OOvvPXPDDz7P+WkP560XfplevZp6fOZsPfupU/ucDG9OrV3Mdz0hLs6am4LQvfIC9T76eaS+/we2n7s2N457h8Smz2tocd9G7f3R9cY+N2HTNQQDcPXE6O337agD6912G8b/cn789OKVLx6+epaVlHqf/ZgxnnvR5Vhm4Eod++xx23GZ91lx91bY2qw3qz/e+tg9/uPbu+fa9Z9xjPDF5Ghf//KvMmdPCEd87j+22XJcVlneKZNVXxfJ0Zh6embMy89fA7sBnizJ1vfUHDm9dyMxpBoxda/ykZ1hz+CBGDhtEn9692Hv3rbjxzofna3PjnQ9zwEffD8Ceu27OXWOfIDNZftk+bQHi2+/MIWiQf6ap02y19mAmPz+bZ158jTlz53H1Pf/hI1uPXGj7fT+wFlfd/eR71u+17Zr8dcJzvPVOSyeOVj3dpH9PYfiQgQxbbQC9e/fiQx/YlLv+9eh8bYasujJrjxzynvvhnn7uRTbfcCS9mptZbtk+rL3Gatw7/omuHL4WU6NM7r3QoDEitmz/AQYAvYrvixQRIyPi0Yg4LyImRsQtEbFcRKwVETdFxP0RcVdErF+0Xysi7o2IhyPiBxHxerG+b0TcFhHji217FYc4FVgrIiZExGnF8R4p9rk3IjYqG8sdETEqIlaIiAsj4r6IeKCsL1Vh+ouzGLrqym3LQ1fpz/QZs+ZvM2M2w1bpD0CvXs2s1Hc5Xp79BgD3P/I0OxzwQ3Y66Eecfuz+ZhlVkyEDVmDqzDfalqe99AZDBqywwLarD+rLiFVW4s5Hpr1n2yd2WIur7v5Pp41TjWHGy7NZddC7t/8PHtiPGS+/2qF91x45hHsfeIL/vv0Os159g/GPTOaFmbM7a6hShy2qPP3TRWxLYNcO9L8OcGBmfjEiLgf2AT4HHJaZ/46I9wPnFH2dCZyZmX+MiMPK+vgvsHdmvhoRg4B7I2IMcCywcWZuDqUgtWyfPwH7ASdExBBgSGaOi4hTgNsz8/NFefu+iPhrZr6ButxWG4/knsuO44mnnueIky9ht+02ZNlletd7WGoAn/jAWoz552Tmzcv51q/afzk2HDGA2yY8V6eRSfD+Ldbh0Sen8OXv/Ib+/VZg4/VG0NzUkedWpc61qMm9d1kC/T+VmROK7/cDI4HtgSvKUqvLFP+/HfDx4vsfgNOL7wGcEhE7AfOAYcC7N4Us2OXALcAJlILHK4v1Hwb2jIiji+VlgRHAfDWDiPgS8CWA1UeMqHiSjWrIKv2Z9sIrbcvTXpzFkMH9528zuB9Ti4zk3LktvPr6WwzoN3/2Z901V2OF5Zbh0cnT2WIDf96qzvSX32DYoHevraEDV2D6ywv+9+AndliLb59/z3vWf3z7tbjuvqeZ25IL2EvquMED+s2XHZzx0mwGD1ipw/sf8sldOOSTpb+GT/jpZaw+dNASH6OWnEYJ6Tv7PN8u+95Cqbw9KzM3L/tsUKGPg4HBwFZFVvEFyiYZX5DMnAq8FBGbUnpX9p+KTQHsU3bsEZn56AL2PzczR2XmqEGDBnfkPBvSFhuMYPJzM3hm2kzemTOXa269n9E7bTJfm9E7bsJl1/8LgDG3T2DHUesSETwzbSZz55buGXtu+sv8+5kXGDFkwHuOIXXU+CdnsNaQfoxYZUV692riEzusxY0LePp5naH96L/CMtz3+Avv2bbPQu5zlBbXBusMY8r0mUx74WXmzJnLX+9+iA9sU+mvu5KWlnnMfvVNAJ58ejpPPvM822yxdmcOV+qQDr0RZgl6FXgqIj6ZmVdEKd24aWY+CNxLqXz9J+CAsn36AS9m5pyI2AVYo1j/GrDiIo71J+AYoF9mPlSsuxn4akR8NTMzIrbIzAeW3Ok1ll69mjn16E/yya+dw7x5yUEf25b13zeEH/3mejbfYAR77LQJB++5HYef+Du23uck+q+0POf9oPQM1b8mTObM391K717NRFNw2jH7MbB/3zqfkZZmLfOSY86/h6u+twfNTU1cevvjPDblFb67/1ZM+M9MbhxXCiA/8YG1ufqe996zuPrgvgwb2Jd7Jk3v6qGrB+rV3MxRX9yTb570W1pakv/90Fa8b8SqnPeHW1l/7eHsuM0GTPr3FL576u957fW3uHvco1zwx9u49JffYG5LC1/5f78BYIXll+WEb+xHr2bv+e6uAur2YEpXi8zOKcMU9xhel5kbF8tHA32Bi4FfAUOA3sBlmXlyRKwD/B5YDrgJODgzhxX3Mf6l2HccsC2wR2Y+HRF/ADYFbgTObne8VYGpwPcz86Ri3XLAGZRK5E2Uyuf/u6jz2HKrUXnXP8cumR+KVKNB+51X7yFIbf5x5gGVG0ld4OCP7cykhx6oS+S26tob5/6nX1m54RL2y703uD8zR3XlMTvyGsGgVCJ+XxHcjQBWy8z7FrVfZj4NbFy2fHrZ5tEL2GUqsG2RATwAWK/Ybyal+x0XdIyD2q0qP94LtDu/zHwL+PKixi1JkrQ4mhoj0dih8vQ5lB5A2RU4mVJZ+Cpg6yU8lq2As4ogdRbw+SXcvyRJkqrUkaDx/Zm5ZUQ8AJCZr0REnyU9kMy8i9J7rSVJktTNdCRonBMRzZTmZiQiBlPKPEqSJDW8RilPd2TKnV8A1wCrRMQPgbuBUzp1VJIkSepWKmYaM/PSiLgf2I3Sk+UfX9DchpIkSY0monGm3OnI09MjgDcpTXvTti4zn+3MgUmSJKn76Mg9jddTup8xKL2JZU3gcWCjThyXJEnSUqFR7mnsSHl6vvfCRcSWwOGdNiJJkiR1O4v97unMHA+8vxPGIkmSpG6qI/c0HlW22ARsCUzrtBFJkiQtRRrkOZgO3dO4Ytn3uZTucbyqc4YjSZKk7miRQWMxqfeKmXl0F41HkiRpqRFAU4OkGhd6T2NE9MrMFmCHLhyPJEmSuqFFZRrvo3T/4oSIGANcAbzRujEzr+7ksUmSJKmb6Mg9jcsCLwG78u58jQkYNEqSpIa32FPRLKUWFTSuUjw5/QjvBoutslNHJUmSpG5lUUFjM9CX+YPFVgaNkiRJOOUOwPTMPLnLRiJJkqRua1FBY4PEzZIkSdWJCKfcAXbrslFIkiSpW1to0JiZL3flQCRJktR9dWTKHUmSJC1Eg1SnG2ZqIUmSJNXATKMkSVINmsw0SpIkaWkUEaMj4vGIeDIijl1Eu30iIiNiVKU+zTRKkiRVKaDbTbkTEc3A2cDuwBRgbESMycxJ7dqtCHwd+FdH+jXTKEmS1LNsAzyZmZMz8x3gMmCvBbT7PvBj4L8d6dSgUZIkqWcZBjxXtjylWNcmIrYEVs/M6zvaqeVpSZKkGtSpOj0oIsaVLZ+bmed2ZMeIaAJ+BhyyOAc0aJQkSVr6zMzMhT28MhVYvWx5eLGu1YrAxsAdUYp4VwPGRMSemVkeiM7HoFGSJKla0S2n3BkLrBMRa1IKFg8ADmrdmJmzgUGtyxFxB3D0ogJG8J5GSZKkHiUz5wJHAjcDjwKXZ+bEiDg5Ivastl8zjZIkSTUIul+qMTNvAG5ot+74hbT9YEf6NNMoSZKkigwaJUmSVJHlaUmSpCqV3ghT71F0DTONkiRJqshMoyRJUg3MNEqSJEkFM42SJEk1iDq9R7CrmWmUJElSRQaNkiRJqsjytCRJUpWcckeSJEkqY6ZRkiSpWgEN8hyMmUZJkiRVZqZRkiSpBk0Nkmo00yhJkqSKDBolSZJUkeVpSZKkKjnljiRJklTGTKMkSVINGuQ5GDONkiRJqsxMoyRJUtWCJhoj1WimUZIkSRUZNEqSJKkiy9OSJElVCnwQRpIkSWpjplGSJKla4eTekiRJUhszjZIkSTVoapCbGs00SpIkqSKDRkmSJFVkeVqSJKlKTrkjSZIklTHTKEmSVAMfhJEkSZIKZholSZJq0CCJRjONkiRJqsygUZIkSRVZnpYkSapS0DgZuEY5T0mSJNXATKMkSVK1AqJBnoQx0yhJkqSKzDRKkiTVoDHyjGYaJUmS1AFmGisIoLmpUf4Noe7u37/9bL2HILVZZ9dv1XsIEgBvPzm13kNoCAaNkiRJVQp897QkSZLUxkyjJElSDRojz2imUZIkSR1gplGSJKkGDXJLo5lGSZIkVWbQKEmSpIosT0uSJFUtfPe0JEmS1MpMoyRJUpWCxsnANcp5SpIkqQZmGiVJkmrgPY2SJElSwaBRkiRJFVmeliRJqkFjFKfNNEqSJKkDzDRKkiRVK3wQRpIkSUupiBgdEY9HxJMRcewCth8WEQ9HxISIuDsiNqzUp0GjJElSlVon9+7qzyLHFNEMnA3sAWwIHLiAoPAPmblJZm4O/AT4WaVzNWiUJEnqWbYBnszMyZn5DnAZsFd5g8x8tWxxBSArdeo9jZIkST3LMOC5suUpwPvbN4qII4CjgD7ArpU6NdMoSZJUg4jo8g8wKCLGlX2+tLjjzsyzM3Mt4DvA9yq1N9MoSZK09JmZmaMWsm0qsHrZ8vBi3cJcBvyq0gHNNEqSJNUg6vCpYCywTkSsGRF9gAOAMfONOWKdssWPAv+u1KmZRkmSpB4kM+dGxJHAzUAzcGFmToyIk4FxmTkGODIiPgTMAV4BPlupX4NGSZKkGnTHub0z8wbghnbrji/7/vXF7dPytCRJkioyaJQkSVJFlqclSZKqVHojTDesT3cCM42SJEmqyEyjJElSDbrjgzCdwUyjJEmSKjLTKEmSVLUgvKdRkiRJKjFolCRJUkWWpyVJkmrggzCSJElSwUyjJElSlZzcW5IkSSpjplGSJKla4T2NkiRJUhuDRkmSJFVkeVqSJKkGlqclSZKkgplGSZKkGvjuaUmSJKlgplGSJKlKATQ1RqLRTKMkSZIqM2iUJElSRZanJUmSauCDMJIkSVLBTKMkSVINnNxbkiRJKphplCRJqoH3NEqSJEkFg0ZJkiRVZHlakiSpSr4RRpIkSSpjplGSJKlq4YMwkiRJUiszjZIkSdUKJ/eWJEmS2hg0SpIkqSLL05IkSTVokOq0mUZJkiRVZqZRkiSpSqXJvRsj12imUZIkSRWZaZQkSapBY+QZzTRKkiSpAwwaJUmSVJHlaUmSpFo0SH3aTKMkSZIqMtMoSZJUg2iQVKOZRkmSJFVkplGSJKkGDTK3t5lGSZIkVWbQKEmSpIosT6smf/3HJL770ytpmTePT++1Pd885MPzbX/7nTl85YRLmPDYswzotwIXnvJ5RgwdyN/+9SgnnTWGd+bMpU/vXpz8tY+z09br1eks1FPced9j/OCsa2mZN4/9PvJ+vnzQbvNtv+/B//DDs//M45On8/P/+xR77LxZ27arbx7LOb//KwCHf+pDfOJ/tu7Ssavn2W27DfjRt/aluamJS/78D864+Nb5tg9fdWXOOfHT9FtxOZqbmjjprD9z6z8m0au5iV9872A2W391mpub+NMN9/Hzi26p01moIxqkOr30ZxojYmREHFTlvq8v6fE0kpaWeXz7J5dzxZmHc+/l3+OqW+7nscnT52tzyZ//Sb+VlmP8NSfylYN24cRf/hmAgf378seffZl/XHYc55zwaQ474Xf1OAX1IC0t8zjxzKs5/9QvcuNvj+G62x/g308/P1+boauuzI+/cwAf222L+dbPevVNfvm7W7jy7K9z1Tlf55e/u4XZr73ZlcNXD9PUFJx2zH588uvnsO1+P2CfD2/FemuuNl+bbx06mmv/Op6dP/VjDj3ut5z+nf0B+PiHtmSZPr3Y4cBT2OXTP+aQvXdg9SED6nEa0nyW+qARGAksMGiMCDOpnej+iU/zvtUHMXL4IPr07sUndt+SG/7+0HxtbrzzIQ786PsB2GvXLfj72MfJTDZdb3WGDO4PwAZrDeGtt+fw9jtzuvoU1IM89NizrDFsICOGDqRP7158dNctuO0fE+drM3y1Aay/1lCiaf68wF1jH2OHrdal/0rL02/F5dlhq3W5877HunL46mG22mgkk5+byTNTX2LO3BauvnU8H9l50/kbZbLiCssCsFLf5Xh+5uxidbL8cn1obm5i2WX78M6cFl57479dfQpaHFGHTx3ULWgsMoSPRsR5ETExIm6JiOUiYq2IuCki7o+IuyJi/aL9RRGxb9n+rVnCU4EdI2JCRHwzIg6JiDERcTtwW0T0jYjbImJ8RDwcEXvV4XR7pOkzZjNs1ZXbloeuujLTZ8yer820F99t06tXMyv1XY6XZ78xX5sxt09gs/VWZ5k+vTt/0Oqxnp85myGr9G9bXm1QP15odz0uzAszZ7f9IwZgtcH9eWFmx/aVFmTI4H5MfeGVtuVpL7zCkMH95mtz6rk3sN8e2/DIdd/n8jO+wjGnXQHAn297gDffeofHbvwhD//lZM669DZmvWrmW/VX70zjOsDZmbkRMAvYBzgX+GpmbgUcDZxToY9jgbsyc/PM/Hmxbktg38zcGfgvsHdmbgnsAvw0olEeju/+Hv3PdE785Z/5+f87oN5DkaQutc//jOIP193Lxv/7f+z3jV/x65M+Q0Sw1UYjaZk3jw32OI7N9zqBIw7elTWGDaz3cLUQpcRf1/+vHuodND6VmROK7/dTKjVvD1wREROA3wBDquj31sx8ufgewCkR8RDwV2AYsOqido6IL0XEuIgYN2PmjCoO3xg68i/poau822bu3BZeff0tBvRbAYCpL7zCp485l1+d9GnWHD646wauHmm1Qf2Y/uKstuXnZ85m1XbX48KsOqgf02eU7TtjFqsO6ti+0oJ0pBLzqb2249q/jgdg7MNPsewyvRnYfwX2HT2K2/4xibkt85j5yuv868HJbLHBiC4dv7Qg9Q4a3y773gIMAGYVWcPWzwbF9rkU442IJqDPIvotr38eDAwGtsrMzYEXgGUXNajMPDczR2XmqMGDDGYWZssN1+A/z87gmakzeWfOXK6+dTx77DT/PTujd9yEP17/LwD+fPsD7LT1ukQEs197k/2/+WtOOGIvtt1srXoMXz3MJuuvztNTZ/Lc9Jd4Z85crr/9AXbbbqMO7bvj1utzz7gnmP3am8x+7U3uGfcEO269fiePWD3Z+EnPsNaIwYwYOpDevZr5xO5bcuOd89/zPfX5l9tmjVh35Kos06c3M195nSnPv8yOxfrll+3DqI1H8u+nX+jyc5Da624PirwKPBURn8zMK4oy8qaZ+SDwNLAVcDmwJ9B6A9xrwIqL6LMf8GJmzomIXYA1Om30DaZXr2Z+csx+7PO1s2lpSQ7ec1s2WGsIp/z6OjbfYAQf2XlTPr3X9hx2wu/Ycu8TWXmlFbjgh58D4LzL7+Sp52bwk/Nv5Cfn3wjA1WcdyeABi/pVSgvXq7mZE776CT7/nXNpaUn23WMb1llzNc747U1ssu5wdtthYx567FkOP/4iXn39Lf72z0n84qKbufG3x9B/peU5/NMf4hNfOQOAIz69O/1XWr6+J6SlWkvLPI75yeVc9YsjaG4OLh1zL49Nfp7vfvmjTHj0WW6882G+d8Y1nHncgRx+4C4kcMRJlwBw/hV3ctbxn+IffzqOAP7wl3uZ+OS0up6PFiEa540wkZn1OXDESOC6zNy4WD4a6AtcDPyKUlm6N3BZZp4cEasCfwaWA24CjsjMvhHRG7gZGAhcBLwCjMrMI4t+BwF/KfoeB2wL7JGZT0fE65nZd1Hj3GqrUXnPv8Yt0XOXqjXztbcrN5K6yDq7fqveQ5AAePvxy5n35ot1Cd023HSLvGTM37v8uKPW7Hd/Zo7qymPWLdOYmU8DG5ctn162efQC2r9AKeBr9Z1i/Rxg13bNLyrbbyaw3ULGsMiAUZIkqZIGSTTW/Z5GSZIkLWERMToiHo+IJyPi2AVsPyoiJkXEQ8XUhBVv3zNolCRJqkU3m9w7IpqBs4E9gA2BAyNiw3bNHqB0O9+mwJXATyqdpkGjJElSz7IN8GRmTs7Md4DLgPlebpKZf8vM1lnj7wWGV+rUoFGSJGnpM6h1Tuni86WybcOA58qWpxTrFuZQ4MZKB+xuU+5IkiQtRer2hpaZS+Lp6Yj4FDAK2LlSW4NGSZKknmUqsHrZ8vBi3Xwi4kPAccDOmVlxTjeDRkmSpBp0w8m9xwLrRMSalILFA4CDyhtExBaUXtc8OjNf7Ein3tMoSZLUg2TmXOBISi8/eRS4PDMnRsTJEbFn0ew0Si8+uSIiJkTEmEr9mmmUJEmqUgdmwKmLzLwBuKHduuPLvn9ocfs00yhJkqSKDBolSZJUkeVpSZKkWnTH+nQnMNMoSZKkisw0SpIk1aBOk3t3OTONkiRJqshMoyRJUg264eTencJMoyRJkioyaJQkSVJFlqclSZJq0CDVaTONkiRJqsxMoyRJUrW668unO4GZRkmSJFVkplGSJKkGTu4tSZIkFQwaJUmSVJHlaUmSpCoFvhFGkiRJamOmUZIkqQYNkmg00yhJkqTKzDRKkiTVokFSjWYaJUmSVJFBoyRJkiqyPC1JklQD3wgjSZIkFcw0SpIk1cDJvSVJkqSCmUZJkqQaNEii0UyjJEmSKjNolCRJUkWWpyVJkmrRIPVpM42SJEmqyEyjJElSlQIn95YkSZLamGmUJEmqVji5tyRJktTGoFGSJEkVWZ6WJEmqQYNUp800SpIkqTIzjZIkSbVokFSjmUZJkiRVZKZRkiSpauHk3pIkSVIrg0ZJkiRVZHlakiSpBr4RRpIkSSqYaZQkSapS0DAz7phplCRJUmVmGiVJkmrRIKlGM42SJEmqyKBRkiRJFVmeliRJqoFvhJEkSZIKZholSZJq4OTekiRJUsFMoyRJUg0aJNFoplGSJEmVGTRKkiSpIsvTkiRJ1QofhJEkSZLaGDRKkiTVJOrwqTCiiNER8XhEPBkRxy5g+04RMT4i5kbEvh05S4NGSZKkHiQimoGzgT2ADYEDI2LDds2eBQ4B/tDRfr2nUZIkqUpBt7yncRvgycycDBARlwF7AZNaG2Tm08W2eR3t1EyjJElSzzIMeK5seUqxriZmGiVJkpY+gyJiXNnyuZl5bmce0KBRkiSpBnWqTs/MzFEL2TYVWL1seXixriaWpyVJknqWscA6EbFmRPQBDgDG1NqpmcYKxo+/f+ZyveOZeo+jBxgEzKz3ICS8FtW9eD0uGWvU8+Dd7UGYzJwbEUcCNwPNwIWZOTEiTgbGZeaYiNgauAZYGfhYRJyUmRstql+Dxgoyc3C9x9ATRMS4RaTRpS7jtajuxOtRnSUzbwBuaLfu+LLvYymVrTvMoFGSJKkGUa+7GruY9zRKkiSpIoNGdZVOnQZAWgxei+pOvB611LA8rS7R2XNHSR3ltajuxOuxh2iM6rSZRkmSJFVmplGSJKkGDZJoNNOozhURh0XEZ4rvh0TE0LJt50fEhvUbnRpdRPSPiMPLlodGxJX1HJMaT0SMjIiDqtz39SU9HmlhDBrVqTLz15n5u2LxEGBo2bYvZOakugxMKukPtAWNmTktM/et33DUoEYCCwwaI8KKYDcXUZ9PPRg0aqGKf/0+FhGXRsSjEXFlRCwfEbtFxAMR8XBEXBgRyxTtT42ISRHxUEScXqw7MSKOjoh9gVHApRExISKWi4g7ImJUkY08rey4h0TEWcX3T0XEfcU+v4mI5nr8LFQfxTX4aEScFxETI+KW4tpZKyJuioj7I+KuiFi/aL9WRNxbXJs/aM3CRETfiLgtIsYX2/YqDnEqsFZxfZ1WHO+RYp97I2KjsrG0Xq8rFNf9fcV/B3u1H7caQxXX50XFn4Wt+7dmCU8Fdiyuw28WfwaOiYjbgdsWcf1KXcqgUZWsB5yTmRsArwJHARcB+2fmJpTui/1KRAwE9gY2ysxNgR+Ud5KZVwLjgIMzc/PMfKts81XFvq32By6LiA2K7ztk5uZAC3Dwkj9FdXPrAGcXr7eaBexDaZqSr2bmVsDRwDlF2zOBM4trc0pZH/8F9s7MLYFdgJ9GRADHAv8prslvtzvun4D9ACJiCDAkM8cBxwG3Z+Y2RV+nRcQKS/qktdRYnOtzYY4F7iquw58X67YE9s3MnVn49St1KdPequS5zLyn+P574P+ApzLziWLdxcARwFmU/mC7ICKuA67r6AEyc0ZETI6IbYF/A+sD9xT9bgWMLf58XA54sfZT0lLmqcycUHy/n1Ipb3vgirK/N5cp/n874OPF9z8ApxffAzglInYC5gHDgFUrHPdy4BbgBErBY+u9jh8G9oyIo4vlZYERwKOLd1rqIRbn+lwct2bmy8X3hV2/z1c5Zi1hjfJGGINGVZLtlmcBA9/TqPRy9G2A3YB9gSOBXRfjOJdR+ov5MeCazMziX9IXZ+Z3qxm4eoy3y763UPrLclaRfe6og4HBwFaZOScinqYU7C1UZk6NiJciYlNKGe/Dik0B7JOZjy/G8dVzLc71OZeiwhcRTUCfRfT7Rtn3xb5+pc5geVqVjIiI7YrvB1EqMY+MiLWLdZ8G/h4RfYF+xQvSvwlstoC+XgNWXMhxrgH2Ag6kFEAC3AbsGxGrAETEgIhYo9YT0lLvVeCpiPgkQJS0Xm/3UioPAhxQtk8/4MXiL9xdgNbraFHXJJRK1MdQurYfKtbdDHy1tTwYEVvUekLqURZ1fT5NqXoCsCfQu/he6Tpc2PWr7iLq8KkDg0ZV8jhwREQ8CqwM/Bz4HKXSy8OUSiW/pvQH3nUR8RBwN6V7H9u7CPh164Mw5Rsy8xVK5b01MvO+Yt0k4HvALUW/twJDlvwpail0MHBoRDwITKT0Dw6AbwBHFdfL2sDsYv2lwKjimv0MpYw2mfkScE9EPFL+MFaZKykFn5eXrfs+pb/sH4qIicWyVG5h1+d5wM7F+u14N5v4ENASEQ9GxDcX0N8Cr1+pq0Vm++qjVBIRI4HrMnPjeo9F6oiIWB54q7i94QDgwMz0SVNJnWbzLbfKv975ry4/7uAVe9+fmaO68pje0yipJ9kKOKsoHc8CPl/f4UhSz2HQqIXKzKcBs4xaamTmXSz4flpJUo0MGiVJkmrQKLNm+iCMJEmSKjLTKEmSVLVomMm9zTRK6lQR0VJMs/RIRFxRPOFcbV9t7+6NiPMjYsNFtP1gRGxfxTGejohBHV3frs3ri9q+gPYnlr1ZRpK6NYNGSZ3treKduhsD7/Dum1UAiIiqKh6Z+YViLs+F+SCl17lJUqcJSvc0dvWnHgwaJXWlu4C1iyzgXRExBpgUEc0RcVpEjI2IhyLiy9D2No2zIuLxiPgrsEprRxFxR0SMKr6PjojxxeTItxVzjB4GfLPIcu4YEYMj4qriGGMjYodi34ERcUtETIyI8+nAuxYi4tqIuL/Y50vttv28WH9bRAwu1q0VETcV+9wVEesvkZ+mJHUh72mU1CWKjOIewE3Fqi2BjTPzqSLwmp2ZW0fEMpTe0nILsAWwHrAhpXf6TgIubNfvYEpv2tip6GtAZr4cEb8GXs/M04t2fwB+npl3R8QISq8D3AA4Abg7M0+OiI8Ch3bgdD5fHGM5YGxEXFW8XWYFYFxmfjMiji/6PhI4FzgsM/8dEe8HzmHx3s0uSXVn0Cipsy0XEROK73cBF1AqG9+XmU8V6z8MbNp6vyKld+2uA+wE/DEzW4BpEXH7AvrfFrizta/MfHkh4/gQsGG8W9dZKUrvTN8J+ESx7/UR8UoHzulrEbF38X31YqwvUXqt5p+K9b8Hri6OsT2lV2+27r9MB44hSd2KQaOkzvZWZm5evqIInt4oXwV8NTNvbtfuI0twHE3Atpn53wWMpcMi4oOUAtDtMvPNiLgDWHYhzbM47qz2PwNJWtp4T6Ok7uBm4CsR0RsgItaNiBWAO4H9i3sehwC7LGDfe4GdImLNYt8BxfrXgBXL2t0CfLV1ISI2L77eCRxUrNsDWLnCWPsBrxQB4/qUMp2tmoDWbOlBlMrerwJPRcQni2NERPjWGqkH8UEYSeo651O6X3F8RDwC/IZSJeQa4N/Ftt8B/2y/Y2bOAL5EqRT8IO+Wh/8C7N36IAzwNWBU8aDNJN59ivskSkHnREpl6mcrjPUmoFdEPAqcSilobfUGsE1xDrsCJxfrDwYOLcY3EdirAz8TSepWIjPrPQZJkqSl0hZbjso77rmvy4/bf/nm+zNzVFce00yjJEmSKjJolCRJUkU+PS1JklStOj6Y0tXMNEqSJKkiM42SJElVCjrw7tEewkyjJEmSKjLTKEmSVIsGSTWaaZQkSVJFBo2SJEmqyPK0JElSDaJB6tNmGiVJklSRmUZJkqQaOLm3JEmSVDDTKEmSVIMGSTSaaZQkSVJlBo2SJEmqyPK0JElSLRqkPm2mUZIkSRWZaZQkSaqBk3tLkiRJBTONkiRJVQqc3FuSJElqY9AoSZKkiiIz6z0GSZKkpVJE3AQMqsOhZ2bm6K48oEGjJEmSKrI8LUmSpIoMGiVJklSRQaMkSZIqMmiUJElSRQaNkiRJquj/A5OK8pFKVlD0AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 792x792 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_preds = np.argmax(output_predictions.predictions, axis=1)\n",
    "names_of_labels = label_names\n",
    "\n",
    "plot_confusion_matrix(y_preds, y_valid, names_of_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push Fine-Tuned Model to HuggingFace Hub (My Profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f63330720eaa402ca48d6aa962438650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Sep25_00-44-45_Brians-Mac-mini/events.out.tfevents.1664084692.Brians-Mac-mini.3612.0: 100%|##…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "remote: Scanning LFS files for validity, may be slow...        \n",
      "remote: LFS file scan complete.        \n",
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-Finanacial_Sentiment_Analysis_5e\n",
      "   baaa797..1a669c1  main -> main\n",
      "\n",
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-Finanacial_Sentiment_Analysis_5e\n",
      "   1a669c1..2942133  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/distilbert-base-uncased-Finanacial_Sentiment_Analysis_5e/commit/1a669c16eb3c03fa5399cf88a5026c02b3ab7e55'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub(commit_message=\"All Dunn!!!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save Datasets to File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d0bd5dd769047a489e68322037a3541",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62e525bb92d348cfb1708bc9f0f2aa69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1afa9adfaf3840a5b92a63bb45f0c89c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "143818"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train'].to_csv('/Users/briandunn/Documents/nlpnn/Datasets/train_ds.csv')\n",
    "dataset['test'].to_csv('/Users/briandunn/Documents/nlpnn/Datasets/test_ds.csv')\n",
    "dataset['valid'].to_csv('/Users/briandunn/Documents/nlpnn/Datasets/eval_ds.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('nlp_ch4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a52fe47989fdc78fafbb981021cec52a6b82df6453830b9ffbd04250493e6cab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}