Binary Classification/Spam Filter- Larger Dataset/DunnBC22-distilbert-base-uncased-SpamFilter-LG.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binary Classification - SpamFilter (Larger Dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-Saved as 'distilbert-base-uncased-SpamFilter-LG' on my HuggingFace profile\n",
    "\n",
    "-Source for the Dataset: https://www.kaggle.com/datasets/uciml/sms-spam-collection-dataset\n",
    "\n",
    "@article{Sanh2019DistilBERTAD,\n",
    "  title={DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter},\n",
    "  author={Victor Sanh and Lysandre Debut and Julien Chaumond and Thomas Wolf},\n",
    "  journal={ArXiv},\n",
    "  year={2019},\n",
    "  volume={abs/1910.01108}\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "NOTE: Redirects are currently not supported in Windows or MacOs.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TOKENIZERS_PARALLELISM']='false'\n",
    "\n",
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score, f1_score\n",
    "from sklearn.dummy import DummyClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import torch\n",
    "from torch.nn.functional import cross_entropy\n",
    "import torch.nn\n",
    "\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "from transformers import AutoModelForSequenceClassification, AutoConfig, AutoTokenizer, AutoModel\n",
    "from transformers import TrainingArguments, Trainer, AdamW, get_scheduler, set_seed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Versions of Many Libraries Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pandas:            1.4.3\n",
      "NumPy:             1.22.4\n",
      "Torch:             1.12.1\n"
     ]
    }
   ],
   "source": [
    "print(\"Pandas:\".ljust(18), pd.__version__)\n",
    "print(\"NumPy:\".ljust(18), np.__version__)\n",
    "print(\"Torch:\".ljust(18), torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Constants/Values Used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "model_ckpt = \"distilbert-base-uncased\"\n",
    "device = \"cpu\"\n",
    "\n",
    "max_input_len = 50\n",
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ingest Data into Pandas DataFrame & Make Minor Edits to Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                               text\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(r'/Users/briandunn/Documents/nlpnn/Datasets/SpamFilter (larger File)/spam.csv', engine='c')\n",
    "\n",
    "data.drop(columns=['Unnamed: 2', 'Unnamed: 3', 'Unnamed: 4'], inplace=True)\n",
    "data.rename(columns={'v2' : 'text', 'v1' : 'label'}, inplace=True)\n",
    "data.drop_duplicates(inplace=True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Visualize Message Length by 'label' Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEHCAYAAABbZ7oVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAYt0lEQVR4nO3debhkdX3n8ffHlk32tq+IIDQCakAHMA0RkxiDosCoIIZRQgxGTRtHRnFL1GQUZ9RxhWhQYjsgKO64a0RAUUQB0yiyCI6KMNI20IRuFnGQ5Tt/nHOlaPrcW/d219K336/nqaeqzvo9t6vrU+f3O0uqCkmS1uRBoy5AkjS+DAlJUidDQpLUyZCQJHUyJCRJnQwJSVInQ0IaoSQvTHL+DKa/JsnTBlmT1MuQ0Bq1X0a/S7JgteE/SlJJFo6otHUiycJ2Ox48l9c5lSTbJDkpyfVJ7khyWZK/mcH8T0ly3TqsZ50uT+uGIaGp/BI4cvJNkscDDxldOVpXkmwMnAPsDOwPbA28DnhHklePsjaNF0NCU/kY8Nc9748GPto7QZJNkrwnyf9NckOSf02yWTtuQZKvJlmV5OYk303yoHbcPyRZluS2JD9N8tR2+H5JLmjnWZ7kxPYLbXJ9T2+nvyXJB5N8J8lLesa/KMmVSVYm+UaSnWe60Um2TnJyu/5lSd6aZF477oVJzm+3eWWSXyY5uGfeXZKc127XOUk+kOT0dvR57fOqJLcn2b9nvjUur8O+SX7STv+RJJu2y7g8ybN6lrlRkpuS7LOGZbwA2Ak4oqp+WVV3VdWZwCuA/5Fkq3YZlWS3nmWe2v49Nge+Djyi3ZbbkzwiyXFJzkjy6fZv8MMke/XMP6PlTfN30BAYEprKhcBWSf6g/ZJ8PnD6atO8A3g0sDewG7AD8KZ23GuA64AJYDvgjUAleQxwDLBvVW0JPAO4pp3nHuBVwAKaX7hPBf4rNKEDnAG8AXgo8FPgSZOFJDm0Xcfh7Tq/C3xyFtt9KnB3uz37AE8HXtIz/o/adS8A3gWcnCTtuE8AP2jrO47my3jSk9vnbapqi6q6oI/lrclRNH+zXWn+9v/UDv8o8Fc90x0CLK+qH61hGQcCX6+q36w2/HPApjR/+07tfAcDv263ZYuq+nU7+lDgs8B8mr/HF5NstBbL0wgZEprO5N7EgcCVwLLJEe0X2WLgVVV1c1XdBrydJkwA7gK2B3Zuf6l+t5qLhd0DbALskWSjqrqmqn4BUFUXV9WFVXV3VV0DfAj4s3Z5hwBXVNXnq+pu4P3A9T21/h3wv6rqynb824G9Z7I3kWS7dj3HVtVvqupG4ISebQK4tqo+XFX3AKe127hdkp2AfYE3VdXvqup84Mt9rHaNy5ti+hOr6ldVdTPwNu5rEjwdOGRyL4AmoD7WsYwFwPLVB7Z/t5va8bN1cVWdUVV3AcfThM4T12J5GiFDQtP5GPCXwAtZramJ5tf6Q4CL2+ahVcCZ7XCAdwM/B85KcnWS1wNU1c+BY2l+ad+Y5FOTTQtJHt02UV2f5FaaL/rJL6xHAL+aXHkbOL0dnTsD7+up5WYgNHs3/doZ2AhY3rOcDwEP65nm98FUVXe0L7do67u5Zxi99U6ha3ldepd5bbte2l/e3wOem2Qbml/mH+9Yxk00YXQ/aTrVF7TjZ6v33+hemn8jm47WU4aEplRV19J0YB8CfH610TcBvwX2rKpt2sfWVbVFO+9tVfWaqnoU8Gzg1ZN9D1X1iar6E5ov5QLe2S7zJOAqYPeq2oqm+Wiy6WU5sOPkyts9md+/p/lyemlPLdtU1WZV9f0ZbPKvgDuBBT3L2Kqq9uxj3uXA/CS9nfuP7Hm9ri653LvMnYDeZpnTaJqcjgAuqKplrNk5wMFtX0Cv59Js/4Xt+zu4/8EKD+953bU9v6+v7YPasafG2SxPI2RIqB8vBg5Yvf26/ZX4YeCEJA8DSLJDkme0r5+ZZLf2y/wWmmame5M8JskBSTYB/h9N0NzbLnZL4Fbg9iSPBV7Ws8qvAY9Pclj7i/fl3P9L5l+BNyTZs13/1kmOmGbbNkmy6eQDuAE4C3hvkq2SPCjJrkn+bJrlTAbqUuC4JBu3HdPP6plkRbudj5puWdN4eZIdk8wH/hH4dM+4LwJPAF7JA/f8en2M5hf+Z9McmrtR++/2fuC4qrqlne4S4C+TzEtyEPc1/UHzt3pokq1XW/YfJjm8/Tc6lvuHzmyWpxEyJDStqvpFVS3tGP0PNE1KF7bNQ+cAj2nH7d6+vx24APhgVZ1L0x/xDpo9ketpmnLe0M7zWprmrdtoAuj3X4BVdRPNL+R3Af8B7EHzpXxnO/4LNHskn2pruZymyWUqt9OE1OTjAJo+mI2BnwAraTrLH9A00+Eomk7f/wDe2tY/Wd8dNH0I32ubsmbbTv8JmiC7GvhFux7adfyWpvN5Fx6450fPdHcCT6PZc7qIJpiPB/6xqt7dM+kraYJuVbttX+xZxlU0BwZc3W7PZJPSl4Dn0fztXgAc3vZPzHZ5GqF40yGtr9qmjOuAo9rwGTtJPg1cVVVvHuI63wQ8uqr+atqJ1/26jwN2G8W6NRjuSWi9kuQZac4U3oT7+isunGa2oUmyb9s89aC2OeVQen4tD2H982maB5cMa52a2wwJrW/2p2liuYmm2eKwtollXDwc+DZNM9b7gZd1nKewziX5W5rmo69X1XnTTS/1w+YmSVIn9yQkSZ0MCUlSp7G4ZPF0FixYUAsXLhx1GZI0Z1188cU3VdXE6sPXi5BYuHAhS5d2HaYvSVpbSa5d03CbmyRJnQwJSVInQ0KS1MmQkCR1GlhItFfV/EGSHye5Islb2uGnprlF4yXtY+9B1SBJWjuDPLrpTprLS9/e3rrw/CRfb8e9rqrOGOC6JUnrwMBCor1r2O3t243ah9cAkaT1yED7JNobi1wC3AicXVUXtaPeluTSJCe0V/OUJI2hgYZEVd1TVXvT3L5wvySPo7m5zGNpbhg/n+amNQ+QZHGSpUmWrlixYpBlShozyeweWveGcnRTVa0CzgUOqqrl1bgT+AiwX8c8S6pqUVUtmph4wJnikuawqjU/phrnBa0HY5BHN00k2aZ9vRlwIHBVku3bYQEOo7nFpCRpDA3y6KbtgdOSzKMJo89U1VeTfCvJBM0dxS4B/m6ANUiS1sIgj266FNhnDcMPGNQ6JUnr1npxFVitndl26NnGK8mQ2AB0fdknBoGkqXntJklSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUyZCQJHUyJCRJnQwJSVInQ0KS1MmQkCR1MiQkSZ0MCUlSJ0NCktTJkJAkdTIkJEmdDAlJUidDQpLUaWAhkWTTJD9I8uMkVyR5Szt8lyQXJfl5kk8n2XhQNUiS1s4g9yTuBA6oqr2AvYGDkjwReCdwQlXtBqwEXjzAGiRJa2FgIVGN29u3G7WPAg4AzmiHnwYcNqgaJElrZ6B9EknmJbkEuBE4G/gFsKqq7m4nuQ7YoWPexUmWJlm6YsWKQZYpSeow0JCoqnuqam9gR2A/4LEzmHdJVS2qqkUTExODKlGSNIWhHN1UVauAc4H9gW2SPLgdtSOwbBg1SJJmbpBHN00k2aZ9vRlwIHAlTVj8RTvZ0cCXBlWDJGntPHj6SWZte+C0JPNowugzVfXVJD8BPpXkrcCPgJMHWIMkaS0MLCSq6lJgnzUMv5qmf0KSNOY841qS1MmQkCR1MiQkSZ0MCUlSJ0NCktTJkJAkdTIkJI3E/PmQzOwBM59n/vzRbuf6bpAn00lSp5UroWrw65kMF82OexKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6DSwkkjwyyblJfpLkiiSvbIcfl2RZkkvaxyGDqkGStHYGeRXYu4HXVNUPk2wJXJzk7HbcCVX1ngGuW5K0DgwsJKpqObC8fX1bkiuBHQa1PknSujeUPokkC4F9gIvaQcckuTTJKUm27ZhncZKlSZauWLFiGGVKGrKZ3kBoNg+tnYGHRJItgM8Bx1bVrcBJwK7A3jR7Gu9d03xVtaSqFlXVoomJiUGXKWkEqgb/0NoZaEgk2YgmID5eVZ8HqKobquqeqroX+DCw3yBrkCTN3iCPbgpwMnBlVR3fM3z7nsmeA1w+qBokSWtnkEc3/THwAuCyJJe0w94IHJlkb6CAa4CXDrAGSdJaGOTRTecDa+o2+rdBrVOStG55xrUkqZMhIUnqNG1IJPlmP8MkSXNPZ59Ekk2BhwAL2hPeJvsXtsIzpyVpgzBVx/VLgWOBRwA/7Bl+K3DiAGuSJI2JzpCoqvcB70vy36rqX4ZYkyRpTPTTcX1Kkn9KsgQgye5JnjnguiRJY6CvkAB+Bzypfb8MeOvAKpIkjY1+QmLXqnoXcBdAVd3Bmk+SkyTNMf2ExO+SbEZzGQ2S7ArcOdCqJEljoZ/LcrwZOBN4ZJKP01yT6YWDLEqSNB6mDYmqOjvJD4En0jQzvbKqbhp4ZZKkkevnjOsABwN/WFVfBR6SxHtASNIGoJ8+iQ8C+wNHtu9vAz4wsIokSWOjnz6JP6qqJyT5EUBVrUyy8YDrkiSNgX72JO5KMo/7jm6aAO4daFWSpLHQT0i8H/gC8LAkbwPOB94+0KokSWOhn6ObPp7kYuCpNEc3HVZVVw68MklzXoZwWu622w5+HXPZtCGR5H8C5wGnVtVvBl+SpA1B1cznSWY3n2avn+amq2mObFqa5AdJ3pvk0AHXJUkaA9OGRFV9pKpeBPw5cDpwRPs8pSSPTHJukp8kuSLJK9vh85OcneRn7bM7g5I0pvo5me5/J/k+cBJN89RfAP18sd8NvKaq9qA5W/vlSfYAXg98s6p2B77ZvpckjaF+mpseCswDVgE3AzdV1d3TzVRVy6vqh+3r24AraW57eihwWjvZacBhM65akjQU/Rzd9ByAJH8APAM4N8m8qtqx35UkWQjsA1wEbFdVy9tR1wPbdcyzGFgMsNNOO/W7KknSOtQZEkmOqaoT27vQ/SnwZGAb4FvAd/tdQZItgM8Bx1bVrek55q2qKskaj1WoqiXAEoBFixZ5PIMkjcBUexIvAk4EDqIJhfdV1a9nsvAkG9EExMer6vPt4BuSbF9Vy5NsD9w4i7olSUPQz9FNx1TVp2cREAFOBq6squN7Rn0ZOLp9fTTwpZksV5I0PFPtSfynJLeuYXhoWoq2mmbZfwy8ALgsySXtsDcC7wA+k+TFwLXAf5lZyZKkYZkqJC6rqn1mu+CqOp/ue2E/dbbLlSQNTz+HwEqSNlBThcRnh1aFJGksdYZEVXk5cEnawNncJEnq1BkSSfZPhnG1d0nSuJpqT+KvgYuTfCrJC5M8fFhFSZLGQ+chsFX1MoAkjwUOBk5NsjVwLnAm8L2qumcoVWpa8+fDypUzn2+m+4rbbgs33zzz9UhaP/Vzgb+rgKuAE5JsRnNfiSOA44FFgy1P/Vq5cjh37LIBUtqwTBsSvarqt8C/tQ9J0hzn0U2SpE6GhCSpUz+3L908yYPa149O8uz2EuCSpDmunz2J84BNk+wAnEVzZddTB1mUJGk89BMSqao7gMOBD1bVEcCegy1LkjQO+gqJJPsDRwFfa4fNG1xJkqRx0U9IHAu8AfhCVV2R5FE0J9RJkua4fk6m+w7wnZ73VwOvGGRRkqTx0BkSSb4CdJ7DW1XPHkhFkqSxMdWexHva58OBhwOnt++PBG4YZFGSpPEw1QX+vgOQ5L1V1XuNpq8kWTrwyiRJI9dPx/XmbWc1AEl2ATafbqYkpyS5McnlPcOOS7IsySXt45DZlS1JGoZ+LvB3LPDtJFcDAXYGFvcx36nAicBHVxt+QlW954GTS5LGzZQh0V6OY2tgd+Cx7eCrqurO6RZcVeclWbjWFUqSRmbK5qaquhf4+6q6s6p+3D6mDYhpHJPk0rY5atuuiZIsTrI0ydIVK1as5SolSbPRT5/EOUlem+SRSeZPPma5vpOAXYG9geXAe7smrKolVbWoqhZNTEzMcnWSpLXRT5/E89rnl/cMK+BRa5h2SlX1+0Nnk3wY+OpMlyFJGp5+zrjeZV2tLMn2VbW8ffsc4PKpppckjda0IdHeO+JlwJPbQd8GPlRVd00z3yeBpwALklwHvBl4SpK9afZErgFeOsu6JUlD0E9z00nARsAH2/cvaIe9ZKqZqurINQw+eUbVSZJGqp+Q2Leq9up5/60kPx5UQZKk8dHP0U33JNl18k179vU9gytJkjQu+tmTeB1w7mpnXP/NQKuSJI2FqS4VfizwfZp7SewOPKYd9dN1cEKdJGk9MFVz047APwM3AmcBzwd2oo+L+0mS5oapLhX+WoAkGwOLgCfRNDMtSbKqqvYYTomSpFHpp09iM2Armgv9bQ38GrhskEVJksbDVH0SS4A9gduAi2j6J46vqpVDqk2SNGJT9UnsBGwCXA8sA64DVg2hJknSmJiqT+KgJKHZm3gS8BrgcUluBi6oqjcPqUZJ0ohM2SdRVQVcnmQVcEv7eCawH821mCRJc9hUfRKvoNmDeBJwF02fxPeBU7DjWpI2CFPtSSwEPgu8qufy3pKkDchUfRKvHmYhkqTx088F/iRJGyhDQpLUyZCQJHXq57IcWk8ko65AWjem+ixPNa5q3deyoTMk5pBh/AcxiDQMftmPj4E1NyU5JcmNSS7vGTY/ydlJftY+bzuo9UuS1t4g+yROBQ5abdjrgW9W1e7AN9v3kqQxNbCQqKrzgJtXG3wocFr7+jTgsEGtX5K09oZ9dNN2PWdvXw9sN+T1S5JmYGSHwLYXD+zsnkqyOMnSJEtXrFgxxMokSZOGHRI3JNkeoH2+sWvCqlpSVYuqatHExMTQCpQk3WfYIfFl4Oj29dHAl4a8fknSDAzyENhPAhcAj0lyXZIXA+8ADkzyM+Bp7XtJ0pga2Ml0VXVkx6inDmqdkqR1y2s3SZI6GRKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6jSwC/xp+JLBr2PbbQe/Dknjw5CYI6rzHn/dktnNJ2nDYXOTJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSeo0kkNgk1wD3AbcA9xdVYtGUYckaWqjPE/iz6vqphGuX5I0DZubJEmdRhUSBZyV5OIki0dUgyRpGqNqbvqTqlqW5GHA2Umuqqrzeidow2MxwE477TSKGiVpgzeSPYmqWtY+3wh8AdhvDdMsqapFVbVoYmJi2CVKkhhBSCTZPMmWk6+BpwOXD7sOSdL0RtHctB3whTTXtX4w8ImqOnMEdUiSpjH0kKiqq4G9hr1eSdLMeQisJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqZMhIUnqZEhIkjoZEpKkToaEJKmTISFJ6mRISJI6GRKSpE6GhCSpkyEhSeo0intca8ia24nPfFzVuq9F0vrFkNgA+GUvabZG0tyU5KAkP03y8ySvH0UNkqTpDT0kkswDPgAcDOwBHJlkj2HXIUma3ij2JPYDfl5VV1fV74BPAYeOoA5J0jRGERI7AL/qeX9dO0ySNGbG9hDYJIuTLE2ydMWKFaMuR5I2SKMIiWXAI3ve79gOu5+qWlJVi6pq0cTExNCKkyTdZxQh8e/A7kl2SbIx8HzgyyOoQ5I0jaGfJ1FVdyc5BvgGMA84paquGHYdkqTppdaDM62SrACuHXUdc9AC4KZRFyHNgJ/Zwdm5qh7Qtr9ehIQGI8nSqlo06jqkfvmZHb6xPbpJkjR6hoQkqZMhsWFbMuoCpBnyMztk9klIkjq5JyFJ6mRIzEFJFia5fNR1SFr/GRKSpE6GxNw1L8mHk1yR5KwkmyX52yT/nuTHST6X5CEASU5NclKSC5NcneQpSU5JcmWSU0e8HZqjkmye5Gvt5/HyJM9Lck2SdyW5LMkPkuzWTvusJBcl+VGSc5Js1w4/LslpSb6b5Nokh/fMf2aSjUa7les/Q2Lu2h34QFXtCawCngt8vqr2raq9gCuBF/dMvy2wP/AqmmtpnQDsCTw+yd5DrFsbjoOAX1fVXlX1OODMdvgtVfV44ETgn9th5wNPrKp9aO5B8/c9y9kVOAB4NnA6cG47/2+B/zzwrZjjDIm565dVdUn7+mJgIfC49hfXZcBRNCEw6SvVHOp2GXBDVV1WVfcCV7TzSuvaZcCBSd6Z5E+r6pZ2+Cd7nvdvX+8IfKP97L6O+392v15Vd7XLm8d9YXMZfnbXmiExd93Z8/oemos5ngoc0/7Keguw6Rqmv3e1ee9lBBeC1NxXVf8HeALNl/lbk7xpclTvZO3zvwAntp/dl7KGz277o+auuu+4fj+764AhsWHZEljettMeNepitGFL8gjgjqo6HXg3TWAAPK/n+YL29dbcd9+Zo4dWpEzZDcx/By4CVrTPW462HG3gHg+8O8m9wF3Ay4AzgG2TXEqzh3BkO+1xwGeTrAS+Bewy/HI3TJ5xLWlsJLkGWFRVXg58TNjcJEnq5J6EJKmTexKSpE6GhCSpkyEhSepkSEiSOhkSkqROhoQkqdP/B9SOWAiuLlkOAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data['message_len'] = data['text'].str.split().apply(len)\n",
    "data.boxplot(\"message_len\", by=\"label\", grid=False, showfliers=False, color=\"Blue\")\n",
    "\n",
    "plt.suptitle('')\n",
    "plt.title('Message Length by Output')\n",
    "plt.xlabel('')\n",
    "plt.ylabel('Words / Tweet')\n",
    "plt.show()\n",
    "\n",
    "data = data.drop(columns=['message_len'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert 'label' Feature from String to Numerical Value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['label'].replace({'ham' : 0, 'spam' : 1}, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert Data from Pandas DataFrame to Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['label', 'text', '__index_level_0__'],\n",
      "    num_rows: 5158\n",
      "})\n",
      "{'label': Value(dtype='int64', id=None), 'text': Value(dtype='string', id=None), '__index_level_0__': Value(dtype='int64', id=None)}\n"
     ]
    }
   ],
   "source": [
    "dataset = Dataset.from_pandas(data)\n",
    "dataset.shuffle(seed=42)\n",
    "print(dataset)\n",
    "print(dataset.features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split Dataset into Train/Test/Valid Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training dataset shape: (4126, 3)\n",
      "Validation dataset shape: (516, 3)\n",
      "Testing dataset shape: (516, 3)\n"
     ]
    }
   ],
   "source": [
    "train_testvalid = dataset.train_test_split(test_size=0.20)\n",
    "test_valid = train_testvalid['test'].train_test_split(test_size=0.50)\n",
    "\n",
    "ds = DatasetDict({\n",
    "    'train' : train_testvalid['train'],\n",
    "    'test' : test_valid['test'],\n",
    "    'valid' : test_valid['train']\n",
    "})\n",
    "\n",
    "print(\"Training dataset shape:\", ds['train'].shape)\n",
    "print(\"Validation dataset shape:\", ds['valid'].shape)\n",
    "print(\"Testing dataset shape:\", ds['test'].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenize Entire Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe721f6923284a0cab6166f71fa81f04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d93b2da0cc024ca6a0caf1e739753efc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0612b055f8834f459b3a5ec4393750a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "def tokenize(batch):\n",
    "    return tokenizer(batch['text'], truncation=True, padding=True, max_length=max_input_len) \n",
    "\n",
    "encoded_data = ds.map(tokenize, batched=True, batch_size=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print Sample to Ensure it Works as Expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['label', 'text', '__index_level_0__', 'input_ids', 'attention_mask']\n",
      "{'label': 0, 'text': 'Ha... U jus ate honey ar? So sweet...', '__index_level_0__': 3664, 'input_ids': [101, 5292, 1012, 1012, 1012, 1057, 18414, 2015, 8823, 6861, 12098, 1029, 2061, 4086, 1012, 1012, 1012, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}\n"
     ]
    }
   ],
   "source": [
    "print(encoded_data['train'].column_names)\n",
    "print(encoded_data['train'][240])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.bias', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight', 'vocab_transform.weight', 'vocab_projector.weight']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "num_labels=2\n",
    "labels_dict = {0 : 'Non-Spam', 1 : 'Spam'}\n",
    "\n",
    "model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt, num_labels=num_labels).to(device))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reminder to Enter HuggingFace Access Token; Make Sure That git lfs is Installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Git LFS initialized.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Used this the first time to install the git lfs; otherwise, it lets me know that git lfs has been initialized\n",
    "!git lfs install\n",
    "\n",
    "# Access Token ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Compute_Metrics Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    f1 = f1_score(labels, preds, average=\"weighted\")\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\"accuracy\": acc, \"f1\": f1}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=64\n",
    "num_of_epochs=3\n",
    "logging_steps = round(((len(ds['train']) / batch_size)), 0) // 2\n",
    "model_name = f\"{model_ckpt}-SpamFilter-LG\"\n",
    "learning_rate=2e-5\n",
    "\n",
    "args = TrainingArguments(output_dir=model_name,\n",
    "                         num_train_epochs=num_of_epochs,                          \n",
    "                         per_device_train_batch_size=batch_size, \n",
    "                         per_device_eval_batch_size=batch_size, \n",
    "                         learning_rate=learning_rate,\n",
    "                         weight_decay=0.01, \n",
    "                         evaluation_strategy=\"epoch\", \n",
    "                         disable_tqdm=False,\n",
    "                         push_to_hub=True, \n",
    "                         hub_strategy='every_save',\n",
    "                         logging_steps=logging_steps,\n",
    "                         log_level=\"error\",\n",
    "                         save_strategy='steps',\n",
    "                         save_steps=logging_steps,\n",
    "                         save_total_limit=12,\n",
    "                         group_by_length=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Instantiate Optimizer (& Learning Rate Scheduler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer_name = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "\n",
    "num_training_steps = num_of_epochs * len(ds['train'])\n",
    "\n",
    "lr_scheduler = get_scheduler(\"linear\",\n",
    "                             optimizer=optimizer_name,\n",
    "                             num_warmup_steps=0,\n",
    "                             num_training_steps=num_training_steps)\n",
    "\n",
    "optimizer = [optimizer_name, lr_scheduler]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/briandunn/Documents/nlpnn/nlp_ch4/bin/distilbert-base-uncased-SpamFilter-LG is already a clone of https://huggingface.co/DunnBC22/distilbert-base-uncased-SpamFilter-LG. Make sure you pull the latest changes with `repo.git_pull()`.\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(model=model, \n",
    "                  args=args, \n",
    "                  compute_metrics=compute_metrics,\n",
    "                  train_dataset=encoded_data['train'], \n",
    "                  eval_dataset=encoded_data['valid'], \n",
    "                  tokenizer=tokenizer, \n",
    "                  optimizers=optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ac44af0060d4c9b97add013841a76ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/195 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2612, 'learning_rate': 1.9948295362740345e-05, 'epoch': 0.49}\n",
      "{'loss': 0.0568, 'learning_rate': 1.9896590725480695e-05, 'epoch': 0.98}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d832568f62a5413bb59759ed77849b00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.056810230016708374, 'eval_accuracy': 0.9786821705426356, 'eval_f1': 0.9791275964512537, 'eval_runtime': 54.1107, 'eval_samples_per_second': 9.536, 'eval_steps_per_second': 0.166, 'epoch': 1.0}\n",
      "{'loss': 0.0294, 'learning_rate': 1.9844886088221038e-05, 'epoch': 1.48}\n",
      "{'loss': 0.03, 'learning_rate': 1.9793181450961385e-05, 'epoch': 1.97}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "945e5d0ad9354dc5a19dd1051335836d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.05329200252890587, 'eval_accuracy': 0.9806201550387597, 'eval_f1': 0.9810789657376363, 'eval_runtime': 53.7662, 'eval_samples_per_second': 9.597, 'eval_steps_per_second': 0.167, 'epoch': 2.0}\n",
      "{'loss': 0.0151, 'learning_rate': 1.974147681370173e-05, 'epoch': 2.46}\n",
      "{'loss': 0.0241, 'learning_rate': 1.9689772176442078e-05, 'epoch': 2.95}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86cc85adf067466eae3225d7f24ba9d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.04997539520263672, 'eval_accuracy': 0.9844961240310077, 'eval_f1': 0.9847762068640159, 'eval_runtime': 53.9707, 'eval_samples_per_second': 9.561, 'eval_steps_per_second': 0.167, 'epoch': 3.0}\n",
      "{'train_runtime': 4114.7674, 'train_samples_per_second': 3.008, 'train_steps_per_second': 0.047, 'train_loss': 0.0684305588595378, 'epoch': 3.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=195, training_loss=0.0684305588595378, metrics={'train_runtime': 4114.7674, 'train_samples_per_second': 3.008, 'train_steps_per_second': 0.047, 'train_loss': 0.0684305588595378, 'epoch': 3.0})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inference Using Fine-Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77097604cded44f582d5029aa7597894",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/9 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.04997539520263672,\n",
       " 'test_accuracy': 0.9844961240310077,\n",
       " 'test_f1': 0.9847762068640159,\n",
       " 'test_runtime': 53.9716,\n",
       " 'test_samples_per_second': 9.561,\n",
       " 'test_steps_per_second': 0.167}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_predictions = trainer.predict(encoded_data['valid'])\n",
    "\n",
    "output_predictions.metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Function to Calculate & Display Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(y_preds, y_true, labels):\n",
    "    cm = confusion_matrix(y_true, y_preds, normalize=\"true\")\n",
    "    fig, ax = plt.subplots(figsize=(6,6))\n",
    "    disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels)\n",
    "    disp.plot(cmap=\"Blues\", values_format=\".2f\", ax=ax)\n",
    "    plt.title('Normalized Confusion Matrix')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Calculate Baseline for Confusion Matrix Calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cace37549ec14f99a11cc2b78910746c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44018a9ee1294a978901858d3a5f2c82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b9ffe94c2b340399b0dd7ee1a897e82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0fcd8ebc7f047719a563bb90616dac7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de55ee72f2d54b26b87d3d3d58159d04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5f26855f5d34b52ad3a3c85c683606d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "((4126, 768), (516, 768))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels=2\n",
    "\n",
    "tokenizer2 = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "def tokenize(batch):\n",
    "    return tokenizer(batch['text'], truncation=True, padding=True, max_length=128, return_tensors=\"np\")\n",
    "\n",
    "encoded_data = ds.map(tokenize, batched=True, batch_size=None)\n",
    "\n",
    "model2 = (AutoModel.from_pretrained(model_ckpt, num_labels=num_labels, output_hidden_states=True).to(device))\n",
    "\n",
    "def extract_hidden_states(batch=True):\n",
    "    inputs = {k:v.to(device) for k,v in batch.items() if k in tokenizer.model_input_names}\n",
    "    with torch.no_grad():\n",
    "        last_hidden_state = model2(**inputs).last_hidden_state\n",
    "    return {\"hidden_state\": last_hidden_state[:,0].cpu().numpy()}\n",
    "\n",
    "encoded_data.set_format('torch', columns=['input_ids', 'attention_mask', 'label'])\n",
    "\n",
    "hidden_data = encoded_data.map(extract_hidden_states, batched=True)\n",
    "\n",
    "X_train = np.array(hidden_data[\"train\"][\"hidden_state\"])\n",
    "X_valid = np.array(hidden_data[\"valid\"][\"hidden_state\"])\n",
    "y_train = np.array(hidden_data[\"train\"][\"label\"])\n",
    "y_valid = np.array(hidden_data[\"valid\"][\"label\"])\n",
    "X_train.shape, X_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAFWCAYAAACivMxMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjV0lEQVR4nO3de5yVZbn/8c93hoMIgiCoKKBoSOAZ8XzOQ6BuyQ4qmmW7tlqSbrP4abXRKG2HWZrpLlPzmG5M3WKCaJZhnuKgmaAooYigAop4QoHh+v2xntE10zCzGGbNmnue75vXer3Wc1j3cz1rFte61n0/B0UEZmaWhqpKB2BmZqVz0jYzS4iTtplZQpy0zcwS4qRtZpYQJ20zs4R0qHQAZmaVUN19m4g1K5v9+li5dGpEjGjBkEripG1muRRrPqDzJ09s9us/ePKK3i0YTsmctM0snwRIlY5ivblP28wsIa60zSy/lF7d6qRtZvmVYPeIk7aZ5ZRcaZuZJSXBSju9rxkzsxxzpW1m+STcPWJmlg4l2T3ipG1m+ZVgpZ1exGZmOeaknSOSHpL0tez5yZLub+H2t5UUklr9F5ykLpLukbRC0u0b0E6Lvy+VIGmKpC9XOo42T2r+o0KctFuQpJckLZHUtWje1yQ9VMGwGhQRt0TEka29XUknSZoh6V1Jr2bJ5YAWaPrzwBbAZhHxheY2Uq73RdIh2RfaXfXm75rNf6jEdi6UdHNT60XEyIi4oZnh5kR2nHZzHxXipN3yqoGzN7QRFbSrv4+kbwGXARdTSLADgKuAUS3Q/DbA8xGxpgXaKpelwL6SNiua92Xg+ZbaQHv83JRN7QWjXGnn3iXAtyVt2tBCSftJmp79jJ8uab+iZQ9JukjSI8D7wHZZFfYNSS9IekfSDyVtL+lRSW9LmiipU/b6npL+IGmppOXZ837riONUSX/Nno/NKt/ax2pJ12fLeki6NquKF0n6kaTqbFm1pJ9KWiZpPnD0ut4UST2A8cCZEXFnRLwXEasj4p6I+E62TmdJl0lanD0uk9Q5W3aIpFcknZv9mnlV0leyZT8AxgEnZPF/tX5FWr/rJtv/+dl7+qKkk+u/LyX+vX4o6ZGsnfslNXa5zlXA/wEn1r5/wAnALfXeq8slLcz+vjMlHZjNHwF8t2g//14UR/3PTXFX2P9IuqOo/Z9IelBK8NCJluZK24AZwEPAt+svkNQLuBf4BbAZ8DPg3nqV1ynAacAmwIJs3qeBPYB9gLHA1cAXgf7ATsDobL0q4LcUqs4BwErgl00FHBETIqJbRHQDhlCoCP83W3w9sAb4BLA7cCTwtWzZfwDHZPOHU+iiWJd9gY2AuxpZ53vZPu4G7ArsBXy/aPmWQA9ga+CrwJWSekbEBRSq9//N9uPaxvZXhe6rXwAjI2ITYD/gqQbWK+XvdRLwFWBzoBMN/N3ruRH4Uvb808AzwOJ660yn8B70An4H3C5po4i4r95+7lr0moY+N7XOBXbOvpAOpPDefTkioolYrQ1y0i6PccA3JfWpN/9o4IWIuCki1kTErcBzwL8VrXN9RMzOlq/O5k2IiLcjYjaF/+T3R8T8iFgBTKGQNImINyLijoh4PyLeAS4CDi41aEldKFSCl0fEFElbAEcB/5lVxkuAn5NVisDxwGURsTAi3gR+3EjzmwHLmui+OBkYHxFLImIp8AMKyajW6mz56oiYDLwLDC51/+pZC+wkqUtEvJq9t/WV8vf6bUQ8HxErgYkUku06RcSjQC9Jgykk7xsbWOfm7G+5JiIuBTrT9H429Lmpbe99Cu/jz4CbgW9GxCtNtJcD7tO2TEQ8A/wBOK/eoq341ypoAYXKsdbCBpp8vej5ygamuwFI2ljSryUtkPQ2MA3YtLY7owTXAnMj4ifZ9DZAR+BVSW9Jegv4NYWqsnZ/iuOtv2/F3gB6q/EjS+q/PwuyeR+1US/pv0+27+sjIt6j0C1xBoV9u1fSJ0uIpzam4r/Xa82I5yZgDHAoDfzykPRtSc9mXTJvUfh10dRdUhr63HwkIp4A5lPoyZ1YQoz5UKXmPyoVcsW23P5dQKH7oPg/+GIKibDYAGBR0fSG/GQ9l0JFtndEdAcOyuY3+QmTdB6wA4WfzrUWAh8CvSNi0+zRPSJ2zJa/SqGLptaARjbxWNbWZxpZp/77M4B/7Too1XvAxkXTWxYvjIipEXEE0JdC9fybEuKpjWlRA+uuj5uAbwCTsyr4I1n3xVgKv2J6RsSmwAo+/huu6/PR6OdG0pkUKvbFWftWexq7K20DiIh5FPqFzyqaPRnYQYXD3jpIOgEYSqEqbwmbUKi838r6Yy8o5UWSRmZxHpf9zK/dh1eB+4FLJXWXVKXCIGhtl8tE4CxJ/ST15F9/WVDU1goK3UZXSvpM9qugo6SRkiZkq90KfF9Sn2xAbxyFn/PN8RRwkKQB2SDo+UX7u4WkUVnf9ocUulnWNtBGWf5eEfEihW6r7zWweBMKYwhLgQ6SxgHdi5a/Dmyr9ThCRNIOwI8ojIOcAoyVtFvzom9nfPSI1TMe+OiY7Yh4g8LA3bkUugvGAsdExLIW2t5lQBdgGfA4cF+JrzsB6AM8q4+PIPlVtuxLFAbY5gDLgd9TqE6hUJ1OBf4OzALubGwjWf/stygMLi6lUMmPodCPDoXEMgN4GvhH1uaPStyH+tt6gMKX5tPATOom2qosjsXAmxQS6NcbaKNsf6+I+GtENPQrYiqFv9vzFLpiPqBu10ftiUNvSJrV1Hay7qibgZ9ExN8j4gUKR6DcVHtkjqVFHkA2szyq6t4vOu81ptmv/+DB82dGxPAWDKkkvmCUmeVXgoeqO2mbWX4lePJoehGbmeWYK20zy6cKHwXSXE7aZpZfCXaPtKmkrQ5dQp02qXQY1sbsPqSxc3YsjxYseIlly5ZteJnsSnvDqNMmdB58fKXDsDbmkSeavOaV5cz+e7fEkXZKstJOL2IzsxxrU5W2mVmrcveImVkiai8YlRgnbTPLqTT7tJ20zSy/EuweSe9rxswsx1xpm1l+uXvEzCwhCXaPOGmbWT4pzYHI9CI2M8sxV9pmll/uHjEzS4ectM3M0iCctM3M0qHskRgPRJqZJcSVtpnllNw9YmaWEidtM7OEOGmbmSUkxaTtgUgzs4S40jazfEr0kD8nbTPLJfnoETOztKSYtN2nbWaWEFfaZpZbKVbaTtpmlltO2mZmqUj06BH3aZtZbklq9qPE9kdImitpnqTzGlg+QNKfJT0p6WlJRzXVppO2mVkZSKoGrgRGAkOB0ZKG1lvt+8DEiNgdOBG4qql23T1iZrnUCsdp7wXMi4j5AJJuA0YBc4rWCaB79rwHsLipRp20zSy3ypy0twYWFk2/Auxdb50LgfslfRPoChzeVKPuHjGz/NIGPKC3pBlFj9OaEcFo4PqI6AccBdwkqdG87ErbzKx5lkXE8EaWLwL6F033y+YV+yowAiAiHpO0EdAbWLKuRl1pm1k+qexHj0wHBkkaKKkThYHGSfXWeRk4DEDSEGAjYGljjbrSNrPcKmefdkSskTQGmApUA9dFxGxJ44EZETEJOBf4jaRzKAxKnhoR0Vi7TtpmllvlPiMyIiYDk+vNG1f0fA6w//q06aRtZrmU6qVZ3adtZpYQV9pmll/pFdpO2maWU/JV/szMkuKkbWaWkBSTtgcizcwS4krbzPIrvULbSdvM8ivF7hEnbTPLpfW5A01b4j5tM7OEuNI2s9xKsdJ20jaz3HLSNjNLSXo520nbzPIrxUrbA5FmZglxpW1m+eQLRpmZpUNAgjnbSdvM8irNk2uctM0stxLM2R6INDNLiSttM8std4+YmaVCaXaPOGmbWS4JqKpKL2s7abeyw/Ydwo/P/TzVVVXcdPejXHbDA3WW99+yJ1eM+yK9N+3G8rff5/RxN7B4yVsA/OCbozjigJ2oknjoiec479LfV2APrBz++Ogczr/099SsXcspo/bjnFOPrLP8w1Wr+foFN/HUcy/Tq0dXrrv43xmw1Wb8+Yln+cEvJ7Fq9Ro6dezA+LM+w0F7Dq7QXlhrKOtApKQRkuZKmifpvHJuKwVVVeKSscfzhbOvYp/jf8TnjtyDwQO3rLPO+LOP47Z7/8YBJ/2YCddMYdyZxwKw1y4D2XvX7Thg9MXsd+JF7D50G/YfNqgSu2EtrKZmLd+ZMJHbL/8Gj0/8PnfcP5Pn5r9aZ52b7n6MHt27MOuuC/n6SYdy4RV3A7DZpt249Wen8+ht3+OqC07hjAturMQuJEtq/qNSypa0JVUDVwIjgaHAaElDy7W9FOyx47bMX7iMBYveYPWaGu58YBZHHbxLnXUGb9eXh2fMBeDhGc8z8qCdAYiAzp060qljBzp37EDHDtUsffPtVt8Ha3kzZ7/Edv17s22/3nTq2IHPHjGMyX95us46U6Y9zeij9wZg1Kd25y/T5xIR7DK4P337bArAkO37svLD1Xy4anVr70Kyam+E0JxHpZSz0t4LmBcR8yNiFXAbMKqM22vz+vbpwaLXl380vfj15fTt06POOrOfX8Qxh+4GwDGH7kr3bl3o2aMr0//xIg/PfIHnplzEc/ddzJ8ef5bnX3q9NcO3Mnl16Qq23qLnR9NbbdGTV5euqLPO4iUfr9OhQzXdu3XhzRXv1Vln0p+eYtfB/encqWP5g24PNqDKbpeVNrA1sLBo+pVsnjXivy6/i/2HfYK/3Pz/2H/YJ1j0+nJqatYysF9vBm+7BTse/X2GHvU9Dhy+A/vutn2lw7U24tl/vsqFV9zNz797YqVDSUbhNPb0Ku2KD0RKOg04DYCO3SobTJmVUlG9tmwFXxp7DQBdu3Ti3w7djbffXcmXP7Mf0595ifdWrgLgj4/NZs+dB/LYU/9svR2wsijlF9hWmxfW2XqLnqxZU8Pb766kV4+uACx6fTmnjL2a//nBKQzs16dVY7fWV85KexHQv2i6Xzavjoi4OiKGR8RwdehSxnAqb9acBWw/oA8DttqMjh2q+ewRw5gyrW7fZa8eXT/6Fj/n1E9zyz2PA/DK68vZf9gnqK6uokN1FfsPG8TzL73W6vtgLW/Y0G3458tLWbBoGatWr+HOB2Yx8qC6Yx0jDtyZW+99AoC7//QkB+25A5JY8c77nHDOr7jgzFHss6t/ea2f5lfZ7bXSng4MkjSQQrI+ETipjNtr82pq1jJ2wkTu+MWZVFeLWyY9znPzX+P804/mqWdfZsq0f3DAHoMYd+axRMCjT87jOxMmAnD3g09y0PAdeOTW7xIRPPjYs9z38DMV3iNrCR06VDNh7PF87qwrqakJTj52H4Zs35eLf/UHdhsygKMO3oVTRu3HGRfcyLDjLqRn965ce9FXAPjNxGm8uHApE66ZwoRrpgBw5y/H0KfXJpXcpWSkeHKNIqJ8jUtHAZcB1cB1EXFRY+tXbbx5dB58fNnisTQtn/7LSodgbcz+ew9n5swZG5RyN95qcAw+/X+a/fqnLjxsZkQM35AYmqOsfdoRMRmYXM5tmJk1S6Knsfsqf2ZmCan40SNmZpVQe8hfapy0zSy3EszZTtpmll8pVtru0zYzS4grbTPLrQQLbSdtM8sppdk94qRtZrlUOHqk0lGsPydtM8upyl5DpLk8EGlmlhBX2maWWwkW2k7aZpZfKXaPOGmbWT4lesEoJ20zy6VUrz3igUgzs4S40jaz3Eqx0nbSNrPcSjBnO2mbWX6lWGm7T9vMLCGutM0sn3zIn5lZOpTotUectM0stxLM2U7aZpZfVQlmbQ9EmpklxJW2meVWgoW2K20zyydltxtr7qO0bWiEpLmS5kk6bx3rHC9pjqTZkn7XVJuutM0st6rKWGlLqgauBI4AXgGmS5oUEXOK1hkEnA/sHxHLJW3eVLtO2maWW2U+5G8vYF5EzM+2dRswCphTtM5/AFdGxHKAiFjSVKPuHjEza57ekmYUPU6rt3xrYGHR9CvZvGI7ADtIekTS45JGNLVRV9pmllsbWGgvi4jhGxhCB2AQcAjQD5gmaeeIeGtdL3ClbWa5JLKzIpv5rwSLgP5F0/2yecVeASZFxOqIeBF4nkISXycnbTPLrSo1/1GC6cAgSQMldQJOBCbVW+f/KFTZSOpNobtkfqMxr98umplZKSJiDTAGmAo8C0yMiNmSxks6NlttKvCGpDnAn4HvRMQbjbXrPm0zy6f1ON66uSJiMjC53rxxRc8D+Fb2KImTtpnlVopnRDppm1kuiTQvGOWkbWa5lWDO9kCkmVlKXGmbWW75zjVmZolQe7tHpKQrgFjX8og4qywRmZm1kvY2EDmj1aIwM6uA9FJ2I0k7Im4onpa0cUS8X/6QzMxsXZo8ekTSvtkpls9l07tKuqrskZmZlVm571xTDqUc8ncZ8GngDYCI+DtwUBljMjMru8LJNWW9YFRZlHT0SEQsrPfNUlOecMzMWkmFK+bmKiVpL5S0HxCSOgJnU7hilZmZtbJSkvYZwOUUbpOzmMKlBM8sZ1BmZq0hwUK76aQdEcuAk1shFjOzVpVi90gpR49sJ+keSUslLZF0t6TtWiM4M7NySXUgspSjR34HTAT6AlsBtwO3ljMoM7PW0F4P+ds4Im6KiDXZ42Zgo3IHZmZm/6qxa4/0yp5OkXQecBuFa5GcQL3b55iZpSi9Hu3GByJnUkjStft1etGyAM4vV1BmZuUmtbMLRkXEwNYMxMystSWYs0s7I1LSTsBQivqyI+LGcgVlZmYNazJpS7oAOIRC0p4MjAT+Cjhpm1nS2uVx2sDngcOA1yLiK8CuQI+yRmVm1gpq717TnEellNI9sjIi1kpaI6k7sAToX+a4zMzKSqh9DUQWmSFpU+A3FI4oeRd4rJxBmZmVXXu7R2StiPhG9vRXku4DukfE0+UNy8zMGtLYyTXDGlsWEbPKE5KZWetIcSCysUr70kaWBfCpFo6F3YcM4JEnftnSzVrieu45ptIhWBvz4dyXW6SdUo7EaGsaO7nm0NYMxMysNYn2V2mbmbVrlbzEanOl+OvAzCy3XGmbWW61y0pbBV+UNC6bHiBpr/KHZmZWPoUzG9vnTRCuAvYFRmfT7wBXli0iM7NWkuLtxkrpHtk7IoZJehIgIpZL6lTmuMzMrAGlJO3VkqopHJuNpD7A2rJGZWbWChI84q+kpP0L4C5gc0kXUbjq3/fLGpWZWZkV7saeXtYu5dojt0iaSeHyrAI+ExHPlj0yM7MyS/GY51JugjAAeB+4p3heRLTMeaRmZhWSYKFdUvfIvXx8g9+NgIHAXGDHMsZlZmYNKKV7ZOfi6ezqf99Yx+pmZkmQ2u9NEOqIiFmS9i5HMGZmrSnBnF1Sn/a3iiargGHA4rJFZGbWSlI8jb2USnuToudrKPRx31GecMzMWke7POQvO6lmk4j4divFY2ZmjWjsdmMdImKNpP1bMyAzs9aSYKHdaKX9Nwr9109JmgTcDrxXuzAi7ixzbGZm5VPhCz81Vyl92hsBb1C4J2Tt8doBOGmbWdJEelm7saS9eXbkyDN8nKxrRVmjMjOzBjWWtKuBbtDgV5GTtpklrXD0SKWjWH+NJe1XI2J8q0ViZtbK2lvSTnB3zMxKV8nbhjVXY0n7sFaLwsyslaXaPbLOy8lGxJutGYiZmTVtvS8YZWbWLqj9nVxjZtautbtrj5iZtVftrk/bzKy9k5r/KK19jZA0V9I8Sec1st7nJIWk4U216aRtZlYG2VVSrwRGAkOB0ZKGNrDeJsDZwBOltOukbWY5Jao24FGCvYB5ETE/IlYBtwGjGljvh8BPgA9KadRJ28xySWxw90hvSTOKHqfV28TWwMKi6VeyeR/HULjnbv+IuLfUuD0QaWb5tOGXZl0WEU32Qa9z81IV8DPg1PV5nSttM7PyWAT0L5rul82rtQmwE/CQpJeAfYBJTQ1GutI2s9wq83Ha04FBkgZSSNYnAifVLoyIFUDv2mlJDwHfjogZjTXqStvMcqkF+rQbFRFrgDHAVOBZYGJEzJY0XtKxzY3blbaZ5Va5z4iMiMnA5Hrzxq1j3UNKadNJ28xyK8Gz2N09YmaWElfaZpZLIs2q1UnbzPJJ7e/ONWZm7Vp6KdtJ28xyqnBp1vTSdopdOmZmueVK28xyK70620nbzHIswd4RJ20zyyslefSI+7TNzBLiStvMcskn15iZJSbF7hEnbTPLrfRStpO2meVVoqexp9ilY2aWW660zSyXPBBpZpaYFLtHnLTNLLfSS9lO2maWYwkW2kl26ZiZ5ZYrbTPLpcJAZHqltpO2meVWit0jTtqt7I+PzuH8S39Pzdq1nDJqP8459cg6yz9ctZqvX3ATTz33Mr16dOW6i/+dAVttxptvvcuXz7uWJ+csYPQx+3DJ2OMrtAdWDoftO4Qfn/t5qququOnuR7nshgfqLO+/ZU+uGPdFem/ajeVvv8/p425g8ZK3ALhwzCiOPGBHAC659j7uemBWa4efKKEEK+2y9WlLuk7SEknPlGsbqampWct3Jkzk9su/weMTv88d98/kufmv1lnnprsfo0f3Lsy660K+ftKhXHjF3QB07tyR755xDOPPPq4SoVsZVVWJS8YezxfOvop9jv8RnztyDwYP3LLOOuPPPo7b7v0bB5z0YyZcM4VxZx4LwJH778gun+zPgSf/N4ef+lPGfPEwNum6USV2w1pJOQcirwdGlLH95Myc/RLb9e/Ntv1606ljBz57xDAm/+XpOutMmfY0o4/eG4BRn9qdv0yfS0TQtUtn9t1tezbq1LESoVsZ7bHjtsxfuIwFi95g9Zoa7nxgFkcdvEuddQZv15eHZ8wF4OEZzzPyoJ0L8wduyaNPzqOmZi3vf7CK2S8s4rB9h7T6PqRKav6jUsqWtCNiGvBmudpP0atLV7D1Fj0/mt5qi568unRFnXUWL/l4nQ4dqunerQtvrnivVeO01tW3Tw8Wvb78o+nFry+nb58eddaZ/fwijjl0NwCOOXRXunfrQs8eXXnmhUUcvu8QunTuSK8eXTlw+A51PmO2brUDkc19VIr7tM0S8F+X38WEsV/gpGP25tEn57Ho9eXU1Kzlz088x7Ch2zD1unNZtvxdpv/jRWrWrq10uGmocMXcXBVP2pJOA04D6D9gQIWjKa9SKqqtNi+ss/UWPVmzpoa3311Jrx5dWztUa0Wl/AJ7bdkKvjT2GgC6dunEvx26G2+/uxKAS387lUt/OxWA3/zwVP65YEkrRZ6+FJN2xU+uiYirI2J4RAzv07tPpcMpq2FDt+GfLy9lwaJlrFq9hjsfmMXIg+r2XY44cGduvfcJAO7+05MctOcOSV4fwUo3a84Cth/QhwFbbUbHDtV89ohhTJlWd6yjV4+uH30Ozjn109xyz+NAYRCzZ/alvuMntmLHQVvxpyeea90dsFZV8Uo7Tzp0qGbC2OP53FlXUlMTnHzsPgzZvi8X/+oP7DZkAEcdvAunjNqPMy64kWHHXUjP7l259qKvfPT6XY4dxzvvfcDq1WuY/JenueOKM/nkdn0ruEfWEmpq1jJ2wkTu+MWZVFeLWyY9znPzX+P804/mqWdfZsq0f3DAHoMYd+axRMCjT87jOxMmAtCxQzWTr/5PAN557wNOG3cDNTXuHilViof8KSLK07B0K3AI0Bt4HbggIq5t7DV77DE8HnliRlnisXT13HNMpUOwNubDuRNZ+/6SDcq4g3faLa76/R+b/frDh/SZGRHDNySG5ihbpR0Ro8vVtplZS0ix0q54n7aZmZXOfdpmllspjvE7aZtZbqXYPeKkbWa5JKAqvZztpG1meeWr/JmZWZm50jazfPK1R8zM0pJgznbSNrN8KgxEppe2nbTNLLfSS9keiDQzS4orbTPLrwRLbSdtM8utFI/TdtI2s9xKcBzSfdpmZilxpW1muZVgoe2kbWY5lmDWdtI2s1wSHog0M0tHotce8UCkmVlCXGmbWW4lWGg7aZtZjiWYtZ20zSyn0rxzjZO2meWWByLNzKysnLTNLJe0gY+StiGNkDRX0jxJ5zWw/FuS5kh6WtKDkrZpqk0nbTPLrzJmbUnVwJXASGAoMFrS0HqrPQkMj4hdgN8DE5pq10nbzHJLG/CvBHsB8yJifkSsAm4DRhWvEBF/joj3s8nHgX5NNeqkbWZWHlsDC4umX8nmrctXgSlNNeqjR8wstzbw6JHekmYUTV8dEVc3Lw59ERgOHNzUuk7aZpZbG3jE37KIGN7I8kVA/6Lpftm8ujFIhwPfAw6OiA+b2qi7R8wsn8p/+Mh0YJCkgZI6AScCk+qEIO0O/Bo4NiKWlNKoK20zy61ynhEZEWskjQGmAtXAdRExW9J4YEZETAIuAboBt6vQV/NyRBzbWLtO2mZmZRIRk4HJ9eaNK3p++Pq26aRtZrkk0jyN3UnbzHIrwZztpG1mOZZg1nbSNrPcSvHSrD7kz8wsIa60zSy3PBBpZpaQBHO2k7aZ5ViCWdt92mZmCXGlbWa5VLiESHqltpO2meWTPBBpZpaUBHO2k7aZ5ViCWdsDkWZmCXGlbWY5VfINetsUJ20zyy0PRJqZJaL0u4a1LU7aZpZfCWZtD0SamSXElbaZ5ZYHIs3MEuKBSDOzhCSYs92nbWaWElfaZpZPvmCUmVlq0svaTtpmlkvClbaZWVISzNkeiDQzS0mbqrRnzZq5rEtHLah0HG1Eb2BZpYOwNsWfiY9t0xKNuHtkA0VEn0rH0FZImhERwysdh7Ud/ky0PJ8RaWaWkvRytpO2meVXgjnbA5Ft2NWVDsDaHH8mzJV2WxUR/g9qdfgz0bLkMyLNzNKS4kCku0faGEkjJM2VNE/SeZWOxypP0nWSlkh6ptKxtDvagEeFOGm3IZKqgSuBkcBQYLSkoZWNytqA64ERlQ7C2gYn7bZlL2BeRMyPiFXAbcCoCsdkFRYR04A3Kx1He5Rgoe2k3cZsDSwsmn4lm2dmZVA7GNmcR6V4INLMckpJDkQ6abcti4D+RdP9snlm1sJSvTSru0falunAIEkDJXUCTgQmVTgmM2tDnLTbkIhYA4wBpgLPAhMjYnZlo7JKk3Qr8BgwWNIrkr5a6Zisctw90sZExGRgcqXjsLYjIkZXOob2KsXuESdtM8stD0SamaUi0WuPuE/bzCwhrrTNLJcqfWZjczlpm1l+JZi13T1idUiqkfSUpGck3S5p4w1o63pJn8+eX9PYxa8kHSJpv2Zs4yVJvUudX2+dd9dzWxdK+vb6xmhtlzbgX6U4aVt9KyNit4jYCVgFnFG8UFKzfp1FxNciYk4jqxwCrHfSNssbJ21rzMPAJ7Iq+GFJk4A5kqolXSJpuqSnJZ0OoIJfZtcD/yOweW1Dkh6SNDx7PkLSLEl/l/SgpG0pfDmck1X5B0rqI+mObBvTJe2fvXYzSfdLmi3pGkr4gSvp/yTNzF5zWr1lP8/mPyipTzZve0n3Za95WNInW+TdtDbHF4yydiOrqEcC92WzhgE7RcSLWeJbERF7SuoMPCLpfmB3YDCFa4FvAcwBrqvXbh/gN8BBWVu9IuJNSb8C3o2In2br/Q74eUT8VdIACmeJDgEuAP4aEeMlHQ2Ucnbgv2fb6AJMl3RHRLwBdAVmRMQ5ksZlbY+hcC/GMyLiBUl7A1cBn2rG22htXIJd2k7a9i+6SHoqe/4wcC2Fbou/RcSL2fwjgV1q+6uBHsAg4CDg1oioARZL+lMD7e8DTKttKyLWdZ3ow4Gh+rik6S6pW7aNz2avvVfS8hL26SxJx2XP+2exvgGsBf43m38zcGe2jf2A24u23bmEbViKEszaTtpW38qI2K14Rpa83iueBXwzIqbWW++oFoyjCtgnIj5oIJaSSTqEwhfAvhHxvqSHgI3WsXpk232r/ntg7VOKZ0S6T9uaYyrwdUkdASTtIKkrMA04Ievz7gsc2sBrHwcOkjQwe22vbP47wCZF690PfLN2QtJu2dNpwEnZvJFAzyZi7QEszxL2JylU+rWqgNpfCydR6HZ5G3hR0heybUjSrk1sw6zVOGlbc1xDob96Vnaz2V9T+NV2F/BCtuxGClemqyMilgKnUeiK+Dsfd0/cAxxXOxAJnAUMzwY65/DxUSw/oJD0Z1PoJnm5iVjvAzpIehb4bwpfGrXeA/bK9uFTwPhs/snAV7P4ZuNbvrVLtdfTTm0gUhFRua2bmVWIpPuARo/lb8KyiGj1Gy47aZuZJcTdI2ZmCXHSNjNLiJO2mVlCnLTNzBLipG1mlhAnbTOzhDhpm5kl5P8Dg9kHzJq/QuwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_preds = np.argmax(output_predictions.predictions, axis=1)\n",
    "labels = {0 : 'Non-Spam', 1 : 'Spam'}\n",
    "\n",
    "plot_confusion_matrix(y_preds, y_valid, labels=labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Push Fine-Tuned Model to HuggingFace Hub (My Profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4a96cfa2f584c0f9c16820b04a7177b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file pytorch_model.bin:   0%|          | 32.0k/255M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9de435a5671c47cbb7b6db24c9689f20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload file runs/Aug29_14-18-14_Brians-Mac-mini/events.out.tfevents.1661800698.Brians-Mac-mini.11789.0: 100%|#…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-SpamFilter-LG\n",
      "   8e28dcd..fe74b68  main -> main\n",
      "\n",
      "To https://huggingface.co/DunnBC22/distilbert-base-uncased-SpamFilter-LG\n",
      "   fe74b68..d12176c  main -> main\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/DunnBC22/distilbert-base-uncased-SpamFilter-LG/commit/fe74b685e0f7616ae618c692625b5b8be2b41972'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.push_to_hub(commit_message=\"All Dunn!!!\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('nlp_ch4')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a52fe47989fdc78fafbb981021cec52a6b82df6453830b9ffbd04250493e6cab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}