Code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7e8bba41",
   "metadata": {},
   "source": [
    "# Lamb - MSDS 453 Project"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3b1d228",
   "metadata": {},
   "source": [
    "## Index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3e3592d",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e50527a9",
   "metadata": {},
   "source": [
    "### Kaggle Database Link"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1706802f",
   "metadata": {},
   "source": [
    "https://www.kaggle.com/datasets/uciml/sms-spam-collection-dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "062cf126",
   "metadata": {},
   "source": [
    "### Other Articles to Consider"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b193ee5",
   "metadata": {},
   "source": [
    "http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "363c67ad",
   "metadata": {},
   "source": [
    "https://www.proquest.com/docview/2798556468?pq-origsite=primo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9ea62c7",
   "metadata": {},
   "source": [
    "https://machinelearningmastery.com/cross-validation-for-imbalanced-classification/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a1db544",
   "metadata": {},
   "source": [
    "https://machinelearningmastery.com/smote-oversampling-for-imbalanced-classification/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5fe9f8d",
   "metadata": {},
   "source": [
    "### Load Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ad0d7b3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools\n",
    "import collections\n",
    "import re\n",
    "import nltk\n",
    "import string\n",
    "import opendatasets as od\n",
    "import pickle\n",
    "from nltk.corpus import stopwords\n",
    "from nltk import bigrams\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn import metrics, svm\n",
    "from sklearn.metrics import precision_score, recall_score, roc_curve, confusion_matrix, jaccard_score, f1_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB\n",
    "from keras.layers import SimpleRNN, LSTM, Dense, Dropout, Activation, Flatten\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, ExtraTreesClassifier, AdaBoostClassifier, AdaBoostClassifier\n",
    "from xgboost import XGBClassifier, XGBRFClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.pipeline import Pipeline\n",
    "from hyperopt import STATUS_OK, Trials, fmin, hp, tpe"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72dcfbe1",
   "metadata": {},
   "source": [
    "### Pickle Load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4388beed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pickled variables\n",
    "# with open('session.pkl', 'rb') as f:\n",
    "#     data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cf304cef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #pickeling file to allow for loading calculation directionly into memory without recomputing\n",
    "# ##as this is a computationally intensive wb this allows for incremental progress to be made without recommputing the whole file\n",
    "# with open('session.pkl', 'wb') as f:\n",
    "#     pickle.dump({\n",
    "#             'cnt' : cnt,\n",
    "#             'ltt' : ltt,\n",
    "#             'lrt' : lrt,\n",
    "#             'gnt' : gnt,\n",
    "#             'bnt' : bnt,\n",
    "#             'mbt' : mbt,\n",
    "#             'srt' : srt,\n",
    "#             'spt' : spt,\n",
    "#             'slt' : slt,\n",
    "#             'rft' : rft,\n",
    "#             'gbt' : gbt,\n",
    "#             'ett' : ett,\n",
    "#             'adt' : adt,\n",
    "#             'xgt' : xgt,\n",
    "#             'xrt' : xrt,\n",
    "#             'cnr' : cnr,\n",
    "#             'ltr' : ltr,\n",
    "#             'lrr' : lrr,\n",
    "#             'gnr' : gnr,\n",
    "#             'bnr' : bnr,\n",
    "#             'mbr' : mbr,\n",
    "#             'srr' : srr,\n",
    "#             'spr' : spr,\n",
    "#             'slr' : slr,\n",
    "#             'rfr' : rfr,\n",
    "#             'gbr' : gbr,\n",
    "#             'etr' : etr,\n",
    "#             'adr' : adr,\n",
    "#             'xgr' : xgr,\n",
    "#             'xrr' : xrr\n",
    "#                 }, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28fd5edb",
   "metadata": {},
   "source": [
    "### Importing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f49d253",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5572, 5)\n"
     ]
    }
   ],
   "source": [
    "#loading corpus into data frame\n",
    "df = pd.read_csv(\"spam.csv\", encoding = \"ISO-8859-1\", engine = \"python\")\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "82e761be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     v1                                                 v2 Unnamed: 2  \\\n",
       "0   ham  Go until jurong point, crazy.. Available only ...        NaN   \n",
       "1   ham                      Ok lar... Joking wif u oni...        NaN   \n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...        NaN   \n",
       "3   ham  U dun say so early hor... U c already then say...        NaN   \n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...        NaN   \n",
       "\n",
       "  Unnamed: 3 Unnamed: 4  \n",
       "0        NaN        NaN  \n",
       "1        NaN        NaN  \n",
       "2        NaN        NaN  \n",
       "3        NaN        NaN  \n",
       "4        NaN        NaN  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "581847bc",
   "metadata": {},
   "source": [
    "### Cleaning Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9844f484",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5572 entries, 0 to 5571\n",
      "Data columns (total 5 columns):\n",
      " #   Column      Non-Null Count  Dtype \n",
      "---  ------      --------------  ----- \n",
      " 0   v1          5572 non-null   object\n",
      " 1   v2          5572 non-null   object\n",
      " 2   Unnamed: 2  50 non-null     object\n",
      " 3   Unnamed: 3  12 non-null     object\n",
      " 4   Unnamed: 4  6 non-null      object\n",
      "dtypes: object(5)\n",
      "memory usage: 217.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8bd168c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>spam</td>\n",
       "      <td>Your free ringtone is waiting to be collected....</td>\n",
       "      <td>PO Box 5249</td>\n",
       "      <td>MK17 92H. 450Ppw 16\"</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>ham</td>\n",
       "      <td>\\Wen u miss someone</td>\n",
       "      <td>the person is definitely special for u..... B...</td>\n",
       "      <td>why to miss them</td>\n",
       "      <td>just Keep-in-touch\\\" gdeve..\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>ham</td>\n",
       "      <td>\\HEY HEY WERETHE MONKEESPEOPLE SAY WE MONKEYAR...</td>\n",
       "      <td>HOWU DOIN? FOUNDURSELF A JOBYET SAUSAGE?LOVE ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>spam</td>\n",
       "      <td>SMS. ac sun0819 posts HELLO:\\You seem cool</td>\n",
       "      <td>wanted to say hi. HI!!!\\\" Stop? Send STOP to ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>710</th>\n",
       "      <td>ham</td>\n",
       "      <td>Height of Confidence: All the Aeronautics prof...</td>\n",
       "      <td>this wont even start........ Datz confidence..\"</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       v1                                                 v2  \\\n",
       "95   spam  Your free ringtone is waiting to be collected....   \n",
       "281   ham                                \\Wen u miss someone   \n",
       "444   ham  \\HEY HEY WERETHE MONKEESPEOPLE SAY WE MONKEYAR...   \n",
       "671  spam         SMS. ac sun0819 posts HELLO:\\You seem cool   \n",
       "710   ham  Height of Confidence: All the Aeronautics prof...   \n",
       "\n",
       "                                            Unnamed: 2             Unnamed: 3  \\\n",
       "95                                         PO Box 5249   MK17 92H. 450Ppw 16\"   \n",
       "281   the person is definitely special for u..... B...       why to miss them   \n",
       "444   HOWU DOIN? FOUNDURSELF A JOBYET SAUSAGE?LOVE ...                    NaN   \n",
       "671   wanted to say hi. HI!!!\\\" Stop? Send STOP to ...                    NaN   \n",
       "710    this wont even start........ Datz confidence..\"                    NaN   \n",
       "\n",
       "                         Unnamed: 4  \n",
       "95                              NaN  \n",
       "281   just Keep-in-touch\\\" gdeve..\"  \n",
       "444                             NaN  \n",
       "671                             NaN  \n",
       "710                             NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#looking at the data in the unnamed columns\n",
    "df[df['Unnamed: 2'].isnull() == False].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "36af2ab8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>spam</td>\n",
       "      <td>Your free ringtone is waiting to be collected....</td>\n",
       "      <td>PO Box 5249</td>\n",
       "      <td>MK17 92H. 450Ppw 16\"</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>ham</td>\n",
       "      <td>\\Wen u miss someone</td>\n",
       "      <td>the person is definitely special for u..... B...</td>\n",
       "      <td>why to miss them</td>\n",
       "      <td>just Keep-in-touch\\\" gdeve..\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>spam</td>\n",
       "      <td>Your free ringtone is waiting to be collected....</td>\n",
       "      <td>PO Box 5249</td>\n",
       "      <td>MK17 92H. 450Ppw 16\"</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1038</th>\n",
       "      <td>ham</td>\n",
       "      <td>Edison has rightly said, \\A fool can ask more ...</td>\n",
       "      <td>GN</td>\n",
       "      <td>GE</td>\n",
       "      <td>GNT:-)\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2170</th>\n",
       "      <td>ham</td>\n",
       "      <td>\\CAN I PLEASE COME UP NOW IMIN TOWN.DONTMATTER...</td>\n",
       "      <td>JUST REALLYNEED 2DOCD.PLEASE DONTPLEASE DONTIG...</td>\n",
       "      <td>U NO THECD ISV.IMPORTANT TOME 4 2MORO\\\"\"</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        v1                                                 v2  \\\n",
       "95    spam  Your free ringtone is waiting to be collected....   \n",
       "281    ham                                \\Wen u miss someone   \n",
       "899   spam  Your free ringtone is waiting to be collected....   \n",
       "1038   ham  Edison has rightly said, \\A fool can ask more ...   \n",
       "2170   ham  \\CAN I PLEASE COME UP NOW IMIN TOWN.DONTMATTER...   \n",
       "\n",
       "                                             Unnamed: 2  \\\n",
       "95                                          PO Box 5249   \n",
       "281    the person is definitely special for u..... B...   \n",
       "899                                         PO Box 5249   \n",
       "1038                                                 GN   \n",
       "2170  JUST REALLYNEED 2DOCD.PLEASE DONTPLEASE DONTIG...   \n",
       "\n",
       "                                    Unnamed: 3                      Unnamed: 4  \n",
       "95                        MK17 92H. 450Ppw 16\"                             NaN  \n",
       "281                           why to miss them   just Keep-in-touch\\\" gdeve..\"  \n",
       "899                       MK17 92H. 450Ppw 16\"                             NaN  \n",
       "1038                                        GE                         GNT:-)\"  \n",
       "2170  U NO THECD ISV.IMPORTANT TOME 4 2MORO\\\"\"                             NaN  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['Unnamed: 3'].isnull() == False].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0b13b3ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>ham</td>\n",
       "      <td>\\Wen u miss someone</td>\n",
       "      <td>the person is definitely special for u..... B...</td>\n",
       "      <td>why to miss them</td>\n",
       "      <td>just Keep-in-touch\\\" gdeve..\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1038</th>\n",
       "      <td>ham</td>\n",
       "      <td>Edison has rightly said, \\A fool can ask more ...</td>\n",
       "      <td>GN</td>\n",
       "      <td>GE</td>\n",
       "      <td>GNT:-)\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2255</th>\n",
       "      <td>ham</td>\n",
       "      <td>I just lov this line: \\Hurt me with the truth</td>\n",
       "      <td>I don't mind</td>\n",
       "      <td>i wil tolerat.bcs ur my someone..... But</td>\n",
       "      <td>Never comfort me with a lie\\\" gud ni8 and swe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3525</th>\n",
       "      <td>ham</td>\n",
       "      <td>\\HEY BABE! FAR 2 SPUN-OUT 2 SPK AT DA MO... DE...</td>\n",
       "      <td>HAD A COOL NYTHO</td>\n",
       "      <td>TX 4 FONIN HON</td>\n",
       "      <td>CALL 2MWEN IM BK FRMCLOUD 9! J X\\\"\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4668</th>\n",
       "      <td>ham</td>\n",
       "      <td>When I was born, GOD said, \\Oh No! Another IDI...</td>\n",
       "      <td>GOD said</td>\n",
       "      <td>\\\"OH No! COMPETITION\\\". Who knew</td>\n",
       "      <td>one day these two will become FREINDS FOREVER!\"</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       v1                                                 v2  \\\n",
       "281   ham                                \\Wen u miss someone   \n",
       "1038  ham  Edison has rightly said, \\A fool can ask more ...   \n",
       "2255  ham      I just lov this line: \\Hurt me with the truth   \n",
       "3525  ham  \\HEY BABE! FAR 2 SPUN-OUT 2 SPK AT DA MO... DE...   \n",
       "4668  ham  When I was born, GOD said, \\Oh No! Another IDI...   \n",
       "\n",
       "                                             Unnamed: 2  \\\n",
       "281    the person is definitely special for u..... B...   \n",
       "1038                                                 GN   \n",
       "2255                                       I don't mind   \n",
       "3525                                   HAD A COOL NYTHO   \n",
       "4668                                           GOD said   \n",
       "\n",
       "                                    Unnamed: 3  \\\n",
       "281                           why to miss them   \n",
       "1038                                        GE   \n",
       "2255  i wil tolerat.bcs ur my someone..... But   \n",
       "3525                            TX 4 FONIN HON   \n",
       "4668          \\\"OH No! COMPETITION\\\". Who knew   \n",
       "\n",
       "                                             Unnamed: 4  \n",
       "281                       just Keep-in-touch\\\" gdeve..\"  \n",
       "1038                                            GNT:-)\"  \n",
       "2255   Never comfort me with a lie\\\" gud ni8 and swe...  \n",
       "3525                CALL 2MWEN IM BK FRMCLOUD 9! J X\\\"\"  \n",
       "4668    one day these two will become FREINDS FOREVER!\"  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['Unnamed: 4'].isnull() == False].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c9e6f19d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5572, 2)\n"
     ]
    }
   ],
   "source": [
    "#the unknown columns are sparsely populated and most that are are populated appear to contain irrelevant information \n",
    "#(such as time or address info).  droping these columns\n",
    "to_drop = ['Unnamed: 2',\"Unnamed: 3\",\"Unnamed: 4\"]\n",
    "df = df.drop(columns = to_drop)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b269d0b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5572 entries, 0 to 5571\n",
      "Data columns (total 2 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   label      5572 non-null   object\n",
      " 1   documents  5572 non-null   object\n",
      "dtypes: object(2)\n",
      "memory usage: 87.2+ KB\n"
     ]
    }
   ],
   "source": [
    "#renamining columns\n",
    "rename_list = {'v1':'label','v2':'documents'}\n",
    "df = df.rename(columns=rename_list)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9d6f0995",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#neither column has any null values, but lets check to make sure there is non-blank text in the documents\n",
    "df_temp = df['documents'].str.len() - df['documents'].str.count(' ')\n",
    "sum(df_temp == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "16ea9dce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ham' 'spam']\n"
     ]
    }
   ],
   "source": [
    "#okay so all the documents contain at least some characters.  Lets check that our label is a binary indicator as expected\n",
    "label_list = df.label.unique()\n",
    "print(label_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d647401f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating one hotkey on label\n",
    "label_binary = pd.get_dummies(df.label)\n",
    "label_binary= label_binary.drop(columns='ham')\n",
    "label_binary = label_binary.rename(columns={'spam':'label_binary'})\n",
    "df = pd.concat([df,label_binary],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "106c070f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5572, 3)\n",
      "[0]\n",
      "[1]\n"
     ]
    }
   ],
   "source": [
    "#checking hotkey join and binary hotkey labeling\n",
    "print(df.shape)\n",
    "print(df[df['label']=='ham'].label_binary.unique())\n",
    "print(df[df['label']=='spam'].label_binary.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "86519632",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "403"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking for duplicates\n",
    "df.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "aac674c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5169, 3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dropping duplicated\n",
    "df = df.drop_duplicates()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b03b2ad",
   "metadata": {},
   "source": [
    "### EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "360f108d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       documents  label_binary\n",
      "label                         \n",
      "ham         4516          4516\n",
      "spam         653           653\n"
     ]
    }
   ],
   "source": [
    "#looking at the frequency of ham versus spam\n",
    "label_count = df.groupby('label').count()\n",
    "print(label_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5f5d15a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5169\n"
     ]
    }
   ],
   "source": [
    "#lets look at how wordy our documents are - first creating a word count\n",
    "documents = df['documents'].tolist()\n",
    "word_count = [] \n",
    "for i in documents:\n",
    "    word_count.append(len(i.split()))\n",
    "print(len(word_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f3264a81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>min</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max</td>\n",
       "      <td>171.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mean</td>\n",
       "      <td>15.340685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>std</td>\n",
       "      <td>11.067417</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label       value\n",
       "0   min    1.000000\n",
       "1   max  171.000000\n",
       "2  mean   15.340685\n",
       "3   std   11.067417"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#calculating mean, standard deviations, min, and max\n",
    "min_val = min(word_count)\n",
    "max_val =max(word_count)\n",
    "mean_val = np.mean(word_count)\n",
    "var_val = np.std(word_count)\n",
    "stat_label = pd.Series(('min','max','mean','std'))\n",
    "stats = pd.Series((min_val,max_val,mean_val,var_val))\n",
    "d = {'label':stat_label,'value':stats}\n",
    "df_stat = pd.DataFrame(data=d)\n",
    "df_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "adee663d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5169, 4)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#adding the word count into the data frame\n",
    "df['word_count'] = np.array(word_count)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bdb06c20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>documents</th>\n",
       "      <th>label_binary</th>\n",
       "      <th>word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>ham</td>\n",
       "      <td>Yup</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>ham</td>\n",
       "      <td>Thanx...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>ham</td>\n",
       "      <td>Okie...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok..</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>782</th>\n",
       "      <td>ham</td>\n",
       "      <td>Beerage?</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    label documents  label_binary  word_count\n",
       "260   ham       Yup             0           1\n",
       "275   ham  Thanx...             0           1\n",
       "283   ham   Okie...             0           1\n",
       "286   ham      Ok..             0           1\n",
       "782   ham  Beerage?             0           1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#looking at a few of these one word documents\n",
    "df[df['word_count'] == 1].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5099d1df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.003869220352099052"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#what percentage of the documents have only 1 word\n",
    "sum(df['word_count'] == 1)/len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5a8edca1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "range(0, 5169)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "range(len(documents))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c5eb1e95",
   "metadata": {},
   "outputs": [],
   "source": [
    "#look at the most common words - first prep a word list\n",
    "word_list = []\n",
    "for i in range(len(documents)):\n",
    "    word_list.append(documents[i].lower().split())\n",
    "master_word_list = list(itertools.chain(*word_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d92ba3ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('i', 2095),\n",
       " ('to', 2055),\n",
       " ('you', 1832),\n",
       " ('a', 1281),\n",
       " ('the', 1223),\n",
       " ('and', 919),\n",
       " ('u', 890),\n",
       " ('in', 785),\n",
       " ('is', 766),\n",
       " ('my', 676),\n",
       " ('for', 653),\n",
       " ('your', 618),\n",
       " ('me', 579),\n",
       " ('of', 552),\n",
       " ('have', 532),\n",
       " ('on', 476),\n",
       " ('call', 468),\n",
       " ('are', 457),\n",
       " ('that', 453),\n",
       " ('it', 440)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#now count the words\n",
    "count_words = collections.Counter(master_word_list)\n",
    "count_words.most_common(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b207acc",
   "metadata": {},
   "source": [
    "### Text Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9ef7ebb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#making text lowercase\n",
    "df['documents_clean'] = df['documents'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "971e3e67",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CGLam\\AppData\\Local\\Temp\\ipykernel_3740\\3020070343.py:2: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  df['documents_clean'] = df['documents_clean'].str.replace(r'https?://\\S+|www\\.\\S+', 'url')\n"
     ]
    }
   ],
   "source": [
    "#replacing URLs with keyword \"URL\"\n",
    "df['documents_clean'] = df['documents_clean'].str.replace(r'https?://\\S+|www\\.\\S+', 'url')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8f10b498",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "179\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\CGLam\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "#loading stop words\n",
    "nltk.download('stopwords')\n",
    "stop_words = set(stopwords.words('english'))\n",
    "print(len(stop_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "02eeb053",
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing stop words\n",
    "df['documents_clean'] = df['documents_clean'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop_words)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f1393629",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CGLam\\AppData\\Local\\Temp\\ipykernel_3740\\2423228234.py:2: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  df['documents_clean'] = df['documents_clean'].str.replace(r'[^\\w\\s]+', '')\n"
     ]
    }
   ],
   "source": [
    "#remove punctuation\n",
    "df['documents_clean'] = df['documents_clean'].str.replace(r'[^\\w\\s]+', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e785dafd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('u', 1001),\n",
       " ('call', 487),\n",
       " ('im', 447),\n",
       " ('2', 443),\n",
       " ('get', 364),\n",
       " ('ur', 316),\n",
       " ('go', 269),\n",
       " ('4', 257),\n",
       " ('ltgt', 254),\n",
       " ('ok', 251),\n",
       " ('free', 243),\n",
       " ('know', 239),\n",
       " ('got', 231),\n",
       " ('like', 231),\n",
       " ('good', 217),\n",
       " ('come', 210),\n",
       " ('ill', 206),\n",
       " ('you', 200),\n",
       " ('time', 199),\n",
       " ('now', 198)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#re-reviewing most common words to see if it makes sense to create any custom stop words\n",
    "word_list_2 = []\n",
    "documents_2 = df['documents_clean'].tolist()\n",
    "for i in range(len(documents_2)):\n",
    "    word_list_2.append(documents_2[i].lower().split())\n",
    "master_word_list_2 = list(itertools.chain(*word_list_2))\n",
    "count_words_2 = collections.Counter(master_word_list_2)\n",
    "count_words_2.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "417a8aea",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating custom stop words\n",
    "custom_stopwords = {'u','im','ur','ill','you'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "08a75443",
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove custom stop words\n",
    "df['documents_clean'] = df['documents_clean'].apply(lambda x: ' '.join([word for word in x.split() if word not in (custom_stopwords)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "67d04734",
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove non-character tokens\n",
    "df['documents_clean'] = df['documents_clean'].apply(lambda x: ' '.join([word for word in x.split() if word.isalpha()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e9a8b950",
   "metadata": {},
   "outputs": [],
   "source": [
    "#applying stemming\n",
    "stemmer = PorterStemmer()\n",
    "df['documents_clean'] = df['documents_clean'].apply(lambda x: ' '.join([stemmer.stem(y) for y in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e22fcfe6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>documents</th>\n",
       "      <th>label_binary</th>\n",
       "      <th>word_count</th>\n",
       "      <th>documents_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>go jurong point crazi avail bugi n great world...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>ok lar joke wif oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>free entri wkli comp win fa cup final tkt may ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>dun say earli hor c alreadi say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>nah think goe usf live around though</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                          documents  label_binary  \\\n",
       "0   ham  Go until jurong point, crazy.. Available only ...             0   \n",
       "1   ham                      Ok lar... Joking wif u oni...             0   \n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...             1   \n",
       "3   ham  U dun say so early hor... U c already then say...             0   \n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...             0   \n",
       "\n",
       "   word_count                                    documents_clean  \n",
       "0          20  go jurong point crazi avail bugi n great world...  \n",
       "1           6                                ok lar joke wif oni  \n",
       "2          28  free entri wkli comp win fa cup final tkt may ...  \n",
       "3          11                    dun say earli hor c alreadi say  \n",
       "4          13               nah think goe usf live around though  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "acff91f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export preprocessed data to excel for further review \n",
    "#df.to_excel('preprocessed.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6538b9db",
   "metadata": {},
   "source": [
    "### Tokenize the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "47a66005",
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_tokenizer(x):\n",
    "    tokenizer = tf.keras.preprocessing.text.Tokenizer()\n",
    "    tokenizer.fit_on_texts(x)\n",
    "    return tokenizer\n",
    "    \n",
    "def encode(x2, tokenizer):\n",
    "    encoded_sentences = tokenizer.texts_to_sequences(x2)\n",
    "    encoded_sentences = tf.keras.preprocessing.sequence.pad_sequences(encoded_sentences, padding='post')\n",
    "    return encoded_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "489d1879",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = define_tokenizer(df['documents_clean'])\n",
    "s_strings = encode(df['documents_clean'],tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "cded4e4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5169"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking that we have appropriate number of documents\n",
    "len(s_strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "78447304",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'go jurong point crazi avail bugi n great world la e buffet cine got amor wat'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#quick look at encoding...text of first clean document\n",
    "df['documents_clean'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8dfa9579",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   2, 2952,  271,  540,  568,  954,   43,   66,  325,  955,   88,\n",
       "       2089,  956,   11, 2953,   64,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#encoding of that document\n",
    "s_strings[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9903ce08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "2952\n",
      "271\n",
      "540\n",
      "568\n",
      "954\n",
      "43\n",
      "66\n",
      "325\n",
      "955\n",
      "88\n",
      "2089\n",
      "956\n",
      "11\n",
      "2953\n",
      "64\n"
     ]
    }
   ],
   "source": [
    "#pulling these words out of the dictionary to make sure we encoded as expected\n",
    "d = tokenizer.word_index\n",
    "print(d['go'])\n",
    "print(d['jurong'])\n",
    "print(d['point'])\n",
    "print(d['crazi'])\n",
    "print(d['avail'])\n",
    "print(d['bugi'])\n",
    "print(d['n'])\n",
    "print(d['great'])\n",
    "print(d['world'])\n",
    "print(d['la'])\n",
    "print(d['e'])\n",
    "print(d['buffet'])\n",
    "print(d['cine'])\n",
    "print(d['got'])\n",
    "print(d['amor'])\n",
    "print(d['wat'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5663926",
   "metadata": {},
   "source": [
    "### Keras Cross Validation Design"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "1da218fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5169, 81)\n",
      "     0     1    2     3     4     5     6    7     8    9  ...  71  72  73  \\\n",
      "0    2  2952  271   540   568   954    43   66   325  955  ...   0   0   0   \n",
      "1    6   226  569   326  1413     0     0    0     0    0  ...   0   0   0   \n",
      "2    9   413  796   797   129  2090  1087  304  2091  206  ...   0   0   0   \n",
      "3  146    40  263  2092    97    73    40    0     0    0  ...   0   0   0   \n",
      "4  750    22  338   696   176   128   339    0     0    0  ...   0   0   0   \n",
      "\n",
      "   74  75  76  77  78  label  \\\n",
      "0   0   0   0   0   0      0   \n",
      "1   0   0   0   0   0      0   \n",
      "2   0   0   0   0   0      1   \n",
      "3   0   0   0   0   0      0   \n",
      "4   0   0   0   0   0      0   \n",
      "\n",
      "                                                 doc  \n",
      "0  go jurong point crazi avail bugi n great world...  \n",
      "1                                ok lar joke wif oni  \n",
      "2  free entri wkli comp win fa cup final tkt may ...  \n",
      "3                    dun say earli hor c alreadi say  \n",
      "4               nah think goe usf live around though  \n",
      "\n",
      "[5 rows x 81 columns]\n"
     ]
    }
   ],
   "source": [
    "#converting tokenized data to pandas dataframe\n",
    "#we will eventually convert back to array but the pd will make the cross validation setup easier\n",
    "s_pandas = pd.DataFrame(data=s_strings)\n",
    "s_pandas['label'] = df['label_binary'].tolist()\n",
    "s_pandas['doc'] = df['documents_clean'].tolist()\n",
    "print(s_pandas.shape)\n",
    "print(s_pandas.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "515f5e9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2584, 80)\n",
      "(2585, 80)\n",
      "(2584, 1)\n",
      "(2585, 1)\n"
     ]
    }
   ],
   "source": [
    "#creating the cross validation datasets\n",
    "y = s_pandas['label'].to_numpy()\n",
    "y = y.reshape(-1,1)\n",
    "x = s_pandas.drop(columns=['label'])\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.5, random_state=9)\n",
    "\n",
    "print(x_train.shape)\n",
    "print(x_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "3f2f0fb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         0     1    2     3    4    5    6     7    8    9  ...  70  71  72  \\\n",
      "2194  4640    14   91    48    0    0    0     0    0    0  ...   0   0   0   \n",
      "2259  4678   253   27  1122    0    0    0     0    0    0  ...   0   0   0   \n",
      "3644    65   487   20    58  986  394   36   798   14  107  ...   0   0   0   \n",
      "107      7  3025  433   199  416    8  204  1108  161   20  ...   0   0   0   \n",
      "2584   208    73  300  1270   43  500  202     0    0    0  ...   0   0   0   \n",
      "\n",
      "      73  74  75  76  77  78  \\\n",
      "2194   0   0   0   0   0   0   \n",
      "2259   0   0   0   0   0   0   \n",
      "3644   0   0   0   0   0   0   \n",
      "107    0   0   0   0   0   0   \n",
      "2584   0   0   0   0   0   0   \n",
      "\n",
      "                                                    doc  \n",
      "2194                                velli good ye pleas  \n",
      "2259                             nohe join today itself  \n",
      "3644      happi sad one thing past is it more good morn  \n",
      "107   know grumpi old peopl mom like better lie alwa...  \n",
      "2584                  yup alreadi thanx print n hand up  \n",
      "\n",
      "[5 rows x 80 columns]\n",
      "[[0]\n",
      " [0]\n",
      " [0]\n",
      " [0]]\n"
     ]
    }
   ],
   "source": [
    "#checking that i split this correctly\n",
    "print(x_train.head())\n",
    "print(y_train[:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "a8fad549",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2584,)\n",
      "(2585,)\n",
      "(2584, 79)\n",
      "(2585, 79)\n"
     ]
    }
   ],
   "source": [
    "#splitting out the text doc from the encoded data\n",
    "train_doc = x_train['doc']\n",
    "test_doc = x_test['doc']\n",
    "x_train = x_train.drop(columns=['doc'])\n",
    "x_test = x_test.drop(columns=['doc'])\n",
    "print(train_doc.shape)\n",
    "print(test_doc.shape)\n",
    "print(x_train.shape)\n",
    "print(x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "d39bef0e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2584\n",
      "2585\n"
     ]
    }
   ],
   "source": [
    "#our data needs to be in array format so we conver the new dataframes back to arrays\n",
    "x_train_array = x_train.to_numpy()\n",
    "x_test_array = x_test.to_numpy()\n",
    "#and checking shaping\n",
    "print(len(x_train_array))\n",
    "print(len(x_test_array))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "f3a4e987",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2584, 79)\n",
      "(2585, 79)\n",
      "(2584, 1)\n",
      "(2585, 1)\n"
     ]
    }
   ],
   "source": [
    "print(x_train_array.shape)\n",
    "print(x_test_array.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "9da7c369",
   "metadata": {},
   "outputs": [],
   "source": [
    "#saving index of records that made it into test versus train\n",
    "train_index = x_train.index\n",
    "test_index = x_test.index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83780414",
   "metadata": {},
   "source": [
    "### One Hotkey Encoding (Count Vectorization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "999ca2bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#creating the one hotkey on clean documents\n",
    "vec = CountVectorizer()\n",
    "X_train_count = vec.fit_transform(df['documents_clean'].values)\n",
    "X_train_count.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "3022dde1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5169"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking shape\n",
    "len(X_train_count.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c7bea37d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5169"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#moving into pandas dataframe\n",
    "df_one = pd.DataFrame(X_train_count.toarray())\n",
    "len(df_one)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "edb0915b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2584, 6793)\n",
      "(2585, 6793)\n"
     ]
    }
   ],
   "source": [
    "#using index to split into train and test\n",
    "df_train = df_one.iloc[train_index]\n",
    "df_test = df_one.iloc[test_index]\n",
    "print(df_train.shape)\n",
    "print(df_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa3d5027",
   "metadata": {},
   "source": [
    "### One Hotkey Encoding (TFID Vectorization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "438e84aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating the one hotkey on clean documents\n",
    "# vec2 = TfidfVectorizer(max_features=7000)\n",
    "# X_train_tfidf = vec2.fit_transform(df['documents_clean'].values)\n",
    "# X_train_tfidf.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "470dd0ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "#checking shape\n",
    "# len(X_train_tfidf.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "76bcdaa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#moving into pandas dataframe\n",
    "# df_vec = pd.DataFrame(X_train_tfidf.toarray())\n",
    "# df_vec.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "05332e8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#using index to split into train and test\n",
    "# df_train = df_vec.iloc[train_index]\n",
    "# df_test = df_vec.iloc[test_index]\n",
    "# print(df_train.shape)\n",
    "# print(df_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d338840",
   "metadata": {},
   "source": [
    "### Defining Metrics Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8789c28d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_metrics(method):\n",
    "    a = print(\"Accuracy:\",metrics.accuracy_score(y_test, method))\n",
    "    p = print(\"Precision:\",metrics.precision_score(y_test, method))\n",
    "    r =print(\"Recall:\",metrics.recall_score(y_test, method))\n",
    "    return a, p, r;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "ae635a88",
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_accuracy(method):\n",
    "    a = metrics.accuracy_score(y_test, method)\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f6157e5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_precision(method):\n",
    "    p = metrics.precision_score(y_test, method)\n",
    "    return p;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "49e48ada",
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_recall(method):\n",
    "    r =metrics.recall_score(y_test, method)\n",
    "    return r;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "3ba54445",
   "metadata": {},
   "outputs": [],
   "source": [
    "def class_f1(method):\n",
    "    r =metrics.f1_score(y_test, method)\n",
    "    return r;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "47f0ef8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_metrics(method):\n",
    "    a = print(\"Accuracy:\",metrics.accuracy_score(y_train, method))\n",
    "    p = print(\"Precision:\",metrics.precision_score(y_train, method))\n",
    "    r =print(\"Recall:\",metrics.recall_score(y_train, method))\n",
    "    return a, p, r;"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aed9fcbf",
   "metadata": {},
   "source": [
    "### Setting Up Tensors for each of the Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "34ab550f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#creating the tensor\n",
    "# ts_train = tf.data.Dataset.from_tensor_slices((x_train_array, y_train))\n",
    "# ts_holdout = tf.data.Dataset.from_tensor_slices((x_test_array, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "0a738d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#padding\n",
    "#ts_train = ts_train.padded_batch(32, padded_shapes=([-1], []))\n",
    "#ts_holdout = ts_holdout.padded_batch(32, padded_shapes=([-1], []))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed395f80",
   "metadata": {},
   "source": [
    "### Establishing Early Stopping Criteria for NN training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "d0b9077c",
   "metadata": {},
   "outputs": [],
   "source": [
    "callback = keras.callbacks.EarlyStopping(monitor='precision',patience=15,min_delta=.001,verbose=1,mode=\"max\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34b60ae5",
   "metadata": {},
   "source": [
    "### Under Sampling Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "bec65de6",
   "metadata": {},
   "outputs": [],
   "source": [
    "under = RandomUnderSampler(sampling_strategy=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45a69068",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "9d78bc2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(solver=\"lbfgs\",random_state=9,warm_start=False,class_weight='balanced')\n",
    "lrm = lr.fit(df_train, y_train.ravel())\n",
    "lrr = lrm.predict(df_train)\n",
    "lrt = lrm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "9e8f24b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9725338491295938\n",
      "Precision: 0.9180887372013652\n",
      "Recall: 0.8512658227848101\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(lrt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "35b2f06c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1.0, 'class_weight': 'balanced', 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1, 'l1_ratio': None, 'max_iter': 100, 'multi_class': 'auto', 'n_jobs': None, 'penalty': 'l2', 'random_state': 9, 'solver': 'lbfgs', 'tol': 0.0001, 'verbose': 0, 'warm_start': False}\n"
     ]
    }
   ],
   "source": [
    "print(lrm.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d425295",
   "metadata": {},
   "source": [
    "### Problems with Naive Bayes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "845d1ca5",
   "metadata": {},
   "source": [
    "http://people.csail.mit.edu/jrennie/papers/icml03-nb.pdf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20af6c20",
   "metadata": {},
   "source": [
    "Problems with Naive Bayes:\n",
    "    Doesnt handle imbalanced data well\n",
    "    Assumes feature indepedence"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a61db471",
   "metadata": {},
   "source": [
    "### Gaussian Naive Bayes Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "184850e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "gn = GaussianNB()\n",
    "gn_steps = [('under', under), ('model', gn)]\n",
    "gn_pipeline = Pipeline(steps=gn_steps)\n",
    "gnm = gn_pipeline.fit(df_train, y_train)\n",
    "gnr = gnm.predict(df_train)\n",
    "gnt = gnm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "c0ecc663",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8096711798839459\n",
      "Precision: 0.3820375335120643\n",
      "Recall: 0.9018987341772152\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(gnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "9dd5f191",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'memory': None, 'steps': [('under', RandomUnderSampler(sampling_strategy=0.5)), ('model', GaussianNB())], 'verbose': False, 'under': RandomUnderSampler(sampling_strategy=0.5), 'model': GaussianNB(), 'under__random_state': None, 'under__replacement': False, 'under__sampling_strategy': 0.5, 'model__priors': None, 'model__var_smoothing': 1e-09}\n"
     ]
    }
   ],
   "source": [
    "print(gnm.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8828248",
   "metadata": {},
   "source": [
    "### Bernouli Naive Bayes Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "a65ed55a",
   "metadata": {},
   "outputs": [],
   "source": [
    "bn = BernoulliNB()\n",
    "bn_steps = [('under', under), ('model', bn)]\n",
    "bn_pipeline = Pipeline(steps=bn_steps)\n",
    "bnm = bn_pipeline.fit(df_train, y_train)\n",
    "bnr = bnm.predict(df_train)\n",
    "bnt = bnm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "8de1d700",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9617021276595744\n",
      "Precision: 0.9909502262443439\n",
      "Recall: 0.6930379746835443\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(bnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "41c17eaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 1.0, 'binarize': 0.0, 'class_prior': None, 'fit_prior': True, 'force_alpha': 'warn'}\n"
     ]
    }
   ],
   "source": [
    "print(bn.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d56fd40",
   "metadata": {},
   "source": [
    "### MultiNomimail Bayes Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "54eb1b2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "mb = MultinomialNB()\n",
    "mb_steps = [('under', under), ('model', mb)]\n",
    "mb_pipeline = Pipeline(steps=mb_steps)\n",
    "mbm = mb_pipeline.fit(df_train, y_train)\n",
    "mbr = mbm.predict(df_train)\n",
    "mbt = mbm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "195345ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9504835589941973\n",
      "Precision: 0.7373737373737373\n",
      "Recall: 0.9240506329113924\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(mbt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "530f4726",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 1.0, 'class_prior': None, 'fit_prior': True, 'force_alpha': 'warn'}\n"
     ]
    }
   ],
   "source": [
    "print(mb.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6f4d6b",
   "metadata": {},
   "source": [
    "### Hypertuned SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "f6aac007",
   "metadata": {},
   "outputs": [],
   "source": [
    "# param_grid_sv = {'C': [0.1,1, 10, 100], 'gamma': [1,0.1,0.01,0.001],'kernel': ['rbf', 'poly', 'sigmoid','linear']}\n",
    "# grid_sv = GridSearchCV(svm.SVC(random_state=9,class_weight='balanced'),param_grid_sv,refit=True,verbose=3,scoring='precision',cv=5)\n",
    "# shm = grid_sv.fit(df_train,y_train.ravel())\n",
    "# print(grid_sv.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a584d279",
   "metadata": {},
   "outputs": [],
   "source": [
    "# svr = grid_sv.predict(df_train)\n",
    "# svt = grid_sv.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "eb582491",
   "metadata": {},
   "outputs": [],
   "source": [
    "# class_metrics(svt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "f7e5483b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_sv_grid = pd.DataFrame(data=grid_sv.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "524f3246",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_sv_grid.to_excel(\"SVC_GridSearch.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0695fa30",
   "metadata": {},
   "source": [
    "###  Best Fit SVM (precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "be3ed1c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "sb = svm.SVC(kernel='poly',random_state=9,C=.1,gamma=1,class_weight='balanced')\n",
    "sbm = sb.fit(df_train, y_train.ravel())\n",
    "sbr = sbm.predict(df_train)\n",
    "sbt = sbm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "5a28dab0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9326885880077369\n",
      "Precision: 0.94375\n",
      "Recall: 0.4778481012658228\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(sbt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "4ca507c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 0.1, 'break_ties': False, 'cache_size': 200, 'class_weight': 'balanced', 'coef0': 0.0, 'decision_function_shape': 'ovr', 'degree': 3, 'gamma': 1, 'kernel': 'poly', 'max_iter': -1, 'probability': False, 'random_state': 9, 'shrinking': True, 'tol': 0.001, 'verbose': False}\n"
     ]
    }
   ],
   "source": [
    "print(sbm.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fc90336",
   "metadata": {},
   "source": [
    "### Best Fit SVM (f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "78092cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "sb2 = svm.SVC(kernel='rbf',random_state=9,C=10,gamma=0.1,class_weight='balanced')\n",
    "sbm2 = sb2.fit(df_train, y_train.ravel())\n",
    "sbr2 = sbm2.predict(df_train)\n",
    "sbt2 = sbm2.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "845e280f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9663442940038685\n",
      "Precision: 0.9872340425531915\n",
      "Recall: 0.7341772151898734\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(sbt2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "2f152c19",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 10, 'break_ties': False, 'cache_size': 200, 'class_weight': 'balanced', 'coef0': 0.0, 'decision_function_shape': 'ovr', 'degree': 3, 'gamma': 0.1, 'kernel': 'rbf', 'max_iter': -1, 'probability': False, 'random_state': 9, 'shrinking': True, 'tol': 0.001, 'verbose': False}\n"
     ]
    }
   ],
   "source": [
    "print(sbm2.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68336e34",
   "metadata": {},
   "source": [
    "### Hyertuned Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "768d4647",
   "metadata": {},
   "outputs": [],
   "source": [
    "# n_estimators = [int(x) for x in np.linspace(start = 100, stop = 2000, num = 10)]\n",
    "# max_features = ['auto', 'sqrt']\n",
    "# max_depth = [int(x) for x in np.linspace(10, 1000, num = 11)]\n",
    "# max_depth.append(None)\n",
    "# min_samples_split = [2, 5, 10]\n",
    "# min_samples_leaf = [1, 2, 4]\n",
    "# bootstrap = [True, False]\n",
    "# random_grid = {\n",
    "#                'n_estimators': n_estimators,\n",
    "#                'max_features': max_features,\n",
    "#                'max_depth': max_depth,\n",
    "#                'min_samples_split': min_samples_split,\n",
    "#                'min_samples_leaf': min_samples_leaf,\n",
    "#                'bootstrap': bootstrap\n",
    "#                 }\n",
    "# print(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "59636522",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rf = RandomForestClassifier(class_weight=\"balanced\",random_state=9)\n",
    "# rf_random = RandomizedSearchCV(estimator = rf, param_distributions = random_grid, n_iter = 100, cv = 5, verbose=2, random_state=9, scoring='precision')\n",
    "# rf_random.fit(df_train, y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "07e83b8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# class_metrics(rf_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "1a39cedc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rhr = rf_random.predict(df_train)\n",
    "# rgt = rf_random.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "f7684fc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_rf_grid = pd.DataFrame(data=rf_random.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "13b048a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_rf_grid.to_excel(\"RF_GridSearch.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b9d2a2d",
   "metadata": {},
   "source": [
    "### Best Fit RF (precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "6d75fbfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(\n",
    "    class_weight=\"balanced\",\n",
    "    random_state=9,\n",
    "    n_estimators=100,\n",
    "    min_samples_split=2,\n",
    "    min_samples_leaf=1,\n",
    "    max_features='sqrt',\n",
    "    max_depth=307,\n",
    "    bootstrap=True)\n",
    "rfm = rf.fit(df_train, y_train.ravel())\n",
    "rfr = rfm.predict(df_train)\n",
    "rft = rfm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "d138796e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9647969052224371\n",
      "Precision: 0.9955947136563876\n",
      "Recall: 0.7151898734177216\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(rft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "79d678e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': 'balanced', 'criterion': 'gini', 'max_depth': 307, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_jobs': None, 'oob_score': False, 'random_state': 9, 'verbose': 0, 'warm_start': False}\n",
      "Max Depth of any Tree:  307\n"
     ]
    }
   ],
   "source": [
    "#looking at default parameters\n",
    "print(rf.get_params())\n",
    "print('Max Depth of any Tree: ',max([estimator.tree_.max_depth for estimator in rf.estimators_]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a15af368",
   "metadata": {},
   "source": [
    "### Best Fit RF (f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "fe729d61",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\sklearn\\ensemble\\_forest.py:424: FutureWarning: `max_features='auto'` has been deprecated in 1.1 and will be removed in 1.3. To keep the past behaviour, explicitly set `max_features='sqrt'` or remove this parameter as it is also the default value for RandomForestClassifiers and ExtraTreesClassifiers.\n",
      "  warn(\n"
     ]
    }
   ],
   "source": [
    "rf2 = RandomForestClassifier(\n",
    "    class_weight=\"balanced\",\n",
    "    random_state=9,\n",
    "    n_estimators=1577,\n",
    "    min_samples_split=10,\n",
    "    min_samples_leaf=1,\n",
    "    max_features='auto',\n",
    "    max_depth=307,\n",
    "    bootstrap=False)\n",
    "rfm2 = rf2.fit(df_train, y_train.ravel())\n",
    "rfr2 = rfm2.predict(df_train)\n",
    "rft2 = rfm2.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "d11b8717",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9822050290135397\n",
      "Precision: 0.972027972027972\n",
      "Recall: 0.879746835443038\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(rft2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "3a20363e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': False, 'ccp_alpha': 0.0, 'class_weight': 'balanced', 'criterion': 'gini', 'max_depth': 307, 'max_features': 'auto', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 1577, 'n_jobs': None, 'oob_score': False, 'random_state': 9, 'verbose': 0, 'warm_start': False}\n",
      "Max Depth of any Tree:  307\n"
     ]
    }
   ],
   "source": [
    "#looking at default parameters\n",
    "print(rf2.get_params())\n",
    "print('Max Depth of any Tree: ',max([estimator.tree_.max_depth for estimator in rf2.estimators_]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "125a45ac",
   "metadata": {},
   "source": [
    "### Hyertuned Gradient Boosted Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "d3a0fbaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# n_estimators = [int(x) for x in np.linspace(start = 100, stop = 500, num = 3)]\n",
    "# max_features = ['log2', 'sqrt']\n",
    "# max_depth = [int(x) for x in np.linspace(1, 10, num = 3)]\n",
    "# max_depth.append(None)\n",
    "# min_samples_split = [2, 5, 10]\n",
    "# min_samples_leaf = [1, 2, 4]\n",
    "# random_grid = {\n",
    "#                'n_estimators': n_estimators,\n",
    "#                'max_features': max_features,\n",
    "#                'max_depth': max_depth,\n",
    "#                'min_samples_split': min_samples_split,\n",
    "#                'min_samples_leaf': min_samples_leaf,\n",
    "#                 }\n",
    "# print(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "680017bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# gb = GradientBoostingClassifier(random_state=9)\n",
    "# gb_grid = GridSearchCV(gb,random_grid, cv = 5, verbose=2, scoring='precision')\n",
    "# gb_grid.fit(df_train, y_train.ravel())\n",
    "# print(gb_grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "c9dcf22c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(gb_grid.best_params_)"
   ]
  },
  {
   "attachments": {
    "image-2.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAABE0AAABICAYAAAADHQuBAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAABo1SURBVHhe7d1tbBTXucDxx1f5UKRUkZpE2MXQdZsWdEVa4FYFQ6tY5JamUlOKKbELrXCVL+lFiIRrFUUQcB2jyBUlIJQXqWrrqLWD4+CEpFJTSJCjJjbObTANUQW3tLaA1CAgUnQrkQ+RfM85c2Z2ZnZ2d/Z91/x/0si7M563M2dmzjx7zpm6GUUAAAAAAAAQ8G/2LwAAAAAAAHwImgAAAAAAAEQgaAIAAAAAABCBPk2q1PXr12V6elo++eQTOwYAAAAAAJTKLbfcIg0NDXL77bfbMQRNqtb7778vjY2N8qlPfcqOAQAAAAAApfLxxx/LpUuXZPHixXYMzXOqlq5hQsAEAAAAAIDy0M/g4dYeBE0AAAAAAAAiEDQBAAAAAACIQNAEAAAAAAAgAkETAAAAAACACLw9p0qdPn1aFi1aZL/VvgsXLthPs8uCBQvsJwAAAABArTt79qwsWbLEfqOmCQAAAAAAQCSCJgAAAAAAABFmYdDkQ3l5R58sbbHDs3+z40vv2h9eTK53x0m5ZsdXncuD0t47R+YMjtsRJfBOr8zZNChX7Ndsrv1+qyx8esJ+q4yJpxfKwoVb5dXLdkSka/LqtoXyzGn7NYsrL7TLnCdKmM5IodO8/YW4OQ8AAAAA0it/0OTMH5OBBW/4o7xnJxfLjw51yMSIGh76oh2T9N6zep0vystX7QjP32R/tu26elK2etODy7jj29931nmowY6Zhf7SK3OezRIM+WBQ2u+ZkL6ft8lcOwolkmNw6mYw94FHZOnmhPS+Y0cAAAAAQJ7KHzS5+1u+wEKDPKc/j3xLvuxMLTEnKPL65xbJ1+2YJF1D5W0RN9iihufap2Wzv6aKDvhsuCT3Drn/83353p12Wi2pb5PDO27IjbbldkQxXZHBn3aIPHdQ2ubZUTVi6X+dk3PnDsn99XYEatRy2XG+Tybu6RXq+AAAAAAoRNU1z9FNXLb+4UNbG8QZ9p+xEwv03rN/kc8Pdcj2r9oRAdflH+O3yud9D8yf/dyt9pP2obz8u/+TPUOFBUqc/SqgZo2u6TE4Lldeb5c5uomNGtpf99UzcGuCuE1wQs1wxgftuPB8WqZlu8t7rUvkow5J2OlzekMPpu/0SYf0ycEHIuqYXH5Vti7UTWD8wzPib5TjNJFxhrhNYIxMyzbT9OcJecadtu3VZPOpwLzB7XH45lu4Sjpfs6NzMP6ESqs5zhCsAXFFBjclp6U25RmXXneaHtxaJbo2j/5+jzoew+p4eP8TN1Cgl6v+112Of9kuXYvFW25wu/X+6CYwpvlRxPSU7fb2S+9vuwy+YJetxrtp45/fv9zwtFjmtckju7vkSZrpAAAAAChAVfZp8lbvK/LLz33X1OY4vuNW+e3vitM/yJcfyhTw+KL8Z/u/5GcbbEDj6kn5We+/5EerbPOeq/8rb4x/WuTPVdBvyVSLJK4/Ijd0bZHNfSLvbpNBfz8cOqjx3JQ8YqevnXrSm768TY1T40cSzvcU/mXf1yVH3+1zHsLd2ilqnNzWJ1P6sxl2iL++yvgbXbJ2bUtEs5xr8uoTnSK/eFvOnTsn5wYfVuPWyL43fyJLnX8QOdguv16QnH6gLSqAESXGsuWAtC/8tTS+qWuTHJaHX+uUITcoU3+/HNLzvblPzRWm+zBpl7+6yz73tuy7z06Kq7tFnvzClNy4odLrzS7p8tWAGH8iIR2LR5xpN6ak7/2WQH8c40+0yIQ6ls50NfTbJk/z1PGwy5NWdTzc6TeCxyOzLmm5S+UTM59at3TINm/d49L7RktyvaHt1o5uTkji7yqvqOlTz62VroPJoMv4EyPS4s57Y0S6VBokAx9HpeNoQqbOq7xp08bM/4Zd+geDsm3zUhnx5r8hO77mTMrF8ntV/j06EgwEAQAAAEAOqrMj2PZVcujbnzEf7/hqo3x9/CP5p/lWWl9+qEMmDols1gER2wxn+9124uWP5C2ZljdktW2a813ZI2flmzl2NGvWUWhzJB20cJvW1LfIutuOylTgyXCt9G22D8+R0zPwL/srLeqxekKmMnaM6ndFpt4XWZqIqmUyJsdeWyNrvnqH833JCnlYjskl/7K3HZZD3/FPPyAn49Q2ibNs5eFBt+nNUlmxTeSvl2KEvE4PSafskz3uduVj94gcdmvefE2naZeMmADCuIx0q8+PumGOudK2LfVBv3QP/iqfnHeDLHOlZe1aOfr3KfPNNHHxtksx263ywgf2u6aDNfZ/5q5aJ2uHp8Sb+1F/8Ga5tOwWmfBlwq5tbn83XfJIVK0kL40KMC8R2CYAAAAAyFV1Bk0qxDSd+d1tclwHRQ59Wn62oc80FfIsXyR7bDBH5DPyvR82iExer0xtk5CJq77H6tvWSYvXzEg9iD90Q3Z8xX4tKfWAOmw/htU3yr/LMTn2Z5tap0/KAXlYVixxvqaaL41xa3TEWnbwu+6/xAvQlFVCEq324wdTMiFd0uJrhmKa2/gsf9SpAeI2v8m5mUqu3p/y1RbxbdccHTQJCtQoMjVffIGSUNOelm47Pg69LFOzxc6fb0e38xKyNBzoAQAAAIAcEDRxXT0pvzx8q+zpXCHmUVp3WHuoQd7q/R+nuU79bWWr8ZKPpXdG/Vpfbr6AQAonCHLsv1c5fYO0HZCHB/3NZ8IuyqXX1khjrE5Zc112JenA0lpJeJ3kdgWaoZjBbYJjzJW2fjvedG5a4sDJ4oRZt+5TpEXXgvG2ayQlaJKWeXtSl3S96c57Q0Z222lxfW1Hct7FHZLIJ3BiglJLfWkNAAAAALkhaBLwL/mHr0nHe29Piyy/TT6rv9z5Jbl3+bT80qt5ojuGnZavt3zJCbLEVHBHsCFXXt8mHR91SUtZapIocxOy9qOXZCSyyc5cSSwONsPw2GYub5t+QZzhJ2lrmYhc+/2v5cB9a6Q5TtAkx2XnpL5R1rx2TMbs/k48Hd0RbFRnplGuvPCkdLWukxb9ID+vRda1qmOX0vlrGqbmRIhpgqKOR6G1KUw/Ikel615fk5zWhLhd3+i+VWIHTQxfYOid3txqmoQkvrDWfsrRB1Ny1LcPAAAAAJCr8gdN9Gt7dZ8hW6fVl2mn/5AiBhEy0W/mMevecFbeEt3pq173i/LyVTXxzhVy6FCD/HarHucMmycXyfFeW/NEN8fpXSVNva/Y6a/IGy3f9fpeKSvf22sS7y6VkVBnrOmNS6+dr2VK5Oi7ieAbcuKob5OD/yHS8ZyznPDbc0znm5tt57F+SzbIPumUVd5baJwh8Iacg+3e+FXH18jbB++PF5CKs+wMrv1+qzPPPZ1yzHQYq+e3ndDW3y97fiHSeY+zTN1R7eFtZrb4ulu8ZiqJo+tkyqtJomuROJ2/utP1kOwINvQGmjlOp7CBTlHnqePxnDoed7n/E/ftOdrR5Hx3dcjSN5Mdrs594BHp8r2V58kv9MUPmpg31/iWfTAhfTnUNAm/OSehO4UN1L6JJ32nxAAAAAAQT92MYj/PEh/KyztekX/80NeJa7npwJDuG8ULuOTu9OnTsmjRIvvNR78WeCwhUw/l/hBZHvqVsgl5ae1UsvNT5cIrj8rC3zQGAyGnn5GFbSKHzxXYlEYvp1TLzmLBggX2U63RAZknJXH+sLTNxuYruomQeTNQ3IAiAAAAAIicPXtWlixJNl2geQ6KbK60/bxPZPM2GfQ1Gbl26a/2U9LE6AGR+xplvv2er1IuG7VoXHpNzRkCJgAAAAAKM2uDJl4zmxxfCVwIr/mPaXp0EzNvP1kqHT9Ndt55x3f2pDShaT+3L0YTnAl5xjdPyvD0RAHLnq3CzXpCQ9w+VKpOvP3S/cakNGMCAAAAgDzMwuY5s0Pa5jk16sKFC/bT7FK7zXMAAAAAAGHh5jkETarUbAuaAAAAAABQ7ejTBAAAAAAAIAaCJgAAAAAAABEImgAAAAAAAEQgaFKlbrnlFvn444/tNwAAAAAAUEr6GVw/i/vREWyVun79ukxPT8snn3xixwAAAAAAgFLRAZOGhga5/fbb7RiCJgAAAAAAAJFongMAAAAAABCBoAkAAAAAAEAEgiYAAAAAAAARCJoAAAAAAABEIGgCAAAAAAAQgaAJAAAAAABABIImAAAAAAAAEQiaAAAAAAAARCBoAgAAAAAAEIGgSS27NCDr6+qkrm69DFyy4zxjsldNW//8Zfu9Cp3cK3UPDEjpt/CyDDxQJ3tP2q9p1UCaRXL2r+7xMfu9mjhpWmeH2kvbfJTyeKRf9tjjFU5nfT5HXouKoKBl2zxYlmtNeVx+fr13rMt93pcjnwX2r26vOoI+pcxnJVXI/YU8XC1ibbdbNstzv5x1hPJ9LajRc9M9ptnLiDWmBo+Hd3+poWsCbh4ETWqWenjavklkYFpmZo7IxkY7+mambxDVWqgssBBVq8YeXym7ukdVHp0xw5Ef1NspBTLpWYOFyhJqfsxJ49FuO6KYTOHLFmZq8oF1dqn/wRFzrKcHWu2Y8ilpPtPUub1lo0j/RWc9MzM7pdlOKrVZ+/BUhSqZhwtR8HZX7N5lA+7udfwmKovoB/HK/GDjpPlsvJ5kulYGAospeT1zPnTuL9PSf2Yl12FUHYImNWtSJodaZf03Mj+ELmsq0kPqTaTYaWZuINtF1pfqIcNqvavJfqoWl2XyjEjPN8v1yFNdSnk8ynusx2SvOoSj5gFWPyyIbJqfrtC/TJpKFsDNd9nNslNv+wsbhathDbg0KcMb1svqjMe62PnMKchvkfXSY8eUSn73F/JwTWncKEf08XqsgHvfhiYp5lX+8vNb5Mg6/SObvo6PSs/ulSUKJJTyHlAabiBs5wo7YlYp87Xy5F5p2LjMKy+Mdu+Slb4fM8ce/6xsutv9IS1dPqyX1eta5dRkVf4EipuZyrioSaMzPdI603/Rfo1leqZ/g5pnoGdGH3rpHp0Z7VZ/1eeeMfsv5n+ccWbY0K/GWGPOfCn/6/+fLKYHWpPLDi8/vG61fR69bvXdP3/rgJ3zYv9MqztPYOhRqaQ5y+0Z02kWnhafs+4c01xvm90Pk9b+fSqpOMc6fDx8+2bT1EtjxZk/l3Rz0tu/vqQMx9rwHyv/9NB4b0huu9kn//LMvoTzgrs/wXkNm88jpynhPBy9fxlEpG0x6f2KWna27U6mR3KI3Eaz/anpUn46L/SoPO7sV+vAqM1Tobzg7k9kHlP5wh4P8z85XMvcvOQtP+XcSJeHs5+bert12vuPSVQ+S8nrrlLnYSVdPsu87lCaRaS32bacjkOQk2a5Xd/1Op00cI5ZPumRn0rn4Wx5oYJ5OLTucF7LvN3ZhPbLn2aZyho+6bbbv7/B+ULr9IbU8zMzu5zINIvPbGeBy4jH5pWL/v3P7fzM93oWHu8N3n77tym8XD0t87np7Js7vxp8+cifD/yDP08E/ye4fj1N/69/HzKem3lcM531F/Na6aRHIN/7ywvq8/q64H6a/YvYdj0+6rwDKomgSc3SF6zwRT4be4HXFyhbyPIuyvYmMq0KQeEbgv/C5VzAnYtsuotdWubG57tA6+/hm4x3Mwut271putP9F2JXaHlJ7o3N/f+IC3sMzr7nmuZJwf0rtezHWqdhT/jY+tPPpLnd3/Cxy8Suz7uZe0Ny/ozHWhnt9q8r4uZs1hG9PYF91AL/6+aF5PoC22L+13eMw/udYb2x2fTJNf/FpfcnZdnh/fAfW+97KI3S5dVwGlWMLTTq7TTb7+SRqP1PyROGW+h09zuqEJhe9DKT0ufh7Oem+WzHG+HjZ0VuQznysBKZz7KsO9P9xZyH5ngEh5R1ZOEsJ9/9yy0PFK6yeThbXqhYHg7ljZT9KjAPR+Zdlz0O3jaF87QVvd1JaddR4La7aZFp3XGY86TAZcRj84qXhuFjm0U4/fO4nmU83oZO0/Ax9qVzmnMzW3nZHRd1Ppr84ytvOedLcj/M8fEd5/D/Z9+n7Jx15JsXQ+ekEU5Hm4bu/6l0rEspY0Zvg97fQvcPKDaa59SqS5NyKs9qdz3b3Sq+PdIZ6mOi/gc7ff2jOFXkhs9P2u9ONUZd3W7f8wPyK93ufH/c6sKXZWD/LukZS9c+fUxO7O6RUa86a71s3N4jwy+d8Kr1qRuGTLvTG1fL+g3DMplD3wo9Y27fL6n7FYdThbO2+o/JdKx1FeKdvnH131gvrUOT4qXKip1OU4zfDGQ5diFu1WRd9VJ9VTdLHZxVgzt/9mPd/Jh/Xc2yuluKW1Wze9TrX6X5m2orz0yadV/+0xGRgaeSx3jFg9K/YZecCLStDX/PkU2fovXvkpVz7rUOPJhMU3VsR7uH5cifnDQdO75LpclqO905P9w0CVLL2r5Jhrs7q+Q8aJX+H9u9UteHB3OuXq3mv+jmtTzy2e4TaZopZc/DGc9NzZdHZcVq9V+nYl3vypKH08i27kz3F7evFFVYdq71zo86OZ8nznJiXquqQoXzcIa8UKk8LJdOyBHpl6e8ZTbLgypf7DruP9sKy8OBskVYgWWN0rJNtQpp+nNyr6zc7ct3ZZBv+auS17Ns52a28nJ6Y6r8POw7f/SynkrdL18+TCmfKRnzcAylu1Y6TXjq6laqXXX6vwpck9w+/tSKR005MfW6UN+0rOD9A4qNoEnNsb3obxd5qhQXO/diZocGdWEPa35sVJZt3CSn/DeyQpkg0C5Z6Vt3XbN6kMsit8Ihgtwbmx3mq4dhO8VlbuRnNpk2qEVr7xvnWAc6Hq1TBTw7vkgC/ays2On1FTB5fliGNzb41t0gm4acfzN0wGOsR3Y12+k19DaLTH0pNN2lHlS9AMBlOfHSsLSuW+0V6Fxjj6v0UA8z3gPFTcwNILv5OKVddtHzcLwHt0rm4azrjnF/QRllywsVysOmX5uhTdLgW3cgrxSYh5sfm1ZXseTy43Q4OWvKGvocVPfbZBCjulX1Pbmg61lr1h89A/dgvZ++Mn8+ebg8hmXT/AaZ3O4EvXeucPq288ofL6ptnj8pnSYorvYn3Q/Auly2fdLsX2U68QVSETSpOfZXhv0iW4reA/uY7FUPzs4beZwLnvnVL0A/aOvosQ6cNBT5Qt3jdR7lDVk6vqOj2/x5D8BuWl/sV7fxIP0/R9aNmp7Mi3vjynCsvUJdcpr+paJcWn353x0CASN9M7fjR+9WBYAaCZwEC/1OQcalf9XRv9Y5AQAnXyR/5XXoNxCs1DWE6IzS49aO0LWq9PXQO0dKkoezF7JdlczD6dcd5/6CskuXFyqch/21jbzBH6wtKA/Xy8YX7DLVfe+UeuDOVpaZFWUNfUztOVhLnZ5W5z250OtZOICoX+5gP8aSex4uvSZp2uAcr+TxcV5aYc77xiZ1BfDXjFN0gDSi02PzAoXjq83+la9WLpAZQZNapS4+y+JWdc2RVzhQN9gtoci57oFdP1A9uKJZdproftzATb003a0ey9zqtbZA5jFVYNVDW8zX4JntGOqR1f4bp74gDx2REyVIE815jVppXrfqvpu+7De9u5vsA7BtdmE+W6YKr6523Swb9/eLbNxSnH2Pdax9hWuzHfazy+T/DFVyvaYlTsEm7u8/uqnOcA77aWpo5Ern/bL+emKrDW/8VfJcPfkrc/44VerVsTfNr2wBTA+hwIgXMMm7dptbq2m2vq7YKSwGZcnDORjTr+7O+kYZR1nycBpx1p3p/lIMzrW0FK90ne15OCovVCYPm6Y8Q5tkS8xrZEF52NxL0ossaxQi270rK1vbONdXBvsCJmkfQu29qZpqUBbjeqbHlaqpR+brWajc63Gauu3an0zny8/vU+dHPs3zlCx5OJ3iXyvdskbyeDn7Zc97VfZr3fCSbNru7rdT9oiq1apV3xshcbMjaAIf227YreY4f1LW+yLnOmjQ4O/HxLYtXRnzoqub9ejXi7nL7gzUbNBRc+fd7Ga6HQIPlr7qus4rzUIPcI0b5SnzOlR3/lIUnHNlCzhqMAVOd//D76bX/Woo5awC3Pxjlf7u9tQ1yOQ63/HQ1bLNr4w2jVXadnbrapfFeGDIcqy9ddlp+5ukP+UXTjdo586f3C7TpMirtrpPmsZSa9CkpX+xGlvmy0N6SOYjJ3CWnGbyYVXUvAjmM7c6s5umuinJ9MCpZJOo5lO+X3ucPmWSaWkH3y/O+8zDUrIpihlyKrQ7hanY1fOLwn3ItdWm05x7+Uku2xkaTBM272EkVh7Owjs31TE90y/TXj7Lsl8lzcOZ81nmdWe+v1SUvt6ZbVUP9uqru43BIPZsy8NZ8kIl87C+vtvaW+78enCPR7HysDOslFPhmhdpyxrZtjvL+WGkv3eV0thvnB8Pgk1dQus2P2iov6G+MyqqCNezYJlADe7xCpz3bl6PezziXc8C5V41uHlB11I0tWLs+KLn4VLJcq10yhrJcnjDS+t9570qawz+03csdE3mDEE8oMrUzeifFVGD9EVTPRBerK2OSfOmL9Sq0Ja8+M4y9legZWOhaqdAyekHAacNcjLvOeOKWqAxgTgdrLlJrlkF0A8CDec7C+vsEcVHHo6tZvPwbC9rZGFqFQYCXED56evHFnmKgAqqCjVNapauDl7OX7xQGvaXKwImqJiIttT67RVqXHHa8dtfxXjYRM0iD2OWszUICJig8pzO6GdFP0KYVQia1Kx608+E7vypGtpYh6tIBofKb1/10k1VnH4kaiZg4rZ7TjMEqyKj+oWri6uhqEE8XdVe5/Haedi8Oa9n4SrfoaFITUJqE3m4NpCH8+Z2plrOgAllCYQ4/aw4ndHn1b8LUEI0zwEAAAAAAIhATRMAAAAAAIAIBE0AAAAAAAAiEDQBAAAAAACIQNAEAAAAAAAgAkETAAAAAACACARNAAAAAAAAIhA0AQAAAAAAiEDQBAAAAAAAIAJBEwAAAAAAgAgETQAAAAAAACIQNAEAAAAAAIhA0AQAAAAAACBC3dDQ0Iz9DAAAAAAAAKtuRrGfAQAAAAAAYIj8PwjORXgdXOCwAAAAAElFTkSuQmCC"
    }
   },
   "cell_type": "markdown",
   "id": "27aad884",
   "metadata": {},
   "source": [
    "![image-2.png](attachment:image-2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "15c80cde",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_gb_grid = pd.DataFrame(data=gb_grid.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "9bcceb9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_gb_grid.to_excel(\"Gradient_Boosted_GridSearch.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4762dee6",
   "metadata": {},
   "source": [
    "### Best Fit Gradient Boosted Trees (precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "a188c77d",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb = GradientBoostingClassifier(warm_start=False,\n",
    "                                random_state=9,\n",
    "                                max_depth=1,\n",
    "                                max_features='log2',\n",
    "                                min_samples_leaf=1,\n",
    "                                min_samples_split=2,\n",
    "                                n_estimators=100)\n",
    "                                \n",
    "gbm = gb.fit(df_train, y_train.ravel())\n",
    "gbr = gbm.predict(df_train)\n",
    "gbt = gbm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "268e70b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8785299806576402\n",
      "Precision: 1.0\n",
      "Recall: 0.006329113924050633\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(gbt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "53351838",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ccp_alpha': 0.0, 'criterion': 'friedman_mse', 'init': None, 'learning_rate': 0.1, 'loss': 'log_loss', 'max_depth': 1, 'max_features': 'log2', 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_iter_no_change': None, 'random_state': 9, 'subsample': 1.0, 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': 0, 'warm_start': False}\n"
     ]
    }
   ],
   "source": [
    "print(gb.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7f5dd2c",
   "metadata": {},
   "source": [
    "### Best Fit Gradient Boosted Trees (f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "605bc80c",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb2 = GradientBoostingClassifier(warm_start=False,\n",
    "                                random_state=9,\n",
    "                                max_depth=None,\n",
    "                                max_features='sqrt',\n",
    "                                min_samples_leaf=4,\n",
    "                                min_samples_split=10,\n",
    "                                n_estimators=100)\n",
    "                                \n",
    "gbm2 = gb2.fit(df_train, y_train.ravel())\n",
    "gbr2 = gbm2.predict(df_train)\n",
    "gbt2 = gbm2.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "2ad4f0ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9806576402321083\n",
      "Precision: 0.975\n",
      "Recall: 0.8639240506329114\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(gbt2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "6e242667",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ccp_alpha': 0.0, 'criterion': 'friedman_mse', 'init': None, 'learning_rate': 0.1, 'loss': 'log_loss', 'max_depth': None, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 4, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_iter_no_change': None, 'random_state': 9, 'subsample': 1.0, 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': 0, 'warm_start': False}\n"
     ]
    }
   ],
   "source": [
    "print(gb2.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36fc8143",
   "metadata": {},
   "source": [
    "### Model - Hyertuned ExtraTrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "ce147392",
   "metadata": {},
   "outputs": [],
   "source": [
    "# n_estimators_e = [int(x) for x in np.linspace(start = 100, stop = 500, num = 3)]\n",
    "# max_features_e = ['log2', 'sqrt']\n",
    "# min_samples_split_e = [2, 5, 10]\n",
    "# min_samples_leaf_e = [1, 2, 4]\n",
    "# bootstrap_e = ['bool',False]\n",
    "# criterion_e = ['gini','log_loss','entropy']\n",
    "# random_grid_e = {\n",
    "#                'n_estimators': n_estimators_e,\n",
    "#                'max_features': max_features_e,\n",
    "#                'min_samples_split': min_samples_split_e,\n",
    "#                'min_samples_leaf': min_samples_leaf_e,\n",
    "#                 'bootstrap': bootstrap_e,\n",
    "#                 'criterion': criterion_e\n",
    "#                 }\n",
    "# print(random_grid_e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "6f2cdf43",
   "metadata": {},
   "outputs": [],
   "source": [
    "# et = ExtraTreesClassifier(warm_start=False,random_state=9,class_weight='Balanced')\n",
    "# et_grid = GridSearchCV(et,random_grid_e, cv = 5, verbose=2, scoring='precision')\n",
    "# et_grid.fit(df_train, y_train.ravel())\n",
    "# print(et_grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "4489a61c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(et_grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "481b3d41",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_et_grid = pd.DataFrame(data=et_grid.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "b718f322",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_et_grid.to_excel(\"ET_GridSearch.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a160c21b",
   "metadata": {},
   "source": [
    "### Best Fit Extra Trees (precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "8bb4e32c",
   "metadata": {},
   "outputs": [],
   "source": [
    "et = ExtraTreesClassifier(\n",
    "        warm_start=False,\n",
    "        random_state=9,\n",
    "        bootstrap=True,\n",
    "        class_weight='balanced',\n",
    "        criterion='log_loss',\n",
    "        max_features='log2',\n",
    "        min_samples_split=10)\n",
    "etm = et.fit(df_train, y_train.ravel())\n",
    "etr = etm.predict(df_train)\n",
    "ett = etm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "95e2e3a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9787234042553191\n",
      "Precision: 0.9961977186311787\n",
      "Recall: 0.8291139240506329\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(ett)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "d4c499c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': 'balanced', 'criterion': 'log_loss', 'max_depth': None, 'max_features': 'log2', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_jobs': None, 'oob_score': False, 'random_state': 9, 'verbose': 0, 'warm_start': False}\n"
     ]
    }
   ],
   "source": [
    "print(et.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d350408b",
   "metadata": {},
   "source": [
    "### Best Fit Extra Trees (F1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "a6408334",
   "metadata": {},
   "outputs": [],
   "source": [
    "et2 = ExtraTreesClassifier(\n",
    "        warm_start=False,\n",
    "        random_state=9,\n",
    "        bootstrap=False,\n",
    "        class_weight='balanced',\n",
    "        criterion='log_loss',\n",
    "        max_features='log2',\n",
    "        min_samples_split=10)\n",
    "etm2 = et2.fit(df_train, y_train.ravel())\n",
    "etr2 = etm2.predict(df_train)\n",
    "ett2 = etm2.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "5ea34702",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9794970986460348\n",
      "Precision: 0.9747292418772563\n",
      "Recall: 0.8544303797468354\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(ett2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "db0a8c5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': False, 'ccp_alpha': 0.0, 'class_weight': 'balanced', 'criterion': 'log_loss', 'max_depth': None, 'max_features': 'log2', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_jobs': None, 'oob_score': False, 'random_state': 9, 'verbose': 0, 'warm_start': False}\n"
     ]
    }
   ],
   "source": [
    "print(et2.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f782cf19",
   "metadata": {},
   "source": [
    "### Hypertuned ADA Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "aed56b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# param_grid_ad = {'n_estimators': [int(x) for x in np.linspace(start = 20, stop = 300, num = 10)], \n",
    "#                  'learning_rate': [.1,.5,1]}\n",
    "# grid_ad = GridSearchCV(AdaBoostClassifier(random_state=9),param_grid_ad,refit=True,verbose=3,scoring='precision',cv=5)\n",
    "# adm = grid_ad.fit(df_train,y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "8568b6af",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(grid_ad.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "eb3c9316",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_ad_grid = pd.DataFrame(data=grid_ad.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "2147e57a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_ad_grid.to_excel(\"AD_GridSearch.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f38b0d55",
   "metadata": {},
   "source": [
    "### Best Fit ADA Boosting (precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "81bc2a6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad = AdaBoostClassifier(random_state=9,learning_rate=0.1,n_estimators=20)\n",
    "adm = ad.fit(df_train, y_train.ravel())\n",
    "adr = adm.predict(df_train)\n",
    "adt = adm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "0e1582fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9056092843326886\n",
      "Precision: 1.0\n",
      "Recall: 0.22784810126582278\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(adt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "c5d7e8ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'algorithm': 'SAMME.R', 'base_estimator': 'deprecated', 'estimator': None, 'learning_rate': 0.1, 'n_estimators': 20, 'random_state': 9}\n"
     ]
    }
   ],
   "source": [
    "print(ad.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2b6de9b",
   "metadata": {},
   "source": [
    "### Best Fit ADA Bososting (f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "19e45c14",
   "metadata": {},
   "outputs": [],
   "source": [
    "ad2 = AdaBoostClassifier(random_state=9,learning_rate=0.5,n_estimators=268)\n",
    "adm2 = ad2.fit(df_train, y_train.ravel())\n",
    "adr2 = adm2.predict(df_train)\n",
    "adt2 = adm2.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "46f1890d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9756286266924564\n",
      "Precision: 0.9501779359430605\n",
      "Recall: 0.8449367088607594\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(adt2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "365933fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'algorithm': 'SAMME.R', 'base_estimator': 'deprecated', 'estimator': None, 'learning_rate': 0.5, 'n_estimators': 268, 'random_state': 9}\n"
     ]
    }
   ],
   "source": [
    "print(ad2.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f98716a1",
   "metadata": {},
   "source": [
    "### XGB Hypertuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "655aff52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# params = {'max_depth': [3, 6, 10],\n",
    "#               'learning_rate': [ 0.1, 0.5],\n",
    "#               'subsample': np.arange(0.5, 1.0, 0.2),\n",
    "#               'colsample_bytree': np.arange(0.5, 1.0, 0.2),\n",
    "#               'colsample_bylevel': np.arange(0.5, 1.0, 0.2),\n",
    "#               'n_estimators': [50, 100],\n",
    "#               'num_class': [1]\n",
    "#               }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "f9bd062a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# grid_xg = GridSearchCV(XGBClassifier(random_state=9,scale_pos_weight=20),params,refit=True,verbose=3,scoring='precision',cv=5,n_jobs=-2)\n",
    "# xgm = grid_xg.fit(df_train,y_train.ravel())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "dfd40d0b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# print(grid_xg.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "91394905",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_xg_grid = pd.DataFrame(data=grid_xg.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "64a6eeaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_xg_grid.to_excel(\"XG_GridSearch.xlsx\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a9903dd",
   "metadata": {},
   "source": [
    "### Model - XGB Classifier (precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "06e05764",
   "metadata": {},
   "outputs": [],
   "source": [
    "xg = XGBClassifier(\n",
    "                    random_state=9,\n",
    "                    colsample_bylevel=0.5, \n",
    "                    colsample_bytree=0.7, \n",
    "                    learning_rate=0.5, \n",
    "                    max_depth=6, \n",
    "                    n_estimators=100, \n",
    "                    num_class=1, \n",
    "                    subsample=0.9,\n",
    "                    scale_pos_weight=20)\n",
    "xgm = xg.fit(df_train, y_train.ravel())\n",
    "xgr = xgm.predict(df_train)\n",
    "xgt = xgm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "03d93d95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9713733075435204\n",
      "Precision: 0.8805031446540881\n",
      "Recall: 0.8860759493670886\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(xgt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "7d2333ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'objective': 'binary:logistic', 'use_label_encoder': None, 'base_score': None, 'booster': None, 'callbacks': None, 'colsample_bylevel': 0.5, 'colsample_bynode': None, 'colsample_bytree': 0.7, 'early_stopping_rounds': None, 'enable_categorical': False, 'eval_metric': None, 'feature_types': None, 'gamma': None, 'gpu_id': None, 'grow_policy': None, 'importance_type': None, 'interaction_constraints': None, 'learning_rate': 0.5, 'max_bin': None, 'max_cat_threshold': None, 'max_cat_to_onehot': None, 'max_delta_step': None, 'max_depth': 6, 'max_leaves': None, 'min_child_weight': None, 'missing': nan, 'monotone_constraints': None, 'n_estimators': 100, 'n_jobs': None, 'num_parallel_tree': None, 'predictor': None, 'random_state': 9, 'reg_alpha': None, 'reg_lambda': None, 'sampling_method': None, 'scale_pos_weight': 20, 'subsample': 0.9, 'tree_method': None, 'validate_parameters': None, 'verbosity': None, 'num_class': 1}\n"
     ]
    }
   ],
   "source": [
    "print(xg.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f0a90b3",
   "metadata": {},
   "source": [
    "### Best Fit XGB Classifier (f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "76029828",
   "metadata": {},
   "outputs": [],
   "source": [
    "xg2 = XGBClassifier(\n",
    "                    random_state=9,\n",
    "                    colsample_bylevel=0.5, \n",
    "                    colsample_bytree=0.5, \n",
    "                    learning_rate=0.1, \n",
    "                    max_depth=10, \n",
    "                    n_estimators=250, \n",
    "                    num_class=1, \n",
    "                    subsample=0.75,\n",
    "                    scale_pos_weight=20)\n",
    "xgm2 = xg2.fit(df_train, y_train.ravel())\n",
    "xgr2 = xgm2.predict(df_train)\n",
    "xgt2 = xgm2.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "c22a50f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9690522243713733\n",
      "Precision: 0.8782051282051282\n",
      "Recall: 0.8670886075949367\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(xgt2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "0def667a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'objective': 'binary:logistic', 'use_label_encoder': None, 'base_score': None, 'booster': None, 'callbacks': None, 'colsample_bylevel': 0.5, 'colsample_bynode': None, 'colsample_bytree': 0.5, 'early_stopping_rounds': None, 'enable_categorical': False, 'eval_metric': None, 'feature_types': None, 'gamma': None, 'gpu_id': None, 'grow_policy': None, 'importance_type': None, 'interaction_constraints': None, 'learning_rate': 0.1, 'max_bin': None, 'max_cat_threshold': None, 'max_cat_to_onehot': None, 'max_delta_step': None, 'max_depth': 10, 'max_leaves': None, 'min_child_weight': None, 'missing': nan, 'monotone_constraints': None, 'n_estimators': 250, 'n_jobs': None, 'num_parallel_tree': None, 'predictor': None, 'random_state': 9, 'reg_alpha': None, 'reg_lambda': None, 'sampling_method': None, 'scale_pos_weight': 20, 'subsample': 0.75, 'tree_method': None, 'validate_parameters': None, 'verbosity': None, 'num_class': 1}\n"
     ]
    }
   ],
   "source": [
    "print(xg2.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "167d1952",
   "metadata": {},
   "source": [
    "### Model XGBRF Classifier Hypertuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "d0b63cc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# params_brf = {'max_depth': [3, 6, 10, 15],\n",
    "#               'learning_rate': [ 0.1, 0.5, 1],\n",
    "#               'subsample': np.arange(0.5, 1.0, 0.1),\n",
    "#               'colsample_bytree': np.arange(0.5, 1.0, 0.1),\n",
    "#               'colsample_bylevel': np.arange(0.5, 1.0, 0.1),\n",
    "#               'n_estimators': [50, 100, 250],\n",
    "#               'num_class': [1]\n",
    "#               }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc59b5f0",
   "metadata": {},
   "source": [
    "### Model - XGBRF Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "02cedc01",
   "metadata": {},
   "outputs": [],
   "source": [
    "xr = XGBRFClassifier(random_state=9)\n",
    "xrm = xr.fit(df_train, y_train)\n",
    "xrr = xrm.predict(df_train)\n",
    "xrt = xrm.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "e0073f61",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.948936170212766\n",
      "Precision: 0.9509803921568627\n",
      "Recall: 0.6139240506329114\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(xrt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "9dfdec04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'colsample_bynode': 0.8, 'learning_rate': 1.0, 'reg_lambda': 1e-05, 'subsample': 0.8, 'objective': 'binary:logistic', 'use_label_encoder': None, 'base_score': None, 'booster': None, 'callbacks': None, 'colsample_bylevel': None, 'colsample_bytree': None, 'early_stopping_rounds': None, 'enable_categorical': False, 'eval_metric': None, 'feature_types': None, 'gamma': None, 'gpu_id': None, 'grow_policy': None, 'importance_type': None, 'interaction_constraints': None, 'max_bin': None, 'max_cat_threshold': None, 'max_cat_to_onehot': None, 'max_delta_step': None, 'max_depth': None, 'max_leaves': None, 'min_child_weight': None, 'missing': nan, 'monotone_constraints': None, 'n_estimators': 100, 'n_jobs': None, 'num_parallel_tree': None, 'predictor': None, 'random_state': 9, 'reg_alpha': None, 'sampling_method': None, 'scale_pos_weight': None, 'tree_method': None, 'validate_parameters': None, 'verbosity': None}\n"
     ]
    }
   ],
   "source": [
    "print(xr.get_params())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3f1a2fb",
   "metadata": {},
   "source": [
    "### Model_CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "c5d51b8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (None, 79, 64)            436480    \n",
      "                                                                 \n",
      " global_average_pooling1d (G  (None, 64)               0         \n",
      " lobalAveragePooling1D)                                          \n",
      "                                                                 \n",
      " dense (Dense)               (None, 64)                4160      \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 32)                2080      \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 1)                 33        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 442,753\n",
      "Trainable params: 442,753\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "md_cnn = tf.keras.Sequential()\n",
    "md_cnn.add(tf.keras.layers.Embedding(input_dim=len(d)+1, output_dim=64, input_length=x_train.shape[1]))\n",
    "md_cnn.add(tf.keras.layers.GlobalAveragePooling1D())\n",
    "md_cnn.add(tf.keras.layers.Dense(64, activation='relu'))\n",
    "md_cnn.add(tf.keras.layers.Dense(32, activation='relu'))\n",
    "md_cnn.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "optimizer = keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95, nesterov=True)\n",
    "md_cnn.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', 'Precision', 'Recall'])\n",
    "\n",
    "md_cnn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "8c190ae7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "81/81 [==============================] - 1s 3ms/step - loss: 0.4626 - accuracy: 0.8696 - precision: 0.0000e+00 - recall: 0.0000e+00\n",
      "Epoch 2/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.3521 - accuracy: 0.8696 - precision: 0.0000e+00 - recall: 0.0000e+00\n",
      "Epoch 3/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.2994 - accuracy: 0.8707 - precision: 1.0000 - recall: 0.0089   \n",
      "Epoch 4/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.1208 - accuracy: 0.9590 - precision: 0.9957 - recall: 0.6884\n",
      "Epoch 5/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0427 - accuracy: 0.9888 - precision: 0.9904 - recall: 0.9228\n",
      "Epoch 6/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0281 - accuracy: 0.9938 - precision: 0.9908 - recall: 0.9614\n",
      "Epoch 7/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0237 - accuracy: 0.9930 - precision: 0.9938 - recall: 0.9525\n",
      "Epoch 8/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0180 - accuracy: 0.9946 - precision: 0.9939 - recall: 0.9644\n",
      "Epoch 9/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0174 - accuracy: 0.9946 - precision: 0.9939 - recall: 0.9644\n",
      "Epoch 10/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0133 - accuracy: 0.9969 - precision: 0.9970 - recall: 0.9792\n",
      "Epoch 11/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0120 - accuracy: 0.9965 - precision: 0.9910 - recall: 0.9822\n",
      "Epoch 12/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0105 - accuracy: 0.9973 - precision: 0.9970 - recall: 0.9822\n",
      "Epoch 13/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0103 - accuracy: 0.9973 - precision: 0.9970 - recall: 0.9822\n",
      "Epoch 14/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0106 - accuracy: 0.9977 - precision: 1.0000 - recall: 0.9822\n",
      "Epoch 15/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0065 - accuracy: 0.9981 - precision: 1.0000 - recall: 0.9852\n",
      "Epoch 16/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0086 - accuracy: 0.9985 - precision: 1.0000 - recall: 0.9881\n",
      "Epoch 17/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0064 - accuracy: 0.9985 - precision: 0.9970 - recall: 0.9911\n",
      "Epoch 18/50\n",
      "81/81 [==============================] - 0s 3ms/step - loss: 0.0075 - accuracy: 0.9981 - precision: 0.9970 - recall: 0.9881\n",
      "Epoch 18: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1c3829779a0>"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md_cnn.fit(x_train_array, y_train, epochs=50, callbacks=[callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "b9db7579",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81/81 [==============================] - 0s 901us/step\n",
      "81/81 [==============================] - 0s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "cnt = md_cnn.predict(x_test_array, batch_size=None)\n",
    "cnr = md_cnn.predict(x_train_array, batch_size=None)\n",
    "cnt = np.concatenate(cnt).round().astype(int)\n",
    "cnr = np.concatenate(cnr).round().astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "5a105c00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9744680851063829\n",
      "Precision: 0.9139072847682119\n",
      "Recall: 0.8734177215189873\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e7097b6",
   "metadata": {},
   "source": [
    "### Model - BiDirectional LSTM with Hidden Layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "a19ce512",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_1 (Embedding)     (None, 79, 100)           682000    \n",
      "                                                                 \n",
      " bidirectional (Bidirectiona  (None, 128)              84480     \n",
      " l)                                                              \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 64)                8256      \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 32)                2080      \n",
      "                                                                 \n",
      " dense_5 (Dense)             (None, 1)                 33        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 776,849\n",
      "Trainable params: 776,849\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#embedding layer and bidirectional layer\n",
    "md_bidir_w_hidden = tf.keras.Sequential()\n",
    "md_bidir_w_hidden.add(tf.keras.layers.Embedding(input_dim= len(d) + 1,output_dim=100,input_length=x_train.shape[1]))\n",
    "md_bidir_w_hidden.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))\n",
    "md_bidir_w_hidden.add(tf.keras.layers.Dense(64, activation='relu'))\n",
    "md_bidir_w_hidden.add(tf.keras.layers.Dense(32, activation='relu'))\n",
    "md_bidir_w_hidden.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n",
    "md_bidir_w_hidden.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "6a619f4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#compile the model\n",
    "optimizer = keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95, nesterov=True)\n",
    "md_bidir_w_hidden.compile(loss=\"binary_crossentropy\",optimizer=optimizer,metrics=['accuracy', 'Precision', 'Recall'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "2fce96b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "81/81 [==============================] - 7s 51ms/step - loss: 0.4085 - accuracy: 0.8603 - precision: 0.1250 - recall: 0.0119\n",
      "Epoch 2/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.3461 - accuracy: 0.8696 - precision: 0.0000e+00 - recall: 0.0000e+00\n",
      "Epoch 3/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.3110 - accuracy: 0.8696 - precision: 0.0000e+00 - recall: 0.0000e+00\n",
      "Epoch 4/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.2762 - accuracy: 0.8727 - precision: 0.9000 - recall: 0.0267\n",
      "Epoch 5/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.1721 - accuracy: 0.9447 - precision: 0.9042 - recall: 0.6439\n",
      "Epoch 6/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0999 - accuracy: 0.9690 - precision: 0.9159 - recall: 0.8398\n",
      "Epoch 7/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0659 - accuracy: 0.9822 - precision: 0.9619 - recall: 0.8991\n",
      "Epoch 8/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0518 - accuracy: 0.9861 - precision: 0.9748 - recall: 0.9169\n",
      "Epoch 9/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0340 - accuracy: 0.9899 - precision: 0.9755 - recall: 0.9466\n",
      "Epoch 10/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0322 - accuracy: 0.9911 - precision: 0.9816 - recall: 0.9496\n",
      "Epoch 11/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0158 - accuracy: 0.9957 - precision: 0.9939 - recall: 0.9733\n",
      "Epoch 12/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0138 - accuracy: 0.9969 - precision: 0.9940 - recall: 0.9822\n",
      "Epoch 13/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0138 - accuracy: 0.9965 - precision: 0.9910 - recall: 0.9822\n",
      "Epoch 14/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0140 - accuracy: 0.9969 - precision: 0.9940 - recall: 0.9822\n",
      "Epoch 15/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0098 - accuracy: 0.9977 - precision: 0.9940 - recall: 0.9881\n",
      "Epoch 16/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0070 - accuracy: 0.9985 - precision: 0.9970 - recall: 0.9911\n",
      "Epoch 17/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0063 - accuracy: 0.9988 - precision: 1.0000 - recall: 0.9911\n",
      "Epoch 18/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0053 - accuracy: 0.9988 - precision: 0.9970 - recall: 0.9941\n",
      "Epoch 19/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0040 - accuracy: 0.9992 - precision: 1.0000 - recall: 0.9941\n",
      "Epoch 20/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0054 - accuracy: 0.9981 - precision: 0.9911 - recall: 0.9941\n",
      "Epoch 21/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0086 - accuracy: 0.9981 - precision: 0.9970 - recall: 0.9881\n",
      "Epoch 22/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0042 - accuracy: 0.9988 - precision: 1.0000 - recall: 0.9911\n",
      "Epoch 23/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0041 - accuracy: 0.9985 - precision: 0.9970 - recall: 0.9911\n",
      "Epoch 24/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0026 - accuracy: 0.9992 - precision: 1.0000 - recall: 0.9941\n",
      "Epoch 25/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0068 - accuracy: 0.9977 - precision: 0.9911 - recall: 0.9911\n",
      "Epoch 26/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0027 - accuracy: 0.9992 - precision: 1.0000 - recall: 0.9941\n",
      "Epoch 27/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0024 - accuracy: 0.9992 - precision: 1.0000 - recall: 0.9941\n",
      "Epoch 28/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0014 - accuracy: 0.9996 - precision: 1.0000 - recall: 0.9970\n",
      "Epoch 29/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0026 - accuracy: 0.9985 - precision: 0.9941 - recall: 0.9941\n",
      "Epoch 30/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0015 - accuracy: 0.9996 - precision: 1.0000 - recall: 0.9970\n",
      "Epoch 31/50\n",
      "81/81 [==============================] - 4s 52ms/step - loss: 0.0014 - accuracy: 0.9988 - precision: 0.9941 - recall: 0.9970\n",
      "Epoch 32/50\n",
      "81/81 [==============================] - 4s 51ms/step - loss: 0.0014 - accuracy: 0.9996 - precision: 1.0000 - recall: 0.9970\n",
      "Epoch 32: early stopping\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1c384199270>"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fit the model\n",
    "md_bidir_w_hidden.fit(x_train_array, y_train, epochs=50, callbacks=[callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "877eab93",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81/81 [==============================] - 2s 16ms/step\n",
      "81/81 [==============================] - 1s 17ms/step\n"
     ]
    }
   ],
   "source": [
    "ltt = md_bidir_w_hidden.predict(x_test_array, batch_size=None)\n",
    "ltr = md_bidir_w_hidden.predict(x_train_array, batch_size=None)\n",
    "ltt = np.concatenate(ltt).round().astype(int)\n",
    "ltr = np.concatenate(ltr).round().astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "17951e84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9756286266924564\n",
      "Precision: 0.953405017921147\n",
      "Recall: 0.8417721518987342\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_metrics(ltt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9ad5c12",
   "metadata": {},
   "source": [
    "### Model - BiDirectional LSTM with Convuluational Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "e8768643",
   "metadata": {},
   "outputs": [],
   "source": [
    "#embedding layer and bidirectional layer\n",
    "#md_combo = tf.keras.Sequential()\n",
    "#md_combo.add(tf.keras.layers.Embedding(input_dim= len(d) + 1,output_dim=64,input_length=x_train.shape[1]))\n",
    "#md_combo.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)))\n",
    "#md_combo.add(tf.keras.layers.GlobalAveragePooling2D(64,input_shape=(3,32,32)))\n",
    "#md_combo.add(Flatten())\n",
    "#md_combo.add(tf.keras.layers.Dense(64, activation='relu'))\n",
    "#md_combo.add(tf.keras.layers.Dense(32, activation='relu'))\n",
    "#md_combo.add(tf.keras.layers.Dense(1, activation='sigmoid'))\n",
    "#md_combo.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "dd14b70c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#compile the model\n",
    "#optimizer = keras.optimizers.SGD(learning_rate=0.02, momentum = 0.95, nesterov=True)\n",
    "#md_combo.compile(loss=\"binary_crossentropy\",optimizer=optimizer,metrics=['accuracy', 'Precision', 'Recall'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "ad9b1244",
   "metadata": {},
   "outputs": [],
   "source": [
    "#fit the model\n",
    "#md_combo.fit(x_train_array, y_train, epochs=50, callbacks=[callback])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "056839e8",
   "metadata": {},
   "source": [
    "### Report (Precision Hyptertuned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "0f6aad24",
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = [lrt, gnt, bnt, mbt, sbt, rft, gbt, ett, adt, xgt, cnt, ltt]\n",
    "method_name = ['LG','GNB','BNB','MNB','SVM','RF','GBT','ET','ADA','XGB','CNN','LTSM']\n",
    "accuracy = []\n",
    "precision = []\n",
    "recall = []\n",
    "f1 = []\n",
    "for i in methods:\n",
    "    accuracy.append(class_accuracy(i))\n",
    "    precision.append(class_precision(i))\n",
    "    recall.append(class_recall(i))\n",
    "    f1.append(class_f1(i))\n",
    "results = pd.DataFrame(\n",
    "    {'Method': method_name,\n",
    "     'Accuracy': accuracy,\n",
    "     'Precision': precision,\n",
    "     'Recall':recall,\n",
    "     'F1':f1\n",
    "    })\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "1f9ecc16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LG</td>\n",
       "      <td>0.972534</td>\n",
       "      <td>0.918089</td>\n",
       "      <td>0.851266</td>\n",
       "      <td>0.883415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GNB</td>\n",
       "      <td>0.809671</td>\n",
       "      <td>0.382038</td>\n",
       "      <td>0.901899</td>\n",
       "      <td>0.536723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BNB</td>\n",
       "      <td>0.961702</td>\n",
       "      <td>0.990950</td>\n",
       "      <td>0.693038</td>\n",
       "      <td>0.815642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MNB</td>\n",
       "      <td>0.950484</td>\n",
       "      <td>0.737374</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>0.820225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM</td>\n",
       "      <td>0.932689</td>\n",
       "      <td>0.943750</td>\n",
       "      <td>0.477848</td>\n",
       "      <td>0.634454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RF</td>\n",
       "      <td>0.964797</td>\n",
       "      <td>0.995595</td>\n",
       "      <td>0.715190</td>\n",
       "      <td>0.832413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>GBT</td>\n",
       "      <td>0.878530</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.006329</td>\n",
       "      <td>0.012579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ET</td>\n",
       "      <td>0.978723</td>\n",
       "      <td>0.996198</td>\n",
       "      <td>0.829114</td>\n",
       "      <td>0.905009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ADA</td>\n",
       "      <td>0.905609</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.227848</td>\n",
       "      <td>0.371134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>XGB</td>\n",
       "      <td>0.971373</td>\n",
       "      <td>0.880503</td>\n",
       "      <td>0.886076</td>\n",
       "      <td>0.883281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CNN</td>\n",
       "      <td>0.974468</td>\n",
       "      <td>0.913907</td>\n",
       "      <td>0.873418</td>\n",
       "      <td>0.893204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LTSM</td>\n",
       "      <td>0.975629</td>\n",
       "      <td>0.953405</td>\n",
       "      <td>0.841772</td>\n",
       "      <td>0.894118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Method  Accuracy  Precision    Recall        F1\n",
       "0      LG  0.972534   0.918089  0.851266  0.883415\n",
       "1     GNB  0.809671   0.382038  0.901899  0.536723\n",
       "2     BNB  0.961702   0.990950  0.693038  0.815642\n",
       "3     MNB  0.950484   0.737374  0.924051  0.820225\n",
       "4     SVM  0.932689   0.943750  0.477848  0.634454\n",
       "5      RF  0.964797   0.995595  0.715190  0.832413\n",
       "6     GBT  0.878530   1.000000  0.006329  0.012579\n",
       "7      ET  0.978723   0.996198  0.829114  0.905009\n",
       "8     ADA  0.905609   1.000000  0.227848  0.371134\n",
       "9     XGB  0.971373   0.880503  0.886076  0.883281\n",
       "10    CNN  0.974468   0.913907  0.873418  0.893204\n",
       "11   LTSM  0.975629   0.953405  0.841772  0.894118"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80a7dd9d",
   "metadata": {},
   "source": [
    "### Report (f1 Hyptertuned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "0ac69268",
   "metadata": {},
   "outputs": [],
   "source": [
    "methods2 = [lrt, gnt, bnt, mbt, sbt2, rft2, gbt2, ett2, adt2, xgt2, cnt, ltt]\n",
    "method_name2 = ['LG','GNB','BNB','MNB','SVM','RF','GBT','ET','ADA','XGB','CNN','LTSM']\n",
    "accuracy2 = []\n",
    "precision2 = []\n",
    "recall2 = []\n",
    "f12 = []\n",
    "for i in methods2:\n",
    "    accuracy2.append(class_accuracy(i))\n",
    "    precision2.append(class_precision(i))\n",
    "    recall2.append(class_recall(i))\n",
    "    f12.append(class_f1(i))\n",
    "results2 = pd.DataFrame(\n",
    "    {'Method': method_name2,\n",
    "     'Accuracy': accuracy2,\n",
    "     'Precision': precision2,\n",
    "     'Recall':recall2,\n",
    "     'F1':f12\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "9ec90bf4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LG</td>\n",
       "      <td>0.972534</td>\n",
       "      <td>0.918089</td>\n",
       "      <td>0.851266</td>\n",
       "      <td>0.883415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GNB</td>\n",
       "      <td>0.809671</td>\n",
       "      <td>0.382038</td>\n",
       "      <td>0.901899</td>\n",
       "      <td>0.536723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BNB</td>\n",
       "      <td>0.961702</td>\n",
       "      <td>0.990950</td>\n",
       "      <td>0.693038</td>\n",
       "      <td>0.815642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MNB</td>\n",
       "      <td>0.950484</td>\n",
       "      <td>0.737374</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>0.820225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM</td>\n",
       "      <td>0.966344</td>\n",
       "      <td>0.987234</td>\n",
       "      <td>0.734177</td>\n",
       "      <td>0.842105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RF</td>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.972028</td>\n",
       "      <td>0.879747</td>\n",
       "      <td>0.923588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>GBT</td>\n",
       "      <td>0.980658</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>0.863924</td>\n",
       "      <td>0.916107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ET</td>\n",
       "      <td>0.979497</td>\n",
       "      <td>0.974729</td>\n",
       "      <td>0.854430</td>\n",
       "      <td>0.910624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ADA</td>\n",
       "      <td>0.975629</td>\n",
       "      <td>0.950178</td>\n",
       "      <td>0.844937</td>\n",
       "      <td>0.894472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>XGB</td>\n",
       "      <td>0.969052</td>\n",
       "      <td>0.878205</td>\n",
       "      <td>0.867089</td>\n",
       "      <td>0.872611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CNN</td>\n",
       "      <td>0.974468</td>\n",
       "      <td>0.913907</td>\n",
       "      <td>0.873418</td>\n",
       "      <td>0.893204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LTSM</td>\n",
       "      <td>0.975629</td>\n",
       "      <td>0.953405</td>\n",
       "      <td>0.841772</td>\n",
       "      <td>0.894118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Method  Accuracy  Precision    Recall        F1\n",
       "0      LG  0.972534   0.918089  0.851266  0.883415\n",
       "1     GNB  0.809671   0.382038  0.901899  0.536723\n",
       "2     BNB  0.961702   0.990950  0.693038  0.815642\n",
       "3     MNB  0.950484   0.737374  0.924051  0.820225\n",
       "4     SVM  0.966344   0.987234  0.734177  0.842105\n",
       "5      RF  0.982205   0.972028  0.879747  0.923588\n",
       "6     GBT  0.980658   0.975000  0.863924  0.916107\n",
       "7      ET  0.979497   0.974729  0.854430  0.910624\n",
       "8     ADA  0.975629   0.950178  0.844937  0.894472\n",
       "9     XGB  0.969052   0.878205  0.867089  0.872611\n",
       "10    CNN  0.974468   0.913907  0.873418  0.893204\n",
       "11   LTSM  0.975629   0.953405  0.841772  0.894118"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d947c8f7",
   "metadata": {},
   "source": [
    "### Creating a Train Data Predictions DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f10c107f",
   "metadata": {},
   "source": [
    "This dataframe will be used to first identify false positives and then calculate the jaccard score amongst false negatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "f3100d0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "yy = s_pandas['label'].iloc[train_index]\n",
    "train_predictions = [lrr, gnr, bnr, mbr, sbr2, rfr2, gbr2, etr2, adr2, xgr2, cnr, ltr]\n",
    "for i in train_predictions:\n",
    "    i = i.tolist()\n",
    "train_results = list(zip(yy,lrr, gnr, bnr, mbr, sbr2, rfr2, gbr2, etr2, adr2, xgr2, cnr, ltr))\n",
    "df_results = pd.DataFrame(data=train_results,columns=['actual','LG','GNB','BNB','MNB','SVM','RF','GBT','ET','ADA','XGB','CNN','LTSM'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "628536e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>True Positive</th>\n",
       "      <th>False Positive</th>\n",
       "      <th>False Negative</th>\n",
       "      <th>True Positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LG</td>\n",
       "      <td>2243</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GNB</td>\n",
       "      <td>1883</td>\n",
       "      <td>364</td>\n",
       "      <td>0</td>\n",
       "      <td>337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BNB</td>\n",
       "      <td>2247</td>\n",
       "      <td>0</td>\n",
       "      <td>74</td>\n",
       "      <td>263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MNB</td>\n",
       "      <td>2174</td>\n",
       "      <td>73</td>\n",
       "      <td>6</td>\n",
       "      <td>331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM</td>\n",
       "      <td>2247</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RF</td>\n",
       "      <td>2247</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>GBT</td>\n",
       "      <td>2247</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ET</td>\n",
       "      <td>2247</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ADA</td>\n",
       "      <td>2247</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>XGB</td>\n",
       "      <td>2238</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CNN</td>\n",
       "      <td>2245</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LTSM</td>\n",
       "      <td>2247</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>336</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Method  True Positive  False Positive  False Negative  True Positive\n",
       "0      LG           2243               4               3            334\n",
       "1     GNB           1883             364               0            337\n",
       "2     BNB           2247               0              74            263\n",
       "3     MNB           2174              73               6            331\n",
       "4     SVM           2247               0               1            336\n",
       "5      RF           2247               0               1            336\n",
       "6     GBT           2247               0               3            334\n",
       "7      ET           2247               0               1            336\n",
       "8     ADA           2247               0               2            335\n",
       "9     XGB           2238               9               1            336\n",
       "10    CNN           2245               2               2            335\n",
       "11   LTSM           2247               0               1            336"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn_list = []\n",
    "fp_list = []\n",
    "fn_list = []\n",
    "tp_list = []\n",
    "for i in range(len(train_predictions)):\n",
    "    tn, fp, fn, tp = confusion_matrix(yy, train_predictions[i], labels=[0, 1]).ravel()\n",
    "    tn_list.append(tn)\n",
    "    fp_list.append(fp)\n",
    "    fn_list.append(fn)\n",
    "    tp_list.append(tp)\n",
    "confusion = list(zip(method_name,tn_list,fp_list,fn_list,tp_list))\n",
    "df_confusion = pd.DataFrame(data=confusion,columns=['Method','True Positive','False Positive','False Negative','True Positive'])\n",
    "df_confusion"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d734c4ee",
   "metadata": {},
   "source": [
    "### Creating a Test Data Predictions DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "ffa379a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "yz = s_pandas['label'].iloc[test_index]\n",
    "test_predictions = [lrt, gnt, bnt, mbt, sbt2, rft2, gbt2, ett2, adt2, xgt2, cnt, ltt]\n",
    "for i in test_predictions:\n",
    "    i = i.tolist()\n",
    "test_results = list(zip(yz,lrt, gnt, bnt, mbt, sbt2, rft2, gbt2, ett2, adt2, xgt2, cnt, ltt))\n",
    "df_results_test = pd.DataFrame(data=test_results,columns=['actual','LG','GNB','BNB','MNB','SVM','RF','GBT','ET','ADA','XGB','CNN','LTSM'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "b4a4a8d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>True Positive</th>\n",
       "      <th>False Positive</th>\n",
       "      <th>False Negative</th>\n",
       "      <th>True Positive</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LG</td>\n",
       "      <td>2245</td>\n",
       "      <td>24</td>\n",
       "      <td>47</td>\n",
       "      <td>269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GNB</td>\n",
       "      <td>1808</td>\n",
       "      <td>461</td>\n",
       "      <td>31</td>\n",
       "      <td>285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BNB</td>\n",
       "      <td>2267</td>\n",
       "      <td>2</td>\n",
       "      <td>97</td>\n",
       "      <td>219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MNB</td>\n",
       "      <td>2165</td>\n",
       "      <td>104</td>\n",
       "      <td>24</td>\n",
       "      <td>292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM</td>\n",
       "      <td>2266</td>\n",
       "      <td>3</td>\n",
       "      <td>84</td>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RF</td>\n",
       "      <td>2261</td>\n",
       "      <td>8</td>\n",
       "      <td>38</td>\n",
       "      <td>278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>GBT</td>\n",
       "      <td>2262</td>\n",
       "      <td>7</td>\n",
       "      <td>43</td>\n",
       "      <td>273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ET</td>\n",
       "      <td>2262</td>\n",
       "      <td>7</td>\n",
       "      <td>46</td>\n",
       "      <td>270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ADA</td>\n",
       "      <td>2255</td>\n",
       "      <td>14</td>\n",
       "      <td>49</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>XGB</td>\n",
       "      <td>2231</td>\n",
       "      <td>38</td>\n",
       "      <td>42</td>\n",
       "      <td>274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CNN</td>\n",
       "      <td>2243</td>\n",
       "      <td>26</td>\n",
       "      <td>40</td>\n",
       "      <td>276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LTSM</td>\n",
       "      <td>2256</td>\n",
       "      <td>13</td>\n",
       "      <td>50</td>\n",
       "      <td>266</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Method  True Positive  False Positive  False Negative  True Positive\n",
       "0      LG           2245              24              47            269\n",
       "1     GNB           1808             461              31            285\n",
       "2     BNB           2267               2              97            219\n",
       "3     MNB           2165             104              24            292\n",
       "4     SVM           2266               3              84            232\n",
       "5      RF           2261               8              38            278\n",
       "6     GBT           2262               7              43            273\n",
       "7      ET           2262               7              46            270\n",
       "8     ADA           2255              14              49            267\n",
       "9     XGB           2231              38              42            274\n",
       "10    CNN           2243              26              40            276\n",
       "11   LTSM           2256              13              50            266"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tn2_list = []\n",
    "fp2_list = []\n",
    "fn2_list = []\n",
    "tp2_list = []\n",
    "for i in range(len(test_predictions)):\n",
    "    tn, fp, fn, tp = confusion_matrix(yz, test_predictions[i], labels=[0, 1]).ravel()\n",
    "    tn2_list.append(tn)\n",
    "    fp2_list.append(fp)\n",
    "    fn2_list.append(fn)\n",
    "    tp2_list.append(tp)\n",
    "confusion2 = list(zip(method_name,tn2_list,fp2_list,fn2_list,tp2_list))\n",
    "df_confusion2 = pd.DataFrame(data=confusion2,columns=['Method','True Positive','False Positive','False Negative','True Positive'])\n",
    "df_confusion2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce8a1e8a",
   "metadata": {},
   "source": [
    "### Creating a DataFrame of False Positives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "2ae2227d",
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = s_pandas['doc'].iloc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "1b62e773",
   "metadata": {},
   "outputs": [],
   "source": [
    "fp_list = []\n",
    "for i in method_name:\n",
    "    fp_list.append(np.where(\n",
    "            ((df_results['actual'] == 1) & (df_results[i] == 0))\n",
    "        |   ((df_results['actual'] == 0) & (df_results[i] == 1))\n",
    "            ,1,0).tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "b4534bdc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fp = pd.DataFrame(data=fp_list)\n",
    "df_fp = df_fp.transpose()\n",
    "df_fp = df_fp.set_axis(method_name,axis=1)\n",
    "df_fp_test = df_fp\n",
    "df_fp_test['docs'] = docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "1b26567c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(method_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "8fef4eb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fp.MNB.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "68f44fbe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LG</th>\n",
       "      <th>GNB</th>\n",
       "      <th>BNB</th>\n",
       "      <th>MNB</th>\n",
       "      <th>SVM</th>\n",
       "      <th>RF</th>\n",
       "      <th>GBT</th>\n",
       "      <th>ET</th>\n",
       "      <th>ADA</th>\n",
       "      <th>XGB</th>\n",
       "      <th>CNN</th>\n",
       "      <th>LTSM</th>\n",
       "      <th>docs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ok lar joke wif oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>free entri wkli comp win fa cup final tkt may ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>nah think goe usf live around though</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   LG  GNB  BNB  MNB  SVM  RF  GBT  ET  ADA  XGB  CNN  LTSM  \\\n",
       "0   0    0    0    0    0   0    0   0    0    0    0     0   \n",
       "1   0    1    0    1    0   0    0   0    0    0    0     0   \n",
       "2   0    0    0    0    0   0    0   0    0    0    0     0   \n",
       "3   0    0    0    0    0   0    0   0    0    0    0     0   \n",
       "4   0    0    0    0    0   0    0   0    0    0    0     0   \n",
       "\n",
       "                                                docs  \n",
       "0                                                NaN  \n",
       "1                                ok lar joke wif oni  \n",
       "2  free entri wkli comp win fa cup final tkt may ...  \n",
       "3                                                NaN  \n",
       "4               nah think goe usf live around though  "
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fp_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "d0911cdc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LG</th>\n",
       "      <th>GNB</th>\n",
       "      <th>BNB</th>\n",
       "      <th>MNB</th>\n",
       "      <th>SVM</th>\n",
       "      <th>RF</th>\n",
       "      <th>GBT</th>\n",
       "      <th>ET</th>\n",
       "      <th>ADA</th>\n",
       "      <th>XGB</th>\n",
       "      <th>CNN</th>\n",
       "      <th>LTSM</th>\n",
       "      <th>docs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>thank much skype wit kz sura didnt get pleasur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1959</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>avatar suppos subtoitl</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      LG  GNB  BNB  MNB  SVM  RF  GBT  ET  ADA  XGB  CNN  LTSM  \\\n",
       "815    0    0    1    1    0   0    0   0    1    0    0     0   \n",
       "1959   1    0    1    1    1   1    1   1    1    1    1     1   \n",
       "\n",
       "                                                   docs  \n",
       "815   thank much skype wit kz sura didnt get pleasur...  \n",
       "1959                             avatar suppos subtoitl  "
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_fp_test[df_fp_test['ADA'] == 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f19ef529",
   "metadata": {},
   "source": [
    "### Jaccard Score Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "79a34f32",
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculate the jaccard scores\n",
    "method1 = []\n",
    "method2 = []\n",
    "jac = []\n",
    "for x in method_name:\n",
    "    for y in method_name:\n",
    "        method1.append(x)\n",
    "        method2.append(y)\n",
    "        jac.append(jaccard_score(df_fp[x],df_fp[y]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "a09027e7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>method2</th>\n",
       "      <th>ADA</th>\n",
       "      <th>BNB</th>\n",
       "      <th>CNN</th>\n",
       "      <th>ET</th>\n",
       "      <th>GBT</th>\n",
       "      <th>GNB</th>\n",
       "      <th>LG</th>\n",
       "      <th>LTSM</th>\n",
       "      <th>MNB</th>\n",
       "      <th>RF</th>\n",
       "      <th>SVM</th>\n",
       "      <th>XGB</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method1</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ADA</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.027027</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.025316</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.090909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BNB</th>\n",
       "      <td>0.027027</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.026316</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.040541</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.012048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNN</th>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.026316</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.005464</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.037500</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ET</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GBT</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.040541</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.012346</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GNB</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005464</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.005420</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.101990</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.010811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LG</th>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.005420</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.061728</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.214286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LTSM</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MNB</th>\n",
       "      <td>0.025316</td>\n",
       "      <td>0.040816</td>\n",
       "      <td>0.037500</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>0.012346</td>\n",
       "      <td>0.101990</td>\n",
       "      <td>0.061728</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>0.112500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RF</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVM</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.012658</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB</th>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.012048</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>0.010811</td>\n",
       "      <td>0.214286</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.112500</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "method2       ADA       BNB       CNN        ET       GBT       GNB        LG  \\\n",
       "method1                                                                         \n",
       "ADA      1.000000  0.027027  0.200000  0.500000  0.250000  0.000000  0.125000   \n",
       "BNB      0.027027  1.000000  0.026316  0.013514  0.040541  0.000000  0.038462   \n",
       "CNN      0.200000  0.026316  1.000000  0.250000  0.400000  0.005464  0.375000   \n",
       "ET       0.500000  0.013514  0.250000  1.000000  0.333333  0.000000  0.142857   \n",
       "GBT      0.250000  0.040541  0.400000  0.333333  1.000000  0.000000  0.250000   \n",
       "GNB      0.000000  0.000000  0.005464  0.000000  0.000000  1.000000  0.005420   \n",
       "LG       0.125000  0.038462  0.375000  0.142857  0.250000  0.005420  1.000000   \n",
       "LTSM     0.500000  0.013514  0.250000  1.000000  0.333333  0.000000  0.142857   \n",
       "MNB      0.025316  0.040816  0.037500  0.012658  0.012346  0.101990  0.061728   \n",
       "RF       0.500000  0.013514  0.250000  1.000000  0.333333  0.000000  0.142857   \n",
       "SVM      0.500000  0.013514  0.250000  1.000000  0.333333  0.000000  0.142857   \n",
       "XGB      0.090909  0.012048  0.166667  0.100000  0.083333  0.010811  0.214286   \n",
       "\n",
       "method2      LTSM       MNB        RF       SVM       XGB  \n",
       "method1                                                    \n",
       "ADA      0.500000  0.025316  0.500000  0.500000  0.090909  \n",
       "BNB      0.013514  0.040816  0.013514  0.013514  0.012048  \n",
       "CNN      0.250000  0.037500  0.250000  0.250000  0.166667  \n",
       "ET       1.000000  0.012658  1.000000  1.000000  0.100000  \n",
       "GBT      0.333333  0.012346  0.333333  0.333333  0.083333  \n",
       "GNB      0.000000  0.101990  0.000000  0.000000  0.010811  \n",
       "LG       0.142857  0.061728  0.142857  0.142857  0.214286  \n",
       "LTSM     1.000000  0.012658  1.000000  1.000000  0.100000  \n",
       "MNB      0.012658  1.000000  0.012658  0.012658  0.112500  \n",
       "RF       1.000000  0.012658  1.000000  1.000000  0.100000  \n",
       "SVM      1.000000  0.012658  1.000000  1.000000  0.100000  \n",
       "XGB      0.100000  0.112500  0.100000  0.100000  1.000000  "
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#put into a matrix\n",
    "d = list(zip(method1,method2,jac))\n",
    "df_j = pd.DataFrame(data=d,columns=('method1','method2','jac'))\n",
    "table = pd.pivot_table(df_j,values='jac',index='method1',columns='method2',aggfunc=np.sum)\n",
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1411ff02",
   "metadata": {},
   "source": [
    "### 3 Approach Voting Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "ea4f9e6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vote_prediction(method1,method2,method3):\n",
    "    pred = []\n",
    "    for i in range(len(method1)):\n",
    "        pred.append(round((method1[i]+method2[i]+method3[i])/3))\n",
    "    return pred;"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1ac92a3",
   "metadata": {},
   "source": [
    "### Ensemble Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "1f8adb3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "en1 = vote_prediction(rft2,ett,gbt2)\n",
    "en2 = vote_prediction(gbt2,ett2,ett)\n",
    "en3 = vote_prediction(ett,ett2,mbt)\n",
    "en4 = vote_prediction(xgt2,gbt2,mbt)\n",
    "en5 = vote_prediction(adt2,lrt,mbt)\n",
    "en6 = vote_prediction(rft2,gbt2,mbt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "77b66e90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LG</td>\n",
       "      <td>0.972534</td>\n",
       "      <td>0.918089</td>\n",
       "      <td>0.851266</td>\n",
       "      <td>0.883415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GNB</td>\n",
       "      <td>0.809671</td>\n",
       "      <td>0.382038</td>\n",
       "      <td>0.901899</td>\n",
       "      <td>0.536723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BNB</td>\n",
       "      <td>0.961702</td>\n",
       "      <td>0.990950</td>\n",
       "      <td>0.693038</td>\n",
       "      <td>0.815642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MNB</td>\n",
       "      <td>0.950484</td>\n",
       "      <td>0.737374</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>0.820225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SVM</td>\n",
       "      <td>0.966344</td>\n",
       "      <td>0.987234</td>\n",
       "      <td>0.734177</td>\n",
       "      <td>0.842105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RF</td>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.972028</td>\n",
       "      <td>0.879747</td>\n",
       "      <td>0.923588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>GBT</td>\n",
       "      <td>0.980658</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>0.863924</td>\n",
       "      <td>0.916107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ET</td>\n",
       "      <td>0.979497</td>\n",
       "      <td>0.974729</td>\n",
       "      <td>0.854430</td>\n",
       "      <td>0.910624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ADA</td>\n",
       "      <td>0.975629</td>\n",
       "      <td>0.950178</td>\n",
       "      <td>0.844937</td>\n",
       "      <td>0.894472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>XGB</td>\n",
       "      <td>0.969052</td>\n",
       "      <td>0.878205</td>\n",
       "      <td>0.867089</td>\n",
       "      <td>0.872611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>CNN</td>\n",
       "      <td>0.974468</td>\n",
       "      <td>0.913907</td>\n",
       "      <td>0.873418</td>\n",
       "      <td>0.893204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LTSM</td>\n",
       "      <td>0.975629</td>\n",
       "      <td>0.953405</td>\n",
       "      <td>0.841772</td>\n",
       "      <td>0.894118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>En1</td>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.985612</td>\n",
       "      <td>0.867089</td>\n",
       "      <td>0.922559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>En2</td>\n",
       "      <td>0.979884</td>\n",
       "      <td>0.992537</td>\n",
       "      <td>0.841772</td>\n",
       "      <td>0.910959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>En3</td>\n",
       "      <td>0.980658</td>\n",
       "      <td>0.978417</td>\n",
       "      <td>0.860759</td>\n",
       "      <td>0.915825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>En4</td>\n",
       "      <td>0.976402</td>\n",
       "      <td>0.920792</td>\n",
       "      <td>0.882911</td>\n",
       "      <td>0.901454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>En5</td>\n",
       "      <td>0.979110</td>\n",
       "      <td>0.936667</td>\n",
       "      <td>0.889241</td>\n",
       "      <td>0.912338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>En6</td>\n",
       "      <td>0.982205</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.886076</td>\n",
       "      <td>0.924092</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Method  Accuracy  Precision    Recall        F1\n",
       "0      LG  0.972534   0.918089  0.851266  0.883415\n",
       "1     GNB  0.809671   0.382038  0.901899  0.536723\n",
       "2     BNB  0.961702   0.990950  0.693038  0.815642\n",
       "3     MNB  0.950484   0.737374  0.924051  0.820225\n",
       "4     SVM  0.966344   0.987234  0.734177  0.842105\n",
       "5      RF  0.982205   0.972028  0.879747  0.923588\n",
       "6     GBT  0.980658   0.975000  0.863924  0.916107\n",
       "7      ET  0.979497   0.974729  0.854430  0.910624\n",
       "8     ADA  0.975629   0.950178  0.844937  0.894472\n",
       "9     XGB  0.969052   0.878205  0.867089  0.872611\n",
       "10    CNN  0.974468   0.913907  0.873418  0.893204\n",
       "11   LTSM  0.975629   0.953405  0.841772  0.894118\n",
       "12    En1  0.982205   0.985612  0.867089  0.922559\n",
       "13    En2  0.979884   0.992537  0.841772  0.910959\n",
       "14    En3  0.980658   0.978417  0.860759  0.915825\n",
       "15    En4  0.976402   0.920792  0.882911  0.901454\n",
       "16    En5  0.979110   0.936667  0.889241  0.912338\n",
       "17    En6  0.982205   0.965517  0.886076  0.924092"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "methods_final = [lrt, gnt, bnt, mbt, sbt2, rft2, gbt2, ett2, adt2, xgt2, cnt, ltt, en1,en2,en3,en4,en5,en6]\n",
    "method_name_final = ['LG','GNB','BNB','MNB','SVM','RF','GBT','ET','ADA','XGB','CNN','LTSM','En1','En2','En3','En4','En5',\"En6\"]\n",
    "accuracy3 = []\n",
    "precision3 = []\n",
    "recall3 = []\n",
    "f13 = []\n",
    "for i in methods_final:\n",
    "    accuracy3.append(class_accuracy(i))\n",
    "    precision3.append(class_precision(i))\n",
    "    recall3.append(class_recall(i))\n",
    "    f13.append(class_f1(i))\n",
    "results3 = pd.DataFrame(\n",
    "    {'Method': method_name_final,\n",
    "     'Accuracy': accuracy3,\n",
    "     'Precision': precision3,\n",
    "     'Recall':recall3,\n",
    "     'F1':f13\n",
    "    })\n",
    "results3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e2089f8",
   "metadata": {},
   "source": [
    "### Testing False Positive Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "17472b65",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ET_train_predictions = etm.predict(df_train).tolist()\n",
    "# ADA_train_predictions = adm.predict(df_train).tolist()\n",
    "# actual = y_train.tolist()\n",
    "# results = list(zip(actual,ET_train_predictions,ADA_train_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "7b5ef848",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_results = pd.DataFrame(data=results,columns=['actual','ET','ADA'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "0f641964",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_results.to_excel(\"FP.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "25838076",
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict_metrics(ET_train_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "7bc10b44",
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict_metrics(ADA_train_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "d21f483b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.metrics import confusion_matrix\n",
    "# tn, fp, fn, tp = confusion_matrix(actual, ADA_train_predictions, labels=[0, 1]).ravel()\n",
    "# print(tn, fp, fn, tp)  # 1 1 1 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "9e74e031",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tn, fp, fn, tp = confusion_matrix(actual, ET_train_predictions, labels=[0, 1]).ravel()\n",
    "# print(tn, fp, fn, tp)  # 1 1 1 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "b4ae4ee7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# actual[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "425e249d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_results['ADA_FP'] = np.where(df_results['ADA'] == 0 & df_results['actual'] == 1,1,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "95bbb848",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# sum(df_results.ADA_FP.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "d9ad592b",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [0,0,0,0,1,0,0,1,0,0]\n",
    "b = [0,1,0,0,0,0,0,1,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "74e9aa02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3333333333333333"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jaccard_score(a,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "id": "131da283",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = [0,1,0,0,0,0,0,0,0,0]\n",
    "d = [0,0,0,0,0,0,0,1,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "id": "a231b484",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jaccard_score(c,d)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "085aba74",
   "metadata": {},
   "source": [
    "### Pickle Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "35a90789",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'srt' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[195], line 10\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;66;03m#pickle predictions\u001b[39;00m\n\u001b[0;32m      2\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124msession.pkl\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mwb\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m      3\u001b[0m     pickle\u001b[38;5;241m.\u001b[39mdump({\n\u001b[0;32m      4\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcnt\u001b[39m\u001b[38;5;124m'\u001b[39m : cnt,\n\u001b[0;32m      5\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mltt\u001b[39m\u001b[38;5;124m'\u001b[39m : ltt,\n\u001b[0;32m      6\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlrt\u001b[39m\u001b[38;5;124m'\u001b[39m : lrt,\n\u001b[0;32m      7\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgnt\u001b[39m\u001b[38;5;124m'\u001b[39m : gnt,\n\u001b[0;32m      8\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbnt\u001b[39m\u001b[38;5;124m'\u001b[39m : bnt,\n\u001b[0;32m      9\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmbt\u001b[39m\u001b[38;5;124m'\u001b[39m : mbt,\n\u001b[1;32m---> 10\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msrt\u001b[39m\u001b[38;5;124m'\u001b[39m : \u001b[43msrt\u001b[49m,\n\u001b[0;32m     11\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mspt\u001b[39m\u001b[38;5;124m'\u001b[39m : spt,\n\u001b[0;32m     12\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mslt\u001b[39m\u001b[38;5;124m'\u001b[39m : slt,\n\u001b[0;32m     13\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrft\u001b[39m\u001b[38;5;124m'\u001b[39m : rft,\n\u001b[0;32m     14\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgbt\u001b[39m\u001b[38;5;124m'\u001b[39m : gbt,\n\u001b[0;32m     15\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mett\u001b[39m\u001b[38;5;124m'\u001b[39m : ett,\n\u001b[0;32m     16\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124madt\u001b[39m\u001b[38;5;124m'\u001b[39m : adt,\n\u001b[0;32m     17\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mxgt\u001b[39m\u001b[38;5;124m'\u001b[39m : xgt,\n\u001b[0;32m     18\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mxrt\u001b[39m\u001b[38;5;124m'\u001b[39m : xrt,\n\u001b[0;32m     19\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcnr\u001b[39m\u001b[38;5;124m'\u001b[39m : cnr,\n\u001b[0;32m     20\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mltr\u001b[39m\u001b[38;5;124m'\u001b[39m : ltr,\n\u001b[0;32m     21\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlrr\u001b[39m\u001b[38;5;124m'\u001b[39m : lrr,\n\u001b[0;32m     22\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgnr\u001b[39m\u001b[38;5;124m'\u001b[39m : gnr,\n\u001b[0;32m     23\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbnr\u001b[39m\u001b[38;5;124m'\u001b[39m : bnr,\n\u001b[0;32m     24\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmbr\u001b[39m\u001b[38;5;124m'\u001b[39m : mbr,\n\u001b[0;32m     25\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msrr\u001b[39m\u001b[38;5;124m'\u001b[39m : srr,\n\u001b[0;32m     26\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mspr\u001b[39m\u001b[38;5;124m'\u001b[39m : spr,\n\u001b[0;32m     27\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mslr\u001b[39m\u001b[38;5;124m'\u001b[39m : slr,\n\u001b[0;32m     28\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrfr\u001b[39m\u001b[38;5;124m'\u001b[39m : rfr,\n\u001b[0;32m     29\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mgbr\u001b[39m\u001b[38;5;124m'\u001b[39m : gbr,\n\u001b[0;32m     30\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124metr\u001b[39m\u001b[38;5;124m'\u001b[39m : etr,\n\u001b[0;32m     31\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124madr\u001b[39m\u001b[38;5;124m'\u001b[39m : adr,\n\u001b[0;32m     32\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mxgr\u001b[39m\u001b[38;5;124m'\u001b[39m : xgr,\n\u001b[0;32m     33\u001b[0m             \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mxrr\u001b[39m\u001b[38;5;124m'\u001b[39m : xrr\n\u001b[0;32m     34\u001b[0m                 }, f)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'srt' is not defined"
     ]
    }
   ],
   "source": [
    "#pickle predictions\n",
    "with open('session.pkl', 'wb') as f:\n",
    "    pickle.dump({\n",
    "            'cnt' : cnt,\n",
    "            'ltt' : ltt,\n",
    "            'lrt' : lrt,\n",
    "            'gnt' : gnt,\n",
    "            'bnt' : bnt,\n",
    "            'mbt' : mbt,\n",
    "            'srt' : srt,\n",
    "            'spt' : spt,\n",
    "            'slt' : slt,\n",
    "            'rft' : rft,\n",
    "            'gbt' : gbt,\n",
    "            'ett' : ett,\n",
    "            'adt' : adt,\n",
    "            'xgt' : xgt,\n",
    "            'xrt' : xrt,\n",
    "            'cnr' : cnr,\n",
    "            'ltr' : ltr,\n",
    "            'lrr' : lrr,\n",
    "            'gnr' : gnr,\n",
    "            'bnr' : bnr,\n",
    "            'mbr' : mbr,\n",
    "            'srr' : srr,\n",
    "            'spr' : spr,\n",
    "            'slr' : slr,\n",
    "            'rfr' : rfr,\n",
    "            'gbr' : gbr,\n",
    "            'etr' : etr,\n",
    "            'adr' : adr,\n",
    "            'xgr' : xgr,\n",
    "            'xrr' : xrr\n",
    "                }, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "eacecd4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "\n",
    "import transformers\n",
    "from tqdm.notebook import tqdm\n",
    "from tokenizers import BertWordPieceTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "id": "3ed9ebc0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1d613be23474eed99d4e363270a7fa4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\huggingface_hub\\file_download.py:123: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\CGLam\\.cache\\huggingface\\hub. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ae2d0421d1040c482038c716bf104e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e168e272512246aa96fd3504951cf3c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "c9be75f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bert_encode(input_text, max_len):\n",
    "    input_ids = []\n",
    "    attension_masks = []\n",
    "    for text in input_text:\n",
    "        output_dict = tokenizer.encode_plus(\n",
    "            text, \n",
    "            add_special_tokens = True,\n",
    "            truncation=True,\n",
    "            max_length = max_len,\n",
    "            pad_to_max_length = True,\n",
    "            return_attention_mask = True\n",
    "        )\n",
    "        input_ids.append(output_dict['input_ids'])\n",
    "        attension_masks.append(output_dict['attention_mask'])\n",
    "    return np.array(input_ids), np.array(attension_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "a7751800",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\transformers\\tokenization_utils_base.py:2304: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "text = s_pandas['doc']\n",
    "target = s_pandas['label']\n",
    "train_input_ids, train_attention_masks = bert_encode(text, 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "d299c9ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(bert_model):\n",
    "    input_ids = tf.keras.Input(shape= (60,), dtype= 'int32')\n",
    "    attention_masks = tf.keras.Input(shape= (60,), dtype= 'int32')\n",
    "    \n",
    "    output = bert_model([input_ids, attention_masks])\n",
    "    output = output[1]\n",
    "    output = tf.keras.layers.Dense(32, activation= 'relu')(output)\n",
    "    output = tf.keras.layers.Dropout(0.2)(output)\n",
    "    output = tf.keras.layers.Dense(1, activation= 'sigmoid')(output)\n",
    "    \n",
    "    model = tf.keras.models.Model(inputs= [input_ids, attention_masks], outputs= output)\n",
    "    model.compile(Adam(lr=1e-5), loss= 'binary_crossentropy', metrics= ['f1'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "id": "331fcb05",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at bert-base-uncased were not used when initializing TFBertModel: ['mlm___cls', 'nsp___cls']\n",
      "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFBertModel were initialized from the model checkpoint at bert-base-uncased.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import TFBertModel\n",
    "bert_model = TFBertModel.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "id": "e9e4e3fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_3\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_7 (InputLayer)           [(None, 60)]         0           []                               \n",
      "                                                                                                  \n",
      " input_8 (InputLayer)           [(None, 60)]         0           []                               \n",
      "                                                                                                  \n",
      " tf_bert_model_2 (TFBertModel)  TFBaseModelOutputWi  109482240   ['input_7[0][0]',                \n",
      "                                thPoolingAndCrossAt               'input_8[0][0]']                \n",
      "                                tentions(last_hidde                                               \n",
      "                                n_state=(None, 60,                                                \n",
      "                                768),                                                             \n",
      "                                 pooler_output=(Non                                               \n",
      "                                e, 768),                                                          \n",
      "                                 past_key_values=No                                               \n",
      "                                ne, hidden_states=N                                               \n",
      "                                one, attentions=Non                                               \n",
      "                                e, cross_attentions                                               \n",
      "                                =None)                                                            \n",
      "                                                                                                  \n",
      " dense_12 (Dense)               (None, 32)           24608       ['tf_bert_model_2[0][1]']        \n",
      "                                                                                                  \n",
      " dropout_114 (Dropout)          (None, 32)           0           ['dense_12[0][0]']               \n",
      "                                                                                                  \n",
      " dense_13 (Dense)               (None, 1)            33          ['dropout_114[0][0]']            \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 109,506,881\n",
      "Trainable params: 109,506,881\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = create_model(bert_model)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "id": "c148d343",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "in user code:\n\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1160, in train_function  *\n        return step_function(self, iterator)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1146, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1135, in run_step  **\n        outputs = model.train_step(data)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 998, in train_step\n        return self.compute_metrics(x, y, y_pred, sample_weight)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1092, in compute_metrics\n        self.compiled_metrics.update_state(y, y_pred, sample_weight)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 577, in update_state\n        self.build(y_pred, y_true)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 483, in build\n        self._metrics = tf.__internal__.nest.map_structure_up_to(\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 631, in _get_metric_objects\n        return [self._get_metric_object(m, y_t, y_p) for m in metrics]\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 631, in <listcomp>\n        return [self._get_metric_object(m, y_t, y_p) for m in metrics]\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 650, in _get_metric_object\n        metric_obj = metrics_mod.get(metric)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\metrics\\__init__.py\", line 181, in get\n        return deserialize(str(identifier))\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\metrics\\__init__.py\", line 136, in deserialize\n        return deserialize_keras_object(\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\utils\\generic_utils.py\", line 769, in deserialize_keras_object\n        raise ValueError(\n\n    ValueError: Unknown metric function: f1. Please ensure this object is passed to the `custom_objects` argument. See https://www.tensorflow.org/guide/keras/save_and_serialize#registering_the_custom_object for details.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "File \u001b[1;32m<timed exec>:1\u001b[0m\n",
      "File \u001b[1;32m~\\anaconda3_\\lib\\site-packages\\keras\\utils\\traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n\u001b[0;32m     68\u001b[0m     \u001b[38;5;66;03m# To get the full stack trace, call:\u001b[39;00m\n\u001b[0;32m     69\u001b[0m     \u001b[38;5;66;03m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[1;32m---> 70\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m e\u001b[38;5;241m.\u001b[39mwith_traceback(filtered_tb) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[0;32m     71\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[0;32m     72\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[1;32m~\\AppData\\Local\\Temp\\__autograph_generated_file_4k9wrn8.py:15\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__train_function\u001b[1;34m(iterator)\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m     14\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m---> 15\u001b[0m     retval_ \u001b[38;5;241m=\u001b[39m ag__\u001b[38;5;241m.\u001b[39mconverted_call(ag__\u001b[38;5;241m.\u001b[39mld(step_function), (ag__\u001b[38;5;241m.\u001b[39mld(\u001b[38;5;28mself\u001b[39m), ag__\u001b[38;5;241m.\u001b[39mld(iterator)), \u001b[38;5;28;01mNone\u001b[39;00m, fscope)\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[0;32m     17\u001b[0m     do_return \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
      "\u001b[1;31mValueError\u001b[0m: in user code:\n\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1160, in train_function  *\n        return step_function(self, iterator)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1146, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1135, in run_step  **\n        outputs = model.train_step(data)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 998, in train_step\n        return self.compute_metrics(x, y, y_pred, sample_weight)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\training.py\", line 1092, in compute_metrics\n        self.compiled_metrics.update_state(y, y_pred, sample_weight)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 577, in update_state\n        self.build(y_pred, y_true)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 483, in build\n        self._metrics = tf.__internal__.nest.map_structure_up_to(\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 631, in _get_metric_objects\n        return [self._get_metric_object(m, y_t, y_p) for m in metrics]\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 631, in <listcomp>\n        return [self._get_metric_object(m, y_t, y_p) for m in metrics]\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\engine\\compile_utils.py\", line 650, in _get_metric_object\n        metric_obj = metrics_mod.get(metric)\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\metrics\\__init__.py\", line 181, in get\n        return deserialize(str(identifier))\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\metrics\\__init__.py\", line 136, in deserialize\n        return deserialize_keras_object(\n    File \"C:\\Users\\CGLam\\anaconda3_\\lib\\site-packages\\keras\\utils\\generic_utils.py\", line 769, in deserialize_keras_object\n        raise ValueError(\n\n    ValueError: Unknown metric function: f1. Please ensure this object is passed to the `custom_objects` argument. See https://www.tensorflow.org/guide/keras/save_and_serialize#registering_the_custom_object for details.\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "history = model.fit(\n",
    "    [train_input_ids, train_attention_masks],\n",
    "    target, \n",
    "    validation_split = 0.5,\n",
    "    epochs = 3,\n",
    "    batch_size = 10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36d2db34",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}