olidTask_snlp(ML_models).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install vaderSentiment\n",
    "!pip install textstat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2s5c91PsEQCA",
    "outputId": "61b36f88-16df-4a3f-c411-d415792e5195"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\stewi_000\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\stewi_000\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\stewi_000\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     C:\\Users\\stewi_000\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "import numpy as np\n",
    "import sys\n",
    "import re\n",
    "import os\n",
    "from collections import Counter\n",
    "from textstat.textstat import *\n",
    "\n",
    "\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('vader_lexicon')\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import *\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "\n",
    "import sklearn as sk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV\n",
    "#from sklearn.pipeline import Pipeline\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jEKsBtiR1x3u"
   },
   "source": [
    "-------------------------------------------------------\n",
    "# **1. Loading Data**\n",
    "-------------------------------------------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "r4xWKf33fTzx"
   },
   "outputs": [],
   "source": [
    "dfs = pd.read_csv('C:/Users/stewi_000/Desktop/UNI/Computerlinguistik/SS 20/SNLP/SNLP-OLID Project/OLIDv1.0/olid-training-v1.0.tsv', sep='\\t', encoding = 'utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IjetJ4GkLaWR"
   },
   "source": [
    "# **2. Preprocessing**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4A226rUbLERs"
   },
   "source": [
    "## **2.1 Automatic Sentiment Analyses of Tweets**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 407
    },
    "id": "M2L6cqQAK_wS",
    "outputId": "df6e3b1a-9173-4864-9353-62fa7d0cc2c3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet</th>\n",
       "      <th>subtask_a</th>\n",
       "      <th>subtask_b</th>\n",
       "      <th>subtask_c</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>compound</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>86426</td>\n",
       "      <td>@USER She should ask a few native Americans wh...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>UNT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>90194</td>\n",
       "      <td>@USER @USER Go home you’re drunk!!! @USER #MAG...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>TIN</td>\n",
       "      <td>IND</td>\n",
       "      <td>0.247</td>\n",
       "      <td>0.753</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.5067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16820</td>\n",
       "      <td>Amazon is investigating Chinese employees who ...</td>\n",
       "      <td>NOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.3400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>62688</td>\n",
       "      <td>@USER Someone should'veTaken\" this piece of sh...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>UNT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.286</td>\n",
       "      <td>0.714</td>\n",
       "      <td>0.000</td>\n",
       "      <td>-0.5574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>43605</td>\n",
       "      <td>@USER @USER Obama wanted liberals &amp;amp; illega...</td>\n",
       "      <td>NOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13235</th>\n",
       "      <td>95338</td>\n",
       "      <td>@USER Sometimes I get strong vibes from people...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>TIN</td>\n",
       "      <td>IND</td>\n",
       "      <td>0.241</td>\n",
       "      <td>0.652</td>\n",
       "      <td>0.108</td>\n",
       "      <td>-0.6240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13236</th>\n",
       "      <td>67210</td>\n",
       "      <td>Benidorm ✅  Creamfields ✅  Maga ✅   Not too sh...</td>\n",
       "      <td>NOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13237</th>\n",
       "      <td>82921</td>\n",
       "      <td>@USER And why report this garbage.  We don't g...</td>\n",
       "      <td>OFF</td>\n",
       "      <td>TIN</td>\n",
       "      <td>OTH</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.805</td>\n",
       "      <td>0.195</td>\n",
       "      <td>0.2924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13238</th>\n",
       "      <td>27429</td>\n",
       "      <td>@USER Pussy</td>\n",
       "      <td>OFF</td>\n",
       "      <td>UNT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13239</th>\n",
       "      <td>46552</td>\n",
       "      <td>#Spanishrevenge vs. #justice #HumanRights and ...</td>\n",
       "      <td>NOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13240 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                              tweet subtask_a  \\\n",
       "0      86426  @USER She should ask a few native Americans wh...       OFF   \n",
       "1      90194  @USER @USER Go home you’re drunk!!! @USER #MAG...       OFF   \n",
       "2      16820  Amazon is investigating Chinese employees who ...       NOT   \n",
       "3      62688  @USER Someone should'veTaken\" this piece of sh...       OFF   \n",
       "4      43605  @USER @USER Obama wanted liberals &amp; illega...       NOT   \n",
       "...      ...                                                ...       ...   \n",
       "13235  95338  @USER Sometimes I get strong vibes from people...       OFF   \n",
       "13236  67210  Benidorm ✅  Creamfields ✅  Maga ✅   Not too sh...       NOT   \n",
       "13237  82921  @USER And why report this garbage.  We don't g...       OFF   \n",
       "13238  27429                                        @USER Pussy       OFF   \n",
       "13239  46552  #Spanishrevenge vs. #justice #HumanRights and ...       NOT   \n",
       "\n",
       "      subtask_b subtask_c    neg    neu    pos  compound  \n",
       "0           UNT       NaN  0.000  1.000  0.000    0.0000  \n",
       "1           TIN       IND  0.247  0.753  0.000   -0.5067  \n",
       "2           NaN       NaN  0.000  0.880  0.120    0.3400  \n",
       "3           UNT       NaN  0.286  0.714  0.000   -0.5574  \n",
       "4           NaN       NaN  0.000  1.000  0.000    0.0000  \n",
       "...         ...       ...    ...    ...    ...       ...  \n",
       "13235       TIN       IND  0.241  0.652  0.108   -0.6240  \n",
       "13236       NaN       NaN  0.000  1.000  0.000    0.0000  \n",
       "13237       TIN       OTH  0.000  0.805  0.195    0.2924  \n",
       "13238       UNT       NaN  0.000  1.000  0.000    0.0000  \n",
       "13239       NaN       NaN  0.000  1.000  0.000    0.0000  \n",
       "\n",
       "[13240 rows x 9 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyzer = SentimentIntensityAnalyzer()\n",
    "\n",
    "dfs['neg'] = dfs['tweet'].apply(lambda x:analyzer.polarity_scores(x)['neg'])\n",
    "dfs['neu'] = dfs['tweet'].apply(lambda x:analyzer.polarity_scores(x)['neu'])\n",
    "dfs['pos'] = dfs['tweet'].apply(lambda x:analyzer.polarity_scores(x)['pos'])\n",
    "dfs['compound'] = dfs['tweet'].apply(lambda x:analyzer.polarity_scores(x)['compound'])\n",
    "dfs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y-QcwvnYVYOl"
   },
   "source": [
    "## **2.2 Tokens Distribution**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "hZZpC8US68X1"
   },
   "outputs": [],
   "source": [
    "all_tweets =dfs[\"tweet\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "d_xINQsb82yT"
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "num_token_inSentence = []\n",
    "for sent in all_tweets:\n",
    "  count = count+1\n",
    "  word_list = len(sent.split())\n",
    "  num_token_inSentence.append(word_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kJ-UHL09-eLx",
    "outputId": "beadd7b9-dc41-4f96-c5c9-07be2431aec9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22.307703927492447\n"
     ]
    }
   ],
   "source": [
    "avg_tokens =sum(num_token_inSentence)/count\n",
    "print(avg_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "SCHbQVvQAFOy"
   },
   "outputs": [],
   "source": [
    "dfs[\"tokens\"] = dfs[\"tweet\"].map(lambda text: re.sub(r\"[^a-zA-Z0-9' ]\", ' ', text)).map(lambda text: text.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "KeIb6eJrAzLj"
   },
   "outputs": [],
   "source": [
    "num = dfs.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 225
    },
    "id": "M4BBQwsjAPpQ",
    "outputId": "ba60fe05-e701-4063-83d7-c8a5d6dc18e6"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAADQCAYAAAA+nmWYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVoElEQVR4nO3de5RdZXnH8e+Pq5DINZDGgA10UWlsJFwKWmybyEWCLoEWNKkEFCxojYArSkG7XHHxh5GLoNWiglxUIKJAieBC0phoXVUIoZEAIYIQkRATQCQMVOrA0z/2e8jmzJmZMydnz7sn8/usddac8+7LefaE87D3nvM+jyICM7PhtlXuAMxsdHLyMbMsnHzMLAsnHzPLwsnHzLJw8jGzLLbJHcBwGDduXEyaNKnP+AsvvMCYMWOGP6A21Tk+x9a5OsfX7diWL1/+dETs0XJhRGzxj4MPPjhaWbJkScvxuqhzfI6tc3WOr9uxAfdEP59LX3aZWRZOPmaWhZOPmWXh5GNmWTj5mFkWlf6pXdIxwBeBrYErI2J+0/L9gauBg4BPR8TFafxNwHdKq+4LfCYiLpM0D/gn4Km07FMR8YMqj6PZpPNub2u9NfPfVXEkZiNXZclH0tbAV4CjgCeAZZIWRsSDpdV+B5wFHF/eNiJWA1NL+1kL3FJa5dJGojKzkanKy65DgUci4tGI+D9gAXBceYWI2BARy4A/DrCfI4BfRcSvqwvVzIZblclnIvCb0usn0thQzQRuaBqbI+k+SVdJ2rXTAM0sH0VFlQwlnQS8MyI+lF7PBg6NiI+1WHce0NN8KSVpO+BJ4M0RsT6NjQeeBgK4AJgQEae12OcZwBkA48ePP3jBggV9Yuzp6WHs2LFDPraVa59ra70pE3ce8r7LOo1vODi2ztU5vm7HNn369OURcUirZVXecH4C2Lv0ei+KRDIUM4B7G4kHoPxc0hXAba02jIivA18HOOSQQ2LatGl91lm6dCmtxgfzgXZvOL9/6Psu6zS+4eDYOlfn+IYztiovu5YB+0naJ53BzAQWDnEfs2i65JI0ofTyBOD+zYrSzLKo7MwnInolzQF+SPGn9qsi4gFJH07LvyrpT4B7gJ2AVySdA0yOiI2SdqT4S9mZTbu+UNJUisuuNS2Wm9kIUOn3fNL3b37QNPbV0vPfUlyOtdr2RWD3FuOzuxymmWXgbzibWRZOPmaWhZOPmWXh5GNmWTj5mFkWTj5mloWTj5ll4eRjZlk4+ZhZFqOiaWC72q1QaGabz2c+ZpaFk4+ZZeHkY2ZZVJp8JB0jabWkRySd12L5/pJ+JuklSZ9oWrZG0kpJKyTdUxrfTdIiSQ+nny6jajYCVZZ8St0rZgCTgVmSJjet1uhe0V8niukRMbWpDON5wOKI2A9YnF6b2QgzErpXNDsOuDY9v5amtjtmNjJUWUD+ROCYpgLyh0XEnBbrzqOpgLykx4BnKSoWfi3VZEbS7yNil9J6z0ZEn0uvTgrIt1sYvl0uIJ9HnWODese3pRSQV4uxoWS6wyPiSUl7AoskPRQRP2l3404KyLdbGL5dLiCfR51jg3rHt6UUkN+s7hUR8WT6uYGiW+mhadH6RhH59HNDV6I1s2FVy+4VksZIen3jOXA0m7pULAROTc9PBW7tatRmNixq2b0CGAfcIqkR4/URcUfa9XzgRkmnA48DJ1V1DGZWnbp2r9gIHNDPPp+h6N9uZiOYJ5ZWaCgTVdfMf1eFkZjVj6dXmFkWTj5mloWTj5ll4eRjZlk4+ZhZFk4+ZpaFk4+ZZeHkY2ZZOPmYWRZOPmaWhZOPmWXh5GNmWdSye4WkvSUtkbRK0gOSzi4tmydpbepqsULSsVUeg5lVo7JZ7aXuFUdRVDVcJmlhRDxYWq3RveL4ps17gbkRcW8qKrZc0qLStpeW6z2b2chTy+4VEbEuIu5Nz58HVgETK4zVzIZZlfV8JgK/Kb1+AjhsqDuRNAk4ELirNDxH0ikUVRDnRsSzLbYrd69g6dKlffbd09PzmvG5U3qHGl7XtBNfnTi2ztU5vuGMrc7dK5A0FrgJOCciNqbhy4EL0r4uAC4BTuvzRjXoXjEUrTpduMtBZ+ocG9Q7PnevACRtS5F4rouImxvjEbE+Il6OiFeAK9jU1cLMRpC6dq8Q8A1gVUR8oWnZhNLLE9jU1cLMRpC6dq94CzAbWClpRdrlp1JB+gslTaW47FoDnFnVMZhZderaveKntL5nRETM7maMZpaHv+FsZlk4+ZhZFk4+ZpaFmwbWRKsGg3On9Pb57pGbC9qWwmc+ZpaFk4+ZZeHkY2ZZOPmYWRZOPmaWhZOPmWXh5GNmWbSVfCTtJekWSU9JWi/pJkmt5mSZmbWl3TOfqynKYUygqFD4/TQ2oE4LyA+0raTdJC2S9HD6uWubx2BmNdJu8tkjIq6OiN70uAbYY6ANSgXkZ1CUyZglaXLTao0C8hcPYdvzgMURsR+wOL02sxGm3eTztKSTJW2dHicDzwyyTccF5AfZ9jjg2vT8Wvp2vjCzEaDd5HMa8F7gt8A64ERa1E1u0qqAfLsdKAbadnxErIOiywWwZ5v7NLMaaWtiaUQ8DrxniPvenALy3Sg+P6K6V7Qyfoe+MdWl64E7MHSuzvHVpnuFpHMj4kJJ/0aLD39EnDXA5ptTQH6gbddLmhAR61I95w2tdjDSule0MndKL5esfO0/UasuFzm4A0Pn6hzfcMY22JnPqvTzng72/WoBeWAtRQH5f+zCtguBU4H56eetHcRmZpkNmHwi4vvp6YsR8d3yMkknDbJtxwXkI2Jjq23TrucDN0o6HXgcGDAOM6undouJnQ98t42x19iMAvItt03jzwBHtBW1mdXWYPd8ZgDHAhMlfam0aCegXndnzWxEGezM50mKy6L3AMtL488DH68qKDPb8g12z+cXwC8kXR8RzV8ENDPrWLv3fCZJ+hzFVIfXNQYjYt9KojKzLd5QJpZeTnGfZzrwTeBbVQVlZlu+dpPPDhGxGFBE/Doi5gHvqC4sM9vStXvZ9QdJWwEPp+/frMVzqsxsM7R75nMOsCNF+YuDgdnAKRXFZGajQLsTS5elpz3AByVtA7wPuKuqwKy1Vp1NW3FnU6u7Ac98JO0k6XxJX5Z0tApzgEcoSmyYmXVksDOfbwHPAj8DPgR8EtgOOD4iVlQbmpltyQZLPvtGxBQASVcCTwNvjIjnK4/MzLZog91wfvVbzRHxMvCYE4+ZdcNgyecASRvT43ngLY3nkjYOtvM2uldI0pfS8vskHZTG3yRpRemxMZXbQNI8SWtLy47t4LjNLLPB5nZt3emOSx0ojqKoTLhM0sKIeLC02gxgv/Q4jOJb1IdFxGpgamk/a4FbSttdGhGv6XhhZiNLlR1LB+1ekV5/Mwo/B3ZJpVHLjgB+FRG/rjBWMxtmVSafdrpXtLPOTOCGprE56TLtKjcNNBuZFDGkphDt77gos/rOiPhQej0bODQiPlZa53bgcxHx0/R6MXBuRCxPr7ejqCn05ohYn8bGU/zVLYALgAkR0aeNT1P3ioMXLFjQJ8aenh7Gjh376uuVa5/rwpF3z/gdYP3/drbtlIk7dzeYJs2/uzqpc2xQ7/i6Hdv06dOXR8QhrZa1O7erE+10rxhsnRnAvY3EA1B+LukK4LZWb76ldq9oV9VdLtyBoXN1jq9O3Ss2RzvdKxZSXEItoLjh/FyjIWAyi6ZLrkbbnPTyBOD+KoIf6TwNw+qusuTTTvcKigLxx1JM13gR+GBje0k7Uvyl7MymXV8oaSrFZdeaFsvNbASo8synne4VAXy0n21fBHZvMT67y2GaWQZV/rXLzKxfTj5mlkWll11Wf74xbbn4zMfMsnDyMbMsnHzMLAsnHzPLwsnHzLJw8jGzLJx8zCwLJx8zy8LJx8yycPIxsywqnV4h6RjgixQlNa6MiPlNy5WWH0tRUuMDEXFvWrYGeB54GehtVEOTtBvwHWASRUmN90bEs1Ueh/WdhjF3Sm/L4muehmHtquzMp9S9YgYwGZglaXLTauXuFWdQdK8omx4RU5vKMJ4HLI6I/YDF6bWZjTAjoXtFs+OAa9Pza4HjuxizmQ2TKi+7WnWmOKyNdSYC6ygqFd4pKYCvpZrMAOMbZVQjYp2kPVu9eVMBeZYuXdpnnZ6enteMz53S2+ahDY/xO9Qvpob+Ymv1ex5uzf+udVPn+IYztiqTj1qMNbfKGGidwyPiyZRcFkl6KCJ+0u6bj/YC8lXrN7aVL7S1fZX3hupcoB3qHd9wxlblZddmda+IiMbPDRTdSg9N66xvXJqlnxu6HrmZVa7K5PNq94rUf2smRbeKsoXAKaln+1tJ3SskjZH0egBJY4Cj2dSlYiFwanp+KnBrhcdgZhWpa/eK8cAtxV/i2Qa4PiLuSMvmAzdKOh14HDipqmMws+rUsntFRDwKHNDPPp+h6N9uI1i75VvB3x3aUvkbzmaWhZOPmWXh5GNmWTj5mFkWTj5mloWTj5ll4eRjZlk4+ZhZFk4+ZpaFk4+ZZeHkY2ZZOPmYWRZOPmaWRaXJR9IxklZLekRSn0LvqY7Pl9Ly+yQdlMb3lrRE0ipJD0g6u7TNPElrJa1Ij2OrPAYzq0ZlJTVK3SuOoqhYuEzSwoh4sLRauXvFYRTdKw4DeoG5EXFvKiq2XNKi0raXRsTFVcVuZtWrZfeKiFjX6N8VEc8DqygKy5vZFqLO3SsAkDQJOBC4q7TeHEmnAPdQnCH1aRro7hXVGs7YhtpNoc7dIaDe8bl7RWOhNBa4CTgnIjam4cuBC9J6FwCXAKf12Ym7V1RqOGNb8/5pQ1q/zt0hoN7xDWdsVf7Xs1ndKyRtS5F4rouImxsrRMT6xnNJVwC3dTdsq5t2S6663OrIUtfuFQK+AayKiC+UN2jqaHoCm7pamNkIUtfuFYcDs4GVklaksU+lgvQXSppKcdm1BjizqmMws+rUtXvFT2l9P4iImN3lMM0sA3/D2cyycPIxsyycfMwsCycfM8vCycfMsnDyMbMsnHzMLAsnHzPLwsnHzLJw8jGzLOpZr8GsA43Z73On9A5YHsWz3+vBZz5mlkUtC8gPtK2k3SQtkvRw+rlrlcdgZtWoLPmUCsjPACYDsyRNblqtXED+DIoqhYNtex6wOCL2Axan12Y2wlR5z+fVAvIAkhoF5MvdK14tIA/8XNIuqVjYpAG2PQ6Ylra/FlgK/EuFx2FbGFdGrIe6FpAfaNvxEbEOIFU93LObQZs1tJukhqqbN8RHciKtawH5drYd+M1L3SuAHkmrW6w2Dnh6KPsdTmfVOD7H1rnB4tPnu/+eQ9hnt393f9rfgroWkN9ugG3XN3p7pUu0Da3evNy9oj+S7omIQwY7kFzqHJ9j61yd4xvO2GpZQH6QbRcCp6bnpwK3VngMZlaRWhaQ72/btOv5wI2STgceB06q6hjMrDq1LCDf37Zp/BngiC6FOOBlWQ3UOT7H1rk6xzdssan4/JuZDS9PrzCzLEZt8hls6scwx7K3pCWSVkl6QNLZabw2U0kkbS3pfyTdVsPYdpH0PUkPpd/h2+oSn6SPp3/T+yXdIOl1OWOTdJWkDZLuL431G4+k89NnZLWkd3YzllGZfNqc+jGceoG5EfEXwFuBj6Z46jSV5GxgVel1nWL7InBHROwPHEARZ/b4JE0EzgIOiYi/pPjjyczMsV0DHNM01jKe9N/gTODNaZt/T5+d7oiIUfcA3gb8sPT6fOD83HGV4rkVOApYDUxIYxOA1Zni2Sv9R/kO4LY0VpfYdgIeI92/LI1nj49N39TfjeKPO7cBR+eOjWL60v2D/a6aPxcUf31+W7fiGJVnPvQ/rSM7SZOAA4G7aJpKAuSaSnIZcC7wSmmsLrHtCzwFXJ0uC6+UNKYO8UXEWuBiiq+ErKP4HtuddYitSX/xVPo5Ga3JZ7Onb1RB0ljgJuCciNiYOx4ASe8GNkTE8tyx9GMb4CDg8og4EHiBmlQ6SPdOjgP2Ad4AjJF0ct6ohqTSz8loTT7tTP0YVpK2pUg810XEzWl4fZpCwkBTSSp2OPAeSWuABcA7JH27JrFB8W/5RETclV5/jyIZ1SG+I4HHIuKpiPgjcDPw1zWJray/eCr9nIzW5NPO1I9hI0nAN4BVEfGF0qLsU0ki4vyI2CsiJlH8nn4UESfXIbYU32+B30h6Uxo6gqL0Sh3iexx4q6Qd07/xERQ3w+sQW1l/8SwEZkraXtI+FHW37u7auw73Tbi6PCimdfwS+BXw6cyxvJ3idPY+YEV6HAvsTnGj9+H0c7fMcU5j0w3n2sQGTAXuSb+//wB2rUt8wGeBh4D7gW8B2+eMDbiB4v7THynObE4fKB7g0+kzshqY0c1Y/A1nM8titF52mVlmTj5mloWTj5ll4eRjZlk4+ZhZFk4+o5CkkHRJ6fUnJM3r0r6vkXRiN/Y1yPuclGawLymNTZG0Ij1+J+mx9Pw/+9lHT9VxWv+cfEanl4C/lzQudyBlQ5wxfTrwzxExvTEQESsjYmpETKX4gtwn0+sjuxyqdYGTz+jUS1Eu8+PNC5rPXBpnB5KmSfqxpBsl/VLSfEnvl3S3pJWS/qy0myMl/Vda791p+60lXSRpmYrW2GeW9rtE0vXAyhbxzEr7v18qGsBI+gzFFzO/KumiwQ621T6alo+T9DNJ75K0h6SbUpzLJB2e1pmXauEslfSopLPS+BhJt0v6Rdr/+waLx5Jc30r1I98D6KEoRbEG2Bn4BDAvLbsGOLG8bvo5Dfg9RcmF7YG1wGfTsrOBy0rb30HxP7b9KL5F+zqKHmr/mtbZnuIbyfuk/b4A7NMizjdQTFHYg2IC6Y+A49OypRR1cvo7xmuAEwfZRw8wnqKCwFFp7Hrg7en5GymmvADMA/47xT4OeAbYFvgH4IrS++6c+993pDx85jNKRTFr/psUxa7atSwi1kXESxRfub8zja+kqBHTcGNEvBIRDwOPAvtT1LE5RdIKig/77hTJCeDuiHisxfv9FbA0iomZvcB1wN8OId7B9rEtxXSCcyNiURo7EvhyinMhsJOk16dlt0fESxHxNMXky/Hp2I+U9HlJfxMRzw0xvlHLyWd0u4zi3smY0lgv6b+LNBlyu9Kyl0rPXym9foXXdkJpnrPT6EL7sUj3ZCJinyhq20Bx5tNKq5IOQzXQPnqB5UC5POhWFAWzGnFOjIjn07Ly8b8MbBMRvwQOpkhCn0uXhNYGJ59RLCJ+B9xIkYAa1lB8mKCoRbNtB7s+SdJW6T7QvhSTEn8IfCSVDkHSn6eiXwO5C/i7dE9ma2AW8OMhxjLQPgI4Ddhfm+p43wnMaWwsaepAO5f0BuDFiPg2ReGwg4YY36hVad8uGxEuofRhA64AbpV0N8UlSX9nJQNZTfEBHw98OCL+IOlKikuze9MZ1VPA8QPtJIqW2OcDSyjOYH4QEUMqPzHYPiLiZUkzge9L2khxGfoVSfdRfD5+Anx4gLeYAlwk6RWKmeIfGUp8o5lntZtZFr7sMrMsnHzMLAsnHzPLwsnHzLJw8jGzLJx8zCwLJx8zy8LJx8yy+H+3Y/uzWM/h8wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dfs[\"num_tokens\"] = dfs[\"tokens\"].map(len)\n",
    "ax = dfs[\"num_tokens\"].hist(bins=20, \n",
    "                                 figsize=(4,3),\n",
    "                                 weights=np.ones(num)/num)\n",
    "ax.set_xlabel(\"Number of Tokens\")\n",
    "ax.set_ylabel(\"Ratio\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "orBlmJFsBeic"
   },
   "outputs": [],
   "source": [
    "# Write DataFrame to CSV File with Default params.\n",
    "dfs.to_csv('C:/Users/stewi_000/Desktop/UNI/Computerlinguistik/SS 20/SNLP/SNLP-OLID Project/Automatic_Sentiment_Analyses_of_Tweets.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RDM3ja54LiaE"
   },
   "source": [
    "## **2.3 Creating Dataframe**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ElwYQWxMWPex",
    "outputId": "b38d5e75-710d-48c1-9fe5-54dde217d337"
   },
   "outputs": [],
   "source": [
    "tweets_list = dfs['tweet'].to_list()\n",
    "tweets_to_string = []\n",
    "\n",
    "for elem in tweets_list:\n",
    "    string = ''.join(elem)\n",
    "    tweets_to_string.append(string)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "id": "Ri67RuP9rquT",
    "outputId": "026daba2-afe3-4660-8e4a-0d5423ab5fa9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>neg</th>\n",
       "      <th>neu</th>\n",
       "      <th>pos</th>\n",
       "      <th>compound</th>\n",
       "      <th>num_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>13240.000000</td>\n",
       "      <td>13240.000000</td>\n",
       "      <td>13240.000000</td>\n",
       "      <td>13240.000000</td>\n",
       "      <td>13240.000000</td>\n",
       "      <td>13240.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>54881.766163</td>\n",
       "      <td>0.120217</td>\n",
       "      <td>0.769305</td>\n",
       "      <td>0.110480</td>\n",
       "      <td>-0.038840</td>\n",
       "      <td>22.474924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>26092.681658</td>\n",
       "      <td>0.144467</td>\n",
       "      <td>0.173044</td>\n",
       "      <td>0.139423</td>\n",
       "      <td>0.513623</td>\n",
       "      <td>15.315337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>10006.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.105000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.986000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>32235.750000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.657000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.476700</td>\n",
       "      <td>10.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>54852.000000</td>\n",
       "      <td>0.077000</td>\n",
       "      <td>0.773500</td>\n",
       "      <td>0.065000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>77743.250000</td>\n",
       "      <td>0.202000</td>\n",
       "      <td>0.904000</td>\n",
       "      <td>0.179000</td>\n",
       "      <td>0.401900</td>\n",
       "      <td>32.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>99986.000000</td>\n",
       "      <td>0.885000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.855000</td>\n",
       "      <td>0.986100</td>\n",
       "      <td>105.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id           neg           neu           pos      compound  \\\n",
       "count  13240.000000  13240.000000  13240.000000  13240.000000  13240.000000   \n",
       "mean   54881.766163      0.120217      0.769305      0.110480     -0.038840   \n",
       "std    26092.681658      0.144467      0.173044      0.139423      0.513623   \n",
       "min    10006.000000      0.000000      0.105000      0.000000     -0.986000   \n",
       "25%    32235.750000      0.000000      0.657000      0.000000     -0.476700   \n",
       "50%    54852.000000      0.077000      0.773500      0.065000      0.000000   \n",
       "75%    77743.250000      0.202000      0.904000      0.179000      0.401900   \n",
       "max    99986.000000      0.885000      1.000000      0.855000      0.986100   \n",
       "\n",
       "         num_tokens  \n",
       "count  13240.000000  \n",
       "mean      22.474924  \n",
       "std       15.315337  \n",
       "min        2.000000  \n",
       "25%       10.000000  \n",
       "50%       18.000000  \n",
       "75%       32.000000  \n",
       "max      105.000000  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "qHdG5TlqoBTk"
   },
   "outputs": [],
   "source": [
    "tweets = dfs[\"tweet\"].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Qyk9dkf4IoQa"
   },
   "source": [
    "# **3. Methods**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t7VvlX3C2-Nw"
   },
   "source": [
    "------------------------------------------\n",
    "## **3.1. Feature Engineering**\n",
    "\n",
    "**Vectorization**\n",
    "\n",
    "The following settings for vectorization were used:\n",
    "\n",
    "\n",
    "\n",
    "*   Word representation: TFIDFVectorizer;\n",
    "*   N-grams: unigram & bigram & trigram;\n",
    "*   Convert to lowercase;\n",
    "*   Try binary counter;\n",
    "*   Use stop words;\n",
    "\n",
    "------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vhXuovOJMB5l"
   },
   "source": [
    "### **3.1.1 Creating List with \"Stopwords\"**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oOeTkDgGL2o7",
    "outputId": "55e9df14-fb5a-4914-e71e-beab68639a2b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['user', 'the', 'is', 'to', 'a', 'and', 'you', 'of', 'are', 'i', 'he', 'in', 'that', 'for', 'she', 'it', 'this', 'on', 'not', 'urluser', 'they', 'with', 'have', 'be', 'gun', 'liberals', 'so', 'all', 'your', 'like', 'what', 'control', 'as', 'but', 'antifa', 'we', 'about', 'just', 'was', 'her', 'if', 'will', 'dont', 'do', 'who', 'its', 'conservatives', 'people', 'no', 'his', 'maga', 'at', 'my', 'by', 'from', 'or', 'how', 'has', 'an', 'their', 'out', 'get', 'can', 'up', 'amp', 'know', 'me', 'one', 'why', 'when', 'because', 'more', 'trump', 'think', 'him', 'would', 'now', 'im', 'should', 'them', 'there', 'our', 'good', 'only', 'right', 'thats', 'us', 'want', 'go', 'were', 'than', 'need', 'see', 'going', 'time', 'never', 'url', 'been', 'being', 'these']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#Tokenize tweets in order to count words\n",
    "s = ''.join(tweets_to_string)\n",
    "#get rid of punctuation\n",
    "sentence = re.sub(r'[^\\w\\s]','',s.lower())\n",
    "tokens = nltk.word_tokenize(sentence)\n",
    "\n",
    "#Tokens Distribution (word occurrences)\n",
    "occurrences = Counter(tokens).most_common(100)\n",
    "dictOftokens = dict(occurrences)\n",
    "\n",
    "#Tokens with occurrences more than 1000\n",
    "most_common_tokens = []\n",
    "for k,v in dictOftokens.items():\n",
    "  most_common_tokens.append(k)\n",
    "\n",
    "#Get most common tokens to add them to stopwords\n",
    "print(most_common_tokens) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6IG8sc578Lst"
   },
   "source": [
    "### **3.1.2 Preprocess Methods**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "hoPDgctAP0Jz"
   },
   "outputs": [],
   "source": [
    "#Creating a list with stopwords\n",
    "stopwords=stopwords = nltk.corpus.stopwords.words(\"english\")\n",
    "stopwords.extend(most_common_tokens)\n",
    "stopwords = list(set(stopwords))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "tfElrM0_n3Ai"
   },
   "outputs": [],
   "source": [
    "\n",
    "#Add preprocess Method\n",
    "def preprocess(text_string):\n",
    "    \"\"\"\n",
    "    Accepts a text string and replaces:\n",
    "    1) urls with URLHERE\n",
    "    2) lots of whitespace with one instance\n",
    "    3) mentions with MENTIONHERE\n",
    "\n",
    "    This allows us to get standardized counts of urls and mentions\n",
    "    Without caring about specific people mentioned\n",
    "    \"\"\"\n",
    "    space_pattern = '\\s+'\n",
    "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
    "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
    "    mention_regex = '@[\\w\\-]+'\n",
    "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
    "    parsed_text = re.sub(giant_url_regex, '', parsed_text)\n",
    "    parsed_text = re.sub(mention_regex, '', parsed_text)\n",
    "    return parsed_text\n",
    "\n",
    "#Add method to get rid of Emojis\n",
    "def remove_emoji(string):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                               u\"\\U00002500-\\U00002BEF\"  # chinese char\n",
    "                               u\"\\U00002702-\\U000027B0\"\n",
    "                               u\"\\U00002702-\\U000027B0\"\n",
    "                               u\"\\U000024C2-\\U0001F251\"\n",
    "                               u\"\\U0001f926-\\U0001f937\"\n",
    "                               u\"\\U00010000-\\U0010ffff\"\n",
    "                               u\"\\u2640-\\u2642\"\n",
    "                               u\"\\u2600-\\u2B55\"\n",
    "                               u\"\\u200d\"\n",
    "                               u\"\\u23cf\"\n",
    "                               u\"\\u23e9\"\n",
    "                               u\"\\u231a\"\n",
    "                               u\"\\ufe0f\"  # dingbats\n",
    "                               u\"\\u3030\"\n",
    "                               \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', string)\n",
    "\n",
    "counter = CountVectorizer()\n",
    "data1 = counter.fit_transform(tweets)\n",
    "count = pd.DataFrame(data = data1.toarray(), columns = counter.get_feature_names())\n",
    "unique_words = count.columns[count.sum(axis = 0) == 1]\n",
    "\n",
    "#Add method to ged rid of unique words\n",
    "def kick_the_word(s):\n",
    "    p = re.compile(r\"\\w+(?:'\\w+)*|[^\\w\\s]\")\n",
    "    b = p.findall(s)\n",
    "    for word in b:\n",
    "        if word in unique_words:\n",
    "            s = s.replace(word, '')\n",
    "    return s\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DLN2Twn98Tgn",
    "outputId": "a2689013-9799-4ac4-ee96-cb586ead7914"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13240"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "8lBvDkwmNJ3d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-20-1058575585fc>:5: FutureWarning: The default value of regex will change from True to False in a future version.\n",
      "  tweets = tweets.str.replace('[^\\w\\s]','')\n"
     ]
    }
   ],
   "source": [
    "#Application of methods\n",
    "tweets = tweets.apply(lambda x: x.replace('#USER#', '').lower())\n",
    "tweets = tweets.apply(remove_emoji)\n",
    "tweets = tweets.apply(kick_the_word)\n",
    "tweets = tweets.str.replace('[^\\w\\s]','')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KY8pdTiqFXft"
   },
   "source": [
    "## **3.2 N-gram Method**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kKx3b3A-GNvI"
   },
   "source": [
    "### **3.2.1 Word N-gram Vectorizer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "pRubp_KDnjNX"
   },
   "outputs": [],
   "source": [
    "tfidf_word = TfidfVectorizer(\n",
    "    analyzer='word',\n",
    "    stop_words=stopwords,\n",
    "    preprocessor=preprocess,\n",
    "    ngram_range=(1, 3),\n",
    "    use_idf=True,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=15000,\n",
    "    min_df=5,\n",
    "    max_df=0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "6Vx60AuKogSe"
   },
   "outputs": [],
   "source": [
    "#Construct tfidf matrix and get relevant scores\n",
    "tfidf_words =tfidf_word.fit_transform(tweets).toarray()\n",
    "vocab_words = {v:i for i, v in enumerate(tfidf_word.get_feature_names())}\n",
    "idf_vals = tfidf_word.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab_words.values()} #keys are indices; values are IDF scores\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4EgIMZ6qGhxr"
   },
   "source": [
    "### **3.2.2 Char N-gram Vectorizer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "mktohBFWGnqG"
   },
   "outputs": [],
   "source": [
    "tfidf_char = TfidfVectorizer(\n",
    "    analyzer='char',\n",
    "    stop_words=stopwords,\n",
    "    preprocessor=preprocess,\n",
    "    ngram_range=(2, 6),\n",
    "    use_idf=True,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=15000,\n",
    "    min_df=5,\n",
    "    max_df=0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "fgaNNYf3Gu7x"
   },
   "outputs": [],
   "source": [
    "#Construct tfidf matrix and get relevant scores\n",
    "tfidf_chars =tfidf_char.fit_transform(tweets).toarray()\n",
    "vocab_chars = {v:i for i, v in enumerate(tfidf_char.get_feature_names())}\n",
    "idf_vals = tfidf_char.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab_chars.values()} #keys are indices; values are IDF scores\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yt9BMeZbNgbK"
   },
   "source": [
    "### **3.2.3 POS-TF-IDF Vectorizer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "BKbC1u6_4oH_"
   },
   "outputs": [],
   "source": [
    "#We can use the TFIDF vectorizer to get a token matrix for the POS tags\n",
    "pos_vectorizer = TfidfVectorizer(\n",
    "    tokenizer=None,\n",
    "    lowercase=False,\n",
    "    preprocessor=None,\n",
    "    ngram_range=(1, 3),\n",
    "    stop_words=None,\n",
    "    use_idf=False,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=15000,\n",
    "    min_df=5,\n",
    "    max_df=0.75,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "TZwFPpxNNnag"
   },
   "outputs": [],
   "source": [
    "#Get POS tags for tweets and save as a string\n",
    "tweet_tags = []\n",
    "list_tokens = []\n",
    "for t in tweets:\n",
    "    sentence = re.sub(r'[^\\w\\s]','',t.lower())\n",
    "    tokens = nltk.word_tokenize(preprocess(sentence))\n",
    "    tags = nltk.pos_tag(tokens)\n",
    "    tag_list = [x[1] for x in tags]\n",
    "    tag_str = \" \".join(tag_list)\n",
    "    tweet_tags.append(tag_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "W2xkferj4uct"
   },
   "outputs": [],
   "source": [
    "#Construct POS TF matrix and get vocab dict\n",
    "pos = pos_vectorizer.fit_transform(pd.Series(tweet_tags)).toarray()\n",
    "pos_vocab = {v:i for i, v in enumerate(pos_vectorizer.get_feature_names())}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "z4kxfC6N4ZIy"
   },
   "source": [
    "## **3.3 Splitting Datasets for Machine Learning Models**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "H1VWSsJCqfbO"
   },
   "outputs": [],
   "source": [
    "ngramm_word = np.concatenate([tfidf_words,pos],axis=1)\n",
    "ngramm_char = np.concatenate([tfidf_chars, pos],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "ZTG_kKsu0Iy8"
   },
   "outputs": [],
   "source": [
    "#Get a list of variable names\n",
    "variables_words = ['']*len(vocab_words)\n",
    "for k,v in vocab_words.items():\n",
    "    variables_words[v] = k\n",
    "\n",
    "variables_chars = ['']*len(vocab_chars)\n",
    "for k,v in vocab_chars.items():\n",
    "    variables_chars[v] = k\n",
    "\n",
    "pos_variables = ['']*len(pos_vocab)\n",
    "for k,v in pos_vocab.items():\n",
    "    pos_variables[v] = k\n",
    "\n",
    "feature_names_words = variables_words+pos_variables\n",
    "feature_names_chars = variables_chars+pos_variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "id": "Qf8YEAPUlaVH"
   },
   "outputs": [],
   "source": [
    "y_notoff = dfs[\"subtask_a\"]\n",
    "y = y_notoff.copy()\n",
    "for i in range(len(y)):\n",
    "    if y_notoff[i] == 'NOT':\n",
    "        y[i] = 1\n",
    "    else:\n",
    "        y[i] = 0\n",
    "y=y.astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "6eOHBOUWq1c8"
   },
   "outputs": [],
   "source": [
    "X_word = pd.DataFrame(ngramm_word)\n",
    "X_char = pd.DataFrame(ngramm_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "wBOHzB5ZOWB1"
   },
   "outputs": [],
   "source": [
    "X_train_word, X_test_word, y_train_word, y_test_word = train_test_split(X_word, y, test_size = 0.3, random_state = 42, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "id": "oZGLk47zLZqI"
   },
   "outputs": [],
   "source": [
    "X_train_char, X_test_char, y_train_char, y_test_char = train_test_split(X_char, y, test_size = 0.3, random_state = 142, shuffle=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vL5Fb7pN5Zr9"
   },
   "source": [
    "# **4. Machine Learning Models**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Iq095_DZqd1J"
   },
   "source": [
    "# **4.1 Evaluation Metrics for Machine Learning Models**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "GT-IJjbn390k"
   },
   "outputs": [],
   "source": [
    "def evaluate_model_Xy(model, X, y, y_pred=None, label=\"Training\", model_name=\"model\"):\n",
    "  if y_pred is None:\n",
    "    y_pred = model.predict(X)\n",
    "\n",
    "  print(label + ' Set')\n",
    "  print(\"Accuracy:\", accuracy_score(y, y_pred))\n",
    "  print()\n",
    "\n",
    "  print(classification_report(y, y_pred, digits=4))\n",
    "  disp = plot_confusion_matrix(model, X, y, \n",
    "                               cmap=plt.cm.Blues, normalize='true')\n",
    "  plt.savefig(model_name + \"_\" + label.lower() + \".png\")\n",
    "  plt.show()\n",
    "  print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "jbwrJRon3-CL"
   },
   "outputs": [],
   "source": [
    "def evaluate_model_ngrammWord(model, model_name=\"model\",\n",
    "                   y_train_pred=None, y_dev_pred=None, y_test_pred=None):\n",
    "  evaluate_model_Xy(model, X_train_word, y_train_word, label=\"Training\", model_name=model_name)\n",
    "  evaluate_model_Xy(model, X_test_word, y_test_word, label=\"Testing\", model_name=model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "jeZJiP8gK0lL"
   },
   "outputs": [],
   "source": [
    "def evaluate_model_ngrammChar(model, model_name=\"model\",\n",
    "                   y_train_pred=None, y_dev_pred=None, y_test_pred=None):\n",
    "  evaluate_model_Xy(model, X_train_char, y_train_char, label=\"Training\", model_name=model_name)\n",
    "  evaluate_model_Xy(model, X_test_char, y_test_char, label=\"Testing\", model_name=model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dMx5i0qp4oP6"
   },
   "source": [
    "### **4.2 LogisticRegression**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "bQZoDSPUtEgz"
   },
   "outputs": [],
   "source": [
    "param_grid = [{}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "UwXXYnLoRI3M"
   },
   "outputs": [],
   "source": [
    "clf_lr = LogisticRegression(penalty='l2',\n",
    "                            tol=1e-4,\n",
    "                            C=5.0,\n",
    "                            fit_intercept=True,\n",
    "                            class_weight='balanced',\n",
    "                            random_state=0,\n",
    "                            solver='lbfgs',\n",
    "                            max_iter=100,\n",
    "                            multi_class='auto',\n",
    "                            verbose=1,\n",
    "                            n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "id": "w0yDoNPks55e"
   },
   "outputs": [],
   "source": [
    "grid_search_lr = GridSearchCV(clf_lr, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_word, y_train_word), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WKXOKH-8z0SQ",
    "outputId": "28eb3c2d-14ea-4fa8-d5f6-7d9aaf500a6b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  3.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.7min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  3.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.6min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  3.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.2min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  3.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.3min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  3.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.8min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  4.5min finished\n"
     ]
    }
   ],
   "source": [
    "clf_lr_word = grid_search_lr.fit(X_train_word, y_train_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "W1Q5jJ6r3ycG"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.9898575744497194\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9828    0.9866    0.9847      3067\n",
      "           1     0.9934    0.9915    0.9924      6201\n",
      "\n",
      "    accuracy                         0.9899      9268\n",
      "   macro avg     0.9881    0.9890    0.9886      9268\n",
      "weighted avg     0.9899    0.9899    0.9899      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbY0lEQVR4nO3de5xd873/8dd7ZhJC7p3cL4QqEookbnEpdRCXHoqSCked+qEazmkdSk/R0p4q9ahbNNUeVXVJkSAqLuk5R0NCm0sRiYYIuSIXoqmGmJnP74+9EzNjsmevZHb2mjXv5+OxH4+99v6u7/qumeQ933X5fpciAjOzrKgodwPMzFqSQ83MMsWhZmaZ4lAzs0xxqJlZplSVuwH1qapDqH2ncjfDEth794HlboIlsHjRm6xatUpbUkdl5x0iatYVVTbWrXwyIkZuyfaSSleote/ENrudVu5mWALTnr+53E2wBA46YN8triNq1rHNrqcWVfbDF8ZWb/EGE0pVqJlZayBQes9cOdTMLBkBFZXlbsUmOdTMLDlt0Wm5knKomVlCPvw0s6xxT83MMkO4p2ZmWSL31MwsY3z108yywxcKzCxLhA8/zSxj3FMzs+zw4aeZZYmASl8oMLMs8Tk1M8sOH36aWda4p2ZmmeKempllhjxMysyyxsOkzCw7fKHAzLLGh59mlhmeT83MssWHn2aWNb5QYGaZ4nNqZpYZ8uGnmWWNe2pmliVyqJlZVuRm83aomVlWSKjCoWZmGeKempllikPNzDLFoWZm2aH8K6UcamaWiJB7amaWLRUVHlFgZhninpqZZUfKz6mltw9pZqklqahXEfWMlDRf0gJJlzXxfRdJj0p6UdJcSWc3V6dDzcwS2XChYEtDTVIlMBY4BhgMfFXS4EbFvgnMi4i9gMOAGyS1L1SvQ83MElOFino1Yz9gQUQsjIj1wHjghEZlAuikXEJ2BN4FagpV6nNqZpaMEl0oqJY0s97y7RFxe/59P2BJve+WAvs3Wv9WYBKwHOgEnBYRdYU26FAzs8QShNqqiBi+qWqa+CwaLR8NvAB8EdgZmCLpmYj426Y26MNPM0ushS4ULAUG1FvuT65HVt/ZwMTIWQC8AexWqFKHmpkl0lIXCoAZwC6SBuVP/o8id6hZ32LgCABJvYBdgYWFKvXhp5kl1wL3qUVEjaQxwJNAJXBHRMyVdH7++3HANcCdkubkt/qdiFhVqF6Hmpklo5YbJhURk4HJjT4bV+/9cuCoJHU61MwsMQ+TMrNsSW+mOdS2xBEH7M6PLz6ZyooKfvvIc9x415QG33fp1IFbrxjNoH7VfLi+hguvuYdXFr4FwHmnfYGzThwBEnc9PJ1x458uwx60DX94bh7fvWECtXV1nHnCgfz7WQ2PZiKCy2+YwJTpc+mwbXvGXnkGe+2Wuyg35pp7eOrZl6nu1onp47+7cZ0fjfs9j0+dQ4VEdfdOjL3yDPr06LJV96uc0txTK+nVz+bGdbVmFRXi+ku/wlf+7ecccNqPOPnoYew6qHeDMhd/7SjmvLqMg0dfyze+/1t+fPHJAOy+Ux/OOnEER3ztpxwy+lqOPngPdhrQoxy7kXm1tXVcet0D3H/TN3jud//JhCdn8df8H5YN/jB9Hq8vWcHMCVfys8tHcfFPfrfxu9OP258HbrrgU/VeeMYRPHvv5Uy95zKOPngI1//q8ZLvS1oUe+WzXMFXslArclxXqzVsyA4sXLqKRctX83FNLROfmsWxh+7ZoMyug/owdcZ8AF5b9A4D+3SnR/dOfG5QL2a8/CbrPvqY2to6ps1+jeMP+3w5diPzZs1dxKD+1ezYr5r27ao46ahhPD51ToMyk6fOYdSx+yGJffccxN/WruPtVe8DMGLoZ+nWebtP1du5Y4eN7/+xbn2qey6l0CZDjeLGdbVafXp0Zdk7721cXr5iDX16dG1Q5uXXlnH84XsBMHTwDgzo3Z2+PbvyyutvMWKfz9Kty3Z02KYdRx40hH69um3N5rcZb61c0+Bn27dnV95auaZhmRVNlFnxfrN1//C2R9nj+Ct44ImZXH7esS3W5taghcZ+lkQpQ62pcV39GheSdK6kmZJmRs26EjanZTX1RygajfC48a4pdO20HVPv/g7nnnooL726lNraOl598x1uumsKD90yhgdvvoC5ry2jprbgcDbbTNF40A25m0cblPnUyJymf7+Nfe+CL/Hy76/hKyOH88sHpm5uE1ulNPfUSnmhoJhxXeQHt94OULFdzyb+CabT8ib+ur+9suFf97UffMiYa+7ZuPziw99n0fLVANw96XnunvQ8AFd840ssX7Gm9I1ug/r2/HSPunejE/p9e3Zrtkwhpxw9nNO+NY7Lzz1uyxvcGiQb0L7VlbKnVsy4rlZr9rzF7DygBwP7foZ2VZW5czXPNDxX07ljB9pVVQLwLyeMYPoLr7P2gw8BqO7WEYD+vbpx/OF78eBTM7GWN3TwQBYuWcmiZatY/3ENE5+axchDGp77POaQPRg/+c9EBDPmvEHnjtvSu7pwqL2+eMXG949PncMuO/YqSfvTSOR6ssW8yqGUPbWN47qAZeTGdZ1ewu1tVbW1dVx6/QNMuPkCKivEPY8+z18Xvs3ZJx0EwK8nTmPXQb34+VVnUlsXzH/jbS784Se9trt+cg7dOm9HTW0dl1x/P++vbT2H3q1JVVUl113yFU656DZq64LRXzqA3Xfuw68nPAvA2ScfzJEHDWHK9HkMO+lqOmzbjluvOGPj+ud879dMm7WA1Wv+zpDjr+Cy/3csZ55wID8YO4kFi1ZQUSEG9O7ODZedVq5dLIN0P01K0dRJh5aqXDoWuJFPxnX9qFD5iu16xja7taV/HK3fu3+6udxNsAQOOmBfZs+auUWJtG3vz8UOZ91SVNlXrxs5q8DUQyVR0ptvmxrXZWatXBkPLYvhEQVmlojI3XyeVg41M0vMPTUzy5Q0XyhwqJlZMj6nZmZZItRik0SWgkPNzBJzT83MMsXn1MwsO3xOzcyyJDf2M72p5lAzs8RSnGkONTNLziMKzCw7Uj6fmkPNzBLZMJ9aWjnUzCyhdM+n5lAzs8RSnGkONTNLSL5QYGYZ4vvUzCxzHGpmlikpzjSHmpkl556amWWHB7SbWZbkJolMb6o51MwssYoUd9XSOyevmaWWVNyr+Xo0UtJ8SQskXbaJModJekHSXEl/bK5O99TMLBG10IB2SZXAWOBIYCkwQ9KkiJhXr0xX4DZgZEQsltSzuXrdUzOzxCpU3KsZ+wELImJhRKwHxgMnNCpzOjAxIhYDRMSK5irdZE9N0i1AbOr7iLio2SabWSYluFBQLWlmveXbI+L2/Pt+wJJ63y0F9m+0/ueAdpKeBjoBN0XEXYU2WOjwc2aB78ysjRK5K6BFWhURwwtU1VjjjlQVMAw4AugAPCfp+Yh4dVMb3GSoRcRvGmxd2j4iPthUeTNrO1rojo6lwIB6y/2B5U2UWZXPng8kTQX2AjYZas2eU5N0oKR5wCv55b0k3Zaw8WaWFcrNp1bMqxkzgF0kDZLUHhgFTGpU5hHgEElVkrYjd3j6SqFKi7n6eSNw9IaNRcSLkg4tYj0zy6iWuE0tImokjQGeBCqBOyJirqTz89+Pi4hXJD0BvATUAb+KiJcL1VvULR0RsaRR6tZuzk6YWesnWu7m24iYDExu9Nm4RsvXA9cXW2cxobZE0ggg8l3Ei2im+2dm2ZbmYVLF3Kd2PvBNcpdflwF755fNrA0qdjRBuUZSNdtTi4hVwOit0BYzayVa9dhPSTtJelTSSkkrJD0iaaet0TgzSycV+SqHYg4/7wXuB/oAfYEHgPtK2SgzS7cWuqWjJIoJNUXEbyOiJv+6mwLDp8ws23JXP1tk7GdJFBr72T3/9v/yU4KMJxdmpwGPbYW2mVkaqfVOEjmLXIhtaP159b4L4JpSNcrM0q1VPqMgIgZtzYaYWeuw4fAzrYoaUSBpD2AwsO2Gz5qb/sPMsqtV9tQ2kHQVcBi5UJsMHAM8CzjUzNqo9EZacVc/TyE3l9HbEXE2uWk/tilpq8wstSSorFBRr3Io5vBzXUTUSaqR1BlYAfjmW7M2rFUffgIz8w8/+CW5K6J/B/5cykaZWbqlONOKGvt5Qf7tuPy8Rp0j4qXSNsvM0koo1WM/C918O7TQdxExuzRNMrNUK+MMHMUo1FO7ocB3AXyxhdvCPrsPZNqfbmnpaq2Euu07ptxNsAQ+mr+4RepplefUIuLwrdkQM2sdBFS2xlAzM9uUVj+iwMysPoeamWVGbqru9KZaMTPfStIZkq7MLw+UtF/pm2ZmaZXm+dSKGSZ1G3Ag8NX88lpgbMlaZGap16ofvALsHxFDJf0FICLeyz8qz8zaIAFVKT78LCbUPpZUSX4Kb0k9yD0p2czaqBRnWlGhdjPwENBT0o/IzdrxvZK2ysxSS2qlw6Q2iIh7JM0iN/2QgBMjwk9oN2vDUpxpRU0SORD4B/Bo/c8iomXGW5hZq9Pa71N7jE8ewLItMAiYDwwpYbvMLKUEZZsAshjFHH7uWX85P3vHeZsobmZZV8Z70IqReERBRMyWtG8pGmNmrYNS/JSCYs6pfbveYgUwFFhZshaZWapl4RF5neq9ryF3jm1CaZpjZq1Bqw21/E23HSPikq3UHjNrBdI8oL3QdN5VEVFTaFpvM2t7co/IK3crNq1Q0zY8MeoFSZMknSnppA2vrdE4M0univyoguZezZE0UtJ8SQskXVag3L6SaiWd0lydxZxT6w6sJvdMgg33qwUwsYh1zSxjWupCQf701ljgSGApMEPSpIiY10S5nwBPFlNvoVDrmb/y+TKfhNkGkaDtZpYxLXRKbT9gQUQszNWp8cAJwLxG5S4kd3GyqFvJCoVaJdARmrwhxaFm1maJiuLvU6uWNLPe8u0RcXv+fT9gSb3vlgL7N9iS1A/4MrkjxS0Otbci4upiKjGztkMk6qmtiojhBapqrHGH6UbgOxFRW+wV10Khlt5rtmZWPoKqlrlRbSkwoN5yf2B5ozLDgfH5QKsGjpVUExEPb6rSQqF2xOa108yyLGFPrZAZwC6SBgHLgFHA6fULRMSgjduV7gR+XyjQoPDDjN/dgsaaWYa1xCSR+ftgx5C7qlkJ3BERcyWdn/9+3ObU60fkmVliLTWgICImA5MbfdZkmEXE14qp06FmZomI4h5DVy4ONTNLRi1z+FkqDjUzSyQ3osChZmYZkt5Ic6iZ2WZIcUfNoWZmSal1zqdmZtYUX/00s8zxhQIzyw610um8zcya4sNPM8sc99TMLFPSG2kONTNLSECle2pmliUpzjSHmpklJZTiA1CHmpkl5p6amWVG7paO9KaaQ83MkpF7amaWMR4mZWaZkZskstyt2DSHmpkl5qufZpYpKT76dKgV4w/T53H5DQ9SW1fHmSeM4FtfO6rB9xHBZTc8yJRpc+mwbXtuu+pM9tptQMF158xfyrevHc+HH31MVVUFP/3OaQwbsiOLl69m/1N/yGcH9gRg+J478rPLv7p1dzjDjjhwd3588SlUVlTw20emc+NvpjT4vkunDtx6xRkM6l/Nh+s/5sJr7uGV198C4LxRh3HWiSNA4q6HpzHuvqfLsAfp0CZ7apLuAI4HVkTEHqXaTqnV1tZxyXX389CtY+jbqytfPOt6jjl0T3bbqc/GMlOmz+P1xSuZNfEqZr78JhdfO54/3HlJwXWvuuVhLj3nGI48aAhPTZvLVTc/zO9/8e8A7NivmmfuvbxMe5xdFRXi+ktP5ctjbmX5O2v4399cwuNT5zD/jbc3lrn47KOZ8+pSzrz0l+yyQy+u/86pnHjBLey+cx/OOnEER5x1Petrannw5gt46tm5LFyysox7VB5pP6dWyhlE7gRGlrD+rWLW3DfZaUA1O/avpn27Kk46ciiT//hSgzKT//gSo47bD0nsu+cg3l+7jrdXvV9wXQnWfvAhAH/7+zp69+iy1fetrRk2ZEcWLlnFomWr+bimlolTZnPsFz7foMyug3ozdcZ8AF5b9A4D+3SnR/dOfG7H3syY8ybrPvqY2to6ps1ewPGH7VWO3Sg/iYoiX+VQslCLiKnAu6Wqf2t5a+X79OvVbeNy317deGvl+43KrGlYpmdX3lqxpuC6//XtU7jy5ocZctz3uPKmh7jymydsLLd4+WoOHX0tx517I9P/sqBUu9bm9OnRhWXvvLdxefk779Gn0R+Tl19bxvGH7w3A0ME7MKB3d/r27Morry9nxD6fpVuX7emwTTuOHDGkwe+2rVGRr3Io+zk1SecC5wIMGDiwzK35tIj41GeN/wA1UQRJBde9Y8Iz/Ne3T+Kfv7gPD02ZzUXX3MPDt11Ir+rOzHn0arp37cgLryxm9H/cznO/+086d+zQErvTpjU1B1jjX9GNv5nCjy8+han3XMa8Bct56dWl1NbW8eqb73DTXVN46NYxfPCPj5j72jJqamu3UsvTxc/9bEZE3A7cDjBs2PAm4qG8+vbs+qm/7r2ruxQus2INvXt0Yf3HNZtc977f/4lrLz4FgBP/aR/+7Uf3ArBN+3Zs074dAHvvPpBB/at5ffEK9hm8Q2l2sA1ZvmLNp3rOb69q2Ote+8GHjLn67o3LLz7yAxYtXw3A3ZOe4+5JzwFwxQVfYvmKNaVvdEqlN9LSPStvKgwdvAOvL17JomWrWP9xDROnzOaYQxuehznm0D0Z/9ifiQhmzHmDzh070Lu6S8F1+/TowrTZrwEwdcar7DSgBwCr3ltLbW0dAG8uXcXCJSvZsV/1Vtzj7Jo9bxE7D+zBwL6foV1VJScdOZTHpzY8P9q5YwfaVVUC8C8njmD6XxZsPPdZ3a0jAP17deP4w/fiwSdnbt0dSJMUH3+WvaeWdlVVlVx36amcfNFYamuD0f98ALvv3Ic7JjwDwL+efAhHHTSEKdPmMvTLP6DDtu0Ye+UZBdcFuPE/T+fyGx6kpraObdtXceN3c7dtTP/LAn487jEqqyqprBA3XDaKbl22L8/OZ0xtbR2XXnc/E27+JpWV4p5Jz/PXhW9z9kkHA/Dric+y66De/Pz7Z1JbV8f8N97mwmvu2bj+XT85h25dtqemppZLrruf99euK9eulF2aDz/V1HmfFqlYug84DKgG3gGuioj/LrTOsGHDY9qf2vBfv1ao275jyt0ES+Cj+fdT948VW5RIu++5T9z1yNNFld1v566zImL4lmwvqZL11CLCd4yaZVV6O2o+/DSzZHKny9Kbag41M0vG86mZWdakONN8S4eZJSWk4l7N1iSNlDRf0gJJlzXx/WhJL+Vf0yU1OzbNPTUzS6wlDj8lVQJjgSOBpcAMSZMiYl69Ym8AX4iI9yQdQ+5G/f0L1euempklUux9t0Xk3n7AgohYGBHrgfHACfULRMT0iNgwLOd5oH9zlTrUzCy54lOtWtLMeq9z69XSD1hSb3lp/rNN+TrweHNN8+GnmSWW4JaOVQVuvm2qkiZHA0g6nFyoHdzcBh1qZpZYC93SsRQYUG+5P7D809vS54FfAcdExOrmKvXhp5klk79PrZhXM2YAu0gaJKk9MAqY1GBT0kBgInBmRLxaTPPcUzOzxFpiREFE1EgaAzwJVAJ3RMRcSefnvx8HXAl8Brgtf4tITXNjSR1qZpaIaLkRBRExGZjc6LNx9d6fA5yTpE6HmpklluYRBQ41M0suxanmUDOzxNI8SaRDzcwSS2+kOdTMbHOkONUcamaWiCeJNLNs8SSRZpY1Kc40h5qZJVXcBJDl4lAzs8RSnGkONTNLpowPXy+KQ83MkktxqjnUzCwx39JhZpnic2pmlh2CCoeamWVLelPNoWZmibTkJJGl4FAzs8RSnGkONTNLzj01M8sUD5Mys0xJb6Q51MwsoSKf6Vk2DjUzS8wjCswsW9KbaQ41M0suxZnmUDOzpORH5JlZdqR9REFFuRtgZtaS3FMzs8TS3FNzqJlZYr6lw8yywzffmlmWpP1CgUPNzBLz4aeZZYp7amaWKSnONIeamW2GFKeaQ83MEhGkepiUIqLcbdhI0kpgUbnbUQLVwKpyN8ISyervbIeI6LElFUh6gtzPpxirImLklmwvqVSFWlZJmhkRw8vdDiuef2etl8d+mlmmONTMLFMcalvH7eVugCXm31kr5XNqZpYp7qmZWaY41MwsUxxqJSRppKT5khZIuqzc7bHmSbpD0gpJL5e7LbZ5HGolIqkSGAscAwwGvippcHlbZUW4E9iqN4tay3Kolc5+wIKIWBgR64HxwAllbpM1IyKmAu+Wux22+RxqpdMPWFJveWn+MzMrIYda6TQ14tf3z5iVmEOtdJYCA+ot9weWl6ktZm2GQ610ZgC7SBokqT0wCphU5jaZZZ5DrUQiogYYAzwJvALcHxFzy9sqa46k+4DngF0lLZX09XK3yZLxMCkzyxT31MwsUxxqZpYpDjUzyxSHmpllikPNzDLFodaKSKqV9IKklyU9IGm7LajrTkmn5N//qtBge0mHSRqxGdt4U9Knnjq0qc8blfl7wm19X9J/JG2jZY9DrXVZFxF7R8QewHrg/Ppf5mcGSSwizomIeQWKHAYkDjWzcnCotV7PAJ/N96L+T9K9wBxJlZKulzRD0kuSzgNQzq2S5kl6DOi5oSJJT0sann8/UtJsSS9K+h9JO5ILz2/le4mHSOohaUJ+GzMkHZRf9zOSnpL0F0m/oIjneEt6WNIsSXMlndvouxvybfkfST3yn+0s6Yn8Os9I2q1FfpqWGX5CeyskqYrcPG1P5D/aD9gjIt7IB8P7EbGvpG2AaZKeAvYBdgX2BHoB84A7GtXbA/glcGi+ru4R8a6kccDfI+Kn+XL3Aj+LiGclDSQ3amJ34Crg2Yi4WtJxQIOQ2oR/zW+jAzBD0oSIWA1sD8yOiIslXZmvewy5B6KcHxGvSdofuA344mb8GC2jHGqtSwdJL+TfPwP8N7nDwj9HxBv5z48CPr/hfBnQBdgFOBS4LyJqgeWS/reJ+g8Apm6oKyI2Na/YPwGDpY0dsc6SOuW3cVJ+3cckvVfEPl0k6cv59wPybV0N1AG/y39+NzBRUsf8/j5Qb9vbFLENa0Mcaq3LuojYu/4H+f/cH9T/CLgwIp5sVO5Ymp/6SEWUgdxpiwMjYl0TbSl63J2kw8gF5IER8Q9JTwPbbqJ45Le7pvHPwKw+n1PLnieBb0hqByDpc5K2B6YCo/Ln3PoAhzex7nPAFyQNyq/bPf/5WqBTvXJPkTsUJF9u7/zbqcDo/GfHAN2aaWsX4L18oO1Grqe4QQWwobd5OrnD2r8Bb0j6Sn4bkrRXM9uwNsahlj2/Ine+bHb+4SG/INcjfwh4DZgD/Bz4Y+MVI2IlufNgEyW9yCeHf48CX95woQC4CBievxAxj0+uwv4AOFTSbHKHwYubaesTQJWkl4BrgOfrffcBMETSLHLnzK7Ofz4a+Hq+fXPxFOnWiGfpMLNMcU/NzDLFoWZmmeJQM7NMcaiZWaY41MwsUxxqZpYpDjUzy5T/D5FkVPGXk+OFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.6815206445115811\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5240    0.5566    0.5398      1333\n",
      "           1     0.7688    0.7446    0.7565      2639\n",
      "\n",
      "    accuracy                         0.6815      3972\n",
      "   macro avg     0.6464    0.6506    0.6482      3972\n",
      "weighted avg     0.6866    0.6815    0.6838      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAayUlEQVR4nO3debxXdb3v8dd7b0RkRjcq4ASKA4IS4oAmWmaBpzKHCvMcy+qq3chz6jR47rlHy4aTmvdaZpesTE3TNDQpySEnNBsYQgQURRxAFNgMCohuN/tz//j9gB97XAv2b//WXvv99LEej99a6/v7ru9my5vvWuv7XUsRgZlZXlRVugFmZu3JoWZmueJQM7NccaiZWa441MwsV7pVugGluvfuHz12H1TpZlgKu/fapdJNsBRWv7aM9evWaGfqqO67f0T9pkRlY9Oq+yNiws4cL61MhVqP3Qdx3Dd+WelmWAofP8r/CHUm3z3/IztdR9RvYtdDPpGo7Ntzr6vZ6QOmlKlQM7POQKDsXrlyqJlZOgKqqivdihY51MwsPe3UZbmycqiZWUo+/TSzvHFPzcxyQ7inZmZ5IvfUzCxnfPfTzPLDNwrMLE+ETz/NLGfcUzOz/PDpp5nliYBq3ygwszzxNTUzy49sn35mt2Vmll1SsqXNajRB0iJJiyVd0sz+r0maW1zmS9osaffW6nSomVl6qkq2tFaFVA1cB0wERgDnSBpRWiYiroqI0RExGvgP4LGIWNNavQ41M0snaS+t7Z7aMcDiiFgSEXXA7cDprZQ/B7itrUodamaWXlV1sgVqJM0qWS4oqWUIsLRkfVlxWxOSegITgKltNc03CswspVQ3CmojYmzLFTURLZT9CPDntk49waFmZjuifYZ0LAP2LVnfB1jeQtlJJDj1BJ9+mllaW56ntpM3CoCZwHBJQyV1pxBc05ocTuoHnATck6R57qmZWUrtM04tIuolTQbuB6qBGyJigaSLivunFIueATwQERuT1OtQM7P02ul5ahExHZjeaNuURus3AjcmrdOhZmbpeZqUmeWGsj1NyqFmZum5p2ZmeSKHmpnlReFp3g41M8sLCVU51MwsR9xTM7NccaiZWa441MwsP0Tzz9fICIeamaUi5J6ameVLVZVnFJhZjrinZmb54WtqZpY37qmZWW74RoGZ5Y6nSZlZfsinn2aWMw41M8sVh5qZ5YZvFJhZ/mQ30xxqZpaSPE3KzHLGp59mli/ZzTSH2s4Ys28/Pn/8AVRLPPDsSqbOXb7d/pGD+vKfHzqYFevfAeAvL67hN3NeBaBX92omnzSM/Qf0JIAfPfYCi1Zs6OgfoctZuOBF7rrjIRoiGHfCEZz6oWObLffyS6/xf668lc98/iO8Z8whW7c3NDRw1X//iv79e3PhF8/qqGZnTpftqUmaAPwQqAZ+HhHfL+fxOlKV4MIThnLpvc+wemMdV585kr+/tJal6zZtV27h6+v59n2Lmnz/fxx/AHOWruOKB5+nW5XYtVt2r1HkRUNDA3fe/iBfvPgT9B/Qhx98/1eMPOJABg2qaVJu2t0zOGzEAU3qePTh2ey99x68/fY7HdTq7JGyffezbH+TJFUD1wETgRHAOZJGlOt4HW34nr157c23WbH+HeobgscXr+bYAwYk+u5uu1Rz+KA+PPjsKgDqG4KNdZvL2Vyj0PsaOHAANQP7061bNWPGHsrTTy1uUu6xR+Zw5HuG07tPz+22r127noXzlzDuhFEd1eTM2hJsbS2VUM7uwTHA4ohYEhF1wO3A6WU8Xofao2d3ajfUbV2v3VjHHr26Nyl3yF69+eHZo7hs4qHsO2A3APbuuytvvF3Pv558INecNYrJ44e5p9YB1q3bQP8Bfbau9x/QhzfWbWhUZj3znnqe944f3eT7d935MB8946RMz3vsKKpSoqUSyvk3aQiwtGR9WXHbdiRdIGmWpFnvblhXxua0r+b+EYpG6y/UbuTzt/6Df/3t0/xh/uv854cOBqBa4sCaXvxx4Qr+berTvF2/mbNHDy5/o7u6xr8gmv4e77rzYT76sZOaDFmY//QL9OnTk/3237uMDew8stxTK+c1teZ+oib/W0XE9cD1AH33O6yZ/+2yqXZjHTW9t/XManp1Z83Guu3KbHp32ynl7KXruKhqKH16dKN2Yx21G+t4bmWhl/DkkjWc5VAru/4DerNu7fqt6+vWrqdvv97blXnl5RXc9IvfA7Bh4yYWzn+R6qoqXnrpNZ6et5iF85fwbn09b2+q4+Zf/oHzzv9wh/4MmdCFJ7QvA/YtWd8HWN5C2U7n+ZUbGNyvB3v12ZXVG+s48aA9+MFD21+f6b/bLqzb9C4Awwf2ogpY/3Y9ALUb3mFIvx68+sbbHDmkX5MbDNb+9tt/EKtWrmV17Tr69e/DnFnP8unPbh9K3/zOBVs/33LTdA4fdSBHjB7OEaOH89GPjQfg+ede4eEHZ3bNQKP44NvsZlpZQ20mMFzSUOBVYBLwqTIer0M1BPz0iZf45mmHUiXxp0UrWbp2ExMO2xOA+55ZyQnDdmfiiL3YHEFdfQNXPfT81u9f/+eX+MopB7FLlXj9zXf44aMvVOpH6TKqq6s4e9IH+Mm1v6WhoYHjjh/FoME1PDFjLkCz19GsOdm++6mI8p3xSToNuIbCkI4bIuK7rZXvu99hcdw3flm29lj7+/hRgyrdBEvhu+d/hJeembdTidRj74Nj/09fm6jsc1dOmB0RY3fmeGmVdZxaREwHppfzGGbWwZTt00+PIzCzVARUVSnR0mZd0gRJiyQtlnRJC2VOljRX0gJJj7VVp6dJmVlq7dFTKxmgfyqFG4szJU2LiIUlZfoDPwEmRMQrkvZsq1731MwstXYap5ZkgP6ngLsi4hWAiFjZVqUONTNLp3hNLcnShiQD9A8GBkh6VNJsSee1ValPP80sFaE0D4mskTSrZP364oD7QlVNNR6O0Q04CjgF2A34i6S/RsRzLR3QoWZmqaW4plbbypCOJAP0lxXr2AhslDQDOBJoMdR8+mlmqbXTNbWtA/QldacwQH9aozL3ACdK6iapJ3As8ExrlbqnZmbptNM4tYiolzQZuJ9tA/QXSLqouH9KRDwj6T5gHtBA4bmM81ur16FmZqkU5n62z+jb5gboR8SURutXAVclrdOhZmapZXlGgUPNzFJLMlugUhxqZpZOF36empnlUFd+npqZ5VK2n6fmUDOz1DKcaQ41M0tJvlFgZjnSnuPUysGhZmapOdTMLFcynGkONTNLzz01M8uPjL94xaFmZqkUHhKZ3VRzqJlZalUZ7qo51MwstQxnmkPNzNKRJ7SbWd5k+JJay6Em6Vqavtllq4i4uCwtMrPM66w3Cma1ss/MuihRuAOaVS2GWkTcVLouqVfxNVVm1sVluKPW9ivyJI2TtJDia6kkHSnpJ2VvmZllU8LX41XqZkKS935eA3wIWA0QEU8B48vYJjPLOCnZUgmJ7n5GxNJGqbu5PM0xs6wTnX/w7VJJxwNRfIvyxbTxhmQzy7cs3/1Mcvp5EfBFYAjwKjC6uG5mXVDSU8/Mnn5GRC1wbge0xcw6iSyffia5+zlM0u8lrZK0UtI9koZ1ROPMLJuUcKmEJKefvwbuAAYBg4E7gdvK2Sgzy7bOPqRDEfGriKgvLrfQyvQpM8u3wt3PZEsltDb3c/fix0ckXQLcTiHMPgnc2wFtM7MsUud9SORsCiG2pfUXluwL4NvlapSZZVunfPRQRAztyIaYWeew5fQzqxLNKJA0EhgB9NiyLSJuLlejzCzbOmVPbQtJlwEnUwi16cBE4AnAoWbWRWU30pLd/TwbOAV4PSLOB44Edi1rq8wssySorlKipRKSnH5uiogGSfWS+gIrAQ++NevCsnz6maSnNktSf+BnFO6IzgH+Xs5GmVm2tdfcT0kTJC2StLg4dKzx/pMlvSFpbnG5tK06k8z9/J/Fj1Mk3Qf0jYh5bTfXzPJIqF3mfkqqBq4DTgWWATMlTYuIhY2KPh4RH05ab2uDb8e0ti8i5iQ9iJnlSPs9geMYYHFELAGQdDtwOtA41FJprad2dSv7Anj/zhy4OcMH9mLahce1d7VWRgOOnlzpJlgK77z8ervUk+KaWo2k0pc4XR8R1xc/DwGWluxbBhzbTB3jJD0FLAe+GhELWjtga4Nv35eszWbWlQioTh5qtRExtpWqGms8r3wOsH9EbJB0GvA7YHhrB0xyo8DMbDvtNKF9GbBvyfo+FHpjW0XEmxGxofh5OrCLpJpW25b6pzGzLq+dQm0mMFzS0OKrAiYB00oLSNpbxXNdScdQyKzVrVWaaJqUmdkWheEaO3+nICLqJU0G7geqgRsiYoGki4r7p1AY/P8FSfXAJmBSRLT66LMk06RE4XHewyLickn7AXtHhMeqmXVR7TVZoHhKOb3Rtikln38M/DhV2xKU+QkwDjinuL6ewtgSM+uiOvWLV4BjI2KMpH8ARMTa4vmvmXVBArpleJpUklB7tzjyNwAkDQQaytoqM8u0DGdaolD7EXA3sKek71K4cPe/y9oqM8ssqX2mSZVLkrmft0qaTeHxQwI+FhF+Q7tZF5bhTEt093M/4C3g96XbIuKVcjbMzLKrsz/O+162vYClBzAUWAQcXsZ2mVlGCSr2AMgkkpx+jipdLz6948IWiptZ3lXwnZ5JpJ5REBFzJB1djsaYWeegDL+lIMk1ta+UrFYBY4BVZWuRmWVaHl6R16fkcz2Fa2xTy9McM+sMOm2oFQfd9o6Ir3VQe8ysE8jyi1dae5x3t+Is+hYf621mXU/hFXmVbkXLWuup/Z3C9bO5kqYBdwIbt+yMiLvK3DYzy6hOPaMA2J3CQ9nez7bxagE41My6oM58o2DP4p3P+WwLsy1afUibmeVbhjtqrYZaNdCbZC9HMLMuQ1R10nFqr0XE5R3WEjPrFETn7alluNlmVjGCbhm+qNZaqJ3SYa0ws06j0/bUImJNRzbEzDqPzj6kw8xsOxnONIeamaUjsv0WdIeamaUjn36aWY4UZhQ41MwsR7IbaQ41M9sBGe6oOdTMLC11zuepmZk1x3c/zSx3fKPAzPJDnfRx3mZmzfHpp5nljntqZpYr2Y00h5qZpSSgOsM9tSyfGptZRknJlrbr0QRJiyQtlnRJK+WOlrRZ0tlt1elQM7OUlPi/VmspvCz9OmAiMAI4R9KIFspdAdyfpHUONTNLrZ16ascAiyNiSUTUAbcDpzdT7kvAVGBlkrY51MwslcKQDiVa2jAEWFqyvqy4bduxpCHAGcCUpO3zjQIzSyfh9bKiGkmzStavj4jrt9XUROPXb14DfCMiNicdRuJQM7PUUkyTqo2IsS3sWwbsW7K+D7C8UZmxwO3FQKsBTpNUHxG/a+mADjUzS6XwkMh2qWomMFzSUOBVYBLwqdICETF063GlG4E/tBZo4FAzsx3Q1p3NJCKiXtJkCnc1q4EbImKBpIuK+xNfRyvlUDOz1Npr7G1ETAemN9rWbJhFxGeS1OlQ2wl/enIh/3H1b9nc0MC/nH48X/7MB7fbf8cfZ/LDmx8EoNduu3L1JZ9k1MH7APDG+re4+Du/5pkXXkOCa//rXI45YliH/wxdzSnjDuO///1sqquq+NU9T3LNTQ9ut/9L/3wKH594NADdqqs4+IC9OeiDl7DuzbcAqKoSj9z8dV5b+QaTvrJDHYlcaI+eWrmULdQk3QB8GFgZESPLdZxK2by5ga9deQd3/3gyg/fqz/s/fRUTx4/i0GGDtpbZf/Ae3PvTf6N/3548+OcFfPl7t/GnG78GwCVX/5ZTxo3gpis+T9279Wx6u65SP0qXUVUlrvr6Jzhj8o9ZvmIdD9/0Nf4442kWvfj61jLX3vIQ197yEAATThzJF85539ZAA7ho0vt47sUV9OnVo8PbnxXteE2tLMo5Tu1GYEIZ66+o2QteYti+NRywTw3dd+nGmaeOYfpj87Yrc+yRw+jftycAR48ayvKV6wB4c8MmnvzHC/zL6eMA6L5LN/r16dmh7e+Kjjr8AJYsreXlV1fzbv1m7npwDqeddESL5c/64FimPjB76/rgPfvzwfcezs33PNkRzc0uiaqESyWULdQiYgawplz1V9prq95gyF4Dtq4P3msAr616o8Xyv7rnST5wfGEGyMuvrqamf2+++K1bGH/u97n4O7eycdM7ZW9zVzdoYD9eXbF26/ryFWsZNLBfs2V323UXThl3GNMenrt12/e+chaX/eh3NDQ0HkrV9SjhUgkVn1Eg6QJJsyTNWlW7qtLNSSyi6f/YLf3D9Pis57hl2l/45uTCDJD6zZt5atFSPnv2icy49RJ69tiVa258sPkvW7tpbvBmM79GACaMH8Xf5i3Zeur5ofeOpHbtep56dmnzX+hCtrz3s8v11JKKiOsjYmxEjB1YM7DSzUls8J79m/yrv3dN03/15z//Khd/59fc+oML2L1/7+J3BzB4z/6MHXkAAB89ZTRPLfJflnJbvnJdk97167XN967PPPUopt6/7dTz2COHMeHEUTx1z7f4xffO58SjD+anl59X9jZnlXtqOTRmxP688MoqXn61lrp367nrwTlMHL/99Zmlr6/hvK//jCnfOo+D9t9r6/a9avoyZK8BPP/SCgBmzFzEIUP37tD2d0VzFr7MgfsNZL/Be7BLt2rOPHUMf5wxr0m5vr16cMKYg7a7Rnr5ddMY+eH/4sjTL+Nz/+uXPD7zOS689OaObH62ZDjVPKRjB3XrVs2VX/8EZ118HZs3B+d+9DgOO3AQN0x9HIDPnnUiV/38j6x5YyNfveI3xe9U8cjN3wDgyq9+nAsuvZG6dzdzwJAarrv0nyv2s3QVmzc38PUr72Dqj75IdbW4ddpfeXbJ65x/5nsB+OVdTwDwT+87kkf+9ixv+Y50i7L8Nik1d22oXSqWbgNOpjBfawVwWUT8orXvHHXU2Pjz32a1VsQyZsDRkyvdBEvhnUV30PDWyp1KpMNGvSduvufRRGWPObD/7FbmfpZF2XpqEXFOueo2swrLbkfNp59mlk7hcll2U82hZmbppHueWodzqJlZahnONIeamaUlv8zYzPIlw5nmUDOzdCo5WyAJh5qZpZfhVHOomVlqHtJhZrnia2pmlh8ep2ZmeePTTzPLDeGempnlTIYzzaFmZjsgw6nmUDOz1LL8kEiHmpmllt1Ic6iZ2Y7IcKo51MwsFT8k0szyxYNvzSxvMpxpDjUzS8sPiTSznMlwpjnUzCwdPyTSzPInw6nmUDOz1Dykw8xyJcvX1Koq3QAz62QEVQmXNquSJkhaJGmxpEua2X+6pHmS5kqaJem9bdXpnpqZ7YCd76pJqgauA04FlgEzJU2LiIUlxR4CpkVESDoCuAM4tLV63VMzs1S2PCQyydKGY4DFEbEkIuqA24HTSwtExIaIiOJqLyBog0PNzFJTwgWoKZ42blkuKKlmCLC0ZH1Zcdv2x5LOkPQscC/w2bba5tNPM0stxY2C2ogY21I1zWxr0hOLiLuBuyWNB74NfKC1AzrUzCy1dpomtQzYt2R9H2B5S4UjYoakAyXVRERtS+V8+mlmqaU4/WzNTGC4pKGSugOTgGnbHUc6SMUElTQG6A6sbq1S99TMLJWENwHaFBH1kiYD9wPVwA0RsUDSRcX9U4CzgPMkvQtsAj5ZcuOgWQ41M0utvWYURMR0YHqjbVNKPl8BXJGmToeamaWX4RkFDjUzSy3DmeZQM7O05FfkmVl+bJlRkFUe0mFmueKempmlluWemkPNzFLzQyLNLD/83k8zy5Os3yhwqJlZaj79NLNccU/NzHIlw5nmUDOzHZDhVHOomVkqgkxPk1IbjybqUJJWAS9Xuh1lUAO0+KROy6S8/s72j4iBO1OBpPso/PkkURsRE3bmeGllKtTyStKsVp7Tbhnk31nn5bmfZpYrDjUzyxWHWse4vtINsNT8O+ukfE3NzHLFPTUzyxWHmpnlikOtjCRNkLRI0mJJl1S6PdY2STdIWilpfqXbYjvGoVYmkqqB64CJwAjgHEkjKtsqS+BGoEMHi1r7cqiVzzHA4ohYEhF1wO3A6RVuk7UhImYAayrdDttxDrXyGQIsLVlfVtxmZmXkUCuf5mb8evyMWZk51MpnGbBvyfo+wPIKtcWsy3Colc9MYLikoZK6A5OAaRVuk1nuOdTKJCLqgcnA/cAzwB0RsaCyrbK2SLoN+AtwiKRlkj5X6TZZOp4mZWa54p6ameWKQ83McsWhZma54lAzs1xxqJlZrjjUOhFJmyXNlTRf0p2Seu5EXTdKOrv4+eetTbaXdLKk43fgGC9JavLWoZa2NyqzIeWxvinpq2nbaPnjUOtcNkXE6IgYCdQBF5XuLD4ZJLWI+HxELGylyMlA6lAzqwSHWuf1OHBQsRf1iKRfA09LqpZ0laSZkuZJuhBABT+WtFDSvcCeWyqS9KikscXPEyTNkfSUpIckHUAhPL9c7CWeKGmgpKnFY8yUdELxu3tIekDSPyT9lATv8Zb0O0mzJS2QdEGjfVcX2/KQpIHFbQdKuq/4ncclHdouf5qWG35DeyckqRuF57TdV9x0DDAyIl4sBsMbEXG0pF2BP0t6AHgPcAgwCtgLWAjc0KjegcDPgPHFunaPiDWSpgAbIuIHxXK/Bv5vRDwhaT8KsyYOAy4DnoiIyyX9E7BdSLXgs8Vj7AbMlDQ1IlYDvYA5EfHvki4t1j2ZwgtRLoqI5yUdC/wEeP8O/DFaTjnUOpfdJM0tfn4c+AWF08K/R8SLxe0fBI7Ycr0M6AcMB8YDt0XEZmC5pIebqf84YMaWuiKipeeKfQAYIW3tiPWV1Kd4jDOL371X0toEP9PFks4oft632NbVQAPwm+L2W4C7JPUu/rx3lhx71wTHsC7Eoda5bIqI0aUbin+5N5ZuAr4UEfc3KncabT/6SAnKQOGyxbiI2NRMWxLPu5N0MoWAHBcRb0l6FOjRQvEoHndd4z8Ds1K+ppY/9wNfkLQLgKSDJfUCZgCTitfcBgHva+a7fwFOkjS0+N3di9vXA31Kyj1A4VSQYrnRxY8zgHOL2yYCA9poaz9gbTHQDqXQU9yiCtjS2/wUhdPaN4EXJX28eAxJOrKNY1gX41DLn59TuF42p/jykJ9S6JHfDTwPPA38P+Cxxl+MiFUUroPdJekptp3+/R44Y8uNAuBiYGzxRsRCtt2F/RYwXtIcCqfBr7TR1vuAbpLmAd8G/lqybyNwuKTZFK6ZXV7cfi7wuWL7FuBHpFsjfkqHmeWKe2pmlisONTPLFYeameWKQ83McsWhZma54lAzs1xxqJlZrvx/bSXrDDH2besAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammWord(clf_lr_word, model_name=\"clf_lr_word\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "4zOB_W-qOETN"
   },
   "outputs": [],
   "source": [
    "grid_search_lr = GridSearchCV(clf_lr, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "gZn55sF2OEfL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  9.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 9.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.9min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.9min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed: 12.0min finished\n"
     ]
    }
   ],
   "source": [
    "clf_lr_char= grid_search_lr.fit(X_train_char, y_train_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "-Nhyi2zVOEqU"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.9982736296935693\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9974    0.9974    0.9974      3114\n",
      "           1     0.9987    0.9987    0.9987      6154\n",
      "\n",
      "    accuracy                         0.9983      9268\n",
      "   macro avg     0.9981    0.9981    0.9981      9268\n",
      "weighted avg     0.9983    0.9983    0.9983      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAYBklEQVR4nO3de5QV5b3m8e/T3SAYFFRAAnghxrtERcAL0Rg9JpCsOR6jx+AlM+Yy6hjjrJxM1LiMJjEnOkZPNIrD4XhY6niEg5EkRBlBE6NGNOGiKGBIEBW5ROiIRqNRu/3NH7saN2337irYu3d19fNx1Vq7ar/11rtx8fDW5X1LEYGZWVE01LsBZmbV5FAzs0JxqJlZoTjUzKxQHGpmVihN9W5AOTX1D/Xdqd7NsAwOP3DPejfBMnjxxRdobm7W9tTRuPNeES1vpSobb22aFxETt+d4WeUr1PruxA77n17vZlgGj/325no3wTKYcOTY7a4jWt5K/ff0b09NGbzdB8woV6FmZj2BQPm9cuVQM7NsBDQ01rsVnXKomVl22q7LcjXlUDOzjHz6aWZF456amRWGcE/NzIpE7qmZWcH47qeZFYdvFJhZkQiffppZwbinZmbF4dNPMysSAY2+UWBmReJramZWHD79NLOicU/NzArFPTUzKwx5mJSZFY2HSZlZcfhGgZkVjU8/zawwPJ+amRWLTz/NrGh8o8DMCsXX1MysMOTTTzMrGvfUzKxI5FAzs6IozebtUDOzopBQg0PNzArEPTUzKxSHmpkVikPNzIpDyZJTDjUzy0TIPTUzK5aGBo8oMLMCcU/NzIrD19TMrGjy3FPL74mxmeVS242CNEuXdUkTJa2UtErSpR18P1DSLyQtlbRc0he7qtM9NTPLrBrDpCQ1AlOAk4C1wEJJcyJiRVmxrwIrIuK/SBoCrJT0HxHxTmf1uqdmZtmIavXUxgOrImJ1ElIzgZPblQlgJ5UqGwC8ArRUqtQ9NTPLLMM1tcGSFpWtT4uIacnnEcBLZd+tBY5st//NwBxgPbAT8PmIeK/SAR1qZpZZhlBrjoixnVXTwbZot/5p4CngBGAf4AFJj0bEXzo7oE8/zSyTKt4oWAvsUbY+klKPrNwXgdlRsgp4HjigUqUONTPLTimXyhYC+0oaJakvMJnSqWa5NcCJAJJ2B/YHVleq1KefZpaNqjNMKiJaJF0IzAMagekRsVzS+cn3U4GrgNskPVM6MpdERHOleh1qZpZZtR6+jYi5wNx226aWfV4PfCpLnQ41M8suvwMKHGq1dNO3z+LTHz+E5s2vc8zkH9S7Ob3KgwtW8K3rf0Lre+/xhZOP4evnbP2PfURw6fU/4YHHltO/X19uufILHHrAHhX3/faNP2Xeo8vo06eRUSMHM+WKsxm4044ALPvjOv7p6hm8/sbfUIP41e0X02+HPt37o7tRrx0m1dUQiKKbce8TnHbRlHo3o9dpbX2Pb147i7tvvIAnZl3OPfMX8/vVG7Yq88CCFTy3ZhOLZ1/JDZedwTeumdnlvp888gAWzLyMx2Zcxj57DuVfbpsPQEtLK+ddcTvXXzqZx2ddzr1T/yd9mhq790d3o7R3PusVfDULtbIhEJOAg4AzJB1Uq+Pl0YInn2PzX96sdzN6ncXLX+Ajewxm75GD6dunic+dNIa5Dz+9VZm5Dz/N5M+ORxLjRo/itdff4k/Nr1Xc94SjDqQpCatxh4xi/cuvAvCr3/6egz86gtH7jQRg10EDaGws9oMFvTLUSDcEwqzqNmx6jRG777Jlffjuu7Bh02vtyry6dZmhg9iw8dVU+wLcOedx/u6Y0r/Rz724EQlO/drNfOLsa7jxjgeq/ZNyRw1KtdRDLa+ppRkCgaRzgXMB6DOghs2x3iKi/UPp0L7T0EERJKXa97rp99PU1MDpk8YB0NLayhNLV/Or279J/359+YcLfsxhB+zJJ8bvv82/Ie966zW1NEMgiIhpETE2IsaqqX8Nm2O9xfChg1j38uYt6+tf3sywwQMrl9n4KsOGDOxy3xn3PsH83yxj2lXnbPmLPXz3QUw4/KPsNmgAO/bry0nHHMzSleX/nhdM9Qa010QtQy3NEAizqhtz0F48t2YTL65r5p13W5j9wBImHfexrcpMOm40M+/7HRHBwmeeZ+cB/Rk2eGDFfR9csIIb73iQu64/jx379d1S14lHHcTyVet482/v0NLSymNLVrH/qGHd+pu7kyj1XtMs9VDL088tQyCAdZSGQJxZw+Plzq3fP4cJR+zLboMGsOzeq7hm2lzunPN4vZtVeE1NjVx78emcetEUWluDs/7+KA7c58NMv+dRAL506rF8asLBPPDYcsac8l369+vDlCvOrrgvwMU/nMXb77RwyldvBmDs6L350bfOYNDOO3LBmSdw4n+9FiROmnAwn/74IfX58d0i32+TUkfXEKpWufQZ4AbeHwLxz5XKN+w4NHbY//Satceqb/PCm+vdBMtgwpFjWbx40XYlUr9h+8Ve/+2mVGX/cO3ExRVm6aiJmj5829EQCDPr4ep4apmGRxSYWSYCGur0uEYaDjUzy8w9NTMrlDzfKHComVk2vqZmZkUiVJVJImvFoWZmmbmnZmaF4mtqZlYcvqZmZkVSGvuZ31RzqJlZZjnONIeamWXnEQVmVhzy6aeZFUjbfGp55VAzs4zyPZ+aQ83MMstxpjnUzCwj+UaBmRWIn1Mzs8JxqJlZoeQ40xxqZpade2pmVhwe0G5mRVKaJDK/qeZQM7PMGnLcVcvvnLxmlltSuqXrejRR0kpJqyRd2kmZ4yU9JWm5pIe7qtM9NTPLRFUa0C6pEZgCnASsBRZKmhMRK8rKDAJuASZGxBpJQ7uq1z01M8usQemWLowHVkXE6oh4B5gJnNyuzJnA7IhYAxARG7uqtNOemqSbgOjs+4i4qMsmm1khZbhRMFjSorL1aRExLfk8Anip7Lu1wJHt9t8P6CPp18BOwI0RcUelA1Y6/VxU4Tsz66VE6Q5oSs0RMbZCVe2170g1AUcAJwL9gcclPRERf+jsgJ2GWkTcvtXRpQ9FxF87K29mvUeVnuhYC+xRtj4SWN9BmeYke/4q6RHgUKDTUOvympqkoyWtAJ5N1g+VdEvGxptZUag0n1qapQsLgX0ljZLUF5gMzGlX5ufAsZKaJO1I6fT02UqVprn7eQPw6baDRcRSScel2M/MCqoaj6lFRIukC4F5QCMwPSKWSzo/+X5qRDwr6X7gaeA94NaIWFap3lSPdETES+1St3VbfoSZ9Xyieg/fRsRcYG67bVPbrf8Q+GHaOtOE2kuSjgEi6SJeRBfdPzMrtjwPk0rznNr5wFcp3X5dBxyWrJtZL5R2NEG9RlJ12VOLiGbgrG5oi5n1ED167Kekj0j6haRNkjZK+rmkj3RH48wsn5RyqYc0p593AbOADwPDgbuBGbVslJnlW5Ue6aiJNKGmiPi/EdGSLHdSYfiUmRVb6e5nVcZ+1kSlsZ+7Jh8fSqYEmUkpzD4P3NcNbTOzPFLPnSRyMaUQa2v9eWXfBXBVrRplZvnWI99REBGjurMhZtYztJ1+5lWqEQWSDgEOAvq1betq+g8zK64e2VNrI+lK4HhKoTYXmAT8BnComfVS+Y20dHc/T6M0l9GfIuKLlKb92KGmrTKz3JKgsUGplnpIc/r5VkS8J6lF0s7ARsAP35r1Yj369BNYlLz84N8o3RF9A/hdLRtlZvmW40xLNfbzguTj1GReo50j4unaNsvM8koo12M/Kz18O6bSdxGxpDZNMrNcq+MMHGlU6qldX+G7AE6ocls4/MA9eey3N1e7WquhXcZdWO8mWAZvr1xTlXp65DW1iPhkdzbEzHoGAY09MdTMzDrT40cUmJmVc6iZWWGUpurOb6qlmflWks6WdEWyvqek8bVvmpnlVZ7nU0szTOoW4GjgjGT9dWBKzVpkZrnXo1+8AhwZEWMkPQkQEZuTV+WZWS8koCnHp59pQu1dSY0kU3hLGkLpTclm1kvlONNShdqPgZ8CQyX9M6VZOy6vaavMLLekHjpMqk1E/IekxZSmHxLwDxHhN7Sb9WI5zrRUk0TuCbwJ/KJ8W0RUZ7yFmfU4Pf05tft4/wUs/YBRwErg4Bq2y8xySlC3CSDTSHP6Obp8PZm947xOiptZ0dXxGbQ0Mo8oiIglksbVojFm1jMox28pSHNN7Z/KVhuAMcCmmrXIzHKtCK/I26nscwula2z31KY5ZtYT9NhQSx66HRAR3+ym9phZD5DnAe2VpvNuioiWStN6m1nvU3pFXr1b0blKTWt7Y9RTkuZI+oKkz7Ut3dE4M8unhmRUQVdLVyRNlLRS0ipJl1YoN05Sq6TTuqozzTW1XYE/U3onQdvzagHMTrGvmRVMtW4UJJe3pgAnAWuBhZLmRMSKDsr9b2BemnorhdrQ5M7nMt4PszaRoe1mVjBVuqQ2HlgVEatLdWomcDKwol25r1G6OZnqUbJKodYIDIAOH0hxqJn1WqIh/XNqgyUtKlufFhHTks8jgJfKvlsLHLnVkaQRwCmUzhS3O9Q2RMT30lRiZr2HyNRTa46IsRWqaq99h+kG4JKIaE17x7VSqOX3nq2Z1Y+gqToPqq0F9ihbHwmsb1dmLDAzCbTBwGcktUTEzzqrtFKonbht7TSzIsvYU6tkIbCvpFHAOmAycGZ5gYgYteW40m3AvZUCDSq/zPiV7WismRVYNSaJTJ6DvZDSXc1GYHpELJd0fvL91G2p16/IM7PMqjWgICLmAnPbbeswzCLinDR1OtTMLBOR7jV09eJQM7NsVJ3Tz1pxqJlZJqURBQ41MyuQ/EaaQ83MtkGOO2oONTPLSj1zPjUzs4747qeZFY5vFJhZcaiHTudtZtYRn36aWeG4p2ZmhZLfSHOomVlGAhrdUzOzIslxpjnUzCwroRyfgDrUzCwz99TMrDBKj3TkN9UcamaWjdxTM7OC8TApMyuM0iSR9W5F5xxqZpaZ736aWaHk+Owz1+NSc+PBBSsYd+r3GHPKd/jRbfM/8H1EcMl1dzPmlO8w4YwfsPT3L3W5788eXMLRp3+fXcd/jSdXvLhl++LlL3DsmVdz7JlX8/Ezr+beh5bW9sfZVm769ln8Yd7VLJh5Wb2bkmtK+V891CzUJE2XtFHSslodozu0tr7HN6+dxd03XsATsy7nnvmL+f3qDVuVeWDBCp5bs4nFs6/khsvO4BvXzOxy3wP3Gc4d1/53jjl8n63qOnCf4Tx0x8U8ete3+MmPL+DrV8+gpaW1e36sMePeJzjtoin1bkautV1TS7PUQy17arcBE2tYf7dYvPwFPrLHYPYeOZi+fZr43EljmPvw01uVmfvw00z+7HgkMW70KF57/S3+1PxaxX33HzWMfffe/QPH27FfX5qaGgF4++13cz0bQhEtePI5Nv/lzXo3I98kGlIu9VCzUIuIR4BXalV/d9mw6TVG7L7LlvXhu+/Chk2vtSvz6tZlhg5iw8ZXU+3bkUXLXuDo07/PhDN+wL9cOnlLyJnlhVIu9VD3a2qSzpW0SNKiTc2b6t2cD4iID2xr/w9QB0WQlGrfjow9ZG8en3U5v7z9Yn5023z+9va7aZtrVnNt7/3sdT21tCJiWkSMjYixQwYPqXdzPmD40EGse3nzlvX1L29m2OCBlctsfJVhQwam2reS/UcNY8f+fXn2ufXb8QvMqs89tR5szEF78dyaTby4rpl33m1h9gNLmHTcx7YqM+m40cy873dEBAufeZ6dB/Rn2OCBqfZt78V1zVtuDKzZ8AqrXnyZPYfvVrPfZ7ZNcpxqfk6tC01NjVx78emcetEUWluDs/7+KA7c58NMv+dRAL506rF8asLBPPDYcsac8l369+vDlCvOrrgvwL0PLeWS6+6mefMbfP7rUxm93wjuuelCHl+6mhtvm09TUyMNDeK6Sz7PboMG1O339za3fv8cJhyxL7sNGsCye6/immlzuXPO4/VuVu7keZiUOrruU5WKpRnA8cBg4GXgyoj490r7HHHE2Hjst4tq0h6rjV3GXVjvJlgGb6+cxXtvbtyuRDpw9OFxx89/nars+H0GLY6IsdtzvKxq1lOLiDNqVbeZ1Vl+O2o+/TSzbEqXy/Kbag41M8sm5/Op+e6nmWVWrZufkiZKWilplaRLO/j+LElPJ8sCSYd2Vad7amaWkaoyfE9SIzAFOAlYCyyUNCciVpQVex74RERsljQJmAYcWaleh5qZZVal08/xwKqIWF2qUzOBk4EtoRYRC8rKPwGM7KpSn36aWSZpTz2T3BvcNgwyWc4tq2oE8FLZ+tpkW2e+DPy/rtrnnpqZZZe+p9Zc4Tm1jmrp8MFZSZ+kFGof7+qADjUzy6xKj3SsBfYoWx8JfGCgs6SPAbcCkyLiz11V6tNPM8tMSrd0YSGwr6RRkvoCk4E5Wx9HewKzgS9ExB/StM09NTPLpkrPqUVEi6QLgXlAIzA9IpZLOj/5fipwBbAbcEtyx7Wlq2FXDjUzy6xaIwoiYi4wt922qWWfvwJ8JUudDjUzy0Tke0SBQ83MMstxpjnUzGwb5DjVHGpmllmeJ4l0qJlZZvmNNIeamW2LHKeaQ83MMvEkkWZWLDmfJNKhZmaZ5TjTHGpmllV1JomsFYeamWWW40xzqJlZNnV8+XoqDjUzyy7HqeZQM7PM/EiHmRWKr6mZWXEIGhxqZlYs+U01h5qZZeJJIs2scHKcaQ41M8vOPTUzKxQPkzKzQslvpDnUzCyjlC8qrhuHmpll5hEFZlYs+c00h5qZZZfjTHOomVlW8ivyzKw48j6ioKHeDTAzqyb31Mwsszz31BxqZpaZH+kws+Lww7dmViR5v1HgUDOzzHz6aWaF4p6amRVKjjPNoWZm2yDHqeZQM7NMBLkeJqWIqHcbtpC0CXix3u2ogcFAc70bYZkU9f/ZXhExZHsqkHQ/pT+fNJojYuL2HC+rXIVaUUlaFBFj690OS8//z3ouj/00s0JxqJlZoTjUuse0ejfAMvP/sx7K19TMrFDcUzOzQnGomVmhONRqSNJESSslrZJ0ab3bY12TNF3SRknL6t0W2zYOtRqR1AhMASYBBwFnSDqovq2yFG4DuvVhUasuh1rtjAdWRcTqiHgHmAmcXOc2WRci4hHglXq3w7adQ612RgAvla2vTbaZWQ051GqnoxG/fn7GrMYcarWzFtijbH0ksL5ObTHrNRxqtbMQ2FfSKEl9gcnAnDq3yazwHGo1EhEtwIXAPOBZYFZELK9vq6wrkmYAjwP7S1or6cv1bpNl42FSZlYo7qmZWaE41MysUBxqZlYoDjUzKxSHmpkVikOtB5HUKukpScsk3S1px+2o6zZJpyWfb6002F7S8ZKO2YZjvCDpA28d6mx7uzJvZDzWdyT9r6xttOJxqPUsb0XEYRFxCPAOcH75l8nMIJlFxFciYkWFIscDmUPNrB4caj3Xo8BHk17UQ5LuAp6R1Cjph5IWSnpa0nkAKrlZ0gpJ9wFD2yqS9GtJY5PPEyUtkbRU0i8l7U0pPL+e9BKPlTRE0j3JMRZKmpDsu5uk+ZKelPSvpHiPt6SfSVosabmkc9t9d33Sll9KGpJs20fS/ck+j0o6oCp/mlYYfkN7DySpidI8bfcnm8YDh0TE80kwvBYR4yTtADwmaT5wOLA/MBrYHVgBTG9X7xDg34Djkrp2jYhXJE0F3oiI65JydwE/iojfSNqT0qiJA4Ergd9ExPckfRbYKqQ68aXkGP2BhZLuiYg/Ax8ClkTENyRdkdR9IaUXopwfEX+UdCRwC3DCNvwxWkE51HqW/pKeSj4/Cvw7pdPC30XE88n2TwEfa7teBgwE9gWOA2ZERCuwXtKvOqj/KOCRtroiorN5xf4OOEja0hHbWdJOyTE+l+x7n6TNKX7TRZJOST7vkbT1z8B7wH8m2+8EZksakPzeu8uOvUOKY1gv4lDrWd6KiMPKNyR/uf9avgn4WkTMa1fuM3Q99ZFSlIHSZYujI+KtDtqSetydpOMpBeTREfGmpF8D/TopHslxX23/Z2BWztfUimce8D8k9QGQtJ+kDwGPAJOTa24fBj7Zwb6PA5+QNCrZd9dk++vATmXl5lM6FSQpd1jy8RHgrGTbJGCXLto6ENicBNoBlHqKbRqAtt7mmZROa/8CPC/pH5NjSNKhXRzDehmHWvHcSul62ZLk5SH/SqlH/lPgj8AzwP8BHm6/Y0RsonQdbLakpbx/+vcL4JS2GwXARcDY5EbECt6/C/td4DhJSyidBq/poq33A02SngauAp4o++6vwMGSFlO6Zva9ZPtZwJeT9i3HU6RbO56lw8wKxT01MysUh5qZFYpDzcwKxaFmZoXiUDOzQnGomVmhONTMrFD+P1bfV3fPUC5AAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.701409869083585\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5375    0.5575    0.5473      1286\n",
      "           1     0.7843    0.7703    0.7772      2686\n",
      "\n",
      "    accuracy                         0.7014      3972\n",
      "   macro avg     0.6609    0.6639    0.6623      3972\n",
      "weighted avg     0.7044    0.7014    0.7028      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAalElEQVR4nO3deZRV1Z328e9TVYBMMlgoOCEqDgQUFSWSiCYOgSQdo2bQpDvdMWk1CXF10hl0rbc76aQzmsFONE1MYhtj1GjjQBqUOAZjTMIQVECJiAOIyIyAIFTV7/3j3oJLjedA3bqnTj0f1l3r3nP23WcXtXjYZ9h7KyIwM8uLqko3wMysIznUzCxXHGpmlisONTPLFYeameVKTaUbUKpnv4HR+4BhlW6GpTCwT49KN8FSWPfqCrZsXK99qaN6/+ERddsSlY1ta2ZFxKR9OV5amQq13gcM4/Qv31TpZlgK7z/poEo3wVL41qXv2+c6om4bvY79UKKy2xdcX7vPB0wpU6FmZl2BQNm9cuVQM7N0BFRVV7oVrXKomVl62qfLcmXlUDOzlHz6aWZ5456ameWGcE/NzPJE7qmZWc747qeZ5YdvFJhZngiffppZzrinZmb54dNPM8sTAdW+UWBmeeJramaWHz79NLO8cU/NzHLFPTUzyw15mJSZ5Y2HSZlZfvhGgZnljU8/zSw3PJ+ameVLtk8/s9syM8uuqupkr3ZImiRpiaSlkq5qYf8XJS0ovhZKqpc0uM2m7cOPZWbdVeNjHe292qxC1cD1wGRgFHCJpFGlZSLimogYGxFjgauB30fE+rbqdaiZWToqnn4mebXtNGBpRCyLiB3A7cD5bZS/BLitvUodamaWXgf01IBDgOUln1cUt7VwOPUBJgHT2qvUNwrMLDUlf6SjVtLcks83RMQNjdW0UD5aqefvgMfbO/UEh5qZpVSYzTtxqK2NiHGt7FsBHFby+VBgZStlLybBqSf49NPM0pJQVbJXO+YAIyWNkNSTQnBNb344DQDOBO5N0jz31MwstRQ9tVZFRJ2kKcAsoBq4MSIWSbqiuH9qsegFwO8iYmuSeh1qZpZaR4QaQETMBGY22Ta1yeebgJuS1ulQM7PUOirUysGhZmbpiJbvW2aEQ83MUhFyT83M8qWqKrsPTjjUzCw199TMLD98Tc3M8sY9NTPLDd8oMLPcSTAEqmIcamaWjnz6aWY541Azs1xxqJlZbvhGgZnlT3YzzaFmZinJw6TMLGd8+mlm+ZLdTHOo7YuTDh3AJycMp0rigWdXc9eTr+6xf/Sw/lz9rmNY/fqbADzx4gbumP8KAH17VvOZiUdy+ODeRMB1v1/GktVbOv1n6G6eWfQCd9/5MBHB+AljOOdd41ss9/KLr3LtNbfysU+8l7EnH7tre0NDAz/49i0MGNiPf/70hZ3V7Mzptj01SZOA/6Iw//jPI+Lb5TxeZ6oSXP72I/jKjGdZt3UH11zwFv7y0kZWbNy2R7nFr27mG7P+1uz7n5gwnPnLN/LdB5+jpkr0qsnuNYq8aGhoYNpvHuSKKz/IwIH9+eF3bmH0CUcxdFhts3K/vWc2x406olkdsx+Zz0FDB7N9+45OanX2SNm++1m2f0lJlpTvykYO6cerm7bz2uY3qWsI/vD8esYfMSjRd3v3qOYtQ/vz4JI1ANQ1BFt31JezuQa8/OIqaocMorZ2IDU11Zx0ynEsfPL5ZuUee/SvnHjSMfTr32eP7Rs3bGbxwmW89W0ndFaTM6sx2Np7VUI5uwdpl5TvUgb37cnarbv/t163dQeD+/ZoVu7Yg/rxw4tG82+TjuWwQb0BGLp/LzZtr+PKM4/kBxeO5jMTR7in1gk2btzMwEH9d30eMKgfmzZtblbm6QXPMeGME5t9/+7/fZi/u2BigoXH86+Dlsgri3L+S0q0pLykyyTNlTR3x5aNZWxOx2rx19Vkbenn177BZbcu4HPTFjJz0SquPu8YAKokjqrty32LX+Pzdy1k+84GLhp7cNnbbC3Z8zd5z52P8N4LJjZ7ZGHR08/Tv18fDjt8aGc2LrOy3FMr5zW1REvKF5egvwFgwPDjW1tyPnPWbd1Bbd+euz4f0Lcn69/YuUeZbTt3n1LOW76Jy6tE/141rNu6g3Vbd/DcmsIyhk+8sJ4Lxw7rnIZ3YwMH9mfjht09s00btjBgQL89yix/eRU3/+L/ANi6dRvPLFxGdVUVL734Kguffp7Fi16grq6O7dt2cMv/zODvP/6eTv0ZMqEbD2hPs6R8l/Pcmi0MG7AfB/bvxfqtO3j7UYP5wcN7Xp8Z2LsHG7cVgm7kkL5IsPnNOgDWbnmTgwfsx8pN2znhkP1ZvmFbs2NYxzps+FDWrN7AurUbGTCwP3+d92yzUPq3r1+26/2tN9/HqNFHMmbsSMaMHcl73z8RgKV/e5lHHpzbPQON4sS32c20sobariXlgVcoLCn/kTIer1M1BPzs8Rf5yuRjqa4SDy5Zw/IN23jX8QcCMOuZ1Uw4cjCTjj+Q+gh21AXfe2jpru//7I8v8fl3HkVNVRWvbd7Ojx5dVqkfpduorq7iog+fzU+vm0ZDQwPjTx/DsINreXz2AgDeNnFsRdvXdWT77qciynfGJ+ndwLXsXlL+G22VHzD8+Dj9yzeVrT3W8d5/0kGVboKl8K1L38dLzzy1T4m039BjYvg//jhR2b99d9K8iBi3L8dLq6zPqbW0pLyZdXHqvqefZpZDAqo8nbeZ5Yl7amaWK1m+UeBQM7N0fE3NzPJEKNOTRGa3ZWaWWVKyV/v1aJKkJZKWSrqqlTJnSVogaZGk37dXp3tqZpZaR1xTK5nJ51wKI5DmSJoeEYtLygwEfgJMioiXJR3YXr3uqZlZOgl7aQlyL8lMPh8B7oqIlwEiYnV7lTrUzCyVwtjPxLN01DbOwlN8XVZSVZKZfI4BBkl6VNI8SR9rr30+/TSz1FKcfa5tY5hUkpl8aoBTgLOB3sATkv4UEc2nky75gplZKh00oiDJTD4rKATjVmCrpNnAiUCroebTTzNLRx02SeSumXwk9aQwk8/0JmXuBc6QVCOpDzAeeKatSt1TM7NUOmo+tYiokzQFmMXumXwWSbqiuH9qRDwj6X7gKaCBwgJOC9uq16FmZil13HxqLc3kExFTm3y+BrgmaZ0ONTNLzcOkzCw/5KmHzCxHGp9TyyqHmpml5lAzs1zJcKY51MwsPffUzCw/PEmkmeVJYZLI7KaaQ83MUqvKcFfNoWZmqWU40xxqZpaO5BsFZpYzGb6k1nqoSfoxzSds2yUirixLi8ws87rqjYK5ndYKM+syROEOaFa1GmoR8cvSz5L6FmefNLNuLsMdtfZnvpV0uqTFFGeblHSipJ+UvWVmlk0JZ72t1M2EJNN5Xwu8C1gHEBFPAhPL2CYzy7iOWsy4HBLd/YyI5U1St748zTGzrBNd/+Hb5ZImAFFcHOFK2ln4wMzyLct3P5Ocfl4BfIbCIqOvAGOLn82sG0p66pnZ08+IWAt8tBPaYmZdRJZPP5Pc/TxS0m8lrZG0WtK9ko7sjMaZWTYp4asSkpx+3grcAQwDDgbuBG4rZ6PMLNu6+iMdiohfRURd8XULbQyfMrN8K9z9TPaqhLbGfg4uvn1E0lXA7RTC7MPAjE5om5llkbruJJHzKIRYY+svL9kXwNfL1Sgzy7YuOfVQRIzozIaYWdfQePqZVYlGFEgaDYwC9mvcFhE3l6tRZpZtXbKn1kjSV4CzKITaTGAy8AfAoWbWTWU30pLd/fwAcDawKiI+DpwI9Cprq8wssySorlKiVyUkOf3cFhENkuok7Q+sBvzwrVk3luXTzyQ9tbmSBgI/o3BHdD7wl3I2ysyyraPGfkqaJGmJpKXFR8ea7j9L0iZJC4qvf2+vziRjPz9dfDtV0v3A/hHxVPvNNbM8EuqQsZ+SqoHrgXOBFcAcSdMjYnGToo9FxHuT1tvWw7cnt7UvIuYnPYiZ5UjHzcBxGrA0IpYBSLodOB9oGmqptNVT+34b+wJ4574cuCVH1/blnsvGd3S1VkaDTp1S6SZYCm++tKpD6klxTa1WUukiTjdExA3F94cAy0v2rQBaCoDTJT0JrAS+EBGL2jpgWw/fviNZm82sOxFQnTzU1kbEuDaqaqrpuPL5wPCI2CLp3cA9wMi2DpjkRoGZ2R46aED7CuCwks+HUuiN7RIRr0fEluL7mUAPSbVtti31T2Nm3V4HhdocYKSkEcWlAi4GppcWkDRUxXNdSadRyKx1bVWaaJiUmVmjwuMa+36nICLqJE0BZgHVwI0RsUjSFcX9Uyk8/P8pSXXANuDiiGhz6rMkw6REYTrvIyPia5IOB4ZGhJ9VM+umOmqwQPGUcmaTbVNL3l8HXJeqbQnK/AQ4Hbik+HkzhWdLzKyb6tILrwDjI+JkSX8FiIgNxfNfM+uGBNRkeJhUklDbWXzyNwAkDQEaytoqM8u0DGdaolD7EXA3cKCkb1C4cPf/ytoqM8ssqWOGSZVLkrGfv5Y0j8L0QwLeHxFeod2sG8twpiW6+3k48Abw29JtEfFyORtmZtnV1afznsHuBVj2A0YAS4C3lLFdZpZRgopNAJlEktPPMaWfi7N3XN5KcTPLuwqu6ZlE6hEFETFf0qnlaIyZdQ3K8CoFSa6pfb7kYxVwMrCmbC0ys0zLwxJ5/Uve11G4xjatPM0xs66gy4Za8aHbfhHxxU5qj5l1AVleeKWt6bxriqPoW53W28y6n8ISeZVuReva6qn9hcL1swWSpgN3Alsbd0bEXWVum5llVJceUQAMpjAp2zvZ/bxaAA41s26oK98oOLB453Mhu8OsUZuTtJlZvmW4o9ZmqFUD/Ui2OIKZdRuiqos+p/ZqRHyt01piZl2C6Lo9tQw328wqRlCT4YtqbYXa2Z3WCjPrMrpsTy0i1ndmQ8ys6+jqj3SYme0hw5nmUDOzdES2V0F3qJlZOvLpp5nlSGFEgUPNzHIku5HmUDOzvZDhjppDzczSUtecT83MrCW++2lmueMbBWaWH8r2dN5Z7kWaWQY1nn4mebVblzRJ0hJJSyVd1Ua5UyXVS/pAe3U61MwsNUmJXu3UUQ1cD0wGRgGXSBrVSrnvALOStM2hZmapKeGrHacBSyNiWUTsAG4Hzm+h3GcpLMu5OknbfE3NzFIRUJ38mlqtpLkln2+IiBuK7w8BlpfsWwGM3+NY0iHABRTWSDk1yQEdamaWWor7BGsjYlxr1bSwrelSAdcCX46I+qQ3JxxqZpaSUMcMlFoBHFby+VBgZZMy44Dbi4FWC7xbUl1E3NNapQ41M0utg57omAOMlDQCeAW4GPhIaYGIGLH7mLoJ+L+2Ag0camaWUuGRjn1PtYiokzSFwl3NauDGiFgk6Yri/ql7U69DzczSUccNaI+ImcDMJttaDLOI+KckdTrUzCw1D5Mys9woTBJZ6Va0zqFmZql10N3PsnComVlqGT77dKjtiwf/uJirv/+/1Dc08A/nT+Bz/3TeHvvvuG8O/3XzAwD07d2L71/1YcYccyjb39zJey67ljd31lFfV8/7zj6Jqy9/TyV+hG7n7NOP51v/+gGqq6r41b1/5NpfPrDH/s/+/dl8cHLhwfWa6iqOOWIoR593FbWD+nHjNy/dVW74wQfwrRtmMPW2Rzuz+ZnRLXtqkm4E3gusjojR5TpOpdTXN/DF797B3ddN4eCDBvLOf7yGyRPHcNyRw3aVGX7wAcz46b8wcP8+PPD4Ij73zdt48KYv0qtnDff+95X069OLnXX1TP7kDzhnwihOHTOijSPavqqqEtd86UNcMOU6Vr62kYd/+UXum/00S15YtavMj295iB/f8hAAk84YzacueQcbX3+Dja+/wcSPfntXPYtnfoMZjzxZkZ+j0rJ+Ta2cA9pvAiaVsf6KmrfoRY48rJYjDq2lZ48aLjz3ZGb+/qk9yow/8UgG7t8HgFPHjGDl6o1AYYaDfn16AbCzrp6ddfWZnp8qL055yxEsW76Wl15Zx866eu56YD7vPvOEVstfdN44pv1uXrPtZ556LC+uWMPyVRvK2dzskqhK+KqEsoVaRMwG1per/kp7dc0mDjlo0K7PBx80iFfXbGq1/K/u/SPnTNg9q0p9fQNnfORbHHPeVZw1/jjGjT6inM01YNiQAbzy2u4gWvnaBoYNGdBi2d69enD26ccz/eEFzfZdeN4pTJvVPOy6kw6apaMsKj71kKTLJM2VNHfN2jWVbk5iEU3H3bZ+8fSxuX/jlulP8NUpu2dVqa6u4rFbr2bRjP9k/qKXWLy06ZA362gt9YZb+DUCMGniGP781DI2vv7GHtt71FQzeeIY7nnor+VoYpfQuO5nt+upJRURN0TEuIgYN6R2SKWbk9jBBw5s9r/+0Nrm/+svfO4VrvzPW/n19y5j8MB+zfYP6N+Ht58ykoeeWFzW9hqsXL2xWe961dqWe9cXnttyb+ycCaN48tnlrFm/uWzt7ArcU8uhk0cN5/mX1/DSK2vZsbOOux6Yz+SJe16fWb5qPR/70s+Y+h8f4+jhB+3avnbDZjZtLvQAtm3fwaN/WcLIIw7Cymv+4pc46vAhHH7wAfSoqebCc0/mvtlPNSu3f9/9eNvJRze7RgrwgXe1fJ2t28lwqvmRjr1UU1PNd7/0IS668nrq64OPvu+tHH/UMG6c9hgAl150Btf8/D7Wb9rKF77zm+J3qnjk5i+zau3rfPqrv6K+oYGGhuCCc05m0hljKvnjdAv19Q186bt3MO1Hn6G6Wvx6+p94dtkqPn7h2wH4n7v+AMB73nEij/z5Wd7YvmOP7/fu1YOzTjuOz33ztk5ve9ZkeZiUWro21CEVS7cBZ1GYA+k14CsR8Yu2vnPKKePi8T/PbauIZcygU6dUugmWwptL7qDhjdX7lEjHjzkpbr730URlTztq4Lw2Joksi7L11CLiknLVbWYVlt2Omk8/zSydwuWy7KaaQ83M0unA+dTKwaFmZqllONMcamaWVvsLFVeSQ83MUstwpjnUzCydSo4WSMKhZmbpZTjVHGpmlpof6TCzXPE1NTPLDz+nZmZ549NPM8sN4Z6ameVMhjPNoWZmeyHDqeZQM7PUsjxJpEPNzFLLbqQ51Mxsb2Q41bzwipml0jhJZJI/7dYlTZK0RNJSSVe1sP98SU9JWlBcSvPt7dXpnpqZpdNBD99KqgauB84FVgBzJE2PiNL1Ih8CpkdESDoBuAM4rq163VMzs9Q6aIW804ClEbEsInYAtwPnlxaIiC2xe3WovkC7K0U51MwspcIkkUle7TgEWF7yeUVx255Hky6Q9CwwA7i0vUodamaWmpTsBdQWr4U1vi4rraaFqpv1xCLi7og4Dng/8PX22uZramaWSspJIte2se7nCuCwks+HAitbqygiZks6SlJtRKxtrZx7amaWXsdcVJsDjJQ0QlJP4GJg+h6HkY5W8TxW0slAT2BdW5W6p2ZmqXXELB0RUSdpCjALqAZujIhFkq4o7p8KXAR8TNJOYBvw4ZIbBy1yqJlZah01SioiZgIzm2ybWvL+O8B30tTpUDOzdARVGR5R4FAzs72Q3VRzqJlZKp4k0sxyJ8OZ5lAzs/TcUzOzXEkwBKpiHGpmllp2I82hZmYplYzrzCSHmpml5nU/zSxfsptpDjUzSy/DmeZQM7O05CXyzCw/sj6iwPOpmVmuuKdmZqlluafmUDOz1PxIh5nlhx++NbM8yfqNAoeamaXm008zyxX31MwsVzKcaQ41M9sLGU41h5qZpSLI9DAptbMuaKeStAZ4qdLtKINaYG2lG2Gp5PV3NjwihuxLBZLup/D3k8TaiJi0L8dLK1OhlleS5kbEuEq3w5Lz76zr8thPM8sVh5qZ5YpDrXPcUOkGWGr+nXVRvqZmZrninpqZ5YpDzcxyxaFWRpImSVoiaamkqyrdHmufpBslrZa0sNJtsb3jUCsTSdXA9cBkYBRwiaRRlW2VJXAT0KkPi1rHcqiVz2nA0ohYFhE7gNuB8yvcJmtHRMwG1le6Hbb3HGrlcwiwvOTziuI2Mysjh1r5tDTi18/PmJWZQ618VgCHlXw+FFhZobaYdRsOtfKZA4yUNEJST+BiYHqF22SWew61MomIOmAKMAt4BrgjIhZVtlXWHkm3AU8Ax0paIekTlW6TpeNhUmaWK+6pmVmuONTMLFccamaWKw41M8sVh5qZ5YpDrQuRVC9pgaSFku6U1Gcf6rpJ0geK73/e1mB7SWdJmrAXx3hRUrNVh1rb3qTMlpTH+qqkL6Rto+WPQ61r2RYRYyNiNLADuKJ0Z3FmkNQi4pMRsbiNImcBqUPNrBIcal3XY8DRxV7UI5JuBZ6WVC3pGklzJD0l6XIAFVwnabGkGcCBjRVJelTSuOL7SZLmS3pS0kOSjqAQnp8r9hLPkDRE0rTiMeZIelvxuwdI+p2kv0r6KQnW8ZZ0j6R5khZJuqzJvu8X2/KQpCHFbUdJur/4ncckHdchf5uWG16hvQuSVENhnrb7i5tOA0ZHxAvFYNgUEadK6gU8Lul3wEnAscAY4CBgMXBjk3qHAD8DJhbrGhwR6yVNBbZExPeK5W4FfhgRf5B0OIVRE8cDXwH+EBFfk/QeYI+QasWlxWP0BuZImhYR64C+wPyI+FdJ/16sewqFBVGuiIjnJI0HfgK8cy/+Gi2nHGpdS29JC4rvHwN+QeG08C8R8UJx+3nACY3Xy4ABwEhgInBbRNQDKyU93EL9bwVmN9YVEa3NK3YOMEra1RHbX1L/4jEuLH53hqQNCX6mKyVdUHx/WLGt64AG4DfF7bcAd0nqV/x57yw5dq8Ex7BuxKHWtWyLiLGlG4r/uLeWbgI+GxGzmpR7N+1PfaQEZaBw2eL0iNjWQlsSj7uTdBaFgDw9It6Q9CiwXyvFo3jcjU3/DsxK+Zpa/swCPiWpB4CkYyT1BWYDFxevuQ0D3tHCd58AzpQ0ovjdwcXtm4H+JeV+R+FUkGK5scW3s4GPFrdNBga109YBwIZioB1HoafYqApo7G1+hMJp7evAC5I+WDyGJJ3YzjGsm3Go5c/PKVwvm19cPOSnFHrkdwPPAU8D/w38vukXI2INhetgd0l6kt2nf78FLmi8UQBcCYwr3ohYzO67sP8BTJQ0n8Jp8MvttPV+oEbSU8DXgT+V7NsKvEXSPArXzL5W3P5R4BPF9i3CU6RbE56lw8xyxT01M8sVh5qZ5YpDzcxyxaFmZrniUDOzXHGomVmuONTMLFf+Pxtx6jex/UmoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(clf_lr_char, model_name=\"clf_lr_char\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "WObBkbO13ypx"
   },
   "outputs": [],
   "source": [
    "del clf_lr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GHGngHlA4vUm"
   },
   "source": [
    "### **4.3 Random Forest**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "id": "LIPSDU75EQCq"
   },
   "outputs": [],
   "source": [
    "clf_rf = RandomForestClassifier(n_estimators=500,\n",
    "                                criterion=\"gini\",\n",
    "                                max_depth=None, \n",
    "                                min_samples_split=2,\n",
    "                                min_samples_leaf=10, \n",
    "                                max_features=\"auto\", \n",
    "                                n_jobs=-1,\n",
    "                                verbose=1,\n",
    "                                random_state=0,\n",
    "                                class_weight='balanced')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "-ZW6qLTd-jji"
   },
   "outputs": [],
   "source": [
    "grid_search_rf = GridSearchCV(clf_rf, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_word, y_train_word), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "id": "9EkLq1MmyMha"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   12.8s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   37.3s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  1.5min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 1.6min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    8.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   34.3s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  1.4min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 1.5min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    9.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   36.1s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed: 18.1min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 18.3min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time=18.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   13.3s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   47.2s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.0min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 2.0min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   14.0s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   48.8s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.0min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.7s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 2.1min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   11.2s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   44.7s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  1.9min finished\n"
     ]
    }
   ],
   "source": [
    "clf_rf_word= grid_search_rf.fit(X_train_word, y_train_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "CGlaAofi51l6"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.9s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    2.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    2.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.7905697022011221\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6666    0.7346    0.6989      3067\n",
      "           1     0.8618    0.8183    0.8394      6201\n",
      "\n",
      "    accuracy                         0.7906      9268\n",
      "   macro avg     0.7642    0.7764    0.7692      9268\n",
      "weighted avg     0.7972    0.7906    0.7929      9268\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.9s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    2.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    2.5s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcq0lEQVR4nO3deZxVdf3H8dd7BkRBQGCQXcUlFc0VcUvcstAyU7Nc2ix/poX2+9Uvs9XSfORSv5+lGGn5szJFTUxMXCr3LVlCFBIlXBhQWdxBxZn5/P64d4bLMHPnHLl37pk772eP83jcc873fs/3gr35nuX7PYoIzMyqRU2lG2BmVkoONTOrKg41M6sqDjUzqyoONTOrKj0q3YBC6tU31HtQpZthKey05cBKN8FSWLL4BV5ZuUIbUkdtvy0jGt5OVDbeXn5nREzYkOOlla1Q6z2IXof8oNLNsBRumXxipZtgKRx12P4bXEc0vE2v7T+dqOw7cybVbfABU8pUqJlZVyBQdq9cZbdlZpZNAmpqky0dVSVNkLRA0kJJZ7exv7+kWyU9LmmepJM7qtOhZmbpScmWolWoFpgEHA6MAU6QNKZVsa8B8yNiV+Ag4OeSNipWr0PNzFLKn34mWYobByyMiEURsQaYAhzVqkwAfSUJ2BR4BWgoVqmvqZlZeh30whIaASwuWK8H9m5V5jJgGrAU6At8JiKailXqnpqZpSPS9NTqJM0sWE5tVVNrrWfY+CgwBxgO7AZcJqlfsea5p2ZmKXV8vazAiogY286+emBUwfpIcj2yQicDF0RuOqGFkp4FdgAea++A7qmZWXqlufs5A9hO0uj8xf/jyZ1qFnoBOBRA0hBge2BRsUrdUzOzlErznFpENEiaCNwJ1AJXRcQ8Safl908GzgOulvRE7sB8OyJWFKvXoWZm6YhS3SggIqYD01ttm1zweSnwkTR1OtTMLL0MjyhwqJlZStkeJuVQM7N0BNR2PASqUhxqZpZeia6plYNDzcxS8umnmVUb99TMrKq4p2ZmVSPBtEKV5FAzs/QSTABZKQ41M0vJNwrMrNr49NPMqkbzfGoZ5VAzs5R8+mlm1cY3CsysqviamplVDfn008yqjXtqZlZN5FAzs2qRm83boWZm1UJCNQ41M6si7qmZWVVxqJlZVXGomVn1UH7JKIeamaUi5J6amVWXmprsjijIbsvMLLMkJVoS1DNB0gJJCyWd3cb+b0mak1+elNQoaWCxOh1qZpaOUizFqpFqgUnA4cAY4ARJYwrLRMTFEbFbROwGfAe4LyJeKVavQ83MUitRT20csDAiFkXEGmAKcFSR8icA13VUqUPNzFJpvlGQMNTqJM0sWE4tqGoEsLhgvT6/bf1jSr2BCcBNHbXPNwrMLLUUw6RWRMTY9qppY1u0U/ZI4KGOTj3BoWZmaalkD9/WA6MK1kcCS9spezwJTj3Bp59m9j6U6JraDGA7SaMlbUQuuKa1caz+wIHALUna5p6amaVWip5aRDRImgjcCdQCV0XEPEmn5fdPzhc9GrgrIlYlqdehZmaplHJEQURMB6a32ja51frVwNVJ63SomVl62R0l5VAzs5SU7WFSDjUzS80D2s2sumQ30xxqG+LQXUfw0y/uQ21NDX+4ewGX3DJ3nf1nHPlBjvvQNgD0qK3hAyP6s+0pf+TtNY3c9qOP0atnDbU1NUz7x7NccOM/K/ETup0HZjzFBb+aRmNTE8dOGMd/HH/IOvv/8vfZ/PaGewDovUkvfnDGMeywzXCeXbyMb55/TUu5+pdeYeLnP8rnjzmgU9ufFd22pyZpAvALcrdrfxMRF5TzeJ2pRuLiL+3H0effwdKVq7j7p5/g9pkvsGDJay1lLr31CS699QkAJuwxitM/tjOvrVoDwFHnTmfVuw30qBW3//jj/G1OPTOfWV6Jn9JtNDY2cf5lN3PlBacypK4/nznjlxy8705su+WQljIjhg7k6p+dTv++vXngsaf40SV/YsqlZzJ61OZMnfyNlnoOPvE8Prz/zpX6KRWVdAaOSinb1b4kI/C7sj23Hcyil9/g+WVv8l5jE1MfXsQRe23Rbvlj99+Gmx5a1LK+6t0GAHrW1tCzRw3R3uAQK5knFrzAqOF1jBo2iI169uCIA3fjnofnrVNm9522on/f3gDssuMWvLzi9fXqefSfzzBq2CCGDxnQKe3OolJNPVQO5byFkXYEfpcybGBvlqxc+yzg0pWrGTagT5tlN9molkN3G8m0fzzbsq1G4v4LP8nTV57EvXOXMmuhe2nl9vKKNxg2eLOW9SGD+/PyyvVDq9nUOx7jgL12WG/77fc9zhEH716OJnYZqlGipRLKGWqJRuBLOrV5BH+8+2YZm1Nabf0jFO2MxZ2w5xb8Y8HLLaeeAE0RjP/2n9np9CnssW0dO47qvv/qd571/37a6038Y85Cpt4xg2+ccsQ629e818A9j8zjo+N3KUsLu4ru2lNLNAI/Iq6IiLERMVa9+paxOaW1dOVqRgxa2zMbPqg3L726us2yx+y3NTc99O82972xeg0Pzn+JQ3dtc8YVK6Ehdf15cflrLesvL3+dzQf2W6/cgkVLOed/b+TSH3+Rzfqt2/t+cMZTjNl2BHUDus5/qyWn7htqaUbgdzmz/72cbYb2Y4vBm9KztoZj9tua22e+sF65fpv0ZP8xw5hesG9Q343p13sjADbuWctBOw/nmaXtnwZZaey8/SheWLKC+hdfYc17DUy/bw4H77vuZd6ly17l6+f+np+edQJbjRy8Xh3T75njU09yZypJlkoo593PlhH4wBJyI/BPLOPxOlVjU3DWVY9w03cnUFsj/njv0zxV/xonfzh3Deb//vYUAB8btxX3zF3C6vyNAYChAzbh8q8eSG2NqKkRNz+yiDtnL27zOFY6PWpr+d7ET3Lqd6+kqamJoz86jm23Gsr1f3kEgM98fF8mX/M3Xn9jNeddOrXlOzdM+joAb7+zhodnP8M5/3lsxX5DNmT77qeijLfdJB0BXMLaEfjnFytfM2Cr6HXID8rWHiu9eZOr5t+pbuGow/bniTmzNyiRNh76gdjyC5cmKvv0RRNmFZkksizK+pxaWyPwzayLq+CpZRIeUWBmqQioqdDjGkk41MwsNffUzKyqZPlGgUPNzNLxNTUzqyZCniTSzKqLe2pmVlV8Tc3MqoevqZlZNcmN/cxuqjnUzCy1DGeaQ83M0svyiILs3pc1s2wq4XxqkiZIWiBpoaSz2ylzkKQ5kuZJuq+jOt1TM7NUmudT2+B61r7H5DBy8y/OkDQtIuYXlNkMuByYEBEvSNq8o3rdUzOzlJL10hL01JK8x+REYGpEvAAQEcs6qtShZmappZj5tq75HST55dSCapK8x+QDwABJ90qaJenzHbXNp59mlo5S3ShYUWSSyCTvMekB7AkcCmwCPCLp0Yh4ur0DOtTMLJUSPqeW5D0m9eSCcRWwStL9wK5Au6Hm008zS61E19Ra3mMiaSNy7zGZ1qrMLcABknpI6g3sDfyrWKXuqZlZaqXoqEVEg6SJwJ2sfY/JPEmn5fdPjoh/SboDmAs0Ab+JiCeL1etQM7PUSjVMqq33mETE5FbrFwMXJ63ToWZm6XhAu5lVk9wkkdlNNYeamaVWk+GumkPNzFLLcKY51MwsHcnzqZlZlcnwJbX2Q03Spaw/ZKFFRJxZlhaZWeZ11RsFMzutFWbWZYjcHdCsajfUIuJ3heuS+uTHX5lZN5fhjlrHYz8l7StpPvnxVpJ2lXR52VtmZtmUcNxnpW4mJBnQfgnwUWAlQEQ8DowvY5vMLONSzKfW6RLd/YyIxa1St7E8zTGzrBNd/+HbxZL2AyI/PciZdDD1h5lVtyzf/Uxy+nka8DVy0+wuAXbLr5tZN5T01DOzp58RsQI4qRPaYmZdRJZPP5Pc/dxa0q2SlktaJukWSVt3RuPMLJuUcKmEJKef1wI3AMOA4cCNwHXlbJSZZVtXf6RDEfGHiGjIL9dQZPiUmVW33N3PZEslFBv7OTD/8Z786+CnkAuzzwC3dULbzCyL1HUniZxFLsSaW/+Vgn0BnFeuRplZtnXJqYciYnRnNsTMuobm08+sSjSiQNLOwBhg4+ZtEfH7cjXKzLKtS/bUmkk6BziIXKhNBw4HHgQcambdVHYjLdndz08BhwIvRcTJ5F753qusrTKzzJKgtkaJlkpIcvr5dkQ0SWqQ1A9YBvjhW7NuLMunn0l6ajMlbQZcSe6O6GzgsXI2ysyyrVRjPyVNkLRA0sL8o2Ot9x8k6XVJc/LLDzuqM8nYz6/mP06WdAfQLyLmdtxcM6tGQiUZ+ympFpgEHAbUAzMkTYuI+a2KPhARH09ab7GHb/coti8iZic9iJlVkdLNwDEOWBgRiwAkTQGOAlqHWirFemo/L7IvgEM25MBt2X3rOh66/sulrtbKaMBeEyvdBEvh3afrS1JPimtqdZIKX+J0RURckf88AlhcsK8e2LuNOvaV9DiwFPjviJhX7IDFHr49OFmbzaw7EVCbPNRWRMTYIlW11npc+Wxgy4h4S9IRwJ+B7YodMMmNAjOzdZRoQHs9MKpgfSS53liLiHgjIt7Kf54O9JRUV7RtqX+NmXV7JQq1GcB2kkbnXxVwPDCtsICkocqf60oaRy6zVharNNEwKTOzZrnHNTb8TkFENEiaCNwJ1AJXRcQ8Safl908m9/D/6ZIagLeB4yOi6NRnSYZJidx03ltHxLmStgCGRoSfVTPrpko1WCB/Sjm91bbJBZ8vAy5L1bYEZS4H9gVOyK+/Se7ZEjPrprr0i1eAvSNiD0n/BIiIV/Pnv2bWDQnokeFhUklC7b38k78BIGkw0FTWVplZpmU40xKF2i+Bm4HNJZ1P7sLd98vaKjPLLKk0w6TKJcnYzz9KmkVu+iEBn4wIv6HdrBvLcKYluvu5BbAauLVwW0S8UM6GmVl2dfXpvG9j7QtYNgZGAwuAncrYLjPLKEHFJoBMIsnp5wcL1/Ozd3ylneJmVu0q+E7PJFKPKIiI2ZL2KkdjzKxrUIbfUpDkmto3ClZrgD2A5WVrkZllWjW8Iq9vwecGctfYbipPc8ysK+iyoZZ/6HbTiPhWJ7XHzLqALL94pdh03j3yo+jbndbbzLqf3CvyKt2K9hXrqT1G7vrZHEnTgBuBVc07I2JqmdtmZhnVpUcUAAPJTcp2CGufVwvAoWbWDXXlGwWb5+98PsnaMGtWdJI2M6tuGe6oFQ21WmBTkr0cwcy6DVHTRZ9TezEizu20lphZlyC6bk8tw802s4oR9MjwRbVioXZop7XCzLqMLttTi4hXOrMhZtZ1dPVHOszM1pHhTHOomVk6IttvQXeomVk68umnmVWR3IiC7IZalnuRZpZRSrh0WI80QdICSQslnV2k3F6SGiV9qqM6HWpmllop3tCen9psEnA4MAY4QdKYdspdCNyZpG0ONTNLSUjJlg6MAxZGxKKIWANMAY5qo9wZ5CamXZakdQ41M0ul+e5nkgWokzSzYDm1oKoRwOKC9fr8trXHkkYARwOTk7bPNwrMLLUUNwpWRMTYdvYlmSzjEuDbEdGYdLZdh5qZpaOSTeddD4wqWB8JLG1VZiwwJX+8OuAISQ0R8ef2KnWomVkqJXz4dgawnaTRwBLgeODEwgIRMbrluNLVwF+KBRo41MzsfShFTy3/DpSJ5O5q1gJXRcQ8Safl9ye+jlbIoWZmqZXq0duImA5Mb7WtzTCLiC8mqdOhZmapCKjN8IgCh5qZpZbhTHOomVlaQhmeGNuhZmapuadmZlUj90hHdlPNoWZm6SQYrF5JDjUzSy3L86k51MwsldwkkZVuRfscamaWmu9+mllVyfDZp0NtQ/zt4fl85+d/orGpic8dtR//9cWPrLP/6edeYuK51/D4U/V8//SPc8bnPtyy7/Jr7+YPf34YJMZsO5xJP/wsG/fq2dk/ods5dN8d+ek3P0VtTQ1/uOVhLvndX9fZ36/Pxvz6vC8wcsgAanvUctk1f+faWx9lxJDN+NWPPs/mg/rRFMHvbn6IX0+5tzI/IgOy3FMr2ySRkq6StEzSk+U6RiU1NjbxrYtu4MZffJVHb/g+N901i6cWvbhOmQH9+nDBN49j4mcPWWf70mWv8evr7+Pu35/FI9d/j6amJqbeNaszm98t1dSIi8/6NMd9/XL2+fRPOPYje7L96KHrlDnluPEsWPQSB5x0AUd+5Rf85OtH07NHLQ0NTXz/kqns8+mf8JGTf8Ypnxq/3ne7i+ZrakmWSijnzLdXAxPKWH9FzZr3HFuPqmOrkXVs1LMHxxy2B9Pvm7tOmcED+7LHTlvSs0ftet9vaGjknXffo6GhkdXvrGHo4P6d1fRua8+dtmLR4hU8v2Ql7zU0MvWvszniwF3WKRPApn16AdCndy9efWM1DY1NvLzyDeYuqAfgrdXv8vRzLzFs8Gad/AsyQqIm4VIJZQu1iLgfeKVc9Vfai8tfZ8SQAS3rw4cM4MXlryf67vDNN+OMzx7KB4/8ATsc/j369dmEQ/bZsVxNtbxhg/uz5OVXW9aXvvwqw1r9Y3LlDffxga2G8q/bz+eh677Ld37+JyLWnYx11LCB7LL9SGbNe64zmp1JpXqbVDlU/B0Fkk5tnr98+YrllW5OYq3/Q4fkF09fe2M10+9/gjm3/Jh/3X4+q99Zw/XTHytxC621tuYAa/3XeMg+O/LE0/XsePj3GH/ST7noW8fRt8/GLfv7bLIRv7/wFL7zPzfx5qp3yt3kTGp+72e366klFRFXRMTYiBg7uG5wpZuT2PDNN1vvX/2hdclOIe997Cm2HD6IugF96dmjliMP3pXH5j5brqZa3tJlr63Xu35pxbq965OO3Ie/3PM4AM/Wr+D5pSvZbsshAPSoreF3F/4HN94xs6VMd+WeWhXaY8yW/PuF5Ty/ZAVr3mtg6l9nc/j4XTr+IjBy6EBmPvEsq99ZQ0Rw34wFbD96SJlbbLPnP882Wwxmi+GD6NmjlmMO24Pb71/3Omj9S68yfq/tgdw10W23HMJzS1YAcOkPTuLp517i8mvv7vS2Z06GU82PdLxPPXrUctFZn+bYMyfR2Bic9Il92HGbYVx10wMAfOnYA3h5xRsc8oWLeHPVO0hi8pR7eeT67zF25634xKG7c9BnL6S2toZdth/JF47ev8K/qPo1NjZx1kU3cNMvv0ZtrfjjtEd5atFLnHzMhwD4v6kPcvFv72DSOZ/loeu+iwQ/vuwWXnl9FfvsujXHf2xv5j2zhPv/mHuR+HmTpvHXh+dX8idVTJaHSamta0MlqVi6DjiI3BtgXgbOiYjfFvvOnnuOjYf+MbMs7bHyGLDXxEo3wVJ4d8ENNK1etkGJtOMHd4/f33JvorLjttlsVpFX5JVF2XpqEXFCueo2swrLbkfNp59mlk7ucll2U82hZmbpeD41M6s2Gc40h5qZpaWSvMy4XBxqZpZahjPND9+aWTpJn7tNknuSJkhaIGmhpLPb2H+UpLmS5uSHU36oozrdUzOz9ErQU5NUC0wCDgPqgRmSpkVE4RPNfwemRURI2gW4AdihWL3uqZlZakr4vw6MAxZGxKKIWANMAY4qLBARb8XaEQJ9yM0OVZRDzcxSk5ItQF3zLDz55dSCakYAiwvW6/PbWh1LR0t6CrgN+FJHbfPpp5mlk+45tRVFhkm1Vct6PbGIuBm4WdJ44Dzgw+t9q4B7amaWWolOP+uBUQXrI4Gl7RXOTzy7jaS6YpU61MwsFZHq9LOYGcB2kkZL2gg4Hpi2zrGkbZV/KE7SHsBGwMpilfr008xSK8VjahHRIGkicCdQC1wVEfMknZbfPxk4Fvi8pPeAt4HPRAdTCznUzCy9Ej18GxHTgemttk0u+HwhcGGaOh1qZpZalieJdKiZWWrZjTSHmpm9HxlONYeamaXiSSLNrLp4kkgzqzYZzjSHmpml5UkizazKZDjTHGpmlk4FX76eiEPNzNLLcKo51MwsNT/SYWZVxdfUzKx6CGocamZWXbKbag41M0uleZLIrHKomVlqGc40h5qZpeeemplVFQ+TMrOqkt1Ic6iZWUoJ3xRVMQ41M0vNIwrMrLpkN9McamaWXoYzzaFmZmnJr8gzs+qR9REFNZVugJl1X5ImSFogaaGks9vYf5KkufnlYUm7dlSne2pmllopemqSaoFJwGFAPTBD0rSImF9Q7FngwIh4VdLhwBXA3sXqdaiZWWoleqRjHLAwIhYBSJoCHAW0hFpEPFxQ/lFgZEeV+vTTzNLR2gdwO1o6MAJYXLBen9/Wni8Dt3dUqXtqZpZKyhsFdZJmFqxfERFXFFTVWrR5TOlgcqH2oY4O6FAzs9RSnH6uiIix7eyrB0YVrI8Elq53LGkX4DfA4RGxsqMD+vTTzFIr0ennDGA7SaMlbQQcD0xb9zjaApgKfC4ink7SNvfUzCy1UtwmiIgGSROBO4Fa4KqImCfptPz+ycAPgUHA5fnpjhqK9PwAh5qZvR8levg2IqYD01ttm1zw+RTglDR1OtTMLBVBpodJKaLNmw0VIWk58Hyl21EGdcCKSjfCUqnWv7MtI2LwhlQg6Q5yfz5JrIiICRtyvLQyFWrVStLMjq4DWLb476zr8t1PM6sqDjUzqyoOtc5xRcdFLGP8d9ZF+ZqamVUV99TMrKo41MysqjjUyqijWT0teyRdJWmZpCcr3RZ7fxxqZVIwq+fhwBjgBEljKtsqS+BqoFMfFrXScqiVT8usnhGxBmie1dMyLCLuB16pdDvs/XOolU/aWT3NrAQcauWTeFZPMysdh1r5JJrV08xKy6FWPh3O6mlmpedQK5OIaACaZ/X8F3BDRMyrbKusI5KuAx4BtpdUL+nLlW6TpeNhUmZWVdxTM7Oq4lAzs6riUDOzquJQM7Oq4lAzs6riUOtCJDVKmiPpSUk3Suq9AXVdLelT+c+/KTbYXtJBkvZ7H8d4TtJ6bx1qb3urMm+lPNaPJP132jZa9XGodS1vR8RuEbEzsAY4rXBnfmaQ1CLilIiYX6TIQUDqUDOrBIda1/UAsG2+F3WPpGuBJyTVSrpY0gxJcyV9BUA5l0maL+k2YPPmiiTdK2ls/vMESbMlPS7p75K2Ihee/5XvJR4gabCkm/LHmCFp//x3B0m6S9I/Jf2aBO/xlvRnSbMkzZN0aqt9P8+35e+SBue3bSPpjvx3HpC0Q0n+NK1q+A3tXZCkHuTmabsjv2kcsHNEPJsPhtcjYi9JvYCHJN0F7A5sD3wQGALMB65qVe9g4EpgfL6ugRHxiqTJwFsR8bN8uWuB/42IByVtQW7UxI7AOcCDEXGupI8B64RUO76UP8YmwAxJN0XESqAPMDsivinph/m6J5J7IcppEfGMpL2By4FD3scfo1Uph1rXsomkOfnPDwC/JXda+FhEPJvf/hFgl+brZUB/YDtgPHBdRDQCSyXd3Ub9+wD3N9cVEe3NK/ZhYIzU0hHrJ6lv/hjH5L97m6RXE/ymMyUdnf88Kt/WlUATcH1++zXAVEmb5n/vjQXH7pXgGNaNONS6lrcjYrfCDfn/c68q3AScERF3tip3BB1PfaQEZSB32WLfiHi7jbYkHncn6SByAblvRKyWdC+wcTvFI3/c11r/GZgV8jW16nMncLqkngCSPiCpD3A/cHz+mtsw4OA2vvsIcKCk0fnvDsxvfxPoW1DuLnKnguTL7Zb/eD9wUn7b4cCADtraH3g1H2g7kOspNqsBmnubJ5I7rX0DeFbScfljSNKuHRzDuhmHWvX5DbnrZbPzLw/5Nbke+c3AM8ATwK+A+1p/MSKWk7sONlXS46w9/bsVOLr5RgFwJjA2fyNiPmvvwv4YGC9pNrnT4Bc6aOsdQA9Jc4HzgEcL9q0CdpI0i9w1s3Pz208Cvpxv3zw8Rbq14lk6zKyquKdmZlXFoWZmVcWhZmZVxaFmZlXFoWZmVcWhZmZVxaFmZlXl/wGyfqGjKrdm3gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.1s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Set\n",
      "Accuracy: 0.6714501510574018\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5088    0.6062    0.5532      1333\n",
      "           1     0.7798    0.7044    0.7402      2639\n",
      "\n",
      "    accuracy                         0.6715      3972\n",
      "   macro avg     0.6443    0.6553    0.6467      3972\n",
      "weighted avg     0.6888    0.6715    0.6775      3972\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.0s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.2s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAAEICAYAAADC7ki9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfDklEQVR4nO3de7xVVd3v8c93rw2iKKKCoICK97uGiJlpKmF4O6RZoj51yqeXUQ966uny2Dkne449dSqr00U9REZ2Ux5TUUoCTEvUUrmICngJ8cIWlZtiIIUbfs8fa+7NYu21954T1tp7rb2+79drvlxzzjHmGHsv948x5pxjDEUEZmb1oKG7K2Bm1lUc8MysbjjgmVndcMAzs7rhgGdmdcMBz8zqhgOemXUbSWMlPStpqaSrS5z/oqSFybZI0mZJe6bJW7K8anoPL7dzv2jsN6i7q2EZHLj3rt1dBcvg1Vde5s21a7Qj18j12z+ieWOqtLFx1ayIGFvqnKQc8BwwBmgC5gKXRMSSdtKfD3wuIs7MmrdFY6pad5HGfoMYPP573V0Ny+Dmq07t7ipYBh//4Bk7fI1o3shOh30kVdq/L7xhQAenRwFLI2IZgKSpwDigvaB1CXDrduYF3KU1s8wEaki3dWwIsLxgvyk51rZEaRdgLHBH1ryFqqqFZ2Y1QEBDLm3qAZLmFexPjojJBVcq1t49tvOBhyNi7XbkbeWAZ2bZKfVtwNURMbKdc03AsIL9ocCKdtKOZ2t3NmveVu7SmllGZevSzgUOkTRcUm/yQW16m9Kk3YH3AXdnzVvMLTwzyy59C69dEdEsaSIwC8gBUyJisaQJyflJSdILgNkRsaGzvJ2V6YBnZtmINK23VCJiBjCj6Nikov2bgZvT5O2MA56ZZaSytPC6gwOemWWX/iltVXHAM7OMVLYubVdzwDOzbIS7tGZWR9zCM7P64C6tmdULATk/tDCzeuF7eGZWH9ylNbN64haemdUNt/DMrC7IQ8vMrJ54aJmZ1Qc/tDCzeuIurZnVhTLOh9fVHPDMLCN3ac2sntToQ4vaDNNm1r1aXk3pbOv0Mhor6VlJSyVd3U6a0yUtlLRY0gMFx1+U9FRybl6pvMXcwjOzbFSeLq2kHHADMIb8sotzJU2PiCUFafoDNwJjI+JlSXsXXeaMiFidtky38Mwsu/K08EYBSyNiWURsAqYC44rSXArcGREvA0TEyh2ptgOemWUmKdUGDJA0r2C7ouAyQ4DlBftNybFChwJ7SPqTpPmSPlZwLoDZyfErSMFdWjPLJD/De+r38FZHxMgOLlUsivYbgROA0cDOwF8kPRIRzwGnRMSKpJt7r6RnImJOR5VxC8/MspFQQ7qtE03AsIL9ocCKEmlmRsSG5F7dHOA4gIhYkfx3JTCNfBe5Qw54ZpZZhi5tR+YCh0gaLqk3MB6YXpTmbuBUSY2SdgFOAp6W1FfSbkld+gJnAYs6K9BdWjPLLEOXtl0R0SxpIjALyAFTImKxpAnJ+UkR8bSkmcCTwBbgpohYJOlAYFpSj0bgloiY2VmZDnhmllk5Ah5ARMwAZhQdm1S0fx1wXdGxZSRd2ywc8MwsG1H6cUMNcMAzs0xEqvtzVckBz8wya2iozeedDnhmlplbeGZWH3wPz8zqiVt4ZlYX/NDCzOpKimFjVckBz8yykbu0ZlZHHPDMrG444JlZXfBDCzOrL7UZ7xzwzCwjeWiZmdURd2nr0KmHDeR/ffBoGhrEbx59mZ/cv7RNmlEH7cX/HHcUjbkG3tiwiY/e+GcAvnHxcZx+xCDWrP8H53/ngTb5rDIeffw5rv/ZDDZv2cK5o0/gsgvet835hx57milT/4AaRK6hgYmfOIdjjzgAgNvv+TO/+8M8CDj3/SP58Hnv6YafoErUZryrbMCTNBb4AfnZTG+KiG9Wsryu1CC45sJj+MSPH+H1dRu5/bOncv/i13j+9fWtaXbr08hXLzyGT/7kUV59cyN77tq79dydc5fzq4de5FuXHN8Nta9Pmzdv4Qc3/ZbvXPMJBu7ZjwlXT+KUkUdwwLCtS52OOOZATjlxIpJ4/sXX+PfvTeWXP/wsy15+nd/9YR6TvjmBxsYcX/qPn3PyCYcydJ8B3fgTdZ9abeFVrCNesMju2cCRwCWSjqxUeV3t2P324KU1G2ha+zbvbA7ueXwFo48avE2a80cM4d6nXuXVNzcCsHb9ptZz85atZd3bm7Cu88zSJoYM3ot9B+1Jr16NnHnKMTw89+lt0uyy806tf8x//8em1s8vN63iyEOH0Wen3jTmchx/5HAefPTpNmXUg7TrWaQJipLGSnpW0lJJV7eT5nRJCyUtlvRAlrzFKtnCa11kN6lcyyK7SzrMVSMG7d6H15JABvD6ur9z7H79t0lzwMBdacyJX3z6ZPru1MgvHnyBu+c3dXFNrcWqtW8xcMDurfsD9+rHkr+2/T4efHQJk389mzff2sA3v/xRAIbvtzc33Xov6/72Njv1buSRx5/jsIOKl1CtH+Vo4RU0isaQX51srqTpEbGkIE1/4EZgbES8nCzJmCpvKZUMeKUW2T2pguV1qTQLauYaxFFD+/PxSX+hT68cU688hSdeeoMXV2/oiipaseIviNJ/uKeedCSnnnQkTyx5gZ9O/QPf++rl7D90by754Kl84dqfsXOf3hy0/2ByNfqkshzKNJY2TaPoUuDOiHgZWpdkTJu3jUoGvDQxgWTF8CsAcrsNrGB1yuu1dX9ncP+dW/cH7d6Hlev+3ibNGxtWsnHTZjZu2sy8ZWs5fN9+DnjdZOBe/Vi1el3r/qo1bzFgj93aTX/ckcNZ8fodvPnWBvr368u5o0dy7uj8mtI/+fVsBu61e7t5e7oy3cNL0yg6FOgl6U/AbsAPIuIXKfO2Ucl/otIssktETI6IkRExMrdz7fwP9NTyNzlgQF+G7rkzvXLi3Hfty/2LX9smzX2LXmPkgXuSaxB9euU4dr/+PL9yfTtXtEo77OAhNL26hldfX8s77zRz/8NP8Z4TD98mTdOra4jI/7v83LIVNDdvZvfddgHgjXX57+71VW8y59EljH7vsV37A1QLZVqXdoCkeQXbFdteqY3iRlEjcAJwLvAB4CuSDk2Zt41KtvBaF9kFXiG/yO6lFSyvS23eElx75yJuuuLd5CTueGw5S19fz/iT9wdg6l9eYtnK9Tz4zCqmf/59bIng9kdf5q+v/Q2A7/7TCEYdtBd79O3NA195Pz+a9Sy3P7a8oyJtBzXmcvyPT57HF//j52zZsoWzzzyB4cMGcfesxwAY94FRzHlkMbMfWEiusYGdevfims9d3Nqauea6W3lr/ds05nJ89pPns9uuO3dUXI8lIEMDb3VEjGznXJpGUVNyjQ3ABklzyC/PmKpBVUwt/5pVgqRzgO+zdZHdr3eUfqdBh8Tg8d+rWH2s/G676tTuroJl8PEPnsHTTz2+Q/3RPoMPjWEf/WGqtEu/c/b89gKepEbgOWA0+UbRXODSiFhckOYI4HryrbvewGPkG0/PdJa3lIq+h1dqkV0zq30NZXhoERHNkiYCs9jaKFosaUJyflJEPC1pJvAksIX8+7yLAErl7axMj7Qws2yUqUvboVKNooiYVLR/HXBdmrydccAzs0xEeVp43cEBz8wyq9GRZQ54ZpZdrY6ldcAzs2zKeA+vqzngmVkmQp4A1Mzqh1t4ZlY3fA/PzOqD7+GZWb3Ij6WtzYjngGdmmdVovHPAM7PsPNLCzOqD3KU1szqRcT68quKAZ2YZpVuRrBo54JlZZjUa7xzwzCwj+aGFmdUJv4dnZnWlVgNebU55YGbdSkq3dX4djZX0rKSlkq4ucf50SeskLUy2awrOvSjpqeT4vDT1dgvPzDIrRwtPUg64ARhDftnFuZKmR8SSoqQPRsR57VzmjIhYnbZMt/DMLJuUrbsUMXEUsDQilkXEJmAqMK6SVXfAM7NM8hOAptuAAZLmFWxXFFxqCFC4+nxTcqzYyZKekPR7SUcVHA9gtqT5Rddtl7u0ZpZZQ/ou7er2FuIm/8C3WBTtLwD2j4j1ks4B7gIOSc6dEhErJO0N3CvpmYiY02G909bazKxFmbq0TcCwgv2hwIrCBBHxVkSsTz7PAHpJGpDsr0j+uxKYRr6L3CEHPDPLRMnkAWm2TswFDpE0XFJvYDwwfduyNFjJhSSNIh+z1kjqK2m35Hhf4CxgUWcFuktrZpmVY6BFRDRLmgjMAnLAlIhYLGlCcn4ScBHwaUnNwEZgfESEpEHAtCQWNgK3RMTMzspsN+BJ+hFt+9OFlb0q/Y9mZj1JuYaWJd3UGUXHJhV8vh64vkS+ZcBxWcvrqIWX6kU+M6svIv+ktha1G/Ai4ueF+5L6RsSGylfJzKpdjc4d0PlDC0knS1oCPJ3sHyfpxorXzMyqU8oHFtU43jbNU9rvAx8A1gBExBPAaRWsk5lVuXKNpe1qqZ7SRsTyomi9uTLVMbNqJzK9eFxV0gS85ZLeA0TyrsxVJN1bM6tPtToBaJou7QTgX8iPcXsFOD7ZN7M6lLY7W42NwE5beMnUK5d1QV3MrEbUapc2zVPaAyX9VtIqSSsl3S3pwK6onJlVJ6Xcqk2aLu0twG3APsC+wG+AWytZKTOrbj35tRRFxC8jojnZfkUHQ87MrGfLP6VNt1WbjsbS7pl8/GMy1/xU8oHuYuCeLqibmVUjqWaf0nb00GI++QDX8pN9quBcAF+rVKXMrLpVY3c1jY7G0g7vyoqYWW1o6dLWolQjLSQdDRwJ9Gk5FhG/qFSlzKy69bgWXgtJXwVOJx/wZgBnAw8BDnhmdao2w126p7QXAaOB1yLiE+Qn3duporUys6olQa5BqbZqk6ZLuzEitkhqltQPWAn4xWOzOlarXdo0Lbx5kvoDPyH/5HYB8FglK2Vm1a1cY2kljZX0rKSlyetvxedPl7RO0sJkuyZt3lLSjKX9TPJxkqSZQL+IeDLNxc2s5xEqy1haSTngBmAM+SUb50qaHhFLipI+GBHnbWfebXT04vGIjs5FxIIOfxoz65nKNxPKKGBpsiAPkqYC44AOg9aO5O2ohffdDs4FcGaKSmVy9NDdefi755f7slZBe5w4sburYBn84/lXynKdDPfwBkgqXBBsckRMTj4PAZYXnGsCTipxjZMlPUF+ke4vRMTiDHm30dGLx2d0ltnM6o+AXPqAtzoiRnZwqWLF4/QXAPtHxHpJ5wB3AYekzNtGmocWZmbbKNPkAU3AsIL9oeRbca0i4q2IWJ98ngH0kjQgTd6S9e60SmZmRcoU8OYCh0ganiwfMR6YXphA0mAl/WdJo8jHrDVp8paSamiZmVmL/CsnO/7UIiKaJU0EZgE5YEpELJY0ITk/ifzAh09LagY2AuMjIoCSeTsrM83QMpGf4v3AiLhW0n7A4Ijwu3hmdapcgyiSbuqMomOTCj5fD1yfNm9n0nRpbwROBi5J9v9G/v0XM6tTPXYRH+CkiBgh6XGAiHgj6TObWR0S0FiN0SyFNAHvneSt5gCQNBDYUtFamVlVq9F4lyrg/RCYBuwt6evkbyL+74rWysyqllSeoWXdIc1Y2l9Lmk9+iigBH4yIpyteMzOrWjUa71I9pd0PeBv4beGxiHi5khUzs+pVhVPdpZKmS3sPWxfz6QMMB54FjqpgvcysSgmqcnLPNNJ0aY8p3E9mUflUO8nNrKer0jVn08g80iIiFkg6sRKVMbPaoBpd1SLNPbx/LdhtAEYAqypWIzOraj19mcbdCj43k7+nd0dlqmNmtaBHBrzkheNdI+KLXVQfM6sBtbqIT0dTvDcmsxm0O9W7mdWf/DKN3V2L7dNRC+8x8vfrFkqaDvwG2NByMiLurHDdzKxK9diRFsCe5CfcO5Ot7+MF4IBnVod66kOLvZMntIvYGuhadDp3vJn1XDXawOsw4OWAXdnOxTLMrKcSDT3wPbxXI+LaLquJmdUEUb4WnqSxwA/IN7BuiohvtpPuROAR4OKIuD059iL5CYk3A80drI7WqqOAV5sh3MwqS9BYhpt4yWtvNwBjyK9CNlfS9IhYUiLdt8ivX1HsjIhYnbbMjh4uj057ETOrHy0tvDJM8T4KWBoRyyJiEzAVGFci3ZXkBzus3NG6txvwImLtjl7czHqmhmQS0M62TgwBlhfsNyXHWkkaAlwATKKtAGZLmi/pijT19jKNZpZZhnt4AyTNK9ifHBGTWy5TIn3xA9HvA/8WEZtLjO44JSJWSNobuFfSMxExp6PKOOCZWSYi3XKHidUdPExoAoYV7A8FVhSlGQlMTYLdAOAcSc0RcVdErACIiJWSppHvIjvgmVkZqWwjLeYCh0gaDrwCjAcuLUwQEcNbi5VuBn4XEXdJ6gs0RMTfks9nAZ2+VeKAZ2aZ5Eda7HjAS8bqTyT/9DUHTImIxZImJOdL3bdrMQiYlrT8GoFbImJmZ2U64JlZZuV6Zy0iZgAzio6VDHQR8fGCz8uA47KW54BnZpn1xKFlZmYlqOfNh2dmVkrGp7RVxQHPzDLryfPhmZltpR44xbuZWSnu0ppZXXELz8zqRm2GOwc8M8tIQM4tPDOrFzUa7xzwzCwroRrt1DrgmVlmbuGZWV3Iv5ZSmxHPAc/Mskm3XkVVcsAzs8w8tMzM6kJ+AtDursX2ccAzs8z8lNbM6kaN9mgd8MrpD39ewpe/ezubt2zho+Pew+c+ftY252c88CRfn/Q7GiQaGxv4xr9exMnHH9RNtbXRJx/B//38ReQaGvjl3X/m+z+/d5vzV/7TaD589okANOYaOPSAwRx81tW8+dbb3VHdqlKuFp6kscAPyK9pcVNEfLOddCcCjwAXR8TtWfIWqljAkzQFOA9YGRFHV6qcarF58xa++O3bmHb9RPYd1J8z//t1nH3aMRx+4D6taU478TDOPu0YJLHor69w+Zen8NjtX+nGWtevhgZx3Zc+wgUTr2fF629y/8+/yO/nPMWzL7zWmuZHv7qPH/3qPgDGnno0n77kDAc7yncPT1IOuAEYQ37JxrmSpkfEkhLpvkV+sZ9MeYtVcpaXm4GxFbx+VZm/+EUOHDaAA4YOoHevRi4cM4IZDzy5TZpdd9mpdZaJtzf+o2a7BT3BCUcdwLLlq3nplTW807yZO+9dwDnvO7bd9B86ayR3zJ7fhTWsYhINKbdOjAKWRsSyiNgETAXGlUh3JXAHsHI78m6jYgEvWQF8baWuX21eXbWOIYP2aN3fd9AevLpqXZt0v/vjE4y66Gtc/LlJ/Ogrl3VlFa3APgN355XX32jdX/H6G+wzcPeSaXfeqRejTz6C6fcv7KLaVT+l3IABkuYVbFcUXGYIsLxgvyk5trUcaQhwAVC8klmneUvp9nt4yS/gCoBh++3XzbXZfhHR5lipf+DOO+M4zjvjOB5esJRvTLqHu268sgtqZ8VKzedW4isEYOxpx/Dok8vcnU1kXJd2dUSM7OBSxYq/he8D/xYRm4u+szR52+j2gBcRk4HJACecMLLTClerfffu36bFMHhA6RYDwCkjDuYzr6xmzZvr2av/rl1RRSuwYuWbbVrkr61u2yIHuHDMCdwxy93ZQmW6G9MEDCvYHwqsKEozEpiaBLsBwDmSmlPmbaNWZ2quOiOO3J/nX17FS6+sZtM7zdx57wLOPm3be0LLlq9qbQk+8cxy3nmnmT1379sd1a17C5a8xEH7DWS/ffeiV2OOC8eM4PdznmyTrl/fPpwy4uA292PrXoY+bQfmAodIGi6pNzAemF6YICKGR8QBEXEAcDvwmYi4K03eUrq9hddTNDbm+PaXPsKHrrqBzZuDy/7buznioH2YcseDAFz+oVOZfv9C/vOeR2lszLFzn1789BuX1+xU2bVu8+YtfOnbt3HHD/+FXE78evojPLPsNT5x4XsB+NmdDwFw7hnH8cdHn+Htv2/qzupWnXIMLYuIZkkTyT99zQFTImKxpAnJ+eL7dp3m7axMlbr3VA6SbgVOJ98MfR34akT8tKM8J5wwMh5+dF5F6mOVsceJE7u7CpbBP569jS1vr9yhaHXEMe+KX9z9p1RpRx3Uf34H9/C6XMVaeBFxSaWubWbdrEY7Ju7Smlkm+dtztRnxHPDMLBvPh2dm9aRG450DnpllpZp9u8ABz8wyq9F454BnZtmke6e4OjngmVl2NRrxHPDMLDO/lmJmdcP38MysPvg9PDOrJ+7SmlldEG7hmVkdqdF454BnZtuhRiOeA56ZZVaOCUC7gwOemWVWm+HOa1qY2fYoz5oWSBor6VlJSyVdXeL8OElPSlqYLPP43oJzL0p6quVcmmq7hWdmmZRrAlBJOeAGYAz5VcjmSpoeEUsKkt0HTI+IkHQscBtweMH5MyJiddoy3cIzs2ySF4/TbJ0YBSyNiGURsQmYCowrTBAR62Prwjt9SbH2bEcc8MwsszL1aIcAywv2m5Jj25YlXSDpGeAe4PKCUwHMljRf0hVp6u0urZlllGkC0AFF99cmR8Tk1gu11aYFFxHTgGmSTgO+Brw/OXVKRKyQtDdwr6RnImJOR5VxwDOzzDK8lbK6g2Uam4BhBftDgRXtXSgi5kg6SNKAiFgdESuS4yslTSPfRe4w4LlLa2aZpO3OpoiJc4FDJA2X1BsYD0zfpizpYCXNSUkjgN7AGkl9Je2WHO8LnAUs6qxAt/DMLLsyvIgXEc2SJgKzgBwwJSIWS5qQnJ8EfAj4mKR3gI3AxckT20Hku7mQj2O3RMTMzsp0wDOzzMo1W0pEzABmFB2bVPD5W8C3SuRbBhyXtTwHPDPLrEZHljngmVlGggYHPDOrH7UZ8RzwzCwTTwBqZnWlRuOdA56ZZecWnpnVjQxDy6qKA56ZZVab4c4Bz8wySjn1U1VywDOzzLwurZnVj9qMdw54ZpZdjcY7Bzwzy0peptHM6kMtj7TwBKBmVjfcwjOzzGq1heeAZ2aZ+bUUM6sPNfzise/hmVkmLQ8tyrAQN5LGSnpW0lJJV5c4P07Sk5IWSpon6b1p85biFp6ZZVaOLq2kHHADMIb8ko1zJU2PiCUFye4DpicL9xwL3AYcnjJvG27hmVlmZWrhjQKWRsSyiNgETAXGFSaIiPUR0bI4d1+2LtTdad5SHPDMLLMM69IOSLqiLdsVBZcZAiwv2G9Kjm1blnSBpGeAe4DLs+Qt5i6tmWWXvke7OiJGZrhKtDkQMY38GrSnAV8D3p82bzEHPDPLRFCuoWVNwLCC/aHAivYSR8QcSQdJGpA1bwtt7R53P0mrgJe6ux4VMABY3d2VsEx66ne2f0QM3JELSJpJ/veTxuqIGNvOdRqB54DRwCvAXODSiFhckOZg4PnkocUI4Lfkg1uus7ylVFULb0e/iGolaV4HzXqrQv7O2tdeANuO6zRLmgjMIh/ApkTEYkkTkvOTgA8BH5P0DrARuDh5iFEyb2dlVlULr6fyH0/t8XfWM/kprZnVDQe8rjG5uytgmfk764HcpTWzuuEWnpnVDQe8Ctqewc3WvSRNkbRS0qLurouVnwNehRQMbj4bOBK4RNKR3VsrS+FmoCyvXVj1ccCrnO0a3GzdKyLmAGu7ux5WGQ54lbNdg5vNrHIc8CpnuwY3m1nlOOBVznYNbjazynHAq5y5wCGShkvqDYwHpndznczqmgNehUREM9AyuPlp4LY0g5ute0m6FfgLcJikJkn/3N11svLxSAszqxtu4ZlZ3XDAM7O64YBnZnXDAc/M6oYDnpnVDQe8GiJps6SFkhZJ+o2kXXbgWjdLuij5fFNHExtIOl3Se7ajjBeTFaZSHS9Ksz5jWf8u6QtZ62j1xQGvtmyMiOMj4mhgEzCh8GQyQ0tmEfHJiFjSQZLTgcwBz6zaOODVrgeBg5PW1x8l3QI8JSkn6TpJcyU9KelTAMq7XtISSfcAe7dcSNKfJI1MPo+VtEDSE5Luk3QA+cD6uaR1eaqkgZLuSMqYK+mUJO9ekmZLelzSj0mxXLOkuyTNl7S4aFV6JH03qct9kgYmxw6SNDPJ86Ckw8vy27S6UFXLNFo6yXqeZwMzk0OjgKMj4oUkaKyLiBMl7QQ8LGk28C7gMOAYYBCwBJhSdN2BwE+A05Jr7RkRayVNAtZHxHeSdLcA/y8iHpK0H/nRJEcAXwUeiohrJZ0LbBPA2nF5UsbOwFxJd0TEGqAvsCAiPi/pmuTaE8mvNTEhIv4q6STgRuDM7fg1Wh1ywKstO0tamHx+EPgp+a7mYxHxQnL8LODYlvtzwO7AIcBpwK0RsRlYIen+Etd/NzCn5VoR0d68cO8HjtTW1ef7SdotKePCJO89kt5I8TNdJemC5POwpK5rgC3AfybHfwXcKWnX5Of9TUHZO6UowwxwwKs1GyPi+MIDyR/+hsJDwJURMaso3Tl0Pj2VUqSB/K2QkyNiY4m6pB6rKOl08sHz5Ih4W9KfgD7tJI+k3DeLfwdmafkeXs8zC/i0pF4Akg6V1BeYA4xP7vHtA5xRIu9fgPdJGp7k3TM5/jdgt4J0s8l3L0nSHZ98nANclhw7G9ijk7ruDryRBLvDybcwWzQALa3US8l3ld8CXpD04aQMSTqukzLMWjng9Tw3kb8/tyBZiObH5Fvy04C/Ak8B/x94oDhjRKwif9/tTklPsLVL+VvggpaHFsBVwMjkocgStj4t/j/AaZIWkO9av9xJXWcCjZKeBL4GPFJwbgNwlKT55O/RXZscvwz456R+i/G0+ZaBZ0sxs7rhFp6Z1Q0HPDOrGw54ZlY3HPDMrG444JlZ3XDAM7O64YBnZnXDAc/M6sZ/AS7reZh8DrNsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammWord(clf_rf_word, model_name=\"clf_rf_word\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "hR2yXGnMPTCW"
   },
   "outputs": [],
   "source": [
    "grid_search_rf = GridSearchCV(clf_rf, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "b2FH56xWPTQt"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   29.8s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  3.6min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.8min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   26.5s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed: 16.1min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed: 18.4min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 18.9min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time=19.1min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   30.8s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.7min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.3s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 4.9min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   31.9s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.8min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 5.0min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   31.2s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.4min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 4.6min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   24.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  4.1min finished\n"
     ]
    }
   ],
   "source": [
    "clf_rf_char= grid_search_rf.fit(X_train_char, y_train_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "id": "S8lJ6iiLPTdM"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    2.0s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    2.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.8917781614156236\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8744    0.7916    0.8309      3114\n",
      "           1     0.8994    0.9425    0.9204      6154\n",
      "\n",
      "    accuracy                         0.8918      9268\n",
      "   macro avg     0.8869    0.8670    0.8757      9268\n",
      "weighted avg     0.8910    0.8918    0.8904      9268\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.8s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    2.1s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcl0lEQVR4nO3deXhV1b3/8fc3CQgyQ5hFBkUUURQFp4ojClxbQb1OVG8dHodb9F61avX+KhVurdb61CoOdaraKtYBUSuiXKtCVSqDzAoOKAjIDCKiQPL9/XF2YhKTc/aGc3J2dj4vn/N4hp2110nqp2sPa33N3RERSYqCfHdARCSbFGoikigKNRFJFIWaiCSKQk1EEqUo3x2oqKBRMy9o0jbf3ZAI9t2jZb67IBEsX7aUDevX2q60Udi8q/uOraG29a1rXnX3wbuyv6jiFWpN2tJs6Jh8d0MiePq2YfnugkRw5pCjd7kN37GV3XqdGWrbb2ffU7zLO4woVqEmInWBgcX3zJVCTUSiMaCgMN+9qJFCTUSis106LZdTCjURiUiHnyKSNBqpiUhiGLEeqcW3ZyISU5YaqYV5ZGrJbLCZLTKzj83sl9V83srMnjezuWb2npn1ydSmQk1EoisoDPdIw8wKgXuAIUBv4Bwz611lsxuB2e5+IHA+8MeMXdupLyQi9VhwoSDMI70BwMfu/qm7bwOeAk6tsk1v4HUAd/8Q6GZm7dM1qlATkWiMKIefxWY2o8LjkgotdQaWVXj9RfBeRXOA0wDMbADQFdgjXfd0oUBEogt/oWCtux9aUyvVvFd1Ke5bgT+a2WxgHvA+sCPdDhVqIhJR1u5T+wLoUuH1HsCKihu4+1fABQBmZsCS4FEjhZqIRGNAYVamSU0HeppZd2A5cDZwbqVdmbUEvgnOuV0MTAmCrkYKNRGJLgs337r7DjMbCbwKFAKPuPsCM7ss+Px+YD/gcTMrARYCF2VqV6EmIhFlb5qUu08EJlZ57/4Kz98FekZpU6EmItFpmpSIJEqMp0kp1EQkmpBToPJFoSYi0WmRSBFJDq2nJiJJo8NPEUmMmK+nplATkYh0+CkiSaMLBSKSKDqnJiKJYTr8FJGk0UhNRJLEFGoikhSp1bwVaiKSFGZYQXxDLb5n+0Qktsws1CNEO5nqfrYws5fMbI6ZLTCzCzK1qVATkciyEWoh637+HFjo7n2BY4E7zKxhunYVaiISWZZGamHqfjrQLCi60hRYj6pJiUhWGdUXt6tesZnNqPD6AXd/IHheXd3Pw6r8/FjgRVJVppoBZ7l7abodKtREJBIj3PmywK7W/TwZmA0cD+wFTDazqekqSunwU0QiKygoCPXIIGPdT1I1P8d7ysekan7um7ZvEb+LiEi2zqmV1/0MTv6fTepQs6KlwAnBPtsDvYBP0zWqw08RiSbaObUahaz7OQZ41MzmBXu93t3XpmtXoSYikWVrRkGIup8rgJOitKlQE5FIIl4oqHUKNRGJLM7TpBRqIhKNaUK7iCSMQk1EEkWhJiKJoQsFIpI88c00hZqIRGSEmQKVNwo1EYlMh58ikizxzTSF2q44rk9HRp97KIUFxpNTPmbsxIWVPr988H6cdkQ3AIoKCujZqTl9rnyOjVu2cfGgXowYuDdm8MRbH/Pg5EV5+Ab1zzszFvH7B16kpNQZdlJ/LjjzuEqfT3zjfR579k0Adm/UkBt+Ppx9enQC4OY7n2Hqex/QumVTnr736trueqzEeaSW0wPjTOuP12UFZtxyXn9G/OENjvmfvzPssG7s06l5pW3um/QBg0a9wqBRr3DLs7N5d9FqNm7ZRq/OLRgxcG+GjpnECTdN5MS+nenevlmevkn9UVJSyq33TeCumy/k2fuu5tUpc/h06apK23Ru34oHb72Uv91zFRefcwL/e/f48s9+fOIh3D36otruduyEXaEjX8GXs1ALuf54nXVwjzZ8tnozS9d8zfaSUl5473NOPrhLjdsPO7wbE6Z9BkDPji2Y+elatm4roaTUmbZoNUP61fyzkh0LFi+jS6c27NGxDQ0aFHHSwL68Oa3y6Lpv7240b7Y7AAf02pPV6zaVf9avTw9aNGtcq32Oq3oZaoRbf7zO6tCqMcvXf1P+euX6b+jQqvr/wTduWMhxfTry8szUysWLlm/k8H3a0apJQxo3LOT4AzvRqfXutdLv+mz1uk20L25Z/rp9cQvWVAitqia8Np0jD+lVCz2re6zAQj3yIZfn1MKsP46ZXQJcAmBN2uSwO9ll1Zwp9aoLEQcGHbQH0z9ew8Yt2wD4aOVX3DNxIX+79gS2fLuDhcs2UlKSdtl1yYLq/j7V/R0Bps/5hBdem87Dt1+e417VTXE+p5bLUAuz/jhBEYYHAIra9KghFuJn5YZv6FxhdNWx9e6s2ri12m2HDejKhH99Xum9cVM/YdzUTwC44fS+rKgw6pPcaF/cglVrN5a/XrV2E8Vtmv9gu4+WrGTMXc9y9+gLadm8SS32sI6I+YT2XB5+hll/vM6avWQd3ds1o0txExoUFnDqgK68+v4XP9iuWeMGHN6rHZNmLav0fptmuwHQufXuDD2kyw9CT7Kv9z57sGz5OpZ/uZ7t23fw2pQ5HHPYfpW2Wbl6A7/4zV8Yc81ZdO3cNk89jTcDzMI9MraVuZjxtWY2O3jMN7MSM2udrs1cjtTK1x8HlpNaf/zcHO6vVpWUOjc+MYNx1xxPYYHx1NRPWLxiE+cf2xOAx9/8CIAh/brw1oKVbN1WUunnHx45kFZNdmN7SSk3/GU6m77ZVuvfob4pKizkustPZeSvHqaktJRTB/Vnr64deHbiNADOGHo4D457nU1ffcOt904AoLCwgL/+8UoAbrztSWbM+5SNX21hyPm/4dIRgxh28oB8fZ08ys5FgAoXEweRGgRNN7MX3b386o273w7cHmz/Y+Aqd1+ftl2v6URQFpjZUOBOvl9//Dfpti9q08ObDR2Ts/5I9r1927B8d0EiOHPI0cyfM2uXEqlRh32863/cHWrbxb8bPLOmEnlmdgTwa3c/OXh9A4C7/7aG7Z8E3nD3B9PtM6c331a3/riI1HEhDy0Du1rMOLVLs92BwcDITDvUjAIRicSAgvC3a+xqMeMyPwbeznToCQo1EdkJWbr4GeVi4tnAuDCNxnf9EBGJrVosZoyZtQCOAV4I0zeN1EQkmmjn1GoUspgxwHDgNXffEqZdhZqIRGJY1haJzFTMOHj9KPBo2DYVaiISWYwnFCjURCS6OE+TUqiJSDRZOqeWKwo1EYkkNfczvqmmUBORyGKcaQo1EYkuwoyCWqdQE5FoYr6emkJNRCIpW08trhRqIhJR/oqqhKFQE5HIYpxpCjURich0oUBEEkT3qYlI4ijURCRRYpxpCjURiS7OIzWtfCsi0YSs+ZmNup/BNscGdT8XmNlbmdrUSE1EIkktElk7dT/NrCVwLzDY3ZeaWbtM7WqkJiKRFZiFemQwAPjY3T91923AU8CpVbY5Fxjv7ksB3H11xr7txPcRkXouS4ef1dX97Fxlm32AVmb2ppnNNLPzMzWqw08RicSiTWhPV8w4TN3PIuAQ4ASgMfCumU1z98U17VChJiKRRTillq6YcZi6n18EbWwBtpjZFKAvED3UzOxuaq6WjLtfWdNnIpJsWZomVV73E1hOqu7nuVW2eQEYa2ZFQEPgMOAP6RpNN1KbkeYzEamnjNQV0F0Vpu6nu39gZpOAuUAp8JC7z0/Xbo2h5u6PVXxtZk3CFhMVkWTL1nz2kHU/bwduD923TBuY2RFmthD4IHjd18zuDbsDEUkYS62nFuaRD2Fu6bgTOBlYB+Duc4CBOeyTiMRctmYU5EKoq5/uvqxK6pbkpjsiEncGYW6szZswobbMzI4E3MwaAlcSHIqKSP0U50Uiwxx+Xgb8nNSdvsuBg4LXIlIPhT30jO3hp7uvBUbUQl9EpI6I8+FnmKufPczsJTNbY2arzewFM+tRG50TkXiykI98CHP4+STwNNAR6AQ8A4zLZadEJN7q+i0d5u5/cfcdweOvpJk+JSLJlrr6Ge6RD+nmfrYOnr4RrEj5FKkwOwt4uRb6JiJxZNlZJDJX0l0omEkqxMp6f2mFzxwYk6tOiUi8xblGQbq5n91rsyMiUjeUHX7GVagZBWbWB+gNNCp7z90fz1WnRCTe6uRIrYyZjQKOJRVqE4EhwD8BhZpIPRXfSAt39fMMUkvpfunuF5BadXK3nPZKRGLLDAoLLNQjH8Icfm5191Iz22FmzYHVgG6+FanH4nz4GWakNiOovfcgqSuis4D3ctkpEYm32ipmHBQy3hQUM55tZjdlajPM3M//DJ7eHyyr29zd52burogkkRGqpmfmdkIUMw5MdfdTwrab7ubbfuk+c/dZYXciIgmSvRU4yosZA5hZWTHjqqEWSbqR2h1pPnPg+F3ZcXX6dmvN23/WgiB1Sav+I/PdBYngu4++yEo7War7WV0x48OqaeMIM5tDqnzeL9x9Qbodprv59rhwfRaR+sSAwvChlq7uZ5hixrOAru7+tZkNBSYAPdPtMMyFAhGRSrI0oT1jMWN3/8rdvw6eTwQamFlx2r5F/jYiUu9lKdTKixkHpQLOBl6suIGZdbDgWNfMBpDKrHXpGg01TUpEpEzqdo3aKWZM6ub/y81sB7AVONvd0y59FmaalJFazruHu482sz2BDu6ue9VE6qnaKmbs7mOBsZH6FmKbe4EjgHOC15tJ3VsiIvVUnS68Ahzm7v3M7H0Ad98QHP+KSD1kQFGMp0mFCbXtwZ2/DmBmbYHSnPZKRGItxpkWKtTuAp4H2pnZb0iduPt/Oe2ViMSWWXamSeVKmLmfT5jZTFLLDxkwzN1VoV2kHotxpoW6+rkn8A3wUsX33H1pLjsmIvFV15fzfpnvC7A0AroDi4D9c9gvEYkpg7wtABlGmMPPAyq+DlbvuLSGzUUk6fJY0zOMyDMK3H2WmfXPRWdEpG6wGFcpCHNO7eoKLwuAfsCanPVIRGItCSXymlV4voPUObbnctMdEakL6myoBTfdNnX3a2upPyJSB8S58Eq65byLgln0NS7rLSL1T6pEXr57UbN0I7X3SJ0/m21mLwLPAFvKPnT38Tnum4jEVJ2eUQC0JrUo2/F8f7+aAwo1kXoo7hcK0g0i2wVXPucD84J/Lwj+Pb8W+iYiMVVbdT8rbNffzErM7IxMbaYbqRUCTQlXHEFE6g2jIAv3qYWt+xlsdxupFXIzShdqK9199E72V0QSyqj1up9XkLqNLNRN/+lCLcZHzSKSNwZF2TmplrHup5l1BoaTOqe/y6F2QsQOikg9EHGklq6YcZhTW3cC17t7Sdh749IVM14fqgURqXci3NKRrphxxrqfwKHAU0GgFQNDzWyHu0+oaYcqkScikWXpnFp53U9gOam6n+dW3MDdu3+/T3sU+Hu6QAOFmohEZGSnCnrIup+RKdREJBrL3oyCTHU/q7z/szBtKtREJJLUjIL43hyhUBORyOIbaQo1EdkJMR6oKdREJCqrm+upiYhUJ1tXP3NFoSYikelCgYgkh9XR5bxFRKqjw08RSRyN1EQkUeIbaQo1EYnIgEKN1EQkSWKcaQo1EYnKsBgfgCrURCQyjdREJDFSt3TEN9UUaiISTcianvkS53voRCSmCsxCPTLJVMzYzE41s7lmNtvMZpjZjzK1qZGaiESSWiQyC+2EK2b8OvCiu7uZHQg8Deybrl2N1EQkMgv5TwblxYzdfRtQVsy4nLt/7e5lZfOa8MMSej+gUBORyMzCPQjqflZ4XFKhmeqKGXf+4b5suJl9CLwMXJipbzr8jOj/3lnIDXc8S0lpKeedeiRX/eykSp+7O7+841kmv72Axo0acu+o8+i7b6q04YE/uYmmu+9GYUEBRUUFvPH49QDMW/QFV9/6FN9+t52iogJ+f/1ZHLJ/t9r+avXCCUfsx2+vOYPCggL+8sI73PnY5Eqft2jWmLG/+ind9yjm223buWLME3zwycryzwsKjDcev46Vqzdx9tU7VewoESLcp5au7meYYsa4+/PA82Y2EBgDnJhuhzkLNTN7BDgFWO3ufXK1n9pUUlLKtb97mufHjqRT+5Yc/x+3M2TgAezbo2P5NpPfWcgnS9cwc/woZsz/jGtufYr/e/Ta8s9fuv+/aNOyaaV2R909gesuHsKgo/bntbcXMOquCfz9T/9dW1+r3igoMG6/7kyGjxzLilUb+cdj1/LKlHksWvJl+TbXXHAy8xZ/wXnXPUjPru25/fozGfafd5d/ftnZx7F4ySqaNWmUj68QC9k6p0a4Ysbl3H2Kme1lZsXuvram7XJ5+PkoMDiH7de6mQs+o0eXYrrtUUzDBkWcNqgfE9+aW2mbiW/N5ex/G4CZ0f+A7mzavJUv125K264ZbN7yLQBffb2VDm1b5Ow71GeH7N+NT5et5fPl69i+o4Txk2cx9JgDK23Tq3sHpkxfBMBHn69iz46tadu6GQCd2rXkpB/tz+MvvFPrfY+VkFc+Q1z9LC9mbGYNSRUzfrHyrmxvC5YEMbN+QENgXbpGczZSC1K1W67az4eVazbRuX2r8ted2rdi5vzPqmyzsfI27VqycvVGOhS3wMw4beRYzIyfDT+Kn52Wujp9y9VncPoV9/CrPz6PuzPp4Wtq5fvUNx3btmD5qg3lr1es2sAhfbpV2mb+R8s55biDmDbnU/r17kqXDq3p1K4la9Zv5parT2fUXRNounv9HaWVycZALWQx49OB881sO7AVOKvChYNq5f2cWnDi8BKALnvumefepFfd77Lq/xlV9+suW3tq0kNX0bFt6j+Q4SPH0rNbB47qtzePPDeVW64+jZ8cfzDPT57FlWOeYMK9V+TiK9Rr1a0BVvXvdedjk/ntNWcw5YlfsvDjFcxd/AUlJaWc/KM+rN2wmTkfLuOofj1rqcfxlM26n5mKGbv7bcBtUdrM+9VPd3/A3Q9190PbFrfNd3fS6tSu5Q/+n75DcYv026zeWH442bFtSwDatm7GKcceyKwFnwEw7u//4sfHHQTAsBMPZtbCz3P3JeqxFas3/mCkXfXUwOYt3zJy9F8ZOOJWLhv1OMUtm/L5inUc1rcHg48+gDkv3MzDt1zA0f334U+jz6/trxAbFvKRD3kPtbqkX++ufLJ0DZ8vX8u27TsYP3kWQwZWPiczZOABPPXye7g70+ctoXnTxnQobsGWrd+VnzfbsvU7/jHtQ/bbqxOQOix6e9ZHAEyZvpgeXeId7nXVrIWfs9eebdmzUxsaFBVy2qB+vDKl8jnR5k0b06CoEIDzhx3JO+9/zOYt3zL6nhfpc8qv6HvqKC668c9Mnb6YS296PB9fIx5inGp5P/ysS4qKCvnddWdy+pX3UFLijPjJ4ey3V0ceeW4qABeefjQnHbU/k99eQL/hN9O4UQPuuemnAKxZt5mfXvcgACU7Sjh98KGceGRvAO78n3O54Y5n2VFSSqOGRdx54zn5+YIJV1JSynW/e5rn7vo5hYXGEy9O48NPv+SC4Nzmn8f/k17dO3Dfr8+jpLSURUu+5IoxT+S51/EU52pSluGc2843bDYOOBYoBlYBo9z94XQ/c8ghh/rb/5qRk/5IbrTqPzLfXZAIvlv0NKXfrN6lRNrvgIP98RfeDLXtgL1azkxzn1pO5PLqp4YbIkkV34GaDj9FJJrU6bL4pppCTUSiifl6ago1EYksxpmmUBORqEzFjEUkWWKcaQo1EYkmn7MFwlCoiUh0MU41hZqIRKZbOkQkUXROTUSSQ/epiUjSxPnwU0sPiUgkRqRqUunbylzMeERQzHiumb1jZn0ztamRmohElo1xWshixkuAY9x9g5kNAR4ADkvXrkZqIhJddhaJDFPM+B13L1tKehqpilNpaaQmIpFFWCSy2MwqLpL4gLs/EDyvrphxulHYRcArmXaoUBORyCIcfu5yMWMAMzuOVKj9KNMOFWoiEl0tFjM2swOBh4Ah7p625ifonJqIRFS2SGSYfzIIU8x4T2A8cJ67Lw7TP43URCSaLN18G7KY8U1AG+DeYLmjHZlqHijURCSybN16G6KY8cXAxVHaVKiJSERaJFJEEibGmaZQE5FotEikiCRPjFNNoSYikcV5lQ6FmohEpnNqIpIcBgUKNRFJlvimmkJNRCIpWyQyrhRqIhJZjDNNoSYi0WmkJiKJomlSIpIo8Y00hZqIRBS2UlS+KNREJLI4zyjQyrciEl12qkmFqfu5r5m9a2bfmdkvwnRNIzURiawW636uB64EhoVtVyM1EYnIKLBwjwzC1P1c7e7Tge1he6dQE5FIymYUhHkQ1P2s8LikQlPV1f3svKv90+GniORSVup+RqFQE5HIsnRLR6i6n1Hp8FNEIqutup87QyM1EYmmFut+mlkHYAbQHCg1s/8Gerv7VzW1q1ATkUiyufRQiLqfX5I6LA1NoSYikcV5RoFCTUQi09xPEUmUGGeaQk1EdkKMU02hJiKRGISZApU35r7LN/BmjZmtAT7Pdz9yoBhYm+9OSCRJ/Zt1dfe2u9KAmU0i9fsJY627D96V/UUVq1BLKjObkWaqiMSQ/mZ1l2YUiEiiKNREJFEUarXjgXx3QCLT36yO0jk1EUkUjdREJFEUaiKSKAq1HMpUKUfix8weMbPVZjY/332RnaNQy5EKlXKGAL2Bc8ysd357JSE8CtTqzaKSXQq13MlYKUfix92nkCrLJnWUQi13clIpR0TSU6jlTk4q5YhIegq13MlJpRwRSU+hljs5qZQjIukp1HLE3XcAZZVyPgCedvcF+e2VZGJm44B3gV5m9oWZXZTvPkk0miYlIomikZqIJIpCTUQSRaEmIomiUBORRFGoiUiiKNTqEDMrMbPZZjbfzJ4xs913oa1HzeyM4PlD6Sbbm9mxZnbkTuzjMzP7QdWhmt6vss3XEff1azP7RdQ+SvIo1OqWre5+kLv3AbYBl1X8MFgZJDJ3v9jdF6bZ5FggcqiJ5INCre6aCuwdjKLeMLMngXlmVmhmt5vZdDOba2aXAljKWDNbaGYvA+3KGjKzN83s0OD5YDObZWZzzOx1M+tGKjyvCkaJR5tZWzN7LtjHdDM7KvjZNmb2mpm9b2Z/IkQdbzObYGYzzWyBmV1S5bM7gr68bmZtg/f2MrNJwc9MNbN9s/LblMRQhfY6yMyKSK3TNil4awDQx92XBMGwyd37m9luwNtm9hpwMNALOABoDywEHqnSblvgQWBg0FZrd19vZvcDX7v774PtngT+4O7/NLM9Sc2a2A8YBfzT3Ueb2b8BlUKqBhcG+2gMTDez59x9HdAEmOXu15jZTUHbI0kVRLnM3T8ys8OAe4Hjd+LXKAmlUKtbGpvZ7OD5VOBhUoeF77n7kuD9k4ADy86XAS2AnsBAYJy7lwArzOwf1bR/ODClrC13r2ldsROB3mblA7HmZtYs2Mdpwc++bGYbQnynK81sePC8S9DXdUAp8Lfg/b8C482safB9n6mw791C7EPqEYVa3bLV3Q+q+EbwH/eWim8BV7j7q1W2G0rmpY8sxDaQOm1xhLtvraYvoefdmdmxpALyCHf/xszeBBrVsLkH+91Y9XcgUpHOqSXPq8DlZtYAwMz2MbMmwBTg7OCcW0fguGp+9l3gGDPrHvxs6+D9zUCzCtu9RupQkGC7g4KnU4ARwXtDgFYZ+toC2BAE2r6kRoplCoCy0ea5pA5rvwKWmNm/B/swM+ubYR9SzyjUkuchUufLZgXFQ/5EakT+PPARMA+4D3ir6g+6+xpS58HGm9kcvj/8ewkYXnahALgSODS4ELGQ76/C3gwMNLNZpA6Dl2bo6ySgyMzmAmOAaRU+2wLsb2YzSZ0zGx28PwK4KOjfArREulShVTpEJFE0UhORRFGoiUiiKNREJFEUaiKSKAo1EUkUhZqIJIpCTUQS5f8DeDuNP2q35hYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.0s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Set\n",
      "Accuracy: 0.7583081570996979\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6526    0.5420    0.5922      1286\n",
      "           1     0.7972    0.8619    0.8283      2686\n",
      "\n",
      "    accuracy                         0.7583      3972\n",
      "   macro avg     0.7249    0.7019    0.7102      3972\n",
      "weighted avg     0.7504    0.7583    0.7518      3972\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.7s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.9s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcdklEQVR4nO3de7xVVb338c+XDQhyE9igAkqoiOIFU8D0KVI7JlgdMrNEu9tjnkeOr87pZleP9vQ8pnVOnbRDZGanTLwnJYpWXtCsQPIGhSFe2CDCBkRFUPbev/PHWuBis/fac8pae8099/fda71ea8451pg/0H6OMcccYygiMDPLix61DsDMrJKc1MwsV5zUzCxXnNTMLFec1MwsV3rWOoBSewzYK/rVj6h1GJZC3151tQ7BUnjxhVVs3rRBu1NH3cDREU1bEpWNLevmR8TU3blfWplKav3qR3DKxdfWOgxLYfy+A2odgqUw6/zTdruOaNrCHuM+lKjs1keurN/tG6aUqaRmZl2BQNl9cuWkZmbpCOiR3ccOTmpmlp5267FcVTmpmVlK7n6aWd64pWZmuSHcUjOzPJFbamaWMx79NLP88ECBmeWJcPfTzHLGLTUzyw93P80sTwTUeaDAzPIkw8/UstuGNLOMKnY/k3w6qkmaKmmZpOWSLmzj+iBJv5b0qKQlkj7ZUZ1OamaWnpTsU7YK1QFXAtOA8cAMSeNbFTsfWBoRE4ATgO9K6l2uXic1M0uvMi21ycDyiFgREa8Dc4DprcoEMECSgP7ABqCpXKV+pmZm6SRohZWol7So5Hh2RMwufh8JrCy51gAc2+r3VwBzgdXAAODDEdFS7oZOamaWXvJpUo0RMbGda21lxmh1fArwCHAScCBwt6QFEfFSu6EljczMrKBiAwUNwH4lx6MotMhKfRK4JQqWA08Dh5Sr1EnNzNKrwEABsBAYK2lM8eH/mRS6mqWeA95VuKX2BsYBK8pV6u6nmaVTofXUIqJJ0kxgPlAHXB0RSySdV7w+C/gmcI2kx4t3/lJENJar10nNzFKq3DSpiJgHzGt1blbJ99XAu9PU6aRmZul5PTUzy5UMT5NyUjOzdORVOswsb9xSM7M8kZOameVFYTVvJzUzywsJ9XBSM7MccUvNzHLFSc3McsVJzczyQ7S9aFBGOKmZWSpCbqmZWb706OEZBWaWI26pmVl++JmameWNW2pmlhseKDCz3PE0KTPLD7n7aWY546RmZrmS5aSW3TfozCyTtg8UJPl0WJc0VdIyScslXdjG9S9IeqT4eUJSs6Qh5ep0UjOz9JTwU64KqQ64EpgGjAdmSBpfWiYiLo+IoyLiKODLwH0RsaFcve5+mlk6qtg0qcnA8ohYASBpDjAdWNpO+RnAdR1V6paamaWWovtZL2lRyefckmpGAitLjhuK59q6357AVODmjmJzS83M0ks+TtAYERNT1BLtlH0f8GBHXU9wUtsth+87gLOOGUUPifufWs+8pS/sdH3c8P5cMOUAGje/BsDDKzcx94k1O65LcNEp49i4ZRvfv29Fp8beXT217Bnuvu0+IlqYMPlwjj9xUpvlVq9cw8+uuJ73n30qhx45FoCtW7Zy+02/Zd2a9UjwnjNOZtToEZ0ZfmZUaPSzAdiv5HgUsLqdsmeSoOsJVU5qkqYC3wfqgKsi4tJq3q8zSfDRifvxnd8vZ8OWbXzjlHE80rCJ1S9t3anck+teaTdhnTxuGM+/tJU+veo6I+Rur6Wlhfm33sOM//0BBg7qz09/cB1jxx/AsL2H7lLunnkPcMDBo3c6f/fc+zjw4Ldw+kffS3NTM9u2bevM8DMj6chmAguBsZLGAKsoJK6z2rjfIOCdwEeSVFq1Z2pJRja6sgOG7snaV15j3ebXaW4J/vzsRt46alDi3w/u24sJIwZx/1PrqxillVq9cg2D6wcxeOgg6nrWMX7Cwfx9yVO7lFv04COMO2Ise/bvu+Pca1tf47kVq5gw+TAA6nrW0advn06LPWsq8UpHRDQBM4H5wF+BGyJiiaTzJJ1XUvQ04K6I2Jwktmq21NKObHQpg/v2ZsPm13ccb3j1dQ6s77dLuYPq+3HxtEN4ccs2rv/LKlZvKrTkZhwzkhv+ssqttE708qbNDBw0YMfxgEEDWL1yTasyr7Dsiac4+zOnc3vJtRc3bGLP/n35zQ13sfb5RvYZOZyTp59A7969Oi3+LKnU3M+ImAfMa3VuVqvja4BrktZZzdHPRCMbks7dPjLy2ssbqxhO9UWrR5zPbniVz9+2hIvu+Bu/e3IdF0wZA8CEEQN5eWsTz27cUoMou7P2nkG/4e6593HSqW/f5ZWFluZgzaq1HH3ckZzz2bPp1bsXD92zsFqBZl6lXr6thmq21BKNbETEbGA2wJAx4zv+ty4jNm55nSH9eu84HrJnb17csvMzlq1NLTu+P7b6JT46cRT996hj7LB+HDVqEEeOGEivuh706VXHuceNZvZDz3Za/N3RgEH9eWnTyzuOX970MgMG7ty6fr7hBX71y0LD4dXNW3nqb8/Qo4cYOXpfBg7qz8j99wXgkCPHdt+k1o0ntKcZ2ehynl7/KsMH7EF9v95s3LKNyaMH86M/PLNTmYF9evLS1iYAxgzdE0m88lozNz36PDc9+jxQGCGdeuhwJ7ROMGLUPmxsfJEXN2xiwMD+LH30SabPmLZTmfO//Kkd3399/XwOOvQAxh1+EFDorq5fu4Ghw4fwzN+fo374zgMM3YUoDJRlVTWTWqKRja6qJeDaRQ187sQD6SGxYMV6Vm/aygkHFf5Fv3f5eibtvxcnHlRPc8C25hZmPfhMbYPu5nrU9eDd009kzlW30tISTJh0GMP2Gcrihx4D4Ojjjiz7+1PefwK3XXcnzc0tDB46kPec8e7OCDuDsr1IpKL1g6BKVi6dCnyPwisdV0fEt8qVHzJmfJxy8bVVi8cqb/y+AzouZJkx6/zTWPXk47uVkfrsc3CM/vgPEpV98rKpD5d5+bYqqvqeWlsjG2bWxan7dj/NLIcE9PBy3maWJ26pmVmuZHmgwEnNzNLxMzUzyxOhSi0SWRVOamaWmltqZpYrfqZmZvnhZ2pmlieFuZ/ZzWpOamaWWoZzmpOamaXnGQVmlh/deD01M8uh7ryempnlUrbXU3NSM7PUMpzTqrrxipnlkQoDBUk+HVYlTZW0TNJySRe2U+YESY9IWiLpvo7qdEvNzFKp1HtqJXsDn0xhT5OFkuZGxNKSMnsBPwSmRsRzkoZ3VK9bamaWWoW2yNuxN3BEvA5s3xu41FnALRHxHEBErO2oUic1M0tNSvYB6rfv61v8nFtSTZK9gQ8GBku6V9LDkj7WUWzufppZaim6n41lNl5JsjdwT+AY4F1AX+AhSX+MiCfbu6GTmpmlU7kJ7Un2Bm6gkBg3A5sl3Q9MANpNau5+mlkqhUUiKzL6uWNvYEm9KewNPLdVmduAd0jqKWlP4Fjgr+UqdUvNzFLrUYGmWkQ0SZoJzOeNvYGXSDqveH1WRPxV0p3AY0ALcFVEPFGuXic1M0utUi/ftrU3cETManV8OXB50jqd1MwsFXlCu5nlTYZXHmo/qUn6AbsOr+4QERdUJSIzy7yuup7aok6Lwsy6DFEYAc2qdpNaRPys9FhSv+K7ImbWzWW4odbxe2qSjpO0lOK7IZImSPph1SMzs2xKOO+zVoMJSV6+/R5wCrAeICIeBaZUMSYzy7gUcz87XaLRz4hY2SrrNlcnHDPLOlGZl2+rJUlSWynpeCCKUxkuoINpCmaWb1ke/UzS/TwPOJ/CkiCrgKOKx2bWDSXtema2+xkRjcDZnRCLmXURWe5+Jhn9PEDSryWtk7RW0m2SDuiM4Mwsm5TwUwtJup+/BG4A9gVGADcC11UzKDPLtq7+Soci4ucR0VT8/IIy06fMLN8Ko5/JPrVQbu7nkOLXe4pbV82hkMw+DNzeCbGZWRYp2fZ3tVJuoOBhCklse/SfKbkWwDerFZSZZVuXXHooIsZ0ZiBm1jVs735mVaIZBZIOB8YDfbafi4j/rlZQZpZtXbKltp2ki4ATKCS1ecA04AHASc2sm8puSks2+vlBCnvurYmIT1LYnmqPqkZlZpklQV0PJfrUQpLu55aIaJHUJGkgsBbwy7dm3ViWu59JWmqLJO0F/JjCiOhi4M/VDMrMsq1Scz8lTZW0TNLy4qtjra+fIGmTpEeKn290VGeSuZ//p/h1VnH/vYER8VjH4ZpZHglVZO6npDrgSuBkCjuxL5Q0NyKWtiq6ICLem7Teci/fHl3uWkQsTnoTM8uRyq3AMRlYHhErACTNAaYDrZNaKuVaat8tcy2Ak3bnxm0ZM2RPfnrWWytdrVXR4Ekzax2CpfBawwsVqSfFM7V6SaWbOM2OiNnF7yOBlSXXGoBj26jjOEmPAquBz0fEknI3LPfy7YnJYjaz7kRAXfKk1hgRE8tU1VrreeWLgdER8YqkU4FfAWPL3TDJQIGZ2U4qNKG9Adiv5HgUhdbYDhHxUkS8Uvw+D+glqb5sbKn/NGbW7VUoqS0ExkoaU9wq4ExgbmkBSfuo2NeVNJlCzlpfrtJE06TMzLYrvK6x+yMFEdEkaSYwH6gDro6IJZLOK16fReHl/3+S1ARsAc6MiLJLnyWZJiUKy3kfEBGXSNof2Cci/K6aWTdVqckCxS7lvFbnZpV8vwK4IlVsCcr8EDgOmFE8fpnCuyVm1k116Y1XgGMj4mhJfwGIiI3F/q+ZdUMCemZ4mlSSpLat+OZvAEgaBrRUNSozy7QM57RESe0/gVuB4ZK+ReHB3deqGpWZZZZUmWlS1ZJk7ue1kh6msPyQgPdHhHdoN+vGMpzTEo1+7g+8Cvy69FxEPFfNwMwsu7r6ct6388YGLH2AMcAy4LAqxmVmGSWo2QKQSSTpfh5RelxcveMz7RQ3s7yr4Z6eSaSeURARiyVNqkYwZtY1KMO7FCR5pvavJYc9gKOBdVWLyMwyLQ9b5A0o+d5E4RnbzdUJx8y6gi6b1Iov3faPiC90Ujxm1gVkeeOVcst59yzOom93WW8z634KW+TVOor2lWup/ZnC87NHJM0FbgQ2b78YEbdUOTYzy6guPaMAGEJhUbaTeON9tQCc1My6oa48UDC8OPL5BG8ks+3KLtJmZvmW4YZa2aRWB/Qn2eYIZtZtiB5d9D215yPikk6LxMy6BNF1W2oZDtvMakbQM8MP1coltXd1WhRm1mV02ZZaRGzozEDMrOvI8isdGX6FzsyyqlIbr0iaKmmZpOWSLixTbpKkZkkf7KhOJzUzS0UUEkeST9l6CtMwrwSmAeOBGZLGt1Pu2xT2B+2Qk5qZpaNC9zPJpwOTgeURsSIiXgfmANPbKPfPFBbRWJskPO/QbmapFGYUJH6mVi9pUcnx7IiYXfw+ElhZcq0BOHane0kjgdMozGhKtI6jk5qZpZZimKAxIiamqKb1i/3fA74UEc1JVwZxUjOz1Co0+NkA7FdyPApY3arMRGBOMaHVA6dKaoqIX7VXqZOamaWkSq2nthAYK2kMsAo4EzirtEBEjNlxV+ka4DflEho4qZlZSttHP3dXcb3GmRRGNeuAqyNiiaTzitdnvZl6ndTMLLVKvXwbEfOAea3OtZnMIuITSep0UjOzdNRFl/M2M2tLpbqf1eKkZmapuaVmZrmS3ZTmpGZmKQmoc0vNzPIkwznNSc3M0hLKcAfUSc3MUnNLzcxyo/BKR3azmpOamaWTcFXbWnFSM7PUsrxHgZOamaVSWCSy1lG0z0nNzFLz6KeZ5UqGe5+Znpeaeb/9w1ImnX4JR5/2b/zHNXftcv3JZ9bw7k99h72P/yw/+Plvd7ne3NzClLMv5cP/8l+dEa4B7zruUP5809d5+JaL+OzHT97l+sB+fbju3z/Dgmsv5A/Xf5Wz3ve2N67178s1l57Dn278Gn+84WtMOmLMLr/vLpTwf7VQtZaapKuB9wJrI+Lwat2nVpqbW/jCZTdw6xUzGbH3Xpz08cuZNuUIDjlg3x1lBg/sx6WfO4Pb73u0zTpmzbmHg8fszcubt3ZW2N1ajx7i8i9+iNNmXsHqF17k9z/7Anfc/zjLnl6zo8ynz5jCshVrmPGvP2LoXv1ZeNPXufGOhWxraubSz32Q3z20lE9c+BN69ayjb5/eNfzT1E7Wn6lVs6V2DTC1ivXX1MNLnuGA/ep5y6h6evfqyQdOPpp59z22U5lhQwZw9GGj6dWzbpffr3phI3c9sISPTT++s0Lu9o457C2sWNnIs6vWs62pmVvuXsyp7zxypzIB9O+3BwD99tyDjS+9SlNzCwP69eH4tx7Iz297CIBtTc289MqWzv4jZEPC7fFqNUJataQWEfcDG6pVf609v24TI/cevON4xN6DeX7dpsS//8q/38zFF7yfHln+T17O7DtsEKte2LjjePULG9l32KCdyvz4hvs4+C378Nc7vsWD132FL3/3JiKC0SOH0vjiK1x50Ue47xdf4vtfPYs9u2lLDQqttSSfWqj5MzVJ50paJGnRusZ1tQ4nsYjWO3klf3h654LHqR88gKMO3b/CUVk5ba0B1vof40lvO5THn2zg0GlfZcrZ/5/LvnAGA/r1oWddHRPG7cfVNy3gnR/5Nq9ufY3PfmLXZ3LdwfZ9P7tdSy2piJgdERMjYuKw+mG1DiexEcP32uW/+vvUDyrzizf86dEV3LngcY78x29wzld+yoKFT3Lu139WrVCtaPXaF3dpXa9p3Ll1ffb73sZv7ik8A326oZFnV69n7Oi9Wb12I6vXvsjDS54FYO7vHmHCuP3ortxSy6Gjx4/mqefW8eyqRl7f1sQtdy9m2pQjO/4hcNHM6Sy5/f/y2NxL+Mn/+yTvmHQws7/58SpHbIuXPsuB+w9j/xFD6dWzjg+cfDR33L/zc9CGNRuZMmkcUHgmetDovXlmVSNr17/Mqhc2ctDo4QBMmTRupwGGbifDWc3vqb1JPXvWcdkXP8TpF1xJc3Nw9j++jUMP3Jerb14AwKdOfwcvNL7ESR+/jJc3b0USs+bcy0PXf5WB/fvWOPruqbm5hS9edgM3/+f51NWJa+f+kb+tWMMnP/B2AH56ywNc/pM7ufKij/DgdV9BgouvuI0NmzYD8MXv3MjsSz5B7151PLOqkfMv+UUt/zg1VamupaSpwPcpbJF3VURc2ur6dOCbQAvQBHw2Ih4oW2dbz4YqFOx1wAkUdlV+AbgoIn5S7jfHHDMxHvzToqrEY9UxeNLMWodgKby27AZaXl27Wxnp0CPeGv99272Jyk4+cK+HI2JiW9ck1QFPAidT2K19ITAjIpaWlOkPbI6IkHQkcENEHFLunlVrqUXEjGrVbWY1VpmG2mRgeUSsAJA0B5gO7EhqEfFKSfl+FN66KcvP1MwslcLjssQzCuq3v91Q/JxbUtVIYGXJcUPx3M73k06T9DfgduBTHcXnZ2pmlk669dQa2+t+0nZ7b5eWWETcCtwqaQqF52v/UO6GbqmZWWoVGvxsAErfixkFrG6vcPGF/gMl1Zer1EnNzFISUrJPBxYCYyWNkdQbOBOYu9OdpINUrEjS0UBvYH25St39NLPUKvFGR0Q0SZoJzKfwSsfVEbFE0nnF67OA04GPSdoGbAE+HB28suGkZmapVPK92oiYB8xrdW5WyfdvA99OU6eTmpmll+F1GJzUzCw1L+dtZrmS5eW8ndTMLB3v+2lmeePup5nlhnBLzcxyJsM5zUnNzN6EDGc1JzUzS61W+w8k4aRmZqllN6U5qZnZm5HhrOakZmapbF8kMquc1MwsHb98a2Z5k+Gc5qRmZmklWgCyZpzUzCy1DOc0JzUzS6eGm68n4qRmZullOKs5qZlZan6lw8xyxc/UzCw/BD0ynNS876eZvQmV2c5Y0lRJyyQtl3RhG9fPlvRY8fMHSRM6qtMtNTNLpVKLREqqA64ETqawW/tCSXMjYmlJsaeBd0bERknTgNnAseXqdUvNzFKrTDuNycDyiFgREa8Dc4DppQUi4g8RsbF4+EdgVEeVOqmZWWpSsg9QL2lRyefckmpGAitLjhuK59pzDnBHR7G5+2lmqaWYJtUYERPbq6aNc9HO/U6kkNTe3tENndTMLLUKDX42APuVHI8CVu9yL+lI4CpgWkSs76hSdz/NLJWkXc8EjbmFwFhJYyT1Bs4E5u58L+0P3AJ8NCKeTBKfW2pmllolZhRERJOkmcB8oA64OiKWSDqveH0W8A1gKPDDYpe3qUx3FnBSM7M3o0L9z4iYB8xrdW5WyfdPA59OU6eTmpmlluEJBU5qZpaWvEWemeVHpWYUVItHP80sV9xSM7PUstxSc1Izs9S8SKSZ5Yf3/TSzPMn6QIGTmpml5u6nmeWKW2pmlisZzmlOamb2JmQ4qzmpmVkqgkxPk1JEmwtN1oSkdcCztY6jCuqBxloHYank9Z/Z6IgYtjsVSLqTwt9PEo0RMXV37pdWppJaXkla1NEaUJYt/mfWdXnup5nlipOameWKk1rnmF3rACw1/zProvxMzcxyxS01M8sVJzUzyxUntSqSNFXSMknLJV1Y63isY5KulrRW0hO1jsXeHCe1KpFUB1wJTAPGAzMkja9tVJbANUCnvixqleWkVj2TgeURsSIiXgfmANNrHJN1ICLuBzbUOg5785zUqmcksLLkuKF4zsyqyEmtetqa8ev3Z8yqzEmtehqA/UqORwGraxSLWbfhpFY9C4GxksZI6g2cCcytcUxmueekViUR0QTMBOYDfwVuiIgltY3KOiLpOuAhYJykBknn1DomS8fTpMwsV9xSM7NccVIzs1xxUjOzXHFSM7NccVIzs1xxUutCJDVLekTSE5JulLTnbtR1jaQPFr9fVW6yvaQTJB3/Ju7xjKRddh1q73yrMq+kvNe/Sfp82hgtf5zUupYtEXFURBwOvA6cV3qxuDJIahHx6YhYWqbICUDqpGZWC05qXdcC4KBiK+oeSb8EHpdUJ+lySQslPSbpMwAquELSUkm3A8O3VyTpXkkTi9+nSlos6VFJv5P0FgrJ81+KrcR3SBom6ebiPRZK+l/F3w6VdJekv0j6EQn28Zb0K0kPS1oi6dxW175bjOV3koYVzx0o6c7ibxZIOqQif5uWG96hvQuS1JPCOm13Fk9NBg6PiKeLiWFTREyStAfwoKS7gLcC44AjgL2BpcDVreodBvwYmFKsa0hEbJA0C3glIr5TLPdL4D8i4gFJ+1OYNXEocBHwQERcIuk9wE5Jqh2fKt6jL7BQ0s0RsR7oByyOiM9J+kax7pkUNkQ5LyL+LulY4IfASW/ir9Fyykmta+kr6ZHi9wXATyh0C/8cEU8Xz78bOHL78zJgEDAWmAJcFxHNwGpJv2+j/rcB92+vKyLaW1fsH4Dx0o6G2EBJA4r3+EDxt7dL2pjgz3SBpNOK3/crxroeaAGuL57/BXCLpP7FP++NJffeI8E9rBtxUutatkTEUaUniv/n3lx6CvjniJjfqtypdLz0kRKUgcJji+MiYksbsSSedyfpBAoJ8riIeFXSvUCfdopH8b4vtv47MCvlZ2r5Mx/4J0m9ACQdLKkfcD9wZvGZ277AiW389iHgnZLGFH87pHj+ZWBASbm7KHQFKZY7qvj1fuDs4rlpwOAOYh0EbCwmtEMotBS36wFsb22eRaFb+xLwtKQziveQpAkd3MO6GSe1/LmKwvOyxcXNQ35EoUV+K/B34HHgv4D7Wv8wItZReA52i6RHeaP792vgtO0DBcAFwMTiQMRS3hiFvRiYImkxhW7wcx3EeifQU9JjwDeBP5Zc2wwcJulhCs/MLimePxs4pxjfErxEurXiVTrMLFfcUjOzXHFSM7NccVIzs1xxUjOzXHFSM7NccVIzs1xxUjOzXPkfAWucTlARNmYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(clf_rf_char, model_name=\"clf_rf_char\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "ie9EHRVp6aGv"
   },
   "outputs": [],
   "source": [
    "del clf_rf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UfgxLqHj46JP"
   },
   "source": [
    "### **4.4 Neural Network**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "id": "BEYfAkeHqcfu"
   },
   "outputs": [],
   "source": [
    "clf_nn = MLPClassifier(alpha=1e-5,\n",
    "                       hidden_layer_sizes=(150,100,50),\n",
    "                       max_iter=300,\n",
    "                       activation = 'relu',\n",
    "                       solver='adam',\n",
    "                       random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "id": "VfM_ewcoT_6x"
   },
   "outputs": [],
   "source": [
    "grid_search_nn = GridSearchCV(clf_nn, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_word, y_train_word), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "id": "incNx66n_1e4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV] END .................................................... total time=19.4min\n",
      "[CV] END .................................................... total time=10.9min\n",
      "[CV] END .................................................... total time= 9.0min\n",
      "[CV] END .................................................... total time=19.8min\n",
      "[CV] END .................................................... total time=16.9min\n"
     ]
    }
   ],
   "source": [
    "clf_nn_word = grid_search_nn.fit(X_train_word, y_train_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "id": "vYyFZP0xyUME"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.9977341389728097\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9984    0.9948    0.9966      3067\n",
      "           1     0.9974    0.9992    0.9983      6201\n",
      "\n",
      "    accuracy                         0.9977      9268\n",
      "   macro avg     0.9979    0.9970    0.9974      9268\n",
      "weighted avg     0.9977    0.9977    0.9977      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaa0lEQVR4nO3deZhU1Z3/8fenG1BUVllEFkXH4IYmCqgYicafBkwcXCNuMSYZ4yj6TJLRmEzURJNfFuPEDYcQY4zRyKgQJYoLWQxG1LDEDQyKqKwKrYBEUejmO39Ugd1td3VdqOq6ffm8nqeep27VueeeapMP59x7z7mKCMzMsqKq0g0wMyslh5qZZYpDzcwyxaFmZpniUDOzTGlX6QbUp3YdQx06VboZlsDH9xlQ6SZYAotef42amhptTR3VnXeLqF1XVNlYt/KRiBi5NcdLKl2h1qET2+19WqWbYQk88dQNlW6CJXD4oUO3uo6oXcd2gz5fVNn3nxnXY6sPmFCqQs3M2gKB0nvmyqFmZskIqKqudCua5VAzs+S0VaflysqhZmYJefhpZlnjnpqZZYZwT83MskTuqZlZxvjqp5llhy8UmFmWCA8/zSxj3FMzs+zw8NPMskRAtS8UmFmW+JyamWWHh59mljXuqZlZprinZmaZIU+TMrOs8TQpM8sOXygws6zx8NPMMsPrqZlZtnj4aWZZ4wsFZpYpPqdmZpkhDz/NLGvcUzOzLJFDzcyyIreat0PNzLJCQlUONTPLEPfUzCxTHGpmlikONTPLDuVfKeVQM7NEhNxTM7NsqaryjAIzyxD31MwsO3xOzcyyJs09tfQOjM0slTZdKCjm1WJd0khJ8yUtkHRZE993kfR7Sc9Kmivp3JbqdE/NzBIrxTQpSdXAOOAYYAkwU9KUiJhXr9iFwLyIOF5ST2C+pDsjYn1z9bqnZmbJiFL11IYBCyJiYT6kJgKjG5UJoJNyle0EvA3UFqrUPTUzSyzBObUekmbV254QERPy7/sCi+t9twQ4pNH+NwFTgGVAJ+C0iNhY6IAONTNLLEGo1UTEkOaqaeKzaLT9GeAZ4NPAnsA0SY9HxDvNHdDDTzNLpIQXCpYA/ett9yPXI6vvXGBy5CwAXgX2LlSpQ83MklORr8JmAntJGiipAzCG3FCzvkXA0QCSegODgIWFKvXw08ySUWmmSUVEraSxwCNANXBrRMyVdH7++/HA1cBtkp7PHZlvRkRNoXodamaWWKluvo2IqcDURp+Nr/d+GXBskjodamaWXHonFDjUtsbRh+7DD79xMtVVVfzm/ie57vZpDb7v0qkjN11+JgP79uD99bVcdPWdvLhwOQBfPe1TnHPCcJC4/b4ZjJ/4WAV+QXb94cl5fPvaSdRt3MjZow/jP85p+I99RPCtaycxbcZcOm7fgXFXnMWBe/cvuO+PJkzlN/fPYOeuOwFw+QXHc8zh+/Hnp//BVeOmsH5DLR3at+N7F41mxNBBrfuDW1map0mVNdQkjQSuJzdeviUiflTO47WmqipxzaWncuLYcSxbsZo//foSHnr8eea/+sbmMt/44rE8/9JSzr70FvbarTfXXHoqJ1x4E/vs0YdzThjO0V/8Ketr67j3+gt49Im5LFy8soK/KDvq6jZy6U/uYfJNF7Jrr64cfc41jDxiMHvv0WdzmT/MmMcri1cwa9IVzHrhNb7x4//lD7/6zxb3Pf/0o7jorKMbHG/nrjvy22u/Sp+eXZj3yjJOvfhm5j74/Vb9za2p2ClQlVK2q5/1pkCMAvYFTpe0b7mO19oO3m83Fi6p4fVlb7Ghto7Jj87muBGDG5QZNLAP02fOB+Dl199kQJ/u9OzeiY8N7M3MF15j3QcbqKvbyBNzXuZzRx5QiZ+RSbPnvs7Afj3YvW8POrRvx0nHHsxD059vUGbq9OcZc9wwJDF08EDeWbuON2rWFLVvYwcM6k+fnl0A2GePPrz/wQY+WL+hbL8vDUo197McynlLRzFTINqsPj27svTNVZu3l61YTZ+eXRuUeeHlpXzuqAMBOGjf3ei/S3d27dWVF19ZzvBP/AvduuxAx+3ac8zh+9G3d7fWbH6mLV+5usHfc9deXVm+cnXDMiuaKLNiTYv73nLPdD55xg8Ze/WdrH7nvY8ce8qfnuGAQf3YrkP70v2gFFKVinpVQjlDrakpEH0bF5J0nqRZkmZF7boyNqe0mvpHKBrdDH3d7dPo2mkHpt/xTc77/Aiee2kJdXUbeem1N7n+9mn87sax3HvDBcx9eSm1dQVnflgC0fiedHI3jDYo85Eb13P/TQvt+6WTP8mcyVcy/Y5vssvOnfnO9b9rUO7FV5bzvZum8N/fGrPljW8j0txTK+c5tWKmQJCfBzYBoGqHXk38TyqdljXxL/0bK9c0KLP23fcZe/Wdm7efve+7vL7sLQDumPIUd0x5CoDL//14lq1YXf5GbyN27fXRXvQu+eHhh2W6NVlm/Ya6ZvfttXPnzZ9/4YThjPn6zzdvL31zFV+49Bfc/N2zGdivZ8l/U6oo3RcKytlTK2YKRJs1Z94i9uzfkwG77kz7dtW5cy+PNzz30nmnjrRvVw3AF0YPZ8Yzr7D23fcB6NEtdwWtX+9ufO6oA7n30VlYaRy07wAWLl7J60trWL+hlsmPzmbkEQ3Pd446Yn8mTv0bEcHM51+l807bs0uPLgX3faPmw3+0HnjsWfbZM3fxYM3a9xjztfFcfuG/cuiBe7TeD60QkevVFvOqhHL21DZPgQCWkpsCcUYZj9eq6uo2cuk19zDphguorhJ3/v4p/rHwDc496XAAfjX5CQYN7M3/XHk2dRuD+a++wUXf/7DXdvuPv0K3zjtQW7eRS665mzVr287QO+3atavmJ5ecyikX30zdxuDM4w9lnz378KtJfwXg3JM/yTGH78e0GfM4+KSr6Lh9e266/KyC+wJ898b7ef6lJUhiQJ/um4eZv7h7Oq8uqeGnv3yYn/7yYQAm3XghPbt3qsCvbw3pvvqpaOokQqkql44DruPDKRA/KFS+aodesd3ep5WtPVZ6bz99Q6WbYAkcfuhQ5syetVWJtP0uH4vdzrmxqLIv/WTk7AKrdJRFWe9Ta2oKhJm1cRUcWhbDMwrMLBGRu/k8rRxqZpaYe2pmlilpvlDgUDOzZHxOzcyyRKgki0SWi0PNzBJzT83MMsXn1MwsO3xOzcyyJDf3M72p5lAzs8RSnGkONTNLzjMKzCw7Ur6emkPNzBLZtJ5aWjnUzCyhdK+n5lAzs8RSnGkONTNLSL5QYGYZ4vvUzCxzHGpmlikpzjSHmpkl556amWWHJ7SbWZbkFolMb6o51MwssaoUd9XSuyavmaWWVNyr5Xo0UtJ8SQskXdZMmSMlPSNprqS/tFSne2pmlohKNKFdUjUwDjgGWALMlDQlIubVK9MVuBkYGRGLJPVqqV731MwssSoV92rBMGBBRCyMiPXARGB0ozJnAJMjYhFARKxoqdJme2qSbgSiue8j4uIWm2xmmZTgQkEPSbPqbU+IiAn5932BxfW+WwIc0mj/jwHtJT0GdAKuj4jbCx2w0PBzVoHvzGwbJXJXQItUExFDClTVWOOOVDvgYOBooCPwpKSnIuKl5g7YbKhFxK8bHF3aMSLeba68mW07SnRHxxKgf73tfsCyJsrU5LPnXUnTgQOBZkOtxXNqkg6TNA94Mb99oKSbEzbezLJCufXUinm1YCawl6SBkjoAY4ApjcrcDxwhqZ2kHcgNT18sVGkxVz+vAz6z6WAR8aykEUXsZ2YZVYrb1CKiVtJY4BGgGrg1IuZKOj///fiIeFHSw8BzwEbgloh4oVC9Rd3SERGLG6Vu3Zb8CDNr+0Tpbr6NiKnA1EafjW+0fQ1wTbF1FhNqiyUNByLfRbyYFrp/ZpZtaZ4mVcx9aucDF5K7/LoU+Hh+28y2QcXOJqjUTKoWe2oRUQOc2QptMbM2ok3P/ZS0h6TfS1opaYWk+yXt0RqNM7N0UpGvSihm+Plb4G6gD7ArcA9wVzkbZWbpVqJbOsqimFBTRPwmImrzrzsoMH3KzLItd/WzJHM/y6LQ3M/u+bd/zi8JMpFcmJ0GPNgKbTOzNFLbXSRyNrkQ29T6r9b7LoCry9UoM0u3NvmMgogY2JoNMbO2YdPwM62KmlEgaX9gX2D7TZ+1tPyHmWVXm+ypbSLpSuBIcqE2FRgF/BVwqJlto9IbacVd/TyF3FpGb0TEueSW/diurK0ys9SSoLpKRb0qoZjh57qI2CipVlJnYAXgm2/NtmFtevgJzMo//OAX5K6I/hP4WzkbZWbpluJMK2ru5wX5t+Pz6xp1jojnytssM0sroVTP/Sx08+1Bhb6LiDnlaZKZpVoFV+AoRqGe2rUFvgvg0yVuC5/YZwBPPH1jqau1Muo2dGylm2AJfDB/UUnqaZPn1CLiqNZsiJm1DQKq22KomZk1p83PKDAzq8+hZmaZkVuqO72pVszKt5J0lqQr8tsDJA0rf9PMLK3SvJ5aMdOkbgYOA07Pb68FxpWtRWaWem36wSvAIRFxkKS/A0TEqvyj8sxsGySgXYqHn8WE2gZJ1eSX8JbUk9yTks1sG5XiTCsq1G4Afgf0kvQDcqt2fKesrTKz1JLa6DSpTSLiTkmzyS0/JOCEiPAT2s22YSnOtKIWiRwAvAf8vv5nEVGa+RZm1ua09fvUHuTDB7BsDwwE5gP7lbFdZpZSgootAFmMYoafg+tv51fv+Gozxc0s6yp4D1oxEs8oiIg5koaWozFm1jYoxU8pKOac2tfrbVYBBwEry9YiM0u1LDwir1O997XkzrFNKk9zzKwtaLOhlr/pdqeIuKSV2mNmbUCaJ7QXWs67XUTUFlrW28y2PblH5FW6Fc0r1LRNT4x6RtIUSWdLOmnTqzUaZ2bpVJWfVdDSqyWSRkqaL2mBpMsKlBsqqU7SKS3VWcw5te7AW+SeSbDpfrUAJhexr5llTKkuFORPb40DjgGWADMlTYmIeU2U+zHwSDH1Fgq1Xvkrny/wYZhtEgnabmYZU6JTasOABRGxMFenJgKjgXmNyl1E7uJkUbeSFQq1amAnaPKGFIea2TZLVBV/n1oPSbPqbU+IiAn5932BxfW+WwIc0uBIUl/gRHIjxa0OteURcVUxlZjZtkMk6qnVRMSQAlU11rjDdB3wzYioK/aKa6FQS+81WzOrHEG70tyotgToX2+7H7CsUZkhwMR8oPUAjpNUGxH3NVdpoVA7esvaaWZZlrCnVshMYC9JA4GlwBjgjPoFImLg5uNKtwEPFAo0KPww47e3orFmlmGlWCQyfx/sWHJXNauBWyNirqTz89+P35J6/Yg8M0usVBMKImIqMLXRZ02GWUR8sZg6HWpmlogo7jF0leJQM7NkVJrhZ7k41MwskdyMAoeamWVIeiPNoWZmWyDFHTWHmpklpba5npqZWVN89dPMMscXCswsO9RGl/M2M2uKh59mljnuqZlZpqQ30hxqZpaQgGr31MwsS1KcaQ41M0tKKMUDUIeamSXmnpqZZUbulo70pppDzcySkXtqZpYxniZlZpmRWySy0q1onkPNzBLz1U8zy5QUjz4das35w4x5fOvae6nbuJGzRw/na188tsH3EcFl197LtCfm0nH7Dtx85dkcuHf/gvuuWvMuX/r2rSxa/jYD+nTnVz/8Ml0778CG2jou/v6dPPuPxdTVbeS044bx9XM/A8DVN09h4oN/Y83a91gy/b9b94+wDbrx8jP5zCf3p2bVWoaP+f+Vbk5qpbmnVrbJ9pJulbRC0gvlOka51NVt5JKf3M0911/AU3d/h0mPzuYfC5c3KDNtxjxeWbSS2ZOv5Lpvn843fjSxxX1/9utpjBg6iNmTr2TE0EH87NePAnDfH+bwwfpaZkz8L/78m29y2++eYNGytwAYecRg/vjrS1rx12/b7nrgKU65eFylm5Fqm86pFfOqhHKuIHIbMLKM9ZfN7LmvsUf/Huzerwcd2rfjpGMOYupfnmtQZupfnmPMZ4chiaGDB7Jm7TreqFlTcN+H/vIcp3/uEABO/9whTH0s97kk3lu3ntraOt5/fz0d2lfTacftARg6eCC79OjSir9+2zbj76+w6p33Kt2MdJOoKvJVCWULtYiYDrxdrvrLafnKNfTt3W3z9q69u7F85ZpGZVY3LNOrK8tXrC6474q3124OqF16dGHlqrUAjD76E+zQsQN7j/ovBh9/BWPPPJpuXXYs2+8z21oq8lUJFT+nJuk84DyA/gMGVLg1ORHxkc8a/6PTRBEkFbVvY7PnvkZ1VRUvPvQDVr/zHsf92884ctje7N6vR5Jmm7WKtD/3s+ILWEbEhIgYEhFDevboWenmALle19I3V23eXvbmqo8MAT9SZsVqdunZpeC+vbp34o2aXK/tjZo19OzWCYB7H57F0cP3pX27anp278QhB+7B319cVLbfZ7a10txTq3iopdFB++7GK4tW8vrSGtZvqGXytDmMGnFAgzKjRgxm4oN/IyKY+fyrdN6pI7v06FJw35EjBnPXA08DcNcDTzPqU7nP++3SncdnzicieHfdB8x64TX22r136/5osyRSnGoVH36mUbt21fzk0s9z8sXjqKsLzvzXQ9lnzz7cOulxAL508hEce/h+THtiLged+D06bt+ecVecVXBfgK+dcwznfutW7pjyJP16d+O2H30ZgK+cOoKxV93B8NN+QABnHH8o++/VF4ArbriPSY/M4r33N7DfZ7/D2aMP47LzPtv6f5RtxC3f/yKHH7wXO3fdiRceuJofTZjKHVOerHSzUifNw081dQ6oJBVLdwFHAj2AN4ErI+KXhfY5+OAh8cTTs8rSHiuPbkPHVroJlsAH8+9m43srtiqR9hn8ibj9/seKKjtsz66zI2LI1hwvqbL11CLi9HLVbWYVlt6OmoefZpZM7nRZelPNoWZmyaR8PTVf/TSzxEp18VPSSEnzJS2QdFkT358p6bn8a4akA1uq0z01M0tIJXmYsaRqYBxwDLAEmClpSkTMq1fsVeBTEbFK0ihgAnBIoXodamaWWImGn8OABRGxMFenJgKjgc2hFhEz6pV/CujXUqUefppZIsUOPfO510PSrHqv8+pV1RdYXG97Sf6z5nwZeKil9rmnZmbJFd9Tqylwn1pTtTR546yko8iF2idbOqBDzcwSK9EtHUuA/vW2+wHLPnIs6QDgFmBURLzVUqUefppZYlJxrxbMBPaSNFBSB2AMMKXhcTQAmAycHREvFdM299TMLJkS3acWEbWSxgKPANXArRExV9L5+e/HA1cAOwM356+41rY07cqhZmaJlWpGQURMBaY2+mx8vfdfAb6SpE6HmpklItI9o8ChZmaJpTjTHGpmtgVSnGoONTNLLM2LRDrUzCyx9EaaQ83MtkSKU82hZmaJeJFIM8uWlC8S6VAzs8RSnGkONTNLqjSLRJaLQ83MEktxpjnUzCyZCj58vSgONTNLLsWp5lAzs8R8S4eZZYrPqZlZdgiqHGpmli3pTTWHmpkl4kUizSxzUpxpDjUzS849NTPLFE+TMrNMSW+kOdTMLKEiH1RcMQ41M0vMMwrMLFvSm2kONTNLLsWZ5lAzs6TkR+SZWXakfUZBVaUbYGZWSu6pmVliae6pOdTMLDHf0mFm2eGbb80sS9J+ocChZmaJefhpZpninpqZZUqKM82hZmZbIMWp5lAzs0QEqZ4mpYiodBs2k7QSeL3S7SiDHkBNpRthiWT1v9luEdFzayqQ9DC5v08xaiJi5NYcL6lUhVpWSZoVEUMq3Q4rnv+btV2e+2lmmeJQM7NMcai1jgmVboAl5v9mbZTPqZlZprinZmaZ4lAzs0xxqJWRpJGS5ktaIOmySrfHWibpVkkrJL1Q6bbYlnGolYmkamAcMArYFzhd0r6VbZUV4TagVW8WtdJyqJXPMGBBRCyMiPXARGB0hdtkLYiI6cDblW6HbTmHWvn0BRbX216S/8zMysihVj5Nzfj1/TNmZeZQK58lQP962/2AZRVqi9k2w6FWPjOBvSQNlNQBGANMqXCbzDLPoVYmEVELjAUeAV4E7o6IuZVtlbVE0l3Ak8AgSUskfbnSbbJkPE3KzDLFPTUzyxSHmpllikPNzDLFoWZmmeJQM7NMcai1IZLqJD0j6QVJ90jaYSvquk3SKfn3txSabC/pSEnDt+AYr0n6yFOHmvu8UZl/JjzWdyX9Z9I2WvY41NqWdRHx8YjYH1gPnF//y/zKIIlFxFciYl6BIkcCiUPNrBIcam3X48C/5HtRf5b0W+B5SdWSrpE0U9Jzkr4KoJybJM2T9CDQa1NFkh6TNCT/fqSkOZKelfRHSbuTC8+v5XuJR0jqKWlS/hgzJR2e33dnSY9K+rukn1PEc7wl3SdptqS5ks5r9N21+bb8UVLP/Gd7Sno4v8/jkvYuyV/TMsNPaG+DJLUjt07bw/mPhgH7R8Sr+WBYExFDJW0HPCHpUeATwCBgMNAbmAfc2qjensAvgBH5urpHxNuSxgP/jIif5sv9FvhZRPxV0gBysyb2Aa4E/hoRV0n6LNAgpJrxpfwxOgIzJU2KiLeAHYE5EfENSVfk6x5L7oEo50fEy5IOAW4GPr0Ff0bLKIda29JR0jP5948DvyQ3LPxbRLya//xY4IBN58uALsBewAjgroioA5ZJ+lMT9R8KTN9UV0Q0t67Y/wP2lTZ3xDpL6pQ/xkn5fR+UtKqI33SxpBPz7/vn2/oWsBH43/zndwCTJe2U/7331Dv2dkUcw7YhDrW2ZV1EfLz+B/n/c79b/yPgooh4pFG542h56SMVUQZypy0Oi4h1TbSl6Hl3ko4kF5CHRcR7kh4Dtm+meOSPu7rx38CsPp9Ty55HgH+X1B5A0sck7QhMB8bkz7n1AY5qYt8ngU9JGpjft3v+87VAp3rlHiU3FCRf7uP5t9OBM/OfjQK6tdDWLsCqfKDtTa6nuEkVsKm3eQa5Ye07wKuSTs0fQ5IObOEYto1xqGXPLeTOl83JPzzk5+R65L8DXgaeB/4H+EvjHSNiJbnzYJMlPcuHw7/fAyduulAAXAwMyV+ImMeHV2G/B4yQNIfcMHhRC219GGgn6TngauCpet+9C+wnaTa5c2ZX5T8/E/hyvn1z8RLp1ohX6TCzTHFPzcwyxaFmZpniUDOzTHGomVmmONTMLFMcamaWKQ41M8uU/wMPzjgegkS8TwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.7187814702920443\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5991    0.4899    0.5390      1333\n",
      "           1     0.7641    0.8344    0.7977      2639\n",
      "\n",
      "    accuracy                         0.7188      3972\n",
      "   macro avg     0.6816    0.6621    0.6683      3972\n",
      "weighted avg     0.7087    0.7188    0.7109      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcSElEQVR4nO3deZgV1bnv8e+vuwGRGRlUEMQZNGoQpwzEISZokuvRaOJwkpwkXqM3JM9NcpJjhpvx5InRm3Nyc9RLiPGaUWMUI4koDomzRpA4gaKIoi3KJCCTQtPv/WMXuLvp3rtK9u5dvfv34ann2VW19qq36YeXtapqraWIwMysXjTUOgAzs0pyUjOzuuKkZmZ1xUnNzOqKk5qZ1ZWmWgdQrO/AITFwxKhah2EZtPrpebeybsVS3nh9tXamjsaBYyNaNqUqG5tWzI6IKTtzvaxyldQGjhjFOf9xfa3DsAw2vrm11iFYBjMu+thO1xEtm+hzYLp63nj08mE7fcGMcpXUzKw7ECi/d66c1MwsGwENjbWOolNOamaWnXbqtlxVOamZWUbufppZvXFLzczqhsh1Sy2/kZlZTqnQUkuzlatJmiJpoaRFki7q4PwgSX+W9Jik+ZI+Xa5Ot9TMLLsKPP2U1AhcDpwENANzJM2MiAVFxT4PLIiIj0gaDiyU9LuI2NxpaDsdmZn1MMmDgjRbaUcBiyJicZKkrgVObVcmgAGSBPQHXgNaSlXqlpqZZSMq9aBgFPBS0X4zcHS7MpcBM4GlwADg4xHRWqpSt9TMLLv0LbVhkuYWbecX19JBze0HE38QeBTYEzgcuEzSwFKhuaVmZhllek9tZURM6uRcM7BX0f5oCi2yYp8GLo7CugOLJD0PHAQ83NkF3VIzs2wENDam20qbA+wvaZyk3sBZFLqaxV4ETgSQNBI4EFhcqlK31MwsuwrcU4uIFklTgdlAI3BVRMyXdEFyfhrwA+BqSU9QSKf/FhErS9XrpGZmGVVumFREzAJmtTs2rejzUuADWep0UjOz7DxMyszqSo6HSTmpmVk2KYdA1YqTmpll50kizax+eD41M6s37n6aWd3I+XxqTmpmlpG7n2ZWb/ygwMzqiu+pmVndkLufZlZv3FIzs3oiJzUzqxeF2byd1MysXkiowUnNzOqIW2pmVlec1MysrjipmVn9EB0vbpcTTmpmlomQW2pmVl8aGjyiwMzqiFtqZlY/fE/NzOpNnltq+e0Ym1kubXtQkGYrW5c0RdJCSYskXdTB+a9KejTZnpS0VdLQUnU6qZlZZmpQqq1kHVIjcDlwMjABOFvShOIyEXFpRBweEYcDXwfujojXStXrpGZm2YhKtdSOAhZFxOKI2AxcC5xaovzZwDXlKnVSM7PMMiS1YZLmFm3nF1UzCnipaL85OdbR9XYFpgA3lIvNDwrMLLMMDwpWRsSkzqrp4Fh0UvYjwP3lup7gpGZmGVVwREEzsFfR/mhgaSdlzyJF1xPc/TSzt0Mpt9LmAPtLGiepN4XENXOHS0mDgPcBN6UJzS01M8tGlRkmFREtkqYCs4FG4KqImC/pguT8tKToacBtEbEhTb1OamaWWaVevo2IWcCsdsemtdu/Grg6bZ1OamaWXX4HFDip7YwXn13C/bfcS0QwfuIE3vneIzost/zlZdz4i+t5/5kfZN+D9wPg8Qcf46l58yFg/BETOPTYw7sw8p7roBH9Of3Q3WkQPLRkDXc8s7LN+f2G7cp5x4xh1YbNADy+dB2zF64A4OyJe3Lw7gNY/2YLF9/5XJfHnid5HiZV1aQmaQrwfyj0l6+MiIureb2u1Nrayn03382HP3kq/Qb2Z8b06xh74DiGjhi6Q7mHbn+A0fuN2X7stWWreGrefE7/72fS2NjIzb+dyZgD9mbwboO7+KfoWQScedgeXHH/C6zZ1MJXjt+HJ15Zx7J1b7Ypt3jVRqY/+OIO3394yRrufe41/nlSh69S9Rhph0DVStWefqYZAtGdLX95GQOHDmLg0EE0NjWy7yH788LTi3co9+TfH2ef8fvSt1/f7cdWr1zNyNG706t3LxoaG9hz7Cief2rH71pljR3alxUbNrNq4xa2RjCveS3v2GNA6u8/t2ojG7dsrWKE3Uelxn5WQzVf6cg6BKJb2fD6BvoPeusfRP9B/dmwru3DmfWvr+f5pxYz4chD2hwfOmIoryx5mTc2bmLL5i28+OwLbFi7rkvi7skG7dKLNZu2bN9fs2kLg3bZsbOy99C+fO2EffncsWPYfUCfrgyx26jE2M9qqWb3s6MhEEe3L5QMmzgfYMDwPasYTvW1/xU+cMu9HHPSu3Z4/D1k+FAOf/cR/OXXM+nVuxe77T4M5Xgm0XqR5p/YS2ve4Lu3Psvmra1MGNmf847Zi3+/fVHVY+tu8tz9rGZSSzUEIiKmA9MBRu53SGdDJHKn38B+rC9qXa1fu55dB/RrU2bF0uXcfv1sAN7Y+AYvPruEhoYGxo3fh/FHTGD8EYXe+N/veJB+A/t3XfA91Jo3tjC4b6/t+4P79mLtGy1tyrzZ0rr984Jl6zlDol/vRjZsdrdzO/XcpJZlCES3M2LPkax9bS2vr36dfgP68dyTz3LiGR9oU+bcL31q++e/3ngHYw/Ym3Hj9wFg0/qN9O2/K+vWrOP5p57jtPPO6NL4e6IXV29ieP/eDN21F2s3tTBx9CB+Pae5TZkBfZpY92Yh0Y0Z0pcG4YTWjoAc57SqJrXtQyCAlykMgTinitfrUg2NDbznlMnc/JubiNbgwHdOYOiI3Zg/50kADm53H6292X+4hTc3vUFDQwPv+dD76NN3l64Iu0drDbjhsVe48N1jaUA8tGQ1r657k3fvPQSA+19YzeGjBvLucUNoDdiytZWri5LeJyeNZr/hu9K/dxPfm3IAtzy1nIeWrKnRT1NL+X76qYjq9fgknQL8lLeGQPywVPmR+x0S5/zH9VWLxypv45tuxXQnMy76GCuem79TGWmX3Q+IsZ/6r1Rln7lkyiMlZumoiqq+p9bREAgz6+bUc7ufZlaHBDTU6HWNNJzUzCwzt9TMrK7k+UGBk5qZZeN7amZWT4QqMklktTipmVlmbqmZWV3xPTUzqx++p2Zm9aQw9jO/Wc1Jzcwyy3FOc1Izs+w8osDM6kfO51PL78smZpZL2+ZTS7OVrUuaImmhpEWSLuqkzHGSHpU0X9Ld5ep0S83MMqrMfGpFizOdRGFS2TmSZkbEgqIyg4ErgCkR8aKkEeXqdUvNzDKrUEstzeJM5wAzIuJFgIhYXq5SJzUzy0aFBwVptjI6Wpyp/aKqBwBDJN0l6RFJnyxXqbufZpZJxvfUhkmaW7Q/PVlsaVtV7bWfirsJOAI4EegLPCjpoYh4prMLOqmZWWYZktrKEtN5p1mcqTmpYwOwQdI9wGFAp0nN3U8zy6xC99S2L84kqTeFxZlmtitzE/BeSU2SdqWwdvBTpSp1S83MMqvE08+IaJE0FZjNW4szzZd0QXJ+WkQ8JelW4HGgFbgyIp4sVa+TmpllU8EB7R0tzhQR09rtXwpcmrZOJzUzy6QwSWR+RxQ4qZlZZg05HiblpGZmmeU4pzmpmVk2yvmAdic1M8ssx7fUOk9qkv6LHd/u3S4ivliViMws97rrg4K5Jc6ZWQ8lCk9A86rTpBYRvyrel9QvGapgZj1cjhtq5YdJSTpW0gKSoQmSDpN0RdUjM7N8UmE+tTRbLaQZ+/lT4IPAKoCIeAyYXMWYzCznKjXzbTWkevoZES+1y7pbqxOOmeWd6P4v374k6V1AJCPpv0iZUfJmVt/y/PQzTffzAuDzFGakfBk4PNk3sx4obdczt93PiFgJnNsFsZhZN5Hn7meap5/7SPqzpBWSlku6SdI+XRGcmeWTUm61kKb7+XvgOmAPYE/gj8A11QzKzPKtu7/SoYj4TUS0JNtvKTF8yszqW+HpZ7qtFkqN/RyafPxbsnLytRSS2ceBm7sgNjPLI3XfSSIfoZDEtkX/uaJzAfygWkGZWb51y6mHImJcVwZiZt3Dtu5nXqUaUSDpEGACsMu2YxHx62oFZWb51i1battI+g5wHIWkNgs4GbgPcFIz66Hym9LSPf08g8KS769GxKcprI7cp6pRmVluSdDYoFRbLaTpfm6KiFZJLZIGAssBv3xr1oPlufuZpqU2V9Jg4BcUnojOAx6uZlBmlm+VGvspaYqkhZIWJa+OtT9/nKS1kh5Ntm+XqzPN2M//kXycliz/PjAiHi8frpnVI6GKjP2U1AhcDpwENANzJM2MiAXtit4bER9OW2+pl28nljoXEfPSXsTM6kjlZuA4ClgUEYsBJF0LnAq0T2qZlGqp/aTEuQBO2JkLd2T0oF340SkHVbpaq6IhR06tdQiWwZuvrKhIPRnuqQ2TVLyI0/SImJ58HgW8VHSuGTi6gzqOlfQYsBT414iYX+qCpV6+PT5dzGbWkwhoTJ/UVkbEpBJVtdd+XPk8YGxErJd0CvAnYP9SF0zzoMDMrI0KDWhvBvYq2h9NoTW2XUS8HhHrk8+zgF6ShpWMLfNPY2Y9XoWS2hxgf0njkqUCzgJmFheQtLuSvq6koyjkrFWlKk01TMrMbJvC6xo7/6QgIlokTQVmA43AVRExX9IFyflpFF7+v1BSC7AJOCsiSk59lmaYlChM571PRHxf0hhg94jwu2pmPVSlBgskXcpZ7Y5NK/p8GXBZpthSlLkCOBY4O9lfR+HdEjProbr1wivA0RExUdI/ACJiddL/NbMeSEBTjodJpUlqW5I3fwNA0nCgtapRmVmu5TinpUpqPwNuBEZI+iGFG3ffqmpUZpZbUmWGSVVLmrGfv5P0CIXphwT8U0R4hXazHizHOS3V088xwEbgz8XHIuLFagZmZvnV3afzvpm3FmDZBRgHLAQOrmJcZpZTgppNAJlGmu7nO4r3k9k7PtdJcTOrdzVc0zONzCMKImKepCOrEYyZdQ/K8SoFae6pfblotwGYCFRm/hIz63bqYYm8AUWfWyjcY7uhOuGYWXfQbZNa8tJt/4j4ahfFY2bdQJ4XXik1nXdTMoq+02m9zaznKSyRV+soOleqpfYwhftnj0qaCfwR2LDtZETMqHJsZpZT3XpEATCUwqRsJ/DW+2oBOKmZ9UDd+UHBiOTJ55O8lcy2KTlJm5nVtxw31EomtUagP+kWRzCzHkM0dNP31F6JiO93WSRm1i2I7ttSy3HYZlYzgqYc31QrldRO7LIozKzb6LYttYh4rSsDMbPuo7u/0mFm1kaOc5qTmpllI/K9CnqeYzOzPFKh+5lmK1uVNEXSQkmLJF1UotyRkrZKOqNcnW6pmVkmhREFO9//TCbMuBw4CWgG5kiaGRELOij3YworuZfllpqZZaaUWxlHAYsiYnFEbAauBU7toNwXKEx3tjxNbE5qZpZZhVZoHwW8VLTfnBwruo5GAacB09LG5u6nmWWkLPOpDZM0t2h/ekRM317RjtoPwfwp8G8RsTXtNZ3UzCyTjE8/V0bEpE7ONQN7Fe2PBpa2KzMJuDZJaMOAUyS1RMSfOrugk5qZZVahl2/nAPtLGge8DJwFnFNcICLGbfss6WrgL6USGjipmVlWqsx03snM2lMpPNVsBK6KiPmSLkjOp76PVsxJzcwyqeTLtxExC5jV7liHySwi/iVNnU5qZpZZt1x4xcysM/lNaU5qZpaRgEa31MysnuQ4pzmpmVlWQjnugDqpmVlmbqmZWd0ovNKR36zmpGZm2aQbrF4zTmpmlpnXKDCzulGYJLLWUXTOSc3MMvPTTzOrKznufTqp7Yw7HljA139yPVtbW/nEqe/iS//ygTbnn3nhVaZ+/7c89nQz37rww3zhE+8H4NkXlvGZb1y1vdySpav4+vkf4sJzju/S+HuiE48dz4++cgaNDQ385qYH+Omvbm9zfmC/Xfj5Dz7F6JFDaGxq5LLf3snv//wQfXo3cfP0/0mfXk00NjUy885/cPH0WZ1cpf71yJaapKuADwPLI+KQal2nVrZubeWrl1zHjZdNZc+RgznhU5dy8uR3cNA+e2wvM2RgPy7+ypncfPdjbb67/94juff3X99ez4RTvsmHjj+sS+PviRoaxKVf+xinTb2MpcvW8NdffZVb7nmChc+/ur3MeWdOZuHiVzn7yz9nt8H9mXP9/+KPt8zhzc0tnHrhz9iwaTNNjQ3ccuWXueOBBcx98oXa/UA1kvd7atVco+BqYEoV66+pR+a/wD57DWPv0cPo3auJ00+ayKy7H29TZvjQAUw8eCy9mho7refuOQvZe/RwxuwxtNoh93hHHLw3i19ayZKXV7GlZSszbp/HKe87tE2ZAPr36wNAv137sPr1jbRsbQVgw6bNAPRqaqRXUyMR7Wee7iFSLo9XqyekVUtqEXEP8Fq16q+1V1asZdTIIdv39xw5hFdWrM1cz4zbHuGjHzyikqFZJ/YYPoiXl63evr902Wr2GD6oTZlfXHc3B+y9O0/d8kPuv+YbfP0n129PXg0N4p7fXcQzt13MXX9/mkfmL+nS+POkQqtJVUXNV5OSdL6kuZLmrli5otbhpNbR/9JZ/2PavKWFW+55gn868Z0VispK6WgOsPa/xhOOGc8TzzQz/uRvMvncH3HJV89kQL9dAGhtDSafezEHf+hbTDx4LOP33WOH+nqCbet+9riWWloRMT0iJkXEpOHDhtc6nNT2HDF4h//1dx82qMQ3dnTHAws47KC9GLHbwEqHZx1YunzNDq3rV1e2bV2f+5Fj+MvfCvdAn29eyZKlq9h/7Mg2ZV5fv4n7HnmWE4+dUP2gc8ottTo0ccJYnntxBUteXsnmLS3MuH0eJ08+tPwXi1w/ey4f/YC7nl1l3oIl7DtmOGP23I1eTY2cftJEbrmn7X3Q5ldXM/nIA4HCPdH9xo7khZdXstvg/gzs3xeAXfr04rijDuTZF5Z1+c+QGznOan6l421qamrkkq99jI9+8XK2bg3O/W/HMH7fPbjqhnsB+MxH38uyla9zwqcuYd2GN5DEtGvv4sE/fJOB/fuy8Y3N3PXw0/znN86u8U/Sc2zd2srXLrmOG372eRobxe9mPsTTi1/l06e/B4D/N+M+Lv3lrVz+nX/m/mu+gQTfu+wmXlu7gYP325MrvvsJGhsaaGgQN94xj9n3PVnjn6h28jxMStV6giPpGuA4Cmv1LQO+ExG/LPWdI46YFPf/fW6pIpYzQ46cWusQLIM3F15H68blO5WRxr/jnfHrm+5KVfaofQc/UmLdz6qoWkstItwEMatX+W2ouftpZtkUbpflN6s5qZlZNjmfT81PP80ss0o9/JQ0RdJCSYskXdTB+VMlPS7p0eR91veUq9MtNTPLSBVZzFhSI3A5cBLQDMyRNDMiFhQVuxOYGREh6VDgOuCgUvW6pWZmmUnptjKOAhZFxOKI2AxcC5xaXCAi1sdbr2j0ozA8tyQnNTPLJG3XM8lpw7YNg0y284uqGgW8VLTfnBxrez3pNElPAzcDnykXn7ufZpZd+t7nyhLvqXVUyw4tsYi4EbhR0mTgB8D7S13QLTUzy0wp/5TRDOxVtD8aWNpZ4WTmn30lDStVqZOamWVWoXtqc4D9JY2T1Bs4C5jZ9jraT8lTCUkTgd7AqlKVuvtpZtlU6D21iGiRNBWYDTQCV0XEfEkXJOenAR8FPilpC7AJ+HiUGdvppGZmmVVqREFEzAJmtTs2rejzj4EfZ6nTSc3MMhH5HlHgpGZmmeU4pzmpmdnbkOOs5qRmZpnleZJIJzUzyyy/Kc1JzczejhxnNSc1M8vEk0SaWX3J+SSRTmpmllmOc5qTmpllVZlJIqvFSc3MMstxTnNSM7Nsarj4eipOamaWXY6zmpOamWXmVzrMrK74npqZ1Q9Bg5OamdWX/GY1JzUzy8STRJpZ3clxTnNSM7Ps3FIzs7riYVJmVlfym9Kc1Mwso5QLFdeMV2g3s8yU8k/ZeqQpkhZKWiTpog7Onyvp8WR7QNJh5ep0S83MsqtAS01SI3A5cBLQDMyRNDMiFhQVex54X0SslnQyMB04ulS9bqmZWWZKuZVxFLAoIhZHxGbgWuDU4gIR8UBErE52HwJGl6vULTUzy0iVWiJvFPBS0X4zpVthnwVuKVepk5qZZZJxRMEwSXOL9qdHxPSiqtqLDq8pHU8hqb2n3AWd1MysmlZGxKROzjUDexXtjwaWti8k6VDgSuDkiFhV7oK+p2ZmmW17raPcVsYcYH9J4yT1Bs4CZra9jsYAM4BPRMQzaWJzS83MMqvEJJER0SJpKjAbaASuioj5ki5Izk8Dvg3sBlyRjGJoKdHyA5zUzCyrCr58GxGzgFntjk0r+nwecF6WOp3UzCwTTz1kZnXHaxSYWV1xS83M6kqOc5qTmpm9DTnOak5qZpaJoFLDpKpCER2OSqgJSSuAJbWOowqGAStrHYRlUq+/s7ERMXxnKpB0K4W/nzRWRsSUnbleVrlKavVK0txyLwxavvh31n15mJSZ1RUnNTOrK05qXWN6+SKWM/6ddVO+p2ZmdcUtNTOrK05qZlZXnNSqqNzyX5Y/kq6StFzSk7WOxd4eJ7UqKVr+62RgAnC2pAm1jcpSuBro0pdFrbKc1Kqn7PJflj8RcQ/wWq3jsLfPSa16Olr+a1SNYjHrMZzUqif18l9mVjlOatWTavkvM6ssJ7XqKbv8l5lVnpNalUREC7Bt+a+ngOsiYn5to7JyJF0DPAgcKKlZ0mdrHZNl42FSZlZX3FIzs7ripGZmdcVJzczqipOamdUVJzUzqytOat2IpK2SHpX0pKQ/Stp1J+q6WtIZyecrSw22l3ScpHe9jWu8IGmHVYc6O96uzPqM1/qupH/NGqPVHye17mVTRBweEYcAm4ELik8mM4NkFhHnRcSCEkWOAzInNbNacFLrvu4F9ktaUX+T9HvgCUmNki6VNEfS45I+B6CCyyQtkHQzMGJbRZLukjQp+TxF0jxJj0m6U9LeFJLnl5JW4nslDZd0Q3KNOZLenXx3N0m3SfqHpJ+TYh1vSX+S9Iik+ZLOb3fuJ0ksd0oanhzbV9KtyXfulXRQRf42rW54hfZuSFIThXnabk0OHQUcEhHPJ4lhbUQcKakPcL+k24B3AgcC7wBGAguAq9rVOxz4BTA5qWtoRLwmaRqwPiL+d1Lu98B/RsR9ksZQGDUxHvgOcF9EfF/Sh4A2SaoTn0mu0ReYI+mGiFgF9APmRcRXJH07qXsqhQVRLoiIZyUdDVwBnPA2/hqtTjmpdS99JT2afL4X+CWFbuHDEfF8cvwDwKHb7pcBg4D9gcnANRGxFVgq6a8d1H8McM+2uiKis3nF3g9MkLY3xAZKGpBc4/TkuzdLWp3iZ/qipNOSz3slsa4CWoE/JMd/C8yQ1D/5ef9YdO0+Ka5hPYiTWveyKSIOLz6Q/OPeUHwI+EJEzG5X7hTKT32kFGWgcNvi2IjY1EEsqcfdSTqOQoI8NiI2SroL2KWT4pFcd037vwOzYr6nVn9mAxdK6gUg6QBJ/YB7gLOSe257AMd38N0HgfdJGpd8d2hyfB0woKjcbRS6giTlDk8+3gOcmxw7GRhSJtZBwOokoR1EoaW4TQOwrbV5DoVu7evA85LOTK4hSYeVuYb1ME5q9edKCvfL5iWLh/ycQov8RuBZ4Ang/wJ3t/9iRKygcB9shqTHeKv792fgtG0PCoAvApOSBxELeOsp7PeAyZLmUegGv1gm1luBJkmPAz8AHio6twE4WNIjFO6ZfT85fi7w2SS++XiKdGvHs3SYWV1xS83M6oqTmpnVFSc1M6srTmpmVlec1MysrjipmVldcVIzs7ry/wHOQJjaoiRDcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammWord(clf_nn_word, model_name=\"clf_nn_word\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "id": "A5_oAXrTUBXe"
   },
   "outputs": [],
   "source": [
    "grid_search_nn = GridSearchCV(clf_nn, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "id": "LOA-K1MOUBfY"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV] END .................................................... total time= 7.3min\n",
      "[CV] END .................................................... total time=12.5min\n",
      "[CV] END .................................................... total time=10.5min\n",
      "[CV] END .................................................... total time=13.4min\n",
      "[CV] END .................................................... total time= 9.7min\n"
     ]
    }
   ],
   "source": [
    "\n",
    "clf_nn_char = grid_search_nn.fit(X_train_char, y_train_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "id": "ZLTitSEeUBo3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.9961156668105309\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9961    0.9923    0.9942      3114\n",
      "           1     0.9961    0.9981    0.9971      6154\n",
      "\n",
      "    accuracy                         0.9961      9268\n",
      "   macro avg     0.9961    0.9952    0.9956      9268\n",
      "weighted avg     0.9961    0.9961    0.9961      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZYElEQVR4nO3deZxU1Z338c+3u1Ex7GGRVdHBBcWFzS0aHR8NOpkxbnEfo8mTOBF9PRPj8uRJTKJZNb5GozgM4xDjEhkdUFGJokkMKhpZXFgMiiCrCrgQFxS7+/f8UQV2N93VdaGq6/bl+86rXq+6VeeeewrMl3PuvedcRQRmZllRVekGmJmVkkPNzDLFoWZmmeJQM7NMcaiZWabUVLoBDammY2iHzpVuhiVw4D6DKt0ES2D5stdZt26dtqWO6i67RtRuKKpsbFj7aESM2ZbjJZWuUNuhMzvufXqlm2EJPP3sryvdBEvg8ENGbXMdUbuBHff6alFlP35hXM9tPmBCqQo1M2sPBErvmSuHmpklI6CqutKtaJFDzcyS0zadlisrh5qZJeThp5lljXtqZpYZwj01M8sSuadmZhnjq59mlh2+UGBmWSI8/DSzjHFPzcyyw8NPM8sSAdW+UGBmWeJzamaWHR5+mlnWuKdmZpninpqZZYY8TcrMssbTpMwsO3yhwMyyxsNPM8sMr6dmZtni4aeZZY0vFJhZpvicmpllhjz8NLOscU/NzLJEDjUzy4rcat4ONTPLCglVOdTMLEPcUzOzTHGomVmmONTMLDuUf6WUQ83MEhFyT83MsqWqyjMKzCxD3FMzs+xI+Tm19PYhzSy1JBX1KqKeMZIWSVos6cpmvu8q6UFJL0paIOn81up0qJlZIpsuFGxrqEmqBsYBxwNDgTMlDW1S7CJgYUQcABwFXC9ph0L1OtTMLDFVqahXK0YDiyNiSURsBCYBJzYpE0Bn5RKyE/AOUFuoUp9TM7NkVLILBf2BFQ22VwIHNylzMzAVWA10Bk6PiPpClbqnZmaJJRh+9pQ0u8Hrmw2raabqaLL9JeAFoB9wIHCzpC6F2uaempkllqCnti4iRrbw3UpgYIPtAeR6ZA2dD/wiIgJYLGkpsDfwXEsHdE/NzBIp1YUCYBYwRNLg/Mn/M8gNNRtaDhwDIKkPsBewpFCl7qmZWXIlOKUWEbWSxgKPAtXAxIhYIOnC/PfjgWuA2yTNyx/1iohYV6heh5qZJaPSTZOKiGnAtCafjW/wfjVwXJI6HWpmlpinSZlZtqQ30xxq2+KYQ/bh55eeQnVVFXc88Aw33P5Yo++7du7IzT84m8H9e/LxxlouvuYuXl7yBgDfOv2LnPeVw0Di9vtnMn7SExX4Bdn1+DML+d71k6mrr+fcEw/l/5zXeAQTEfzf6yfz2MwFdNxpB8ZddQ4H7D2w4L4XfG8ii5etAWD9Bxvo2qkjM+66knsfmcVNd/xhc90LFq/miTsuZ9ieA9ro17a97banJmkMcCO5k4C3RsQvynm8tlRVJa67/DROGjuO1Wve44+/vYzfPzmPRUvf3Fzm0q8dx7xXVnHu5bcyZNc+XHf5aXzlopvZZ/e+nPeVwzjma79iY20d/3Pjt5n+9AKWrFhbwV+UHXV19Vx+7b1Mufki+vXuxjHnXceYI4ax9+59N5d5fOZCXluxhtmTr2L2/Ne59Jf/zeO/+W7BfSf+7ILN+3//hil06dQRgNPGjOK0MaMAWLh4NWd/d0LmAy3NoVa2WzqKnNfVbo3Yd1eWrFzHstVv82ltHVOmz+GEI4c1KrPX4L7MmLUIgFeXvcWgvj3o1aMzew7uw6z5r7Phk0+pq6vn6bmv8uWj9q/Ez8ikOQuWMXhAT3br35MdOtRw8nEj+P2MeY3KTJsxjzNOGI0kRg0bzN/e38Cb69YXtW9EcP/jz3PKcSO2OPbk6bOb/TxrSjWhvRzKeZ9aMfO62q2+vbqx6q13N2+vXvMefXt1a1Rm/qur+PLRBwAwfOiuDNylB/16d+Pl197gsIP+ju5dd6bjjh049vB96d+ne1s2P9PeWPteoz/Pfr278cba9xqXWdNMmTXri9r3medfo3ePzuwxqPcWx77vsec5+UvbQaiVZu5nWZRz+FnMvC7y0yZyUyc6dCpjc0qruX+EoskMjxtuf4yff+cUZtx5BQsXr+alV1ZSV1fPK6+/xY23P8Z9N43lww2fsODVVdTWFZzOZglE04k25G4YbVRmi9k4ub/TYvadPH1Os8E1e/7rdNypA0P36Jeswe1Qmoef5Qy1YuZ1ERETgAkAVTv3buY/qXRa3cy/9G+uXd+ozPsffszYa+7avP3i/T9i2eq3Abhz6rPcOfVZAH7wL//I6jXvlb/R24l+vbfsRe/Sq2uTMt2bLbPx07qC+9bW1vHQEy/yx99etsVxp0yfs10MPUs4ob0syjn8LGZeV7s1d+Fy9hjYi0H9Pk+HmurcuZcnG5976dKpIx1qqgH45xMPY+YLr/H+hx8D0LN7rlc6oE93vnz0AfzP9Nlt+wMybPjQQSxZsZZlq9ax8dNapkyfw5gjGp/vPP6I/Zg07TkiglnzltKl007s0rNrq/s+MWsRQ3bts8Xpgvr6eh744wucvB2Emsj1aot5VUI5e2qb53UBq8jN6zqrjMdrU3V19Vx+3b1M/vW3qa4Sdz34LH9d8ibnn3w4AL+Z8jR7De7Dv//wXOrqg0VL3+Tin3zWa7v9l9+ge5edqa2r57Lr7mH9+xsq9VMyp6ammmsvO41TL7mFuvrg7H88hH326MtvJj8FwPmnfIFjD9+Xx2YuZMTJV9Nxpw7c/INzCu67yX0t9MZmPv8a/Xp3Y7f+PdvmR1ZUuq9+Kpo7iVCqyqUTgBv4bF7XTwuVr9q5d+y49+lla4+V3jt/+XWlm2AJHH7IKObOmb1NibTTLnvGrufdVFTZV64dM6fAKh1lUdb71Jqb12Vm7VwFh5bF8IwCM0tE5G4+TyuHmpkl5p6amWVKmi8UONTMLBmfUzOzLBEq2SKR5eBQM7PE3FMzs0zxOTUzyw6fUzOzLMnN/UxvqjnUzCyxFGeaQ83MkvOMAjPLjpSvp+ZQM7NENq2nllYONTNLKN3rqTnUzCyxFGeaQ83MEpIvFJhZhvg+NTPLHIeamWVKijPNoWZmybmnZmbZ4QntZpYluUUi05tqDjUzS6wqxV219K7Ja2apJRX3ar0ejZG0SNJiSVe2UOYoSS9IWiDpz63V6Z6amSWiEk1ol1QNjAOOBVYCsyRNjYiFDcp0A24BxkTEckm9W6vXPTUzS6xKxb1aMRpYHBFLImIjMAk4sUmZs4ApEbEcICLWtFZpiz01STcB0dL3EXFJq002s0xKcKGgp6TZDbYnRMSE/Pv+wIoG360EDm6y/55AB0lPAJ2BGyPi9kIHLDT8nF3gOzPbToncFdAirYuIkQWqaqppR6oGGAEcA3QEnpH0bES80tIBWwy1iPhto6NLn4uID1sqb2bbjxLd0bESGNhgewCwupky6/LZ86GkGcABQIuh1uo5NUmHSloIvJzfPkDSLQkbb2ZZodx6asW8WjELGCJpsKQdgDOAqU3KPAAcIalG0s7khqcvF6q0mKufNwBf2nSwiHhR0pFF7GdmGVWK29QiolbSWOBRoBqYGBELJF2Y/358RLws6RHgJaAeuDUi5heqt6hbOiJiRZPUrduaH2Fm7Z8o3c23ETENmNbks/FNtq8Driu2zmJCbYWkw4DIdxEvoZXun5llW5qnSRVzn9qFwEXkLr+uAg7Mb5vZdqjY2QSVmknVak8tItYBZ7dBW8ysnWjXcz8l7S7pQUlrJa2R9ICk3duicWaWTiryVQnFDD9/B9wD9AX6AfcCd5ezUWaWbiW6paMsigk1RcQdEVGbf91JgelTZpZtuaufJZn7WRaF5n72yL/9U35JkEnkwux04OE2aJuZpZHa7yKRc8iF2KbWf6vBdwFcU65GmVm6tctnFETE4LZsiJm1D5uGn2lV1IwCSfsBQ4GdNn3W2vIfZpZd7bKntomkHwJHkQu1acDxwFOAQ81sO5XeSCvu6uep5NYyejMizie37MeOZW2VmaWWBNVVKupVCcUMPzdERL2kWkldgDWAb74124616+EnMDv/8IP/JHdF9APguXI2yszSLcWZVtTcz2/n347Pr2vUJSJeKm+zzCythFI997PQzbfDC30XEXPL0yQzS7UKrsBRjEI9tesLfBfA35e4LRy0zyCe/stNpa7Wyqj7qLGVboIl8Mmi5SWpp12eU4uIo9uyIWbWPgiobo+hZmbWknY/o8DMrCGHmpllRm6p7vSmWjEr30rSOZKuym8PkjS6/E0zs7RK83pqxUyTugU4FDgzv/0+MK5sLTKz1GvXD14BDo6I4ZKeB4iId/OPyjOz7ZCAmhQPP4sJtU8lVZNfwltSL3JPSjaz7VSKM62oUPs1cB/QW9JPya3a8f2ytsrMUktqp9OkNomIuyTNIbf8kICvRISf0G62HUtxphW1SOQg4CPgwYafRURp5luYWbvT3u9Te5jPHsCyEzAYWATsW8Z2mVlKCSq2AGQxihl+Dmu4nV+941stFDezrKvgPWjFSDyjICLmShpVjsaYWfugFD+loJhzat9psFkFDAfWlq1FZpZqWXhEXucG72vJnWObXJ7mmFl70G5DLX/TbaeIuKyN2mNm7UCaJ7QXWs67JiJqCy3rbWbbn9wj8irdipYVatqmJ0a9IGmqpHMlnbzp1RaNM7N0qsrPKmjt1RpJYyQtkrRY0pUFyo2SVCfp1NbqLOacWg/gbXLPJNh0v1oAU4rY18wyplQXCvKnt8YBxwIrgVmSpkbEwmbK/RJ4tJh6C4Va7/yVz/l8FmabRIK2m1nGlOiU2mhgcUQsydWpScCJwMIm5S4md3GyqFvJCoVaNdAJmr0hxaFmtt0SVcXfp9ZT0uwG2xMiYkL+fX9gRYPvVgIHNzqS1B84idxIcZtD7Y2IuLqYSsxs+yES9dTWRcTIAlU11bTDdANwRUTUFXvFtVCopfearZlVjqCmNDeqrQQGNtgeAKxuUmYkMCkfaD2BEyTVRsT9LVVaKNSO2bp2mlmWJeypFTILGCJpMLAKOAM4q2GBiBi8+bjSbcBDhQINCj/M+J1taKyZZVgpFonM3wc7ltxVzWpgYkQskHRh/vvxW1OvH5FnZomVakJBREwDpjX5rNkwi4ivFVOnQ83MEhHFPYauUhxqZpaMSjP8LBeHmpklkptR4FAzswxJb6Q51MxsK6S4o+ZQM7Ok1D7XUzMza46vfppZ5vhCgZllh9rpct5mZs3x8NPMMsc9NTPLlPRGmkPNzBISUO2empllSYozzaFmZkkJpXgA6lAzs8TcUzOzzMjd0pHeVHOomVkyck/NzDLG06TMLDNyi0RWuhUtc6iZWWK++mlmmZLi0Weq56WmxuMzFzLqlKsZftKP+Lfbpm/xfURwxa/uZfhJP+LwM3/Gi39d0eq+9z8+l0O/+hN6jL6Y5xcu2/z5xk9ruejHd3DYGT/lC2f9nKfmvFLeH2eN3PSDs3nl0Z8zc9L3Kt2UVFOR/6uEsoWapImS1kiaX65jtIW6unouu/Ye7r3x2zx7z/eZPH0Of13yRqMyj81cyGvL1zJnyg+54XtncukvJrW67z579OP2a/83hx20R6O6fnvf0wDMnPT/uO/msXz/hvuor69vg19qAHc/9CynXjKu0s1ItU3n1Ip5VUI5e2q3AWPKWH+bmLPgdXYf2JPdBvRkhw41nHzscKb9+aVGZab9+SXO+IfRSGLUsMGsf38Db65bX3DfvQbvwpDd+mxxvEVL3+TIUXsB0KtHZ7p26sjzLy8v/w81AGY+/xrv/u2jSjcj3SSqinxVQtlCLSJmAO+Uq/628sba9fTv033zdr8+3Xlj7fomZd5rXKZ3N95Y815R+za135D+/H7GPGpr61i2ah0v/HUFq956t0S/xqw0VOSrEip+oUDSN4FvAgwcNKjCrdlSRGzxWdN/gJopgqSi9m3qnH86lFdef4uj//laBvbtwej9B1NTXZ2kyWZl5ed+tiIiJgATAEaMGNlMPFRWv97dGvWUVr/1Lrv07Fq4zJr32KVXVzZ+Wtvqvk3V1FTzs++csnn7uAuuZ/eBvbb1Z5iVVHojzVc/WzV86K68tnwty1atY+OntUx5bC7HH7l/ozLHHzmMSQ8/R0Qwa95SunTqyC49uxa1b1MffbyRDzd8AsCf/vIyNTVV7L1737L9PrOtkuLxZ8V7amlXU1PNtZd/lVMuGUddXXD2Px3CPnv0ZeLkJwG44JQjOO7wfXns6QUMP+nHdNypA+OuOqfgvgAP/elFrvjVvax79wNO/9fxDNuzP5NvGsu6d97nlIvHUVUl+vbqxvgfn1ex3749uvUnX+PwEUP4fLdOzH/oGn4xYRp3Tn2m0s1KnTQPP9XceZ+SVCzdDRwF9ATeAn4YEf9VaJ8RI0bG03+ZXZb2WHl0HzW20k2wBD5ZdA/1H63ZpkTaZ9hBcfsDTxRVdvQe3eZExMhtOV5SZeupRcSZ5arbzCosvR01Dz/NLJnc6bL0pppDzcySSfl6ar76aWaJlerip6QxkhZJWizpyma+P1vSS/nXTEkHtFane2pmlpBK8jBjSdXAOOBYYCUwS9LUiFjYoNhS4IsR8a6k48nd03pwoXodamaWWImGn6OBxRGxJFenJgEnAptDLSJmNij/LDCgtUo9/DSzRIodehaRe/2BFQ22V+Y/a8nXgd+3Vql7amaWXPE9tZ6SGt58OiE/NbKlWpq9cVbS0eRC7QutHdChZmaJJbilY12Bm29XAgMbbA8AVm9xLGl/4Fbg+Ih4u7UDevhpZolJxb1aMQsYImmwpB2AM4CpjY+jQcAU4NyIKGoZaPfUzCyZEt2nFhG1ksYCjwLVwMSIWCDpwvz344GrgM8Dt+SvuNa2Nu3KoWZmiZVqRkFETAOmNflsfIP33wC+kaROh5qZJSLSPaPAoWZmiaU40xxqZrYVUpxqDjUzSyzNi0Q61MwssfRGmkPNzLZGilPNoWZmiXiRSDPLlpQvEulQM7PEUpxpDjUzS6o0i0SWi0PNzBJLcaY51MwsmQo+fL0oDjUzSy7FqeZQM7PEfEuHmWWKz6mZWXYIqhxqZpYt6U01h5qZJeJFIs0sc1KcaQ41M0vOPTUzyxRPkzKzTElvpDnUzCyhIh9UXDEONTNLzDMKzCxb0ptpDjUzSy7FmeZQM7Ok5EfkmVl2pH1GQVWlG2BmVkruqZlZYmnuqTnUzCwx39JhZtnhm2/NLEvSfqHAoWZmiXn4aWaZ4p6amWVKijPNoWZmWyHFqeZQM7NEBKmeJqWIqHQbNpO0FlhW6XaUQU9gXaUbYYlk9e9s14jotS0VSHqE3J9PMdZFxJhtOV5SqQq1rJI0OyJGVrodVjz/nbVfnvtpZpniUDOzTHGotY0JlW6AJea/s3bK59TMLFPcUzOzTHGomVmmONTKSNIYSYskLZZ0ZaXbY62TNFHSGknzK90W2zoOtTKRVA2MA44HhgJnShpa2VZZEW4D2vRmUSsth1r5jAYWR8SSiNgITAJOrHCbrBURMQN4p9LtsK3nUCuf/sCKBtsr85+ZWRk51MqnuRm/vn/GrMwcauWzEhjYYHsAsLpCbTHbbjjUymcWMETSYEk7AGcAUyvcJrPMc6iVSUTUAmOBR4GXgXsiYkFlW2WtkXQ38Aywl6SVkr5e6TZZMp4mZWaZ4p6amWWKQ83MMsWhZmaZ4lAzs0xxqJlZpjjU2hFJdZJekDRf0r2Sdt6Gum6TdGr+/a2FJttLOkrSYVtxjNclbfHUoZY+b1Lmg4TH+pGk7yZto2WPQ6192RARB0bEfsBG4MKGX+ZXBkksIr4REQsLFDkKSBxqZpXgUGu/ngT+Lt+L+pOk3wHzJFVLuk7SLEkvSfoWgHJulrRQ0sNA700VSXpC0sj8+zGS5kp6UdIfJO1GLjz/Nd9LPEJSL0mT88eYJenw/L6flzRd0vOS/oMinuMt6X5JcyQtkPTNJt9dn2/LHyT1yn+2h6RH8vs8KWnvkvxpWmb4Ce3tkKQacuu0PZL/aDSwX0QszQfD+ogYJWlH4GlJ04GDgL2AYUAfYCEwsUm9vYD/BI7M19UjIt6RNB74ICJ+lS/3O+DfIuIpSYPIzZrYB/gh8FREXC3pH4BGIdWCC/LH6AjMkjQ5It4GPgfMjYhLJV2Vr3ssuQeiXBgRr0o6GLgF+Put+GO0jHKotS8dJb2Qf/8k8F/khoXPRcTS/OfHAftvOl8GdAWGAEcCd0dEHbBa0h+bqf8QYMamuiKipXXF/hcwVNrcEesiqXP+GCfn931Y0rtF/KZLJJ2Ufz8w39a3gXrgv/Of3wlMkdQp/3vvbXDsHYs4hm1HHGrty4aIOLDhB/n/c3/Y8CPg4oh4tEm5E2h96SMVUQZypy0OjYgNzbSl6Hl3ko4iF5CHRsRHkp4AdmqheOSP+17TPwOzhnxOLXseBf5FUgcASXtK+hwwAzgjf86tL3B0M/s+A3xR0uD8vj3yn78PdG5Qbjq5oSD5cgfm384Azs5/djzQvZW2dgXezQfa3uR6iptUAZt6m2eRG9b+DVgq6bT8MSTpgFaOYdsZh1r23ErufNnc/MND/oNcj/w+4FVgHvDvwJ+b7hgRa8mdB5si6UU+G/49CJy06UIBcAkwMn8hYiGfXYX9MXCkpLnkhsHLW2nrI0CNpJeAa4BnG3z3IbCvpDnkzpldnf/8bODr+fYtwEukWxNepcPMMsU9NTPLFIeamWWKQ83MMsWhZmaZ4lAzs0xxqJlZpjjUzCxT/j9QXdOd1BCwQgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.7150050352467271\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5647    0.5226    0.5428      1286\n",
      "           1     0.7793    0.8071    0.7930      2686\n",
      "\n",
      "    accuracy                         0.7150      3972\n",
      "   macro avg     0.6720    0.6648    0.6679      3972\n",
      "weighted avg     0.7098    0.7150    0.7120      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEICAYAAAA++2N3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcUElEQVR4nO3deZgV1Z3/8fenuxEEZNFGVEBAxAXiEsQ9LtGYoPkljpoYl4xZHzUZYjK/yWKWmcwk5pkkJr/JooYQQ8xMIkSjiUSJmMV9GUFFBRRFMOwisglioOnv7497G2839O0qvbdvdfXnlaee51bVuadO0+mPp5ZzShGBmVle1NW6AWZmleRQM7NccaiZWa441MwsVxxqZpYrDjUzyxWHmpnVjKQJkhZIWijpyl3s7y/pD5KelDRP0sc6rDNLz6n12mNg9B20X62bYSn0qFetm2ApbFi9nNc2rHtLv7T6fsMjmrYkKhtbXp4ZERN2tU9SPfAccAawDJgFXBgR80vKfAXoHxFfkjQIWADsExFb2ztmQ+KfpBP0HbQf771qaq2bYSkMHdir1k2wFKZcce5briOattDz4PMTlX19zrWNZXYfAyyMiEUAkqYBZwPzS8oEsIckAX2BtUBTuWNmKtTMrCsQqCJXroYAS0vWlwHHtilzDTAdWAHsAXwoIprLVepQM7N0BNTVJy3dKGl2yfrkiJhcUlNbba+HvQeYA5wGjAL+JOn+iNjY3gEdamaWnhJfllsTEePb2bcMGFayPpRCj6zUx4BvR+Hi/0JJi4FDgEfbO6DvfppZSsXTzyRLebOA0ZJGStoNuIDCqWapJcDpAJIGAwcDi8pV6p6amaWXvKfWrohokjQRmAnUA1MiYp6ky4v7JwHfBG6Q9DSF09UvRcSacvU61MwsHVGpGwVExAxgRpttk0o+rwDenaZOh5qZpaSK9NSqxaFmZuklv/vZ6RxqZpZSxZ5TqwqHmpmlI3z6aWY5456ameWHTz/NLE8E1PtGgZnlia+pmVl++PTTzPLGPTUzyxX31MwsN+RhUmaWNx4mZWb54RsFZpY3Pv00s9yo4Hxq1eBQM7OUfPppZnnjGwVmliu+pmZmuSGffppZ3rinZmZ5IoeameVFYTZvh5qZ5YWE6hxqZpYj7qmZWa441MwsV7Icatl92MTMskkplo6qkiZIWiBpoaQrd7H/C5LmFJe5krZL2rNcnQ41M0tFCCnZUrYeqR64FjgTGANcKGlMaZmIuDoijoyII4EvA/dGxNpy9fr008xSq6urSH/oGGBhRCwCkDQNOBuY3075C4GpHbatEi0zs+4lRU+tUdLskuXSkmqGAEtL1pcVt+3qeL2BCcAtHbXNPTUzSyfh9bKiNRExvkxNbUU7Zd8HPNjRqSc41MzsTajQ3c9lwLCS9aHAinbKXkCCU0/w6aeZpVSpGwXALGC0pJGSdqMQXNN3Op7UHzgFuC1J+9xTM7PUKjFMKiKaJE0EZgL1wJSImCfp8uL+ScWi5wB3RcTmJPU61MwsHVXu4duImAHMaLNtUpv1G4AbktbpUDOz1LI8osChZmapOdTMLDdabhRklUPNzNLLbqY51MwsJVVsmFRVONTMLDWffubU2H36cv6RQ6gTPLB4LTOffbnV/oMG9eHTJ45gzeatADyxfAN3zF/NwN178LFjh9GvVwMRcP+iV/jr86/U4kfodhY/9yL33H4fzc3BYUeP5ZhTdj2CZ9Wyl5j6k5t47wUTOOiw0QA89sATzJ09D4DGfRp5z3nvoqFHN/0Tym6mVTfUJE0AfkjhwbrrI+Lb1TxeZ5LgwnFD+MG9i1m3ZRtffteBPLViIys3/r1VuefXbObaB15stW17BDfPWcnS9Vvo2VDHV88YzTMvbdrpu1ZZzc3N/HX6PZz38XPYo19ffn3dbxh1yEj2GrzXTuXuv/NBho/ef8e2Vzds4omHn+Qjn/swPXo0cPuNM1jw1HOMPWpM28N0C1nuqVXtxDjJXEld2cg9e7N601bWbN7K9uZg9pL1HLFfv0Tf3fh6E0vXbwHg703NrNz4OgN271HN5hqF3teAvQYwYM/+1DfUc8jho3nhmUU7lZvz8JOMHjuK3n17t9re3NxM07Ymmrc3s21bE3369emspmdK0iFStQq+al7t2zFXUkRsBVrmSsqFAbv3YN1r23asr9uybZfBdMBevfnau0fzmZNGsG+/njvt36t3D/YfsDuLX3mtqu012LRhE3v077tjvW//vry6sfXIm1c3bOL5eS9w+LGHtdq+R/++jH/HOK7/7i/46X9eT89ePRkxenintDuLumuoJZ4rKa+WrNvCV+54lqvuep67n3+FT504otX+ng11XHbCcG6as4LXm5pr08huru2f3T133MdJE07c6e7e61te54VnFvGJz3+ES7/8CbZt3cb8J57tvIZmjOqUaKmFal5TSzRXUnHSuEsB+jTuW8XmVNb6LdsY2PuNntnA3Xuwfsu2VmVKg2ruqle5sE702a2ezVu3Uye47IThPLpkPU8s39hp7e7O+vbvy6sbNu1Y37RhE33bnEK+tHw1M6bdCcCW115n8YIXqauvo3l7M/0G9ttxSjp67ChWLlnJmLcf0nk/QIZk+ZpaNUMt0VxJETEZmAzQeMDY9iaIy5wX177G3n13Y68+PVi/pYnx+w/g548saVWmX68GNr7eBMCIPXenDti8dTsAlxw9jFUbX+fPz63p7KZ3W/sMGcz6NevZsHYDffv15dmnnuesD72nVZlPfuGjOz7f+ds/ccDBIzhwzChWLl3FqqWr2LZ1Gw09GljywlIGDxncyT9BRlRwQHs1VDPUdsyVBCynMFfSRVU8XqdqDpj2+Ao+e/IB1AkeXLyOlRv/zsmjCi+6ue+FtYwb2p9TRu3F9gi2bW/mZ8XQG9XYm+NHDGTZ+i187YzC4wK/f3oVc1e9WrOfpzuoq6/jne8/lVt+cRsRzbztqLE0Dt6LJ//3aQCOaHMdrdS+w/Zh9NsO5FfXTKOuTuy93yAOO2ZsZzU9U0Th7n9WKaJ6nSNJZwE/4I25kr5VrnzjAWPjvVclmtzSMmLowF61boKlMOWKc1n5/Ny3FEm99jkohv3jjxKVXfi9Mx8rM513VVT1ObVdzZVkZl1fXY1uAiTRTR+HNrM3Tdk+/XSomVkqwj01M8sZ99TMLFe66yMdZpZHvqZmZnki5EkizSxf3FMzs1zxNTUzyw9fUzOzPCmM/cxuqmX3ap+ZZZaUbOm4Hk2QtEDSQklXtlPmVElzJM2TdG9HdbqnZmapVWJEQcmU/2dQmKpslqTpETG/pMwA4DpgQkQskbR3h217yy0zs+5FFZvOO8mU/xcBt0bEEoCIWN1RpQ41M0ulZT61Cpx+Jpny/yBgoKR7JD0m6ZKOKvXpp5mllOqlKo2SZpesTy7Odl2oaGdtJ3hsAI4CTgd2Bx6W9EhEPNfeAR1qZpZaipufa8pMEplkyv9lxTo2A5sl3QccAbQbaj79NLN0VLhRkGTpwI4p/yXtRmHK/+ltytwGnCSpQVJv4FjgmXKVuqdmZqlU6jm1iGiSNBGYyRtT/s+TdHlx/6SIeEbSncBTQDNwfUTMLVevQ83MUqvUw7e7mvI/Iia1Wb8auDppnQ41M0stwwMKHGpmll6Wh0k51MwsHQ9oN7M8KUwSmd1Uc6iZWWp1Ge6qOdTMLLUMZ5pDzczSkXyjwMxyJsOX1NoPNUk/ZufBpTtExBVVaZGZZV5XvVEwu8w+M+umROEOaFa1G2oR8cvSdUl9iiPlzayby3BHreNZOiQdL2k+xZHxko6QdF3VW2Zm2ZRw1tta3UxIMvXQD4D3AK8ARMSTwMlVbJOZZVylXrxSDYnufkbE0japu706zTGzrBNd/+HbpZJOAKI4kdsVdDBJm5nlW5bvfiY5/bwc+CcKL0RYDhxZXDezbijpqWdmTz8jYg1wcSe0xcy6iCyffia5+3mApD9IelnSakm3STqgMxpnZtmkhEstJDn9vBG4CdgX2A+4GZhazUaZWbZ19Uc6FBH/ExFNxeVXlBk+ZWb5Vrj7mWyphXJjP/csfrxb0pUUXgkfwIeAOzqhbWaWReq6k0Q+RiHEWlp/Wcm+AL5ZrUaZWbZ1yamHImJkZzbEzLqGltPPrEo0okDS24AxQK+WbRHx39VqlJllW5fsqbWQ9HXgVAqhNgM4E3gAcKiZdVPZjbRkdz8/AJwOrIqIjwFHAD2r2iozyywJ6uuUaKmFJKG2JSKagSZJ/YDVgB++NevGKvWcmqQJkhZIWlh8yqLt/lMlbZA0p7j8W0d1JrmmNlvSAOBnFO6IbgIeTfA9M8upSlxSk1QPXAucASwDZkmaHhHz2xS9PyL+T9J6k4z9/HTx4yRJdwL9IuKppAcws3wRqtTYz2OAhRGxCEDSNOBsoG2opVLu4dtx5fZFxONv5cBm1kVVbgaOIcDSkvVlwLG7KHe8pCeBFcDnI2JeuUrL9dS+X2ZfAKeVq/jNGD5wd356/uGVrtaqaODRE2vdBEvh78tXV6SeFI90NEoqfYnT5IiY3FLNLsq3HYL5ODA8IjZJOgv4PTC63AHLPXz7zo7ba2bdjYD65KG2JiLGt7NvGTCsZH0ohd7YDhGxseTzDEnXSWosTom2S0nufpqZtVKhAe2zgNGSRhZn1b4AmF5aQNI+KnYLJR1DIbNeKVep39BuZqlV4hG0iGiSNBGYCdQDUyJinqTLi/snUXhO9lOSmoAtwAURUXaWIIeamaVSmKq7MncKImIGhZFKpdsmlXy+BrgmTZ1JZr6VpA+3PPQmaf9iN9DMuqksz6eW5JradcDxwIXF9VcpPDBnZt1Ul37xCnBsRIyT9ARARKwrXtQzs25IQENXnqUD2FYczhAAkgYBzVVtlZllWoYzLVGo/Qj4HbC3pG9RuBvxtaq2yswyS6rYMKmqSDL289eSHqMw/ZCAf4gIv6HdrBvLcKYlmiRyf+A14A+l2yJiSTUbZmbZ1dWn876DN17A0gsYCSwAxlaxXWaWUYKaTQCZRJLTz8NK14uzd1zWTnEzy7saPoOWROoRBRHxuKSjq9EYM+salOG3FCS5pvZ/S1brgHHAy1VrkZllWh5ekbdHyecmCtfYbqlOc8ysK+iyoVZ86LZvRHyhk9pjZl1Al3zvp6SG4tQg7U7rbWbdT+EVebVuRfvK9dQepXD9bI6k6cDNwOaWnRFxa5XbZmYZ1aVHFAB7Uphp8jTeeF4tAIeaWTfUlW8U7F288zmXN8KsRdmZJ80s3zLcUSsbavVAX5K98cXMug1R10WfU1sZEd/otJaYWZcgum5PLcPNNrOaETRk+KJauVA7vdNaYWZdRpftqUXE2s5siJl1HV39kQ4zs1YynGkONTNLRyR7DV2tONTMLB359NPMcqQwosChZmY5kt1Iy/apsZllVKXe0C5pgqQFkhZKurJMuaMlbZf0gY7qdKiZWUpCSraUraUwX+O1wJnAGOBCSWPaKfcdYGaS1jnUzCyVlrufSZYOHAMsjIhFEbEVmAacvYtyn6Ew2/bqJO3zNTUzSy3FjYJGSbNL1idHxOTi5yHA0pJ9y4BjS78saQhwDoWpzxK98MmhZmbpKNV03msiYnz7Ne2k7QxAPwC+FBHbkx7ToWZmqVTw4dtlwLCS9aHAijZlxgPTioHWCJwlqSkift9epQ41M0utQi9emQWMljQSWA5cAFxUWiAiRpYc8wbg9nKBBg41M3sTKhFpxRc7TaRwV7MemBIR8yRdXtw/6c3U61Azs1QE1FdoREFEzABmtNm2yzCLiI8mqdOhZmapZXiUlEPNzNISyvBAKYeamaXmnpqZ5UbhkY7spppDzczSSThYvVYcamaWmudTM7PcKEwSWetWtM+hZmap+e6nmeVKhs8+PZ/aW/Hnh+Zz9HnfYNw5/85/3XDXTvufe3EV7/749xh8wuf48f/8udW+SVPv5vgPfYvjz7+Kn9x4d2c1uds7/fhDefS3/8pjt36dz33kjJ329+vTi6n/7zLu//WVPPSbr3LR+47bse/H/3oxz838Tx6a9pXObHImKeH/aqFqoSZpiqTVkuZW6xi1tH17M1/47k3c/MNP88hNX+OWux7j2UUrW5UZ2K8P3/6XDzLxw6e12j5/4Qp++fuH+Msvv8D9N36ZmQ/M5YUliea/s7egrk5c/cXz+eBnr+O486/ivHcfxcEj92lV5pMfPJkFi1Zx0sXf5n2X/ZCrPnsOPRrqAZh6+yN84Ipra9H0TGm5ppZkqYVq9tRuACZUsf6aemzeixwwrJERQxvZrUcD554xjhn3PtWqzKA992Dc2OE7/ihaPPfiKo4+bAS9e+1GQ0M9J447kNvvebIzm98tHTV2BIuWruFvy19hW9N2bv3T45x1yuGtygTQt09PAPr07sm6ja/RtL0ZgIeeeIF1G1/r7GZnj0RdwqUWqhZqEXEfsLZa9dfaypc3MGTwwB3r+w0eyMqXNyT67qGj9uOhJxaydv0mXnt9K396aB7LX1pXraZa0b6D+rf6d17x0jr2HdS/VZmf3XQvB43Yh2f++C0enPoVvvz93xLRdt5CU8KlFmp+o0DSpcClAMP237/GrUluV/9HT/ofpoNH7sNnLzmDcyZeQ5/ePRk7eggN9fUdf9Hekl3NAdb213jacYfy9HPLeP+nfsTIoY387pqJPDznBV7d/HontTL7sv7ez5rfKIiIyRExPiLGD2ocVOvmJLbf3gN2+q/+Po39y3yjtX88+wTu/dWVzJj8zwzs14cDhnWdn72rWrF6/U6961VrWveuL37fcdx+d+FSwOJla/jbilcYPXxwp7azK8hyT63modZVjRsznBeWvMzflq9h67Ymbv3T45x58uEdf7Ho5bWvArB01Vpuv/tJPvCe9qZxt0p5fP7fGLX/IPbfby96NNRz7hnj+ON9ra+DLlu1jpOPPhgoXBM9cPhgXly+phbNzbYMp1rNTz+7qoaGer77xfM574pr2b49uPj9x3HoqH2Zcsv9AHz8vJN4ac1GTvvId3l18+tIYtK0e3j4N1+lX9/dueRL17Nuw2YaGuq5+ovnM6Bf7xr/RPm3fXszX/zuTdzyo3+ivl78evojPLtoFR879x0A/OLWB7j653dy7dc/zINTv4IE/3HNbazdsBmA66/6KCceNZq9BvRl7u3f5NuTZ/Cr6Q/X8keqmSyffqpaF0ElTQVOpfCyhJeAr0fEz8t956ijxseD/zu7XBHLmIFHT6x1EyyFvy+4iebXVr+lRDr0sLfHf992T6Kyx4wa8FiZt0lVRdV6ahFxYbXqNrMay25HzaefZpZO4XJZdlPNoWZm6Xg+NTPLmwxnmkPNzNJSpV5mXBUONTNLLcOZ5lAzs3RqOVogCYeamaWX4VTzMCkzS61Sk0RKmiBpgaSFkq7cxf6zJT0laY6k2ZLe0VGd7qmZWWqVuKYmqR64FjgDWAbMkjQ9IuaXFPsLMD0iQtLhwE3AIeXqdU/NzNIpPqeWZOnAMcDCiFgUEVuBacDZpQUiYlO8MZazD4V5PMtyqJlZahU6/RwCLC1ZX1bc1vpY0jmSngXuAD7eUaUONTNLRaTqqTUWr4W1LJe2qaqtnXpiEfG7iDgE+Afgmx21z9fUzCy1FJfU1pSZpWMZMKxkfSiwor2KIuI+SaMkNUZEu5PcuadmZulVZpLIWcBoSSMl7QZcAExvdRjpQBWHL0gaB+wGvFKuUvfUzCy1SkwSGRFNkiYCM4F6YEpEzJN0eXH/JOA84BJJ24AtwIeig0kgHWpmllqlnr2NiBnAjDbbJpV8/g7wnTR1OtTMLL0MjyhwqJlZKp4k0szyxZNEmlneZDjTHGpmlpYniTSznMlwpjnUzCwdTxJpZvmT4VRzqJlZan6kw8xyxdfUzCw/BHUONTPLl+ymmkPNzFJpmSQyqxxqZpZahjPNoWZm6bmnZma54mFSZpYr2Y00h5qZpZTwnZ4141Azs9Q8osDM8iW7meZQM7P0MpxpDjUzS0sVeUVetTjUzCyVrI8o8BvazSxX3FMzs9Sy3FNzqJlZan6kw8zyI+MP3/qampml0nKjIMnSYV3SBEkLJC2UdOUu9l8s6ani8pCkIzqq0z01M0utEqefkuqBa4EzgGXALEnTI2J+SbHFwCkRsU7SmcBk4Nhy9TrUzCy1Cp1+HgMsjIhFhTo1DTgb2BFqEfFQSflHgKEdVerTTzNLTQkXoFHS7JLl0pJqhgBLS9aXFbe15xPAHztqm3tqZpZe8p7amogYn6KW2GVB6Z0UQu0dHR3QoWZmqQgqNUxqGTCsZH0osGKn40mHA9cDZ0bEKx22L2KXwVgTkl4G/lbrdlRBI7Cm1o2wVPL6OxseEYPeSgWS7qTw75PEmoiY0E49DcBzwOnAcmAWcFFEzCspsz/wV+CSNtfX2m9flkItryTNLtMFtwzy76xzSDoL+AFQD0yJiG9JuhwgIiZJuh44jzc6O00d/V4cap3AfyBdj39nXZfvfppZrjjUOsfkWjfAUvPvrIvy6aeZ5Yp7amaWKw61KuposK5lj6QpklZLmlvrttib41CrkpLBumcCY4ALJY2pbassgRuAXT5XZV2DQ616dgzWjYitQMtgXcuwiLgPWFvrdtib51CrnrSDdc2sAhxq1ZN4sK6ZVY5DrXoSDdY1s8pyqFXPLGC0pJGSdgMuAKbXuE1muedQq5KIaAImAjOBZ4CbSmcfsGySNBV4GDhY0jJJn6h1mywdjygws1xxT83McsWhZma54lAzs1xxqJlZrjjUzCxXHGpdiKTtkuZImivpZkm930JdN0j6QPHz9eUG20s6VdIJb+IYL0ra6QUd7W1vU2ZTymP9u6TPp22j5Y9DrWvZEhFHRsTbgK3A5aU7izODpBYRn4yI+WWKnAqkDjWzWnCodV33AwcWe1F3S7oReFpSvaSrJc2S9JSkywBUcI2k+ZLuAPZuqUjSPZLGFz9PkPS4pCcl/UXSCArh+c/FXuJJkgZJuqV4jFmSTix+dy9Jd0l6QtJPSfDKW0m/l/SYpHlt3t6NpO8X2/IXSYOK20ZJurP4nfslHVKRf03LDb/MuAsqvi/xTODO4qZjgLdFxOJiMGyIiKMl9QQelHQX8HbgYOAwYDAwH5jSpt5BwM+Ak4t17RkRayVNAjZFxPeK5W4E/isiHii+l3EmcCjwdeCBiPiGpPcCrUKqHR8vHmN3YJakW4ovrO0DPB4R/yLp34p1T6Tw7oDLI+J5SccC1wGnvYl/Rssph1rXsrukOcXP9wM/p3Ba+GhELC5ufzdweMv1MqA/MBo4GZgaEduBFZL+uov6jwPua6krItqbV+xdwBi98ZbufpL2KB7j3OJ375C0LsHPdIWkc4qfhxXb+grQDPymuP1XwK2S+hZ/3ptLjt0zwTGsG3GodS1bIuLI0g3FP+7NpZuAz0TEzDblzqLjqY+UoAwULlscHxFbdtGWxOPuJJ1KISCPj4jXJN0D9GqneBSPu77tv4FZKV9Ty5+ZwKck9QCQdJCkPsB9wAXFa277Au/cxXcfBk6RNLL43T2L218F9igpdxeFU0GK5Y4sfrwPuLi47UxgYAdt7Q+sKwbaIRR6ii3qgJbe5kUUTms3AoslfbB4DEk6ooNjWDfjUMuf6ylcL3u8+PKQn1Lokf8OeB54GvgJcG/bL0bEyxSug90q6UneOP37A3BOy40C4ApgfPFGxHzeuAv7H8DJkh6ncBq8pIO23gk0SHoK+CbwSMm+zcBYSY9RuGb2jeL2i4FPFNs3D0+Rbm14lg4zyxX31MwsVxxqZpYrDjUzyxWHmpnlikPNzHLFoWZmueJQM7NccaiZWa78f53tncQ9euLFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(clf_nn_char, model_name=\"clf_nn_char\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "id": "Z2lQ9qrayUQK"
   },
   "outputs": [],
   "source": [
    "del clf_nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "N8AqkvOlAWOf"
   },
   "source": [
    "### **4.5 KNeighborsClassifier**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "id": "lv3JDSb8EQCu"
   },
   "outputs": [],
   "source": [
    "knn=KNeighborsClassifier(algorithm='auto',\n",
    "                     leaf_size=30,\n",
    "                     metric='minkowski',\n",
    "                     metric_params=None, \n",
    "                     n_jobs=1, \n",
    "                     n_neighbors=18,\n",
    "                     p=2,\n",
    "                     weights='uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "id": "U7n-Gx4xU9AU"
   },
   "outputs": [],
   "source": [
    "grid_search_knn = GridSearchCV(knn, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_word, y_train_word), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "id": "JGzxAEiLAzaq"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV] END .................................................... total time=  11.3s\n",
      "[CV] END .................................................... total time=   6.7s\n",
      "[CV] END .................................................... total time=   6.1s\n",
      "[CV] END .................................................... total time=   7.1s\n",
      "[CV] END .................................................... total time=   6.5s\n"
     ]
    }
   ],
   "source": [
    "\n",
    "knn_word = grid_search_knn.fit(X_train_word, y_train_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "id": "76lbTBq5EQCv"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.7041432887354337\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9265    0.1151    0.2048      3067\n",
      "           1     0.6946    0.9955    0.8183      6201\n",
      "\n",
      "    accuracy                         0.7041      9268\n",
      "   macro avg     0.8106    0.5553    0.5115      9268\n",
      "weighted avg     0.7714    0.7041    0.6152      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZLklEQVR4nO3deZhV1Z3u8e9bVSDIKBQgMigqzqhRxMQIEuwkoG3bDrEdko5Gr9qJem/SbeJNbjt2WhNjx76Kl+AQTWxFvZIWlYjRdiAOV1ARBSUhIFCiQjE4oljF7/5xTmFRVJ06G87h7Nq8nzznec7ee+211yke36w9rLUVEZiZZUVVpRtgZlZKDjUzyxSHmpllikPNzDLFoWZmmVJT6QY016dvbQwesmulm2EJvPZmfaWbYAnEx6uI9R9qa+qo7rlrRMO64o63buWMiBi/NcdLKlWhNnjIrjz0+DOVboYlsPeZv650EyyBT2des9V1RMM6dtj7lKLKfjJnYu1WHzChVIWamXUEAqX3ypVDzcySEVBVXelWtMmhZmbJaasuy5WVQ83MEvLpp5lljXtqZpYZwj01M8sSuadmZhnju59mlh2+UWBmWSJ8+mlmGeOempllh08/zSxLBFT7RoGZZYmvqZlZdvj008yyxj01M8sU99TMLDPkYVJmljUeJmVm2eEbBWaWNT79NLPM8HxqZpYtPv00s6zxjQIzyxRfUzOzzJBPP80sa9xTM7MskUPNzLIiN5u3Q83MskJCVQ41M8sQ99TMLFMcamaWKQ41M8sO5T8p5VAzs0SE3FMzs2ypqvKIAjPLEPfUzCw7Un5NLb19SDNLLUlFfYqoZ7ykBZIWSrqkle29JD0o6RVJ8ySd1V6dDjUzS6TpRsHWhpqkamAiMAHYDzhN0n4tin0PmB8RBwFjgeskdS5Ur0PNzBJTlYr6tGMUsDAiFkXEemAKcHyLMgH0UC4huwOrgYZClfqampklo0Q3CmolzW62PDkiJue/DwKWNdtWBxzeYv8bgWnAcqAH8HcRsaHQAR1qZpZYglCrj4iRbVXTyrposfx1YA4wDtgD+IOkmRHxflsH9OmnmSVWohsFdcCQZsuDyfXImjsLmBo5C4HFwD6FKnWomVkipbpRAMwChksalr/4fyq5U83mlgJHA0gaAOwNLCpUqU8/zSy5EjynFhENki4AZgDVwG0RMU/S+fntk4CrgNslvZo/6o8ior5QvQ41M0tGpRsmFRHTgekt1k1q9n058LUkdTrUzCwxD5Mys2xJb6Y51LbG0y+8wU9veoANGzbwjQmHc+5p4zbZ/pelK/jxtfcwb2Ed3z9rAmefMhaAt1es5Yc/u5v6NR9QJXHKsV/k2yeOrsAv2P4c/YUhXH32kVRXid8+9jrXT315k+09d+zMr/7H0Qyu7U51dRU3PjCHu/5rAQD/cNyBfOuv9gVg/pJVfO+GJ/j0s8Zt/hvSIM09tbLe/WxvXFdH1ti4gStv+B23/Os5PHzrxTz0xMssXPLOJmV69+jKT753PGd/Y+wm66urq7jk/OP4/W0/5J4bLuSuB57ZbF8rvaoqce25o/nGVQ/xxYumcNKRe7L34J02KXPOhANYsGwNo39wH8f98wP8y5lH0KmmioF9unHesSMYd/H/5Yj/fg9VVeLEI/es0C+prGLvfFYq+MoWakWO6+qw5i5Yyq679GXILn3p3KmGY8cezOPPzNukTN+denDgPkOpqd70z9y/b0/2Hz4YgO47dmH3oQN4t77NZwmtRA4d3p9Fb7/Hknc/4LOGDUz940KOGbXbJmUigu5dOwHQrUsn1nz4KQ2NuQfYa6qr6NK5huoqseMONbyz+qNt/RNSI82hVs7Tz43jugAkNY3rml/GY24z79a/x879e29cHtCvN3PfWJK4nrp3VvP6wrc4aJ+hJWydtWZgn268Vf95EC1f9RGH7tV/kzI3T3+Nu348gddv/Xu6d+3M2dc9SgS8vfojbnhgDq9O/hafrG/giTnLeOKVum39E1Ijza/IK+fpZ2vjuga1LCTpXEmzJc1evWplGZtTWtFyMAe5hxKT+Gjdp1x0xR38+LvH071blxK1zNrSWseh5b/juC8M4dXF9ex79m8Y84N7+fl/G02Prp3o1a0zx4waxsHn38m+Z/+GHbt04pSjhm+bhqdQmntq5Qy1YsZ1ERGTI2JkRIzs07dfGZtTWjv368U7K9ZuXH535Vr69+1Z9P6fNTRy0eV3cNzRh/C10SPK0EJrafmqjxhU223j8i59u212CnnGuH146PnFACx+532WrPiA4YN3YuxBg1ny7vusev8TGho38ODzixi1987btP2poe031IoZ19Vhjdh7CG++Vc+yt1ex/rMGHn5yDuOO2L+ofSOCn/ziXnbfdQBnnXxUmVtqTV768wr2GNibof170KmmihOP3JPfz3pzkzJ19R8y5sDcCUW/Xl3Zc5devPnO+9St/JCRew2ga+fcFZujDhzMgro12/onpILI9XqL+VRCOa+pbRzXBbxFblzX6WU83jZVU13NpReewDmX3EzjhuCk8YcxfLedufvBZwE47bgjWLn6fU767r/z4cefUCVxx9SZTL/1Yt5Y9DYPPPYiew0byPHn/RsAP/jOBI46fN9K/qTMa9wQ/PDmmdx/2V9TXSX+4/E3eGPZGs76eu7+1a9nzOfae2cz8aJxPHP9KUjiit8+z+oPPmH1B58w7blFPHndyTRuCOYuWskdj2bi8vAWSPfbpBStXRwqVeXSMcD1fD6u66eFyh948KHx0OPPlK09Vnp7n/nrSjfBEvh05jVsWLtkqxKpy857xa7fvqGosn/6+fgXC0w9VBZlffi2tXFdZtbBVfDUshgeUWBmiYjcg8xp5VAzs8TcUzOzTEnzjQKHmpkl42tqZpYlQiWbJLIcHGpmlph7amaWKb6mZmbZ4WtqZpYlubGf6U01h5qZJZbiTHOomVlyHlFgZtkhn36aWYY0zaeWVg41M0so3fOpOdTMLLEUZ5pDzcwSkm8UmFmG+Dk1M8sch5qZZUqKM82hZmbJuadmZtnhAe1mliW5SSLTm2oONTNLrCrFXbX0zslrZqklFfdpvx6Nl7RA0kJJl7RRZqykOZLmSXqqvTrdUzOzRFSiAe2SqoGJwFeBOmCWpGkRMb9Zmd7ATcD4iFgqqX979bqnZmaJVam4TztGAQsjYlFErAemAMe3KHM6MDUilgJExIr2Km2zpybpBiDa2h4RF7XbZDPLpAQ3CmolzW62PDkiJue/DwKWNdtWBxzeYv+9gE6SngR6AP8eEb8pdMBCp5+zC2wzs+2UyN0BLVJ9RIwsUFVLLTtSNcChwNFAV+A5Sc9HxJ/aOmCboRYRd2xydKlbRHzUVnkz236U6ImOOmBIs+XBwPJWytTns+cjSU8DBwFthlq719QkfUnSfOD1/PJBkm5K2Hgzywrl5lMr5tOOWcBwScMkdQZOBaa1KPMAMFpSjaQdyZ2evl6o0mLufl4PfL3pYBHxiqQxRexnZhlVisfUIqJB0gXADKAauC0i5kk6P799UkS8LukRYC6wAbglIl4rVG9Rj3RExLIWqdu4JT/CzDo+UbqHbyNiOjC9xbpJLZavBa4tts5iQm2ZpCOAyHcRL6Kd7p+ZZVuah0kV85za+cD3yN1+fQs4OL9sZtuhYkcTVGokVbs9tYioB87YBm0xsw6iQ4/9lLS7pAclrZS0QtIDknbfFo0zs3RSkZ9KKOb08y7gXmAgsAtwH3B3ORtlZulWokc6yqKYUFNE/DYiGvKfOykwfMrMsi1397MkYz/LotDYzz75r0/kpwSZQi7M/g54eBu0zczSSB13ksgXyYVYU+vPa7YtgKvK1SgzS7cO+Y6CiBi2LRtiZh1D0+lnWhU1okDSAcB+QJemde1N/2Fm2dUhe2pNJF0GjCUXatOBCcAfAYea2XYqvZFW3N3Pk8nNZfRORJxFbtqPHcraKjNLLQmqq1TUpxKKOf1cFxEbJDVI6gmsAPzwrdl2rEOffgKz8y8/uJncHdEPgRfK2SgzS7cUZ1pRYz+/m/86KT+vUc+ImFveZplZWgmleuxnoYdvDym0LSJeKk+TzCzVKjgDRzEK9dSuK7AtgHElbgudqkX/Xl3aL2jpUTev0i2wJNavK0k1HfKaWkR8ZVs2xMw6BgHVHTHUzMza0uFHFJiZNedQM7PMyE3Vnd5UK2bmW0n6pqRL88tDJY0qf9PMLK3SPJ9aMcOkbgK+BJyWX/4AmFi2FplZ6nXoF68Ah0fEIZJeBoiINflX5ZnZdkhATYpPP4sJtc8kVZOfwltSP3JvSjaz7VSKM62oUPvfwO+A/pJ+Sm7Wjv9V1laZWWpJHXSYVJOI+A9JL5KbfkjA30aE39Buth1LcaYVNUnkUOBj4MHm6yJiaTkbZmbp1dGfU3uYz1/A0gUYBiwA9i9ju8wspQQVmwCyGMWcfo5ovpyfveO8NoqbWdZV8Bm0YiQeURARL0k6rByNMbOOQSl+S0Ex19R+0GyxCjgEWFm2FplZqmXhFXk9mn1vIHeN7f7yNMfMOoIOG2r5h267R8TF26g9ZtYBpHlAe6HpvGsioqHQtN5mtv3JvSKv0q1oW6GmNb0xao6kaZK+JenEps+2aJyZpVNVflRBe5/2SBovaYGkhZIuKVDuMEmNkk5ur85irqn1AVaReydB0/NqAUwtYl8zy5hS3SjIX96aCHwVqANmSZoWEfNbKfczYEYx9RYKtf75O5+v8XmYNYkEbTezjCnRJbVRwMKIWJSrU1OA44H5LcpdSO7mZFGPkhUKtWqgO7T6QIpDzWy7JaqKf06tVtLsZsuTI2Jy/vsgYFmzbXXA4ZscSRoEnEDuTHGrQ+3tiLiymErMbPshEvXU6iNiZIGqWmrZYboe+FFENBZ7x7VQqKX3nq2ZVY6gpjQPqtUBQ5otDwaWtygzEpiSD7Ra4BhJDRHxn21VWijUjt6ydppZliXsqRUyCxguaRjwFnAqcHrzAhExbONxpduBhwoFGhR+mfHqrWismWVYKSaJzD8HewG5u5rVwG0RMU/S+fntk7akXr8iz8wSK9WAgoiYDkxvsa7VMIuIM4up06FmZomI4l5DVykONTNLRqU5/SwXh5qZJZIbUeBQM7MMSW+kOdTMbAukuKPmUDOzpNQx51MzM2uN736aWeb4RoGZZYc66HTeZmat8emnmWWOe2pmlinpjTSHmpklJKDaPTUzy5IUZ5pDzcySEkrxCahDzcwSc0/NzDIj90hHelPNoWZmycg9NTPLGA+TMrPMyE0SWelWtM2hZmaJ+e6nmWVKis8+Uz0uNTUee3Y+h510JYeccDm/vP3RzbZHBD/6xX0ccsLlfPm0f+WVN5YVve8Nv32MnQ67gFVrPwRg6fJVDDzy+4w+/WpGn34137/67vL9MNvMDf98Bn+acTXPTvlxpZuSairyf5VQtp6apNuAvwZWRMQB5TpOuTU2buDin9/L7268gF0G9Gbct69lwpgR7LP7wI1l/vDsfP6ydCUvTr2M2a+9yT9eM4XHbr+43X3r3lnDky+8weCdd9rkmLsNqmXmXf9zm/5Oy7n7oee5+d6nmHTF31e6KamV9mtq5eyp3Q6ML2P928SL895k9yG17Da4ls6dajjxq4cw/am5m5SZ/tRcTj12FJI4bMQw3vtgHe/Uv9fuvj/55f1cfuHfpnrGg+3Nsy//hTXvf1zpZqSbRFWRn0ooW6hFxNPA6nLVv628vfI9Bg34vCe1y4CdeHvley3KrN20TP/evL1ibcF9pz81l4H9ejNir8GbHXPp8lWMOeMajj33ep59eWGpf5LZVlORn0qo+I0CSecC5wIMGTq0wq3ZXERstq7l/wG1UgRJbe778Sfr+bdfz+D+Gy/YbPuA2p68+uCV9OndnTmvL+WMf5rMc/f8hJ7du27xbzArpbS/97PiNwoiYnJEjIyIkf1q+1W6OZvZpX9v3np3zcbl5e+uYefaXoXLrFjLzv16tbnv4rqVLFm+itGnX82Bf3Mpy1es5ahv/ox3699nh86d6NO7OwAH7zuUYYNr+cvSFWX+lWbJpLmnVvFQS7tD9tuVvyxdyZK36ln/WQNT//ASE8YcuEmZCWNGMOXhF4gIZr26mJ7du7Jzba82991/z0H8+dFrmDvtSuZOu5Jd+vfmqTt/xIDantSv+YDGxg0AvFlXz6JlK9ltUG0lfrpZ21KcahU//Uy7mppqfv7DUzjpook0NgZn/M0X2XePgdx2/0wAvnPSaL725f35wzPzOOSEK+japRMTL/1mwX0LefblhVw96WGqa6qprhLXXXIqO/XqVvbfaTm3/MuZfPnQ4fTt3Z3XHrqKayZP585pz1W6WamT5tNPtXbdpyQVS3cDY4Fa4F3gsoi4tdA+hx46Mp75f7PL0h4rj50O2/y6oKXXpwvuZcPHK7YqkfYd8YX4zQNPFlV21B69X4yIkVtzvKTK1lOLiNPKVbeZVVh6O2o+/TSzZHKXy9Kbag41M0vG86mZWdakONP8SIeZJSWk4j7t1iSNl7RA0kJJl7Sy/QxJc/OfZyUd1F6d7qmZWWKlOP2UVA1MBL4K1AGzJE2LiPnNii0GjoqINZImAJOBwwvV656amSVS7HO3ReTeKGBhRCyKiPXAFOD45gUi4tmIaBqW8zyw+WDpFhxqZpZc8alWK2l2s8+5zWoZBCxrtlyXX9eWs4Hft9c0n36aWWIJHumoL/DwbWuVtDoaQNJXyIXake0d0KFmZomV6JGOOmBIs+XBwPLNj6UDgVuACRGxqr1KffppZsnkn1Mr5tOOWcBwScMkdQZOBaZtcihpKDAV+FZE/KmY5rmnZmaJlWJEQUQ0SLoAmAFUA7dFxDxJ5+e3TwIuBfoCN+UfEWlobyypQ83MEhGlG1EQEdOB6S3WTWr2/RzgnCR1OtTMLLE0jyhwqJlZcilONYeamSWW5kkiHWpmllh6I82hZmZbIsWp5lAzs0Q8SaSZZYsniTSzrElxpjnUzCyp4iaArBSHmpklluJMc6iZWTIVfPl6URxqZpZcilPNoWZmifmRDjPLFF9TM7PsEFQ51MwsW9Kbag41M0uklJNEloNDzcwSS3GmOdTMLDn31MwsUzxMyswyJb2R5lAzs4SKfKdnxTjUzCwxjygws2xJb6Y51MwsuRRnmkPNzJKSX5FnZtmR9hEFVZVugJlZKbmnZmaJpbmn5lAzs8T8SIeZZYcfvjWzLEn7jQKHmpkl5tNPM8sU99TMLFNSnGkONTPbAilONYeamSUiSPUwKUVEpduwkaSVwJJKt6MMaoH6SjfCEsnqv9muEdFvayqQ9Ai5v08x6iNi/NYcL6lUhVpWSZodESMr3Q4rnv/NOi6P/TSzTHGomVmmONS2jcmVboAl5n+zDsrX1MwsU9xTM7NMcaiZWaY41MpI0nhJCyQtlHRJpdtj7ZN0m6QVkl6rdFtsyzjUykRSNTARmADsB5wmab/KtsqKcDuwTR8WtdJyqJXPKGBhRCyKiPXAFOD4CrfJ2hERTwOrK90O23IOtfIZBCxrtlyXX2dmZeRQK5/WRvz6+RmzMnOolU8dMKTZ8mBgeYXaYrbdcKiVzyxguKRhkjoDpwLTKtwms8xzqJVJRDQAFwAzgNeBeyNiXmVbZe2RdDfwHLC3pDpJZ1e6TZaMh0mZWaa4p2ZmmeJQM7NMcaiZWaY41MwsUxxqZpYpDrUORFKjpDmSXpN0n6Qdt6Ku2yWdnP9+S6HB9pLGSjpiC47xpqTN3jrU1voWZT5MeKzLJf1T0jZa9jjUOpZ1EXFwRBwArAfOb74xPzNIYhFxTkTML1BkLJA41MwqwaHWcc0E9sz3op6QdBfwqqRqSddKmiVprqTzAJRzo6T5kh4G+jdVJOlJSSPz38dLeknSK5Iel7QbufD8fr6XOFpSP0n3548xS9KX8/v2lfSopJcl/Yoi3uMt6T8lvShpnqRzW2y7Lt+WxyX1y6/bQ9Ij+X1mStqnJH9Nywy/ob0DklRDbp62R/KrRgEHRMTifDC8FxGHSdoBeEbSo8AXgL2BEcAAYD5wW4t6+wE3A2PydfWJiNWSJgEfRsQv8uXuAn4ZEX+UNJTcqIl9gcuAP0bElZKOBTYJqTZ8J3+MrsAsSfdHxCqgG/BSRPyjpEvzdV9A7oUo50fEnyUdDtwEjNuCP6NllEOtY+kqaU7++0zgVnKnhS9ExOL8+q8BBzZdLwN6AcOBMcDdEdEILJf0X63U/0Xg6aa6IqKtecX+CthP2tgR6ympR/4YJ+b3fVjSmiJ+00WSTsh/H5Jv6ypgA3BPfv2dwFRJ3fO/975mx96hiGPYdsSh1rGsi4iDm6/I/8f9UfNVwIURMaNFuWNof+ojFVEGcpctvhQR61ppS9Hj7iSNJReQX4qIjyU9CXRpo3jkj7u25d/ArDlfU8ueGcA/SOoEIGkvSd2Ap4FT89fcBgJfaWXf54CjJA3L79snv/4DoEezco+SOxUkX+7g/NengTPy6yYAO7XT1l7Amnyg7UOup9ikCmjqbZ5O7rT2fWCxpG/kjyFJB7VzDNvOONSy5xZy18teyr885FfkeuS/A/4MvAr8H+CpljtGxEpy18GmSnqFz0//HgROaLpRAFwEjMzfiJjP53dhrwDGSHqJ3Gnw0nba+ghQI2kucBXwfLNtHwH7S3qR3DWzK/PrzwDOzrdvHp4i3VrwLB1mlinuqZlZpjjUzCxTHGpmlikONTPLFIeamWWKQ83MMsWhZmaZ8v8B2tJ9gdpcXcIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.6900805639476334\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8864    0.0878    0.1597      1333\n",
      "           1     0.6833    0.9943    0.8100      2639\n",
      "\n",
      "    accuracy                         0.6901      3972\n",
      "   macro avg     0.7848    0.5410    0.4849      3972\n",
      "weighted avg     0.7515    0.6901    0.5918      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbYklEQVR4nO3debxd49338c/3nESSkklORhmEEnMMMUVLUCTqrqmUqqq2D0p4bo8anvtlKk8nbnfNTdNWUcptCKJCRN1EE9oMhgyKICKJyEAIEnHO+T1/7J045zjZZy85++x11vm+X6/9eu211rWude1Evq41XNdSRGBmlhUV5W6AmVlzcqiZWaY41MwsUxxqZpYpDjUzy5R25W5AXZv3qIoBAweVuxmWwKzXl5a7CZZArHmPWPuxNqaOyi6DIqpXF3e81csmRsTIjTleUqkKtQEDB/HYU8+WuxmWwFbHXV/uJlgCnz533UbXEdWr6TDk+KLKrnnhpqqNPmBCqQo1M2sNBErvlSuHmpklI6Cistyt2CCHmpklp426LFdSDjUzS8inn2aWNe6pmVlmCPfUzCxL5J6amWWM736aWXb4RoGZZYnw6aeZZYx7amaWHT79NLMsEVDpGwVmliW+pmZm2eHTTzPLGvfUzCxT3FMzs8yQh0mZWdZ4mJSZZYdvFJhZ1vj008wyw/OpmVm2+PTTzLLGNwrMLFN8Tc3MMkM+/TSzrHFPzcyyRA41M8uK3GzeDjUzywoJVTjUzCxD3FMzs0xxqJlZpjjUzCw7lP+klEPNzBIRck/NzLKlosIjCswsQ9xTM7PsSPk1tfT2Ic0stSQV9SminpGSXpE0T9JFjWzvKulhSS9KmiPp1KbqdKiZWSLrbhRsbKhJqgRuAkYBOwAnStqhQbGzgLkRMRQYAVwjaZNC9TrUzCwxVaioTxP2AuZFxBsRsRa4GziyQZkAOiuXkJsB7wHVhSr1NTUzS0aJbhRUSZpeZ3lsRIzNf98CeLvOtoXA3g32vxEYDywGOgPfiYjaQgd0qJlZYglCbXlEDNtQNY2siwbLhwEvAAcBWwOTJD0TER9u6IA+/TSzxJrpRsFCYECd5f7kemR1nQqMi5x5wJvAdoUqdaiZWSLNdaMAmAZsI2lw/uL/CeRONetaABwMIKk3MAR4o1ClPv00s+Sa4Tm1iKiWNBqYCFQCt0TEHEln5LePAa4EbpU0K3/UCyNieaF6HWpmloyab5hUREwAJjRYN6bO98XAoUnqdKiZWWIeJmVm2ZLeTHOoJfX0P17mihsfpLamluO/uQ8/OengetsjgitueICnnnuZjh034eqLTmSnbfsD8Md7n+aeR55DiG236svVF55Ahw7tmfvaIi7+r3v5dG01lZUVXHnusQzdflA5fl7mHbzHlvzy9IOorBB/njiLa+/9Z73tXTfrwI3/PpLBfbuxZm01Z187kZffyl3CueHfD+OwvbZm+cpPGH7mrWVofXqkuadW0rufTY3ram1qamq57Lpx/OnXpzHxtgt5+MmZvDZ/Sb0yT/3jZeYvXM6Td/4HvzjvOC75zX0ALFm2ktvuf4aHfncuj916AbW1tTz85PMA/Op3D3PODw7jkT/+lHN/OJJfjflri/+2tqCiQlx95jc47tL72eeMP3HsAdsxZECPemXOO34fZr2xlK+ddRs/ueZRfnn6geu33fXEHL59yX0t3ezUKfbOZ7mCr2ShVuS4rlblxX8tYNAWVQzs14NN2rfjiIN2Y9KU2fXKPDFlNkcfNgxJ7Lbjlnz40WqWrsg9J1hTU8uaTz+jurqG1Ws+o3dVVyD3H8lHH68BYNXHa+hV1aVlf1gbsce2fXhj8fu8teQDPquuZdzkf3H4vlvXKzNkYA8mv7AAgNcWvsfA3l3p2e0rAEydvZD3V61p8XanUZpDrZSnn+vHdQFIWjeua24Jj1lSS5Z9QN+e3dYv9+3ZjRfmvtWgzIf1yvTp2Y0lyz5gl+0G8OPvjOBrx19Jxw7t+dqeQ/j6nkMAuGT0UZxy/u/45W8fpjZque/Gc1ri57Q5fXt0ZtHyVeuXFy//iD2G9K1XZvabSzliv214bu4idt+2DwN6daFfVWeWrfykpZubaml+RV4pTz8bG9e1RcNCkk6TNF3S9BUrCj5+kgINR3B88dpCNFoGPlj1CU9Mmc3Td1/Ms/dfzurVa3nw8dyQuDsfmsLFZx3JlHsv5eKzjuLCq/67NM1v4xrrOETU//u69p5/0m2zjky+4fuc9q3deOn1pdTUFBxq2CaluadWylArZlwXETE2IoZFxLAePapK2JyN16dnN95ZtnL98jvLVn7hVLFvz671yixZtpLeVV2ZMuNV+vfdnB7dNqN9u0oO239nZsyZD8D9E6czcv9dADh8xFBe+teCUv+UNmnx8lVsUdV5/XK/qs1Y8t5H9cqsWr2W0b95jP3Pvp0z/vNRqrp24q0lH7R0U9NNbTfUihnX1arsMmQA8xcu4+13VrD2s2r++uTzfGP4TvXKHDx8Jx6YOJ2I4Pk58+m8aUd69ehCv17deWHuW6xes5aIYOrM1/jqoN4A9O7RhX+88DoAU2e+xpb9e7b4b2sLZr66hK37dWdg7660b1fBMftvx6PPvV6vTJdNO9C+Xe6fxfcP25mpsxeyavXacjQ3tUSu11vMpxxKeU1t/bguYBG5cV3fLeHxSq5du0ou/9/HcMr5Y6mtreW4UXux7eA+3PnQVABOOnI4B+6zPU/942UOPOkXdOzQnqsuPBGAXXcYxMgDhvJv/+u/aFdZwQ7bbMEJR+wLwC9+ejxX3vgg1TU1dNikPT8/77iy/cYsq6kNLvjt37j//x1LZUUFdz4+i38tWMGphw8F4E8TXmTIgM357XmHU1NbyysLVnD2dRPX7/+HC77JfrsMoEeXTsy+/XR+dccU7nh89oYOl2HpfpuUGl5TaNbKpcOBa/l8XNfPC5Ufutse8dhTz5asPdb8tjru+nI3wRL49LnrqP3w7Y1KpI59to1Bp9xQVNlXrxo5o8DUQyVR0odvGxvXZWatXBlPLYvhEQVmlojIPcicVg41M0vMPTUzy5Q03yhwqJlZMr6mZmZZItRsk0SWgkPNzBJzT83MMsXX1MwsO3xNzcyyJDf2M72p5lAzs8RSnGkONTNLziMKzCw75NNPM8uQdfOppZVDzcwSSvd8ag41M0ssxZnmUDOzhOQbBWaWIX5Ozcwyx6FmZpmS4kxzqJlZcu6pmVl2eEC7mWVJbpLI9KaaQ83MEqtIcVctvXPymllqScV9mq5HIyW9ImmepIs2UGaEpBckzZH0dFN1uqdmZomomQa0S6oEbgIOARYC0ySNj4i5dcp0A24GRkbEAkm9mqrXPTUzS6xCxX2asBcwLyLeiIi1wN3AkQ3KfBcYFxELACJiaVOVbrCnJukGIDa0PSLOabLJZpZJCW4UVEmaXmd5bESMzX/fAni7zraFwN4N9t8WaC/pKaAzcF1E3F7ogIVOP6cX2GZmbZTI3QEt0vKIGFagqoYadqTaAXsABwOdgGclPRcRr27ogBsMtYi4rd7RpU0j4uMNlTeztqOZnuhYCAyos9wfWNxImeX57PlY0mRgKLDBUGvympqkfSXNBV7OLw+VdHPCxptZVig3n1oxnyZMA7aRNFjSJsAJwPgGZR4Cvi6pnaSvkDs9fblQpcXc/bwWOGzdwSLiRUn7F7GfmWVUczymFhHVkkYDE4FK4JaImCPpjPz2MRHxsqTHgJeAWuAPETG7UL1FPdIREW83SN2aL/MjzKz1E8338G1ETAAmNFg3psHy1cDVxdZZTKi9LWk4EPku4jk00f0zs2xL8zCpYp5TOwM4i9zt10XArvllM2uDih1NUK6RVE321CJiOXBSC7TFzFqJVj32U9JWkh6WtEzSUkkPSdqqJRpnZumkIj/lUMzp51+Ae4C+QD/gXuCuUjbKzNKtmR7pKIliQk0R8eeIqM5/7qDA8Ckzy7bc3c9mGftZEoXGfm6e//o/+SlB7iYXZt8BHmmBtplZGqn1ThI5g1yIrWv96XW2BXBlqRplZunWKt9REBGDW7IhZtY6rDv9TKuiRhRI2gnYAei4bl1T03+YWXa1yp7aOpIuA0aQC7UJwCjg74BDzayNSm+kFXf389vk5jJaEhGnkpv2o0NJW2VmqSVBZYWK+pRDMaefqyOiVlK1pC7AUsAP35q1Ya369BOYnn/5we/J3RH9CPhnKRtlZumW4kwrauznmfmvY/LzGnWJiJdK2ywzSyuhVI/9LPTw7e6FtkXEzNI0ycxSrYwzcBSjUE/tmgLbAjiomdtCuwrRfdNNmrtaK6Vl88vdAkui+tNmqaZVXlOLiANbsiFm1joIqGyNoWZmtiGtfkSBmVldDjUzy4zcVN3pTbViZr6VpO9JujS/PFDSXqVvmpmlVZrnUytmmNTNwL7AifnlVcBNJWuRmaVeq37xCrB3ROwu6XmAiHg//6o8M2uDBLRL8elnMaH2maRK8lN4S+pJ7k3JZtZGpTjTigq164EHgF6Sfk5u1o6LS9oqM0stqZUOk1onIu6UNIPc9EMCjooIv6HdrA1LcaYVNUnkQOAT4OG66yJiQSkbZmbp1dqfU3uEz1/A0hEYDLwC7FjCdplZSgnKNgFkMYo5/dy57nJ+9o7TN1DczLKujM+gFSPxiIKImClpz1I0xsxaB6X4LQXFXFP7P3UWK4DdgWUla5GZpVoWXpHXuc73anLX2O4vTXPMrDVotaGWf+h2s4g4v4XaY2atQJoHtBeazrtdRFQXmtbbzNqe3Cvyyt2KDSvUtHVvjHpB0nhJJ0s6Zt2nJRpnZulUkR9V0NSnKZJGSnpF0jxJFxUot6ekGknfbqrOYq6pbQ6sIPdOgnXPqwUwroh9zSxjmutGQf7y1k3AIcBCYJqk8RExt5FyvwYmFlNvoVDrlb/zOZvPw2ydSNB2M8uYZrqkthcwLyLeyNWpu4EjgbkNyp1N7uZkUY+SFQq1SmAzaPSBFIeaWZslKop/Tq1K0vQ6y2MjYmz++xbA23W2LQT2rnckaQvgaHJnihsdau9ExBXFVGJmbYdI1FNbHhHDClTVUMMO07XAhRFRU+wd10Khlt57tmZWPsq9o7cZLAQG1FnuDyxuUGYYcHc+0KqAwyVVR8SDG6q0UKgd/OXaaWZZlrCnVsg0YBtJg4FFwAnAd+sWiIjB648r3Qr8tVCgQeGXGb+3EY01swxrjkki88/BjiZ3V7MSuCUi5kg6I799zJep16/IM7PEmmtAQURMACY0WNdomEXED4qp06FmZomI4l5DVy4ONTNLRs1z+lkqDjUzSyQ3osChZmYZkt5Ic6iZ2ZeQ4o6aQ83MklLrnE/NzKwxvvtpZpnjGwVmlh1qpdN5m5k1xqefZpY57qmZWaakN9IcamaWkIBK99TMLEtSnGkONTNLSijFJ6AONTNLzD01M8uM3CMd6U01h5qZJSP31MwsYzxMyswyIzdJZLlbsWEONTNLzHc/zSxTUnz26VArxhNT5/J/r7mPmtpaTj5yOOf+4NB62yOCi665j0lT5tCp4ybcfNnJDN1uQMF9fzX2EW5/cCo9um0GwCVnfYtD99uRex6dxg1/fmJ93XPmLebpP1/IzkP6t9CvzbaD992eX573bSorKvjzQ1O59rZJ9bZ37dyJGy/5HoP7V7Fm7WecfeWdvPz6OwCcfsIITjlqOEjc/uAUxtz1VBl+QTq0yZ6apFuAI4ClEbFTqY5TajU1tZx/1T08cONo+vXuxkGnXM2o/Xdmu636ri8zaepcXl+wjBnjLmP67Pmc96u7eeLW85vc9ycnHsjZJ3+j3vGOH7Unx4/aE4A58xZx0nljHWjNpKJCXH3B8Rw9+kYWv7uSJ287n0cnz+KVN5esL3PeqYcx69WFnHzB79lmUG+uvvB4jjrzBrbfui+nHDWcg0+5mrXVNdx3/Zk8/vc5vPH2sjL+ovJI+zW1Us4gciswsoT1t4gZc+az1YAqtuxfxSbt23HMIbsz4emX6pWZ8PRLnPDNvZDEnjsP5oNVq1my/IOi9i3k/okzOPawPZr7J7VZe+y4JW+8vZy3Fq3gs+oaxk2ayeEH7FKvzJDBfZg87RUAXnvrXQb23Zyem3dm2y37MG3WfFZ/+hk1NbVMmTmPI0YMLcfPKD+JiiI/5VCyUIuIycB7paq/pbyz7AO26N19/XK/3t15Z9kHDcqsrF+mVzfeWbqyyX1/f+9k9jvxF4y+4g5WfvjJF479wKSZHHvosOb8OW1a355dWfTu++uXF7/7Pn17dq1XZvZrizjiwF0B2H2HQQzoszn9enXj5dcXM3y3r9K966Z06tCeQ4bvWO/vtq1RkZ9yKPtcb5JOkzRd0vRly9PXlY+IL6xr+D+gRoogqeC+Pzz26zz/wOU8c+dF9K7qwsXXjqtXbvrs+XTq2J4dvtrvS7fd6mtsDrCGf0XX3jaJbl2+wuQ7L+K07xzAS68upKamllfnv8t1t0/igRtHc9/1ZzHntUVU19S0UMvTZd17P9PaUyv7jYKIGAuMBdhjj2GNxEN59evV7Qv/d+9T1bVwmaUr6dOzK2s/q97gvr16dFm//pSj9uM7546pV+e4x2dw7GHupTWnxUtXfqHnvGR5/V73qo/XMPqKO9Yvv/jQz3hr8QoA7hj/LHeMfxaAS878NxYvXVn6RqdUii+plb+nlna77zCI1xcs461Fy1n7WTXjJs1k1P71r8OM2n9n7n7kn0QE02a9SZfNOtGnqmvBfev+Y/rrUy+y/daf33iora3lob89z7GH+Hpac5o59y22HtiTgf160L5dJcccsjuPTq5/jbPLZp1o364SgO8fNZypz89j1cdrAKjqnrtT3b93d444cCj3TZzesj8gTVJ8/ln2nlratWtXyVUXHM+x59xETU1w0rf2Yfut+3LL/c8AudPIQ/fbkUlT5rD70T+jU8f23HTp9wruC3DZ9Q8y69WFSGJg3835zX+cuP6YU5+fR79e3diyf1XL/+AMq6mp5YKr7uH+68+islLcOf45/vXGEk495msA/Gnc3xkyuA+/vfxkampreeXNJZx95Z3r97/91z+me9dNqa6u4fyr7uGDVavL9VPKLs3DpNTYdZ9mqVi6CxgBVAHvApdFxB8L7bPHHsNiyj/a8P/9WqHue44udxMsgU9fuYfaT5ZuVCJtv/NucftDTxVVdq+tu82IiBa9jlKynlpEnNh0KTNrldLbUfPpp5klk7tclt5Uc6iZWTKeT83MsibFmeZHOswsKSEV92myJmmkpFckzZN0USPbT5L0Uv4zVVKTY9PcUzOzxJrj9FNSJXATcAiwEJgmaXxEzK1T7E3ggIh4X9Iocg/q712oXvfUzCyRYp+7LSL39gLmRcQbEbEWuBs4sm6BiJgaEeuG5TwHNDlljUPNzJIrPtWq1o3tzn9Oq1PLFsDbdZYX5tdtyI+AR5tqmk8/zSyxBI90LC/w8G1jlTQ6GkDSgeRC7WtNHdChZmaJNdMjHQuBAXWW+wOLv3gs7QL8ARgVESuaqtSnn2aWTP45tWI+TZgGbCNpsKRNgBOA8fUOJQ0ExgEnR8SrxTTPPTUzS6w5RhRERLWk0cBEoBK4JSLmSDojv30McCnQA7g5/4hIdVNjSR1qZpaIaL4RBRExAZjQYN2YOt9/DPw4SZ0ONTNLLM0jChxqZpZcilPNoWZmiaV5kkiHmpkllt5Ic6iZ2ZeR4lRzqJlZIp4k0syyxZNEmlnWpDjTHGpmllRxE0CWi0PNzBJLcaY51MwsmTK+fL0oDjUzSy7FqeZQM7PE/EiHmWWKr6mZWXYIKhxqZpYt6U01h5qZJdKck0SWgkPNzBJLcaY51MwsOffUzCxTPEzKzDIlvZHmUDOzhIp8p2fZONTMLDGPKDCzbElvpjnUzCy5FGeaQ83MkpJfkWdm2ZH2EQUV5W6AmVlzck/NzBJLc0/NoWZmifmRDjPLDj98a2ZZkvYbBQ41M0vMp59mlinuqZlZpqQ40xxqZvYlpDjVHGpmlogg1cOkFBHlbsN6kpYBb5W7HSVQBSwvdyMskaz+nQ2KiJ4bU4Gkx8j9+RRjeUSM3JjjJZWqUMsqSdMjYli522HF899Z6+Wxn2aWKQ41M8sUh1rLGFvuBlhi/jtrpXxNzcwyxT01M8sUh5qZZYpDrYQkjZT0iqR5ki4qd3usaZJukbRU0uxyt8W+HIdaiUiqBG4CRgE7ACdK2qG8rbIi3Aq06MOi1rwcaqWzFzAvIt6IiLXA3cCRZW6TNSEiJgPvlbsd9uU51EpnC+DtOssL8+vMrIQcaqXT2IhfPz9jVmIOtdJZCAyos9wfWFymtpi1GQ610pkGbCNpsKRNgBOA8WVuk1nmOdRKJCKqgdHAROBl4J6ImFPeVllTJN0FPAsMkbRQ0o/K3SZLxsOkzCxT3FMzs0xxqJlZpjjUzCxTHGpmlikONTPLFIdaKyKpRtILkmZLulfSVzairlslfTv//Q+FBttLGiFp+Jc4xnxJX3jr0IbWNyjzUcJjXS7pp0nbaNnjUGtdVkfErhGxE7AWOKPuxvzMIIlFxI8jYm6BIiOAxKFmVg4OtdbrGeCr+V7U/0j6CzBLUqWkqyVNk/SSpNMBlHOjpLmSHgF6ratI0lOShuW/j5Q0U9KLkv4maUty4Xluvpf4dUk9Jd2fP8Y0Sfvl9+0h6XFJz0v6HUW8x1vSg5JmSJoj6bQG267Jt+Vvknrm120t6bH8Ps9I2q5Z/jQtM/yG9lZIUjty87Q9ll+1F7BTRLyZD4YPImJPSR2AKZIeB3YDhgA7A72BucAtDertCfwe2D9f1+YR8Z6kMcBHEfGf+XJ/AX4TEX+XNJDcqIntgcuAv0fEFZK+CdQLqQ34Yf4YnYBpku6PiBXApsDMiDhP0qX5ukeTeyHKGRHxmqS9gZuBg77EH6NllEOtdekk6YX892eAP5I7LfxnRLyZX38osMu662VAV2AbYH/groioARZLerKR+vcBJq+rKyI2NK/YN4AdpPUdsS6SOuePcUx+30ckvV/EbzpH0tH57wPybV0B1AL/nV9/BzBO0mb533tvnWN3KOIY1oY41FqX1RGxa90V+X/cH9ddBZwdERMblDucpqc+UhFlIHfZYt+IWN1IW4oedydpBLmA3DciPpH0FNBxA8Ujf9yVDf8MzOryNbXsmQj8RFJ7AEnbStoUmAyckL/m1hc4sJF9nwUOkDQ4v+/m+fWrgM51yj1O7lSQfLld818nAyfl140CujfR1q7A+/lA245cT3GdCmBdb/O75E5rPwTelHRc/hiSNLSJY1gb41DLnj+Qu142M//ykN+R65E/ALwGzAJ+CzzdcMeIWEbuOtg4SS/y+enfw8DR624UAOcAw/I3Iuby+V3YnwH7S5pJ7jR4QRNtfQxoJ+kl4ErguTrbPgZ2lDSD3DWzK/LrTwJ+lG/fHDxFujXgWTrMLFPcUzOzTHGomVmmONTMLFMcamaWKQ41M8sUh5qZZYpDzcwy5f8DJkpf7QyRrPQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammWord(knn_word, model_name=\"knn_word\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "id": "dk2jpzHwU-UV"
   },
   "outputs": [],
   "source": [
    "grid_search_knn = GridSearchCV(knn, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "id": "woV5TuSTU-dh"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV] END .................................................... total time=  13.4s\n",
      "[CV] END .................................................... total time=  11.7s\n",
      "[CV] END .................................................... total time=  11.5s\n",
      "[CV] END .................................................... total time=  11.7s\n",
      "[CV] END .................................................... total time=  11.7s\n"
     ]
    }
   ],
   "source": [
    "\n",
    "knn_char = grid_search_knn.fit(X_train_char, y_train_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "id": "1fd0rCvmU-ne"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.6890375485541649\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8841    0.0857    0.1563      3114\n",
      "           1     0.6825    0.9943    0.8094      6154\n",
      "\n",
      "    accuracy                         0.6890      9268\n",
      "   macro avg     0.7833    0.5400    0.4829      9268\n",
      "weighted avg     0.7502    0.6890    0.5900      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAbT0lEQVR4nO3deZRU1b328e/TDQqRUZpRAYlRIs6CqJgoxqhgTJynGGO8SRQjJsvrdchdSdT43ptEkzfEKYQkhjhErwMqXlEkg6KiBgSVKSriACIyKEgUA939u39Ugd1tU11HurpOn34+a9Vadc7Ztc+u7tVP7zPsfRQRmJllRUW5G2Bm1pwcamaWKQ41M8sUh5qZZYpDzcwypV25G1DX9j2qov+AgeVuhiUw95W3y90ESyA+fJfY8L62po7KLgMjqtcXt7/1K6dGxKit2V9SqQq1/gMGMvXRp8rdDEtg0Enjyt0ES+Bfz1y71XVE9Xq2HXxKUWU/fO6Gqq3eYUKpCjUzaw0ESu+ZK4eamSUjoKKy3K3YIoeamSWnrTotV1IONTNLyIefZpY17qmZWWYI99TMLEvknpqZZYyvfppZdvhCgZllifDhp5lljHtqZpYdPvw0sywRUOkLBWaWJT6nZmbZ4cNPM8sa99TMLFPcUzOzzJCHSZlZ1niYlJllhy8UmFnW+PDTzDLD86mZWbb48NPMssYXCswsU3xOzcwyQz78NLOscU/NzLJEDjUzy4rcbN4ONTPLCglVONTMLEPcUzOzTHGomVmmONTMLDuUf6WUQ83MEhFyT83MsqWiwiMKzCxD3FMzs+xI+Tm19PYhzSy1JBX1KqKeUZJelLRI0mWNbO8q6QFJz0uaL+nspup0qJlZIpsuFGxtqEmqBG4ARgNDgNMlDWlQ7HxgQUTsDYwEfiFpm0L1OtTMLDFVqKhXE4YDiyJicURsAO4Ajm1QJoDOyiVkJ+AdoLpQpT6nZmbJKNGFgipJs+osT4iICfn3OwBL6mxbChzQ4PPXA5OBZUBn4NSIqC20Q4eamSWWINRWRcSwLVXTyLposHwU8BzwBWBnYJqkxyPivS3t0IefZpZYM10oWAr0r7O8I7keWV1nA5MiZxHwKvDZQpU61Mwskea6UADMBHaRNCh/8v80coeadb0BHA4gqTcwGFhcqFIffppZcs1wn1pEVEsaC0wFKoGbImK+pDH57eOBq4CJkubm93ppRKwqVK9DzcySUfMNk4qIKcCUBuvG13m/DDgySZ0ONTNLzMOkzCxb0ptpDrWkHntmIVdefx+1NbWc+qUDOe+Mw+ttjwiuvO5eHn16IR06bMPPLzudPXbdEYDf3/UY//Pg0wgx+NN9uebS09h22/YATJz0ODff+wTtKis47MAhfH/Ml1v8u7UFhw8dxE/GHE5lhbjl4RcYd9cz9bZ37bQt1184mkF9u/Hhhhou+OVDLHw9dwrnugtHcdTwnVm15gNGnPeHcjQ/NdLcUyvp1c+mxnW1NjU1tfzoV5OY+LNzeOSPlzL5r7N5+bXl9co8+sxCXlu6ir/d9p/85KKT+cEv7wZg+co1TLzncSb/5kKmTryEmtpaHvjrHACemvMyf35iHg/9/mIemXgp3z51ZEt/tTahokJcc/4XOfmHd3Hgub/nxJG7MXhAj3plLjr1IOa+soLPfWci5/38QX4y5qN/WrdPm8dJP7i7pZudOsVe+SxX8JUs1Ioc19WqPP+PNxi4QxUD+vVgm/bt+PIX9mXak/PqlZn25DxOOGoYkth3951475/rWbE6d59gTU0tH/5rI9XVNXz44UZ6VXUF4Nb7ZzDmq4ez7Ta5jnNV984t+8XaiKG79mXxsjW8vnwtG6trmfTYQo4+8DP1ygwe0IPpz78OwMtL32FA7y707PYpAGbMW8q769a3eLvTqE2GGsWN62pVlq9cS9+e3TYv9+nZjeUr19Yr8/bK9+qV6Zsv06dnN7596kgOPuUqDjjxCjp36sAh+w8G4NUlK5k5dzHHnTeOU793Pc//442W+DptTt+qTry5ct3m5WWr1tG3R/1/IPMWr+CYEbsCsN+ufejfqyv9qvxPpqFmGvtZEqUMtcbGde3QsJCkcyTNkjRr9eqCt5+UXXxsBMfHzy00XgbWrvuAaU/OY/odP+Dpe67gg/UbuPeR3JC4mppa1q77gHtv/B7fH/Nlxl5xMxEfr8e2jho5u93w9zXurmfo1qkD068/i3O+MpQXXnmbmpqCQw3bpDT31Ep5oaCYcV3kB7dOANh736Gp/kvu27Mbb61cs3l5+co19K7qUq9Mn55d65V5a+Uaeld15YlnX6J/3+3p0a0TAEcdsiez57/G8UcOo0/Proz6/F5IYp/dBlJRId5Z+/7mstY8lq1axw49P+p19avqzPLV/6xXZt0HGxj7y4c2Lz8/8Vxef7t+b7zNSzagvcWVsqdWzLiuVmWvwf15belKlry1mg0bq3ngr3P44og96pX54og9mDR1FhHBnPmv0Xm7DvTq0YV+vbozZ8HrrP9wAxHBjNkvs/PA3gAc+bk9mTHnZQAWL1nBxo01bN91uxb/flk3+6W32Llfdwb07kr7dhWccOhuPPT0onplumy3Le3b5f4svj5qL2bMXcK6DzaUo7mpJXJHH8W8yqGUPbXN47qAN8mN6/pqCfdXcu3aVXLl907g6xdPoLa2lpNHD2fXQX247f4ZAJxx7AgOO3A3/vbMQkae8d903LY9V196OgD7DhnI6EP35phv/3/aVVYwZJcdOP2YgwA4+ejhXPKzOzjqG1fTvn0lP//+6an+T9ha1dQGl/z6z9zz/06mslLc9shc/vHGas4+eh8A/jDlOQb378Gv/+NL1NTW8uIbq7lg3Ee9tt9d+mUO3qs/Pbp0ZN4t5/HTW57g1kfmlunblFO6nyalUp67kXQ0MI6PxnX9V6Hye+87NKY++lTJ2mPNb9BJ48rdBEvgX89cS+17S7cqkTr02TUGnnVdUWVfunrUswWmHiqJkt5829i4LjNr5cp4aFkMjygws0RE7kbmtHKomVli7qmZWaak+UKBQ83MkvE5NTPLEqFmmySyFBxqZpaYe2pmlik+p2Zm2eFzamaWJbmxn+lNNYeamSWW4kxzqJlZch5RYGbZkfL51BxqZpbIpvnU0sqhZmYJpXs+NYeamSWW4kxzqJlZQvKFAjPLEN+nZmaZ41Azs0xJcaY51MwsOffUzCw7PKDdzLIkN0lkelPNoWZmiVWkuKuW3jl5zSy1pOJeTdejUZJelLRI0mVbKDNS0nOS5kt6rKk63VMzs0TUTAPaJVUCNwBHAEuBmZImR8SCOmW6ATcCoyLiDUm9mqrXPTUzS6xCxb2aMBxYFBGLI2IDcAdwbIMyXwUmRcQbABGxoqlKt9hTk3QdEFvaHhHfbbLJZpZJCS4UVEmaVWd5QkRMyL/fAVhSZ9tS4IAGn98VaC/pUaAz8KuIuLnQDgsdfs4qsM3M2iiRuwJapFURMaxAVQ017Ei1A4YChwMdgackPR0RL21ph1sMtYj4Y729S9tFxPtbKm9mbUcz3dGxFOhfZ3lHYFkjZVbls+d9SdOBvYEthlqT59QkHSRpAbAwv7y3pBsTNt7MskK5+dSKeTVhJrCLpEGStgFOAyY3KHM/8HlJ7SR9itzh6cJClRZz9XMccNSmnUXE85IOKeJzZpZRzXGbWkRUSxoLTAUqgZsiYr6kMfnt4yNioaSHgReAWuB3ETGvUL1F3dIREUsapG7NJ/kSZtb6iea7+TYipgBTGqwb32D5GuCaYussJtSWSBoBRL6L+F2a6P6ZWbaleZhUMfepjQHOJ3f59U1gn/yymbVBxY4mKNdIqiZ7ahGxCjijBdpiZq1Eqx77KenTkh6QtFLSCkn3S/p0SzTOzNJJRb7KoZjDzz8BdwJ9gX7AXcDtpWyUmaVbM93SURLFhJoi4paIqM6/bqXA8Ckzy7bc1c9mGftZEoXGfm6ff/u3/JQgd5ALs1OBB1ugbWaWRmq9k0Q+Sy7ENrX+3DrbAriqVI0ys3Rrlc8oiIhBLdkQM2sdNh1+plVRIwok7QEMATpsWtfU9B9mll2tsqe2iaTLgZHkQm0KMBp4AnCombVR6Y204q5+nkRuLqPlEXE2uWk/ti1pq8wstSSorFBRr3Io5vBzfUTUSqqW1AVYAfjmW7M2rFUffgKz8g8/+C25K6L/BP5eykaZWbqlONOKGvv5nfzb8fl5jbpExAulbZaZpZVQqsd+Frr5dr9C2yJidmmaZGapVsYZOIpRqKf2iwLbAvhCM7eFdhWi23bbNHe1Vkqr3ih3CyyJ6g3NUk2rPKcWEYe1ZEPMrHUQUNkaQ83MbEta/YgCM7O6HGpmlhm5qbrTm2rFzHwrSV+T9KP88gBJw0vfNDNLqzTPp1bMMKkbgYOA0/PL64AbStYiM0u9Vv3gFeCAiNhP0hyAiHg3/6g8M2uDBLRL8eFnMaG2UVIl+Sm8JfUk96RkM2ujUpxpRYXatcC9QC9J/0Vu1o4flLRVZpZaUisdJrVJRNwm6Vly0w8JOC4i/IR2szYsxZlW1CSRA4APgAfqrosIj48xa6Na+31qD/LRA1g6AIOAF4HdS9guM0spQdkmgCxGMYefe9Zdzs/ece4WiptZ1pXxHrRiJB5REBGzJe1fisaYWeugFD+loJhzav9eZ7EC2A9YWbIWmVmqZeEReZ3rvK8md47tntI0x8xag1YbavmbbjtFxMUt1B4zawXSPKC90HTe7SKiutC03mbW9uQekVfuVmxZoaZtemLUc5ImSzpT0gmbXi3RODNLp4r8qIKmXk2RNErSi5IWSbqsQLn9JdVIOqmpOos5p7Y9sJrcMwk23a8WwKQiPmtmGdNcFwryp7duAI4AlgIzJU2OiAWNlPsZMLWYeguFWq/8lc95fBRmm0SCtptZxjTTKbXhwKKIWJyrU3cAxwILGpS7gNzFyaJuJSsUapVAJ2j0hhSHmlmbJSqKv0+tStKsOssTImJC/v0OwJI625YCB9Tbk7QDcDy5I8WtDrW3IuLHxVRiZm2HSNRTWxURwwpU1VDDDtM44NKIqCn2imuhUEvvNVszKx/lntHbDJYC/ess7wgsa1BmGHBHPtCqgKMlVUfEfVuqtFCoHf7J2mlmWZawp1bITGAXSYOAN4HTgK/WLRARgzbvV5oI/G+hQIPCDzN+Zysaa2YZ1hyTRObvgx1L7qpmJXBTRMyXNCa/ffwnqdePyDOzxJprQEFETAGmNFjXaJhFxDeKqdOhZmaJiOIeQ1cuDjUzS0bNc/hZKg41M0skN6LAoWZmGZLeSHOomdknkOKOmkPNzJJS65xPzcysMb76aWaZ4wsFZpYdaqXTeZuZNcaHn2aWOe6pmVmmpDfSHGpmlpCASvfUzCxLUpxpDjUzS0ooxQegDjUzS8w9NTPLjNwtHelNNYeamSUj99TMLGM8TMrMMiM3SWS5W7FlDjUzS8xXP80sU1J89OlQK8afZyzg+7+4m5raWs48dgQXfuPIetsjgst+cTfTnpxPxw7bcOPlZ7L3Z/sX/OxPJzzIzffNoEe3TgD88PyvcOTBu3PnQzO57pY/b657/qJlPHbLpew5eMcW+rbZdvhBu/GTi06isqKCW+6fwbg/Tqu3vWvnjlz/w68xaMcqPtywkQuuuo2Fr7wFwLmnjeSs40aAxM33Pcn42x8twzdIhzbZU5N0E3AMsCIi9ijVfkqtpqaWi6++k3uvH0u/3t34wlnXMPqQPfnsp/tuLjNtxgJeeWMlz066nFnzXuOin97Bnyde3ORnzzv9MC4484v19nfK6P05ZfT+AMxf9CZnXDTBgdZMKirENZecwvFjr2fZ22v46x8v5qHpc3nx1eWby1x09lHMfWkpZ17yW3YZ2JtrLj2F475zHbvt3JezjhvB4Wddw4bqGu6+9js88sR8Fi9ZWcZvVB5pP6dWyhlEJgKjSlh/i3h2/mt8un8VO+1YxTbt23HCEfsx5bEX6pWZ8tgLnPal4Uhi/z0HsXbdepavWlvUZwu5Z+qznHjU0Ob+Sm3W0N13YvGSVbz+5mo2Vtcwadpsjj50r3plBg/qw/SZLwLw8utvM6Dv9vTcvjO77tSHmXNfY/2/NlJTU8uTsxdxzMi9y/E1yk+ioshXOZQs1CJiOvBOqepvKW+tXMsOvbtvXu7XuztvrVzboMya+mV6deOtFWua/Oxv75rOwaf/N2N/fCtr3vvgY/u+d9psTjxyWHN+nTatb8+uvPn2u5uXl739Ln17dq1XZt7Lb3LMYfsAsN+QgfTvsz39enVj4SvLGLHvZ+jedTs6btueI0bsXu9329aoyFc5lH2uN0nnSJoladbKVenrykfEx9Y1/AfUSBEkFfzsv534eebcewWP33YZvau68INxk+qVmzXvNTp2aM+Qz/T7xG23+hqbA6zhr2jcH6fRrcunmH7bZZxz6qG88NJSampqeem1t/nVzdO49/qx3H3t+cx/+U2qa2paqOXpsum5n2ntqZX9QkFETAAmAAwdOqyReCivfr26fey/e5+qroXLrFhDn55d2bCxeouf7dWjy+b1Zx13MKdeOL5enZMeeZYTj3IvrTktW7HmYz3n5avq97rXvf8hY3986+bl5++/kteXrQbg1slPcevkpwD44Xe+zLIVa0rf6JRK8Sm18vfU0m6/IQN55Y2VvP7mKjZsrGbStNmMPqT+eZjRh+zJHQ/+nYhg5txX6dKpI32quhb8bN0/pv999Hl22/mjCw+1tbXc/5c5nHiEz6c1p9kLXmfnAT0Z0K8H7dtVcsIR+/HQ9PrnOLt06kj7dpUAfP24EcyYs4h1738IQFX33JXqHXt355jD9ubuqbNa9gukSYqPP8veU0u7du0qufqSUzjxuzdQUxOc8ZUD2W3nvtx0z+NA7jDyyIN3Z9qT89nv+Cvp2KE9N/zoawU/C3D5tfcx96WlSGJA3+355X+evnmfM+Ysol+vbuy0Y1XLf+EMq6mp5ZKr7+Sea8+nslLcNvlp/rF4OWef8DkA/jDpCQYP6sOvrziTmtpaXnx1ORdcddvmz9/8s2/Rvet2VFfXcPHVd7J23fpyfZWyS/MwKTV23qdZKpZuB0YCVcDbwOUR8ftCnxk6dFg8+Uwb/u/XCnXff2y5m2AJ/OvFO6n9YMVWJdJue+4bN9//aFFlh+/c7dmIaNHzKCXrqUXE6U2XMrNWKb0dNR9+mlkyudNl6U01h5qZJeP51Mwsa1Kcab6lw8ySElJxryZrkkZJelHSIkmXNbL9DEkv5F8zJDU5Ns09NTNLrDkOPyVVAjcARwBLgZmSJkfEgjrFXgUOjYh3JY0md6P+AYXqdU/NzBIp9r7bInJvOLAoIhZHxAbgDuDYugUiYkZEbBqW8zTQ5JQ1DjUzS674VKvaNLY7/zqnTi07AEvqLC/Nr9uSbwIPNdU0H36aWWIJbulYVeDm28YqaXQ0gKTDyIXa55raoUPNzBJrpls6lgL96yzvCCz7+L60F/A7YHRErG6qUh9+mlky+fvUink1YSawi6RBkrYBTgMm19uVNACYBJwZES8V0zz31MwsseYYURAR1ZLGAlOBSuCmiJgvaUx++3jgR0AP4Mb8LSLVTY0ldaiZWSKi+UYURMQUYEqDdePrvP8W8K0kdTrUzCyxNI8ocKiZWXIpTjWHmpklluZJIh1qZpZYeiPNoWZmn0SKU82hZmaJeJJIM8sWTxJpZlmT4kxzqJlZUsVNAFkuDjUzSyzFmeZQM7Nkyvjw9aI41MwsuRSnmkPNzBLzLR1mlik+p2Zm2SGocKiZWbakN9UcamaWSHNOElkKDjUzSyzFmeZQM7Pk3FMzs0zxMCkzy5T0RppDzcwSKvKZnmXjUDOzxDyiwMyyJb2Z5lAzs+RSnGkONTNLSn5EnpllR9pHFFSUuwFmZs3JPTUzSyzNPTWHmpkl5ls6zCw7fPOtmWVJ2i8UONTMLDEffppZprinZmaZkuJMc6iZ2SeQ4lRzqJlZIoJUD5NSRJS7DZtJWgm8Xu52lEAVsKrcjbBEsvo7GxgRPbemAkkPk/v5FGNVRIzamv0llapQyypJsyJiWLnbYcXz76z18thPM8sUh5qZZYpDrWVMKHcDLDH/zlopn1Mzs0xxT83MMsWhZmaZ4lArIUmjJL0oaZGky8rdHmuapJskrZA0r9xtsU/GoVYikiqBG4DRwBDgdElDytsqK8JEoEVvFrXm5VArneHAoohYHBEbgDuAY8vcJmtCREwH3il3O+yTc6iVzg7AkjrLS/PrzKyEHGql09iIX98/Y1ZiDrXSWQr0r7O8I7CsTG0xazMcaqUzE9hF0iBJ2wCnAZPL3CazzHOolUhEVANjganAQuDOiJhf3lZZUyTdDjwFDJa0VNI3y90mS8bDpMwsU9xTM7NMcaiZWaY41MwsUxxqZpYpDjUzyxSHWisiqUbSc5LmSbpL0qe2oq6Jkk7Kv/9docH2kkZKGvEJ9vGapI89dWhL6xuU+WfCfV0h6T+SttGyx6HWuqyPiH0iYg9gAzCm7sb8zCCJRcS3ImJBgSIjgcShZlYODrXW63HgM/le1N8k/QmYK6lS0jWSZkp6QdK5AMq5XtICSQ8CvTZVJOlRScPy70dJmi3peUl/kbQTufC8MN9L/LyknpLuye9jpqSD85/tIekRSXMk/YYinuMt6T5Jz0qaL+mcBtt+kW/LXyT1zK/bWdLD+c88LumzzfLTtMzwE9pbIUntyM3T9nB+1XBgj4h4NR8MayNif0nbAk9KegTYFxgM7An0BhYANzWotyfwW+CQfF3bR8Q7ksYD/4yIn+fL/Qn4ZUQ8IWkAuVETuwGXA09ExI8lfQmoF1Jb8G/5fXQEZkq6JyJWA9sBsyPiIkk/ytc9ltwDUcZExMuSDgBuBL7wCX6MllEOtdalo6Tn8u8fB35P7rDw7xHxan79kcBem86XAV2BXYBDgNsjogZYJumvjdR/IDB9U10RsaV5xb4IDJE2d8S6SOqc38cJ+c8+KOndIr7TdyUdn3/fP9/W1UAt8D/59bcCkyR1yn/fu+rse9si9mFtiEOtdVkfEfvUXZH/436/7irggoiY2qDc0TQ99ZGKKAO50xYHRcT6RtpS9Lg7SSPJBeRBEfGBpEeBDlsoHvn9rmn4MzCry+fUsmcqcJ6k9gCSdpW0HTAdOC1/zq0vcFgjn30KOFTSoPxnt8+vXwd0rlPuEXKHguTL7ZN/Ox04I79uNNC9ibZ2Bd7NB9pnyfUUN6kANvU2v0rusPY94FVJJ+f3IUl7N7EPa2McatnzO3Lny2bnHx7yG3I98nuBl4G5wK+Bxxp+MCJWkjsPNknS83x0+PcAcPymCwXAd4Fh+QsRC/joKuyVwCGSZpM7DH6jibY+DLST9AJwFfB0nW3vA7tLepbcObMf59efAXwz3775eIp0a8CzdJhZprinZmaZ4lAzs0xxqJlZpjjUzCxTHGpmlikONTPLFIeamWXK/wG90Fr733LGzgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.6915911379657603\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8352    0.0591    0.1104      1286\n",
      "           1     0.6882    0.9944    0.8135      2686\n",
      "\n",
      "    accuracy                         0.6916      3972\n",
      "   macro avg     0.7617    0.5268    0.4619      3972\n",
      "weighted avg     0.7358    0.6916    0.5858      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAblklEQVR4nO3deZhU1Z3/8fenGxRUNm022cQNN6LBDdG4RgPRjEucuIUYk4yaiJmJjoZkEk30N2PikqhxIWgcd/m5oJKIW9wQlwREUYGgCCKr0CqKgmJ3f+ePKrC703TXha6u27c/r+ep56l776lzT9H0p89dzrmKCMzMsqKs1A0wM2tODjUzyxSHmpllikPNzDLFoWZmmdKu1A2obauKiujXf0Cpm2EJvPrmklI3wRKIT1cQn6/SxtRR3nlARNXqwva3evmjETF8Y/aXVKpCrV//ATzx7N9K3QxLoO+Ii0vdBEvgs5dv2Og6omo1mw76VkFlP33l2oqN3mFCqQo1M2sNBErvmSuHmpklI6CsvNStWC+Hmpklp406LVdUDjUzS8iHn2aWNe6pmVlmCPfUzCxL5J6amWWMr36aWXb4QoGZZYnw4aeZZYx7amaWHT78NLMsEVDuCwVmliU+p2Zm2eHDTzPLGvfUzCxT3FMzs8yQh0mZWdZ4mJSZZYcvFJhZ1vjw08wyw/OpmVm2+PDTzLLGFwrMLFN8Ts3MMkM+/DSzrHFPzcyyRA41M8uK3GzeDjUzywoJlTnUzCxD3FMzs0xxqJlZpjjUzCw7lH+llEPNzBIRck/NzLKlrMwjCswsQ9xTM7PsSPk5tfT2Ic0stSQV9CqgnuGSZkuaI2l0A9u7SPqzpOmSZkg6rak6HWpmlsjaCwUbG2qSyoFrgRHALsBJknapV+wsYGZE7A4cDFwhaZPG6nWomVliKlNBrybsA8yJiLkRsQYYBxxdr0wAnZRLyC2A94Gqxir1OTUzS0aJLhRUSJpaa3lsRIzNv+8DLKi1bSGwb73PXwNMABYDnYATIqKmsR061MwssQShVhkRe62vmgbWRb3lrwGvAIcC2wGPS3o2Ij5a3w59+GlmiTXThYKFQL9ay33J9chqOw0YHzlzgHnATo1V6lAzs0Sa60IBMAXYQdLA/Mn/E8kdatb2DnAYgKSewCBgbmOV+vDTzJJrhvvUIqJK0ijgUaAcuCkiZkg6M799DHAxcLOk1/J7/WlEVDZWr0PNzJJR8w2TioiJwMR668bUer8YOCJJnQ41M0vMw6TMLFvSm2kOtaSeenEWF141nuqa4KSjhjJq5FfrbI8ILrhqPE++MIuOHdrz+5+fzOBBuQs8Q4//NZtv1oHyMtGuvJyJfzoXgJlvLmL05Xfzyeo19Ou1JX+4cCSdNu/Q4t+tLThs7+25ZNQIysvEbROnceVdk+ts77JFB645/xgG9u7Gp59XcfalDzLr7WXrtpeViaeuP4MllR9x4n/d2dLNT40099SKevWzqXFdrU11dQ2/+N293Hb5GTx1+2ge/Os03pi3tE6ZJ1+cxbwFy5k87r/47Xkn8LPL76mz/Z6rz+Kxm89fF2gA5/12HD878xs8cetPGX7gYMbc+WSLfJ+2pqxMXPbvR/Kvo29n6GnX8s1DBzNoQPc6Zc495UBem7OUA/7ten54yf1cMmpEne1nHjeUN95Z3pLNTp1Cr3yWKviKFmoFjutqVV6ZNZ9t+lYwoE8Fm7Rvx9Ff/TKPTX6tTpnHnn2N44fvjST23G0bPvp4Ne9WfthovW+9s4yhe2wHwIF7D2LiM9OL9h3asj136sPcRe8zf8kHfF5VzfgnX+frw+re8jRoQHcmTcvdMfDmgkr69+pK926bA7B1RWeOGLojt06c1uJtT5s2GWoUNq6rVVmy/EN69+i2brlX964sWV43sJZWfsjWtcr07tGVpflQk8TJ54xhxPcu5/YHn19XZtC2vXls8usA/OWpV1j87ooifou2q3dFZxYt++LntbjyQ3p371SnzOtvLeWor+wMwJCd+tCvZxe2rugMwP+cNZwL//gYNTX1b3pve5pp7GdRFDPUGhrX1ad+IUmnS5oqaep7lY3eflJ6Dfxfrv/XKBoqkz+rev/1/84jN/0nt11xBreMn8yLr7wFwBU/O4lbxk9mxPcu5+NVn9G+fXmzN92goY5D/Z/XlXdNpmunjkwaeyanH7svr765lOrqGr42dEcqV3zC9DeXtExjUy7NPbViXigoZFwX+cGtYwH2GLJnqv8E9u7RhSXLPli3vHT5Cnrl/4qvK9O9C4trlVmybAU982V6VXQBoKJbJ4YfOJhXZs5n6B7bsf2Antz5+x8CMPedZTzxwsxif5U2afHyj+jTo8u65a0rurC0cmWdMitXfcaoSx9Ytzz9zv9g/tIVHHfoYIYPG8Th++7Appu0o9Nmm/LHnx3HGZeMb6nmp0eyAe0trpg9tULGdbUqu+/Un3kLKnln8Xus+byKB//6Mofvv1udMkccsBv3PjKFiOCl19+m0xYd6VnRhVWrP+PjVZ8CsGr1Z0yaMptB2/YGoPKD3C9WTU0NV93yGCOPHtayX6yNmPaPxWzXZ0v69+pK+3blHHfobjz8wj/qlOm8eQfat8v1lL9z5J48/+p8Vq76jItu/Cu7nfA7dj/5Sr5/8b08+/K8thlo5Ce+VWGvUihmT23duC5gEblxXScXcX9F165dORef801OOWcMNTU1nHDkvgzatje3PfAcACOP2Z9D99uFJ1+YxQEn/D86dNiE3/38JACWv7+SH/z8JiB3FfWYw4dwyNDcuZsHHp/GLeNztxaMOOhLnHBk/dlXrDlU19Rw/h8mct9vR1JeXsYdD7/MP95ezmnfyE0i8b9/nsqgARVcP/o4qmtqmD1/OWdf9mCJW51G6X6alKKhk0DNVbn0deBKvhjX9d+Nld9jyJ7xxLN/K1p7rPn1HXFxqZtgCXz28g3UrFy8UYnUodeOMeDUPxRU9o1Lh7/UyNRDRVHUm28bGtdlZq1cCQ8tC+ERBWaWiMjdyJxWDjUzS8w9NTPLlDRfKHComVkyPqdmZlki1GyTRBaDQ83MEnNPzcwyxefUzCw7fE7NzLIkN/YzvanmUDOzxFKcaQ41M0vOIwrMLDtSPp+aQ83MElk7n1paOdTMLKF0z6fmUDOzxFKcaQ41M0tIvlBgZhni+9TMLHMcamaWKSnONIeamSXnnpqZZYcHtJtZluQmiUxvqjnUzCyxshR31dI7J6+ZpZZU2KvpejRc0mxJcySNXk+ZgyW9ImmGpGeaqtM9NTNLRM00oF1SOXAtcDiwEJgiaUJEzKxVpitwHTA8It6R1KOpet1TM7PEylTYqwn7AHMiYm5ErAHGAUfXK3MyMD4i3gGIiGVNVbrenpqkPwCxvu0R8eMmm2xmmZTgQkGFpKm1lsdGxNj8+z7AglrbFgL71vv8jkB7SU8DnYCrIuLWxnbY2OHn1Ea2mVkbJXJXQAtUGRF7NVJVffU7Uu2APYHDgI7AC5JejIg31rfD9YZaRNxSZ+/S5hHxyfrKm1nb0Ux3dCwE+tVa7gssbqBMZT57PpE0CdgdWG+oNXlOTdJ+kmYCs/LLu0u6LmHjzSwrlJtPrZBXE6YAO0gaKGkT4ERgQr0yDwJfkdRO0mbkDk9nNVZpIVc/rwS+tnZnETFd0oEFfM7MMqo5blOLiCpJo4BHgXLgpoiYIenM/PYxETFL0iPAq0ANcGNEvN5YvQXd0hERC+qlbvWGfAkza/1E8918GxETgYn11o2pt3wZcFmhdRYSagskDQMi30X8MU10/8ws29I8TKqQ+9TOBM4id/l1EbBHftnM2qBCRxOUaiRVkz21iKgETmmBtphZK9Gqx35K2lbSnyUtl7RM0oOStm2JxplZOqnAVykUcvh5J3A30BvYGrgHuKuYjTKzdGumWzqKopBQU0TcFhFV+dftNDJ8ysyyLXf1s1nGfhZFY2M/t8y/fSo/Jcg4cmF2AvBQC7TNzNJIrXeSyJfIhdja1p9Ra1sAFxerUWaWbq3yGQURMbAlG2JmrcPaw8+0KmhEgaTdgF2ADmvXNTX9h5llV6vsqa0l6ULgYHKhNhEYAUwGHGpmbVR6I62wq5/Hk5vLaGlEnEZu2o9Ni9oqM0stCcrLVNCrFAo5/FwdETWSqiR1BpYBvvnWrA1r1YefwNT8ww9uIHdF9GPg78VslJmlW4ozraCxnz/Kvx2Tn9eoc0S8WtxmmVlaCaV67GdjN98OaWxbREwrTpPMLNVKOANHIRrrqV3RyLYADm3mtlAusfmmfhRpq7LyvVK3wJKormqWalrlObWIOKQlG2JmrYPIdUDSyt0iM0us1Y8oMDOrzaFmZpmRm6o7valWyMy3kvRtSRfkl/tL2qf4TTOztErzfGqFDJO6DtgPOCm/vBK4tmgtMrPUa9UPXgH2jYghkl4GiIgP8o/KM7M2SEC7FB9+FhJqn0sqJz+Ft6Tu5J6UbGZtVIozraBQuxq4H+gh6b/Jzdrxi6K2ysxSS2qlw6TWiog7JL1EbvohAcdEhJ/QbtaGpTjTCpoksj+wCvhz7XUR8U4xG2Zm6dXa71N7iC8ewNIBGAjMBnYtYrvMLKUEJZsAshCFHH4Orr2cn73jjPUUN7OsK+E9aIVIPKIgIqZJ2rsYjTGz1kEpfkpBIefUzqm1WAYMAZYXrUVmlmpZeERep1rvq8idY7uvOM0xs9ag1YZa/qbbLSLivBZqj5m1Amke0N7YdN7tIqKqsWm9zaztyT0ir9StWL/Gmrb2iVGvSJogaaSk49a+WqJxZpZOZflRBU29miJpuKTZkuZIGt1Iub0lVUs6vqk6CzmntiXwHrlnEqy9Xy2A8QV81swyprkuFORPb10LHA4sBKZImhARMxso91vg0ULqbSzUeuSvfL7OF2G2ViRou5llTDOdUtsHmBMRc3N1ahxwNDCzXrmzyV2cLOhWssZCrRzYAhq8IcWhZtZmibLC71OrkDS11vLYiBibf98HWFBr20Jg3zp7kvoAx5I7UtzoUFsSERcVUomZtR0iUU+tMiL2aqSq+up3mK4EfhoR1YVecW0s1NJ7zdbMSkfQrnluVFsI9Ku13BdYXK/MXsC4fKBVAF+XVBURD6yv0sZC7bANa6eZZVnCnlpjpgA7SBoILAJOBE6uXSAiBq7br3Qz8JfGAg0af5jx+xvRWDPLsOaYJDJ/H+woclc1y4GbImKGpDPz28dsSL1+RJ6ZJdZcAwoiYiIwsd66BsMsIr5bSJ0ONTNLRBT2GLpScaiZWTJqnsPPYnGomVkiuREFDjUzy5D0RppDzcw2QIo7ag41M0tKrXM+NTOzhvjqp5llji8UmFl2qJVO521m1hAffppZ5rinZmaZkt5Ic6iZWUICyt1TM7MsSXGmOdTMLCmhFB+AOtTMLDH31MwsM3K3dKQ31RxqZpaM3FMzs4zxMCkzy4zcJJGlbsX6OdTMLDFf/TSzTEnx0adDrRB/fX4mP7viXqprahh59DB+8t0j6myPCEZfcS+PPzeDjh024boLR7L7Tv0a/exvxj7ErQ88z1ZdtwDgl2f9C0fsvysAr7+5iHMuuYuVH3+KysSTt5xPh03bt+A3zq7D9tuZS849nvKyMm578HmuvOXxOtu7dOrINb/8NgP7VvDpms85++I7mPXWEgDOOPFgTj1mGEjc+sBzjLnr6RJ8g3Rokz01STcBRwHLImK3Yu2n2Kqrazjv0ru5/5pRbN2zK4eeehkjDhzMTtv2Xlfm8edn8tY7y3lp/IVMff1tzv3NOP5683lNfvaHJx3C2SO/Wmd/VVXVnHHBLYz59XcYvGNf3l/xMe3blbfod86qsjJx2fnf4thR17D43RU8ect5PDzpNWbPW7quzLmnfY3X3ljIyPNvYIcBPbnsp9/imB/9gZ23682pxwzjsFMvY01VNfde/SMemzyDuQuWl/AblUbaz6kVcwaRm4HhRay/Rbw042227VfBNn0r2KR9O447fAgTn3m1TpmJz7zKiUfugyT2HjyQD1euZmnlhwV9tr4n//YPdt2+D4N37AvAll23oLw8zRO9tB577roNcxdUMn/Re3xeVc34x6fx9YO+VKfMoIG9mDRlNgBvzn+X/r23pPuWndhxm15Mee1tVn/2OdXVNTw3bQ5HHbx7Kb5G6UmUFfgqhaL9tkTEJOD9YtXfUpYs/5A+PbutW966ZzeWLP+wXpkVdcv06MqSZSua/OwN90xi/5P+h1EX3c6Kj1YB8Nb8ZUjwzbOv4aBv/4arbq17eGQbrnf3Lix694N1y4vf/YDe3bvUKfP6m4s46pA9ABiyywD69dqSrXt0ZdZbixn25e3p1mVzOm7ansOH7VrnZ9vWqMBXKZS8CyDpdElTJU1dXpm+rnxE/NO6+n+AGiiCpEY/+71vfoWX7/8Vz94xmp4VnfnFleMBqKqu5sXpcxl78Xd5+MZzeOjp6Tzz99kb/T2s4TnA6v+Irrzlcbp23oxJd4zm9BMO4tU3FlJdXcMbb7/LVbc+zv3XjOLeq89ixpuLqKqubqGWp8va536mtadW8gsFETEWGAuw5557NRAPpbV1j67/9Ne9V0WXxsssW0Gv7l1Y83nVej/bY6vO69afesz+nPCTMbm6enZl/y9vv+4CwuHDdmX67AUctM+g5v9ybcziZSv+qee8tLJur3vlJ58y6qLb1y1Pf/DXzF/8HgC3T3iB2ye8AMAvf/QNFi9bUfxGp1SKT6mVvqeWdkN2GcBb7yxn/qJK1nxexfjHpzHiwLrnYUYcOJhxD/2diGDKa/PovEVHelV0afSztX+Z/vL0dHbeLnfx4LChuzBjziJWfbqGqqpqnps2h0EDe7XcF86waTPns13/7vTfeivatyvnuMOH8PCkuuc4O2/Rcd2Fme8cM4znX57Dyk8+BaCiW+4PTd+e3TjqkN2599GpLfsF0iTFx58l76mlXbt25Vx6/rf45o+vpbo6OOVfhrLzdr256b5ngdxh5BH778rjz81gyLG/pmOH9lx7wbcb/SzAhVc/wGtvLEQS/Xtvye9/fhIAXTtvxo9OPpTDvnMpSBy+/6587YBWe/E4Vaqrazj/0ru57+qzKC8Xd0x4kX/MXcppxx0AwP+On8yggb24/lcjqa6pYfa8pZx98R3rPn/rb39Aty6bU1VVzXmX3s2HK1eX6quUXJqHSamh8z7NUrF0F3AwUAG8C1wYEX9q7DN77rlXPPe3NvzXrxXqtveoUjfBEvhs9t3UrFq2UYm08+Avx60PPl1Q2X226/pSROy1MftLqmg9tYg4qVh1m1mJpbej5sNPM0smd7osvanmUDOzZDyfmpllTYozzbd0mFlSQirs1WRN0nBJsyXNkTS6ge2nSHo1/3peUpNj09xTM7PEmuPwU1I5cC1wOLAQmCJpQkTMrFVsHnBQRHwgaQS5G/X3baxe99TMLJFC77stIPf2AeZExNyIWAOMA46uXSAino+ItcNyXgT6NlWpQ83Mkis81SrWju3Ov06vVUsfYEGt5YX5devzfeDhpprmw08zSyzBLR2Vjdx821AlDY4GkHQIuVA7oKkdOtTMLLFmuqVjIdCv1nJfYPE/70tfAm4ERkTEe01V6sNPM0smf59aIa8mTAF2kDRQ0ibAicCEOruS+gPjgZER8UYhzXNPzcwSa44RBRFRJWkU8ChQDtwUETMknZnfPga4ANgKuC5/i0hVU2NJHWpmlohovhEFETERmFhv3Zha738A/CBJnQ41M0sszSMKHGpmllyKU82hZmaJpXmSSIeamSWW3khzqJnZhkhxqjnUzCwRTxJpZtniSSLNLGtSnGkONTNLqrAJIEvFoWZmiaU40xxqZpZMCR++XhCHmpkll+JUc6iZWWK+pcPMMsXn1MwsOwRlDjUzy5b0pppDzcwSac5JIovBoWZmiaU40xxqZpace2pmlikeJmVmmZLeSHOomVlCBT7Ts2QcamaWmEcUmFm2pDfTHGpmllyKM82hZmZJyY/IM7PsSPuIgrJSN8DMrDm5p2ZmiaW5p+ZQM7PEfEuHmWWHb741syxJ+4UCh5qZJebDTzPLFPfUzCxTUpxpDjUz2wApTjWHmpklIkj1MClFRKnbsI6k5cD8UrejCCqAylI3whLJ6s9sQER035gKJD1C7t+nEJURMXxj9pdUqkItqyRNjYi9St0OK5x/Zq2Xx36aWaY41MwsUxxqLWNsqRtgifln1kr5nJqZZYp7amaWKQ41M8sUh1oRSRouabakOZJGl7o91jRJN0laJun1UrfFNoxDrUgklQPXAiOAXYCTJO1S2lZZAW4GWvRmUWteDrXi2QeYExFzI2INMA44usRtsiZExCTg/VK3wzacQ614+gALai0vzK8zsyJyqBVPQyN+ff+MWZE51IpnIdCv1nJfYHGJ2mLWZjjUimcKsIOkgZI2AU4EJpS4TWaZ51ArkoioAkYBjwKzgLsjYkZpW2VNkXQX8AIwSNJCSd8vdZssGQ+TMrNMcU/NzDLFoWZmmeJQM7NMcaiZWaY41MwsUxxqrYikakmvSHpd0j2SNtuIum6WdHz+/Y2NDbaXdLCkYRuwj7cl/dNTh9a3vl6ZjxPu61eS/jNpGy17HGqty+qI2CMidgPWAGfW3pifGSSxiPhBRMxspMjBQOJQMysFh1rr9Sywfb4X9ZSkO4HXJJVLukzSFEmvSjoDQDnXSJop6SGgx9qKJD0taa/8++GSpkmaLukJSduQC8+f5HuJX5HUXdJ9+X1MkbR//rNbSXpM0suS/kgBz/GW9ICklyTNkHR6vW1X5NvyhKTu+XXbSXok/5lnJe3ULP+alhl+QnsrJKkduXnaHsmv2gfYLSLm5YPhw4jYW9KmwHOSHgO+DAwCBgM9gZnATfXq7Q7cAByYr2vLiHhf0hjg44i4PF/uTuD3ETFZUn9yoyZ2Bi4EJkfERZKOBOqE1Hp8L7+PjsAUSfdFxHvA5sC0iDhX0gX5ukeReyDKmRHxpqR9geuAQzfgn9EyyqHWunSU9Er+/bPAn8gdFv49Iubl1x8BfGnt+TKgC7ADcCBwV0RUA4slPdlA/UOBSWvrioj1zSv2VWAXaV1HrLOkTvl9HJf/7EOSPijgO/1Y0rH59/3ybX0PqAH+f3797cB4SVvkv+89tfa9aQH7sDbEoda6rI6IPWqvyP9yf1J7FXB2RDxar9zXaXrqIxVQBnKnLfaLiNUNtKXgcXeSDiYXkPtFxCpJTwMd1lM88vtdUf/fwKw2n1PLnkeBH0pqDyBpR0mbA5OAE/Pn3HoDhzTw2ReAgyQNzH92y/z6lUCnWuUeI3coSL7cHvm3k4BT8utGAN2aaGsX4IN8oO1Erqe4Vhmwtrd5MrnD2o+AeZL+Nb8PSdq9iX1YG+NQy54byZ0vm5Z/eMgfyfXI7wfeBF4Drgeeqf/BiFhO7jzYeEnT+eLw78/AsWsvFAA/BvbKX4iYyRdXYX8NHChpGrnD4HeaaOsjQDtJrwIXAy/W2vYJsKukl8idM7sov/4U4Pv59s3AU6RbPZ6lw8wyxT01M8sUh5qZZYpDzcwyxaFmZpniUDOzTHGomVmmONTMLFP+D3Miak/25NmaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(knn_char, model_name=\"knn_char\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "id": "YUn_xfDEBaSb"
   },
   "outputs": [],
   "source": [
    "del knn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D5G3DhkMNqrn"
   },
   "source": [
    "# **5. Other Features**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "id": "jEMyD2APTq0o"
   },
   "outputs": [],
   "source": [
    "# #Add Word Tokenizer without Stemming\n",
    "def basic_tokenize(tweet):\n",
    "     \"\"\"Removes punctuation & excess whitespace, sets to lowercase.\n",
    "        Returns a list of tokens.\"\"\"\n",
    "     list_tokens = []\n",
    "     for t in tweet:\n",
    "       sentence = re.sub(r'[^\\w\\s]','',t.lower())\n",
    "       list_tokens.append(nltk.word_tokenize(sentence))\n",
    "     tokens = [item for sublist in list_tokens for item in sublist]\n",
    "     return tokens\n",
    "\n",
    "def other_features(tweet):\n",
    "    \"\"\"This function takes a string and returns a list of features.\n",
    "    These include Sentiment scores, Text and Readability scores,\n",
    "    as well as Twitter specific features\"\"\"\n",
    "    words = basic_tokenize(tweet) #Get text only\n",
    "    syllables = []\n",
    "    for word in words:\n",
    "      syllables.append(textstat.syllable_count(word))\n",
    "\n",
    "    num_chars = sum(len(w) for w in words)\n",
    "    num_chars_total = len(tweet)\n",
    "    num_words = len(words)\n",
    "    num_unique_terms = len(set(words))\n",
    "\n",
    "    avg_syl = []\n",
    "    for s in syllables:\n",
    "      avg_syl.append(int(s+0.001)/int(num_words+0.001))\n",
    "    avg = sum(avg_syl) / len(avg_syl)\n",
    "    ###Modified FK grade, where avg words per sentence is just num words/1\n",
    "    FKRA = round(float(0.39 * float(num_words)/1.0) + float(11.8 * avg) - 15.59,1)\n",
    "    ##Modified FRE score, where sentence fixed to 1\n",
    "    FRE = round(206.835 - 1.015*(float(num_words)/1.0) - (84.6*float(avg)),2)\n",
    "\n",
    "    twitter_objs = []\n",
    "    retweet = 0\n",
    "    if \"rt\" in words:\n",
    "      retweet = 1\n",
    "    features = [FKRA, FRE, avg, num_chars, num_chars_total, num_words,\n",
    "                num_unique_terms,  retweet]\n",
    "    return features\n",
    "\n",
    "\n",
    "def get_feature_array(tweets):\n",
    "  feats = []\n",
    "  for t in tweets:\n",
    "    feats.append(other_features(t))\n",
    "  return np.array(feats)\n",
    "\n",
    "\n",
    "  \n",
    "other_features_names = [\"FKRA\", \"FRE\",\"avg_syl_per_word\", \"num_chars\", \"num_chars_total\", \\\n",
    "                        \"num_words\", \"num_unique_words\", \"is_retweet\"]\n",
    "\n",
    "                       \n",
    "\n",
    "feats = get_feature_array(tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_M05gIqmUCFp"
   },
   "source": [
    "### **5.1 Adding New Features to Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "id": "CHVrujFCUJRC"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13240, 19472)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M_new_words = np.concatenate([tfidf_words,pos,feats],axis=1)\n",
    "M_new_words.shape\n",
    "\n",
    "M_new_chars = np.concatenate([tfidf_chars,pos,feats],axis=1)\n",
    "M_new_chars.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "wJbEphrJV_Sl"
   },
   "outputs": [],
   "source": [
    "X_new_words = pd.DataFrame(M_new_words)\n",
    "X_train_word, X_test_word, y_train_word, y_test_word = train_test_split(X_new_words, y, test_size = 0.3, random_state = 42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new_chars = pd.DataFrame(M_new_chars)\n",
    "X_train_char, X_test_char, y_train_char, y_test_char = train_test_split(X_new_chars, y, test_size = 0.3, random_state = 42, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_M05gIqmUCFp"
   },
   "source": [
    "### **5.2 Testing MAchine Learning Models on Data with New Features**\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_M05gIqmUCFp"
   },
   "source": [
    "#### **5.2.1 Linear Regression Model**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_lr = LogisticRegression(penalty='l2',\n",
    "                            tol=1e-4,\n",
    "                            C=5.0,\n",
    "                            fit_intercept=True,\n",
    "                            class_weight='balanced',\n",
    "                            random_state=0,\n",
    "                            solver='lbfgs',\n",
    "                            max_iter=100,\n",
    "                            multi_class='auto',\n",
    "                            verbose=1,\n",
    "                            n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = [{}]\n",
    "grid_search_lr = GridSearchCV(c_lr, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed: 59.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time=59.8min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.6min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.3min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.3min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:  8.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 8.2min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed: 11.4min finished\n"
     ]
    }
   ],
   "source": [
    "c_lr_char = grid_search_lr.fit(X_train_char, y_train_char)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.9001942166594734\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8184    0.8976    0.8562      3067\n",
      "           1     0.9468    0.9015    0.9236      6201\n",
      "\n",
      "    accuracy                         0.9002      9268\n",
      "   macro avg     0.8826    0.8995    0.8899      9268\n",
      "weighted avg     0.9043    0.9002    0.9013      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEKCAYAAABzM8J8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdKElEQVR4nO3de5xVdb3/8dd7ZlAxQSRQkIuikYqZhoi38lJHA0/9yEuKmv6yehgW2elnmnX6aWk9Mq3HsRLjcDykmcnRIMWk0PxlqGhyCS+Q6IghFxXGW15Ths/vj70G9owze68le7PX7Hk/fazHY6+1vvNd35mR93zX5bu+igjMzOpFQ60bYGZWSQ41M6srDjUzqysONTOrKw41M6srDjUzqysONTOrGUnjJC2X1Czpwk727yTpt5IelvSgpA+Uq9OhZmY1IakRmAKMB0YBp0oa1aHYt4AlEfFB4EzgJ+XqdaiZWa2MBZojYkVEvAXMACZ0KDMKuAsgIh4Ddpe0S6lKm6rR0ndLTb1D2/SpdTMsgwP2GV7rJlgGT6/8Oy0tLdqSOhr77hax4Y1UZeON9XMjYlwXu4cAq4rWVwMHdyjzEHACcK+kscBuwFDgua6Oma9Q26YP2+59Sq2bYRnc98BPa90Ey+DwQw7a4jpiwxtsu9fJqcq+uWTK3pIWFm2aFhHTks+dhWvHcZuXAT+RtAR4BPgrsKHUMXMVambWHQiU+spVS0SM6WLfamBY0fpQYG1xgYj4B3AWgCQBTyVLlxxqZpaNgIbGStS0ABgpaQSwBpgInNbuUFI/4PXkmtsXgHlJ0HXJoWZm2WmLLssBEBEbJE0G5gKNwPSIWCppUrJ/KrAP8EtJrcAy4PPl6nWomVlGmU4/S4qIOcCcDtumFn2+HxiZpU6HmpllV4GeWrU41MwsG1Gxnlo1ONTMLCO5p2ZmdaYydz+rwqFmZhlV7kZBNTjUzCwb4dNPM6sz7qmZWf3w6aeZ1RMBjb5RYGb1xNfUzKx++PTTzOqNe2pmVlfcUzOzuiEPkzKzeuNhUmZWP3yjwMzqjU8/zaxu5Px9avltmZnlVHL6mWYpV5M0TtJySc2SLuxk/46SbpP0kKSlks4qV6d7amaWXQVuFEhqBKYAx1CYLm+BpNkRsayo2JeBZRHxSUkDgeWSbkhml+q8aVvcMjPredoe6yi3lDYWaI6IFUlIzQAmdCgTQJ9kzs8dgBfwZMZmVlGq2N3PIcCqovXVwMEdylwFzKYwyXEf4JSI2FiqUvfUzCy79D21AZIWFi1nF9fSSc3RYf3jwBJgV+AA4CpJfUs1zT01M8tM6R/paImIMV3sWw0MK1ofSqFHVuws4LKICKBZ0lPA3sCDXR3QPTUzy6TwNm+lWspYAIyUNELSNsBECqeaxZ4GPkbhmLsAewErSlXqnpqZZSOhhi1/+DYiNkiaDMwFGoHpEbFU0qRk/1TgUuBaSY9QyNNvRERLqXodamaWWYbTz5IiYg4wp8O2qUWf1wLHZqnToWZmmVUq1KrBoWZmmTnUzKx+iM4fxsgJh5qZZSJS3dmsGYeamWXW0JDfp8EcamaWmXtqZlY/fE3NzOqNe2pmVjd8o8DM6k4lhklVi0PNzLKRTz/NrM441MysrjjUzKxu+EaBmdWf/GaaQ83MMpKHSZlZncnz6Wd+47Yb+tgh+/Dgzd9m0cyL+Lczj3nH/h379Ob6y7/AvTdcyB9/8XX22WNwDVppbf54/zLGnnQpB57wXa687o537H/8789y7Od+zKDDv8bPfnVXDVqYY0q51EBVQ63clPL1pKFBXHHBp/n0V3/OIad8nxM/fiB7jRjUrsx5nz2WRx5fw4dPv4xzvnM9PzjvxBq11lpbN3LB5Tdz00/O4f7/+Xdmzl3EYyueaVdmp77v4bKvn8Tk0z9ao1bmV4UmXimbEZLOl7QkWR6V1Cqpf6k6qxZqRVPKjwdGAadKGlWt49XagfvuxorVLaxc+zxvb2hl1h2LOO6I/dqV2WvEYOYtWA7AEyufY/jg/gzs36cWze3xFi1dyYihA9h9yAC26dXECcceyO/nPdKuzMD+fRg9ajeamhpr1Mp8Shto5UItTUZExBURcUBEHAB8E/hzRLxQqt5q9tTSTClfNwYP7Mea517ctL523UsMHtivXZlHn1jDJ47eH4DRo3Zj2KD+7Lpz+zK2dTyz/iWG7LLTpvVdd+7HM+tfql2DupkK9dSyZsSpwI3lKq1mqHU2pfyQKh6vpjr7/UWHyaav/OWd9OuzPfN+9Q3OPvkIHn58Na2tG7dSC61YdJwHnMLzV5aOGpRqKSN1RkjaHhgHzCxXaTXvfqaZUp5kGvrCVPS9dqhic6pr7bp3/uV/dv3L7cq88tqbTL70hk3rD93yHVaufX6rtdE223Xnd/asBw3csYYt6l4y3P0cIGlh0fq0iJjWVk0n5Tv5cwPAJ4H7yp16QnV7ammmlCcipkXEmIgYo6beVWxOdS1e9jR7DhvI8F3fS6+mxsI1mnvaX6Ppu0NveiXXZ86ccBjzlzzJK6+9WYvm9nijRw1nxar1rFzTwltvb2DWHYsY95H9yn+hbRrQnvL0s6Xt33eyTCuqKVVGJCaS4tQTqttT2zSlPLAmadRpVTxeTbW2buSCK25m5k+/RGODuOG2B3hsxbOcdcLhAPxi1n3sNWIXfn7xGbRuDJY/9Sxf+d4NZWq1amlqauTy8z/NSedeTevG4PRPHsI+ew7mFzPvBeCsEz/Mcy3/4KOfvYJXXnuTBompM+7m/hnfou8O3fePbyWIzi+3vAupMkLSjsCRwGdStS86u7hQIZKOA65k85Ty3y9VvmH7nWPbvU+pWnus8l74y09r3QTL4PBDDmLxooVbFEnbDXp/DDsj3e+9+UfjF0XEmK72d5YRkibB5pnaJX0WGBcRE9Mcs6ojCjqbUt7Mur+GCr0ksrOMaAuzovVrgWvT1ulhUmaWjSp2+lkVDjUzy0RUrqdWDQ41M8vMPTUzqyt5fkuHQ83MsvE1NTOrJ0J+SaSZ1Rf31MysrviampnVD19TM7N6Uhj7md9Uc6iZWWY5zjSHmpll5xEFZlY/5NNPM6sjFXyfWlU41Mwso3TT39WKQ83MMstxpjnUzCwj+UaBmdWRvD+nlt9RqWaWWxWazBhJ4yQtl9Qs6cIuyhwlaYmkpZL+XK5O99TMLLNKdNQkNQJTgGMoTJe3QNLsiFhWVKYfcDWFiVeelrRzuXrdUzOzzCrUUxsLNEfEioh4C5gBTOhQ5jRgVkQ8DRAR68pV6lAzs2ySAe1pljKGAKuK1lcn24q9H9hJ0t2SFkk6s1ylPv00s0wKL4lMff45QNLCovVpRbO0d1ZJx4mIm4ADgY8BvYH7JT0QEY93dUCHmpll1pD+olpLicmMVwPDitaHAms7KdMSEa8Br0maB+wPdBlqPv00s8wqdPq5ABgpaYSkbYCJwOwOZW4FPiKpSdL2wMHA30pV6p6amWWiCg1oj4gNkiYDc4FGYHpELJU0Kdk/NSL+JukPwMPARuCaiHi0VL0ONTPLrFIDCiJiDjCnw7apHdavAK5IW2eXoSbpZ7zzol3xgc5NexAzqy/ddZjUwhL7zKyHEoU7oHnVZahFxHXF65Lek9yBMLMeLscdtfJ3PyUdKmkZyR0HSftLurrqLTOzfEo5mqBWg97TPNJxJfBx4HmAiHgIOKKKbTKznKvQIx1VkeruZ0Ss6pC6rdVpjpnlncj08O1WlybUVkk6DIjkAblzKfPwm5nVtzzf/Uxz+jkJ+DKFgaZrgAOSdTPrgdKeeub29DMiWoDTt0JbzKybyPPpZ5q7n3tIuk3SeknrJN0qaY+t0TgzyyelXGohzennr4GbgMHArsDNwI3VbJSZ5Vt3f6RDEXF9RGxIll9RYviUmdW3wt3PdEstlBr72T/5+KdkQoQZFMLsFOD2rdA2M8sjZXpJ5FZX6kbBIgoh1tb6LxbtC+DSajXKzPItz1PklRr7OWJrNsTMuoe208+8SjWiQNIHgFHAdm3bIuKX1WqUmeVbt+yptZF0MXAUhVCbA4wH7gUcamY9VH4jLd3dz5MozOTybEScRWHSg22r2iozyy0JGhuUaqmFNKefb0TERkkbJPUF1gF++NasB8vz6WeantrCZOr3/6JwR3Qx8GA1G2Vm+VapsZ+SxklaLqk5eXSs4/6jJL0saUmyXFSuzjRjP7+UfJyazOrSNyIeLt9cM6tHQhUZ+ympEZgCHENhfs8FkmZHxLIORe+JiE+krbfUw7ejS+2LiMVpD2JmdaRyb+AYCzRHxAoASTOACUDHUMukVE/txyX2BfDRLTlwZz60z3Du+8vPKl2tVdFOB02udRMsg38uf7oi9WS4pjZAUvEkTtMiYlryeQiwqmjfagqTFXd0qKSHKMze/vWIWFrqgKUevj06XZvNrCcR0Jg+1FoiYkyJqjrqOK58MbBbRLwq6TjgFmBkqQOmuVFgZtZOhQa0rwaGFa0PpdAb2yQi/hERryaf5wC9JA0o2bbM342Z9XgVCrUFwEhJI5KpAiYCs4sLSBqk5FxX0lgKmfV8qUpTDZMyM2tTeFxjy+8URMQGSZOBuUAjMD0ilkqalOyfSuHh/3MkbQDeACZGRMlXn6UZJiUKr/PeIyIukTQcGBQRflbNrIeq1GCB5JRyTodtU4s+XwVclaltKcpcDRwKnJqsv0Lh2RIz66G69cQrwMERMVrSXwEi4sXk/NfMeiABTTkeJpUm1N5OnvwNAEkDgY1VbZWZ5VqOMy1VqP0U+C2ws6TvU7hw9+2qtsrMckuqzDCpakkz9vMGSYsovH5IwKciwjO0m/VgOc60VHc/hwOvA7cVb4uIyoy3MLNup7u/zvt2Nk/Ash0wAlgO7FvFdplZTglq9gLINNKcfu5XvJ68veOLXRQ3s3pXwzk908g8oiAiFks6qBqNMbPuQTmepSDNNbX/U7TaAIwG1letRWaWa/UwRV6fos8bKFxjm1md5phZd9BtQy156HaHiDh/K7XHzLqBPE+8Uup13k3JKPouX+ttZj1PYYq8Wreia6V6ag9SuH62RNJs4GbgtbadETGrym0zs5zq1iMKgP4UXsr2UTY/rxaAQ82sB+rONwp2Tu58PsrmMGtT8iVtZlbfctxRKxlqjcAOpJscwcx6DNHQTZ9TeyYiLtlqLTGzbkHku6dW6h5GjpttZjUjaGpQqqVsVdI4ScslNUu6sES5gyS1SjqpXJ2lQu1jZVtkZj1OW09tS1/nnTwHOwUYD4wCTpU0qotyP6QwQUtZXYZaRLyQpgIz63kakhdFllvKGAs0R8SKiHgLmAFM6KTcVyiMYlqXqm1ZvhEzM8jUUxsgaWHRcnZRNUOAVUXrq5NtRcfREOB4YCoped5PM8tEZOoNtUTEmBJVddTxyYorgW9ERGvaoVkONTPLRhUbUbAaGFa0PhRY26HMGGBGEmgDgOMkbYiIW7qq1KFmZpkURhRUJNQWACMljQDWABOB04oLRMSITceVrgV+VyrQwKFmZu9CJSIteWHGZAp3NRuB6RGxVNKkZH/q62jFHGpmllmlHr6NiDnAnA7bOg2ziPhsmjodamaWkbrn+9TMzDqT8e7nVudQM7PMuvv71MzMNlM3fZ23mVlnfPppZnXHPTUzqyv5jTSHmpllJKDRPTUzqyc5zjSHmpllJZTjE1CHmpll5p6amdWNwiMd+U01h5qZZZNi/oFacqiZWWYeJmVmdaPwkshat6JrDjUzy8x3P82sruT47NOhltUf5y/jmz/+Da0bN3LGhMP42mePbbc/Irjwx7/hzvuW0nu7bbj64jPYf+/C3BJTb/wT190yHyI481OHc85pRwPwyOOrOe+yGbz6+j8ZPvi9TLv0f9N3h95b/XvraT526D784LyTaGxo4Ppb53PldXe2279jn95c9X8/w4ihA3jzrbf5yqU38Lcnn6lRa/Mlzz21qg22lzRd0jpJj1brGFtba+tGzr/8Jm7+yZd44KZvM/OORTy2ov3/5HfOX8aTT69n0ayLufJbp3LeZTMAWNa8lutumc9d153PPb/+JnPvfZQnny7MzfrV7/2ai788gfkz/p1PHL0/P7v+rq3+vfU0DQ3iigtO5tNfvZpDTv4eJx57IHuNGNSuzHlnfZxHHl/Nh0/7AedcfD0/OO+kGrU2X9quqaVZytYljZO0XFKzpAs72T9B0sOSliTzhn64XJ3VfIPItcC4Kta/1S1a+nf2GDaA3YcOYJteTZxwzGjm/PnhdmXm/PlhJv7rWCRx0H4jePmVN3i25WUe//uzHLTf7my/3TY0NTVy+Oj38bu7HwKg+el1HDb6fQAcNXZvbvvTkq39rfU4B+67OytWtbByzfO8vaGVWXcu5rgjP9iuzF4jBjFvwXIAnlj5HMMH92dg/z61aG6+pJydvdwdUkmNwBRgPDAKOFXSqA7F7gL2j4gDgM8B15RrXtVCLSLmAS9Uq/5aeGb9ywzZZadN67vushPPrH+5Q5mX2pfZuR/PrHuJffbclfl/beaFl17l9Tff4s75S1nz3IsA7L3HYH4/7xEAbr1r8abtVj2DB+7Y7ue89rkXGTxwx3ZlHn1iDZ84+gAARo/ajWGD+rPrzv22YivzSymXMsYCzRGxIiLeAmYAE4oLRMSrEdE2wfF7eOdkx+9Q83e9STq7bUr69S3ra92ckjb/bDfr+MeokyJIYq8Rg/jqmcdw/OSrOOncKew7cghNjY0AXHXR6Vxz8zyOOuOHvPr6P+nVq7Eazbcinb0PrOPv7srr7qRf3+2Zd8OFnH3KkTz8+GpaWzdupRbmV9u8nyl7agPa/n0ny9lFVQ0BVhWtr062tT+edLykx4DbKfTWSqr5jYKImAZMAzjwwDFlU7iWdt253zv+ug8asGPpMuteYlDSAzhjwmGcMeEwAC6ZMnvTX/337z6IWVdNBqB55XPcce/San4bRuH30rHX/WxL+173K6+9yeRLfrVp/aFbv8vKtc9vtTbmWYbbBC0RMSZDNe/IgIj4LfBbSUcAlwL/UuqANe+pdSejR+3Gk0+vZ+WaFt56ewOz7lzM+CPaX4cZf8R+zLj9QSKCBY88Rd8dem8KvvUvvALAqmdf4Hd/eoiTPj6m3faNGzfyo+lzOevEstdCbQstXraSPYcPZPiu76VXUyMnHDOa389rf3207w696dVU6DWf+anDmP/XZl557c1aNDd/KnP+uRoYVrQ+FFjbVeHkktaekgaUqrTmPbXupKmpkcsvOJkTz51Ca2tw+v86hH32HMz0mfcA8LkTP8Kxh+/LnfctZfTx36X3dr2YctFnNn39md+4hhdffo2mpkauuOBk+vXdHoCZcxdyzW/mAfCJow7g9E8esvW/uR6mtXUjF1x+EzN/+mUaG8UNsx/gsRXPctYJhT8ov5h1L3uNGMTPv3MGrRs3svypZ/nKpTfUuNX5UaFhUguAkZJGAGuAicBpxQUkvQ94MiJC0mhgG6Bkd1mdXSeqBEk3AkcBA4DngIsj4r9Lfc2BB46J+/6ysCrtserY6aDJtW6CZfDP5Tex8fV1W5RI++z3ofjlrXenKjt2z36LSpx+Iuk44EqgEZgeEd+XNAkKM7VL+gZwJvA28AZwfkTcW+qYVeupRcSp1arbzGqsQs/eRsQcYE6HbVOLPv8Q+GGWOn36aWaZFC6X5XdEgUPNzLLx+9TMrN7kONMcamaWlTyZsZnVlxxnmkPNzLJJOa6zZhxqZpZdjlPNoWZmmfmRDjOrK76mZmb1w8+pmVm98emnmdUN4Z6amdWZHGeaQ83M3oUcp5pDzcwyq9BLIqvCoWZmmeU30hxqZvZu5DjVHGpmlkneXxLp2aTMLJvk4ds0S9mqpHGSlktqlnRhJ/tPl/RwssyXtH+5Ot1TM7PMKtFPk9QITAGOoTBd3gJJsyNiWVGxp4AjI+JFSeMpzBF8cKl6HWpmllHFXhI5FmiOiBUAkmYAE4BNoRYR84vKP0BhbtCSfPppZpllOP0cIGlh0XJ2UTVDgFVF66uTbV35PPD7cm1zT83MMsn4ksiWEvN+dlZNpxMRSzqaQqh9uNwBHWpmll1lbn6uBoYVrQ8F1r7jUNIHgWuA8RFRcnZ28Omnmb0LSvlfGQuAkZJGSNoGmAjMbnccaTgwCzgjIh5P0zb31Mwss0rcJ4iIDZImA3OBRmB6RCyVNCnZPxW4CHgvcHVyc2JDidNZwKFmZlkJGir07G1EzAHmdNg2tejzF4AvZKnToWZm70J+RxQ41MwsE78k0szqTo4zzaFmZtm5p2ZmdaVCw6SqwqFmZpnlN9IcamaWUdrXCtWKQ83MMsvzSyIdamaWXX4zzaFmZtnlONMcamaWlTxFnpnVj7yPKPCrh8ysrrinZmaZ5bmn5lAzs8z8SIeZ1Q8/fGtm9STvNwocamaWWZ5PP33308wyyzDvZ5l6NE7ScknNki7sZP/eku6X9E9JX0/TNvfUzCyzSvTTJDUCU4BjKEyXt0DS7IhYVlTsBeBc4FNp63VPzcyyU8qltLFAc0SsiIi3gBnAhOICEbEuIhYAb6dtmntqZpaJIMswqQGSFhatT4uIacnnIcCqon2rgYO3tH25CrXFixe19O6llbVuRxUMAFpq3QjLpF5/Z7ttaQWLFy+a27uXBqQs3hIR47rY11kyxrts1ia5CrWIGFjrNlSDpIXlJmC1fPHvrGslQiqr1cCwovWhwNotrdTX1MysVhYAIyWNkLQNMBGYvaWV5qqnZmY9R0RskDQZmAs0AtMjYqmkScn+qZIGAQuBvsBGSf8GjIqIf3RVryK2+BTWypB0dtHFUesG/DvrvhxqZlZXfE3NzOqKQ62Kyg0BsfyRNF3SOkmP1rot9u441KqkaAjIeGAUcKqkUbVtlaVwLVCpRxasBhxq1VN2CIjlT0TMozDe0Loph1r1dDYEZEiN2mLWYzjUqqcqQ0DMrDSHWvVUZQiImZXmUKueqgwBMbPSHGpVEhEbgLYhIH8DboqIpbVtlZUj6UbgfmAvSaslfb7WbbJsPKLAzOqKe2pmVlccamZWVxxqZlZXHGpmVlccamZWVxxq3YikVklLJD0q6WZJ229BXddKOin5fE2pwfaSjpJ02Ls4xt+ld07Q0dX2DmVezXis76Sd7Nbqm0Ote3kjIg6IiA8AbwGTincmbwbJLCK+0GEC2Y6OAjKHmlktONS6r3uA9yW9qD9J+jXwiKRGSVdIWiDpYUlfBFDBVZKWSbod2LmtIkl3SxqTfB4nabGkhyTdJWl3CuH5taSX+BFJAyXNTI6xQNLhyde+V9Idkv4q6T9JMZ2tpFskLZK0VNLZHfb9OGnLXZIGJtv2lPSH5GvukbR3RX6aVjc88Uo3JKmJwnva/pBsGgt8ICKeSoLh5Yg4SNK2wH2S7gA+BOwF7AfsAiwDpneodyDwX8ARSV39I+IFSVOBVyPiR0m5XwP/ERH3ShpOYdTEPsDFwL0RcYmkfwXahVQXPpccozewQNLMiHgeeA+wOCLOk3RRUvdkYBowKSKekHQwcDXw0XfxY7Q65VDrXnpLWpJ8vgf4bwqnhQ9GxFPJ9mOBD7ZdLwN2BEYCRwA3RkQrsFbS/+uk/kOAeW11RURX7xX7F2CUNs/S3VdSn+QYJyRfe7ukF1N8T+dKOj75PCxp6/PARuB/ku2/AmZJ2iH5fm8uOva2KY5hPYhDrXt5IyIOKN6Q/ON+rXgT8JWImNuh3HGUf/WRUpSBwmWLQyPijU7aknrcnaSjKATkoRHxuqS7ge26KB7JcV/q+DMwK+ZravVnLnCOpF4Akt4v6T3APGBics1tMHB0J197P3CkpBHJ1/ZPtr8C9CkqdweFU0GScgckH+cBpyfbxgM7lWnrjsCLSaDtTaGn2KYBaOttnkbhtPYfwFOSPp0cQ5L2L3MM62EcavXnGgrXyxYnk4f8J4Ue+W+BJ4BHgJ8Df+74hRGxnsJ1sFmSHmLz6d9twPFtNwqAc4ExyY2IZWy+C/td4AhJiymcBj9dpq1/AJokPQxcCjxQtO81YF9JiyhcM7sk2X468PmkfUvxK9KtA7+lw8zqintqZlZXHGpmVlccamZWVxxqZlZXHGpmVlccamZWVxxqZlZXHGpmVlf+P+kY/s0r/dyQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.7155085599194361\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5718    0.6062    0.5885      1333\n",
      "           1     0.7948    0.7707    0.7826      2639\n",
      "\n",
      "    accuracy                         0.7155      3972\n",
      "   macro avg     0.6833    0.6884    0.6856      3972\n",
      "weighted avg     0.7200    0.7155    0.7175      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAa5klEQVR4nO3de7xVVb338c93r81FFAQEQUAQFUSQVARJU6K8gdbxaDcvpzp1yuxE9nSx7Hmdp5unPGY9Tycvh6h8zLwdS01MlMw0tFK5iCQkSngBUWDLVUFg7/07f6zFZrHZlzVhr73mnvv79rVer7XWHHvMseHF1zHmnGMMRQRmZllRVekGmJm1JYeamWWKQ83MMsWhZmaZ4lAzs0yprnQDilX3ODC69h5Y6WZYAoP77FfpJlgCa1atYOP6N7QvdeR6DYuo3VpS2di6dnZETNmX8yWVqlDr2nsgIz59Q6WbYQlc9aGxlW6CJfCFj5y5z3VE7Va6HfXhksq+vfD6fvt8woRSFWpm1hEIlN4rVw41M0tGQFWu0q1olkPNzJLTPl2WKyuHmpkl5OGnmWWNe2pmlhnCPTUzyxK5p2ZmGeO7n2aWHb5RYGZZIjz8NLOMcU/NzLLDw08zyxIBOd8oMLMs8TU1M8sODz/NLGvcUzOzTHFPzcwyQ54mZWZZ42lSZpYdvlFgZlnj4aeZZYbXUzOzbPHw08yyJsU3CtIbt2aWXjsf62jt1Wo1miJpqaRlkq5o4vjlkhYWXs9KqpPUt6U6HWpmlowKw89SXi1WoxxwPTAVGA1cKGl0cZmIuCYijouI44CvA3+MiHUt1etQM7Pk2qandiKwLCKWR8R24A7g3BbKXwjc3lqlDjUzS0xSSS+gn6R5Ra9LiqoZDKwo+ryy8F1T5+sBTAHuaq1tvlFgZonkV/Mu+Tm1mogY30JVjUUzZd8P/Km1oSc41MwsKQlVtcnDtyuBQ4s+DwFWNVP2AkoYeoKHn2a2FxIMP1syFxghabikruSDa2YT5zoQeDdwbyltc0/NzBJLMPxsVkTUSpoGzAZywI0RsVjSpYXj0wtFzwN+FxFvlVKvQ83MEmuLUAOIiFnArEbfTW/0+SbgplLrdKiZWTKi6Uv8KeFQM7NEREnXyyrGoWZmiVVVpfceo0PNzBJzT83MssPX1Mwsa9xTM7PM8I0CM8ucNpomVRYONTNLRh5+mlnGONTMLFMcamaWGb5RYGbZk95Mc6iZWULyNCkzyxgPP80sW9KbaQ61ffHOI/rypbNGUiUx8+lV3Pznl/coM25Yb7545kiqc2LDlh189uYFAPzb+4/mXSP6sf6t7Vz0kyfbu+md1tOLlnHjL2dTX1/PaZOP5/z3n7Lb8afmL+X2ux6hSiKXq+ITF5/F0UcNBeC3s5/k948sIIAzJh/P+6a8swK/QTp02p6apCnAf5JfqvdnEfEf5Txfe6oSXD7lKD5/69Os2bSNmz41gceer+HFml0rDh/QrZqvTh3FF257mtWbttGnR5eGY7995jV+NXcl3zx3dFPVWxnU1dfz0188wDe+9k8c1LcXX/vGz5gw7igOHdy/oczYMcOZMG4kknjpldX88Lpfc+33P8crK9bw+0cWcPW3P0V1dY4rr7mVcceNYNDAgyr4G1VGifsPVEzZrvaVsvtyRzZ6UC9Wrt/Kqg1vU1sfPLR4NZOO6rdbmbOOGcAjz61h9aZtAKzfsqPh2MJXNrBp6w6s/Sz7+6sMHNCHgQf3oUt1jlPeOYa585fuVma/7l0b/sFu27a94f3KVTWMPHII3bp1IZerYsyoYTw177l2/x3Soo02XimLcvbUGnZfBpC0c/flJWU8Z7s5uFd3Vm96u+Hzmk3bGDO4125lhh7Ug+qqKm746Dj275bjjqdW8MCi19u7qVawbv1m+vU9sOFz3769eOHvr+5R7sl5z3HLnQ+zadNb/O8vXwjA0CH9ue3Xf2Dz5i107dqFBc+8wBHDB7Vb29Oms879bGr35YmNCxV2bL4EoMuBB5exOeUXjbZhzVWJUYf05HO3LKBbdY6ff2I8z67cyIp1WyvTwE6u8d8PQFOdiYnjRzFx/CgWP/cyt9/1KN+64qMMGdyffzznXXz76lvo3r0rhw0dSC7FjzWUW5qHn+UMtZJ2X46IGcAMgB6Djmpud+bUWbPpbQb06t7w+eBe3ah5c1ujMtvYsOUN3t5Rz9s76nn6lQ2MGNDToVYhB/XtSc26jQ2f163bRN/ePZstP2bUMK5bvZ5Nm7fQq2cPTp98PKdPPh6AW+98mIP69mr2ZzMt5RPay/m/miS7L3c4f1u1mUP79uCQ3t2prhJnjBnAnOdrdisz5/m1HDe0NzmJbtVVjBnci5dqStq60MrgyMMH89rr61i9Zj07aut4/InFjB83crcyr61eRxS6dMtfeo3aujp6HrAfABs35v/u1tZs5Il5z3HKSce07y+QEiLfwy3lVQnl7Kk17L4MvEp+9+WLyni+dlUXwQ8eXMqPLzqeKsF9z7zGi2vf4rxxgwG4Z8GrvFSzhSf+/ga3fmYi9RHMfHoVy9fm/2Fced4Yxg3rQ+8eXbjvC+9ixh+Xc9/C1yr5K2VeLlfFpz42lSuvuZX6+uC9k45j6JCDmf3wPADOOm08T8z9G48+vojqXBVdu1bzpc99oKFXcs2P72Tzm1vJ5XJ8+uNTOWD//Sr561RQuu9+Kpq60NBWlUtnAz9i1+7L322pfI9BR8WIT99QtvZY27vqQ2Mr3QRL4AsfOZMXFi/cp0TqPnBkDPv4tSWVff77U+ZHxPh9OV9SZX1Orandl82sg6vg0LIUnlFgZokIqOqkj3SYWUa5p2ZmmZLmGwUONTNLxtfUzCxLhFK9SGR6W2ZmqdVWD99KmiJpqaRlkq5opsxkSQslLZb0x9bqdE/NzBJri2tqRSv5nEF+BtJcSTMjYklRmd7ADcCUiHhFUqsTxN1TM7NkSuyllZB7DSv5RMR2YOdKPsUuAu6OiFcAImJNa5U61Mwskfzcz5LXU+snaV7R65KiqppayWdwo9ONBPpIelTSfEkfa619Hn6aWWIJRp81LUyTKmUln2rgBOA0YD/gL5KeiIjnmzuhQ83MEmujGQWlrOSzknwwvgW8JWkOcCzQbKh5+GlmyajNlvNuWMlHUlfyK/nMbFTmXuBUSdWSepBfaPZvLVXqnpqZJbJzPbV9FRG1kqYBs9m1ks9iSZcWjk+PiL9JehBYBNST38Dp2ZbqdaiZWUJtt55aUyv5RMT0Rp+vAa4ptU6Hmpkl5mlSZpYd8tJDZpYhO59TSyuHmpkl5lAzs0xJcaY51MwsOffUzCw7vEikmWVJfpHI9KaaQ83MEqtKcVfNoWZmiaU40xxqZpaM5BsFZpYxKb6k1nyoSbqWPRdsaxARl5WlRWaWeh31RsG8dmuFmXUYIn8HNK2aDbWI+EXxZ0n7F1afNLNOLsUdtdZXvpV0kqQlFFablHSspBvK3jIzS6cSV72t1M2EUpbz/hFwFvAGQEQ8A0wqY5vMLOXaajPjcijp7mdErGiUunXlaY6ZpZ3o+A/frpB0MhCFzREuo5WND8ws29J897OU4eelwOfIbzL6KnBc4bOZdUKlDj1TO/yMiBrg4nZoi5l1EGkefpZy9/NwSfdJWitpjaR7JR3eHo0zs3RSia9KKGX4eRtwJ3AIMAj4FXB7ORtlZunW0R/pUET8MiJqC69baGH6lJllW/7uZ2mvSmhp7mffwttHJF0B3EE+zD4C3N8ObTOzNFLHXSRyPvkQ29n6zxQdC+DKcjXKzNKtQy49FBHD27MhZtYx7Bx+plVJMwokHQOMBrrv/C4ibi5Xo8ws3TpkT20nSd8EJpMPtVnAVOBxwKFm1kmlN9JKu/v5QeA04PWI+ARwLNCtrK0ys9SSIFelkl6VUMrwc2tE1EuqldQLWAP44VuzTizNw89SemrzJPUGfkr+jugC4KlyNsrM0q2t5n5KmiJpqaRlhUfHGh+fLGmjpIWF1zdaq7OUuZ//Wng7XdKDQK+IWNR6c80si4TaZO6npBxwPXAGsBKYK2lmRCxpVPSxiHhfqfW29PDtuJaORcSCUk9iZhnSditwnAgsi4jlAJLuAM4FGodaIi311H7YwrEA3rsvJ27K0Yf05E//57S2rtbKqM+EaZVugiWw7cVVbVJPgmtq/SQVb+I0IyJmFN4PBlYUHVsJTGyijpMkPQOsAr4SEYtbOmFLD9++p7Q2m1lnIiBXeqjVRMT4FqpqrPG88gXAsIh4U9LZwG+AES2dsJQbBWZmu2mjCe0rgUOLPg8h3xtrEBGbIuLNwvtZQBdJ/VpsW+Lfxsw6vTYKtbnACEnDC1sFXADMLC4gaaAKY11JJ5LPrDdaqrSkaVJmZjvlH9fY9zsFEVEraRowG8gBN0bEYkmXFo5PJ//w/2cl1QJbgQsiosWlz0qZJiXyy3kfHhHfkTQUGBgRflbNrJNqq8kChSHlrEbfTS96fx1wXaK2lVDmBuAk4MLC583kny0xs06qQ2+8AkyMiHGSngaIiPWF8a+ZdUICqlM8TaqUUNtRePI3ACT1B+rL2iozS7UUZ1pJofZj4B7gYEnfJX/h7t/K2iozSy2pbaZJlUspcz9vlTSf/PJDAv4xIrxDu1knluJMK+nu51BgC3Bf8XcR8Uo5G2Zm6dXRl/O+n10bsHQHhgNLgTFlbJeZpZSgYgtAlqKU4efY4s+F1Ts+00xxM8u6Cu7pWYrEMwoiYoGkCeVojJl1DErxLgWlXFP7UtHHKmAcsLZsLTKzVMvCFnk9i97Xkr/Gdld5mmNmHUGHDbXCQ7cHRMTl7dQeM+sA0rzxSkvLeVcXZtE3u6y3mXU++S3yKt2K5rXUU3uK/PWzhZJmAr8C3tp5MCLuLnPbzCylOvSMAqAv+UXZ3suu59UCcKiZdUId+UbBwYU7n8+yK8x2anGRNjPLthR31FoMtRxwAKVtjmBmnYao6qDPqb0WEd9pt5aYWYcgOm5PLcXNNrOKEVSn+KJaS6HmXYXNbA8dtqcWEevasyFm1nF09Ec6zMx2k+JMc6iZWTIi3bugO9TMLBl5+GlmGZKfUeBQM7MMSW+kOdTMbC+kuKPmUDOzpNQx11MzM2uK736aWeb4RoGZZYfSvZx3mnuRZpZCO4efpbxarUuaImmppGWSrmih3ARJdZI+2FqdDjUzS0xSSa9W6sgB1wNTgdHAhZJGN1PuamB2KW1zqJlZYirx1YoTgWURsTwitgN3AOc2Ue7z5LflXFNK23xNzcwSEZAr/ZpaP0nzij7PiIgZhfeDgRVFx1YCE3c7lzQYOI/8HikTSjmhQ83MEktwn6AmIsY3V00T3zXeKuBHwNcioq7UmxMONTNLSKhtJkqtBA4t+jwEWNWozHjgjkKg9QPOllQbEb9prlKHmpkl1kZPdMwFRkgaDrwKXABcVFwgIobvOqduAn7bUqCBQ83MEso/0rHvqRYRtZKmkb+rmQNujIjFki4tHJ++N/U61MwsGbXdhPaImAXMavRdk2EWEf9cSp0ONTNLzNOkzCwz8otEVroVzXOomVlibXT3sywcamaWWIpHnw61ffH7Py/h6z/8NXX19Xz03JP54j+fudvxOx+Yy3/e/BAA++/XjR9e8RHGjhzC29t2cM4lP2Lbjlrqauv4h9OO5+ufOacSv0Knc9pJR3PVlz9IrqqKX977Z370i4d2O/75fzqND03NP7henati5GEDOfLMK+jX5wBu/N4nG8oNG3QQV824n+m3P9qezU+NTtlTk3Qj8D5gTUQcU67zVEpdXT2Xf/9O7rluGoMG9Oa9H7+GqZPGMurwQxrKDBt0EPf/5H/Ru1cPHvrTYr74vdv5/U2X061rNff+12Uc0KMbO2rrmPqp/8vpJ49mwtjhLZzR9lVVlbjmqx/mvGnXsWr1Bv7wi8t5YM5fWfri6w1lrr3lYa695WEAppx6DJ+98D1s2LSFDZu2MOni/2ioZ8ms73L/I89U5PeotLRfUyvnhPabgCllrL+i5i9+icMP7cdhQ/rRtUs1558xjll/XLRbmYnHHk7vXj0AmDB2OKvWbADyKxwc0KMbADtq69hRW5fq9amy4oQxh7F8RQ0vv/oGO2rruPuhBZz97nc0W/4DZ47nrt/N3+P7d084ipdWrmXF6+vL2dz0kqgq8VUJZQu1iJgDrCtX/ZX22tqNDB7Qp+HzoAF9eG3txmbL//LeP3P6ybtWVamrq+fUi65i5JlXMHniKMYfc1g5m2vAIf0P5NXVu4Jo1er1HNL/wCbL7tetC6eddDQz/7Bwj2Pnn3kCd83eM+w6kzZapaMsKr70kKRLJM2TNG9tzdpKN6dkEY3n3TZ/8fSxec9zy8y/8K1pu1ZVyeWqeOy2r7P4/n9nweKXWbKs8ZQ3a2tN9Yab+GsEYMqksTy5aDkbNm3Z7fsu1TmmThrLbx5+uhxN7BB27vvZ6XpqpYqIGRExPiLG9+/Xv9LNKdmgg3vv8X/9gf32/L/+sy+8ymX/fhu3/uAS+vY+YI/jB/bswSknjODhvywpa3sNVq3ZsEfv+vWapnvX55/RdG/s9JNH88xzK1i7bnPZ2tkRuKeWQeNGD+Pvr6zl5Vdr2L6jlrsfWsDUSbtfn1nx+jo+9tWfMv3bH+PIYQMavq9Zv5mNm/M9gK1vb+fRp5Yy4rABWHktWPIyRwztz9BBB9GlOsf5Z4zjgTmL9ijXa//uvGvckXtcIwX44FlNX2frdFKcan6kYy9VV+f4/lc/zAcuu566uuDif3gnRx9xCDfe9RgAn/zAqVzzswdYt/EtvnL1fxd+popHbv4ar9ds4l+/9Uvq6uuprw/OO30cU04dW8lfp1Ooq6vnq9+/k7t+/DlyOXHrzCd4bvnrfOL8UwD4/3c/DsA57zmWR558ji1vb9/t5/fr1oXJJ47ii9+7vd3bnjZpnialpq4NtUnF0u3AZPJrIK0GvhkRP2/pZ044YXz86cl5LRWxlOkzYVqlm2AJbFt6J/Vb1uxTIh099vi4+d5HSyp74hG957ewSGRZlK2nFhEXlqtuM6uw9HbUPPw0s2Tyl8vSm2oONTNLpg3XUysHh5qZJZbiTHOomVlSrW9UXEkONTNLLMWZ5lAzs2QqOVugFA41M0suxanmUDOzxPxIh5lliq+pmVl2+Dk1M8saDz/NLDOEe2pmljEpzjSHmpnthRSnmkPNzBJL8yKRDjUzSyy9keZQM7O9keJU88YrZpbIzkUiS/mv1bqkKZKWSlom6Yomjp8raZGkhYWtNE9prU731MwsmTZ6+FZSDrgeOANYCcyVNDMiiveLfBiYGREh6R3AncColup1T83MEmujHfJOBJZFxPKI2A7cAZxbXCAi3oxdu0PtD7S6U5R7amaWUKJFIvtJKt4ibkZEzCi8HwysKDq2Epi4x9mk84CrgIOBc1o7oUPNzBJLMPysaWGLvKZq2aMnFhH3APdImgRcCZze0gk9/DSzREodepaQeyuBQ4s+DwFWNVc4IuYAR0jq11KlDjUzS65tUm0uMELScEldgQuAmbudRjpShbGupHFAV+CNlir18NPMEmuLVToiolbSNGA2kANujIjFki4tHJ8OfAD4mKQdwFbgI0U3DprkUDOzxNpqllREzAJmNfpuetH7q4Grk9TpUDOzZARVKZ5R4FAzs72Q3lRzqJlZIl4k0swyJ8WZ5lAzs+TcUzOzTEkwTardOdTMLLH0RppDzcwSkvf9NLOs8b6fZpYt6c00h5qZJZfiTHOomVlS8hZ5ZpYdaZ9R4PXUzCxT3FMzs8TS3FNzqJlZYn6kw8yyww/fmlmWpP1GgUPNzBLz8NPMMsU9NTPLlBRnmkPNzPZCilPNoWZmiQhSPU1KrewL2q4krQVernQ7yqAfUFPpRlgiWf07GxYR/felAkkPkv/zKUVNREzZl/MllapQyypJ8yJifKXbYaXz31nH5bmfZpYpDjUzyxSHWvuYUekGWGL+O+ugfE3NzDLFPTUzyxSHmpllikOtjCRNkbRU0jJJV1S6PdY6STdKWiPp2Uq3xfaOQ61MJOWA64GpwGjgQkmjK9sqK8FNQLs+LGpty6FWPicCyyJieURsB+4Azq1wm6wVETEHWFfpdtjec6iVz2BgRdHnlYXvzKyMHGrl09SMXz8/Y1ZmDrXyWQkcWvR5CLCqQm0x6zQcauUzFxghabikrsAFwMwKt8ks8xxqZRIRtcA0YDbwN+DOiFhc2VZZayTdDvwFOErSSkn/Uuk2WTKeJmVmmeKempllikPNzDLFoWZmmeJQM7NMcaiZWaY41DoQSXWSFkp6VtKvJPXYh7pukvTBwvuftTTZXtJkSSfvxTlekrTHrkPNfd+ozJsJz/UtSV9J2kbLHodax7I1Io6LiGOA7cClxQcLK4MkFhGfioglLRSZDCQONbNKcKh1XI8BRxZ6UY9Iug34q6ScpGskzZW0SNJnAJR3naQlku4HDt5ZkaRHJY0vvJ8iaYGkZyQ9LOkw8uH5xUIv8VRJ/SXdVTjHXEnvKvzsQZJ+J+lpST+hhH28Jf1G0nxJiyVd0ujYDwtteVhS/8J3R0h6sPAzj0ka1SZ/mpYZ3qG9A5JUTX6dtgcLX50IHBMRLxaCYWNETJDUDfiTpN8BxwNHAWOBAcAS4MZG9fYHfgpMKtTVNyLWSZoOvBkRPyiUuw34fxHxuKSh5GdNHA18E3g8Ir4j6Rxgt5BqxicL59gPmCvproh4A9gfWBARX5b0jULd08hviHJpRLwgaSJwA/DevfhjtIxyqHUs+0laWHj/GPBz8sPCpyLixcL3ZwLv2Hm9DDgQGAFMAm6PiDpglaQ/NFH/O4E5O+uKiObWFTsdGC01dMR6SepZOMf5hZ+9X9L6En6nyySdV3h/aKGtbwD1wH8Xvr8FuFvSAYXf91dF5+5WwjmsE3GodSxbI+K44i8K/7jfKv4K+HxEzG5U7mxaX/pIJZSB/GWLkyJiaxNtKXnenaTJ5APypIjYIulRoHszxaNw3g2N/wzMivmaWvbMBj4rqQuApJGS9gfmABcUrrkdAryniZ/9C/BuScMLP9u38P1moGdRud+RHwpSKHdc4e0c4OLCd1OBPq209UBgfSHQRpHvKe5UBezsbV5Efli7CXhR0ocK55CkY1s5h3UyDrXs+Rn562ULCpuH/IR8j/we4AXgr8B/AX9s/IMRsZb8dbC7JT3DruHffcB5O28UAJcB4ws3Ipaw6y7st4FJkhaQHwa/0kpbHwSqJS0CrgSeKDr2FjBG0nzy18y+U/j+YuBfCu1bjJdIt0a8SoeZZYp7amaWKQ41M8sUh5qZZYpDzcwyxaFmZpniUDOzTHGomVmm/A+Saudiok3RiAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(c_lr_char, model_name=\"lr_chars_newFeatures\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_M05gIqmUCFp"
   },
   "source": [
    "#### **5.2.2 Random Forest Classifier**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_rf = RandomForestClassifier(n_estimators=500,\n",
    "                                criterion=\"gini\",\n",
    "                                max_depth=None, \n",
    "                                min_samples_split=2,\n",
    "                                min_samples_leaf=10, \n",
    "                                max_features=\"auto\", \n",
    "                                n_jobs=-1,\n",
    "                                verbose=1,\n",
    "                                random_state=0,\n",
    "                                class_weight='balanced')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_rf = GridSearchCV(clf_rf, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   22.6s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.6min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 2.7min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   18.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.5min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 2.6min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   15.4s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   56.5s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.4min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 2.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   16.2s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.5min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 2.6min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   16.7s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.6min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 2.7min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   10.1s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:   46.5s\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.1min finished\n"
     ]
    }
   ],
   "source": [
    "clf_rf_char= grid_search_rf.fit(X_train_char, y_train_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.8944756150194216\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8775    0.7917    0.8324      3067\n",
      "           1     0.9017    0.9453    0.9230      6201\n",
      "\n",
      "    accuracy                         0.8945      9268\n",
      "   macro avg     0.8896    0.8685    0.8777      9268\n",
      "weighted avg     0.8937    0.8945    0.8930      9268\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.2s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.4s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAchklEQVR4nO3deZhcZZ328e/dHQKB7BtJWMIiW1iCYQnLGAMCElQ2GWQREfFF1MAMg6Dj67CqCIyXiBEZQF5EEF6QsMmuAmFVkhACCftOQshCEiCEJd2/+aNON91Nd9U56aqu09X3h6suuqpOPeepzpU7z1me56eIwMysVtRVuwNmZuXkUDOzmuJQM7Oa4lAzs5riUDOzmtKr2h1oqW6tflG3zrBqd8My2GL9gdXugmUw//XXWPr2YnWmjfr+oyNWrUy1baxcdFdE7NuZ/WWVr1BbZxj9v/TTanfDMrjunP2r3QXL4ND9JnS6jVi1kjW3ODTVth/M+u3QTu8wo1yFmpl1BwLl98yVQ83MshFQV1/tXnTIoWZm2alTp+UqyqFmZhn58NPMao1HamZWM0SuR2r57ZmZ5ZQKI7U0j1ItSftKelbSC5J+1M77gyTdKGm2pH9K2qZUmw41M8uurj7dowhJ9cBvgUnAGOBwSWPabPZjYFZEbAd8A/h1ya6t1hcysx4suVCQ5lHczsALEfFSRHwEXAsc0GabMcDfACLiGWAjSesWa9ShZmbZiCyHn0MlTW/xOK5FS+sBr7d4/kbyWktPAAcDSNoZGA2sX6x7vlBgZtmlv1CwOCJ27KiVdl5ruxT3L4BfS5oFPAk8DqwqtkOHmpllVLb71N4ANmjxfH1gfssNIuId4BgASQJeTh4dcqiZWTYC6ssyTeoxYDNJGwPzgMOAI1rtShoIvJ+cc/s2MC0Jug451MwsuzLcfBsRqyRNBu4C6oHLI2KOpOOT9y8GtgKulNQAzAWOLdWuQ83MMirfNKmIuB24vc1rF7f4+RFgsyxtOtTMLDtPkzKzmpLjaVIONTPLJuUUqGpxqJlZdl4k0sxqh9dTM7Na48NPM6sZOV9PzaFmZhn58NPMao0vFJhZTfE5NTOrGfLhp5nVGo/UzKyWyKFmZrWisJq3Q83MaoWE6hxqZlZD8jxSy+8lDDPLLUmpHinaKVXMeICkWyU9IWmOpGNKtelQM7PMyhFqKYsZfx+YGxFjgYnALyX1LtauQ83MslGGR3FpihkH0C+pJNUXeBuXyDOzchLpDi0TQyVNb/H8koi4JPm5vWLG49t8fgpwC4XSef2Ar0VEY7EdOtTMLLO6ui4rZvxFYBawJ7ApcI+kB4qVyfPhp5llVqYLBSWLGVMoZDw1Cl6gUMh4y2KNOtTMLJvynVNrLmacnPw/jMKhZkuvAV8AkLQusAXwUrFGffhpZpmV4z61lMWMzwaukPQkhZj8YUQsLtauQ83MMsl4oaCoFMWM5wP7ZGnToWZmmXmalJnVDuV7mpRDzcwyc6iZWU1xqJlZzSjnhYJKcKiZWXb5zTSHmpllpEzTpLqcQ83MMvPhp5nVlvxmmkOtMyZuM5Kzj9iBOolrHniRKbfPbfX+d/fdioN32QiA+jqx2aj+bPtvU1m24iOO3WsLjpywKRJcPe1FLrvn2Sp8g57n4RnP8t+X3EpjY3DgPjvxzX+d2Or9O+59nD/ccD8Aa6/Vmx9970A232QUAGdecD0PPvYMgwb05bqLTurqrudKnkdqFT0wLrVUb3dWJ/Hzr+/Ikb+6l4k/uY0Dxo9ms1H9W23zuzufZu8z7mDvM+7gnBue4JFnF7JsxUdssd4AjpywKV/66V3sdfod7D12PTYe3q9K36TnaGho5Nzf3cyFZx7D9RedxF33z+Kl195qtc2oEYO55BfHce2Uf+fYw77Az6bc2PzeV/bagd+c+a2u7nbupF2ho1rBV7FQS7lUb7f12U2G8MrC93ht0Qo+bmjk5n+8yhe3X7/D7Q8cP5qb/vEqAJuN7M/Mlxaz8qMGGhqDR55dyKRxHX/WymPOc6+zwcghrD9iCGus0Yt9Jozl/kdbj67HbjWa/n3XBmDbLTdg4eLlze+N22YT+vfr06V9zqseGWqkW6q32xoxsA/z317R/PzNpe8zctDa7W7bp3c9E7cZye0zCot8PjNvOeM3H86gdXrTp3c9e247ilGD2/+slc/CJe+w7rABzc+HDx3AwiUdrjXIzXdPZ7cdN++KrnU7qlOqRzVU8pxamqV6kXQccBxA3TpDK9id8mrvH6GItot2Fuw9dj2mv7CYZSs+AuCFN9/hojvmcu0P9mTFB6uY+/pSVjW2/1krp0//jjsaTEyf/SI33/0Yl513fIX71D3l+ZxaJUMtzVK9JOuVXwLQa8gm3eZv9ptLVzJq8DrNz0cOWpsFy1a2u+0B40dz0z9eafXaNQ+8xDUPFNa6+9HBY3lz6fsV66sVDB8ygLcWfXI4uXDxcoYN7v+p7Z5/+U3OvvAGLjzzGAb2X+dT7/d4OZ/QXsnDzzRL9XZbs15ewsbr9mODoeuwRn0dB4wfzd2z5n1qu3591mCXzYdz5+NvtHp9SL81AVhv8Nrst8P6nwo9K78xm6/P6/OXMG/B23z88SrunvYEE8a3Ps27YOEyTvn5VZx18tcYvd6wKvU030RhhJvmUbKt0nU/T5E0K3k8JalB0uBibVZypNa8VC8wj8JSvUdUcH9dqqEx+L9XTedP/7EH9XXi2gdf4rn5yzlq4mcA+ON9LwAwadz6TJuzgJUfNbT6/GXf/xyD+q7Jxw2N/Piq6Sx//+Mu/w49Ta/6ek45fn9OOO1yGhob2X/vHdl09Lr8+fZHAThkv1249Nq/svydFZx70U0A1NfX8ccLTgDgx+ddw4wnX2LZOyvY7+ifc9yRe3PgPjtV6+tUUXkuArS4mLg3hUHQY5JuiYjmqzcRcT5wfrL9V4CTIuLtou12dB6oHCTtB1zAJ0v1/qzY9r2GbBL9v/TTivXHyu+Bc/avdhcsg0P3m8CcJ2Z2KpHWGrF5jD76N6m2fe68fWd0VE1K0q7AGRHxxeT5fwJExDkdbP8n4N6IuLTYPit68217S/WaWTeX8tAy0dm6n4VdSmsD+wKTS+3QMwrMLBMBdelv1+hs3c8mXwEeKnXoCQ41M1sNZbr4meVi4mHANWkaze/6IWaWW2WaUZCm7ieSBgCfB25O0zeP1Mwsm2zn1DqUsu4nwEHA3RGxooOmWnGomVkmQmVbJLJU3c/k+RXAFWnbdKiZWWY5nlDgUDOz7PI8TcqhZmbZlOmcWqU41Mwsk8Lcz/ymmkPNzDLLcaY51MwsuwwzCrqcQ83Mssn5emoONTPLpGk9tbxyqJlZRtUrqpKGQ83MMstxpjnUzCwj+UKBmdUQ36dmZjXHoWZmNSXHmeZQM7PsPFIzs9qR8wntXs7bzDIpLBKZ7lGyrRLFjJNtJibFjOdIur9Umx6pmVlmdV1UzFjSQOAiYN+IeE3S8JJ963TPzKzHkdI9StgZeCEiXoqIj4BrgQPabHMEMDUiXgOIiIWlGnWomVkmUqZqUkMlTW/xOK5FU+0VM16vze42BwZJuk/SDEnfKNU/H36aWWYZJhR0tphxL2AH4AtAH+ARSY9GxHMd7bDDUJP0m3Z28MmeI07s6D0zq21lmiaVppjxGxSCcQWwQtI0YCyQPdSA6avZUTOrYaJwBbQMmosZA/MoFDM+os02NwNTJPUCegPjgV8Va7TDUIuIP7R8LmmdtMVEzay2lWOglqaYcUQ8LelOYDbQCFwWEU8Va7fkOTVJuwK/B/oCG0oaC3wnIr7Xua9kZt2SyreeWspixucD56dtM83VzwuALwJLkh08AUxIuwMzqz1luqWjIlJd/YyI19skc0NlumNmeSfKc/NtpaQJtdcl7QaEpN7AicDTle2WmeVZnheJTHP4eTzwfQo3xc0Dtk+em1kPlPbQM7eHnxGxGDiyC/piZt1Eng8/S47UJG0i6VZJiyQtlHSzpE26onNmlk9K+aiGNIeffwKuA0YCo4DrgWsq2Skzy7cMcz+7XJpQU0T8MSJWJY+rKDJ9ysxqW+HqZ7pHNRSb+zk4+fHeZPG2aymE2deA27qgb2aWR0q3AGS1FLtQMINCiDX1/jst3gvg7Ep1yszyrVvWKIiIjbuyI2bWPTQdfuZVqhkFkrYBxgBrNb0WEVdWqlNmlm/dcqTWRNLpwEQKoXY7MAl4EHComfVQ+Y20dFc/D6Gw6uSCiDiGwgJta1a0V2aWWxLU1ynVoxrSHH6ujIhGSask9QcWAr751qwHy/PhZ5qR2vSkTNWlFK6IzgT+WclOmVm+lWvuZ6m6n0nNz+VJ3c9Zkk4r1WaauZ9Ni0FenKxA2T8iZpfurpnVIqEuq/uZeCAivpy23WI3344r9l5EzEy7EzOrIeVbgaO57ieApKa6n21DLZNiI7VfFnkvgD07s+P2jN1oMA9d3rbuguXZoJ0mV7sLlsGHz79RlnYynFMbKqllEadLIuKS5Of26n6Ob6eNXSU9QaHS1A8iYk6xHRa7+XaPdH02s55EQH36UOts3c+ZwOiIeE/SfsBNwGbFdugK7WaWWZkmtJes+xkR70TEe8nPtwNrSBpatG+Zv42Z9XhlCrXmup9JqYDDgFtabiBphJJjXUk7U8isJcUaTTVNysysSeF2jc5fKUhT95PCzf/flbQKWAkcFhFFlz5LM01KFJbz3iQizpK0ITAiInyvmlkPVa7JAqXqfkbEFGBKpr6l2OYiYFfg8OT5uxTuLTGzHqpbF14BxkfEOEmPA0TE0uT418x6IAG9cjxNKk2ofZzc+RsAkoYBjRXtlZnlWo4zLVWoXQjcCAyX9DMKJ+5+UtFemVluSeWZJlUpaeZ+Xi1pBoXlhwQcGBGu0G7Wg+U401Jd/dwQeB+4teVrEfFaJTtmZvnV3Zfzvo1PCrCsBWwMPAtsXcF+mVlOCaq2AGQaaQ4/t235PFm94zsdbG5mta6KNT3TyDyjICJmStqpEp0xs+5BOa5SkOac2n+0eFoHjAMWVaxHZpZrtVAir1+Ln1dROMd2Q2W6Y2bdQbcNteSm274RcUoX9cfMuoE8F14ptpx3r2QWfYfLeptZz1MokVftXnSs2EjtnxTOn82SdAtwPbCi6c2ImFrhvplZTnXrGQXAYAqLsu3JJ/erBeBQM+uB8n6hoNggcnhy5fMp4Mnk/3OS/z/VBX0zs5zqqrqfLbbbSVKDpENKtVlspFYP9CVdcQQz6zFEXRnuU0tb9zPZ7lwKK+SWVCzU3oyIs1azv2ZWo0SX1/08gcJtZKlu+i8Wajk+ajazqhH0Sn9SrVN1PyWtBxxE4Zx+p0PtC2kaMLOeJeNIrbN1Py8AfhgRDWnvjStWzPjtVC2YWY9Tpls6Stb9BHYErk0CbSiwn6RVEXFTR426RJ6ZZVamc2rNdT+BeRTqfh7RcoOI2PiTfeoK4C/FAg0camaWkShPFfSUdT8zc6iZWTYq34yCUnU/27z+zTRtOtTMLJPCjIL83hzhUDOzzPIbaQ41M1sNOR6oOdTMLCt1z/XUzMzaU66rn5XiUDOzzHyhwMxqh7rpct5mZu3x4aeZ1RyP1MyspuQ30hxqZpaRgHqP1MysluQ40xxqZpaVUI4PQB1qZpaZR2pmVjMKt3TkN9UcamaWTcqantWS53vozCyn6qRUj1JKFTOWdICk2ZJmSZou6V9KtemRmpllUlgksgztpCtm/DfglogISdsB1wFbFmvXIzUzy0wp/yuhuZhxRHwENBUzbhYR70VEU9m8dfh0Cb1PcaiZWWZSugdJMeMWj+NaNNNeMeP1Pr0vHSTpGeA24Ful+ubDz4z++vBc/vOXf6ahsZGjDtiNk765T6v3I4If/fLP3PPQHPqs1ZuLTj+KsVsWShtut/9p9F17Terr6ujVq457r/whAL+45DauvOlhhgzsC8B/fX9/9tl96679Yj3EF3bdinNOPoT6ujr+ePPDXPCHe1q9P6BfH6b819fZeP2hfPDRx5xw9tU8/eKbADxx85m89/6HNDQ2smpVI3sefV41vkIuZLhPrbPFjImIG4EbJU0Azgb2KrbDioWapMuBLwMLI2KbSu2nKzU0NHLKeddx45TJjFp3IHsefT6TJmzLlpuMbN7mnofn8uJri5gx9XSmP/UKJ//iWv56xSnN79968b81h1dL3z18D044quiflXVSXZ04/9RDOWjyFOa/tYy//+EU7pj2JM++vKB5m5OP+SJPPvcGR516KZuNXpfzf3goB37vN83vf+X4X/P28hXV6H5ulOucGumKGTeLiGmSNpU0NCIWd7RdJQ8/rwD2rWD7XW7GnFfYZIOhbLT+UHqv0YuD9x7H7ffPbrXN7ffP5rAv7Ywkdtp2Y5a/u5IFi5dXqcfW0g5bb8RLry/m1XlL+HhVA1Pvmcl+n9+u1TZbbDyCaY89C8Dzr77FhiMHM2xwv2p0N79SXvlMcfWzuZixpN4Uihnf0npX+oySJUEkjQN6A0uKNVqxUIuIacDblWq/Gt5ctJz11h3U/HzUuoN4c9HyNtssa73N8IG8uXAZUFiu5eDJU5h41LlcMfXBVp+79Ppp7H74z5l81lUse+f9yn2JHmzksAHMe2tp8/P5by1l5LABrbZ56vl5fHmP7QEYN2Y0G4wYzKjhA4HCqYWpUyZz75WncvRBu3dVt3NJKR/FRMQqoKmY8dPAdU3FjJsKGgNfBZ6SNIvCldKvtbhw0K6qn1NLThweB7DBhhtWuTfFtfe7bPuPUXu/7qa1p+687CRGDhvIorff5aDJU9hsoxHsPu4zfOurn+OUYychwc8u/gs/uWAqU077eiW+Qo/W3hpgbf+8LvjDPZxz8iFMu/pHzH1hPrOfe4OGhkYA9v32r1iweDlDB/XlximTef6VBTz8+Itd0fVcKWfdz1LFjCPiXODcLG1W/epnRFwSETtGxI7Dhg6rdneKGjV84Kf+pR8xdEDxbRYuY0QyGhg5bCAAwwb348sTt2PmnFcAGD6kP/X1ddTV1XH0gbszY86rlf0iPdT8hcs+NdJue2rg3RUfMPmsq5hw5C84/vQrGTqwL6/OLxztNG27eOl7/OW+2YzbeqMu63velGOkVilVD7XuZNyY0bz42iJenbeYjz5exdR7ZjJpQutzMpMmbMu1t/2TiOCxJ1+mf98+jBg6gBUrP+TdFR8AsGLlh/z90WfYatNRAK3+Yv3lvifYatORWPnNnPsqm244jA1HDWGNXvUcvPc47pjW+pxo/759WKNXPQDfOHA3Hn78Bd5d8QFrr9WbvmuvCcDaa/Vmz1225OkXOzynXftynGpVP/zsTnr1que8Uw/lqyf+loaG4Mj9d2GrTUdy+Q0PAPCtr36OfXbfmnsemsO4g86kz1pr8NvkMHLRknf5+qmXAtCwqoGv7rsje+02BoDTL7yJJ597A0lsOHIwv/rx4dX5gjWuoaGRU8+7jhsu/D719eLqWx7lmZcWcMzBhZk3/2/qg2yx8Qh+d8ZRNDQ28uzLCzjh7KsBGDakH1ed938AqO9Vzw13Tudvjzxdte9SbXmuJqUS59xWv2HpGmAiMBR4Czg9In5f7DM77LBjPPSP6RXpj1XGoJ0mV7sLlsGHz15H4/sLO5VIW2372bjy5vtSbbvzpgNnFLlPrSIqNlKLCA83zGpVfgdqPvw0s2wKp8vym2oONTPLJufrqTnUzCyzHGeaQ83MspKLGZtZbclxpjnUzCybas4WSMOhZmbZ5TjVHGpmlplv6TCzmuJzamZWO3J+n5pX6TCzzMpUTSpN3c8jk7qfsyU9LGlsqTY9UjOzTER5Rmop636+DHw+IpZKmgRcAowv1q5HamaWWZmWU0tT9/PhiGhadfVRCsVZinKomVl26VOt03U/WzgWuKNU13z4aWaZZVgkstN1PwEk7UEh1P6l1A4damaWWZkufqaq+ylpO+AyYFJEFC2PBz78NLPVUZ6Tamnqfm4ITAWOiojn0nTNIzUzy6Rci0RGxCpJTXU/64HLm+p+Ju9fDJwGDAEuSlYGWVVqeXCHmpllU8abb1PU/fw28O0sbTrUzCyzHE8ocKiZWVZeJNLMakyOM82hZmbZeJFIM6s9OU41h5qZZeZFIs2spvicmpnVDkGdQ83Makt+U82hZmaZlGuRyEpxqJlZZjnONIeamWXnkZqZ1RRPkzKzmpLfSHOomVlGynndT4eamWWW5xkFXs7bzLIrU428FMWMt5T0iKQPJf0gTdc8UjOzzMoxTktZzPht4ETgwLTteqRmZhmJOqV7lJCmmPHCiHgM+Dht7xxqZpZJ04yCNA/KW8w4FR9+mlkllaWYcRYONTPLrEy3dKQqZpyVDz/NLDOl/K+EksWMV4dHamaWTZluvk1TzFjSCGA60B9olPTvwJiIeKejdh1qZpZJOZceSlHMeAGFw9LUHGpmllmeZxQ41MwsM8/9NLOakuNMc6iZ2WrIcao51MwsE0GaKVBVo4hO38BbNpIWAa9Wux8VMBRYXO1OWCa1+mc2OiKGdaYBSXdS+P2ksTgi9u3M/rLKVajVKknTi0wVsRzyn1n35RkFZlZTHGpmVlMcal3jkmp3wDLzn1k35XNqZlZTPFIzs5riUDOzmuJQq6BSlXIsfyRdLmmhpKeq3RdbPQ61CmlRKWcSMAY4XNKY6vbKUrgC6NKbRa28HGqVU7JSjuVPREyjUJbNuimHWuVUpFKOmRXnUKucilTKMbPiHGqVU5FKOWZWnEOtcipSKcfMinOoVUhErAKaKuU8DVwXEXOq2ysrRdI1wCPAFpLekHRstftk2XialJnVFI/UzKymONTMrKY41MyspjjUzKymONTMrKY41LoRSQ2SZkl6StL1ktbuRFtXSDok+fmyYpPtJU2UtNtq7OMVSZ+qOtTR6222eS/jvs6Q9IOsfbTa41DrXlZGxPYRsQ3wEXB8yzeTlUEyi4hvR8TcIptMBDKHmlk1ONS6rweAzySjqHsl/Ql4UlK9pPMlPSZptqTvAKhgiqS5km4Dhjc1JOk+STsmP+8raaakJyT9TdJGFMLzpGSU+DlJwyTdkOzjMUm7J58dIuluSY9L+h9S1PGWdJOkGZLmSDquzXu/TPryN0nDktc2lXRn8pkHJG1Zlt+m1QxXaO+GJPWisE7bnclLOwPbRMTLSTAsj4idJK0JPCTpbuCzwBbAtsC6wFzg8jbtDgMuBSYkbQ2OiLclXQy8FxH/nWz3J+BXEfGgpA0pzJrYCjgdeDAizpL0JaBVSHXgW8k++gCPSbohIpYA6wAzI+JkSaclbU+mUBDl+Ih4XtJ44CJgz9X4NVqNcqh1L30kzUp+fgD4PYXDwn9GxMvJ6/sA2zWdLwMGAJsBE4BrIqIBmC/p7+20vwswramtiOhoXbG9gDFS80Csv6R+yT4OTj57m6SlKb7TiZIOSn7eIOnrEqAR+P/J61cBUyX1Tb7v9S32vWaKfVgP4lDrXlZGxPYtX0j+cq9o+RJwQkTc1Wa7/Si99JFSbAOF0xa7RsTKdvqSet6dpIkUAnLXiHhf0n3AWh1sHsl+l7X9HZi15HNqtecu4LuS1gCQtLmkdYBpwGHJObeRwB7tfPYR4POSNk4+Ozh5/V2gX4vt7qZwKEiy3fbJj9OAI5PXJgGDSvR1ALA0CbQtKYwUm9QBTaPNIygc1r4DvCzpX5N9SNLYEvuwHsahVnsuo3C+bGZSPOR/KIzIbwSeB54Efgfc3/aDEbGIwnmwqZKe4JPDv1uBg5ouFAAnAjsmFyLm8slV2DOBCZJmUjgMfq1EX+8EekmaDZwNPNrivRXA1pJmUDhndlby+pHAsUn/5uAl0q0Nr9JhZjXFIzUzqykONTOrKQ41M6spDjUzqykONTOrKQ41M6spDjUzqyn/C2VSeuwaE7KWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Set\n",
      "Accuracy: 0.7532729103726082\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6660    0.5311    0.5910      1333\n",
      "           1     0.7851    0.8655    0.8234      2639\n",
      "\n",
      "    accuracy                         0.7533      3972\n",
      "   macro avg     0.7256    0.6983    0.7072      3972\n",
      "weighted avg     0.7452    0.7533    0.7454      3972\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.6s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcbElEQVR4nO3de5hcVZnv8e+vOwmQ+5UACZcA4RKuQiDCQIhcJEFHBkTlMt45iBp5juOg4DjqyHFQOM54ZojGiDk4KkEQlAAhwYOGIOKYEAmQQCAmQEIISZOQCwSS7n7PH1Vpqpvu6r2TqtTu3b+PTz1P7dqr1n67m7yutddeaykiMDPLi7paB2BmVklOamaWK05qZpYrTmpmlitOamaWKz1qHUCpPfsNij7D9qt1GJZCrx6qdQiWwsZXXuKNjRt26Y9W3//AiMaticrG1nVzImLirlwvrUwltT7D9uN9191a6zAshQOG7FXrECyFn1x14S7XEY1b2ePwDycq++bjU4bu8gVTylRSM7OuQKDs3rlyUjOzdATU1dc6ig45qZlZesruvVQnNTNLyd1PM8sbt9TMLDeEW2pmlidyS83Mcsajn2aWHx4oMLM8Ee5+mlnOuKVmZvnh7qeZ5YmAeg8UmFme+J6ameVHtruf2Y3MzLJLSvbqtBpNlLRU0jJJ17RzfoCkeyQtkrRY0ic7q9NJzczSU12yV7kqpHpgCjAJGANcImlMm2KfB5ZExHHABOB7knqVq9dJzczSSdpK67yldjKwLCKWR8Q24Dbg/DZlAugnSUBfYD3QWK5S31Mzs/SST5MaKmlByfG0iJhWfD8CWFlybhUwrs33bwJmAquBfsBHIqK53AWd1MwspVQDBQ0RMbbjit4h2hyfCzwOnAkcAvxW0sMRsamjC7r7aWbpVab7uQrYv+R4JIUWWalPAndFwTJgBXBEuUqd1MwsnR3rqe3iQAEwHxgtaVTx5v/FFLqapV4EzgKQNBw4HFherlJ3P80spco8pxYRjZImA3OAemB6RCyWdGXx/FTgOuAWSU8WLsxXIqKhXL1OamaWXoXWU4uIWcCsNp9NLXm/Gnhvmjqd1MwsPU+TMrPcULanSTmpmVl6bqmZWZ7ISc3M8qKwmreTmpnlhYTqnNTMLEfcUjOzXHFSM7NccVIzs/wQ7a+vkRFOamaWipBbamaWL3V1nlFgZjnilpqZ5YfvqZlZ3rilZma54YECM8sdT5Mys/yQu59mljNOamaWK1lOatl9gs7MMmnHQEGSV6d1SRMlLZW0TNI17Zy/WtLjxddTkpokDS5Xp5OamaWnhK9yVUj1wBRgEjAGuETSmNIyEXFjRBwfEccD1wIPRcT6cvW6+2lm6ahi06ROBpZFxHIASbcB5wNLOih/CTCjs0rdUjOz1CrU/RwBrCw5XlX8rL3r9QYmAnd2VqlbamaWXvJxgqGSFpQcT4uIaWVqiQ7q+Vvgkc66nuCktkuO2qcfH3nXftRJ/GH5emY/s7bV+cOG9eHzp42i4fVtACxctZH7lrxCjzpx9ZmH0qNe1Es8tvI17ln8Si1+hG5nxdLn+f2984jm4OiTjmLchLHtlluz8hVu/eHtvP+SiRx2zGjWr9vAvTPubzm/cf1GTj373Zx42rt2V+iZkmL0syEi2v8lF1pm+5ccjwRWd1D2YhJ0PaHKSU3SROD/APXAzRHxnWpeb3eS4NITR/Dvc5ezYet2vnrOaBat3sjLm95qVe65hte56eEVrT5rbA7+be5feauxmXrBl886lKfWbGbFq2/szh+h22lububBmXO56NMX0K9/X34x5ZcceuQohgwf8o5y82Y/wkGjD2j5bPCwQXzsqktbzv/o+umMPuqQ3Rp/ViQd2UxgPjBa0ijgJQqJ69J2rjcAOAP4+ySVVu2eWpKRja5s1ODerN28jYbXt9HUHMx/8TWOGzEg8fffamwGoL5O1Nep40a3Vcyala8wcMhABg4eQH2Peg4/bjTLnl7+jnJ/+eMiRh99CL379m63nheXrWTgkAH0H9S/2iFnViXuqUVEIzAZmAM8DdweEYslXSnpypKiFwAPRMTrSWKrZkst7chGlzJwr56s37qt5fi1N7Yzasg7/xEcPKQ3/3zuYWzcup07Hl/d0pKT4GvnHMawvr2Yu+xVVqx3K63atmzaQr8BfVuO+/Xvy8srW3f7N2/cwrIlf+VDl1/IA6sebLeeZ554jiOOPayqsWZdpeZ+RsQsYFabz6a2Ob4FuCVpndUc/Uw0siHpCkkLJC14a9OGKoZTWUnucL64YSvX3vs01815lt8918DnThv1dtmA6x54lq/cs4RRg3uz34A9qxqvddAYbvOHnHvvPE6f+DcdPrLQ1NjEX59ezmHHjK54fF1JpR6+rYZqttQSjWwUR0KmAQw5+Kgu0wnbsHU7g/fq1XI8sHdPXtu6vVWZN4tdTICnXt7MpSeKvr3q2bKtqeXzrdubWbpuC0ft04/VG9+sfuDdWL/+fdm8cUvL8eZNW+jbv0+rMmteWst9M2YDsPWNN1m+9HlUV9dy/2zFs88zfL9h9OnXfte0W+jGE9rTjGx0Oc+vf4O9+/ViSJ9evLZ1OycdMJCbH32hVZn+e/Zg05uNABw0eC/qgC3bmui7Rz1NzcHW7c30rBdHDu/L7KfXtnMVq6R9Rg7ntYbX2Lh+I33792Xpouc47+JzW5X5H1/+RMv72Xf8loOPOKjVgMAzi57liOMO310hZ5Io3D7JqmomtUQjG11Vc8CMhS/xP884mDrBI8vX8/Kmtxh/SGEkbd5fX+XEkQM449ChNEWwvamZacWkN2DPnnxy3AHUqfAfx4IXN/Lky5tr+eN0C3X1dZz5gQncOf1umqOZo8cexdDhQ1j0308CcNy4Y8p+f/u27bzw3ErOueDM3RFuhmV7kUhFVK/HJ+k84PsUHumYHhHfLld+yMFHxfuuu7Vq8VjlHTBkr1qHYCn85KoLefnZp3YpI+25z2Fx4Mf/M1HZZ2+Y+FiZ59SqoqrPqbU3smFmXZy6b/fTzHJIQJ2X8zazPHFLzcxyJcsDBU5qZpaO76mZWZ4IVWqRyKpwUjOz1NxSM7Nc8T01M8sP31MzszwpzP3MblZzUjOz1DKc05zUzCw9zygws/zoxuupmVkOdef11Mwsl7K9npqTmpmlluGcVtWNV8wsj1QYKEjy6rQqaaKkpZKWSbqmgzITJD0uabGkhzqr0y01M0ulUs+plewNfA6FPU3mS5oZEUtKygwEfgBMjIgXJe3dWb1uqZlZahXaIq9lb+CI2Abs2Bu41KXAXRHxIkBEdLpDkZOamaUmJXsBQ3fs61t8XVFSTZK9gQ8DBkmaK+kxSR/rLDZ3P80stRTdz4YyG68k2Ru4B3AicBawF/CopD9FxLMdXdBJzczSqdyE9iR7A6+ikBhfB16XNA84Dugwqbn7aWapFBaJrMjoZ8vewJJ6UdgbeGabMncDp0vqIak3MA54ulylbqmZWWp1FWiqRUSjpMnAHN7eG3ixpCuL56dGxNOSZgNPAM3AzRHxVLl6ndTMLLVKPXzb3t7AETG1zfGNwI1J63RSM7NU5AntZpY3GV55qOOkJuk/eefwaouIuKoqEZlZ5nXV9dQW7LYozKzLEIUR0KzqMKlFxE9LjyX1KT4rYmbdXIYbap0/pybpFElLKD4bIuk4ST+oemRmlk0J533WajAhycO33wfOBV4FiIhFwPgqxmRmGZdi7udul2j0MyJWtsm6TdUJx8yyTlTm4dtqSZLUVko6FYjiVIar6GSagpnlW5ZHP5N0P68EPk9hSZCXgOOLx2bWDSXtema2+xkRDcBluyEWM+sistz9TDL6ebCkeyStk7RW0t2SDt4dwZlZNinhqxaSdD9vBW4H9gX2A+4AZlQzKDPLtq7+SIci4mcR0Vh8/Zwy06fMLN8Ko5/JXrVQbu7n4OLb3xe3rrqNQjL7CHDfbojNzLJIyba/q5VyAwWPUUhiO6L/TMm5AK6rVlBmlm1dcumhiBi1OwMxs65hR/czqxLNKJB0NDAG2HPHZxHxX9UKysyyrUu21HaQ9A1gAoWkNguYBPwBcFIz66aym9KSjX5eRGHPvTUR8UkK21PtUdWozCyzJKivU6JXLSTpfm6NiGZJjZL6A2sBP3xr1o1lufuZpKW2QNJA4McURkQXAn+uZlBmlm2VmvspaaKkpZKWFR8da3t+gqSNkh4vvr7eWZ1J5n5+rvh2anH/vf4R8UTn4ZpZHglVZO6npHpgCnAOhZ3Y50uaGRFL2hR9OCLen7Tecg/fnlDuXEQsTHoRM8uRyq3AcTKwLCKWA0i6DTgfaJvUUinXUvtemXMBnLkrF27PQYP2YtpHjqt0tVZFg06aXOsQLIW3Vq2tSD0p7qkNlVS6idO0iJhWfD8CWFlybhUwrp06TpG0CFgN/GNELC53wXIP374nWcxm1p0IqE+e1BoiYmyZqtpqO698IXBgRGyRdB7wG2B0uQsmGSgwM2ulQhPaVwH7lxyPpNAaaxERmyJiS/H9LKCnpKFlY0v905hZt1ehpDYfGC1pVHGrgIuBmaUFJO2jYl9X0skUctar5SpNNE3KzGyHwuMauz5SEBGNkiYDc4B6YHpELJZ0ZfH8VAoP/39WUiOwFbg4IsoufZZkmpQoLOd9cER8S9IBwD4R4WfVzLqpSk0WKHYpZ7X5bGrJ+5uAm1LFlqDMD4BTgEuKx5spPFtiZt1Ul954BRgXESdI+gtARGwo9n/NrBsS0CPD06SSJLXtxSd/A0DSMKC5qlGZWaZlOKclSmr/Afwa2FvStyncuPtaVaMys8ySKjNNqlqSzP38haTHKCw/JODvIsI7tJt1YxnOaYlGPw8A3gDuKf0sIl6sZmBmll1dfTnv+3h7A5Y9gVHAUuCoKsZlZhklqNkCkEkk6X4eU3pcXL3jMx0UN7O8q+GenkmknlEQEQslnVSNYMysa1CGdylIck/tH0oO64ATgHVVi8jMMi0PW+T1K3nfSOEe253VCcfMuoIum9SKD932jYird1M8ZtYFZHnjlXLLefcozqLvcFlvM+t+Clvk1TqKjpVrqf2Zwv2zxyXNBO4AXt9xMiLuqnJsZpZRXXpGATCYwqJsZ/L282oBOKmZdUNdeaBg7+LI51O8ncx2KLtIm5nlW4YbamWTWj3Ql2SbI5hZtyHquuhzai9HxLd2WyRm1iWIrttSy3DYZlYzgh4ZvqlWLqmdtduiMLMuo8u21CJi/e4MxMy6jiw/0pHhR+jMLKsqtfGKpImSlkpaJumaMuVOktQk6aLO6nRSM7NURCFxJHmVracwDXMKMAkYA1wiaUwH5b5LYX/QTjmpmVk6KnQ/k7w6cTKwLCKWR8Q24Dbg/HbKfYHCIhprk4TnHdrNLJXCjILE99SGSlpQcjwtIqYV348AVpacWwWMa3UtaQRwAYUZTYnWcXRSM7PUUgwTNETE2BTVtH2w//vAVyKiKenKIE5qZpZahQY/VwH7lxyPBFa3KTMWuK2Y0IYC50lqjIjfdFSpk5qZpaRKrac2HxgtaRTwEnAxcGlpgYgY1XJV6Rbg3nIJDZzUzCylHaOfu6q4XuNkCqOa9cD0iFgs6cri+ak7U6+TmpmlVqmHbyNiFjCrzWftJrOI+ESSOp3UzCwdddHlvM3M2lOp7me1OKmZWWpuqZlZrmQ3pTmpmVlKAurdUjOzPMlwTnNSM7O0hDLcAXVSM7PU3FIzs9woPNKR3azmpGZm6SRc1bZWnNTMLLUs71HgpGZmqRQWiax1FB1zUjOz1Dz6aWa5kuHep5Parvh/f1zCtd/7FU3NzXz0/FP54ife2+r8s8+vYfK3fs6iZ1bxtc++ny989GwA3nxrO++74vu8tb2RpsYmPnDWu7j2M++rxY/Q7Zx1ypFc/6WLqK+r42d3/5Hv//S3rc7377MnP7ru44wcPoj6HvXc9PMHufWeP3HogXsz/V8/1VLuwP2GcP20+5g6Y+5u/gmyoVu21CRNB94PrI2Io6t1nVppamrm6htu59c3TWa/4QM58+M3Mmn8MRxx8L4tZQb178N3vvQh7ntoUavv7tGrB3f/8Cr69t6D7Y1NTLr83zj71DGcdMyotpexCqqrEzd++cNcMPkmVr/yGr/76dXcP+9Jlq5Y01Lm8g+NZ+nyNVzyDz9iyMC+zP/VP3PH/fNZ9sJaxl/2nZZ6lsz6Nvf9flFHl8q1rN9Tq+YKIrcAE6tYf009tvh5Dt5/KAeNHEqvnj248JwTmPXQE63KDBvcjxOOOpCePepbfS6Jvr33AGB7YxPbG5syvepBXpx41EEsX9nACy+9yvbGJu767ULOO+PYVmUC6Nun8Lfp03sPNmx6g8am5lZlzjjpcJ5ftY6VazbsrtCzJeH2eLUaIa1aUouIecD6atVfay+v28iI4YNajvcbPoiX121M/P2mpmZOv/R6DnvvNUwYdwRjjz6oClFaqX2HDeClV95ORKtf2cC+wwa0KvPj2x/isIP24en7v80jM77Ktd/7FRGtNzi68L0ncuecx3ZLzFmlhK9aqPlab5KukLRA0oJ1DetqHU5ibf9Dh3Q3T+vr63j41mtZfN//YuHiF1iyrO0mOlZp7bWG2/4Zz3z3kTz57CqOnPRPjL/sem64+kP067Nny/mePeqZNP4YfvPgX6odbmbt2Pez27XUkoqIaRExNiLGDhs6rNbhJLbf3gPf8f/6+wwdUOYb7RvQrzennTiaBx9dUsnwrB2r1772jtb1mobWrevL/vbd3Fu8V7ZiVQMvrH6V0QcObzl/9qljWPTMStat37x7gs4ot9Ry6IQxB/LXF9fxwksNbNveyF2/Xcik8cd2/kWgYcNmNm5+A4Ctb25j7p+XMvqg4Z18y3bVwiUvcMgBwzhgvyH07FHPheecwP3zWt8HXbVmA+NPOhwo3BM99MDhPP9SQ8v5i84dy50PdO+uJ1CxrCZpoqSlkpZJuqad8+dLekLS48Ue3Wmd1elHOnZSjx713PDlD/PBq6bQ1BRc9oF3c+Qh+zL9zocB+NQHT+eVhk2c+fEb2Pz6m0hi6m1zefSX/8Sahk187ps/o6m5mebm4IKzT2Di6cfU+CfKv6amZr58w+3c+R+fp75e/GLmn3hm+Ro+eWHh38n/vesP3PiT2Uz5xt/zyIyvIsG/3HQ36ze+DsBee/RkwslH8MV/nVHLHyMTKtG1lFQPTAHOobCx8XxJMyOitNvyIDAzIkLSscDtwBFl623v3lAlSJoBTKCwq/IrwDci4iflvnPiiWPjkf9eUJV4rDoGnTS51iFYCm8tvZ3mN9buUkY68ph3xX/dPTdR2ZMPGfhYRIxt75ykU4BvRsS5xeNrASLi+jLlp0fEkeWuWbWWWkRcUq26zazGKnPDbASwsuR4FTDuHZeSLgCuB/YGOn1K3ffUzCyVwu2yZP8Dhu54uqH4uqJNVW29o+sYEb+OiCOAvwOu6yw+31Mzs3TSrafW0FH3k0LLbP+S45FAh882RcQ8SYdIGhoRDR2Vc0vNzFKr0ODnfGC0pFGSegEXAzNbXUc6VMUHDCWdAPQCXi1XqVtqZpaSKjKtLyIaJU0G5gD1FAYBFku6snh+KvBB4GOStgNbgY9EJ6ObTmpmllqlJgtExCxgVpvPppa8/y7w3TR1OqmZWSq1nC2QhJOamaWX4azmpGZmqXXLRSLNLL+yvPyfk5qZpeN9P80sb9z9NLPcEG6pmVnOZDinOamZ2U7IcFZzUjOz1Gq1/0ASTmpmllp2U5qTmpntjAxnNSc1M0tlxyKRWeWkZmbp+OFbM8ubDOc0JzUzS6syi0RWi5OamaWW4ZzmpGZm6XiRSDPLnwxnNSc1M0vNj3SYWa5k+Z6a9/00s3QEdQlfnVYlTZS0VNIySde0c/4ySU8UX3+UdFxndbqlZmY7YdebapLqgSnAORR2a58vaWZELCkptgI4IyI2SJoETAPGlavXSc3MUqngIpEnA8siYjmApNuA84GWpBYRfywp/ydgZGeVuvtpZqkp4QsYKmlByeuKkmpGACtLjlcVP+vIp4H7O4vNLTUzSy1FS60hIsZ2VE07n0X719N7KCS10zq7oJOamaVWoWlSq4D9S45HAqvbudaxwM3ApIh4tbNK3f00s9RSdD/LmQ+MljRKUi/gYmBmq+tIBwB3AR+NiGeTxOaWmpmlogotPRQRjZImA3OAemB6RCyWdGXx/FTg68AQ4AfF1mFjme4s4KRmZjuhUjMKImIWMKvNZ1NL3l8OXJ6mTic1M0svwzMKnNTMLLUM5zQnNTNLS94iz8zyo4IzCqrCj3SYWa64pWZmqWW5peakZmapeZFIM8sP7/tpZnmS9YECJzUzS83dTzPLFbfUzCxXMpzTnNTMbCdkOKs5qZlZKoJMT5NSRLur59aEpHXAC7WOowqGAg21DsJSyevf7MCIGLYrFUiaTeH3k0RDREzcleullamklleSFnS2sJ1li/9mXZfnfppZrjipmVmuOKntHtNqHYCl5r9ZF+V7amaWK26pmVmuOKmZWa44qVWRpImSlkpaJumaWsdjnZM0XdJaSU/VOhbbOU5qVSKpHpgCTALGAJdIGlPbqCyBW4Dd+rCoVZaTWvWcDCyLiOURsQ24DTi/xjFZJyJiHrC+1nHYznNSq54RwMqS41XFz8ysipzUqqe9Gb9+fsasypzUqmcVsH/J8UhgdY1iMes2nNSqZz4wWtIoSb2Ai4GZNY7JLPec1KokIhqBycAc4Gng9ohYXNuorDOSZgCPAodLWiXp07WOydLxNCkzyxW31MwsV5zUzCxXnNTMLFec1MwsV5zUzCxXnNS6EElNkh6X9JSkOyT13oW6bpF0UfH9zeUm20uaIOnUnbjG85LesetQR5+3KbMl5bW+Kekf08Zo+eOk1rVsjYjjI+JoYBtwZenJ4sogqUXE5RGxpEyRCUDqpGZWC05qXdfDwKHFVtTvJd0KPCmpXtKNkuZLekLSZwBUcJOkJZLuA/beUZGkuZLGFt9PlLRQ0iJJD0o6iELy/GKxlXi6pGGS7ixeY76kvyl+d4ikByT9RdKPSLCPt6TfSHpM0mJJV7Q5971iLA9KGlb87BBJs4vfeVjSERX5bVpueIf2LkhSDwrrtM0ufnQycHRErCgmho0RcZKkPYBHJD0AvAs4HDgGGA4sAaa3qXcY8GNgfLGuwRGxXtJUYEtE/O9iuVuBf4+IP0g6gMKsiSOBbwB/iIhvSXof0CpJdeBTxWvsBcyXdGdEvAr0ARZGxJckfb1Y92QKG6JcGRHPSRoH/AA4cyd+jZZTTmpdy16SHi++fxj4CYVu4Z8jYkXx8/cCx+64XwYMAEYD44EZEdEErJb0u3bqfzcwb0ddEdHRumJnA2OkloZYf0n9ite4sPjd+yRtSPAzXSXpguL7/Yuxvgo0A78sfv5z4C5JfYs/7x0l194jwTWsG3FS61q2RsTxpR8U/3G/XvoR8IWImNOm3Hl0vvSREpSBwm2LUyJiazuxJJ53J2kChQR5SkS8IWkusGcHxaN43dfa/g7MSvmeWv7MAT4rqSeApMMk9QHmARcX77ntC7ynne8+CpwhaVTxu4OLn28G+pWUe4BCV5BiueOLb+cBlxU/mwQM6iTWAcCGYkI7gkJLcYc6YEdr81IK3dpNwApJHypeQ5KO6+Qa1s04qeXPzRTuly0sbh7yIwot8l8DzwFPAj8EHmr7xYhYR+E+2F2SFvF29+8e4IIdAwXAVcDY4kDEEt4ehf0XYLykhRS6wS92EutsoIekJ4DrgD+VnHsdOErSYxTumX2r+PllwKeL8S3GS6RbG16lw8xyxS01M8sVJzUzyxUnNTPLFSc1M8sVJzUzyxUnNTPLFSc1M8uV/w/Z7Z8AIloM0AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(clf_rf_char, model_name=\"clf_rf_char_newFeatures\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_M05gIqmUCFp"
   },
   "source": [
    "#### **5.2.2 Neural Network**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_nn = MLPClassifier(alpha=1e-5,\n",
    "                       hidden_layer_sizes=(150,100,50),\n",
    "                       max_iter=300,\n",
    "                       activation = 'relu',\n",
    "                       solver='adam',\n",
    "                       random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search_nn = GridSearchCV(clf_nn, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV] END .................................................... total time= 8.0min\n",
      "[CV] END .................................................... total time=32.7min\n",
      "[CV] END .................................................... total time= 8.5min\n",
      "[CV] END .................................................... total time=14.0min\n",
      "[CV] END .................................................... total time=11.2min\n"
     ]
    }
   ],
   "source": [
    "clf_nn_char = grid_search_nn.fit(X_train_char, y_train_char)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.9974104445403539\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9958    0.9964    0.9961      3067\n",
      "           1     0.9982    0.9979    0.9981      6201\n",
      "\n",
      "    accuracy                         0.9974      9268\n",
      "   macro avg     0.9970    0.9972    0.9971      9268\n",
      "weighted avg     0.9974    0.9974    0.9974      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAX/UlEQVR4nO3de5QV5Z3u8e/T3dwMN7VBlIui8YbiBbmoGMfEUSGZFcfLOHibMZelHmPMSnLioCvRk5jMeHSY0SM4DDEsk+MRBqOZoBLRTEYxIBFBRSAhQVREjNARjBcS7eZ3/ti7cdN2766C3r2rq5/PWnutXXu/9da7YfHwvlX1vqWIwMwsL2qq3QAzs47kUDOzXHGomVmuONTMLFccamaWK3XVbkAp1fUJ9exX7WZYCscfOaLaTbAUXnnlZRoaGrQnddT2PzCicXuisrF9y8KImLQnx0srW6HWsx+9Dr+g2s2wFBb/anq1m2ApTJwwdo/riMbtif+d/um5GfV7fMCUMhVqZtYVCJTdM1cONTNLR0BNbbVb0SaHmpmlpz06LVdRDjUzS8nDTzPLG/fUzCw3hHtqZpYnck/NzHLGVz/NLD98ocDM8kR4+GlmOeOempnlh4efZpYnAmp9ocDM8sTn1MwsPzz8NLO8cU/NzHLFPTUzyw15mpSZ5Y2nSZlZfvhCgZnljYefZpYbXk/NzPLFw08zyxtfKDCzXPE5NTPLDXn4aWZ5456ameWJHGpmlheF1bwdamaWFxKqcaiZWY64p2ZmueJQM7NccaiZWX6o+Mooh5qZpSLknpqZ5UtNjWcUmFmOuKdmZvmR8XNq2e1DmllmSUr0SlDPJElrJa2TNLWV7wdIelDS85JWS/pce3U61MwsleYLBXsaapJqgRnAZGAUcKGkUS2KfQlYExHHAqcB0yT1LFevQ83MUlONEr3aMR5YFxHrI+J9YC5wdosyAfRTISH7Am8CjeUq9Tk1M0tHHXahYCjwasn2RmBCizLTgfnAJqAf8LcRsaNcpe6pmVlqKYaf9ZKeKXldXlpNK1VHi+2zgOeAA4DjgOmS+pdrm3tqZpZaip5aQ0SMbeO7jcDwku1hFHpkpT4H3BwRAayT9BJwBPB0Wwd0T83MUumoCwXAMuBQSSOLJ/+nUBhqltoAnA4gaT/gcGB9uUrdUzOz9DrglFpENEq6GlgI1AKzI2K1pCuL388EbgLulvRC8aj/EBEN5ep1qJlZOuq4aVIRsQBY0OKzmSXvNwFnpqnToWZmqXmalJnlS3YzzaFWSXd862LOOuVoGra+zclT/rHazelWfr5kDddN+zFNO3Zw6dkn89XLdh3BRARTp/2Yxxavpk/vntx546Uce8Twsvt+798eYsGildRIDNqnHzNuvIT9Bw0EYNXvXuNr/zSHt9/5E6oRv/jhtfTu1aNTf3NnynJPraJXP9ub15V3cx5ayvnXzKh2M7qdpqYdfOOWedx3+1UsnfdN7n90Ob9Z//ouZR5bsoYXN2xh+QM3ctv1F/L1m+e2u++XLz2dxXOu58l7r+OsU47mlrt+BkBjYxNX3PBDpk2dwlPzvslDM79Cj7razv3RnSjplc9qBV/FQi3hvK5cW/Lsi2z943vVbka3s3z1yxw8vJ6DhtXTs0cd554xhgVPrNylzIInVjLlM+ORxLjRI3nr7e38vuGtsvv279tn5/7vbv/zzn+0v/jVbzjq40MZfdgwAPYZ2Jfa2nzfLZXlUKvk8HPnvC4ASc3zutZU8JhmvL7lLYbut/fO7QP225vlq15uUWbbrmUGD+T1zdva3femO+cz9+Gn6d+3Dw/OvAaAF1/ZjATnfXk6DVvf4dwzT+Arf3dGZX5cRmT5EXmV/O+ktXldQ1sWknR58xSKaNxeweZYd1G4+XxXLTsNrRRBUrv7fuuqz7L64e/yN5PG8v15iwBobGpi6fPrmXXTZfzsrq/x8OPP88TTa/foN2RdlntqlQy1JPO6iIhZETE2Isaqrk8ru5ilc8Dggbz2xtad25ve2MqQ+gHly2zexpBBAxLtC3D+pHHM/8Vzhbr2G8jE4z/OvgP7slfvnpxx8lE8v/bVj+yTG+q+oZZkXpdZhxsz6kBe3LCFV15r4P0PGnngsRVMPvWYXcpMPnU0cx9+mohg2Qsv0b9vH4bUDyi774sbNu/c/5FFKznsoP0AOP3EUaxe9xrv/el9GhubWLxiHYePHNJ5P7iTiULvNcmrGip5Tm3nvC7gNQrzui6q4PEy567vXsbEEw5l34F9WfXQTdw8awH3zH+q2s3Kvbq6Wm659gLOu2YGTU3BxZ89kSMP2Z/Z9z8JwOfP+wRnTjyKxxavZsw536ZP7x7MuOGSsvsCfHv6T/ndK5upqRHDh+zDv1w3BYCB/ffiqos+xel/dwtInDHxKM465ejq/PhOke2nSam1cwgdVrn0aeA2PpzX9b1y5Wv2Ghy9Dr+gYu2xjrd12fRqN8FSmDhhLMuXP7NHidR7yGFx4N/fkajsb2+ZtLzMKh0VUdGbb1ub12VmXVwVh5ZJeEaBmaUioCbDt3Q41MwsNffUzCxXsnyhwKFmZun4nJqZ5YlQhy0SWQkONTNLzT01M8sVn1Mzs/zwOTUzy5PC3M/spppDzcxSy3CmOdTMLD3PKDCz/JCHn2aWI83rqWWVQ83MUsr2emoONTNLLcOZ5lAzs5TkCwVmliO+T83McsehZma5kuFMc6iZWXruqZlZfnhCu5nlSWGRyOymmkPNzFKryXBXLbtr8ppZZknJXu3Xo0mS1kpaJ2lqG2VOk/ScpNWSnmivTvfUzCwVddCEdkm1wAzgDGAjsEzS/IhYU1JmIHAnMCkiNkga3F697qmZWWo1SvZqx3hgXUSsj4j3gbnA2S3KXAQ8EBEbACJic3uVttlTk3QHEG19HxHXtNtkM8ulFBcK6iU9U7I9KyJmFd8PBV4t+W4jMKHF/ocBPSQ9DvQDbo+IH5U7YLnh5zNlvjOzbkoUroAm1BARY8tU1VLLjlQdcAJwOtAHeErS0oj4bVsHbDPUIuKHuxxd+lhEvNtWeTPrPjrojo6NwPCS7WHAplbKNBSz511Ji4BjgTZDrd1zapJOkrQG+HVx+1hJd6ZsvJnlhQrrqSV5tWMZcKikkZJ6AlOA+S3K/BT4hKQ6SXtRGJ7+ulylSa5+3gac1XywiHhe0qkJ9jOznOqI29QiolHS1cBCoBaYHRGrJV1Z/H5mRPxa0iPASmAHcFdErCpXb6JbOiLi1Rap27Q7P8LMuj7RcTffRsQCYEGLz2a22L4VuDVpnUlC7VVJJwNR7CJeQzvdPzPLtyxPk0pyn9qVwJcoXH59DTiuuG1m3VDS2QTVmknVbk8tIhqAizuhLWbWRXTpuZ+SDpb0oKQtkjZL+qmkgzujcWaWTUr4qoYkw897gXnA/sABwH3AnEo2ysyyrYNu6aiIJKGmiPi/EdFYfN1DmelTZpZvhaufHTL3syLKzf3cp/j2v4tLgsylEGZ/CzzcCW0zsyxS110kcjmFEGtu/RUl3wVwU6UaZWbZ1iWfURARIzuzIWbWNTQPP7Mq0YwCSUcDo4DezZ+1t/yHmeVXl+ypNZN0I3AahVBbAEwGfgk41My6qexGWrKrn+dTWMvo9xHxOQrLfvSqaKvMLLMkqK1Rolc1JBl+bo+IHZIaJfUHNgO++dasG+vSw0/gmeLDD75P4YroO8DTlWyUmWVbhjMt0dzPq4pvZxbXNeofESsr2ywzyyqhTM/9LHfz7Zhy30XEiso0ycwyrYorcCRRrqc2rcx3AXyqg9vC8UeOYPGvpnd0tVZBe4+7utpNsBT+vHZDh9TTJc+pRcQnO7MhZtY1CKjtiqFmZtaWLj+jwMyslEPNzHKjsFR3dlMtycq3knSJpBuK2yMkja9808wsq7K8nlqSaVJ3AicBFxa33wZmVKxFZpZ5XfrBK8CEiBgj6VmAiNhafFSemXVDAuoyPPxMEmofSKqluIS3pEEUnpRsZt1UhjMtUaj9H+AnwGBJ36Owasc3K9oqM8ssqYtOk2oWEf9P0nIKyw8J+OuI8BPazbqxDGdaokUiRwDvAQ+WfhYRHTPfwsy6nK5+n9rDfPgAlt7ASGAtcFQF22VmGSWo2gKQSSQZfo4u3S6u3nFFG8XNLO+qeA9aEqlnFETECknjKtEYM+salOGnFCQ5p/a1ks0aYAywpWItMrNMy8Mj8vqVvG+kcI7t/so0x8y6gi4basWbbvtGxDc6qT1m1gVkeUJ7ueW86yKisdyy3mbW/RQekVftVrStXNOanxj1nKT5ki6VdG7zqzMaZ2bZVFOcVdDeqz2SJklaK2mdpKllyo2T1CTp/PbqTHJObR/gDxSeSdB8v1oADyTY18xypqMuFBRPb80AzgA2AsskzY+INa2U+9/AwiT1lgu1wcUrn6v4MMyaRYq2m1nOdNAptfHAuohYX6hTc4GzgTUtyn2ZwsXJRLeSlQu1WqAvtHpDikPNrNsSNcnvU6uX9EzJ9qyImFV8PxR4teS7jcCEXY4kDQXOoTBS3ONQez0ivpOkEjPrPkSqnlpDRIwtU1VLLTtMtwH/EBFNSa+4lgu17F6zNbPqEdR1zI1qG4HhJdvDgE0tyowF5hYDrR74tKTGiPjPtiotF2qn7147zSzPUvbUylkGHCppJPAaMAW4qLRARIzceVzpbuChcoEG5R9m/OYeNNbMcqwjFoks3gd7NYWrmrXA7IhYLenK4vczd6dePyLPzFLrqAkFEbEAWNDis1bDLCIuS1KnQ83MUhHJHkNXLQ41M0tHHTP8rBSHmpmlUphR4FAzsxzJbqQ51MxsN2S4o+ZQM7O01DXXUzMza42vfppZ7vhCgZnlh7roct5mZq3x8NPMcsc9NTPLlexGmkPNzFISUOuempnlSYYzzaFmZmkJZXgA6lAzs9TcUzOz3Cjc0pHdVHOomVk6ck/NzHLG06TMLDcKi0RWuxVtc6iZWWq++mlmuZLh0adDLYmfL1nDddN+TNOOHVx69sl89bIzd/k+Ipg67cc8tng1fXr35M4bL+XYI4aX3fdbt/+EhU+uokePWkYOq2fGDZcwoN9evLntHf5+6g94ds0rXPhXJ3LrtRd0+u/tzu741sWcdcrRNGx9m5On/GO1m5NZWe6pVWyyvaTZkjZLWlWpY3SGpqYdfOOWedx3+1UsnfdN7n90Ob9Z//ouZR5bsoYXN2xh+QM3ctv1F/L1m+e2u+8nJxzBkrnXs3jO9RwyYjD/cvejAPTq1YPrr/wrvvOVczr3hxoAcx5ayvnXzKh2MzKt+Zxaklc1VHIFkbuBSRWsv1MsX/0yBw+v56Bh9fTsUce5Z4xhwRMrdymz4ImVTPnMeCQxbvRI3np7O79veKvsvp868Ujq6moBGHf0SDa9sQ2Aj/XpxUnHHULvnj069XdawZJnX2TrH9+rdjOyTaIm4asaKhZqEbEIeLNS9XeW17e8xdD99t65fcB+e/P6lrdalNm2a5nBA3l987ZE+wLcM/8p/vLkURVovVllKOGrGqp+Tk3S5cDlAMNHjKhyaz4qIj7yWcv/gFopgqRE+/7z7Eeoq6vhgsnj9qSZZp0m68/9rPoClhExKyLGRsTYQfWDqt2cjzhg8EBee2Przu1Nb2xlSP2A8mU2b2PIoAHt7jvnoaU8+stVzLrpskwvumfWUpZ7alUPtawbM+pAXtywhVdea+D9Dxp54LEVTD71mF3KTD51NHMffpqIYNkLL9G/bx+G1A8ou+/Pl6zh9h/9nHunXcFevXtW46eZ7b4Mp1rVh59ZV1dXyy3XXsB518ygqSm4+LMncuQh+zP7/icB+Px5n+DMiUfx2OLVjDnn2/Tp3YMZN1xSdl+Aa2+dx5/fb+ScL00HYOzog/jX6y4E4JjP3sDb7/6JDz5oZMETK7n/ji9xxMH7V+HXdz93ffcyJp5wKPsO7Muqh27i5lkLuGf+U9VuVuZkefip1s77dEjF0hzgNKAeeAO4MSJ+UG6fE04YG4t/9UxF2mOVsfe4q6vdBEvhz2vnseO9zXuUSEeOPj5+9NPHE5Udf8jA5RExdk+Ol1bFemoRcWGl6jazKstuR83DTzNLp3C6LLup5lAzs3Qyvp6ar36aWWoddfFT0iRJayWtkzS1le8vlrSy+Foi6dj26nRPzcxSUofcVympFpgBnAFsBJZJmh8Ra0qKvQT8RURslTQZmAVMKFevQ83MUuug4ed4YF1ErC/UqbnA2cDOUIuIJSXllwLD2qvUw08zSyXp0DNB7g0FXi3Z3lj8rC1fAH7WXqXuqZlZesl7avWSSm8+nRURs8rU0uqNs5I+SSHUTmnvgA41M0stxS0dDWVuvt0IDC/ZHgZs+sixpGOAu4DJEfGH9g7o4aeZpSYle7VjGXCopJGSegJTgPm7HkcjgAeASyPit0na5p6amaXTQfepRUSjpKuBhUAtMDsiVku6svj9TOAGYF/gzuIV18b2pl051MwstY6aURARC4AFLT6bWfL+i8AX09TpUDOzVES2ZxQ41MwstQxnmkPNzHZDhlPNoWZmqWV5kUiHmpmllt1Ic6iZ2e7IcKo51MwsFS8SaWb5kvFFIh1qZpZahjPNoWZmaXXMIpGV4lAzs9QynGkONTNLp4oPX0/EoWZm6WU41RxqZpaab+kws1zxOTUzyw9BjUPNzPIlu6nmUDOzVLxIpJnlToYzzaFmZum5p2ZmueJpUmaWK9mNNIeamaWU8EHFVeNQM7PUPKPAzPIlu5nmUDOz9DKcaQ41M0tLfkSemeVH1mcU1FS7AWZmHck9NTNLLcs9NYeamaXmWzrMLD98862Z5UnWLxQ41MwsNQ8/zSxX3FMzs1zJcKY51MxsN2Q41RxqZpaKINPTpBQR1W7DTpK2AK9Uux0VUA80VLsRlkpe/84OjIhBe1KBpEco/Pkk0RARk/bkeGllKtTyStIzETG22u2w5Px31nV57qeZ5YpDzcxyxaHWOWZVuwGWmv/OuiifUzOzXHFPzcxyxaFmZrniUKsgSZMkrZW0TtLUarfH2idptqTNklZVuy22exxqFSKpFpgBTAZGARdKGlXdVlkCdwOderOodSyHWuWMB9ZFxPqIeB+YC5xd5TZZOyJiEfBmtdthu8+hVjlDgVdLtjcWPzOzCnKoVU5rM359/4xZhTnUKmcjMLxkexiwqUptMes2HGqVsww4VNJIST2BKcD8KrfJLPccahUSEY3A1cBC4NfAvIhYXd1WWXskzQGeAg6XtFHSF6rdJkvH06TMLFfcUzOzXHGomVmuONTMLFccamaWKw41M8sVh1oXIqlJ0nOSVkm6T9Jee1DX3ZLOL76/q9xke0mnSTp5N47xsqSPPHWorc9blHkn5bH+l6T/mbaNlj8Ota5le0QcFxFHA+8DV5Z+WVwZJLWI+GJErClT5DQgdaiZVYNDret6Evh4sRf135LuBV6QVCvpVknLJK2UdAWACqZLWiPpYWBwc0WSHpc0tvh+kqQVkp6X9F+SDqIQnl8t9hI/IWmQpPuLx1gmaWJx330lPSrpWUn/ToLneEv6T0nLJa2WdHmL76YV2/JfkgYVPztE0iPFfZ6UdESH/GlabvgJ7V2QpDoK67Q9UvxoPHB0RLxUDIa3ImKcpF7AYkmPAscDhwOjgf2ANcDsFvUOAr4PnFqsa5+IeFPSTOCdiPjnYrl7gX+NiF9KGkFh1sSRwI3ALyPiO5I+A+wSUm34fPEYfYBlku6PiD8AHwNWRMTXJd1QrPtqCg9EuTIifidpAnAn8Knd+GO0nHKodS19JD1XfP8k8AMKw8KnI+Kl4udnAsc0ny8DBgCHAqcCcyKiCdgk6Ret1H8isKi5rohoa12xvwRGSTs7Yv0l9Sse49zivg9L2prgN10j6Zzi++HFtv4B2AH8R/Hze4AHJPUt/t77So7dK8ExrBtxqHUt2yPiuNIPiv+43y39CPhyRCxsUe7TtL/0kRKUgcJpi5MiYnsrbUk8707SaRQC8qSIeE/S40DvNopH8bjbWv4ZmJXyObX8WQj8D0k9ACQdJuljwCJgSvGc2/7AJ1vZ9yngLySNLO67T/Hzt4F+JeUepTAUpFjuuOLbRcDFxc8mA3u309YBwNZioB1BoafYrAZo7m1eRGFY+0fgJUl/UzyGJB3bzjGsm3Go5c9dFM6XrSg+POTfKfTIfwL8DngB+DfgiZY7RsQWCufBHpD0PB8O/x4Ezmm+UABcA4wtXohYw4dXYb8NnCppBYVh8IZ22voIUCdpJXATsLTku3eBoyQtp3DO7DvFzy8GvlBs32q8RLq14FU6zCxX3FMzs1xxqJlZrjjUzCxXHGpmlisONTPLFYeameWKQ83McuX/A6NbQooRPF5DAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.7079556898288016\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.5696    0.5311    0.5497      1333\n",
      "           1     0.7710    0.7973    0.7839      2639\n",
      "\n",
      "    accuracy                         0.7080      3972\n",
      "   macro avg     0.6703    0.6642    0.6668      3972\n",
      "weighted avg     0.7034    0.7080    0.7053      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZlUlEQVR4nO3deZRV5Znv8e+vCkSZRxHFARVQnBUc0q0hRg16b+LVTA59051hGdNN7LYT0yarVwaT3NuJ7bqaFi9q9NrGiMF2ACMOSdp56DAEBzAg4sAgMiOTQlU994+zC04V1Kmz5Zw6u3b9Pq6z1tl7v/Xup0Af33fv/T5bEYGZWV7U1ToAM7NKclIzs1xxUjOzXHFSM7NccVIzs1zpVusAivXoMyB6Dx5W6zAshR7d62sdgqWw/r1lbNmwVnvSR33fgyMatpbVNraueiwiJuzJ+dLKVFLrPXgYn7rm7lqHYSmMHtqr1iFYCjdPvHCP+4iGrfQY/YWy2n4wd9LgPT5hSplKambWGQiU3StXTmpmlo6AuuxednBSM7P0tEeX5arKSc3MUvL008zyxiM1M8sN4ZGameWJPFIzs5zx3U8zyw/fKDCzPBGefppZznikZmb54emnmeWJgHrfKDCzPPE1NTPLD08/zSxvPFIzs1zxSM3MckNeJmVmeeNlUmaWH75RYGZ54+mnmeWG66mZWb54+mlmeeMbBWaWK76mZma5IU8/zSxvPFIzszyRk5qZ5UWhmreTmpnlhYTqnNTMLEeyPFLL7i0MM8ssSWV9yuhngqQFkhZJuno3x6+SNDf5vCqpUdLAUn06qZlZapVIapLqgUnAucAY4GJJY4rbRMS1EXF8RBwPfBd4KiLWlurXSc3M0lGKT2knA4siYnFEbAPuAc4v0f5iYEp7nTqpmVkqorxRWjJSGyxpVtHnsqKuDgCWFG0vTfbtek6pJzABuK+9+HyjwMxSq6srezy0OiLGtnFsd2O5aKPtp4Hn2pt6gpOamX0EFbr7uRQ4sGh7OLC8jbYXUcbUEzz9NLO0KndNbSYwUtIISXtRSFzTdzmd1A/4ODCtnPA8UjOz1CoxUouIBkkTgceAeuD2iJgn6fLk+OSk6QXA4xGxuZx+ndTMLJXmGwWVEBEzgBmt9k1utX0HcEe5fTqpmVlqXiZlZvmhbC+TclIzs9Sc1MwsV5zUzCw3KnmjoBqc1MwsvezmNCc1M0tJqZZJdTgnNTNLzdNPM8uX7OY0J7U9cfSwPlxy4gFI4pk31jDjtZUtjo/etzffPH0EqzdvA2D2kvU8NO89utWJq886nO51ddTVwax3NjDt1RW1+BW6nMUL3uL305+iKYLjxh3FaZ8Yt9t27y5ZwZ2TpnL+JedyxLEjWbNqHdN+vfPB9/Vr3+f0s09l3OkndFTomdJlR2qSJgA3UFjX9cuI+Jdqnq8jSfBXJw3nuifeYO3W7Xz/nFHMXbaB5e9/2KLd66s2ccPTb7bY19AUXPufb/BhQxP1gu+eNZJX3n2fxWu2dOSv0OU0NTXx+INPctHXLqBPv97cceM9jBxzKIOHDtql3ROPPMeIUQft2DdoyAC+8g+X7jg+6ae3Merowzo0/qwot1R3rVTtal85pXo7s0MH9mTlpg9ZtXkbjU3Bf72zjuOH9yv75z9saAKgvk7UZ3jJSZ68u+Q9BgzqR/9B/ajvVs+Y40bx+vzFu7Sb/dxLjD76cHr27rnbft5etIT+g/rRb0DfaoecWZV6R0E1VPMWRtpSvZ1K/57dWbtl+47tdVu2M2Cf7ru0O2xwL340YTRXfvxQ9u+79479Evxwwmiuv+Bo5q3Y6FFaB9i4YRN9+vfZsd2nX282bti0S5uF897ghFOPabOf+S8tZMzxo6sWZ2egOpX1qYVqJrWySvVKuqy51O8HG9dXMZzKKqdk59trt3DV9Pn84NEF/H7hKr55xoidbQN++OgCvjVtPiMG9eSAfntjNdBqNPH7h55i/Ll/0eYjC40NjSyav5gjjjm8I6LLrCyP1Kp5Ta2sUr0RcQtwC8CgEWPaKuWbOeu2bGdgz50jswE9u7N+6/YWbT5IppgAr7y7kXqJ3nvVs2lb4479W7c3smDlJo4e1odlGz6ofuBdWJ9+vdm4fuOO7Y0bNtGnb68WbVYsXcm0KY8AsHXzByz+81vU1dcx6qjC9bM3FrzF0AP2pVeflj/XpXThBe1pSvV2Om+u3cLQPj0Y3Gsv1m3dzikHDeDm599u0abv3t14/4MGAEYM7IkEm7Y10qdHPQ1NhYTWvV6MGdqHR1rdObXKGzZ8KGvXrGf92g306dub+S8t5DMXTWjR5htXf3nH999OfZzDjxixI6EBvDZ3IWOOG9VhMWeR2GWAmynVTGo7SvUCyyiU6r2kiufrUE0Bd81ayj+OP5Q6iWcXr2X5+x8w/vDCnbQnF61h7IH9+cTIQTQ1wbbGJiY//xYA/fbpzldPPYg6CQEz31nPS8vfr90v00XU1ddxzvnj+c1tDxJNwbHjxjBkv0H86cWXATjh1GNL/vz2bdt5c9E7fOrCMzsi3AzL9t1PRVRvxifpPOB6dpbq/Wmp9oNGjIlPXXN31eKxyhs9tAtPwzqhmydeyPKFr+xRRtp7v1Fx8F//W1ltF/58wuwSb5Oqiqo+p7a7Ur1m1smp604/zSyHBNRl+NlKJzUzS80jNTPLlSzfKHBSM7N0fE3NzPJEyEUizSxfPFIzs1zxNTUzy4+MX1PL7sTYzDKpsPazMlU6JE2QtEDSIklXt9FmvKS5kuZJeqq9Pj1SM7PUKjFSKyokezaFAhgzJU2PiPlFbfoDNwETIuIdSfu216+TmpmlVqEVBTsKyQJIai4kO7+ozSXA/RHxDkBEtFvOxtNPM0tHFZt+llNIdhQwQNKTkmZL+lJ7nXqkZmappKynNljSrKLtW5LCsM1dtda6bFA34CTgk8A+wAuSXoyIhW2d0EnNzFJKVU9tdYnSQ+UUkl2a9LEZ2CzpaeA4oM2k5umnmaUmlfdpx45CspL2olBIdnqrNtOA0yV1k9QTOAV4rVSnHqmZWTqqzI2CiGiQNBF4jJ2FZOdJujw5PjkiXpP0KPAy0ETh/cGvlurXSc3MUml+Tq0SdldINiImt9q+Fri23D6d1MwsNS+TMrNcyXBOc1Izs/Q8UjOz/Mj4gnYnNTNLpVAkMrtZzUnNzFKry/BQzUnNzFLLcE5zUjOzdCTfKDCznMnwJbW2k5qkf2PXFfM7RMQVVYnIzDKvs94omFXimJl1UaJwBzSr2kxqEfHvxduSeiXlP8ysi8vwQK390kOSTpM0n6Tch6TjJN1U9cjMLJvKrHpbq5sJ5dRTux74FLAGICJeAs6oYkxmlnEVqqdWFWXd/YyIJa2ybmN1wjGzrBOd/+HbJZI+BkRSnfIK2qk8aWb5luW7n+VMPy8H/o7CW16WAccn22bWBZU79czs9DMiVgOXdkAsZtZJZHn6Wc7dz0MlPSRplaSVkqZJOrQjgjOzbFKZn1ooZ/p5NzAVGAbsD9wLTKlmUGaWbZ39kQ5FxK8ioiH53EWJ5VNmlm+Fu5/lfWqh1NrPgcnXJyRdDdxDIZl9EXi4A2IzsyxS5y0SOZtCEmuO/utFxwL4cbWCMrNs65SlhyJiREcGYmadQ/P0M6vKWlEg6WhgDLB3876IuLNaQZlZtnXKkVozST8AxlNIajOAc4FnASc1sy4quymtvLufnwM+CayIiC8DxwE9qhqVmWWWBPV1KutTC+VMP7dGRJOkBkl9gZWAH74168KyPP0sZ6Q2S1J/4FYKd0TnAH+sZlBmlm2VWvspaYKkBZIWJY+OtT4+XtIGSXOTz/fb67OctZ9/m3ydLOlRoG9EvNx+uGaWR0IVWfspqR6YBJwNLAVmSpoeEfNbNX0mIv57uf2Wevj2xFLHImJOuScxsxypXAWOk4FFEbEYQNI9wPlA66SWSqmR2nUljgVw5p6ceHcOGdiT2y8+vtLdWhUNGDex1iFYCh8ufa8i/aS4pjZYUvFLnG6JiFuS7wcAS4qOLQVO2U0fp0l6CVgOfDsi5pU6YamHbz9RXsxm1pUIqC8/qa2OiLElumqt9bryOcDBEbFJ0nnAg8DIUics50aBmVkLFVrQvhQ4sGh7OIXR2A4R8X5EbEq+zwC6SxpcMrbUv42ZdXkVSmozgZGSRiSvCrgImF7cQNJ+Sua6kk6mkLPWlOq0rGVSZmbNCo9r7PmdgohokDQReAyoB26PiHmSLk+OT6bw8P83JDUAW4GLIqJk6bNylkmJQjnvQyPiGkkHAftFhJ9VM+uiKrVYIJlSzmi1b3LR9xuBG1PFVkabm4DTgIuT7Y0Uni0xsy6qU794BTglIk6U9CeAiFiXzH/NrAsS0C3Dy6TKSWrbkyd/A0DSEKCpqlGZWaZlOKeVldR+ATwA7CvppxQu3P1zVaMys8ySKrNMqlrKWfv5a0mzKZQfEvA/IsJvaDfrwjKc08q6+3kQsAV4qHhfRLxTzcDMLLs6eznvh9n5Apa9gRHAAuCoKsZlZhklqFkByHKUM/08png7qd7x9Taam1ne1fCdnuVIvaIgIuZIGleNYMysc1CG31JQzjW1fyzarANOBFZVLSIzy7Q8vCKvT9H3BgrX2O6rTjhm1hl02qSWPHTbOyKu6qB4zKwTyPKLV0qV8+6WrKJvs6y3mXU9hVfk1TqKtpUaqf2RwvWzuZKmA/cCm5sPRsT9VY7NzDKqU68oAAZSKMp2JjufVwvASc2sC+rMNwr2Te58vsrOZNasZJE2M8u3DA/USia1eqA35b0cwcy6DFHXSZ9TezcirumwSMysUxCdd6SW4bDNrGYE3TJ8Ua1UUvtkh0VhZp1Gpx2pRcTajgzEzDqPzv5Ih5lZCxnOaU5qZpaOyPZb0J3UzCwdefppZjlSWFHgpGZmOZLdlOakZmYfQYYHapm+3mdmmSSk8j7t9iRNkLRA0iJJV5doN05So6TPtdenk5qZpdJ897OcT8l+CkVoJwHnAmOAiyWNaaPdz4DHyonPSc3MUqtL3tLe3qcdJwOLImJxRGwD7gHO3027b1J4hcDKsmJL84uYmSEqNf08AFhStL002bfzVNIBwAXA5HLD840CM0sl5cO3gyXNKtq+JSJuKeqqtdZlza4H/ikiGst9L4KTmpmlluLFK6sjYmwbx5YCBxZtDweWt2ozFrgnOd9g4DxJDRHxYFsndFIzs9Qq9ETHTGCkpBHAMuAi4JLiBhExYsc5pTuA35ZKaOCkZmYpCaivwINqydvqJlK4q1kP3B4R8yRdnhwv+zpaMSc1M0utUg/fRsQMYEarfbtNZhHxN+X06aRmZikJZXihlJOamaWW5WVSTmpmlkrhkY7sZjUnNTNLRx6pmVnOuJ6ameVGoUhkraNom5OamaXmu59mlisZnn06qVXS75+fz3ev+w8am5r4n+d/jCv/5pwWx6c+MpMb7vwdAL326cF1V3+RY0YNr0WoBnzytCP539/6HPV1dfxq2vNc/++/a3G8b6+9ufnHf83woQOo71bPjXf9gbsferFG0WZLlkdqVSs9JOl2SSslvVqtc2RJY2MTV/18Kvfe8Le8OPWfue/x2fx58bst2hy8/yAevvkfeG7K97jqqxO48n9NqVG0Vlcnrv3OF/j839/EqV/4CZ895yRGj9ivRZuvff4MFixewemX/guf/voN/OTvL6B7t/oaRZwdzdfUyvnUQjXrqd0BTKhi/5kye95bHHrgYA4ZPpi9unfjwrNPZMZTL7doc8pxh9K/b08Axh0zguUr19cgUgM46ahDWLxkNW8vW8P2hkbu/90czvv4sS3aBNC7Vw8AevXswbr3t9DQ2FSDaDOmzAKRtbpDWrWkFhFPA2ur1X/WvLtqAwcMHbBje/+hA3h31YY22/9q2vOc9bFdKhdbBxk2pB/L3lu3Y3v5e+sYNqRfiza3Tn2KUYfsx2uP/JTnpnyP7173H0S0LvfVNanMTy3U/JqapMuAywAOPOigGkfz0e3uX/a2/kf1zKyF3DX9BR659coqR2Vt2V09sNZ/hWeeeiSvLFzKZ77xC0YMH8wDN07khblvsHHzBx0UZTZl/b2fNS/nHRG3RMTYiBg7ZPCQWofzke2/b/9d/s+/3+B+u7R79fVlXPGTu/n1v17GwP69OzJEK7J85fpdRtYrVrccWV/66VP57RMvAfDm0tW8vXwNIw8e2qFxZlWWR2o1T2p5ceKYg3njnVW8vWw127Y3cP/v5nDuGS2v0SxZsZYvfedWJv/oSxzu/zhqas78tznsoCEctP8guner58KzT+SRp1teA126Yh1njBsNwJCBfTj84KG8tWx1LcLNngxntZpPP/OiW7d6fv6dL/DZKybR2Bhc+plTOfKwYdx+3zMAfOWzp3PtLx9h7YbNfPtnv0l+po4n7vynWobdZTU2NvGdn0/lvl/8HfX14tfTX+TPi1fw5Qv/EoD/d/+zXHvbo0z6wV/x3JTvIcGPbpzG2g2baxx5NmR5+qlqXfiUNAUYT6Gu+HvADyLitlI/c9JJY+O5/5pVqollzIBxE2sdgqXw4YKpNG1ZuUcZ6chjTog7pz1ZVtuTD+s/u8Q7CqqiaiO1iLi4Wn2bWY1ld6Dm6aeZpVO4XJbdrOakZmbpuJ6ameVNhnOak5qZpaU0LzPucE5qZpZahnOak5qZpVPL1QLlcFIzs/QynNWc1MwsNT/SYWa54mtqZpYfGX9OzVU6zCw1lflPu/1IEyQtkLRI0tW7OX6+pJclzZU0S9JfttenR2pmloqozEhNUj0wCTgbWArMlDQ9IuYXNfsDMD0iQtKxwFTgiFL9eqRmZqlVqJzaycCiiFgcEduAe4DzixtExKbYWUqoF4VXR5TkpGZm6ZWf1QYn08bmz2VFvRwALCnaXprsa3kq6QJJfwYeBr7SXmiefppZaimKRK4uUU9td53sMhKLiAeABySdAfwYOKtkbOVGZmbWrELTz6XAgUXbw4HlbTVO3lB3mKTBpTp1UjOz9CqT1WYCIyWNkLQXcBEwvcVppMOVrJ6XdCKwF7CmVKeefppZKpUqEhkRDZImAo8B9cDtETFP0uXJ8cnAZ4EvSdoObAW+GO28g8BJzczSqeDDtxExA5jRat/kou8/A36Wpk8nNTNLLcMLCpzUzCwtF4k0s5zJcE5zUjOzdFwk0szyJ8NZzUnNzFJzkUgzyxVfUzOz/BDUOamZWb5kN6s5qZlZKpUqElktTmpmllqGc5qTmpml55GameWKl0mZWa5kN6U5qZlZSsr4ez+d1MwsNa8oMLN8yW5Oc1Izs/QynNOc1MwsLaV5RV6Hc1Izs1SyvqLAr8gzs1zxSM3MUsvySM1JzcxS8yMdZpYffvjWzPIk6zcKnNTMLDVPP80sVzxSM7NcyXBOc1Izs48gw1nNSc3MUhFkepmUIqLWMewgaRXwdq3jqILBwOpaB2Gp5PXv7OCIGLInHUh6lMKfTzlWR8SEPTlfWplKanklaVZEjK11HFY+/511Xl77aWa54qRmZrnipNYxbql1AJaa/846KV9TM7Nc8UjNzHLFSc3McsVJrYokTZC0QNIiSVfXOh5rn6TbJa2U9GqtY7GPxkmtSiTVA5OAc4ExwMWSxtQ2KivDHUCHPixqleWkVj0nA4siYnFEbAPuAc6vcUzWjoh4Glhb6zjso3NSq54DgCVF20uTfWZWRU5q1bO7Fb9+fsasypzUqmcpcGDR9nBgeY1iMesynNSqZyYwUtIISXsBFwHTaxyTWe45qVVJRDQAE4HHgNeAqRExr7ZRWXskTQFeAEZLWirpq7WOydLxMikzyxWP1MwsV5zUzCxXnNTMLFec1MwsV5zUzCxXnNQ6EUmNkuZKelXSvZJ67kFfd0j6XPL9l6UW20saL+ljH+Ecb0na5a1Dbe1v1WZTynP9UNK308Zo+eOk1rlsjYjjI+JoYBtwefHBpDJIahHxtYiYX6LJeCB1UjOrBSe1zusZ4PBkFPWEpLuBVyTVS7pW0kxJL0v6OoAKbpQ0X9LDwL7NHUl6UtLY5PsESXMkvSTpD5IOoZA8r0xGiadLGiLpvuQcMyX9RfKzgyQ9LulPkm6mjPd4S3pQ0mxJ8yRd1urYdUksf5A0JNl3mKRHk595RtIRFfnTtNzwG9o7IUndKNRpezTZdTJwdES8mSSGDRExTlIP4DlJjwMnAKOBY4ChwHzg9lb9DgFuBc5I+hoYEWslTQY2RcS/Ju3uBv5PRDwr6SAKqyaOBH4APBsR10j6b0CLJNWGryTn2AeYKem+iFgD9ALmRMS3JH0/6XsihReiXB4Rr0s6BbgJOPMj/DFaTjmpdS77SJqbfH8GuI3CtPCPEfFmsv8c4Njm62VAP2AkcAYwJSIageWS/nM3/Z8KPN3cV0S0VVfsLGCMtGMg1ldSn+QcFyY/+7CkdWX8TldIuiD5fmAS6xqgCfhNsv8u4H5JvZPf996ic/co4xzWhTipdS5bI+L44h3Jf9ybi3cB34yIx1q1O4/2Sx+pjDZQuGxxWkRs3U0sZa+7kzSeQoI8LSK2SHoS2LuN5pGcd33rPwOzYr6mlj+PAd+Q1B1A0ihJvYCngYuSa27DgE/s5mdfAD4uaUTyswOT/RuBPkXtHqcwFSRpd3zy9Wng0mTfucCAdmLtB6xLEtoRFEaKzeqA5tHmJRSmte8Db0r6fHIOSTqunXNYF+Oklj+/pHC9bE7y8pCbKYzIHwBeB14B/i/wVOsfjIhVFK6D3S/pJXZO/x4CLmi+UQBcAYxNbkTMZ+dd2B8BZ0iaQ2Ea/E47sT4KdJP0MvBj4MWiY5uBoyTNpnDN7Jpk/6XAV5P45uES6daKq3SYWa54pGZmueKkZma54qRmZrnipGZmueKkZma54qRmZrnipGZmufL/AaeshScHKkAKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(clf_nn_char, model_name=\"clf_nn_char_newFeatures\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "avffFQxr_Y3i"
   },
   "source": [
    "### **5.2 TF-IDF without stopwords**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "id": "xuYH5P5h_Xsi"
   },
   "outputs": [],
   "source": [
    "tfidf_char_no_stopwords = TfidfVectorizer(\n",
    "    analyzer='char',\n",
    "    preprocessor=preprocess,\n",
    "    ngram_range=(2, 6),\n",
    "    use_idf=True,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=10000,\n",
    "    min_df=5,\n",
    "    max_df=0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "id": "hTqkaPzz_5dk"
   },
   "outputs": [],
   "source": [
    "#Construct tfidf matrix and get relevant scores\n",
    "tfidf =tfidf_char_no_stopwords.fit_transform(tweets).toarray()\n",
    "vocab = {v:i for i, v in enumerate(tfidf_char_no_stopwords.get_feature_names())}\n",
    "idf_vals = tfidf_char_no_stopwords.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "id": "EX8U6lyW_5o4"
   },
   "outputs": [],
   "source": [
    "array = np.concatenate([tfidf,pos],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "id": "1HCa_yzV_5y-"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13240, 14464)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "id": "Mzrrt-LoAmL_"
   },
   "outputs": [],
   "source": [
    "X_new_noStopWords = pd.DataFrame(array)\n",
    "X_train_char, X_test_char, y_train_char, y_test_char = train_test_split(X_new_noStopWords, y, test_size = 0.3, random_state = 42, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_nn = MLPClassifier(alpha=1e-5,\n",
    "                       hidden_layer_sizes=(150,100,50),\n",
    "                       max_iter=300,\n",
    "                       activation = 'relu',\n",
    "                       solver='adam',\n",
    "                       random_state=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "id": "V0FH8DvkAvJv"
   },
   "outputs": [],
   "source": [
    "grid_search_nn = GridSearchCV(clf_nn, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "id": "GkviYvxrAvPd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV] END .................................................... total time= 8.9min\n",
      "[CV] END .................................................... total time= 7.7min\n",
      "[CV] END .................................................... total time= 8.8min\n",
      "[CV] END .................................................... total time=10.2min\n",
      "[CV] END .................................................... total time=16.3min\n"
     ]
    }
   ],
   "source": [
    "clf_nn_chars = grid_search_nn.fit(X_train_char, y_train_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "id": "vDZYfd-pAvVA"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.9968709538195943\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9980    0.9925    0.9953      3067\n",
      "           1     0.9963    0.9990    0.9977      6201\n",
      "\n",
      "    accuracy                         0.9969      9268\n",
      "   macro avg     0.9972    0.9958    0.9965      9268\n",
      "weighted avg     0.9969    0.9969    0.9969      9268\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaXUlEQVR4nO3deZhU1b3u8e/b3aAosw2IDIrGOI8MTpFovBrwmGMcEjVqnHLVJOhzE49GcxM9iSeTHm80EQ8hHqIejRwNaFBRwZMoRtQwOCAYFFGZVGhFQgwGu/ndP6rA7ra7ujZUde3evJ/nqeepXbX22qva5GWtvfdaWxGBmVlWVFW6AWZmpeRQM7NMcaiZWaY41MwsUxxqZpYpNZVuQGOq6RLq3K3SzbAEDtxrcKWbYAksefMN6urqtCV1VHffOaJ+XVFlY92qRyNi1JYcL6l0hVrnbmyz52mVboYl8NQzv6h0EyyBIw4dvsV1RP06ttnjy0WV/fD5sbVbfMCEUhVqZtYRCJTeM1cONTNLRkBVdaVb0SqHmpklpy06LVdWDjUzS8jDTzPLGvfUzCwzhHtqZpYlck/NzDLGVz/NLDt8ocDMskR4+GlmGeOempllh4efZpYlAqp9ocDMssTn1MwsOzz8NLOscU/NzDLFPTUzywx5mpSZZY2nSZlZdvhCgZlljYefZpYZXk/NzLLFw08zyxpfKDCzTPE5NTPLDHn4aWZZ456amWWJHGpmlhW51bwdamaWFRKqcqiZWYa4p2ZmmeJQM7NMcaiZWXYo/0oph5qZJSLknpqZZUtVlWcUmFmGuKdmZtnhc2pmljVp7qmld2BsZqm08UJBMa8265JGSVooaZGkK1v4voekByS9IGm+pPPaqtM9NTNLrBTTpCRVA2OBY4FlwCxJUyJiQaNi3wQWRMQXJPUBFkq6KyLWt1ave2pmlowoVU9tBLAoIhbnQ2oicGKzMgF0U66yrsB7QH2hSt1TM7PEEpxTq5U0u9H2+IgYn38/AFja6LtlwCHN9r8ZmAKsALoBp0XEhkIHdKiZWWIJQq0uIoa1Vk0Ln0Wz7c8DzwOfA3YDpkt6MiL+2toBPfw0s0RKeKFgGTCo0fZAcj2yxs4DJkfOIuB1YM9ClTrUzCw5FfkqbBawu6QhkjoDp5Mbaja2BDgGQFI/YA9gcaFKPfw0s2RUmmlSEVEvaQzwKFANTIiI+ZIuzn8/DrgWuE3SvNyR+U5E1BWq16FmZomV6ubbiJgKTG322bhG71cAxyWp06FmZsmld0KBQ21LHHPoXvzkslOorqriv37/NDfeMb3J9z26deHm75/JkAG1fLi+nkuuvYuXF78FwEWnfZZzvng4SNxx/0zGTXy8Ar8gux57egHfvWESDRs2cPaJh/F/zmn6j31EcNUNk5g+cz5dtu3M2KvP4oA9BxXc9/zvTmDRmysBWPO3dfTo2oUZd13JkhXvcuhpP+JTg/sCMGzfXfh/V53ejr+2/aV5mlRZQ03SKOAmcuPlWyPip+U8XnuqqhLXX/ElThozlhUr3+cPt1/Ow0/OY+Hrb28qc9m5xzHvleWcfcWt7L5zP66/4kt88Zs3s9eu/Tnni4dzzLn/zvr6Bn530zeY9tR8Fi9dVcFflB0NDRu44rp7mXzzN9mpb0+OOed6Rh25H3vu2n9TmcdmLuC1pSuZPelqZr/0Bpf97L957Df/UnDfCT8+f9P+37txMt27dtm0vcuAWmbc9YlZPplU7BSoSinb1c9GUyBGA3sDZ0jau1zHa29D99mZxcvqeHPFu3xU38DkaXM4fuR+TcrsMaQ/M2YtBODVN99hcP/e9OndjU8P6cesl95g3T8+oqFhA0/NfZUTjtq/Ej8jk+bMf5MhA2vZZUAtnTvVcPJxQ3l4xrwmZabOmMfpx49AEsP3G8Jf167j7bo1Re0bEdz/2HOcctzQ9vxZqVKquZ/lUM5bOoqZAtFh9e/Tk+XvrN60vWLl+/Tv07NJmZdeXc4JRx8AwMF778ygHXuzU9+evPzaWxx+0Kfo1WM7umzTiWOP2IcB/Xq1Z/Mz7a1V7zf5e+7UtydvrXq/aZmVLZRZuaaofZ9+7jX69u7GbvnhJsCSFe/y2bN+xgkX3cTTzy0q7Q9KIVWpqFcllHP4WcwUCCRdCFwIQKeuZWxOabX0j1A0uxn6xjum85Nvn8KMO7/DgkUrePGVZTQ0bOCVN97hpjumc98vx/DBun8w/9Xl1DcUnPlhCUTze9LJ3TDapMwnblzP/TctZt9J0+Zw8uc/7qX1q+3Oi1N+SO+e2/P8y0s46/JfM3Pid5sMT7MmzcPPcoZaMVMgyM8DGw9QtV3fFv4nlU4rWviX/u1Va5qUWfvBh4y59q5N2y/c/6+8ueJdAO6c8gx3TnkGgO9//QusWPl++Ru9ldip7yd70Tv26dGsTK8Wy6z/qKHgvvX1DTz4+Av84fbLN322TedObNO5EwAH7jWYIQNreW3JKg7ae3DJf1sqKN2hVs7hZzFTIDqsuQuWsNugPgzeaQc61VTnzr082fTcS/euXehUUw3AV088nJnPv8baDz4EoLZXrlc6sF8vTjj6AH43bTZWGgfvPZjFS1fx5vI61n9Uz+Rpcxh1ZNPznaOP3JeJU/9MRDBr3ut077otO9b2aHPfx2ctZPed+zX5B61u9Voa8j3tN5bXsXjpKnYZsEP7/NgKELlebTGvSihnT23TFAhgObkpEF8p4/HaVUPDBq64/l4m/eIbVFeJux54hr8sfpvzTj4CgN9Mfoo9hvTjP645m4YNwcLX3+aSf/u413bHz75Gr+7bUd+wgcuvv4c1a9dV6qdkTk1NNddd/iVOvfQWGjYEZ37hUPbarT+/mfQnAM475TMce8Q+TJ+5gKEn/5Au23bi5u+fVXDfje6bNucTFwhmPvcaP/nVQ9RUV1FdXcUNV55Grx7bt98PbnfpvvqpaOkkQqkql44HbuTjKRA/KlS+aru+sc2ep5WtPVZ67z37i0o3wRI44tDhzJ0ze4sSadsdPx07n/PLosq+ct2oOQVW6SiLst6n1tIUCDPr4Co4tCyGZxSYWSIid/N5WjnUzCwx99TMLFPSfKHAoWZmyficmplliVBJFoksF4eamSXmnpqZZYrPqZlZdvicmpllSW7uZ3pTzaFmZomlONMcamaWnGcUmFl2pHw9NYeamSWycT21tHKomVlC6V5PzaFmZomlONMcamaWkHyhwMwyxPepmVnmONTMLFNSnGkONTNLzj01M8sOT2g3syzJLRKZ3lRzqJlZYlUp7qqld01eM0stqbhX2/VolKSFkhZJurKVMkdJel7SfElPtFWne2pmlohKNKFdUjUwFjgWWAbMkjQlIhY0KtMTuAUYFRFLJPVtq1731MwssSoV92rDCGBRRCyOiPXARODEZmW+AkyOiCUAEbGyrUpb7alJ+iUQrX0fEZe22WQzy6QEFwpqJc1utD0+Isbn3w8Aljb6bhlwSLP9Pw10kvQ40A24KSLuKHTAQsPP2QW+M7OtlMhdAS1SXUQMK1BVc807UjXAUOAYoAvwtKRnIuKV1g7YaqhFxO1Nji5tHxEftFbezLYeJbqjYxkwqNH2QGBFC2Xq8tnzgaQZwAFAq6HW5jk1SYdJWgC8nN8+QNItCRtvZlmh3HpqxbzaMAvYXdIQSZ2B04Epzcr8HjhSUo2k7cgNT18uVGkxVz9vBD6/8WAR8YKkkUXsZ2YZVYrb1CKiXtIY4FGgGpgQEfMlXZz/flxEvCzpEeBFYANwa0S8VKjeom7piIilzVK3YXN+hJl1fKJ0N99GxFRgarPPxjXbvh64vtg6iwm1pZIOByLfRbyUNrp/ZpZtaZ4mVcx9ahcD3yR3+XU5cGB+28y2QsXOJqjUTKo2e2oRUQec2Q5tMbMOokPP/ZS0q6QHJK2StFLS7yXt2h6NM7N0UpGvSihm+Plb4B6gP7ATcC9wdzkbZWbpVqJbOsqimFBTRPxXRNTnX3dSYPqUmWVb7upnSeZ+lkWhuZ+982//mF8SZCK5MDsNeKgd2mZmaaSOu0jkHHIhtrH1FzX6LoBry9UoM0u3DvmMgogY0p4NMbOOYePwM62KmlEgaV9gb2DbjZ+1tfyHmWVXh+ypbSTpGuAocqE2FRgN/AlwqJltpdIbacVd/TyV3FpGb0fEeeSW/dimrK0ys9SSoLpKRb0qoZjh57qI2CCpXlJ3YCXgm2/NtmIdevgJzM4//ODX5K6I/g34czkbZWbpluJMK2ru5zfyb8fl1zXqHhEvlrdZZpZWQqme+1no5tuDC30XEXPL0yQzS7UKrsBRjEI9tRsKfBfA50rcFg7aazBPPfvLUldrZdRr+JhKN8ES+MfCJSWpp0OeU4uIo9uzIWbWMQio7oihZmbWmg4/o8DMrDGHmpllRm6p7vSmWjEr30rSWZKuzm8PljSi/E0zs7RK83pqxUyTugU4DDgjv70WGFu2FplZ6nXoB68Ah0TEwZKeA4iI1flH5ZnZVkhATYqHn8WE2keSqskv4S2pD7knJZvZVirFmVZUqP0CuA/oK+lH5Fbt+F5ZW2VmqSV10GlSG0XEXZLmkFt+SMAXI8JPaDfbiqU404paJHIw8HfggcafRURp5luYWYfT0e9Te4iPH8CyLTAEWAjsU8Z2mVlKCSq2AGQxihl+7td4O796x0WtFDezrKvgPWjFSDyjICLmShpejsaYWcegFD+loJhzat9utFkFHAysKluLzCzVsvCIvG6N3teTO8c2qTzNMbOOoMOGWv6m264RcXk7tcfMOoA0T2gvtJx3TUTUF1rW28y2PrlH5FW6Fa0r1LSNT4x6XtIUSWdLOnnjqz0aZ2bpVJWfVdDWqy2SRklaKGmRpCsLlBsuqUHSqW3VWcw5td7Au+SeSbDxfrUAJhexr5llTKkuFORPb40FjgWWAbMkTYmIBS2U+xnwaDH1Fgq1vvkrny/xcZhtFAnabmYZU6JTaiOARRGxOFenJgInAgualbuE3MXJom4lKxRq1UBXaPGGFIea2VZLVBV/n1qtpNmNtsdHxPj8+wHA0kbfLQMOaXIkaQBwErmR4haH2lsR8cNiKjGzrYdI1FOri4hhBapqrnmH6UbgOxHRUOwV10Khlt5rtmZWOYKa0tyotgwY1Gh7ILCiWZlhwMR8oNUCx0uqj4j7W6u0UKgds3ntNLMsS9hTK2QWsLukIcBy4HTgK40LRMSQTceVbgMeLBRoUPhhxu9tQWPNLMNKsUhk/j7YMeSualYDEyJivqSL89+P25x6/Yg8M0usVBMKImIqMLXZZy2GWUScW0ydDjUzS0QU9xi6SnGomVkyKs3ws1wcamaWSG5GgUPNzDIkvZHmUDOzzZDijppDzcySUsdcT83MrCW++mlmmeMLBWaWHeqgy3mbmbXEw08zyxz31MwsU9IbaQ41M0tIQLV7amaWJSnONIeamSUllOIBqEPNzBJzT83MMiN3S0d6U82hZmbJyD01M8sYT5Mys8zILRJZ6Va0zqFmZon56qeZZUqKR58OtdY8NnMBV93wOxo2bODsEw/nW+ce1+T7iODKG37H9Kfm02XbztxyzdkcsOeggvuuXvMB5393Akveeo/B/Xvzm59cQM/u27H+o3q+9eO7ee7lJVRVVfHTy07hM0M/zdoPPuT4//3zTcdcsfJ9vjx6OD+57NT2+0NsZX75/TP5/Gf2pW71Wg4//ceVbk5qpbmnVrbJ9pImSFop6aVyHaNcGho2cPl193DvTd/gmXu+x6Rpc/jL4realJk+cwGvLVnFnMnXcON3z+Cyn05sc9+f3z6dkcP3YM7kaxg5fA9+fvs0AG6/7ykAZk78v9x38xi+d+N9bNiwgW7bb8uTv71q02tQ/96ccPSB7feH2Ard/eAznHrp2Eo3I9U2nlMr5lUJ5VxB5DZgVBnrL5s5899g10G17DKwls6dajj52IOZ+sSLTcpMfeJFTv+nEUhi+H5DWLN2HW/XrSm478NPvMgZJxwCwBknHMLUx3OfL3z9bUYO3wOAPr270aNrF557eUmT4722ZCWr3lvL4QftVu6fv1Wb+dxrrP7r3yvdjHSTqCryVQllC7WImAG8V676y+mtVWsY0K/Xpu2d+vXirVVrmpV5v2mZvj15a+X7Bfdd+d5adqztAcCOtT1YtXotAPvuPoCHZ8yjvr6BN5fX8fxflrL8ndVNjjfp0TmcfOzBqV7yxbYeKvJVCRU/pybpQuBCgEGDB1e4NTkR8YnPmmdJC0WQVNS+zZ31z4fxyhvvcPRXr2NQ/96M2H8INdXVTcpMnj6HcT/4apttNys3P/ezDRExHhgPMHTosBaiov3t1Ldnk57SindWb+phtVpm5fvs2KcH6z+qb3Xfvr278XbdGnas7cHbdWvo06sbADU11fz426ds2ue4829g10F9Nm3Pe2UZ9Q0NHLhXOkLfLL2Rlu5VeSvm4L135rUlq3hzeR3rP6pn8vS5jB65f5Myo0fux8SH/kxEMGve63Tv2oUda3sU3HfUyP24+8FnAbj7wWcZ/dnc53//cD0frPsHAH989mVqaqrYc9f+m4416dE5nHLcsPb46WbFSfH4s+I9tTSqqanmuiu+zCmXjqWhITjznw9lr936M2HSkwCcf8qRHHfEPkx/aj4Hn/QDumzbibFXn1VwX4BvnXMs5101gTunPM3Afr247acXAFD33lpOuWQsVVWif5+ejPvBOU3ac/9jc7nnpq+3419g63Xrv53LEUN3Z4eeXXnpwWv56fip3Dnl6Uo3K3XSPPxUS+eASlKxdDdwFFALvANcExH/WWifoUOHxVPPzi5Le6w8eg0fU+kmWAL/WHgPG/6+cosSaa/9Doo7fv94UWVH7NZzTkS06zCjbD21iDijXHWbWYWlt6Pm4aeZJZM7XZbeVHOomVkyKV9PzVc/zSyxUl38lDRK0kJJiyRd2cL3Z0p6Mf+aKemAtup0T83MElJJZrZIqgbGAscCy4BZkqZExIJGxV4HPhsRqyWNJndP6yGF6nWomVliJRp+jgAWRcTiXJ2aCJwIbAq1iJjZqPwzwMC2KvXw08wSKXbomc+9WkmzG70ubFTVAGBpo+1l+c9acwHwcFvtc0/NzJIrvqdWV+A+tZZqafHGWUlHkwu1z7R1QIeamSVWols6lgGDGm0PBFZ84ljS/sCtwOiIeLetSj38NLPEpOJebZgF7C5piKTOwOnAlKbH0WBgMnB2RLxSTNvcUzOzZEp0n1pE1EsaAzwKVAMTImK+pIvz348DrgZ2AG7JX3Gtb2valUPNzBIr1YyCiJgKTG322bhG778GfC1JnQ41M0tEpHtGgUPNzBJLcaY51MxsM6Q41RxqZpZYmheJdKiZWWLpjTSHmpltjhSnmkPNzBLxIpFmli0pXyTSoWZmiaU40xxqZpZUaRaJLBeHmpklluJMc6iZWTIVfPh6URxqZpZcilPNoWZmifmWDjPLFJ9TM7PsEFQ51MwsW9Kbag41M0vEi0SaWeakONMcamaWnHtqZpYpniZlZpmS3khzqJlZQkU+qLhiHGpmlphnFJhZtqQ30xxqZpZcijPNoWZmScmPyDOz7Ej7jIKqSjfAzKyU3FMzs8TS3FNzqJlZYr6lw8yywzffmlmWpP1CgUPNzBLz8NPMMsU9NTPLlBRnmkPNzDZDilPNoWZmiQhSPU1KEVHpNmwiaRXwZqXbUQa1QF2lG2GJZPW/2c4R0WdLKpD0CLm/TzHqImLUlhwvqVSFWlZJmh0RwyrdDiue/5t1XJ77aWaZ4lAzs0xxqLWP8ZVugCXm/2YdlM+pmVmmuKdmZpniUDOzTHGolZGkUZIWSlok6cpKt8faJmmCpJWSXqp0W2zzONTKRFI1MBYYDewNnCFp78q2yopwG9CuN4taaTnUymcEsCgiFkfEemAicGKF22RtiIgZwHuVbodtPoda+QwAljbaXpb/zMzKyKFWPi3N+PX9M2Zl5lArn2XAoEbbA4EVFWqL2VbDoVY+s4DdJQ2R1Bk4HZhS4TaZZZ5DrUwioh4YAzwKvAzcExHzK9sqa4uku4GngT0kLZN0QaXbZMl4mpSZZYp7amaWKQ41M8sUh5qZZYpDzcwyxaFmZpniUOtAJDVIel7SS5LulbTdFtR1m6RT8+9vLTTZXtJRkg7fjGO8IekTTx1q7fNmZf6W8Fj/KulfkrbRsseh1rGsi4gDI2JfYD1wceMv8yuDJBYRX4uIBQWKHAUkDjWzSnCodVxPAp/K96L+KOm3wDxJ1ZKulzRL0ouSLgJQzs2SFkh6COi7sSJJj0saln8/StJcSS9I+h9Ju5ALz2/le4lHSuojaVL+GLMkHZHfdwdJ0yQ9J+lXFPEcb0n3S5ojab6kC5t9d0O+Lf8jqU/+s90kPZLf50lJe5bkr2mZ4Se0d0CSasit0/ZI/qMRwL4R8Xo+GNZExHBJ2wBPSZoGHATsAewH9AMWABOa1dsH+DUwMl9X74h4T9I44G8R8e/5cr8Ffh4Rf5I0mNysib2Aa4A/RcQPJf0T0CSkWnF+/hhdgFmSJkXEu8D2wNyIuEzS1fm6x5B7IMrFEfGqpEOAW4DPbcaf0TLKodaxdJH0fP79k8B/khsW/jkiXs9/fhyw/8bzZUAPYHdgJHB3RDQAKyT9oYX6DwVmbKwrIlpbV+x/AXtLmzpi3SV1yx/j5Py+D0laXcRvulTSSfn3g/JtfRfYAPx3/vM7gcmSuuZ/772Njr1NEcewrYhDrWNZFxEHNv4g/3/uDxp/BFwSEY82K3c8bS99pCLKQO60xWERsa6FthQ9707SUeQC8rCI+Lukx4FtWyke+eO+3/xvYNaYz6llz6PA1yV1ApD0aUnbAzOA0/Pn3PoDR7ew79PAZyUNye/bO//5WqBbo3LTyA0FyZc7MP92BnBm/rPRQK822toDWJ0PtD3J9RQ3qgI29ja/Qm5Y+1fgdUlfyh9Dkg5o4xi2lXGoZc+t5M6Xzc0/PORX5Hrk9wGvAvOA/wCeaL5jRKwidx5ssqQX+Hj49wBw0sYLBcClwLD8hYgFfHwV9gfASElzyQ2Dl7TR1keAGkkvAtcCzzT67gNgH0lzyJ0z+2H+8zOBC/Ltm4+XSLdmvEqHmWWKe2pmlikONTPLFIeamWWKQ83MMsWhZmaZ4lAzs0xxqJlZpvx/M8g39OubQsQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing Set\n",
      "Accuracy: 0.7293554884189325\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6282    0.4741    0.5404      1333\n",
      "           1     0.7637    0.8583    0.8082      2639\n",
      "\n",
      "    accuracy                         0.7294      3972\n",
      "   macro avg     0.6959    0.6662    0.6743      3972\n",
      "weighted avg     0.7182    0.7294    0.7183      3972\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcaklEQVR4nO3de5xVdb3/8dd7Brxwv4qKioiI4gUjxEtHNMtCzymPXUU7ZdbD7ET+OqcsK8uT1e9nWqebGJGSnS6SJiUFip1KQdMCyRsoiqAyIJeRO6IwM5/fH3sBm2Fmz16y9+w9a95PH+vx2Gut7/6u74aHH77f9b0pIjAzy4qaShfAzKyUHNTMLFMc1MwsUxzUzCxTHNTMLFO6VLoA+br17hu9Dxpc6WJYCjsa3XvekWxZu5LXNq/XvuRR22tIRMO2otLGtrWzI2L8vjwvraoKar0PGsxlP5he6WJYCnXrX6t0ESyFmddM2Oc8omEb+4/4QFFpX3ts0oB9fmBKVRXUzKwjEKh631w5qJlZOgJqaitdilY5qJlZetqn13Jl5aBmZim5+WlmWeOampllhnBNzcyyRK6pmVnGuPfTzLLDHQVmliXCzU8zyxjX1MwsO9z8NLMsEVBbvR0F1Rtuzax6ScUdbWaj8ZIWS1oi6eoW7veW9HtJj0taKOmjbeXpoGZmKSXNz2KOQrlItcAk4DxgJDBB0shmyT4FLIqIUcDZwHck7VcoXwc1M0uvNDW1scCSiFgaEduBacAFzdIE0FOSgB7AOqChUKZ+p2Zm6RXfUTBA0vy88ykRMSX5PBhYnnevDji12fdvAmYAK4GewAcjoqnQAx3UzCydIt+XJeojYkxrObVwrfn68O8EHgPOAYYBf5Q0NyI2tfZANz/NLL2a2uKOwuqAw/PODyNXI8v3UWB65CwBlgHHFixayp9iZp1eaToKgHnAcElDk5f/F5FrauZ7CXgbgKRBwAhgaaFM3fw0s/RKME0qIhokTQRmA7XA1IhYKOmK5P5k4OvAbZKeJNdc/UJE1BfK10HNzNIp4XpqETELmNXs2uS8zyuBd6TJ00HNzFLyNCkzyxqvp2ZmmeKlh8wsM+Tmp5lljWtqZpYlclAzs6zIrebtoGZmWSGhGgc1M8sQ19TMLFMc1MwsUxzUzCw7RMsroVUJBzUzS0XINTUzy5aaGs8oMLMMcU3NzLLD79TMLGtcUzOzzHBHgZlljqdJmVl2yM1PM8uYag5q1TvYxMyqlqSijiLyGS9psaQlkq5u4f5Vkh5LjqckNUrqVyhPBzUzS2VnR8G+BjVJtcAk4DxgJDBB0sj8NBFxY0ScHBEnA18EHoiIdYXydVAzs/RU5FHYWGBJRCyNiO3ANOCCAuknALe3lanfqZlZOko1TWqApPl551MiYkryeTCwPO9eHXBqi4+UugHjgYltPdBBzcxSS9FRUB8RY1rLpoVr0UradwEPtdX0BAc1M3sjStP5WQccnnd+GLCylbQXUUTTExzU9smyZ1/g/j/MoakpOPGU4xl7Vsv/IK2qW83tP7qDf75oPMecOJx1a9czc9o9u+5vXLeRM95+GqPf8qb2KnqndfzBPfjAyYOpETy4bB2zn1m7x/1jBnbn399yJPVbtwPwjxUbmbloDV1qxOfeOowutaJWYkHdRn6/cHUlfkJVKNGQjnnAcElDgRXkAtfFLTyrN3AW8KFiMi1rUJM0Hvg+UAvcEhHXl/N57ampqYk/z7if9152IT179eCXN/+aYccOpf+g/nulm3vvQwwZfsSua/0G9uXfPn3xrvtTrp/K0SOHtWv5OyMJJowezPceWMb6bTv44tuP5omVm3h50+t7pHuufiuTHnxhj2sNTcF3H1jK6w1N1Ag+f87RPPXyZpate7Udf0F1KHa4RlsiokHSRGA2uRgxNSIWSroiuT85SXohcF9EbC0m37IFtbzu2nPJVTPnSZoREYvK9cz2tKpuNX3696FPv94AHHvScJ5/euleQe2xhx9n+PHDWLViTYv5vPT8cvr0602vvr3KXubObmi/bqzZsn1XLWz+SxsYdWgvXt60to1v5rze0ARAbY2orRHR6uuf7CvV4NuImAXManZtcrPz24Dbis2znEM60nbXdihbNm6hZ+8eu8579O7B5k17/kOyeeMWnlv4PCedemKr+Sx+4jlGjDqmbOW03foc2JX1r+7Ydb5+2w76HNh1r3RH9e/GNe8YzqfPPJJDeu2/67oE15w7nG+/eyRPr97MC+u2tUu5q5FqVNRRCeUMai111w5unkjS5ZLmS5r/6qb1ZSxO+TX/K7x/5hzOHP+WVru/Gxsaef7ppRxzwvDyF86K8tL6bXxp5jN8477n+Mtzr/DJtxy5614EfOOPz3H1H57myH7dODQv4HU2pZpRUA7lfKdWVHdtMmZlCsAhw0/oMPX5Hr17sHnjll3nWzZuoUev7nukWb1iDbOm3QvAtldfY9niF6iprdn1/mzZsy8w6NCBdO/Zrf0K3olt2LaDvt1218z6HtiVDdt27JHmtaSJCfDUqs1MqBHd96tl6/bGXde37Wji2TVbOP6Qnqxs9j6uU+jEE9rTdNd2OAcPHsSG+g1sXLeRHr168MwTz3H+B9+5R5qPX3Xprs/3/uaPHDXiyD06BBY//iwjRo1oryJ3ei+se5WDeuxH/+5d2bCtgTFH9OHWR17aI02vA7qw6bUGAI7sdyA1wNbtjfTYv5bGpmDbjia61opjB/Vk9jMtvyfNOpFrilercga1orprO6qa2hre+u6zueundxPRxAlvPp4Bg/rz+N+eBGBUgfdoADu27+DFJct5+4XntEdxDWgKmLZgJf9n3FHUCB5atp6XN73OuGG5+dFznl/H6MN6c9aw/jRGsKOxiZ8kQa/3AV25dOzh1CS1lEeXb+DJlzdX8udUUHUvEqmI8rX4JJ0PfI/d3bXfLJT+kOEnxGU/mF628ljp1a1/rdJFsBRmXjOB+qUL9ykiHXDwMTHkIz8sKu2zN4x/tMCMgrIo6zi1lrprzayDU+dtfppZBgmo8XLeZpYlrqmZWaZUc0eBg5qZpeN3amaWJUJpFolsdw5qZpaaa2pmlil+p2Zm2eF3amaWJbm5n9Ub1RzUzCy1Ko5pDmpmlp5nFJhZdnTi9dTMLIM683pqZpZJ1b2emoOamaVWxTGtrBuvmFkWKddRUMzRZlbSeEmLJS2RdHUrac6W9JikhZIeaCtP19TMLJVSjVMrZm9gSX2Am4HxEfGSpIPaytc1NTNLrURb5BWzN/DFwPSIeAkgItrc7cZBzcxSk4o7gAE79/VNjsvzsilmb+BjgL6S7pf0qKQPt1U2Nz/NLLUUzc/6AhuvFLM3cBfgzcDbgAOBhyU9EhHPtvZABzUzS6d0E9qL2Ru4jlxg3ApslTQHGAW0GtTc/DSzVHKLRJak93PX3sCS9iO3N/CMZmnuBs6U1EVSN+BU4OlCmbqmZmap1ZSgqhYRDZImArPZvTfwQklXJPcnR8TTku4FngCagFsi4qlC+TqomVlqpRp829LewBExudn5jcCNxebpoGZmqcgT2s0sa6p45aHWg5qkH7J39+ouEXFlWUpkZlWvo66nNr/dSmFmHYbI9YBWq1aDWkT8LP9cUvdkrIiZdXJVXFFre5yapNMlLSIZGyJplKSby14yM6tORc77rFRnQjGDb78HvBN4BSAiHgfGlbFMZlblUsz9bHdF9X5GxPJmUbexPMUxs2onSjP4tlyKCWrLJZ0BRDKV4UramKZgZtlWzb2fxTQ/rwA+RW5JkBXAycm5mXVCxTY9q7b5GRH1wCXtUBYz6yCquflZTO/nUZJ+L2mtpDWS7pZ0VHsUzsyqk4o8KqGY5uevgDuAQ4BDgTuB28tZKDOrbh19SIci4ucR0ZAcv6DA9Ckzy7Zc72dxRyUUmvvZL/n4l2TrqmnkgtkHgZntUDYzq0Yqbvu7SinUUfAouSC2s/SfyLsXwNfLVSgzq24dcumhiBjangUxs45hZ/OzWhU1o0DSCcBI4ICd1yLif8pVKDOrbh2ypraTpGuBs8kFtVnAecCDgIOaWSdVvSGtuN7P95Hbc29VRHyU3PZU+5e1VGZWtSSorVFRRyUU0/zcFhFNkhok9QLWAB58a9aJVXPzs5ia2nxJfYCfkOsRXQD8vZyFMrPqVqq5n5LGS1osaUkydKz5/bMlbZT0WHJ8ta08i5n7+e/Jx8nJ/nu9IuKJtotrZlkkVJK5n5JqgUnAueR2Yp8naUZELGqWdG5E/Eux+RYafDu60L2IWFDsQ8wsQ0q3AsdYYElELAWQNA24AGge1FIpVFP7ToF7AZyzLw9uyaG9DuAr5x5T6mytjPqeMrHSRbAUXl+1tiT5pHinNkBS/iZOUyJiSvJ5MLA8714dcGoLeZwu6XFgJfC5iFhY6IGFBt++tbgym1lnIqC2+KBWHxFjCmTVXPN55QuAIRGxRdL5wO+A4YUeWExHgZnZHko0ob0OODzv/DBytbFdImJTRGxJPs8CukoaULBsqX+NmXV6JQpq84DhkoYmWwVcBMzITyDpYCVtXUljycWsVwplWtQ0KTOznXLDNfa9pyAiGiRNBGYDtcDUiFgo6Yrk/mRyg/8/KakB2AZcFBEFlz4rZpqUyC3nfVREXCfpCODgiPBYNbNOqlSTBZIm5axm1ybnfb4JuClV2YpIczNwOjAhOd9MbmyJmXVSHXrjFeDUiBgt6R8AEbE+af+aWSckoEsVT5MqJqjtSEb+BoCkgUBTWUtlZlWtimNaUUHtB8BvgYMkfZPci7tryloqM6taUmmmSZVLMXM/fynpUXLLDwn414jwDu1mnVgVx7Siej+PAF4Ffp9/LSJeKmfBzKx6dfTlvGeyewOWA4ChwGLg+DKWy8yqlKBiC0AWo5jm54n558nqHZ9oJbmZZV0F9/QsRuoZBRGxQNIp5SiMmXUMquJdCop5p/afeac1wGigNOuXmFmHk4Ut8nrmfW4g947trvIUx8w6gg4b1JJBtz0i4qp2Ko+ZdQDVvPFKoeW8uySz6Ftd1tvMOp/cFnmVLkXrCtXU/k7u/dljkmYAdwJbd96MiOllLpuZVakOPaMA6EduUbZz2D1eLQAHNbNOqCN3FByU9Hw+xe5gtlPBRdrMLNuquKJWMKjVAj0obnMEM+s0RE0HHaf2ckRc124lMbMOQXTcmloVF9vMKkbQpYpfqhUKam9rt1KYWYfRYWtqEbGuPQtiZh1HNQ/pqOIhdGZWrUq18Yqk8ZIWS1oi6eoC6U6R1CjpfW3l6aBmZqmIXOAo5iiYT24a5iTgPGAkMEHSyFbSfYvc/qBtclAzs3SUa34Wc7RhLLAkIpZGxHZgGnBBC+k+TW4RjTXFFM87tJtZKrkZBUW/UxsgaX7e+ZSImJJ8Hgwsz7tXB5y6x7OkwcCF5GY0FbWOo4OamaWWopugPiLGpMim+cD+7wFfiIjGYlcGcVAzs9RK1PlZBxyed34YsLJZmjHAtCSgDQDOl9QQEb9rLVMHNTNLSaVaT20eMFzSUGAFcBFwcX6CiBi666nSbcAfCgU0cFAzs5R29n7uq2S9xonkejVrgakRsVDSFcn9yW8kXwc1M0utVINvI2IWMKvZtRaDWURcWkyeDmpmlo466HLeZmYtKVXzs1wc1MwsNdfUzCxTqjekOaiZWUoCal1TM7MsqeKY5qBmZmkJVXED1EHNzFJzTc3MMiM3pKN6o5qDmpmlU+SqtpXioGZmqVXzHgUOamaWSm6RyEqXonUOamaWmns/zSxTqrj1WdXzUqve//51Eae89zpGX/hffPe2+/a6/+wLq3jHZd9m0Bmf4Yc//9+97jc2NjHukuv54H/8qD2Ka8DbTj+Ov//mKzw6/Vo+85Fz97rfq/sB3P7fn2DuL6/mr7/+Mhe/67Td93ocyG3Xf4y/3XkNj9xxDaecOHSv73cWKvK/SihbTU3SVOBfgDURcUK5nlMpjY1NXHXDHfz2pokcOqgP53zkRs4bdyLHHnXIrjR9e3Xn+s++n5kPPN5iHpOn/YVjhg5i89bX2qvYnVpNjbjx8x/gwok3sXL1Bv78s6u4Z86TLF62aleaj79/HIuXrmLCf/6Y/n16MO83X+HOe+axo6GR6z/7Pv708CIuvfpWunap5cAD9qvgr6mcan+nVs6a2m3A+DLmX1GPLnyBow4fwJGHDWC/rl14z7mjmfXAE3ukGdivJ6OPH0LXLrV7fX/F6vXc9+BCPnzBGe1V5E7vzccfydLl9by44hV2NDQy/Y8LOP+sk/ZIE0CP7vsD0L3b/qzf9CoNjU307H4AZ7xpGD+/+2EAdjQ0smnLtvb+CdWhyO3xKtVDWragFhFzgHXlyr/SXl67kcGD+u46P3RQX15eu7Ho73/pv+/ia1f+KzXV/E9exhwysDcrVq/fdb5y9XoOGdh7jzQ/ueMBjjnyYJ6+55s8dPuX+OJ3fkNEMGRwf+o3bGHStR/igV98ge9/+WK6ddKaGuRqa8UclVDxd2qSLpc0X9L8tfVrK12cokU038mr+Jen9859kgF9e3LycUeUuFRWSEtrgDX/azzntON48tk6jjvvy4y75P9xw1Xvp2f3A+hSW8uoEYcz9TdzOetD3+LV117nM5fu/U6uM9i572enq6kVKyKmRMSYiBgzcMDAShenaIce1Gevf/UPHtC7wDd2+9vjS7l37pOc9O6v8rEv/ZS5857l8q/8rFxFtcTKNRv2ql2vqt+zdn3Ju07jD3/JvQNdVlfPiytfYfiQQaxcs56Vazbw6MIXAZjxp8cYNeJwOivX1DJo9MghPP/SWl5cUc/2HQ1M/+MCzht3UttfBK6deAELZ36DJ2Zcx63/96OcecoxTPn6R8pcYluw6EWGHTGQIw7tT9cutbzn3NHcM2fP96B1q9Yz7pQRQO6d6NFDBvHCinrWvLKZFavXc/SQgwAYd8qIPToYOp0qjmoep/YGdelSyw2f/wDvvXISjY3BJe8+jeOGHcLUu+YCcNl7z2R1/SbO+cgNbN76GpKYPO1+Hv71l+nV48AKl75zamxs4vM33MFdP/gUtbXilzMe4Zmlq/joe/4JgJ9Of5Abb72XSdd+iIdu/xISfO2mu1m3cSsAn//2nUy57lL261rLCyvq+dR1v6jkz6moap4mpZbeDZUkY+l24GxyuyqvBq6NiFsLfefNbx4TD/1tflnKY+XR95SJlS6CpfD64jtoenXNPkWk4058U/zP3fcXlXbssD6PRsSY1u5LGg98n9y+n7dExPXN7l8AfB1oAhqAz0TEg4WeWbaaWkRMKFfeZlZhJaioSaoFJgHnAnXAPEkzImJRXrI/ATMiIiSdBNwBHFsoX79TM7NUcq/LSjKjYCywJCKWRsR2YBpwQX6CiNgSu5uT3ckNJSzIQc3M0knWUyvmAAbsHLKVHJfn5TQYWJ53Xpdc2/Nx0oWSngFmApe1VTx3FJhZailan/UF3qm1lM1eNbGI+C3wW0njyL1fe3uhBzqomVlKKtVmxnVA/mC/w4CVrSWOiDmShkkaEBH1raVz89PMUkvR/CxkHjBc0lBJ+wEXATP2fI6OVhJBJY0G9gNeKZSpa2pmlkqpxtVGRIOkicBsckM6pkbEQklXJPcnA+8FPixpB7AN+GC0MQ7NQc3M0ivR2NuImAXManZtct7nbwHfSpOng5qZpeblvM0sU6p4lpSDmpml5H0/zSxr3Pw0s8wQrqmZWcZUcUxzUDOzN6CKo5qDmpmlVs2LRDqomVlq1RvSHNTM7I2o4qjmoGZmqexcJLJaOaiZWToefGtmWVPFMc1BzczSKtkikWXhoGZmqVVxTHNQM7N0Krj5elEc1MwsvSqOag5qZpaah3SYWab4nZqZZYegxkHNzLKleqOa9/00s1R2LhJZgn0/kTRe0mJJSyRd3cL9SyQ9kRx/lTSqrTxdUzOz1EpRT5NUC0wCziW3W/s8STMiYlFesmXAWRGxXtJ5wBTg1EL5OqiZWWol6igYCyyJiKW5PDUNuADYFdQi4q956R8BDmsrUwc1M0stxTSpAZLm551PiYgpyefBwPK8e3UUroV9DLinrQc6qJlZaikqavURMSZFNtFiQumt5ILaP7X1QAc1M0ul2E6AItQBh+edHwas3Pt5Ogm4BTgvIl5pK1P3fppZairyvzbMA4ZLGippP+AiYMYez5GOAKYD/xYRzxZTNtfUzCy9EtTUIqJB0kRgNlALTI2IhZKuSO5PBr4K9AduTt7jNRRozgIOamb2BpRq6G1EzAJmNbs2Oe/zx4GPp8nTQc3MUpK3yDOz7Ng5o6BauaPAzDLFNTUzS62aa2oOamaWmheJNLPs8L6fZpYl1d5R4KBmZqm5+WlmmeKampllShXHNAc1M3sDqjiqOaiZWSqCqp4mpYgW12SrCElrgRcrXY4yGADUV7oQlkpW/86GRMTAfclA0r3k/nyKUR8R4/fleWlVVVDLKknz21ouxaqL/846Ls/9NLNMcVAzs0xxUGsfU9pOYlXGf2cdlN+pmVmmuKZmZpnioGZmmeKgVkaSxktaLGmJpKsrXR5rm6SpktZIeqrSZbE3xkGtTCTVApOA84CRwARJIytbKivCbUC7Dha10nJQK5+xwJKIWBoR24FpwAUVLpO1ISLmAOsqXQ574xzUymcwsDzvvC65ZmZl5KBWPi3N+PX4GbMyc1Arnzrg8Lzzw4CVFSqLWafhoFY+84DhkoZK2g+4CJhR4TKZZZ6DWplERAMwEZgNPA3cERELK1sqa4uk24GHgRGS6iR9rNJlsnQ8TcrMMsU1NTPLFAc1M8sUBzUzyxQHNTPLFAc1M8sUB7UORFKjpMckPSXpTknd9iGv2yS9L/l8S6HJ9pLOlnTGG3jGC5L22nWotevN0mxJ+az/kvS5tGW07HFQ61i2RcTJEXECsB24Iv9msjJIahHx8YhYVCDJ2UDqoGZWCQ5qHddc4OikFvUXSb8CnpRUK+lGSfMkPSHpEwDKuUnSIkkzgYN2ZiTpfkljks/jJS2Q9LikP0k6klzw/I+klnimpIGS7kqeMU/SW5Lv9pd0n6R/SPoxRezjLel3kh6VtFDS5c3ufScpy58kDUyuDZN0b/KduZKOLcmfpmWGd2jvgCR1IbdO273JpbHACRGxLAkMGyPiFEn7Aw9Jug94EzACOBEYBCwCpjbLdyDwE2Bckle/iFgnaTKwJSK+naT7FfDdiHhQ0hHkZk0cB1wLPBgR10n6Z2CPINWKy5JnHAjMk3RXRLwCdAcWRMRnJX01yXsiuQ1RroiI5ySdCtwMnPMG/hgtoxzUOpYDJT2WfJ4L3EquWfj3iFiWXH8HcNLO92VAb2A4MA64PSIagZWS/txC/qcBc3bmFRGtrSv2dmCktKsi1ktSz+QZ70m+O1PS+iJ+05WSLkw+H56U9RWgCfh1cv0XwHRJPZLfe2fes/cv4hnWiTiodSzbIuLk/AvJ/9xb8y8Bn46I2c3SnU/bSx+piDSQe21xekRsa6EsRc+7k3Q2uQB5ekS8Kul+4IBWkkfy3A3N/wzM8vmdWvbMBj4pqSuApGMkdQfmABcl79wOAd7awncfBs6SNDT5br/k+magZ166+8g1BUnSnZx8nANcklw7D+jbRll7A+uTgHYsuZriTjXAztrmxeSatZuAZZLenzxDkka18QzrZBzUsucWcu/LFiSbh/yYXI38t8BzwJPAj4AHmn8xItaSew82XdLj7G7+/R64cGdHAXAlMCbpiFjE7l7YrwHjJC0g1wx+qY2y3gt0kfQE8HXgkbx7W4HjJT1K7p3Zdcn1S4CPJeVbiJdIt2a8SoeZZYpramaWKQ5qZpYpDmpmlikOamaWKQ5qZpYpDmpmlikOamaWKf8f07y8CbswlKYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(clf_nn_chars, model_name=\"clf_nn_chars_noStopWords\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5B5xWgLEsW_6"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "id": "NXr0nSA8sXQW"
   },
   "outputs": [],
   "source": [
    "clf_rf = RandomForestClassifier(n_estimators=500,\n",
    "                                criterion=\"gini\",\n",
    "                                max_depth=None, \n",
    "                                min_samples_split=2,\n",
    "                                min_samples_leaf=10, \n",
    "                                max_features=\"auto\", \n",
    "                                n_jobs=-1,\n",
    "                                verbose=1,\n",
    "                                random_state=0,\n",
    "                                class_weight='balanced')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "id": "xaXdoukasXc-"
   },
   "outputs": [],
   "source": [
    "grid_search_rf = GridSearchCV(clf_rf, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, shuffle=True,\n",
    "                                              random_state=42).split(X_train_char, y_train_char), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "id": "7sc0LppbsXl2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   40.5s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  3.4min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.9s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 4.2min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   30.0s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  3.4min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.5min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   23.6s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  3.1min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.8s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.2min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   28.9s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  3.2min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.3min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   26.2s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  3.2min finished\n",
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    0.5s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] END .................................................... total time= 3.3min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   15.9s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.7min finished\n"
     ]
    }
   ],
   "source": [
    "clf_rf_chars= grid_search_rf.fit(X_train_char, y_train_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "id": "XwhqwGT2yiL_"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.4s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set\n",
      "Accuracy: 0.8952309020284851\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8748    0.7975    0.8344      3067\n",
      "           1     0.9040    0.9436    0.9234      6201\n",
      "\n",
      "    accuracy                         0.8952      9268\n",
      "   macro avg     0.8894    0.8705    0.8789      9268\n",
      "weighted avg     0.8944    0.8952    0.8939      9268\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.4s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.6s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcGUlEQVR4nO3de5xVVf3/8dd7BhDljoCAhqh5wwuGt9Q0zbxglqJWZFlZ/bS+kd9veal8/NLC+pX67ZsVXlIzsky/lHj7aaaZhtevIIICJV4QFFSuEiBemPl8/zh7xpmROWdvOGfOnjPvp4/98FzWWXudmQefWWvtvdZHEYGZWa2oq3YDzMzKyUHNzGqKg5qZ1RQHNTOrKQ5qZlZTulW7AS3V9ewTdb0GV7sZlsFu2/WvdhMsg8UvLWLVyuXanDrq+24fsWF9qrKxftlfIuLYzTlfVvkKar0G03vsxGo3wzL44yXjqt0Ey+CTYw/d7Dpiw3q22PVTqcq+OevyQZt9woxyFdTMrDMQKL8zVw5qZpaNgLr6areiXQ5qZpadNmtarqIc1MwsIw8/zazWuKdmZjVD5Lqnlt+WmVlOqdBTS3OUqkk6VtIzkp6T9J2NvD9A0i2SnpL0uKQ9S9XpoGZm2dXVpzuKkFQPXA6MBUYBn5E0qk2x84FZEbE38Hng5yWbtklfyMy6sORCQZqjuAOA5yLihYh4G7gJOKFNmVHAfQAR8U9gpKRtilXqoGZm2Ygsw89Bkma0OM5oUdO2wEstnr+cvNbSbOAkAEkHANsD2xVrni8UmFl26S8ULI+I/dqrZSOvtd2K+yfAzyXNAp4GngQ2FDuhg5qZZVS2+9ReBt7X4vl2wJKWBSLiX8DpAJIELEiOdjmomVk2AurLskxqOrCzpB2AxcB44NRWp5L6A28kc25fAaYlga5dDmpmll0Zbr6NiA2SJgB/AeqB6yJirqSvJu9fBewOXC+pAZgHfLlUvQ5qZpZR+ZZJRcRdwF1tXruqxeNHgZ2z1OmgZmbZeZmUmdWUHC+TclAzs2xSLoGqFgc1M8vOm0SaWe3wfmpmVms8/DSzmpHz/dQc1MwsIw8/zazW+EKBmdUUz6mZWc2Qh59mVmvcUzOzWiIHNTOrFYXdvB3UzKxWSKguv0Etv7N9ZpZbklIdKeoplfezn6Q7JM2WNFfS6aXqdFAzs8zKEdRS5v38OjAvIkYDhwM/ldSjWL0OamaWWZl6amnyfgbQJ0m60htYSYlsUg5qZpaNMhzFpcn7OYlCnoIlFFLk/XtENBar1EHNzDIR6XppKp3MOE3ez2OAWcBwYB9gkqS+xdrnq59mllldXVmSGZfM+0kh5+dPIiKA5yQtAHYDHm+3bWlbZmbWpExzas15P5PJ//HA7W3KLAKOTM65DbAr8EKxSt1TM7Ns0s2XlZQy7+dFwGRJTydn/XZELC9Wr4OamWVWrhUFKfJ+LgGOzlKng5qZZdJ0oSCvHNTMLLM8L5NyUDOzbOQF7WZWYxzUzKymOKiZWc3whQIzqz35jWkOamaWkTItk+pwDmpmlpmHn2ZWW/Ib0xzUyumIvYZx0an7U18nbpj2HJPunNvq/T5bdufyMw9h24G96FYvrvzzPG56qOjaXKugh2c8w39efTsNjcG4o/fn9E8d0er9u+5/ksl/egCArXr24Pyvj2OXHYdXoaX5k+eeWkUHxqX2H68ldRI/Pu0ATv2vv3HY+Xcw7sCR7DK8X6sypx+5C/MXr+bIC+7kpJ/cy4Xj96V7fX7nJmpZQ0MjF195K7/8wZe4+cpvcfe02byw6LVWZbbdZgDX/uRMplz+Tf7PZ47kh7+cWqXW5kvaHTqqFfgq9i8q5f7jNeMDO27NgtfWsGjZWt5paOTW/3mRYz6wXasyEdC7Z6Fz3GuLbry+7m02NBbdxNMqZM78l9hu+NZsN2xrunfvxjGHjeaBx+a1KjN61Ej69tkKgL12HcFrK1ZXo6m51CWDGun2H68ZwwZsxZKVbzQ/f2XVGwwbsFWrMtfd9ww7D+/H7MtO5v4fHs/3/jCDaLvPp3WIZStWM3RQ/+bnQwb1Y2mRoHXrPdM5ZN9dO6BlnYPqlOqohkrOqW1s//ED2xZKtvc9A0BbbV3B5lTWxv4oRZudiY/YczhzFq3i5Iv/ysghvZly7kd57JmlrH3znQ5qpTXZ2B8TtTP7PX3289x6z3Suu/RrFW5V59FV59TS7D9ORFwdEftFxH51PYtuPZ5rS1a+wfCB7/bMhg3YildXrW9VZvyhO3HXE4sAeHHpWhYtW8vOwzrvd+7Mhgzqx6vLX29+vnT5agZv/d7fxfwFr3DRL/7Ezy74Av379urAFuaYuu7wM83+4zVj1oIV7LhNH0YM6kX3+jpOPHAk9zz5cqsyi1es49BRwwAY1LcnOw3ry8Jla6vR3C5vj12246XFK1j86kreeWcDf5k2mw8fuHurMq8sXcU5P/odF539abbfdnCVWpo/ojAySXOUrKt0MuNzJc1KjjmSGiQNLFZnJYefzfuPA4sp7D9+agXPV1UNjcH5v5/OjeccSX2duPHB53lmyWo+f8TOAFx//7P81+1P8/OvHMT9F30MSfxwypOsXPtWlVveNXWrr+fbXzuBr3/v1zQ2NvKJo/Znp+2H8qe7HgPglOM+yDU33sfqf73Bj6+4FYD6+jpu+PlZVWx1XpSnF9biYuJRFDpB0yXdHhHNV2wi4lLg0qT8x4FvRsTKovVGBWeqJR0HXMa7+4//qFj5blvvGL3HTqxYe6z8Hr1kXLWbYBl8cuyhzJk9c7MiUs+hu8T2X/hlqrLzLzn2ifaySUk6CPh+RByTPP8uQET8uJ3yfwDuj4hrip2zojffbmz/cTPr5FIOLRODJM1o8fzqiLg6eZzqYiKApK2AY4EJpU7oFQVmlomAuvS3axTL+5nqYmLi48DDpYae4KBmZpugTBc2s1xMHA/cmKZSr9Exs8w6MJkxkvoBHwZuS9M299TMLJtsc2rtSpnMGGAccE9ErEtTr4OamWUiVLZNIkslM06eTwYmp63TQc3MMsvxKikHNTPLLs9rPx3UzCybMs2pVYqDmpllUlj7md+o5qBmZpnlOKY5qJlZdhlWFHQ4BzUzy0YefppZDWnaTy2vHNTMLKPq7WqbhoOamWWW45jmoGZmGckXCsyshvg+NTOrOQ5qZlZTchzTHNTMLLs899S8862ZZZMy52c58n4mZQ5P8n7OlfT3UnW6p2ZmmRQ2ieyYvJ+S+gNXAMdGxCJJQ0rV656amWVWJ6U6SjgAeC4iXoiIt4GbgBPalDkVmBoRiwAiYmnJtm3C9zGzLq5Mw8+N5f3ctk2ZXYABkh6Q9ISkz5eq1MNPM8tE2Ra0F0tmnCbvZzdgX+BIYEvgUUmPRcT89k7ooGZmmWWYUiuWzDhN3s+XkzrWAeskTQNGA9mDmqRf0n62ZCLirPbeM7PaVqZlUs15P4HFFPJ+ntqmzG3AJEndgB7AgcDPilVarKc2o8h7ZtZFicIV0M2VJu9nRPxD0t3AU0AjcG1EzClWb7tBLSJ+2/K5pF5pk4maWW0r13r2lHk/LwUuTd22UgUkHSRpHvCP5PloSVekPYGZ1RgV9lNLc1RDmls6LgOOAVYARMRs4LAKtsnMcq5cKwoqIdXVz4h4qU3UbahMc8ws7wRpbqytmjRB7SVJBwMhqQdwFslQ1My6pjxvEplm+PlV4OsU7vRdDOyTPDezLijt0DO3w8+IWA58tgPaYmadRJ6Hn2mufu4o6Q5JyyQtlXSbpB07onFmlk9KeVRDmuHnH4ApwDBgOPBH4MZKNsrM8q2z39KhiPhdRGxIjt9TZPmUmdW2wtXPdEc1FFv7OTB5eH+yI+VNFILZp4E7O6BtZpZHKs8mkZVS7ELBExSCWFPrz2zxXgAXVapRZpZvec5RUGzt5w4d2RAz6xyahp95lWpFgaQ9gVFAz6bXIuL6SjXKzPKtU/bUmki6EDicQlC7CxgLPAQ4qJl1UfkNaemufp5CYSvdVyPidAq7Tm5R0VaZWW5JUF+nVEc1pBl+ro+IRkkbJPUFlgK++dasC8vz8DNNT21GknvvGgpXRGcCj1eyUWaWbx2VzDhJZLw6SWY8S9IFpepMs/bz35KHVyXb6vaNiKdKN9fMapFIldOzdD0pkhknHoyI49PWW+zm2zHF3ouImWlPYmY1pHw7cDQnMwaQ1JTMuG1Qy6RYT+2nRd4L4CObc+KNGT1yIA9P/ly5q7UKGrD/hGo3wTJ469mXy1JPmfJ+biyZ8YEbqeMgSbMppM87JyLmFjthsZtvj0jXZjPrSgTUpw9qxfJ+pklmPBPYPiLWSjoOuBXYudgJ01woMDNrpUwL2ksmM46If0XE2uTxXUB3SYOKti3ztzGzLq9MQa05mXGSKmA8cHvLApKGKhnrSjqAQsxaUazSVMukzMyaFG7X6JhkxhRu/v+apA3AemB8RBTd+izNMilR2M57x4iYKGkEMDQifK+aWRfVUcmMI2ISMClT21KUuQI4CPhM8nwNhXtLzKyL6tSJV4ADI2KMpCcBImJVMv41sy5IQLccL5NKE9TeSe78DQBJg4HGirbKzHItxzEtVVD7BXALMETSjyhM3P3firbKzHJLKs8yqUpJs/bzBklPUNh+SMCJEeEM7WZdWI5jWqqrnyOAN4A7Wr4WEYsq2TAzy6/Ovp33nbybgKUnsAPwDLBHBdtlZjklqNoGkGmkGX7u1fJ5snvHme0UN7NaV8WcnmlkXlEQETMl7V+JxphZ56AcZylIM6f2rRZP64AxwLKKtcjMcq0WUuT1afF4A4U5tpsr0xwz6ww6bVBLbrrtHRHndlB7zKwTyHPilWLbeXdLVtG3u623mXU9hRR51W5F+4r11B6nMH82S9LtwB+BdU1vRsTUCrfNzHKqU68oAAZS2JTtI7x7v1oADmpmXVDeLxQU60QOSa58zgGeTv4/N/n/nA5om5nlVEfl/WxRbn9JDZJOKVVnsZ5aPdCbdMkRzKzLEHVluE8tbd7PpNzFFHbILalYUHslIiZuYnvNrEaJDs/7+Q0Kt5Gluum/WFDL8ajZzKpG0K08k2ol835K2hYYR2FOf7OD2pEZG2hmXUDGnlqxZMZpprYuA74dEQ1p740rlsx4ZaoazKzLyXBLR7FkxiXzfgL7ATclAW0QcJykDRFxa3sndIo8M8usTHNqzXk/gcUU8n6e2rJAROzw7jk1Gfj/xQIaOKiZWUaiPFnQU+b9zMxBzcyyUflWFJTK+9nm9S+mqdNBzcwyKawoyO/NEQ5qZpZZfkOag5qZbYIcd9Qc1MwsK3XO/dTMzDamXFc/K8VBzcwy84UCM6sd6qTbeZuZbYyHn2ZWc9xTM7Oakt+Q5qBmZhkJqHdPzcxqSY5jmoOamWUllOMBqIOamWXmnpqZ1YzCLR35jWoOamaWTcqcntWS53vozCyn6qRURymlkhlLOkHSU5JmSZoh6UOl6nRPzcwyKWwSWYZ60iUzvg+4PSJC0t7AFGC3YvW6p2ZmmSnlfyU0JzOOiLeBpmTGzSJibUQ0pc3rxXtT6L2Hg5qZZSalO0jyfrY4zmhRzcaSGW/73nNpnKR/AncCXyrVNg8/M/rrI/P47k//RENjI6edcDDf/OLRrd6PCL7z0z9x78Nz2bJnD6648DRG71ZIbbj3Jy6g91ZbUF9XR7duddx//bebP3f1fz/ANVOm0a2+jqM+tCcTzzqxI79Wl3HkQbvz47NPob6ujt/d9giX/fbeVu/367Mlk773OXbYbhBvvv0O37joBv7x/CvN79fVifuvP49Xlq5m/Lc2KdlRTchwn1qxvJ9pkhkTEbcAt0g6DLgI+GixE1YsqEm6DjgeWBoRe1bqPB2poaGRcy+Zwi2TJjB8m/585AuXMvawvdhtx2HNZe59ZB7PL1rGE1MvZMacFzn7Jzfx18nnNr9/x1X/ztb9e7eq98EZ87nr70/z0I3fZYse3Vm2ck2HfaeupK5OXHrepxg3YRJLXnudv/32XP487WmeWfBqc5mzTz+Gp+e/zGnnXcPO22/Dpd/+FCf+2y+b3//q+COYv+A1+vTqWY2vkAvlmlMjXTLjZhExTdJOkgZFxPL2ylVy+DkZOLaC9Xe4J+a+yI7vG8TI7QbRo3s3TjpqDHf9/alWZe76+1OM/9gBSGL/vXZg9Zr1vLp8ddF6r7v5Qf7jC0exRY/uAAwe2Kdi36Er23ePkbzw0nIWLl7BOxsamHrvTI778N6tyuy6w1CmTX8GgGcXvsaIYQObfx/Dh/Tn6A/twfW3PdLhbc+VlFc+U1z9bE5mLKkHhWTGt7c+ld6vZEsQSWOAHsCKYpVWLKhFxDRgZaXqr4ZXlq1m220GND8fvs0AXlm2uk2Z11uXGdKfV5a+DhS2azlpwiQOP+1iJk99qLnMcwuX8uis5/noFy/lY2dcxsy5Cyv7RbqoYYP7sfi1Vc3Pl7y2imGD+7UqM+fZxRx/xD4AjBm1Pe8bOpDhQ/oD8P++dTIX/uJWGhtLzlXXPKU8iomIDUBTMuN/AFOakhk3JTQGTgbmSJpF4Urpp1tcONioqs+pJROHZwC8b8SIKremuI39LNv+MdrYj7tp76m7r/0mwwb3Z9nKNYybMImdRw7lkDHvZ0NDI6+veYN7f3MOM+ct5PTzr2PWrd/P9Z5VndHGfp5tf1+X/fZefnz2KUy74TvMe24JT81/mYaGRo750J4sX7WG2f98iUPG7NxBLc6ncub9LJXMOCIuBi7OUmfVg1pEXA1cDbDvvvvl+k/g8CH93/OXfuigfsXLLH2doUlvYNjg/kBheHn84Xszc+6LHDLm/Ww7pD8fP2I0kth3j5HUSax4fS2DBngYWk5Llr7+np5226mBNeveZMLE3zc/n33bD1i4ZAUnHb0vxx66F0cdvAdbbNGdPr168quJn+fMC67vsPbnSZ7/3PqWjgzGjNqe5xctY+Hi5bz9zgam3juTsYe1npMZe9he3HTn40QE059eQN/eWzJ0UD/WrX+LNeveBGDd+rf422P/ZPedhgNw3OF7M236fACeW/gab7+z4T0XE2zzzZy3kJ1GDGbE8K3p3q2ek44aw5+ntZ4T7dt7S7p3qwfg8ycezCNPPseadW8y8fLb2fP47zH6hAv58vm/4cHp87tsQAPKM/6skKr31DqTbt3queS8T3HyWZfT0BB89hMfZPedhnHdzQ8C8KWTD+XoQ/bg3ofnMmbcD9iyZ3cuv+BzACxbsYbPnXcNAA0bGjj52P346MGjAPjcJw5iwsQbOOjTP6JH93qu/P5pHnpWQENDI+ddMoWbf/F16uvFDbc/xj9feJXTTyqsvPnN1IfYdYehXPn902hobOSZBa/yjYtuqHKr8ynP2aRUYs5t0yuWbgQOBwYBrwEXRsSvi31m3333i4f/Z0ZF2mOVMWD/CdVugmXw1jNTaHxj6WZFpN33+kBcf9sDqcoesFP/J4rcp1YRFeupRcRnKlW3mVVZfjtqHn6aWTaF6bL8RjUHNTPLJuf7qTmomVlmOY5pDmpmlpVyfXXeQc3MMstxTHNQM7NsqnhfbSoOamaWXY6jmoOamWXmWzrMrKZ4Ts3MakfO71PzLh1mllmZskmlyfv52STv51OSHpE0ulSd7qmZWSaiPD21lHk/FwAfjohVksZS2HvxwGL1uqdmZpmVaTu1NHk/H4mIpl1XH6OQnKUoBzUzy648US1V3s8Wvgz8uVSlHn6aWWYZNokcJKnlJolXJ1v4Q8q8nwCSjqAQ1D5U6oQOamaWWYYptWLJjFPl/ZS0N3AtMDYiiqbHAw8/zWxTlGf4mSbv5whgKnBaRMxP0zT31Mwsk3JtEhkRGyQ15f2sB65ryvuZvH8VcAGwNXBFsjPIhlLbgzuomVk2Zbz5NkXez68AX8lSp4OamWWW4wUFDmpmlpU3iTSzGpPjmOagZmbZeJNIM6s9OY5qDmpmlpk3iTSzmuI5NTOrHYI6BzUzqy35jWoOamaWSbk2iawUBzUzyyzHMc1Bzcyyc0/NzGqKl0mZWU3Jb0hzUDOzjJTzvJ8OamaWWZ5XFHg7bzPLrkw58lIkM95N0qOS3pJ0TpqmuadmZpmVo5+WMpnxSuAs4MS09bqnZmYZiTqlO0pIk8x4aURMB95J2zoHNTPLpGlFQZqDJO9ni+OMFlVlTWacioefZlZJxfJ+pk5mnIWDmpllVqZbOlIlM87Kw08zy0wp/yuhZDLjTeGempllU6abb9MkM5Y0FJgB9AUaJf0HMCoi/tVevQ5qZpZJObceSpHM+FUKw9LUHNTMLLM8ryhwUDOzzLz208xqSo5jmoOamW2CHEc1BzUzy0SQZglU1Shis2/gLRtJy4CF1W5HBQwClle7EZZJrf7Oto+IwZtTgaS7Kfx80lgeEcduzvmyylVQq1WSZhRZKmI55N9Z5+UVBWZWUxzUzKymOKh1jKur3QDLzL+zTspzamZWU9xTM7Oa4qBmZjXFQa2CSmXKsfyRdJ2kpZLmVLsttmkc1CqkRaacscAo4DOSRlW3VZbCZKBDbxa18nJQq5ySmXIsfyJiGoW0bNZJOahVTkUy5ZhZcQ5qlVORTDlmVpyDWuVUJFOOmRXnoFY5FcmUY2bFOahVSERsAJoy5fwDmBIRc6vbKitF0o3Ao8Cukl6W9OVqt8my8TIpM6sp7qmZWU1xUDOzmuKgZmY1xUHNzGqKg5qZ1RQHtU5EUoOkWZLmSPqjpK02o67Jkk5JHl9bbLG9pMMlHbwJ53hR0nuyDrX3epsyazOe6/uSzsnaRqs9Dmqdy/qI2Cci9gTeBr7a8s1kZ5DMIuIrETGvSJHDgcxBzawaHNQ6rweB9ye9qPsl/QF4WlK9pEslTZf0lKQzAVQwSdI8SXcCQ5oqkvSApP2Sx8dKmilptqT7JI2kEDy/mfQSD5U0WNLNyTmmSzok+ezWku6R9KSkX5Eij7ekWyU9IWmupDPavPfTpC33SRqcvLaTpLuTzzwoabey/DStZjhDeyckqRuFfdruTl46ANgzIhYkgWF1ROwvaQvgYUn3AB8AdgX2ArYB5gHXtal3MHANcFhS18CIWCnpKmBtRPxnUu4PwM8i4iFJIyismtgduBB4KCImSvoY0CpIteNLyTm2BKZLujkiVgC9gJkRcbakC5K6J1BIiPLViHhW0oHAFcBHNuHHaDXKQa1z2VLSrOTxg8CvKQwLH4+IBcnrRwN7N82XAf2AnYHDgBsjogFYIulvG6n/g8C0proior19xT4KjJKaO2J9JfVJznFS8tk7Ja1K8Z3OkjQuefy+pK0rgEbgv5PXfw9MldQ7+b5/bHHuLVKcw7oQB7XOZX1E7NPyheQf97qWLwHfiIi/tCl3HKW3PlKKMlCYtjgoItZvpC2p191JOpxCgDwoIt6Q9ADQs53ikZz39bY/A7OWPKdWe/4CfE1SdwBJu0jqBUwDxidzbsOAIzby2UeBD0vaIfnswOT1NUCfFuXuoTAUJCm3T/JwGvDZ5LWxwIASbe0HrEoC2m4UeopN6oCm3uapFIa1/wIWSPpkcg5JGl3iHNbFOKjVnmspzJfNTJKH/IpCj/wW4FngaeBK4O9tPxgRyyjMg02VNJt3h393AOOaLhQAZwH7JRci5vHuVdgfAIdJmklhGLyoRFvvBrpJegq4CHisxXvrgD0kPUFhzmxi8vpngS8n7ZuLt0i3NrxLh5nVFPfUzKymOKiZWU1xUDOzmuKgZmY1xUHNzGqKg5qZ1RQHNTOrKf8LdDpGyo1gtVoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.7s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.5s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing Set\n",
      "Accuracy: 0.751007049345418\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6575    0.5386    0.5922      1333\n",
      "           1     0.7865    0.8583    0.8208      2639\n",
      "\n",
      "    accuracy                         0.7510      3972\n",
      "   macro avg     0.7220    0.6985    0.7065      3972\n",
      "weighted avg     0.7432    0.7510    0.7441      3972\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend ThreadingBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  42 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=4)]: Done 192 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=4)]: Done 442 tasks      | elapsed:    1.3s\n",
      "[Parallel(n_jobs=4)]: Done 500 out of 500 | elapsed:    1.4s finished\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATUAAAEGCAYAAAAE8QIHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAceElEQVR4nO3de7xVdZ3/8debA4hcRUBEEEIFFBWNwEszIdlYYFOMloXaVFY/s1/kr8tYTjU62a/fz7RmapIGSRm7KXkrMfGSjRe8FWBegsIIL1yUO8hN4ZzzmT/2BjeHc/ZZS/Y+e5113k8f+/HYa63v/q4v+PDj9/5VRGBmlhedal0AM7NKclAzs1xxUDOzXHFQM7NccVAzs1zpXOsClDqgV9/o2X9QrYthKRzQpa7WRbAUNq1eyfbNG7Q/edT1HhZRvyNR2tix9t6ImLQ/70srU0GtZ/9BvOeKG2tdDEth1MAetS6CpXDttLP3O4+o38EBoz6UKO1rT03vv98vTClTQc3M2gOBsttz5aBmZukI6JTdbgcHNTNLT/vVLVdVDmpmlpKbn2aWN66pmVluCNfUzCxP5JqameWMRz/NLD88UGBmeSLc/DSznHFNzczyw81PM8sTAXXZHSjIbrg1s+ySkn1azUaTJC2RtFTSpc087yPpTklPS1ok6YLW8nRQM7OUis3PJJ9yuUh1wHRgMjAaOFfS6CbJPgssjogTgInAdyV1LZevg5qZpVeZmtpJwNKIWBYRO4HZwJQmaQLoJUlAT2ADUF8uU/epmVl6yQcK+ktaUHI9MyJmFr8PBpaXPFsBnNzk99cAc4BVQC/gwxHRWO6FDmpmlk7C/rKidRExrqWcmrnX9HT19wBPAacDRwK/kTQvIl5t6YVufppZep3qkn3KWwEcXnI9hEKNrNQFwO1RsBR4Hji6bNFS/lHMrMOrzEABMB8YIWl4sfN/KoWmZqmXgHcBSBoIjAKWlcvUzU8zS68Cy6Qiol7SNOBeoA6YFRGLJF1UfD4D+CZwg6RnKTRXvxIR68rl66BmZulUcD+1iJgLzG1yb0bJ91XAu9Pk6aBmZil5mZSZ5Y33UzOzXPHWQ2aWG3Lz08zyxjU1M8sTOaiZWV4UdvN2UDOzvJBQJwc1M8sR19TMLFcc1MwsVxzUzCw/RPM7oWWEg5qZpSLkmpqZ5UunTl5RYGY54pqameWH+9TMLG9cUzOz3PBAgZnljpdJmVl+yM1PM8uZLAe17E42MbPMkpTokyCfSZKWSFoq6dJmnl8i6ani54+SGiQdXC5PBzUzS2X3QMH+BjVJdcB0YDIwGjhX0ujSNBFxdUScGBEnAv8MPBQRG8rl66BmZukp4ae8k4ClEbEsInYCs4EpZdKfC9zUWqbuUzOzdJRqmVR/SQtKrmdGxMzi98HA8pJnK4CTm32l1B2YBExr7YUOamaWWoqBgnURMa6lbJq5Fy2kfR/waGtNT3BQM7M3ozKDnyuAw0uuhwCrWkg7lQRNT3BQ2y/HDerFeWMHI4l5f13P3D+t2ev5qEN68rl3DGfdtp0ALFy+iTsXrd7zXILL3jOSTdt38f2Hn2/TsndUy5a8wP1zHqIxghPGH8up7xzfbLqXl7/CT6bfzJTzJnP0mBEAvLbjde6+9X7Wrl6PgDPPOYPBwwa1Yemzo0JTOuYDIyQNB1ZSCFznNfOuPsBpwEeSZFrVoCZpEvB9oA64LiKurOb72pIEH3nbEL77wF/ZsGMXl717JE+t3MyqV1/fK91f1m5tMWCdMXIAL29+nQO7eLymLTQ2NnLfrx5k6qfOolefntxwzWxGjD6C/gP77ZPugbsfZfjIoXvdv3/OQxwxahhn/eN7aahvYNeu+rYsfmYkna7RmoiolzQNuJdCjJgVEYskXVR8PqOY9CzgvojYliTfqv3XlGS4tj074uDurNn6Omu37aShMfjdSxs5cUifxL/ve2AXxhzWm4eXra9iKa3Uy8tX07dfHw7q14e6znWMPmEkf1m8bJ90Cx99mlHHHUX3nt333Hv9tddZ/vxKxow/FoC6znV0O/CANit71lRqnlpEzI2IkRFxZER8q3hvRklAIyJuiIipSctWzSpC2uHaduWg7l3YsH3XnuuN23fR98Au+6Q7sn8PvjFpFF847QgO691tz/1zxw7mlqdWES11i1rFbdm8lV4H9dpz3atPT7Zs3rpPmucW/ZW3nnL8Xvc3bXiV7j0O5K5bfsOs79/I3FvvZ+fOXXRU6qREn1qoZlBrbrh2cNNEki6UtEDSgte2bKpicSorybDNixu2c8mcxVx+zxLuf24tn5swHIATDuvNq6/X8+LGHVUvp7WiSW3i/jsfYuLkv9lnykJjYyOvrFrD2FPG8In/cx5dunbhiQcW0FFVqqZWDdXsU0s0XFucszIToN/w0e2m3rJx+y4O7v5Gzaxv9y5s2rH3/7lfq2/c8/3Zl7dQJ9Gzax1HDejBiYN7M2bQaLrUiW5d6vhfpw7lR4+/1Gbl74h69enJlk1b9lxv2byVXr177JXmlRVruOOmuwHYse01lv35BTrVdeKwoYfSq09PDht6KABHH38UTzzYQYNaB17Qnma4tt15fsN2BvY6gP49urJxxy5OHtqXax97ca80vbt15tXXCp3Jww/ujgRbdzZw29Mvc9vTLwOFEdJJRw9wQGsDg4YMZMP6TWzasJlevXuy+OnneP/USXul+cylF+z5/uub7+Ooo4cz8tgjAejdpxfr126k34C+vLB0Of0OKbsEMbfEPhXcTKlmUEs0XNteNQb8bMEKvjjxCDpJPLJsA6tefY2JRxVG0h5cup5xhx/EO0f0o7ERdjY0MuOxF2pb6A6uU10n3j1lIr+4/ldEYzBm/GgGHNqPPzzxDABvPWVM2d+fMWUid950Dw0NDRx0cB/ee84ZbVHsDMr2JpGKKvZUSzoT+B5vDNd+q1z6fsNHx3uuuLFq5bHKGzWwR+uJLDOunXY2q557dr8iUrdDR8awj/0gUdrnrpq0sMyKgqqo6jy1iJgLzK3mO8ysjanjNj/NLIcEdPJ23maWJ66pmVmuZHmgwEHNzNJxn5qZ5YlQmk0i25yDmpml5pqameWK+9TMLD/cp2ZmeVJY+5ndqOagZmapZTimOaiZWXpeUWBm+dGB91MzsxzqyPupmVkuZXs/NQc1M0stwzGtqgevmFkeqTBQkOTTalbSJElLJC2VdGkLaSZKekrSIkkPtZana2pmlkql5qmVnA18BoUzTeZLmhMRi0vSHAT8EJgUES9JOqS1fF1TM7PUKnREXpKzgc8Dbo+IlwAiYk1rmTqomVlqUrIP0H/3ub7Fz4Ul2SQ5G3gk0FfSg5IWSvpoa2Vz89PMUkvR/FxX5uCVJGcDdwbeBrwLOBB4XNITEfFcSy90UDOzdCq3oD3J2cArKATGbcA2SQ8DJwAtBjU3P80slcImkRUZ/dxzNrCkrhTOBp7TJM0dwDskdZbUHTgZ+FO5TF1TM7PUOlWgqhYR9ZKmAffyxtnAiyRdVHw+IyL+JOke4BmgEbguIv5YLl8HNTNLrVKTb5s7GzgiZjS5vhq4OmmeDmpmloq8oN3M8ibDOw+1HNQk/YB9h1f3iIiLq1IiM8u89rqf2oI2K4WZtRuiMAKaVS0GtYj4cem1pB7FuSJm1sFluKLW+jw1SadKWkxxboikEyT9sOolM7NsSrjus1aDCUkm334PeA+wHiAingYmVLFMZpZxKdZ+trlEo58RsbxJ1G2oTnHMLOtEZSbfVkuSoLZc0tuBKC5luJhWlimYWb5lefQzSfPzIuCzFLYEWQmcWLw2sw4oadMzs83PiFgHnN8GZTGzdiLLzc8ko59HSLpT0lpJayTdIemItiicmWWTEn5qIUnz80bgZmAQcBhwC3BTNQtlZtnW3qd0KCJ+GhH1xc/PKLN8yszyrTD6mexTC+XWfh5c/PpA8eiq2RSC2YeBu9qgbGaWRUp2/F2tlBsoWEghiO0u/adLngXwzWoVysyyrV1uPRQRw9uyIGbWPuxufmZVohUFko4DRgPddt+LiJ9Uq1Bmlm3tsqa2m6TLgYkUgtpcYDLwCOCgZtZBZTekJRv9/CCFM/deiYgLKBxPdUBVS2VmmSVBXScl+tRCkubnjoholFQvqTewBvDkW7MOLMvNzyQ1tQWSDgJ+RGFE9Eng99UslJllW6XWfkqaJGmJpKXFqWNNn0+UtFnSU8XPZa3lmWTt5/8ufp1RPH+vd0Q803pxzSyPhCqy9lNSHTAdOIPCSezzJc2JiMVNks6LiL9Pmm+5ybdjyz2LiCeTvsTMcqRyO3CcBCyNiGUAkmYDU4CmQS2VcjW175Z5FsDp+/Pi5rzl4O7MOvfESmdrVdR3/LRaF8FSeH3F6orkk6JPrb+k0kOcZkbEzOL3wcDykmcrgJObyeNUSU8Dq4B/iohF5V5YbvLtO5OV2cw6EgF1yYPauogYVyarppquK38SGBYRWyWdCfwKGFHuhUkGCszM9lKhBe0rgMNLrodQqI3tERGvRsTW4ve5QBdJ/cuWLfWfxsw6vAoFtfnACEnDi0cFTAXmlCaQdKiKbV1JJ1GIWevLZZpomZSZ2W6F6Rr7P1IQEfWSpgH3AnXArIhYJOmi4vMZFCb/f0ZSPbADmBoRZbc+S7JMShS28z4iIq6QNBQ4NCI8V82sg6rUYoFik3Juk3szSr5fA1yTqmwJ0vwQOBU4t3i9hcLcEjProNr1wSvAyRExVtIfACJiY7H9a2YdkIDOGV4mlSSo7SrO/A0ASQOAxqqWyswyLcMxLVFQ+w/gl8Ahkr5FoePu61UtlZllllSZZVLVkmTt588lLaSw/ZCAf4gIn9Bu1oFlOKYlGv0cCmwH7iy9FxEvVbNgZpZd7X0777t44wCWbsBwYAlwbBXLZWYZJajZBpBJJGl+Hl96Xdy949MtJDezvKvhmZ5JpF5REBFPShpfjcKYWfugDJ9SkKRP7Ysll52AscDaqpXIzDItD0fk9Sr5Xk+hj+226hTHzNqDdhvUipNue0bEJW1UHjNrB7J88Eq57bw7F1fRt7itt5l1PIUj8mpdipaVq6n9nkL/2VOS5gC3ANt2P4yI26tcNjPLqHa9ogA4mMKmbKfzxny1ABzUzDqg9jxQcEhx5POPvBHMdiu7SZuZ5VuGK2plg1od0JNkhyOYWYchOrXTeWovR8QVbVYSM2sXRPutqWW42GZWM4LOGe5UKxfU3tVmpTCzdqPd1tQiYkNbFsTM2o8sT+nI8BQ6M8uqSh28ImmSpCWSlkq6tEy68ZIaJH2wtTwd1MwsFVEIHEk+ZfMpLMOcDkwGRgPnShrdQrpvUzgftFUOamaWjgrNzySfVpwELI2IZRGxE5gNTGkm3ecobKKxJknxfEK7maVSWFGQuE+tv6QFJdczI2Jm8ftgYHnJsxXAyXu9SxoMnEVhRVOifRwd1MwstRTDBOsiYlyKbJpO7P8e8JWIaEi6M4iDmpmlVqHBzxXA4SXXQ4BVTdKMA2YXA1p/4ExJ9RHxq5YydVAzs5RUqf3U5gMjJA0HVgJTgfNKE0TE8D1vlW4Afl0uoIGDmpmltHv0c38V92ucRmFUsw6YFRGLJF1UfD7jzeTroGZmqVVq8m1EzAXmNrnXbDCLiI8nydNBzczSUTvdztvMrDmVan5Wi4OamaXmmpqZ5Up2Q5qDmpmlJKDONTUzy5MMxzQHNTNLSyjDDVAHNTNLzTU1M8uNwpSO7EY1BzUzSyfhrra14qBmZqll+YwCBzUzS6WwSWStS9EyBzUzS82jn2aWKxlufWZ6XWrm3f/YYsZ/4ArGnvWv/PsN9+3z/LkXXuHdn/gOA9/+eX7w0/v3ed7Q0MiE86/kw1/4z7YorgHvOvUYfn/rv7Dw9sv5/MfO2Od57x7duOnfPs28n1/KY7/4Gue975Q3nvU8kBuu/CS/u+XrPHHz1xl//PB9ft9RKOE/tVC1mpqkWcDfA2si4rhqvadWGhoaueSqm/nlNdM4bOBBnP6xq5k84XiOPmLQnjR9e/fgyi+dw10PPd1sHjNmP8DI4QPZsu21tip2h9apk7j6yx/irGnXsGr1Jv77x5dw98PPsuT5V/ak+dQ5E1iy7BXO/eK19DuoJ/Nv/RduuXs+u+obuPJLH+S3jy/m45deT5fOdRzYrWsN/zS1k/U+tWrW1G4AJlUx/5pauOgFjji8P28Z0p+uXTpz9hljmfvQM3ulGXBwL8YeO4wunev2+f3K1Ru575FFfHTK29uqyB3e2459C8uWr+PFlevZVd/A7b95kjNPG7NXmgB69jgAgB7dD2Djq9upb2ikV49uvP2tR/LTOx4HYFd9A69u3dHWf4RsSHg8Xq1GSKsW1CLiYWBDtfKvtZfXbmbwwL57rg8b2JeX125O/Puv/tttfOPif6BTlv+XlzODBvRh5eqNe65Xrd7IoAF99krzo5sfYuRbDuVPd3+LR2/6Kv/83VuJCIYN7se6TVuZfvlHeOhnX+H7XzuP7h20pgaF2lqSTy3UvE9N0oWSFkhasHbd2loXJ7GIpid5Je88vWfes/Tv24sTjxla4VJZOc3tAdb0X+PppxzDs8+t4JjJX2PC+f+fqy45h149utG5ro4TRh3OrFvncdpHvs32117n8x/ft0+uI9h97meHq6klFREzI2JcRIwb0H9ArYuT2GGHHLTP//UP7d+nzC/e8Lunl3HPvGcZ8/7L+ORX/4t585/jwn/5cbWKakWr1mzap3b9yrq9a9fnv+8Ufv1AoQ/0+RXreHHVekYMG8iqNRtZtWYTCxe9CMCc3z7FCaMOp6NyTS2Hxo4exl9fWsuLK9exc1c9t//mSSZPGNP6D4HLp01h0V3/l2fmXMH1/+8C3jF+JDO/+bEql9ieXPwiRw4dwNDD+tGlcx1nnzGWux/eux90xSsbmTB+FFDoEz1q2EBeWLmONeu3sHL1Ro4adggAE8aP2muAocPJcFTzPLU3qXPnOq768of4wMXTaWgIzn//KRxz5CBm3TYPgE984B2sXvcqp3/sKrZsew1JzJj9II//4mv07nlgjUvfMTU0NPLlq27mtv/4LHV14udznuDPy17hgrP/FoD/uv0Rrr7+HqZf/hEevemrSPCNa+5gw+ZtAHz5O7cw84qP07VLHS+sXMdnr/hZLf84NZXlZVJqrm+oIhlLNwETKZyqvBq4PCKuL/ebt71tXDz6uwVVKY9VR9/x02pdBEvh9SU307h9zX5FpGOOf2v85I4HE6U96ciDFkbEuJaeS5oEfJ/CuZ/XRcSVTZ5PAb4JNAL1wOcj4pFy76xaTS0izq1W3mZWYxWoqEmqA6YDZwArgPmS5kTE4pJkvwXmRERIGgPcDBxdLl/3qZlZKoXusoqsKDgJWBoRyyJiJzAbmFKaICK2xhvNyR4UphKW5aBmZukU91NL8gH6756yVfxcWJLTYGB5yfWK4r29XyedJenPwF3AJ1orngcKzCy1FK3PdWX61JrLZp+aWET8EvilpAkU+tf+rtwLHdTMLCVV6jDjFUDpZL8hwKqWEkfEw5KOlNQ/Ita1lM7NTzNLLUXzs5z5wAhJwyV1BaYCc/Z+j45SMYJKGgt0BdaXy9Q1NTNLpVLzaiOiXtI04F4KUzpmRcQiSRcVn88APgB8VNIuYAfw4WhlHpqDmpmlV6G5txExF5jb5N6Mku/fBr6dJk8HNTNLzdt5m1muZHiVlIOamaXkcz/NLG/c/DSz3BCuqZlZzmQ4pjmomdmbkOGo5qBmZqlleZNIBzUzSy27Ic1BzczejAxHNQc1M0tl9yaRWeWgZmbpePKtmeVNhmOag5qZpVWxTSKrwkHNzFLLcExzUDOzdGp4+HoiDmpmll6Go5qDmpml5ikdZpYr7lMzs/wQdHJQM7N8yW5U87mfZpbK7k0iK3DuJ5ImSVoiaamkS5t5fr6kZ4qfxySd0FqerqmZWWqVqKdJqgOmA2dQOK19vqQ5EbG4JNnzwGkRsVHSZGAmcHK5fB3UzCy1Cg0UnAQsjYhlhTw1G5gC7AlqEfFYSfongCGtZeqgZmappVgm1V/SgpLrmRExs/h9MLC85NkKytfCPgnc3doLHdTMLLUUFbV1ETEuRTbRbELpnRSC2t+29kIHNTNLJekgQAIrgMNLrocAq/Z9n8YA1wGTI2J9a5l69NPMUlPCf1oxHxghabikrsBUYM5e75GGArcD/xgRzyUpm2tqZpZeBWpqEVEvaRpwL1AHzIqIRZIuKj6fAVwG9AN+WOzHqy/TnAUc1MzsTajU1NuImAvMbXJvRsn3TwGfSpOng5qZpSQfkWdm+bF7RUFWeaDAzHLFNTUzSy3LNTUHNTNLzZtEmll++NxPM8uTrA8UOKiZWWpufppZrrimZma5kuGY5qBmZm9ChqOag5qZpSLI9DIpRTS7J1tNSFoLvFjrclRBf2BdrQthqeT139mwiBiwPxlIuofC308S6yJi0v68L61MBbW8krSgte1SLFv876z98tpPM8sVBzUzyxUHtbYxs/UkljH+d9ZOuU/NzHLFNTUzyxUHNTPLFQe1KpI0SdISSUslXVrr8ljrJM2StEbSH2tdFntzHNSqRFIdMB2YDIwGzpU0uralsgRuANp0sqhVloNa9ZwELI2IZRGxE5gNTKlxmawVEfEwsKHW5bA3z0GtegYDy0uuVxTvmVkVOahVT3Mrfj1/xqzKHNSqZwVweMn1EGBVjcpi1mE4qFXPfGCEpOGSugJTgTk1LpNZ7jmoVUlE1APTgHuBPwE3R8Si2pbKWiPpJuBxYJSkFZI+WesyWTpeJmVmueKampnlioOameWKg5qZ5YqDmpnlioOameWKg1o7IqlB0lOS/ijpFknd9yOvGyR9sPj9unKL7SVNlPT2N/GOFyTtc+pQS/ebpNma8l3/Kumf0pbR8sdBrX3ZEREnRsRxwE7gotKHxZ1BUouIT0XE4jJJJgKpg5pZLTiotV/zgKOKtagHJN0IPCupTtLVkuZLekbSpwFUcI2kxZLuAg7ZnZGkByWNK36fJOlJSU9L+q2kt1AInl8o1hLfIWmApNuK75gv6W+Kv+0n6T5Jf5B0LQnO8Zb0K0kLJS2SdGGTZ98tluW3kgYU7x0p6Z7ib+ZJOroif5uWGz6hvR2S1JnCPm33FG+dBBwXEc8XA8PmiBgv6QDgUUn3AW8FRgHHAwOBxcCsJvkOAH4ETCjmdXBEbJA0A9gaEd8pprsR+PeIeETSUAqrJo4BLgceiYgrJL0X2CtIteATxXccCMyXdFtErAd6AE9GxJckXVbMexqFA1Euioi/SDoZ+CFw+pv4a7ScclBrXw6U9FTx+zzgegrNwt9HxPPF++8GxuzuLwP6ACOACcBNEdEArJL0383kfwrw8O68IqKlfcX+Dhgt7amI9ZbUq/iOs4u/vUvSxgR/poslnVX8fnixrOuBRuAXxfs/A26X1LP4572l5N0HJHiHdSAOau3Ljog4sfRG8T/ubaW3gM9FxL1N0p1J61sfKUEaKHRbnBoRO5opS+J1d5ImUgiQp0bEdkkPAt1aSB7F925q+ndgVsp9avlzL/AZSV0AJI2U1AN4GJha7HMbBLyzmd8+DpwmaXjxtwcX728BepWku49CU5BiuhOLXx8Gzi/emwz0baWsfYCNxYB2NIWa4m6dgN21zfMoNGtfBZ6XdE7xHZJ0QivvsA7GQS1/rqPQX/Zk8fCQaynUyH8J/AV4FvhP4KGmP4yItRT6wW6X9DRvNP/uBM7aPVAAXAyMKw5ELOaNUdhvABMkPUmhGfxSK2W9B+gs6Rngm8ATJc+2AcdKWkihz+yK4v3zgU8Wy7cIb5FuTXiXDjPLFdfUzCxXHNTMLFcc1MwsVxzUzCxXHNTMLFcc1MwsVxzUzCxX/gfthrJrWJQg3gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate_model_ngrammChar(clf_rf_chars, model_name=\"clf_rf_chars_noStopWords\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": " \"hate_speech_snlp.ipynb\"",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}