NLP/NLP_With_Disaster_Tweets/NLP_with_disaster_tweets-with_pretrained_glove.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d0b6ba9-f744-4430-a0b0-b5f83e13eecf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.model_selection import GridSearchCV, StratifiedKFold\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3087da1e-e6fc-4e93-b0ad-d061ec1d13ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "## getting the data\n",
    "df_train = pd.read_csv(\"data/train.csv\")\n",
    "df_test = pd.read_csv(\"data/test.csv\")\n",
    "sample_submission = pd.read_csv(\"data/sample_submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "415c453f-2e85-488f-a070-0865f0369fc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  \\\n",
       "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
       "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
       "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
       "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d3613a7d-d526-4c81-be0b-ab82dd7e7311",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "keyword       61\n",
       "location    2533\n",
       "text           0\n",
       "target         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## checking for missing\n",
    "df_train.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "32e14424-354f-47a2-a7dd-826088b2b86f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "keyword\n",
       "fatalities               45\n",
       "deluge                   42\n",
       "armageddon               42\n",
       "sinking                  41\n",
       "damage                   41\n",
       "                         ..\n",
       "forest%20fire            19\n",
       "epicentre                12\n",
       "threat                   11\n",
       "inundation               10\n",
       "radiation%20emergency     9\n",
       "Name: count, Length: 221, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## checking the keyword column\n",
    "df_train[\"keyword\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "52bebafc-1905-4f92-8ac1-4744692559de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>48</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>Birmingham</td>\n",
       "      <td>@bbcmtd Wholesale Markets ablaze http://t.co/l...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>49</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>Est. September 2012 - Bristol</td>\n",
       "      <td>We always try to bring the heavy. #metal #RT h...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>50</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>AFRICA</td>\n",
       "      <td>#AFRICANBAZE: Breaking news:Nigeria flag set a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>52</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>Philadelphia, PA</td>\n",
       "      <td>Crying out for more! Set me ablaze</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>53</td>\n",
       "      <td>ablaze</td>\n",
       "      <td>London, UK</td>\n",
       "      <td>On plus side LOOK AT THE SKY LAST NIGHT IT WAS...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7575</th>\n",
       "      <td>10826</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>TN</td>\n",
       "      <td>On the bright side I wrecked http://t.co/uEa0t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7577</th>\n",
       "      <td>10829</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>#NewcastleuponTyne #UK</td>\n",
       "      <td>@widda16 ... He's gone. You can relax. I thoug...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7579</th>\n",
       "      <td>10831</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>Vancouver, Canada</td>\n",
       "      <td>Three days off from work and they've pretty mu...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7580</th>\n",
       "      <td>10832</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>London</td>\n",
       "      <td>#FX #forex #trading Cramer: Iger's 3 words tha...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7581</th>\n",
       "      <td>10833</td>\n",
       "      <td>wrecked</td>\n",
       "      <td>Lincoln</td>\n",
       "      <td>@engineshed Great atmosphere at the British Li...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5080 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  keyword                       location  \\\n",
       "31       48   ablaze                     Birmingham   \n",
       "32       49   ablaze  Est. September 2012 - Bristol   \n",
       "33       50   ablaze                         AFRICA   \n",
       "34       52   ablaze               Philadelphia, PA   \n",
       "35       53   ablaze                     London, UK   \n",
       "...     ...      ...                            ...   \n",
       "7575  10826  wrecked                             TN   \n",
       "7577  10829  wrecked         #NewcastleuponTyne #UK   \n",
       "7579  10831  wrecked              Vancouver, Canada   \n",
       "7580  10832  wrecked                        London    \n",
       "7581  10833  wrecked                        Lincoln   \n",
       "\n",
       "                                                   text  target  \n",
       "31    @bbcmtd Wholesale Markets ablaze http://t.co/l...       1  \n",
       "32    We always try to bring the heavy. #metal #RT h...       0  \n",
       "33    #AFRICANBAZE: Breaking news:Nigeria flag set a...       1  \n",
       "34                   Crying out for more! Set me ablaze       0  \n",
       "35    On plus side LOOK AT THE SKY LAST NIGHT IT WAS...       0  \n",
       "...                                                 ...     ...  \n",
       "7575  On the bright side I wrecked http://t.co/uEa0t...       0  \n",
       "7577  @widda16 ... He's gone. You can relax. I thoug...       0  \n",
       "7579  Three days off from work and they've pretty mu...       0  \n",
       "7580  #FX #forex #trading Cramer: Iger's 3 words tha...       0  \n",
       "7581  @engineshed Great atmosphere at the British Li...       0  \n",
       "\n",
       "[5080 rows x 5 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[df_train[\"location\"].notna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a28b781e-a5f4-4394-a929-e162d6cbf73d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target\n",
       "0    4342\n",
       "1    3271\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "096a961f-65e2-474c-bd94-6c4cc25d92f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([\"HELP I'M IN A WHIRLWIND OF NOSTALGIA\",\n",
       "       'Check out Ameribag Healthy Back Bag Shoulder Cross Body Backpack Khaki Tan Beige Nylon http://t.co/r4k7TyLofJ @eBay',\n",
       "       '#ModiMinistry Rly tragedy in MP: Some live to recount horror http://t.co/s43wE7Oe2i',\n",
       "       \"@MI_Country_Hick pfft! I wish I had a bot like that. Instead I'm daily inundated with 140 characters of the same unhinged machismo bullshit.\",\n",
       "       'that horrible sinking feeling when you\\x89Ûªve been at home on your phone for a while and you realise its been on 3G this whole time'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## checking the nature of the tweets\n",
    "\n",
    "df_train[\"text\"].sample(5).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "789a3db4-9661-45e4-a9b1-b178b7eeddde",
   "metadata": {},
   "outputs": [],
   "source": [
    "## for cleaning function\n",
    "## - remove urls\n",
    "## - remove # tags\n",
    "## - remove html special characters eg &amp;\n",
    "## - remove @text\n",
    "## - remove [01:04 UTC]\n",
    "## - strip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2d60ad2e-12cd-4956-9e59-bc08f066ca59",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "## function to preprocessing of text\n",
    "def process_text(text):\n",
    "    ## patterns to remove\n",
    "    rem_pat_1 = \"([@]|https?:)\\S*\"\n",
    "    rem_pat_2 = \"&\\S+;\"\n",
    "    rem_pat_3 = \"\\[\\d+:\\d+.+\\]\" ## removing timestamp. eg. [01:04 UTC]\n",
    "    rem_pat_4 = \"[\\-_.+#]\" ## to remove symbols (make sure to bring last to avoid affecting first two patterns)\n",
    "    combined_rem_pat = f\"({rem_pat_1})|({rem_pat_2})|({rem_pat_3})|({rem_pat_4})\"\n",
    "\n",
    "    text = re.sub(combined_rem_pat, \"\", text) ## removing text that match patterns\n",
    "    text = text.strip() ## removing trailing white spaces\n",
    "    text = text.lower() ## lowercasing\n",
    "\n",
    "    return text\n",
    "\n",
    "## function for tokenizing of string\n",
    "def tokenize(text):\n",
    "    return re.split(\"\\s+\", text)\n",
    "\n",
    "## function to remove stop words\n",
    "def remove_stopwords(token_list):\n",
    "    l = []\n",
    "    for word in token_list:\n",
    "        if word not in stopwords:\n",
    "            l.append(word)\n",
    "    return l\n",
    "\n",
    "def full_text_process(text):\n",
    "    text = process_text(text)\n",
    "    # text = tokenize(text)\n",
    "    # text = remove_stopwords(text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6ea6ad0f-f9d9-4617-a111-eeb636216b2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "## creating datatransformers\n",
    "class GloveVectorizer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, filepath=\".\", encoding=\"utf-8\"):\n",
    "        self.filepath = filepath\n",
    "        self.encoding = encoding\n",
    "\n",
    "    def fitX(self, X):\n",
    "        self.unique_words = set()\n",
    "        self.vectors = []\n",
    "        self.word2idx = {}\n",
    "\n",
    "        ## getting unique words\n",
    "        for text in X:\n",
    "            self.unique_words |= set(text.split())\n",
    "\n",
    "        ## builing word to index dictionary and vector matrix\n",
    "        idx = 0\n",
    "        with open(self.filepath, encoding=self.encoding) as  file:\n",
    "            for line in file:\n",
    "                line = line.split()\n",
    "                word = line[0]\n",
    "                if word in self.unique_words:\n",
    "                    self.word2idx[word] = idx\n",
    "                    self.vectors.append(line[1:])\n",
    "                    idx+=1\n",
    "\n",
    "        self.vectors = np.array(self.vectors, dtype=np.float32)\n",
    "\n",
    "    def fitY(self, y):\n",
    "        ## creating integer labels for each category\n",
    "        unique = {x for x in y}\n",
    "        self.label2int = dict([(v,i) for i,v in enumerate(unique)])\n",
    "\n",
    "\n",
    "    def transformX(self, X):\n",
    "        N = len(X)\n",
    "        transformed_x = np.zeros((N, self.vectors.shape[1]), dtype=np.float32)\n",
    "        for i in range(N):\n",
    "            mat = []\n",
    "            line = X[i].lower().split()\n",
    "            for word in line:\n",
    "                if word in self.word2idx:\n",
    "                    mat.append(self.vectors[self.word2idx[word]])\n",
    "\n",
    "            if len(mat) > 0: transformed_x[i] = np.mean(mat, axis=0)\n",
    "            else: print(f\"Sentence at index:{i} has no word in the vector dictionary\")\n",
    "\n",
    "        return np.array(transformed_x)\n",
    "\n",
    "    def transformY(self, y):\n",
    "        return np.array([self.label2int[word] for word in y])\n",
    "        \n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        self.fitX(X)\n",
    "\n",
    "        if y is not None: self.fitY(y)\n",
    "\n",
    "        return self\n",
    "    \n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        if y is not None: return self.transformX(X), self.transformY(y)\n",
    "        else: return self.transformX(X)\n",
    "\n",
    "    def fit_transform(self, X, y=None):\n",
    "        self.fit(X,y)\n",
    "        return self.transform(X,y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1c2f0e9d-009b-4bff-8c5f-9506b8a01653",
   "metadata": {},
   "outputs": [],
   "source": [
    "## getting the training and testing processed data\n",
    "## train\n",
    "train_text = df_train[\"text\"].apply(lambda x: full_text_process(x)).values\n",
    "train_target = df_train[\"target\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "02a21d6a-3a34-4248-8715-eec82f334e56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'our deeds are the reason of this earthquake may allah forgive us all'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6fde0bbf-14b8-4f6f-8ca2-85790555b4eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_state = 42\n",
    "# X_train, X_test, y_train, y_test = train_test_split(train_text, train_target, test_size=0.3, shuffle=True, random_state=random_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e6bb7d75-e45d-461f-8aba-6f68d4602d48",
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_path = \"../../../Machine_Learning_With_Python/NLP_LEARN/data/glove6B/glove.6B.50d.txt\"\n",
    "vectorizer = GloveVectorizer(glove_path, encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "78516039-f917-46fe-b680-a41e46720430",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GloveVectorizer(filepath=&#x27;../../../Machine_Learning_With_Python/NLP_LEARN/data/glove6B/glove.6B.50d.txt&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GloveVectorizer</label><div class=\"sk-toggleable__content\"><pre>GloveVectorizer(filepath=&#x27;../../../Machine_Learning_With_Python/NLP_LEARN/data/glove6B/glove.6B.50d.txt&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "GloveVectorizer(filepath='../../../Machine_Learning_With_Python/NLP_LEARN/data/glove6B/glove.6B.50d.txt')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.fit(train_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "af4ca60e-c80a-4e84-9e05-e1d0a423969e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence at index:24 has no word in the vector dictionary\n",
      "Sentence at index:783 has no word in the vector dictionary\n",
      "Sentence at index:2586 has no word in the vector dictionary\n",
      "Sentence at index:3667 has no word in the vector dictionary\n",
      "Sentence at index:3681 has no word in the vector dictionary\n",
      "Sentence at index:3683 has no word in the vector dictionary\n",
      "Sentence at index:4092 has no word in the vector dictionary\n",
      "Sentence at index:4504 has no word in the vector dictionary\n",
      "Sentence at index:5115 has no word in the vector dictionary\n",
      "Sentence at index:5353 has no word in the vector dictionary\n",
      "Sentence at index:5983 has no word in the vector dictionary\n",
      "Sentence at index:5987 has no word in the vector dictionary\n",
      "Sentence at index:5988 has no word in the vector dictionary\n",
      "Sentence at index:5998 has no word in the vector dictionary\n",
      "Sentence at index:6313 has no word in the vector dictionary\n",
      "Sentence at index:6522 has no word in the vector dictionary\n",
      "Sentence at index:6705 has no word in the vector dictionary\n",
      "Sentence at index:6907 has no word in the vector dictionary\n",
      "Sentence at index:7210 has no word in the vector dictionary\n"
     ]
    }
   ],
   "source": [
    "X = vectorizer.transform(train_text)\n",
    "y = train_target"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ae2299f-e593-4283-a0cb-0af5e29438bb",
   "metadata": {},
   "source": [
    "### GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9e5ad016-e2b5-4c32-a1ae-38076883487c",
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bf4477c-46b0-4362-8155-02a0cbcb34de",
   "metadata": {},
   "source": [
    "#### RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "e34dcfd5-78ed-461e-ac6d-b3af50f62588",
   "metadata": {},
   "outputs": [],
   "source": [
    "# params = {\n",
    "#     \"n_estimators\": [200, 300],\n",
    "#     \"max_depth\": [ 8, 10],\n",
    "# }\n",
    "\n",
    "# grid = GridSearchCV(RandomForestClassifier(), params, cv=kf, scoring=\"accuracy\", verbose=10)\n",
    "# grid.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8b1928f7-2b28-48ac-bac0-f7d561c7e113",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score: 0.7874681083655521\n",
      "Best Parameters: {'max_depth': 10, 'n_estimators': 200}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Score:\", grid.best_score_)\n",
    "print(\"Best Parameters:\", grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54f6c7e4-77b2-44ca-8eb7-2c9d6f779a1e",
   "metadata": {},
   "source": [
    "### ExtraTreesClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "befa604f-f5c0-412d-85ea-437749785116",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n",
      "[CV 1/5; 1/4] START max_depth=16, n_estimators=600..............................\n",
      "[CV 1/5; 1/4] END max_depth=16, n_estimators=600;, score=0.807 total time=  16.5s\n",
      "[CV 2/5; 1/4] START max_depth=16, n_estimators=600..............................\n",
      "[CV 2/5; 1/4] END max_depth=16, n_estimators=600;, score=0.807 total time=  18.2s\n",
      "[CV 3/5; 1/4] START max_depth=16, n_estimators=600..............................\n",
      "[CV 3/5; 1/4] END max_depth=16, n_estimators=600;, score=0.785 total time=  17.6s\n",
      "[CV 4/5; 1/4] START max_depth=16, n_estimators=600..............................\n",
      "[CV 4/5; 1/4] END max_depth=16, n_estimators=600;, score=0.793 total time=  16.2s\n",
      "[CV 5/5; 1/4] START max_depth=16, n_estimators=600..............................\n",
      "[CV 5/5; 1/4] END max_depth=16, n_estimators=600;, score=0.793 total time=  16.2s\n",
      "[CV 1/5; 2/4] START max_depth=16, n_estimators=700..............................\n",
      "[CV 1/5; 2/4] END max_depth=16, n_estimators=700;, score=0.806 total time=  17.8s\n",
      "[CV 2/5; 2/4] START max_depth=16, n_estimators=700..............................\n",
      "[CV 2/5; 2/4] END max_depth=16, n_estimators=700;, score=0.806 total time=  17.2s\n",
      "[CV 3/5; 2/4] START max_depth=16, n_estimators=700..............................\n",
      "[CV 3/5; 2/4] END max_depth=16, n_estimators=700;, score=0.784 total time=  12.3s\n",
      "[CV 4/5; 2/4] START max_depth=16, n_estimators=700..............................\n",
      "[CV 4/5; 2/4] END max_depth=16, n_estimators=700;, score=0.798 total time=  16.0s\n",
      "[CV 5/5; 2/4] START max_depth=16, n_estimators=700..............................\n",
      "[CV 5/5; 2/4] END max_depth=16, n_estimators=700;, score=0.797 total time=  14.9s\n",
      "[CV 1/5; 3/4] START max_depth=18, n_estimators=600..............................\n",
      "[CV 1/5; 3/4] END max_depth=18, n_estimators=600;, score=0.805 total time=  18.3s\n",
      "[CV 2/5; 3/4] START max_depth=18, n_estimators=600..............................\n",
      "[CV 2/5; 3/4] END max_depth=18, n_estimators=600;, score=0.810 total time=  17.7s\n",
      "[CV 3/5; 3/4] START max_depth=18, n_estimators=600..............................\n",
      "[CV 3/5; 3/4] END max_depth=18, n_estimators=600;, score=0.790 total time=  17.2s\n",
      "[CV 4/5; 3/4] START max_depth=18, n_estimators=600..............................\n",
      "[CV 4/5; 3/4] END max_depth=18, n_estimators=600;, score=0.797 total time=  17.1s\n",
      "[CV 5/5; 3/4] START max_depth=18, n_estimators=600..............................\n",
      "[CV 5/5; 3/4] END max_depth=18, n_estimators=600;, score=0.798 total time=  17.2s\n",
      "[CV 1/5; 4/4] START max_depth=18, n_estimators=700..............................\n",
      "[CV 1/5; 4/4] END max_depth=18, n_estimators=700;, score=0.805 total time=  21.8s\n",
      "[CV 2/5; 4/4] START max_depth=18, n_estimators=700..............................\n",
      "[CV 2/5; 4/4] END max_depth=18, n_estimators=700;, score=0.809 total time=  20.5s\n",
      "[CV 3/5; 4/4] START max_depth=18, n_estimators=700..............................\n",
      "[CV 3/5; 4/4] END max_depth=18, n_estimators=700;, score=0.787 total time=  20.2s\n",
      "[CV 4/5; 4/4] START max_depth=18, n_estimators=700..............................\n",
      "[CV 4/5; 4/4] END max_depth=18, n_estimators=700;, score=0.795 total time=  20.6s\n",
      "[CV 5/5; 4/4] START max_depth=18, n_estimators=700..............................\n",
      "[CV 5/5; 4/4] END max_depth=18, n_estimators=700;, score=0.793 total time=  21.4s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-7 {color: black;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=ExtraTreesClassifier(),\n",
       "             param_grid={&#x27;max_depth&#x27;: [16, 18], &#x27;n_estimators&#x27;: [600, 700]},\n",
       "             scoring=&#x27;accuracy&#x27;, verbose=10)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-15\" type=\"checkbox\" ><label for=\"sk-estimator-id-15\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=ExtraTreesClassifier(),\n",
       "             param_grid={&#x27;max_depth&#x27;: [16, 18], &#x27;n_estimators&#x27;: [600, 700]},\n",
       "             scoring=&#x27;accuracy&#x27;, verbose=10)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-16\" type=\"checkbox\" ><label for=\"sk-estimator-id-16\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: ExtraTreesClassifier</label><div class=\"sk-toggleable__content\"><pre>ExtraTreesClassifier()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-17\" type=\"checkbox\" ><label for=\"sk-estimator-id-17\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">ExtraTreesClassifier</label><div class=\"sk-toggleable__content\"><pre>ExtraTreesClassifier()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=ExtraTreesClassifier(),\n",
       "             param_grid={'max_depth': [16, 18], 'n_estimators': [600, 700]},\n",
       "             scoring='accuracy', verbose=10)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {\n",
    "    \"n_estimators\": [600, 700],\n",
    "    \"max_depth\": [16, 18],\n",
    "}\n",
    "\n",
    "grid = GridSearchCV(ExtraTreesClassifier(), params, cv=kf, scoring=\"accuracy\", verbose=10)\n",
    "grid.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "fc082f5e-2bcc-4e14-aa9b-71dde072ac86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score: 0.7998154448262861\n",
      "Best Parameters: {'max_depth': 18, 'n_estimators': 600}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Score:\", grid.best_score_)\n",
    "print(\"Best Parameters:\", grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32e7275b-8afa-4b98-ae7e-033cb55b67df",
   "metadata": {},
   "source": [
    "#### XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "9e16ac0f-a3f2-466d-a00c-8019d852f1e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n",
      "[CV 1/5; 1/12] START learning_rate=0.05, max_depth=6, n_estimators=200..........\n",
      "[CV 1/5; 1/12] END learning_rate=0.05, max_depth=6, n_estimators=200;, score=0.805 total time=   3.1s\n",
      "[CV 2/5; 1/12] START learning_rate=0.05, max_depth=6, n_estimators=200..........\n",
      "[CV 2/5; 1/12] END learning_rate=0.05, max_depth=6, n_estimators=200;, score=0.809 total time=   3.0s\n",
      "[CV 3/5; 1/12] START learning_rate=0.05, max_depth=6, n_estimators=200..........\n",
      "[CV 3/5; 1/12] END learning_rate=0.05, max_depth=6, n_estimators=200;, score=0.798 total time=   2.6s\n",
      "[CV 4/5; 1/12] START learning_rate=0.05, max_depth=6, n_estimators=200..........\n",
      "[CV 4/5; 1/12] END learning_rate=0.05, max_depth=6, n_estimators=200;, score=0.792 total time=   3.3s\n",
      "[CV 5/5; 1/12] START learning_rate=0.05, max_depth=6, n_estimators=200..........\n",
      "[CV 5/5; 1/12] END learning_rate=0.05, max_depth=6, n_estimators=200;, score=0.787 total time=   2.8s\n",
      "[CV 1/5; 2/12] START learning_rate=0.05, max_depth=6, n_estimators=300..........\n",
      "[CV 1/5; 2/12] END learning_rate=0.05, max_depth=6, n_estimators=300;, score=0.802 total time=   4.4s\n",
      "[CV 2/5; 2/12] START learning_rate=0.05, max_depth=6, n_estimators=300..........\n",
      "[CV 2/5; 2/12] END learning_rate=0.05, max_depth=6, n_estimators=300;, score=0.808 total time=   6.3s\n",
      "[CV 3/5; 2/12] START learning_rate=0.05, max_depth=6, n_estimators=300..........\n",
      "[CV 3/5; 2/12] END learning_rate=0.05, max_depth=6, n_estimators=300;, score=0.796 total time=   5.1s\n",
      "[CV 4/5; 2/12] START learning_rate=0.05, max_depth=6, n_estimators=300..........\n",
      "[CV 4/5; 2/12] END learning_rate=0.05, max_depth=6, n_estimators=300;, score=0.787 total time=   5.6s\n",
      "[CV 5/5; 2/12] START learning_rate=0.05, max_depth=6, n_estimators=300..........\n",
      "[CV 5/5; 2/12] END learning_rate=0.05, max_depth=6, n_estimators=300;, score=0.791 total time=   5.3s\n",
      "[CV 1/5; 3/12] START learning_rate=0.05, max_depth=8, n_estimators=200..........\n",
      "[CV 1/5; 3/12] END learning_rate=0.05, max_depth=8, n_estimators=200;, score=0.809 total time=   8.2s\n",
      "[CV 2/5; 3/12] START learning_rate=0.05, max_depth=8, n_estimators=200..........\n",
      "[CV 2/5; 3/12] END learning_rate=0.05, max_depth=8, n_estimators=200;, score=0.810 total time=   6.6s\n",
      "[CV 3/5; 3/12] START learning_rate=0.05, max_depth=8, n_estimators=200..........\n",
      "[CV 3/5; 3/12] END learning_rate=0.05, max_depth=8, n_estimators=200;, score=0.793 total time=   6.4s\n",
      "[CV 4/5; 3/12] START learning_rate=0.05, max_depth=8, n_estimators=200..........\n",
      "[CV 4/5; 3/12] END learning_rate=0.05, max_depth=8, n_estimators=200;, score=0.789 total time=   6.4s\n",
      "[CV 5/5; 3/12] START learning_rate=0.05, max_depth=8, n_estimators=200..........\n",
      "[CV 5/5; 3/12] END learning_rate=0.05, max_depth=8, n_estimators=200;, score=0.788 total time=   8.6s\n",
      "[CV 1/5; 4/12] START learning_rate=0.05, max_depth=8, n_estimators=300..........\n",
      "[CV 1/5; 4/12] END learning_rate=0.05, max_depth=8, n_estimators=300;, score=0.809 total time=   9.8s\n",
      "[CV 2/5; 4/12] START learning_rate=0.05, max_depth=8, n_estimators=300..........\n",
      "[CV 2/5; 4/12] END learning_rate=0.05, max_depth=8, n_estimators=300;, score=0.807 total time=  11.5s\n",
      "[CV 3/5; 4/12] START learning_rate=0.05, max_depth=8, n_estimators=300..........\n",
      "[CV 3/5; 4/12] END learning_rate=0.05, max_depth=8, n_estimators=300;, score=0.798 total time=  10.5s\n",
      "[CV 4/5; 4/12] START learning_rate=0.05, max_depth=8, n_estimators=300..........\n",
      "[CV 4/5; 4/12] END learning_rate=0.05, max_depth=8, n_estimators=300;, score=0.790 total time=  10.9s\n",
      "[CV 5/5; 4/12] START learning_rate=0.05, max_depth=8, n_estimators=300..........\n",
      "[CV 5/5; 4/12] END learning_rate=0.05, max_depth=8, n_estimators=300;, score=0.786 total time=  10.7s\n",
      "[CV 1/5; 5/12] START learning_rate=0.05, max_depth=10, n_estimators=200.........\n",
      "[CV 1/5; 5/12] END learning_rate=0.05, max_depth=10, n_estimators=200;, score=0.804 total time=  12.6s\n",
      "[CV 2/5; 5/12] START learning_rate=0.05, max_depth=10, n_estimators=200.........\n",
      "[CV 2/5; 5/12] END learning_rate=0.05, max_depth=10, n_estimators=200;, score=0.806 total time=  12.3s\n",
      "[CV 3/5; 5/12] START learning_rate=0.05, max_depth=10, n_estimators=200.........\n",
      "[CV 3/5; 5/12] END learning_rate=0.05, max_depth=10, n_estimators=200;, score=0.789 total time=  12.6s\n",
      "[CV 4/5; 5/12] START learning_rate=0.05, max_depth=10, n_estimators=200.........\n",
      "[CV 4/5; 5/12] END learning_rate=0.05, max_depth=10, n_estimators=200;, score=0.781 total time=  12.6s\n",
      "[CV 5/5; 5/12] START learning_rate=0.05, max_depth=10, n_estimators=200.........\n",
      "[CV 5/5; 5/12] END learning_rate=0.05, max_depth=10, n_estimators=200;, score=0.783 total time=  12.4s\n",
      "[CV 1/5; 6/12] START learning_rate=0.05, max_depth=10, n_estimators=300.........\n",
      "[CV 1/5; 6/12] END learning_rate=0.05, max_depth=10, n_estimators=300;, score=0.807 total time=  15.9s\n",
      "[CV 2/5; 6/12] START learning_rate=0.05, max_depth=10, n_estimators=300.........\n",
      "[CV 2/5; 6/12] END learning_rate=0.05, max_depth=10, n_estimators=300;, score=0.807 total time=  16.2s\n",
      "[CV 3/5; 6/12] START learning_rate=0.05, max_depth=10, n_estimators=300.........\n",
      "[CV 3/5; 6/12] END learning_rate=0.05, max_depth=10, n_estimators=300;, score=0.791 total time=  15.2s\n",
      "[CV 4/5; 6/12] START learning_rate=0.05, max_depth=10, n_estimators=300.........\n",
      "[CV 4/5; 6/12] END learning_rate=0.05, max_depth=10, n_estimators=300;, score=0.784 total time=  15.5s\n",
      "[CV 5/5; 6/12] START learning_rate=0.05, max_depth=10, n_estimators=300.........\n",
      "[CV 5/5; 6/12] END learning_rate=0.05, max_depth=10, n_estimators=300;, score=0.790 total time=  15.5s\n",
      "[CV 1/5; 7/12] START learning_rate=0.1, max_depth=6, n_estimators=200...........\n",
      "[CV 1/5; 7/12] END learning_rate=0.1, max_depth=6, n_estimators=200;, score=0.809 total time=   4.3s\n",
      "[CV 2/5; 7/12] START learning_rate=0.1, max_depth=6, n_estimators=200...........\n",
      "[CV 2/5; 7/12] END learning_rate=0.1, max_depth=6, n_estimators=200;, score=0.810 total time=   3.4s\n",
      "[CV 3/5; 7/12] START learning_rate=0.1, max_depth=6, n_estimators=200...........\n",
      "[CV 3/5; 7/12] END learning_rate=0.1, max_depth=6, n_estimators=200;, score=0.791 total time=   3.8s\n",
      "[CV 4/5; 7/12] START learning_rate=0.1, max_depth=6, n_estimators=200...........\n",
      "[CV 4/5; 7/12] END learning_rate=0.1, max_depth=6, n_estimators=200;, score=0.778 total time=   3.6s\n",
      "[CV 5/5; 7/12] START learning_rate=0.1, max_depth=6, n_estimators=200...........\n",
      "[CV 5/5; 7/12] END learning_rate=0.1, max_depth=6, n_estimators=200;, score=0.783 total time=   3.7s\n",
      "[CV 1/5; 8/12] START learning_rate=0.1, max_depth=6, n_estimators=300...........\n",
      "[CV 1/5; 8/12] END learning_rate=0.1, max_depth=6, n_estimators=300;, score=0.810 total time=   5.2s\n",
      "[CV 2/5; 8/12] START learning_rate=0.1, max_depth=6, n_estimators=300...........\n",
      "[CV 2/5; 8/12] END learning_rate=0.1, max_depth=6, n_estimators=300;, score=0.814 total time=   5.5s\n",
      "[CV 3/5; 8/12] START learning_rate=0.1, max_depth=6, n_estimators=300...........\n",
      "[CV 3/5; 8/12] END learning_rate=0.1, max_depth=6, n_estimators=300;, score=0.788 total time=   5.0s\n",
      "[CV 4/5; 8/12] START learning_rate=0.1, max_depth=6, n_estimators=300...........\n",
      "[CV 4/5; 8/12] END learning_rate=0.1, max_depth=6, n_estimators=300;, score=0.789 total time=   5.2s\n",
      "[CV 5/5; 8/12] START learning_rate=0.1, max_depth=6, n_estimators=300...........\n",
      "[CV 5/5; 8/12] END learning_rate=0.1, max_depth=6, n_estimators=300;, score=0.780 total time=   5.4s\n",
      "[CV 1/5; 9/12] START learning_rate=0.1, max_depth=8, n_estimators=200...........\n",
      "[CV 1/5; 9/12] END learning_rate=0.1, max_depth=8, n_estimators=200;, score=0.806 total time=   7.9s\n",
      "[CV 2/5; 9/12] START learning_rate=0.1, max_depth=8, n_estimators=200...........\n",
      "[CV 2/5; 9/12] END learning_rate=0.1, max_depth=8, n_estimators=200;, score=0.812 total time=   6.9s\n",
      "[CV 3/5; 9/12] START learning_rate=0.1, max_depth=8, n_estimators=200...........\n",
      "[CV 3/5; 9/12] END learning_rate=0.1, max_depth=8, n_estimators=200;, score=0.787 total time=   6.8s\n",
      "[CV 4/5; 9/12] START learning_rate=0.1, max_depth=8, n_estimators=200...........\n",
      "[CV 4/5; 9/12] END learning_rate=0.1, max_depth=8, n_estimators=200;, score=0.784 total time=   7.1s\n",
      "[CV 5/5; 9/12] START learning_rate=0.1, max_depth=8, n_estimators=200...........\n",
      "[CV 5/5; 9/12] END learning_rate=0.1, max_depth=8, n_estimators=200;, score=0.784 total time=   6.6s\n",
      "[CV 1/5; 10/12] START learning_rate=0.1, max_depth=8, n_estimators=300..........\n",
      "[CV 1/5; 10/12] END learning_rate=0.1, max_depth=8, n_estimators=300;, score=0.802 total time=   8.9s\n",
      "[CV 2/5; 10/12] START learning_rate=0.1, max_depth=8, n_estimators=300..........\n",
      "[CV 2/5; 10/12] END learning_rate=0.1, max_depth=8, n_estimators=300;, score=0.810 total time=   8.7s\n",
      "[CV 3/5; 10/12] START learning_rate=0.1, max_depth=8, n_estimators=300..........\n",
      "[CV 3/5; 10/12] END learning_rate=0.1, max_depth=8, n_estimators=300;, score=0.791 total time=   9.0s\n",
      "[CV 4/5; 10/12] START learning_rate=0.1, max_depth=8, n_estimators=300..........\n",
      "[CV 4/5; 10/12] END learning_rate=0.1, max_depth=8, n_estimators=300;, score=0.783 total time=   9.1s\n",
      "[CV 5/5; 10/12] START learning_rate=0.1, max_depth=8, n_estimators=300..........\n",
      "[CV 5/5; 10/12] END learning_rate=0.1, max_depth=8, n_estimators=300;, score=0.786 total time=   9.2s\n",
      "[CV 1/5; 11/12] START learning_rate=0.1, max_depth=10, n_estimators=200.........\n",
      "[CV 1/5; 11/12] END learning_rate=0.1, max_depth=10, n_estimators=200;, score=0.808 total time=   9.3s\n",
      "[CV 2/5; 11/12] START learning_rate=0.1, max_depth=10, n_estimators=200.........\n",
      "[CV 2/5; 11/12] END learning_rate=0.1, max_depth=10, n_estimators=200;, score=0.804 total time=   9.4s\n",
      "[CV 3/5; 11/12] START learning_rate=0.1, max_depth=10, n_estimators=200.........\n",
      "[CV 3/5; 11/12] END learning_rate=0.1, max_depth=10, n_estimators=200;, score=0.787 total time=   9.3s\n",
      "[CV 4/5; 11/12] START learning_rate=0.1, max_depth=10, n_estimators=200.........\n",
      "[CV 4/5; 11/12] END learning_rate=0.1, max_depth=10, n_estimators=200;, score=0.781 total time=   9.6s\n",
      "[CV 5/5; 11/12] START learning_rate=0.1, max_depth=10, n_estimators=200.........\n",
      "[CV 5/5; 11/12] END learning_rate=0.1, max_depth=10, n_estimators=200;, score=0.790 total time=  10.3s\n",
      "[CV 1/5; 12/12] START learning_rate=0.1, max_depth=10, n_estimators=300.........\n",
      "[CV 1/5; 12/12] END learning_rate=0.1, max_depth=10, n_estimators=300;, score=0.809 total time=  11.4s\n",
      "[CV 2/5; 12/12] START learning_rate=0.1, max_depth=10, n_estimators=300.........\n",
      "[CV 2/5; 12/12] END learning_rate=0.1, max_depth=10, n_estimators=300;, score=0.807 total time=  11.7s\n",
      "[CV 3/5; 12/12] START learning_rate=0.1, max_depth=10, n_estimators=300.........\n",
      "[CV 3/5; 12/12] END learning_rate=0.1, max_depth=10, n_estimators=300;, score=0.789 total time=  12.3s\n",
      "[CV 4/5; 12/12] START learning_rate=0.1, max_depth=10, n_estimators=300.........\n",
      "[CV 4/5; 12/12] END learning_rate=0.1, max_depth=10, n_estimators=300;, score=0.787 total time=  12.7s\n",
      "[CV 5/5; 12/12] START learning_rate=0.1, max_depth=10, n_estimators=300.........\n",
      "[CV 5/5; 12/12] END learning_rate=0.1, max_depth=10, n_estimators=300;, score=0.792 total time=  11.3s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-7 {color: black;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None, device=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     feature_types=None, gamma=None,\n",
       "                                     grow_policy=None, importance_typ...\n",
       "                                     max_cat_threshold=None,\n",
       "                                     max_cat_to_onehot=None,\n",
       "                                     max_delta_step=None, max_depth=None,\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     multi_strategy=None, n_estimators=None,\n",
       "                                     n_jobs=None, num_parallel_tree=None,\n",
       "                                     random_state=None, ...),\n",
       "             param_grid={&#x27;learning_rate&#x27;: [0.05, 0.1], &#x27;max_depth&#x27;: [6, 8, 10],\n",
       "                         &#x27;n_estimators&#x27;: [200, 300]},\n",
       "             scoring=&#x27;accuracy&#x27;, verbose=10)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None, device=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     feature_types=None, gamma=None,\n",
       "                                     grow_policy=None, importance_typ...\n",
       "                                     max_cat_threshold=None,\n",
       "                                     max_cat_to_onehot=None,\n",
       "                                     max_delta_step=None, max_depth=None,\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     multi_strategy=None, n_estimators=None,\n",
       "                                     n_jobs=None, num_parallel_tree=None,\n",
       "                                     random_state=None, ...),\n",
       "             param_grid={&#x27;learning_rate&#x27;: [0.05, 0.1], &#x27;max_depth&#x27;: [6, 8, 10],\n",
       "                         &#x27;n_estimators&#x27;: [200, 300]},\n",
       "             scoring=&#x27;accuracy&#x27;, verbose=10)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, device=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              multi_strategy=None, n_estimators=None, n_jobs=None,\n",
       "              num_parallel_tree=None, random_state=None, ...)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, device=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              multi_strategy=None, n_estimators=None, n_jobs=None,\n",
       "              num_parallel_tree=None, random_state=None, ...)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None, device=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     feature_types=None, gamma=None,\n",
       "                                     grow_policy=None, importance_typ...\n",
       "                                     max_cat_threshold=None,\n",
       "                                     max_cat_to_onehot=None,\n",
       "                                     max_delta_step=None, max_depth=None,\n",
       "                                     max_leaves=None, min_child_weight=None,\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     multi_strategy=None, n_estimators=None,\n",
       "                                     n_jobs=None, num_parallel_tree=None,\n",
       "                                     random_state=None, ...),\n",
       "             param_grid={'learning_rate': [0.05, 0.1], 'max_depth': [6, 8, 10],\n",
       "                         'n_estimators': [200, 300]},\n",
       "             scoring='accuracy', verbose=10)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {\n",
    "    \"n_estimators\": [200, 300],\n",
    "    \"max_depth\": [6, 8, 10],\n",
    "    \"learning_rate\": [0.05, 0.1]\n",
    "}\n",
    "\n",
    "grid = GridSearchCV(XGBClassifier(), params, cv=kf, scoring=\"accuracy\", verbose=10)\n",
    "grid.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "454858a8-2a74-44e8-8f7a-d4b0b70f5701",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score: 0.7983689429621839\n",
      "Best Parameters: {'learning_rate': 0.05, 'max_depth': 6, 'n_estimators': 200}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Score:\", grid.best_score_)\n",
    "print(\"Best Parameters:\", grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c303e8b5-d247-427a-b2bf-2b45b01ce7c1",
   "metadata": {},
   "source": [
    "### Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "352ce788-5dfe-4b03-bba5-b7d36ed57d90",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_on_dataframe(model, data):\n",
    "    '''Make a prediction on a dataframe'''\n",
    "    pred_text = df_test[\"text\"].apply(lambda x: full_text_process(x)).values\n",
    "    pred_text = vectorizer.transform(pred_text)\n",
    "    return grid.predict(pred_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "17e4d6ee-d121-4830-a7e9-896c57f6fa82",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence at index:14 has no word in the vector dictionary\n",
      "Sentence at index:748 has no word in the vector dictionary\n",
      "Sentence at index:1433 has no word in the vector dictionary\n",
      "Sentence at index:1554 has no word in the vector dictionary\n",
      "Sentence at index:1733 has no word in the vector dictionary\n",
      "Sentence at index:2379 has no word in the vector dictionary\n",
      "Sentence at index:2562 has no word in the vector dictionary\n",
      "Sentence at index:2567 has no word in the vector dictionary\n",
      "Sentence at index:2569 has no word in the vector dictionary\n",
      "Sentence at index:2571 has no word in the vector dictionary\n",
      "Sentence at index:3015 has no word in the vector dictionary\n"
     ]
    }
   ],
   "source": [
    "# pred_text = df_test[\"text\"].apply(lambda x: full_text_process(x)).values\n",
    "# pred_text = vectorizer.transform(pred_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c20efddc-829a-4d4e-b46f-d192f1505b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# preds = grid.predict(pred_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "8418d8ff-29ca-4b7d-b6c8-02f4677af265",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## creating submission dataframe\n",
    "# ids = df_test[\"id\"].values\n",
    "\n",
    "# sub_df = pd.DataFrame({\"id\": ids, \"target\": preds.astype(int)})\n",
    "\n",
    "# ## saving as csv\n",
    "# if not os.path.exists(\"x__submissions\"):\n",
    "#     os.mkdir(\"x__submissions\")\n",
    "\n",
    "# sub_df.to_csv(\"x__submissions/sub_glove_1.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71617e0c-49ff-4489-be49-5120c2241636",
   "metadata": {},
   "source": [
    "### Saving Necessary Objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "222493fa-4ca2-48a6-a8fa-7077f42391fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "306336a5-e2d8-403d-aeda-82cdcbbbf9a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "## creating transformer\n",
    "class TweetTransformer():\n",
    "    def __init__(self, word2idx, vectors):\n",
    "        self.word2idx = word2idx\n",
    "        self.vectors = vectors\n",
    "\n",
    "    ## function to preprocessing of text\n",
    "    def process_text(self, text):\n",
    "        ## patterns to remove\n",
    "        rem_pat_1 = \"([@]|https?:)\\S*\"\n",
    "        rem_pat_2 = \"&\\S+;\"\n",
    "        rem_pat_3 = \"\\[\\d+:\\d+.+\\]\" ## removing timestamp. eg. [01:04 UTC]\n",
    "        rem_pat_4 = \"[\\-_.+#]\" ## to remove symbols (make sure to bring last to avoid affecting first two patterns)\n",
    "        combined_rem_pat = f\"({rem_pat_1})|({rem_pat_2})|({rem_pat_3})|({rem_pat_4})\"\n",
    "    \n",
    "        text = re.sub(combined_rem_pat, \"\", text) ## removing text that match patterns\n",
    "        text = text.strip() ## removing trailing white spaces\n",
    "        text = text.lower() ## lowercasing\n",
    "    \n",
    "        return text\n",
    "    \n",
    "    \n",
    "    def full_text_process(self, text):\n",
    "        text = self.process_text(text)\n",
    "        return text\n",
    "\n",
    "\n",
    "    def transform(self, X):\n",
    "        \n",
    "        if isinstance(X, str):\n",
    "            X = [self.full_text_process(X)]\n",
    "            \n",
    "        N = len(X)\n",
    "        X = [self.full_text_process(x) for x in X]\n",
    "            \n",
    "        transformed_x = np.zeros((N, self.vectors.shape[1]), dtype=np.float32)\n",
    "        for i in range(N):\n",
    "            mat = []\n",
    "            line = X[i].lower().split()\n",
    "            for word in line:\n",
    "                if word in self.word2idx:\n",
    "                    mat.append(self.vectors[self.word2idx[word]])\n",
    "\n",
    "            if len(mat) > 0: transformed_x[i] = np.mean(mat, axis=0)\n",
    "            else: print(f\"Sentence at index:{i} has no word in the vector dictionary\")\n",
    "\n",
    "        return np.array(transformed_x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d13a1d22-d21b-4028-899a-3f36a87e1107",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-6 {color: black;}#sk-container-id-6 pre{padding: 0;}#sk-container-id-6 div.sk-toggleable {background-color: white;}#sk-container-id-6 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-6 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-6 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-6 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-6 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-6 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-6 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-6 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-6 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-6 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-6 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-6 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-6 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-6 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-6 div.sk-item {position: relative;z-index: 1;}#sk-container-id-6 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-6 div.sk-item::before, #sk-container-id-6 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-6 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-6 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-6 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-6 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-6 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-6 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-6 div.sk-label-container {text-align: center;}#sk-container-id-6 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-6 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-6\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>ExtraTreesClassifier(max_depth=16, n_estimators=500)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-14\" type=\"checkbox\" checked><label for=\"sk-estimator-id-14\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">ExtraTreesClassifier</label><div class=\"sk-toggleable__content\"><pre>ExtraTreesClassifier(max_depth=16, n_estimators=500)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "ExtraTreesClassifier(max_depth=16, n_estimators=500)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimator = ExtraTreesClassifier(**{'max_depth': 16, 'n_estimators': 500})\n",
    "estimator.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "43a287e8-70a8-4e79-98d5-4fb75a27819f",
   "metadata": {},
   "outputs": [],
   "source": [
    "## creating transformer\n",
    "transformer = TweetTransformer(vectorizer.word2idx, vectorizer.vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "10b30ac3-72eb-448a-aba1-58c94751f0e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_folder = \"x__serialized\"\n",
    "if not os.path.exists(save_folder):\n",
    "    os.mkdir(save_folder)\n",
    "\n",
    "## saving model\n",
    "with open(os.path.join(save_folder, \"disaster_tweets_classifier.pickle\"), \"wb\") as file:\n",
    "    pickle.dump(estimator, file)\n",
    "\n",
    "# ## saving transformer\n",
    "# with open(os.path.join(save_folder, \"disaster_tweets_transformer.pickle\"), \"wb\") as file:\n",
    "#     pickle.dump(transformer, file)\n",
    "\n",
    "## saving word2idx and vectors\n",
    "with open(os.path.join(save_folder, \"disaster_tweets_word_vectors.pickle\"), \"wb\") as file:\n",
    "    pickle.dump({\"word2idx\": vectorizer.word2idx, \"vectors\": vectorizer.vectors}, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "63a3b9b7-72ec-4957-8efa-a6d1bbe5bdbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence at index:24 has no word in the vector dictionary\n",
      "Sentence at index:783 has no word in the vector dictionary\n",
      "Sentence at index:2586 has no word in the vector dictionary\n",
      "Sentence at index:3667 has no word in the vector dictionary\n",
      "Sentence at index:3681 has no word in the vector dictionary\n",
      "Sentence at index:3683 has no word in the vector dictionary\n",
      "Sentence at index:4092 has no word in the vector dictionary\n",
      "Sentence at index:4504 has no word in the vector dictionary\n",
      "Sentence at index:5115 has no word in the vector dictionary\n",
      "Sentence at index:5353 has no word in the vector dictionary\n",
      "Sentence at index:5983 has no word in the vector dictionary\n",
      "Sentence at index:5987 has no word in the vector dictionary\n",
      "Sentence at index:5988 has no word in the vector dictionary\n",
      "Sentence at index:5998 has no word in the vector dictionary\n",
      "Sentence at index:6313 has no word in the vector dictionary\n",
      "Sentence at index:6522 has no word in the vector dictionary\n",
      "Sentence at index:6705 has no word in the vector dictionary\n",
      "Sentence at index:6907 has no word in the vector dictionary\n",
      "Sentence at index:7210 has no word in the vector dictionary\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimator.predict(transformer.transform(train_text))[:30]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}