Understanding Anova & Chi Square Test in Titanic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyforest import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"~/Downloads/TitanicTrain.csv\")\n",
    "test = pd.read_csv(\"~/Downloads/TitanicTest.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the World of ML, Generalization is the key to win ML Competitions.\n",
    "\n",
    "The Generalized Data will get you the winning edge."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Cabin_Status</th>\n",
       "      <th>Titles</th>\n",
       "      <th>Family</th>\n",
       "      <th>Family_On_Board</th>\n",
       "      <th>Fare_Per_Person</th>\n",
       "      <th>titles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>No Cabin Found</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>Mr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>Cabin Allotted</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>Mrs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>No Cabin Found</td>\n",
       "      <td>Miss</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>Miss</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>Cabin Allotted</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>Mrs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>No Cabin Found</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>Mr</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived  Pclass     Sex   Age     Fare Embarked    Cabin_Status Titles  \\\n",
       "0         0       3    male  22.0   7.2500        S  No Cabin Found     Mr   \n",
       "1         1       1  female  38.0  71.2833        C  Cabin Allotted    Mrs   \n",
       "2         1       3  female  26.0   7.9250        S  No Cabin Found   Miss   \n",
       "3         1       1  female  35.0  53.1000        S  Cabin Allotted    Mrs   \n",
       "4         0       3    male  35.0   8.0500        S  No Cabin Found     Mr   \n",
       "\n",
       "   Family Family_On_Board  Fare_Per_Person titles  \n",
       "0       1         Singles           7.2500     Mr  \n",
       "1       1         Singles          71.2833    Mrs  \n",
       "2       1         Singles           7.9250   Miss  \n",
       "3       1         Singles          53.1000    Mrs  \n",
       "4       1         Singles           8.0500     Mr  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Singles', 'Large Family', 'Small Family'], dtype=object)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.Family_On_Board.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>Family_On_Board</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Large Family</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Singles</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Small Family</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Survived</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Survived</th>\n",
       "      <td>60</td>\n",
       "      <td>31</td>\n",
       "      <td>446</td>\n",
       "      <td>252</td>\n",
       "      <td>59</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Family_On_Board Large Family     Singles      Small Family    \n",
       "Survived                   0   1       0    1            1   0\n",
       "Survived                  60  31     446  252           59  43"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(train.groupby(\"Family_On_Board\")[\"Survived\"].value_counts()).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Age is Dependant on gender or Title, Family on Board\n",
    "\n",
    "### Hypothesis Test - Numerical Vs Numerical\n",
    "\n",
    "Because I have a Hypothesis (Opinion) about a particular situation that it might be true and it might not be true.\n",
    "\n",
    "Types of Test\n",
    "\n",
    "1. Z Test - Population metrics are available such as Pop Mean or SD.\n",
    "\n",
    "2. T Test - This test is suitable when the sample mean or sd is available.\n",
    "\n",
    "### Is there any example for the above tests.\n",
    "\n",
    "The Average IQ of Adult Population is 100. A Pyschological Researcher belives that it has changed. He takes a random sample of 100 people and performs the IQ Test and he finds that the sample average IQ is now 105 with sd of 10.\n",
    "\n",
    "Find out if the statement about the population is correct or not.\n",
    "\n",
    "T test is heavily inclined test towards Sample where n is less than or equal to 30 rows of data.\n",
    "\n",
    "### Hyp Test - Numerical Vs Categorical\n",
    "\n",
    "#### ANOVA - Analysis of Variance\n",
    "\n",
    "1. If I have more than two categories/groups to consider\n",
    "2. Homogeneity of Variance and Normality is the basic assumption to be applied on the data.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Libraries for ANOVA\n",
    "import scipy.stats as stats\n",
    "import statsmodels.api as sms\n",
    "from statsmodels.formula.api import ols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the Statement - Age is Dependant on Titles\n",
    "\n",
    "### Null Hyp\n",
    "\n",
    "H0: The Age of the Passengers is equal across the Gender. In other words, the Age is indifferent from the Gender.\n",
    "\n",
    "### Alt Hyp:\n",
    "\n",
    "H1: The Age of the Passenger is Dependant on the Gender.\n",
    "\n",
    "### Level of Significance\n",
    "\n",
    "Alpha =0.05\n",
    "\n",
    "### Test Inference\n",
    "\n",
    "If the P Value is less than the LOS, then we reject the Null Hypothesis because there is a very low possibility for the H0 to be true in all respects and hence the test holds significance as per the statistics generated."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "newtrain = train.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Mr', 'Mrs', 'Miss', 'Master', 'Others', 'Ms'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newtrain.Titles.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "F_onewayResult(statistic=9.19313968472156, pvalue=0.0025002091122819548)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.f_oneway(newtrain.loc[newtrain.Sex==\"male\", \"Age\"],\n",
    "              newtrain.loc[newtrain.Sex==\"female\", \"Age\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Anova Table - FINDING IF AGE IS DEPENDANT ON SEX, TITLES AND FAMILY ON BOARD\n",
    "model = ols(\"Age~Family_On_Board+Sex+Titles\", data = newtrain).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>df</th>\n",
       "      <th>sum_sq</th>\n",
       "      <th>mean_sq</th>\n",
       "      <th>F</th>\n",
       "      <th>PR(&gt;F)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Family_On_Board</th>\n",
       "      <td>2.0</td>\n",
       "      <td>17798.285492</td>\n",
       "      <td>8899.142746</td>\n",
       "      <td>71.030338</td>\n",
       "      <td>2.581870e-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sex</th>\n",
       "      <td>1.0</td>\n",
       "      <td>263.285072</td>\n",
       "      <td>263.285072</td>\n",
       "      <td>2.101464</td>\n",
       "      <td>1.475161e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Titles</th>\n",
       "      <td>5.0</td>\n",
       "      <td>36783.477943</td>\n",
       "      <td>7356.695589</td>\n",
       "      <td>58.718978</td>\n",
       "      <td>9.155593e-53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Residual</th>\n",
       "      <td>876.0</td>\n",
       "      <td>109750.978344</td>\n",
       "      <td>125.286505</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    df         sum_sq      mean_sq          F        PR(>F)\n",
       "Family_On_Board    2.0   17798.285492  8899.142746  71.030338  2.581870e-29\n",
       "Sex                1.0     263.285072   263.285072   2.101464  1.475161e-01\n",
       "Titles             5.0   36783.477943  7356.695589  58.718978  9.155593e-53\n",
       "Residual         876.0  109750.978344   125.286505        NaN           NaN"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms.stats.anova_lm(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inference\n",
    "\n",
    "The Age is dependant on Family on Board, Titles and Gender as the P Value is less than 0.05 in all the cases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'combined' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-53e014255a8b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcombined\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Titles\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Age\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdescribe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'combined' is not defined"
     ]
    }
   ],
   "source": [
    "combined.groupby(\"Titles\")[\"Age\"].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This shows that the Age is Dependant on the Gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'combined' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-df5276fb3f7a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcombined\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'combined' is not defined"
     ]
    }
   ],
   "source": [
    "combined.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Problem: I want to find out if the Survival is Dependant on the Age\n",
    "\n",
    "Step01: H0: Survival is totally independent of the Age.\n",
    "\n",
    "Step02: H1: Survival is heavily relying on Age\n",
    "\n",
    "Step03: Choose the Level of Alpha/Significance -0.05\n",
    "\n",
    "Step 04: Find the P Value and Conclude the F Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# F Test\n",
    "model = ols(\"Survived~Age\", data = newtrain).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>df</th>\n",
       "      <th>sum_sq</th>\n",
       "      <th>mean_sq</th>\n",
       "      <th>F</th>\n",
       "      <th>PR(&gt;F)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Age</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.523436</td>\n",
       "      <td>0.523436</td>\n",
       "      <td>2.217958</td>\n",
       "      <td>0.13677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Residual</th>\n",
       "      <td>883.0</td>\n",
       "      <td>208.387298</td>\n",
       "      <td>0.235999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             df      sum_sq   mean_sq         F   PR(>F)\n",
       "Age         1.0    0.523436  0.523436  2.217958  0.13677\n",
       "Residual  883.0  208.387298  0.235999       NaN      NaN"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms.stats.anova_lm(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inference - Since we find that the P VAlue is greater than 5% and hence we fail to reject the H0 which is that the Survival is Not Dependant on the Age "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Cabin_Status</th>\n",
       "      <th>Titles</th>\n",
       "      <th>Family</th>\n",
       "      <th>Family_On_Board</th>\n",
       "      <th>Fare_Per_Person</th>\n",
       "      <th>titles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>No Cabin Found</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>Mr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>Cabin Allotted</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>Mrs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>No Cabin Found</td>\n",
       "      <td>Miss</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>Miss</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>Cabin Allotted</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>Mrs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>No Cabin Found</td>\n",
       "      <td>Mr</td>\n",
       "      <td>1</td>\n",
       "      <td>Singles</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>Mr</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived  Pclass     Sex   Age     Fare Embarked    Cabin_Status Titles  \\\n",
       "0         0       3    male  22.0   7.2500        S  No Cabin Found     Mr   \n",
       "1         1       1  female  38.0  71.2833        C  Cabin Allotted    Mrs   \n",
       "2         1       3  female  26.0   7.9250        S  No Cabin Found   Miss   \n",
       "3         1       1  female  35.0  53.1000        S  Cabin Allotted    Mrs   \n",
       "4         0       3    male  35.0   8.0500        S  No Cabin Found     Mr   \n",
       "\n",
       "   Family Family_On_Board  Fare_Per_Person titles  \n",
       "0       1         Singles           7.2500     Mr  \n",
       "1       1         Singles          71.2833    Mrs  \n",
       "2       1         Singles           7.9250   Miss  \n",
       "3       1         Singles          53.1000    Mrs  \n",
       "4       1         Singles           8.0500     Mr  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newtrain.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chi Square Test\n",
    "\n",
    "When we wish to find out the relationship between the Cat Variables\n",
    "\n",
    "For e.g. I want to find out if the Survival is based on PClass\n",
    "\n",
    "H0: The Survival is Independent of PClass\n",
    "\n",
    "H1: The Survival is Dependent of the PClass\n",
    "\n",
    "Test of Hypothesis - Since the variables are categorical in nature and hence we would apply Chi Square Goodness of Fit Test.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>Survived</th>\n",
       "      <th colspan=\"5\" halign=\"left\">0</th>\n",
       "      <th colspan=\"6\" halign=\"left\">1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Titles</th>\n",
       "      <th>Mr</th>\n",
       "      <th>Miss</th>\n",
       "      <th>Mrs</th>\n",
       "      <th>Master</th>\n",
       "      <th>Others</th>\n",
       "      <th>Miss</th>\n",
       "      <th>Mrs</th>\n",
       "      <th>Mr</th>\n",
       "      <th>Master</th>\n",
       "      <th>Others</th>\n",
       "      <th>Ms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Titles</th>\n",
       "      <td>435</td>\n",
       "      <td>55</td>\n",
       "      <td>25</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>126</td>\n",
       "      <td>97</td>\n",
       "      <td>80</td>\n",
       "      <td>23</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Survived    0                           1                         \n",
       "Titles     Mr Miss Mrs Master Others Miss Mrs  Mr Master Others Ms\n",
       "Titles    435   55  25     17     15  126  97  80     23     11  1"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(newtrain.groupby(\"Survived\")[\"Titles\"].value_counts()).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "array = np.array([[435,55,25,17,15,0],[80,126,97,23,11,1]])\n",
    "chistats, pvalue, df, exp_freq = stats.chi2_contingency(array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The P Value is 2.6822979231163236e-59\n"
     ]
    }
   ],
   "source": [
    "#print(chistats)\n",
    "print(\"The P Value is\",pvalue)\n",
    "#print(df)\n",
    "#print(exp_freq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inference\n",
    "\n",
    "Since the P Value is Less than 0.05 and Hence Reject the Ho which is that the Survival is NOT BASED on P Value. Here we find that the Survival has a relation with the Passenger Class.\n",
    "\n",
    "#### Title and PClass appears to be a very strong variable as per Chi Sq Test.\n",
    "\n",
    "We will build a Random Forest Model to find if this is correct or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = newtrain.drop([\"Survived\", \"titles\"], axis = 1)\n",
    "y = newtrain.Survived"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "ohetrain = pd.get_dummies(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "xtrain,xtest,ytrain,ytest = train_test_split(ohetrain,y,test_size = 0.20,\n",
    "                                             random_state=123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(criterion = \"entropy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\It\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "rfmodel = rf.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = rf.predict(xtest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Classification Model Metrics & Finding Feature Importance\n",
    "\n",
    "1. Accuracy\n",
    "2. Classification Report\n",
    "3. Confusion Matrix\n",
    "4. Cohen's Kappa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accuracy\n",
    "from sklearn.metrics import matthews_corrcoef, accuracy_score,classification_report, confusion_matrix, cohen_kappa_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Accuracy of RF Model is 0.8022598870056498\n"
     ]
    }
   ],
   "source": [
    "print(\"The Accuracy of RF Model is\", accuracy_score(ytest, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAD8CAYAAAB9532ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XuYVNWZ7/HvT2hBBQlBYxiNNhq8cNFWWjLeW42OYxwM0QxekmCi4THH8ZzjDCYkePIQjXcnUWNMJBk1Ot5GR8+DIYpExVtUaKChAeMVjOAVcyQKgtK8549ajUVb3V1FX6qa/fs8Tz29L2ut/e5l4Vtr7V27FBGYmZnZlm+rcgdgZmZm3cNJ38zMLCOc9M3MzDLCSd/MzCwjnPTNzMwywknfzMwsI5z0zczMMsJJ38zMLCOc9M3MzDKid7kDMMu3ww47RHV1dbnDMDPrMebOnbsyInYspqyTvlWU6upq6uvryx2GmVmPIenVYst6et/MzCwjnPTNzMwywknfzMwsI5z0zczMMsI38llFaVyxiupJ08sdhplZt1l22Ve67Vge6ZuZmWWEk74VTdJYSSFp73LHYmZmpXPSt1KcCjwJnFLuQMzMrHRO+lYUSf2AQ4AzSUlf0laSrpe0WNLvJf1B0slp3yhJj0maK2mGpMFlDN/MzHDSt+J9FXgwIl4A/irpAOBrQDUwEjgLOAhAUhXwC+DkiBgF3Ahc3FrDkiZIqpdU37RmVdeehZlZhvnufSvWqcDVafnOtF4F3B0RG4A3JT2a9u8FjABmSgLoBbzRWsMRMRWYCtBn8NDokujNzMxJ39onaRBwFDBCUpBL4gHc11oVYHFEHNRNIZqZWRE8vW/FOBm4JSJ2i4jqiPgCsBRYCZyUru3vBNSl8s8DO0raON0vaXg5Ajczs0846VsxTuXTo/r/Bv4OWA4sAm4AngVWRcRH5D4oXC5pAdAAHNx94ZqZWSGe3rd2RURdgW3XQu6u/oj4IF0CmA00pv0NwOHdGaeZmbXNSd866veSPgNsDVwUEW92pLGROw+gvhsfSWlmliVO+tYhhWYBzMysMvmavpmZWUY46ZuZmWWEk76ZmVlGOOmbmZllhJO+mZlZRjjpm5mZZYSTvpmZWUY46ZuZmWWEk76ZmVlG+Il8VlEaV6yietL0codhZl1kmR+zXVYe6ZuZmWWER/olkNRE+hW55KsRsayLjrUMeB/YALwFfKujP2Yj6WbgCGBVaveciHi6Y5GamVlP4ZF+aT6MiJq817JiKknqtZnHOzIi9gPqgR8VW6md450fETXAJOCGEtr0B0Qzsx7OSb+DJFVLekLSvPQ6OG2vk/SopNtJswOSviFptqQGSTeU8GHgceCLqY1jJT2djnW3pH5p+zJJP5b0JPD1EtvcQ9KDkuamc9k7bb9Z0s8kPQpcLumIFHuDpPmS+ivnSkmLJDVKGpd3/rMk3SPpz5Juk6SiO9bMzDqdR2+l2UZSQ1peGhFjgbeBYyJiraShwB1AbSozGhgREUsl7QOMAw6JiI8lXQ+cDtxSxHFPABol7QBcAHw5IlZL+gHwr8CFqdzaiDi0yHP5Jz65VDEVODsiXpT0JeB64Ki0b890vCZJ95O7JPBU+rCxFvgaUAPsB+wAzJH0eKq7PzAceB14CjgEeLJlIJImABMAem2/Y5Hhm5lZqZz0S/NhmhrPVwVcJ6kGaCKXJJvNjoilafloYBS5pAiwDbkPDG15NN1HsJBcsj8UGAY8ldrYGsi/Jn9XEedwpaQLgHeAM1PyPhi4O28g3iev/N0R0ZSWnwJ+Juk24N6IWC7pUOCOVOYtSY8BBwJ/S+e/HCB9WKqmQNKPiKnkPnjQZ/DQKOIczMxsMzjpd9x55G6024/c5ZK1eftW5y0L+F1E/LCEto+MiJUbG8hl5ZkRcWor5Ve3sj3f+RFxT16b2wPvFfgw86k2I+IySdOB44FnJH2Z3Hm1Zl3echN+v5mZlZWv6XfcAOCNiNgAfBNo7Tr9w8DJkj4HIOmzknYr8VjPAIdIar4Wv62kPdup06aI+BuwVNLXU5uStF+hspL2iIjGiLic3M2Fe5O7N2CcpF6SdgQOB2Z3JCYzM+saTvoddz0wXtIz5Kb2C462I2IJuSn6hyQtBGYCg0s5UES8A5wB3JHaeIZc4u2o08lN9S8AFgMntlLuf6cb9hYAHwIPAPeRu/ywAHgE+H5Hv1poZmZdQxG+hGqVo7a2Nurr68sdhplZjyFpbkTUtl/SI30zM7PM8I1VZSbpWTa9Wx7gmxHRWKh8kW3+ktzX4/JdExE3bW6bZmbW8znpl1lEfKkL2jyns9s0M7Oez9P7ZmZmGeGkb2ZmlhFO+mZmZhnhpG9mZpYRTvpmZmYZ4aRvZmaWEU76ZmZmGeHv6VtFaVyxiupJ08sdhlmPsuyyr5Q7BOshPNI3MzPLCI/0rVWSmoD8xwF/NSKWlSkcMzPrICd9a8uHEVFTaiVJvSKiqSsCMjOzzefpfSuJpGpJT0ial14Hp+11kh6VdDtpdkDSNyTNltQg6QZJvcoavJlZxnmkb23ZRlJDWl4aEWOBt4FjImKtpKHAHUDz7ziPBkZExFJJ+wDjgEMi4mNJ1wOnA7d08zmYmVnipG9tKTS9XwVcJ6kGaAL2zNs3OyKWpuWjgVHAHEkA25D7wPApkiYAEwB6bb9j50VvZmabcNK3Up0HvAXsR+7y0Nq8favzlgX8LiJ+2F6DETEVmArQZ/DQ6LxQzcwsn6/pW6kGAG9ExAbgm0Br1+kfBk6W9DkASZ+VtFs3xWhmZgU46VuprgfGS3qG3NT+6kKFImIJcAHwkKSFwExgcLdFaWZmn+LpfWtVRPQrsO1FYN+8TT9M22cBs1qUvQu4q+siNDOzUjjpW0UZufMA6v1IUTOzLuHpfTMzs4xw0jczM8sIJ30zM7OMcNI3MzPLCCd9MzOzjHDSNzMzywgnfTMzs4xw0jczM8sIJ30zM7OMcNI3MzPLCD+G1ypK44pVVE+aXu4wrIdY5kc2m5XEI30zM7OMcNI3MzPLCCf9Hk7SIEkN6fWmpBV5639KZaolnZZXp07S7zsxhpB0a956b0nvdOYxzMys43xNv4eLiHeBGgBJU4APIuKqFsWqgdOA27sojNXACEnbRMSHwDHAikIFJfWOiPVdFIeZmbXBI/0tmKQP0uJlwGFp9H9eizLbSbpR0hxJ8yWdmLYPlzQ71VkoaWg7h3sAaL6r6lTgjrxjTJE0VdJDwC0F4pwgqV5SfdOaVZt5tmZm1h4n/WyYBDwRETUR8fMW+yYDj0TEgcCRwJWStgPOBq6JiBqgFljezjHuBE6R1BfYF3i2xf5RwIkRcVrLihExNSJqI6K217YDSj45MzMrjqf37VhgjKSJab0vsCvwNDBZ0i7AvRHxYluNRMRCSdXkRvl/KFBkWpr6NzOzMnHSNwEnRcTzLbY/J+lZclP2MySdFRGPtNPWNOAqoA4Y1GLf6s4I1szMNp+n97PhfaB/K/tmAOdKEoCk/dPf3YFXIuJacsl83yKOcyNwYUQ0djxkMzPrbE762bAQWC9pQcsb+YCLgCpgoaRFaR1gHLBIUgOwNwVuwGspIpZHxDWdGLeZmXUiRUS5YzDbqLa2Nurr68sdhplZjyFpbkTUFlPWI30zM7OM8I18VhRJg4CHC+w6Oj0gyMzMKpyTvhUl/8l/ZmbWM3l638zMLCOc9M3MzDLCSd/MzCwjnPTNzMwywknfzMwsI5z0zczMMsJJ38zMLCP8PX2rKI0rVlE9aXq5w7Ausuyyr5Q7BLNM80jfzMwsI5z0K4ykyZIWS1ooqUHSlzqp3cNSuw2StumMNgsco07S77uibTMz6zhP71cQSQcBJwAHRMQ6STsAW3dS86cDV0XETZ3UnpmZ9TAe6VeWwcDKiFgHEBErI+J1SaMkPSZprqQZkgZL6i1pjqQ6AEmXSrq4UKOSzgL+GfixpNvStvNT/YWSfpK2VUv6s6TfSlok6TZJX5b0lKQXJY1O5UZL+pOk+envXgWOuZ2kG9Mx5ks6sSs6zMzMiuekX1keAr4g6QVJ10s6QlIV8Avg5IgYBdwIXBwR64EzgF9JOgY4DvhJoUYj4rfANOD8iDhd0rHAUGA0uR/RGSXp8FT8i8A1wL7A3sBpwKHAROBHqcyfgcMjYn/gx8AlBQ47GXgkIg4EjgSulLRdofgkTZBUL6m+ac2q4nrKzMxK5un9ChIRH0gaBRxGLlHeBfwUGAHMlATQC3gjlV8s6VbgfuCgiPioyEMdm17z03o/ch8C/gIsjYhGAEmLgYcjIiQ1AtWp/ADgd5KGAgFUtXKMMZImpvW+wK7AcwXOeyowFaDP4KFR5DmYmVmJnPQrTEQ0AbOAWSnRngMsjoiDWqkyEngP2KmEwwi4NCJu2GSjVA2sy9u0IW99A5+8Xy4CHo2IsanOrFaOcVJEPF9CXGZm1oU8vV9BJO2VRs/NasiNjHdMN/khqUrS8LT8NWAQcDhwraTPFHmoGcB3JPVL7ews6XMlhDoAWJGWz2jjGOcqTU9I2r+E9s3MrAs46VeWfuSmzZdIWggMI3fN/GTgckkLgAbg4HRn/2XAmRHxAnAduWvx7YqIh4DbgafTbMI9QP8S4rwCuFTSU+QuNxRyEblp/4WSFqV1MzMrI0X4EqpVjj6Dh8bg8VeXOwzrIn4in1nnkzQ3ImqLKetr+lZRRu48gHonBjOzLuGkv4WRdB8wpMXmH0TEjHLEY2ZmlcNJfwsTEWPLHYOZmVUm38hnZmaWEU76ZmZmGeGkb2ZmlhFO+mZmZhnhpG9mZpYRTvpmZmYZ4aRvZmaWEU76ZmZmGeGH81hFaVyxiupJ08sdRkXwc+rNrLN5pJ9hkpokNUhaJOluSdu2UXaKpIndGZ+ZmXUuJ/1s+zAiaiJiBPARcHa5AzIzs67jpG/NngC+CCDpW5IWSlog6daWBSV9V9KctP+/m2cIJH09zRoskPR42jZc0uw0o7BQ0tBuPSszM9vI1/QNSb2BfwQelDQcmAwcEhErJX22QJV7I+I3qe5PgTOBXwA/Bv4hIlZI+kwqezZwTUTcJmlroFdXn4+ZmRXmkX62bSOpAagH/gL8B3AUcE9ErASIiL8WqDdC0hOSGoHTgeFp+1PAzZK+yyfJ/WngR5J+AOwWER+2bEzSBEn1kuqb1qzqzPMzM7M8TvrZ1nxNvyYizo2IjwAB0U69m4F/iYiRwE+AvgARcTZwAfAFoEHSoIi4HRgDfAjMkHRUy8YiYmpE1EZEba9tB3TayZmZ2aac9K2lh4F/ljQIoJXp/f7AG5KqyI30SWX3iIhnI+LHwErgC5J2B16JiGuBacC+XX4GZmZWkK/p2yYiYrGki4HHJDUB84EzWhT7P8CzwKtAI7kPAQBXphv1RO7DwwJgEvANSR8DbwIXdvlJmJlZQYpobybXrPv0GTw0Bo+/utxhVAQ/nMfMiiFpbkTUFlPWI32rKCN3HkC9k52ZWZfwNX0zM7OMcNI3MzPLCCd9MzOzjHDSNzMzywgnfTMzs4xw0jczM8sIJ30zM7OMcNI3MzPLCCd9MzOzjHDSNzMzywg/htcqSuOKVVRPml7uMLqEn6VvZuXmkb6ZmVlGOOmbmZllhJN+DyFpkKSG9HpT0oq89T+lMtWSTsurUyfp950YQ0i6NW+9t6R3OvMYZmbWdXxNv4eIiHeBGgBJU4APIuKqFsWqgdOA27sojNXACEnbRMSHwDHAikIFJfWOiPVdFIeZmW0Gj/S3AJI+SIuXAYel0f95LcpsJ+lGSXMkzZd0Yto+XNLsVGehpKHtHO4BoPmOtFOBO/KOMUXSVEkPAbdsRttmZtaFnPS3LJOAJyKiJiJ+3mLfZOCRiDgQOBK4UtJ2wNnANRFRA9QCy9s5xp3AKZL6AvsCz7bYPwo4MSJOK7ZtSRMk1Uuqb1qzquiTNTOz0nh6PzuOBcZImpjW+wK7Ak8DkyXtAtwbES+21UhELJRUTW6U/4cCRaalqX+KbTsipgJTAfoMHholnZWZmRXNI/3sEHBSmgWoiYhdI+K5iLgdGAN8CMyQdFQRbU0DriJvaj/P6uaFzWzbzMy6iJP+luV9oH8r+2YA50oSgKT909/dgVci4lpyyXzfIo5zI3BhRDS2VWgz2zYzsy7ipL9lWQisl7Sg5Y18wEVAFbBQ0qK0DjAOWCSpAdgbuKW9g0TE8oi4poh4Sm7bzMy6jiJ8CdUqR21tbdTX15c7DDOzHkPS3IioLaasR/pmZmYZ4bv3bROSBgEPF9h1dHpAkJmZ9VBO+raJ/Cf/mZnZlsXT+2ZmZhnhpG9mZpYRTvpmZmYZ4aRvZmaWEU76ZmZmGeGkb2ZmlhFO+mZmZhnh7+lbRWlcsYrqSdPLHUaHLLvsK+UOwcysII/0zczMMsJJP2MkNUlqyHtVd0KbZ0v6Vlq+WdLJHW3TzMw6n6f3s+fDiOjUx+xGxK87sz0zM+saHukbkqolPSFpXnodnLbXSXpM0n9JekHSZZJOlzRbUqOkPVK5KZImtmjzaEn35a0fI+ne7j0zMzPL56SfPdvkTe03J+W3gWMi4gBgHHBtXvn9gP8FjAS+CewZEaOB3wLntnGcR4B9JO2Y1r8N3FSooKQJkuol1TetWbXZJ2ZmZm3z9H72FJrerwKuk1QDNAF75u2bExFvAEh6GXgobW8EjmztIBERkm4FviHpJuAg4FutlJ0KTAXoM3holH5KZmZWDCd9AzgPeIvcqH4rYG3evnV5yxvy1jfQ/vvnJuD+1N7dEbG+U6I1M7PN4qRvAAOA5RGxQdJ4oFdnNBoRr0t6HbgAOKYz2jQzs83na/oGcD0wXtIz5Kb2V3di27cBr0XEkk5s08zMNoMifAnVuo6k64D5EfEfxZTvM3hoDB5/dRdH1bX8RD4z606S5kZEbTFlPb1vXUbSXHKzBv9WbJ2ROw+g3knTzKxLOOlbl4mIUeWOwczMPuFr+mZmZhnhpG9mZpYRTvpmZmYZ4aRvZmaWEU76ZmZmGeGkb2ZmlhFO+mZmZhnhpG9mZpYRTvpmZmYZ4SfyWUVpXLGK6knTyx1Gyfy8fTPrCTzSNzMzywgn/R5K0mRJiyUtlNQg6UvljglA0gfljsHMzArz9H4PJOkg4ATggIhYJ2kHYOsyh2VmZhXOI/2eaTCwMiLWAUTEyoh4XdIoSY9JmitphqTBknpLmiOpDkDSpZIubq1hScskXSLpaUn1kg5Ibb0s6exUpp+khyXNk9Qo6cRW2jo/HXuhpJ90fjeYmVkpnPR7poeAL0h6QdL1ko6QVAX8Ajg5/aTtjcDFEbEeOAP4laRjgOOA9hLwaxFxEPAEcDNwMvD3wIVp/1pgbEQcABwJ/Lsk5Tcg6VhgKDAaqAFGSTq80MEkTUgfMOqb1qwqqSPMzKx4nt7vgSLiA0mjgMPIJd27gJ8CI4CZKf/2At5I5RdLuhW4HzgoIj5q5xDT0t9GoF9EvA+8L2mtpM8Aq4FLUhLfAOwM7AS8mdfGsek1P633I/ch4PEC5zMVmArQZ/DQKLYfzMysNE76PVRENAGzgFmSGoFzgMVphF7ISOA9csm5PevS3w15y83rvYHTgR2BURHxsaRlQN8WbQi4NCJuKOJ4ZmbWDTy93wNJ2kvS0LxNNcBzwI7pJj8kVUkanpa/BgwCDgeuTaP1jhgAvJ0S/pHAbgXKzAC+I6lfimFnSZ/r4HHNzKwDPNLvmfoBv0jJez3wEjCB3BT5tZIGkPtve7Wkt4DLgKMj4jVJ1wHXAOM7cPzbgPsl1QMNwJ9bFoiIhyTtAzydLjd8AHwDeLsDxzUzsw5QhC+hWuXoM3hoDB5/dbnDKJmfyGdm5SJpbkTUFlPWI32rKCN3HkC9E6iZWZdw0s8oSfcBQ1ps/kFEzChHPGZm1vWc9DMqIsaWOwYzM+tevnvfzMwsIzzSNzOzbvXxxx+zfPly1q5dW+5QepS+ffuyyy67UFVVtdltOOmbmVm3Wr58Of3796e6upoWT/C2VkQE7777LsuXL2fIkJa3YxXP0/tmZtat1q5dy6BBg5zwSyCJQYMGdXh2xEnfzMy6nRN+6Tqjz5z0zcwsk958801OOeUU9thjD4YNG8bxxx/PCy+8ULDssmXLGDFiRDdH2Pl8Td/MzMqqetL0Tm2vmCdkRgRjx45l/Pjx3HnnnQA0NDTw1ltvseeee3ZqPJXESd8qSuOKVZ3+P4D2+BG6Ztnz6KOPUlVVxdlnn71xW01NDRHB+eefzwMPPIAkLrjgAsaNG7dJ3Ztvvpn6+nquu+46AE444QQmTpxIXV0d/fr145xzzuGPf/wjAwcO5JJLLuH73/8+f/nLX7j66qsZM2YMN998M9OmTWPNmjW8/PLLjB07liuuuIKmpibOPPNM6uvrkcR3vvMdzjvvvE49byd9MzPLnEWLFjFq1KhPbb/33ntpaGhgwYIFrFy5kgMPPJDDDz+86HZXr15NXV0dl19+OWPHjuWCCy5g5syZLFmyhPHjxzNmzBggN6swf/58+vTpw1577cW5557L22+/zYoVK1i0aBEA7733XuecbB5f0zczM0uefPJJTj31VHr16sVOO+3EEUccwZw5c4quv/XWW3PccccBMHLkSI444giqqqoYOXIky5Yt21ju6KOPZsCAAfTt25dhw4bx6quvsvvuu/PKK69w7rnn8uCDD7L99tt39um1n/QlfV7SnZJelrRE0h8ktXrBQ1K1pEWt7PutpGGlBpl+P36WpAZJz0mamrbXSDq+iPpFlSsxplnpp2Wb12slzSqxjT1Tf76Uzuu/JO3URvk6Sb9vZd8f0k/tFnvsKZJWpD5tkHRZKbGXKh1vYlcew8ysWMOHD2fu3Lmf2l7ML8/27t2bDRs2bFzP/xpdVVXVxrvst9pqK/r06bNxef369RvLNW8H6NWrF+vXr2fgwIEsWLCAuro6fvnLX3LWWWeVfmLtaDPpKxf5fcCsiNgjIoYBPwJaTUxtiYizImLJZlS9Fvh5RNRExD7AL9L2GqCYZF5suVJ9TtI/bk5FSX2B6cCvIuKL6bx+Bey4Oe1FxPERUepcUHOf1kTEpM05rplZT3TUUUexbt06fvOb32zcNmfOHAYOHMhdd91FU1MT77zzDo8//jijR4/epG51dTUNDQ1s2LCB1157jdmzZ3dKTCtXrmTDhg2cdNJJXHTRRcybN69T2s3X3kj/SODjiPh184aIaIiIJyT1k/SwpHmSGiWdmFevt6TfSVoo6R5J28LG0XFtWv5A0sWSFkh6pq0RLjAYWJ4XQ6OkrYELgXFppDpO0mhJf5I0P/3dq5Vym4w6JS1KMxTbSZqeYlokadynItnUlcAFLTdK6ivpptQv8yUdWaDuacDTEXF/3nk9GhHNsTyR+naepIPz6m0v6b406/JrSVulYy6TtEOq+5yk30haLOkhSdu0cx75sR+dYm6UdKOkPvntp+WNsxqpL29M/21fkfQ/89qaLOl5SX8E9io2BjOzriaJ++67j5kzZ7LHHnswfPhwpkyZwmmnnca+++7Lfvvtx1FHHcUVV1zB5z//+U3qHnLIIQwZMoSRI0cyceJEDjjggE6JacWKFdTV1VFTU8MZZ5zBpZde2int5lNbUxnpf+BDIuJTtw9K6g1sGxF/S8ngGWAosBuwFDg0Ip6SdCOwJCKuSoliYkTUSwpgTETcL+kK4G8R8dNW4vg2cDXwJ+Ah4KaIeE/SGUBtRPxLKrc9sCYi1kv6MvC9iDipQLkpwAcRcVVaXwScAIwCjouI76btAyJiVSsxzQImAlcAFwHvA1dFRJ2kfwNGRMS3Je2dYt4zItbm1f8Z8GpEXFOg7W2BDRGxVtJQ4I6IqJVUBzwIDANeTcs3RMQ9kpYBtUA/4KV0vg2S/guYFhH/2eIYU4DvAu+kTT8AHgNeBI6OiBck3QLMi4irm9uPiJXpg1vzuU4BjiX3AbE/8DzweWBf4GbgS+RuGJ0H/Lq5z1vEMgGYANBr+x1H7fK9mwp1eZfx3ftm3eu5555jn332KXcYPVKhvpM0NyJqi6nfkRv5BFwiaSHwR2BnPpn2fy0inkrL/wkcWqD+R0Dz9em5QHVrB4qIm4B9gLuBOuCZ5hFoCwOAu1MS/zkwvITzAWgEvizpckmHtZbwW/gpnx7tHwrcmmL/M7kEXcoXP6uA30hqJHfO+fdBzI6IVyKiCbiDwn27NCIa0nJbfZs/vT+D3Gh8aUQ0P53id0Axt61Oj4h1EbESeJvc++Aw4L6IWBMRfwOmtVY5IqZGRG1E1PbadkARhzMzs83RXtJfTG70W8jp5K4/j4qIGuAtoG/a13L6oNB0wsfxyTRDE+18fTAiXo+IGyPiRGA9UOjRSBcBj0bECOCf8uJpaT2bnnvfdIwXyJ1vI3CppB+3FVOq80iq//d5m4t5VmJbfXseuf7cj9zofev8Q7YMoUD9dXnL7fZtnrbizu+zlv3a2vHavyPGzMy6TXtJ/xGgj6TvNm+QdKCkI8iNqt+OiI/TNevd8urtKumgtHwq8GRHgpR0nKSqtPx5YBCwgtyUev+8ogPSdoAz8ra3LLcMOCC1dwAwJC3/HbnLA/8JXNVcpggXA9/PW3+c3IcilPumw67kpr3z3Q4cLGnj3HI6z5HpPN6IiA3AN4FeefVGSxqSruWPo4N928KfgWpJX0zr3yQ35Q+5Pmv+kHJSEW09DoyVtI2k/uQ+hJmZWRm1mfTTSHwscIxyX9lbDEwBXgduA2qV+9ra6eQSRrPngPFp6v+z5O5K74hjgUWSFgAzgPMj4k3gUWBY8w165K6vXyrpKTZNlC3L/TfwWUkNwPeA5unskcDstH0yuan7dkXEH/jk2jjA9UCvND1/F3BGRKxrUedDcvcRnCvpRUlLyH1QeTvVHy/pGXKXBVbnVX0auAxYRO7eifuKibHI81gLfJvcJZJGYAPQfBPnT4BrJD1BbjTfXlvzyJ17A7n+fqKz4jSznq+Yr8bZpjqjz9q8kc+su9XW1kZ9fX37Bc2sx1q6dCn9+/f3z+uWICJ49913ef/99xkyZMgm+0q5kc+P4TUzs261yy67sHz5ct555532C9tGffv2ZZddduknb2gjAAAEBklEQVRQGxWV9CVNBr7eYvPdEXFxOeIBkHQf6Zp/nh+ku93NzKxEVVVVnxqtWveoqKSfknvZEnwhETG23DGYmZl1Bv/gjpmZWUY46ZuZmWWE7963iiLpfT79TAP7xA7AynIHUcHcP+1zH7WtJ/bPbhFR1I+1VdQ1fTPg+WK/epJFkurdP61z/7TPfdS2Lb1/PL1vZmaWEU76ZmZmGeGkb5VmarkDqHDun7a5f9rnPmrbFt0/vpHPzMwsIzzSNzMzywgnfesW6WeDn5f0kqRJBfb3kXRX2v+spOq8fT9M25+X9A/dGXd32dz+kVQt6cP0C5INkn7dsu6Woog+OlzSPEnrJZ3cYt/49GuWL0oa331Rd58O9k9T3ntoWvdF3b2K6KN/lbRE0kJJD0vaLW/flvEeigi//OrSF7mfOX4Z2B3YGlgADGtR5n8Av07LpwB3peVhqXwfcr+B8DLQq9znVEH9Uw0sKvc5VEgfVQP7ArcAJ+dt/yzwSvo7MC0PLPc5VUr/pH0flPscKqSPjgS2Tcvfy/t3tsW8hzzSt+4wGngpIl6JiI+AO4ETW5Q5EfhdWr4HOFq539w8EbgzItZFxFLgpdTelqQj/ZMV7fZRRCyLiIXAhhZ1/wGYGRF/jYj/B8wEjuuOoLtRR/onK4rpo0cjYk1afQZo/km7LeY95KRv3WFn4LW89eVpW8EyEbEeWAUMKrJuT9eR/gEYImm+pMckHdbVwZZJR94Hfg+1r6+keknPSPpq54ZWMUrtozOBBzazbsXyE/msOxQakbb82khrZYqp29N1pH/eAHaNiHcljQL+r6ThEfG3zg6yzDryPvB7qH27RsTrknYHHpHUGBEvd1JslaLoPpL0DaAWOKLUupXOI33rDsuBL+St7wK83loZSb2BAcBfi6zb0212/6TLHu8CRMRcctcs9+zyiLtfR94Hfg+1IyJeT39fAWYB+3dmcBWiqD6S9GVgMjAmItaVUrcncNK37jAHGCppiKStyd2I1vIO4WlA8x2xJwOPRO4OmmnAKenu9SHAUGB2N8XdXTa7fyTtKKkXQBqlDSV3k9GWppg+as0M4FhJAyUNBI5N27Ykm90/qV/6pOUdgEOAJV0Wafm020eS9gduIJfw387bteW8h8p9J6Ff2XgBxwMvkBuJTk7bLiT3jwugL3A3uRv1ZgO759WdnOo9D/xjuc+lkvoHOAlYTO5O5HnAP5X7XMrYRweSG5GtBt4FFufV/U7qu5eAb5f7XCqpf4CDgcb0HmoEziz3uZSxj/4IvAU0pNe0Le095CfymZmZZYSn983MzDLCSd/MzCwjnPTNzMwywknfzMwsI5z0zczMMsJJ38zMLCOc9M3MzDLCSd/MzCwj/j9EboqPaeFK1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Finding the Important Variables/Features\n",
    "pd.DataFrame(rf.feature_importances_, index = ohetrain.columns, \n",
    "             columns = [\"Columns\"]).sort_values(by = \n",
    "                                                \"Columns\").tail(10).plot(kind = \"barh\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAHNRJREFUeJzt3XucVVX5x/HPd2ZAQEAQFVEjr6BmiYpmqeQ9sfKSluIlNX6ReflVpqVlpmaZdvFSxk/QvOcliUo0qVA0TZOLeCFNS0QQRCQVFZSLz++PvYeO48ycM3DmnHVmvu/Xa784Z+29134G8Zl1nrP22ooIzMwsPXXVDsDMzJrnBG1mlignaDOzRDlBm5klygnazCxRTtBmZolygrY1Jqm7pDskvS7pN2vQz9GS/lTO2KpB0h8lHVftOKz2OUF3IpKOkjRV0puS5ueJZPcydH040B/oFxGfW91OIuKmiNi/DPG8h6Q9JYWk3zZp3z5vn1xiP+dKurHYcRExPCKuW81wzVZxgu4kJJ0GXAr8kCyZDgR+CRxchu4/CDwTESvK0Fd7WQh8XFK/grbjgGfKdQFl/P+UlY3/MXUCktYBzgdOjojfRsRbEbE8Iu6IiDPyY9aSdKmkefl2qaS18n17Spor6RuSXs5H3yfk+84DzgGOyEfmI5uONCVtmo9UG/L3x0t6TtIbkmZJOrqg/YGC8z4uaUpeOpki6eMF+yZL+r6kB/N+/iRpvVb+GpYBvwOOzM+vBz4P3NTk7+oySXMkLZY0TdIeefsBwLcLfs7HCuL4gaQHgSXA5nnb/+T7R0u6vaD/iyRNkqSS/wNap+UE3Tl8DOgGjG/lmO8AuwJDgO2BXYCzC/ZvCKwDbAyMBK6Q1Dcivkc2Kr81InpGxNWtBSJpbeByYHhE9AI+Dsxo5rh1gTvzY/sBPwPubDICPgo4AdgA6Aqc3tq1geuBL+SvPwnMBOY1OWYK2d/BusCvgd9I6hYRdzf5ObcvOOdYYBTQC5jdpL9vAB/Jf/nsQfZ3d1x4jQUrgRN059APeKVICeJo4PyIeDkiFgLnkSWeRsvz/csj4i7gTWDwasbzLrCdpO4RMT8iZjZzzKeAZyPihohYERE3A08Dnyk45pqIeCYilgK3kSXWFkXE34B1JQ0mS9TXN3PMjRGxKL/mT4G1KP5zXhsRM/NzljfpbwlwDNkvmBuBUyNibpH+zAAn6M5iEbBeY4mhBRvx3tHf7LxtVR9NEvwSoGdbA4mIt4AjgBOB+ZLulLR1CfE0xrRxwfuXViOeG4BTgL1o5hNFXsZ5Ki+rvEb2qaG10gnAnNZ2RsQjwHOAyH6RmJXECbpzeAh4GziklWPmkX3Z12gg7//4X6q3gB4F7zcs3BkREyNiP2AA2ah4bAnxNMb04mrG1OgG4CTgrnx0u0pegvgWWW26b0T0AV4nS6wALZUlWi1XSDqZbCQ+D/jm6odunY0TdCcQEa+TfZF3haRDJPWQ1EXScEkX54fdDJwtaf38y7ZzyD6Sr44ZwDBJA/MvKM9q3CGpv6SD8lr0O2SlkpXN9HEXMCifGtgg6QhgW2DCasYEQETMAj5BVnNvqhewgmzGR4Okc4DeBfsXAJu2ZaaGpEHABWRljmOBb0pqtRRj1sgJupOIiJ8Bp5F98beQ7GP5KWQzGyBLIlOBx4EngOl52+pc68/ArXlf03hvUq0j++JsHvAfsmR5UjN9LAI+nR+7iGzk+emIeGV1YmrS9wMR0dyng4nAH8mm3s0m+9RRWL5ovAlnkaTpxa6Tl5RuBC6KiMci4lmymSA3NM6QMWuN/GWymVmaPII2M0uUE7SZWaKcoM3MEuUEbWaWqNZuXKiq7gNH+NtLe5+lL5xX7RAsSYPWeG2TtuScpS/cXJG1VDyCNjNLVLIjaDOzSkpxpVgnaDMzoK7VpWqqI72IzMyqwCNoM7NEpfgMBSdoMzMgxTkTTtBmZrjEYWaWLCdoM7NEeRaHmVmiPII2M0uUE7SZWaKEp9mZmSXJI2gzs0TV1aWXDtOLyMysKjyCNjNLkkscZmaJcoI2M0uUXOIwM0uTR9BmZomqq6uvdgjv4wRtZoZLHGZmyXKJw8wsUU7QZmaJconDzCxR8q3eZmZp8kNjzcwS5RKHmVmi/CWhmVmqXOIwM0tUegNoJ2gzMwDq0svQTtBmZuARtJlZqsI1aDOzRKWXn52gzcwAqEsvQztBm5mBp9mZmSWrvjwJWtJg4NaCps2Bc4A+wJeAhXn7tyPirtb6coI2M4OyjaAj4p/AkKxL1QMvAuOBE4BLIuInpfblBG1mBu31JeE+wL8jYvbqLMaU4Mw/M7MqqFPJm6RRkqYWbKNa6PVI4OaC96dIelzSryT1LRpSWX4wM7Nap9K3iBgTEUMLtjHv607qChwE/CZvGg1sQVb+mA/8tFhILnGYmQFRX/bx6nBgekQsAGj8E0DSWGBCsQ48gjYzgzaNoEs0goLyhqQBBfsOBZ4s1oFH0GZmUNZ50JJ6APsBXy5ovljSECCA55vsa5YTtJkZlPVOwohYAvRr0nZsW/txgjYzA6/FYWaWLN/qbWaWqDLd6l1OTtBmZuARtJlZstLLz07QqTl15HCOH7E3EcHMp+cw6vT/Y9edBnHhd46ma9cGHn1iFieecSUrV75b7VCtgs466zImT55Cv37rMGHCFQBcdNGvuPfeR+jSpQsDB27IhRd+ld69e1Y50toVCa4H7RtVErJR/76cdMIB7PapbzN0v29SX1/HEQfvxlU/+wpfOOVyhu73TV6Yu5BjDh9W7VCtwj772X246qpz39O2225DmDDhCu644+dsuunGXHnl7dUJrqOQSt8qpN0StKStJX1L0uWSLstfb9Ne1+soGhrq6d6tK/X1dXTv3pUlS9/mnWXL+deslwC454EnOGT4LlWO0ipt5523Y511er2nbffdd6ShoR6AIUMG89JLr1QjtI6j/HcSrrF2SdCSvgXcQvajPAJMyV/fLOnM9rhmRzBvwatcOmYCzzz8C2ZNHc3ixUu4/Y6H6dJQz44f2RyAQw/8KJts1K9IT9bZjBv3Z4YN26naYdS2+rrStwpprxr0SOBDEbG8sFHSz4CZwI+aOylfsm8UQEPfoTT03LKdwktTn3XW5tP7DWWb3f6X1xYv4dejv8qRh+7OF075ORefcyxrdW3gL/c/wYoVrj/bf40efSv19fUcdNCe1Q6ltqVXgm63BP0usBEwu0n7gHxfs/Il+8YAdB84ItoptmTtvft2PD/nZV75zxsA/O7uKey60yBuGf8A+x5+HgD77PFhttp8QGvdWCcyfvwkJk+ewrXXXsDqLAhvBRL8krC9EvTXgEmSngXm5G0DgS2BU9rpmjVvzouvsMuOW9G9W1eWvr2MvXbbjumPP8f6/XqzcNFiunZt4BsnHcRFP/9dtUO1BNx//zTGjh3HjTdeSPfu3aodTu3rLAk6Iu6WNAjYBdiY7MPDXGBKRKxsj2t2BFNm/Jvxd/2dh+76IStWvstjM5/n6l9P4tzTP8/wfXakrk6MvfEv3Pe3mdUO1SrstNN+zCOPPMGrry5m2LDjOfXUoxgz5naWLVvOCSd8F4Dttx/M+eefXOVIa1ekl59RRJqVhM5Y4rDilr5wXrVDsCQNWuP0uvmXx5Wcc5678rCKpHPfqGJmBp2nxGFmVnMSvG3PCdrMDLxYkplZslziMDNLU3gEbWaWqAYnaDOzNHkEbWaWKNegzcwSlV5+doI2M4M0n6jiBG1mBi5xmJklq94J2swsTZ7FYWaWKJc4zMwS5QRtZpYm3+ptZpaqBL8kLLoCqqSvSuqtzNWSpkvavxLBmZlVTJ1K3yoVUgnHfDEiFgP7A+sDJwA/ateozMwqrUYTdGM0BwLXRMRjJHlTpJnZGlAbtmJdSX0k3S7paUlPSfqYpHUl/VnSs/mffYv1U0qCnibpT2QJeqKkXsC7JZxnZlYzok4lbyW4DLg7IrYGtgeeAs4EJkXEVsCk/H2rSvmScCQwBHguIpZI6kdW5jAz6zjKNItDUm9gGHA8QEQsA5ZJOhjYMz/sOmAy8K3W+iplBB3AtsD/5u/XBrq1MWYzs7TVq+RN0ihJUwu2UQU9bQ4sBK6R9KikqyStDfSPiPkA+Z8bFAuplBH0L8lKGnsD5wNvAOOAndv205uZpauuDU/1jogxwJgWdjcAOwKnRsTfJV1GCeWMZmMq4ZiPRsTJwNt5YK8CXVfnYmZmqZJK34qYC8yNiL/n728nS9gLJA3IrqUBwMvFOiolQS+XVE9W6kDS+vhLQjPrYMqVoCPiJWCOpMF50z7AP4A/AMflbccBvy8WUykljsuB8cAGkn4AHA6cXcJ5ZmY1Q+W91ftU4CZJXYHnyCZW1AG3SRoJvAB8rlgnRRN0RNwkaRrZbwEBh0TEU2sSuZlZatpSgy4mImYAQ5vZtU9b+imaoCUNBJYAdxS2RcQLbbmQmVnKVMYEXS6llDjuJKs/i2x63WbAP4EPtWNcZmYVleBidiWVOD5c+F7SjsCX2y0iM7MqSHA56LYvNxoR0yV5DrSZdSg1OYKWdFrB2zqy+XwL2y0iM7MqqMkEDfQqeL2CrCY9rn3CMTOrjroEF+wvpQZ9XiUCMTOrppoaQUu6g/zuweZExEHtEpGZWRXUVIIGflKxKMzMqqymEnRE3FfJQMzMqqkmp9lJ2gq4kGxN6FXrQEfE5u0Yl5lZRdXUCLrANcD3gEuAvcgW/UjwRzEzW30pzuIo5e7z7hExCVBEzI6Ic8kW7zcz6zDKuB502ZQygn5bUh3wrKRTgBcp4VEtZma1JMUSRykj6K8BPcieSbgTcAz/XXTazKxDqKkRtKTDgQkRMSVvehM/zdvMOqgUZ3G0NoI+GnhB0vWShuePvTIz65Dq6kvfKhZTSzsi4lBgS2ASWXljjqTRkoZVKjgzs0pJscTRag06IhZHxHURMRz4MDAD+LmkORWJzsysQiSVvFVKSetBS+oLfBY4AlgXr2ZnZh1MirM4WvuSsBdwCDCCbA3oPwAXAPdGRIuLKJmZ1aKaStDALGAiMBq4OyKWVyakzPP/HFHJy1mN2HXcy9UOwRL08GGD1riPWkvQAyNiScUiMTOrooZaeqq3k7OZdSZ1Sq9y2+aHxpqZdUQp3qjiBG1mRmnrXlSaH3llZkbtlTj8yCsz6zRqqsThR16ZWWfSUEsJupEfeWVmnYESLHGUUhe/huxmlRVkj7y6HrihPYMyM6u0OpW+VSymEo7xI6/MrMOra8NWKX7klZkZac7iWJ1HXh2LH3llZh1Mg0rfSiGpXtKjkibk76+VNEvSjHwbUjSmYgf4kVdm1hm0Q235q8BTQO+CtjMi4vZSOyhlFse9NHPDSkS4Dm1mHUY5SxySNgE+BfwAOG11+ymlBn16wetuwGFkMzrMzDqMMo+gLwW+CfRq0v4DSeeQPUrwzIh4p9WYil0lIqYVbA9GxGnAR1c3ajOzFLVlFoekUZKmFmyjGvuR9Gng5YiY1uQSZwFbAzuTPZnqW8ViKqXEsW6Tn2EnYMNi55mZ1ZK2lDgiYgwwpoXduwEHSTqQrOrQW9KNEXFMvv8dSdfw3upEs0opcUwjq0GLrLQxCxhZwnlmZjWjXAv2R8RZZKNlJO0JnB4Rx0gaEBHzlT119hDgyaIxlXC9bSLi7cIGSWu1PWwzs3RV4AaUmyStTzbYnQGcWOyEUhL038geGlvooWbazMxqVnvcqBIRk4HJ+es2z3xrbT3oDYGNge6SdiDL+pDN6evR1guZmaWsppYbBT4JHA9sAvyU/yboxcC32zcsM7PKqqknqkTEdcB1kg6LiHEVjMnMrOJSHEGX8ktjJ0l9Gt9I6ivpgnaMycys4urrouStUkpJ0MMj4rXGNxHxKnBg+4VkZlZ5tbrcaL2ktRpvSZTUHfA0OzPrUFJcbrSUBH0jMCm/8yWAL5I9VcXMrMNIsQZdynKjF0t6HNiXbCbH9yNiYrtHZmZWQTWZoAEi4m7gbgBJu0m6IiJObtfIzMwqqEuNljjIV/4fARxBthbHb9szKDOzSqupEbSkQcCRZIl5EXAr2YNj96pQbGZmFVNTCRp4Gvgr8JmI+BeApK9XJCozswqrTzBBtzal7zDgJeBeSWMl7cN/b/c2M+tQ6lT6VrGYWtoREeMj4giyJwBMBr4O9Jc0WtL+FYrPzKwi6hQlbxWLqdgBEfFWRNwUEZ8mWzhpBnBmu0dmZlZBXVT6ViltumsxIv4TEVf6id5m1tGkWOIoaZqdmVlHV6u3epuZdXgpzuJwgjYzo/bmQZuZdRrleqp3OTlBm5kB9a5Bm5mlKcEBtBO0mRm4Bm1mliwnaDOzRLkGbWaWKM/iMDNLlEscZmaJ8p2EZmaJ8locVtSCl17jh2ffwqJFb1An8ZnDPsrnjt6Dq664mwcmz6ROos+6Pfn2+Uew3gbrVDtcq6DxBwzlrRUreTeClRGccM9jjNp2IMM26se7Ebz6znK+P/VZXnl7WbVDrUkJlqBRRHq/NQAWLP1DmoG1s1cWLmbRK4sZvM0mLHnrbf5nxGX88JLjWb//OqzdsxsAt//6AZ5/bgGnn31YlaOtvIPvWrfaIVTN+AOGcvw9M3h92YpVbT0a6lmyYiUAn99iAJv27sHFj/67WiFWzcOH7b7GBYp75t1Vcs7Ze6MDK1IQ8Qg6Meut35v11u8NQI+1u/HBzTdg4cuvs+kW/Vcd8/bSZSjBeplVXmNyBujWUF/FSGpfl7r0xoRO0Amb/+J/ePbpeWz74YEAjP35H7l7wjR69uzGZWNPrHJ0VmkBXL77dgQwftZ8fj9rAQAnfuiDDB+4AW8uX8HJ9z9R1RhrWYqzOCpedpF0Qiv7RkmaKmnqDVdPrGRYyVmy5B2+e/r1nHrGQatKG186dTjjJp7NfgfuyG9vebDKEVqljZr8OMfdM4OvPziTwzffiCHrZZ+0/m/mbA7+4xQmzlnI4VtsVOUoa1eKT1SpRl38vJZ2RMSYiBgaEUOPHfnJSsaUlBXLV/Ldb1zPfgfuwCf2+fD79u87fAfum+SRUmfT+OXfq+8s5755i9i2b6/37P/TnIXstXG/aoTWIdS1YWuNpG6SHpH0mKSZks7L2zeT9HdJz0q6VVLXUmIqO0mPt7A9AfQv2kEnFhFcdN5tfHCzDTji2E+sap8ze+Gq1w/eN5OBm21QjfCsSrrV19EjrzF3q69jl/59eG7xW3wg/3QFsMeAdZn9xtJqhVjzpNK3It4B9o6I7YEhwAGSdgUuAi6JiK2AV4GRxTpqrxp0f+CTeRCFBPytna7ZITwx43kmTpjO5lttyBc//zMgK23c+btHmPP8QlQnNhzQl298p/PN4OjM1u3WhYt23RaA+jr40wsLeXjBa1y469YM7NmdAF5a8g4XTf9XdQOtYeUqXUQ2Ne7N/G2XfAtgb+CovP064FxgdGt9tVeCngD0jIgZTXdImtxO1+wQPrLDZtw/48fva//YHttUIRpLxby33uHYSY++r/2sh5+uQjQdU1vKCZJGAaMKmsZExJiC/fXANGBL4Arg38BrEdE4R3IusHGx67RLgo6IFofuEXFUS/vMzKpFbbiTME/GY1rZvxIYIqkPMB5oboRV9IIp3jxjZlZxasNWqoh4DZgM7Ar0kdQ4KN4EmFfsfCdoMzPK9yWhpPXzkTOSugP7Ak8B9wKH54cdB/y+WEy+UcXMjLaNjIsYAFyX16HrgNsiYoKkfwC3SLoAeBS4ulhHTtBmZpRvudGIeBzYoZn254Bd2tKXE7SZGSXNb644J2gzM8pa4igbJ2gzM5ygzcySleJqdk7QZmZ4BG1mliw/k9DMLFGexWFmlqgUb6t2gjYzwyNoM7NkJZifnaDNzMDT7MzMkuUEbWaWqATzsxO0mRm07YkqleIEbWaGR9BmZsnyNDszs0TVVzuAZjhBm5nhEbSZWcLSy9BO0GZmgJygzczSJKW3XJITtJkZ4BKHmVmilOCCo07QZma4xGFmljCXOMzMkuRZHGZmiXKCNjNLlJTezd5O0GZmgGvQZmaJconDzCxZnmZnZpYkj6DNzBKlBNcbTW9Mb2ZWBaK+5K1oX9KvJL0s6cmCtnMlvShpRr4dWKwfJ2gzMyCbxVHqVtS1wAHNtF8SEUPy7a5inbjEYWZGeUscEXG/pE3XtB+PoM3MgLaMoCWNkjS1YBtV4kVOkfR4XgLpW+xgJ2gzM7LlRkvdImJMRAwt2MaUcInRwBbAEGA+8NNiJzhBm5kBZa5Bv09ELIiIlRHxLjAW2KXYOa5Bm5kBde28HrSkARExP397KPBka8eDE7SZWa58CVrSzcCewHqS5gLfA/aUNAQI4Hngy8X6cYI2M6O8dxJGxIhmmq9uaz9O0GZmgFezMzNLVIq3ejtBm5lBSbdwV5oiotoxWBGSRpU4z9I6Ef+76Pg8D7o2lHqXknUu/nfRwTlBm5klygnazCxRTtC1wXVGa47/XXRw/pLQzCxRHkGbmSXKCdrMLFFO0ImTdICkf0r6l6Qzqx2PVV9zz7uzjskJOmGS6oErgOHAtsAISdtWNypLwLU0/7w762CcoNO2C/CviHguIpYBtwAHVzkmq7KIuB/4T7XjsPbnBJ22jYE5Be/n5m1m1gk4QaetueW1PC/SrJNwgk7bXOADBe83AeZVKRYzqzAn6LRNAbaStJmkrsCRwB+qHJOZVYgTdMIiYgVwCjAReAq4LSJmVjcqq7b8eXcPAYMlzZU0stoxWfvwrd5mZonyCNrMLFFO0GZmiXKCNjNLlBO0mVminKDNzBLlBG2rSFopaYakJyX9RlKPNehrT0kT8tcHtbYSn6Q+kk5ajWucK+n0Zq77UJO2BkkLJA1oS19m1eYEbYWWRsSQiNgOWAacWLhTmTb/m4mIP0TEj1o5pA/Q5gTdgvuBTSRtWtC2L/BkRMwv0zXMKsIJ2lryV2BLSZtKekrSL4HpwAck7S/pIUnT85F2T1i1dvXTkh4APtvYkaTjJf0if91f0nhJj+Xbx4EfAVvko/cf58edIWmKpMclnVfQ13fy9bH/AgxuGnREvAv8BjiioPlI4Ob8/C/l/T4maVxznxIkTZY0NH+9nqTn89f1kn5cENeX8/YBku4v+PSxx+r+pZsVcoK295HUQLYG9RN502Dg+ojYAXgLOBvYNyJ2BKYCp0nqBowFPgPsAWzYQveXA/dFxPbAjsBM4Ezg3/no/QxJ+wNbkS23OgTYSdIwSTuRJdsdyH4B7NzCNW7Oj0PSWsCBwLh8328jYuf8+k8BbbkLbyTwekTsnF/7S5I2A44CJkbEEGB7YEYb+jRrUUO1A7CkdJfUmFz+ClwNbATMjoiH8/ZdyR4e8KAkgK5ktx1vDcyKiGcBJN0IjGrmGnsDXwCIiJXA65L6Njlm/3x7NH/fkyxh9wLGR8SS/BrNrksSEVMk9ZQ0GNgGeDgiXs13byfpArKySk+y2+hLtT/wEUmH5+/XyeOaAvxKUhfgdxHhBG1l4QRthZbmo8BV8iT8VmET8OeIGNHkuCGUbylUARdGxJVNrvG1NlzjFrJR9Dbk5Y3ctcAhEfGYpOOBPZs5dwX//XTZrUlcp0bE+5K6pGHAp4AbJP04Iq4vMU6zFrnEYW31MLCbpC0BJPWQNAh4GthM0hb5cSNaOH8S8JX83HpJvYE3yEbHjSYCXyyobW8saQOyLwAPldRdUi+yckpLbgaOIRuxF460ewHz89Hu0S2c+zywU/768IL2icBX8nORNEjS2pI+CLwcEWPJPnXs2EpcZiXzCNraJCIW5iPPm/P6LsDZEfGMpFHAnZJeAR4Atmumi68CY/IV2FYCX4mIhyQ9mD8E9Y95HXob4KF8BP8mcExETJd0K1mNdzZZGaalOP8haQkwLSIKPwF8F/h7fv4TvPcXQ6OfALdJOha4p6D9KmBTYLqywBYCh5CNws+QtDyP9QstxWXWFl7NzswsUS5xmJklygnazCxRTtBmZolygjYzS5QTtJlZopygzcwS5QRtZpao/weiBby9sNH/0gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.heatmap(confusion_matrix(ytest, pred), annot = True, cmap = \"YlGnBu\")\n",
    "plt.title(\"Confusion Matrix\")\n",
    "plt.xlabel(\"Predicted Values\")\n",
    "plt.ylabel(\"Actual Values\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.88      0.84       101\n",
      "           1       0.82      0.70      0.75        76\n",
      "\n",
      "    accuracy                           0.80       177\n",
      "   macro avg       0.81      0.79      0.79       177\n",
      "weighted avg       0.80      0.80      0.80       177\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(ytest,pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5891092392385753\n"
     ]
    }
   ],
   "source": [
    "print(cohen_kappa_score(ytest, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5940836504464574\n"
     ]
    }
   ],
   "source": [
    "print(matthews_corrcoef(ytest, pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dtree = DecisionTreeClassifier(criterion = \"entropy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_df = dtree.fit(xtrain,ytrain).predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7853107344632768"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest,pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "## KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_knn = knn.fit(xtrain,ytrain).predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7627118644067796"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, pred_knn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "gbm = GradientBoostingClassifier()\n",
    "pred_gbm = gbm.fit(xtrain,ytrain).predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8248587570621468"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, pred_gbm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "bagg = BaggingClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_bagg = bagg.fit(xtrain,ytrain).predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8192090395480226"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, pred_bagg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "ada = AdaBoostClassifier(base_estimator=rf, n_estimators=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_ada = ada.fit(xtrain,ytrain).predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8022598870056498"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, pred_ada)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensemble of These Models - Voting Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "vc = VotingClassifier(estimators=[(\"rf\",rf),(\"GBM\", gbm),\n",
    "                                  (\"bagg\", bagg),\n",
    "                                  (\"Tree\",dtree),\n",
    "                                 (\"Ada\", ada),(\"knn\", knn)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_vc = vc.fit(xtrain,ytrain).predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8305084745762712"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, pred_vc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6483909415971394"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohen_kappa_score(ytest, pred_vc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6528906797548438"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matthews_corrcoef(ytest, pred_vc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}