Multi-class Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi-Class Classification - Different Approaches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "# %matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import Imputer # For treating missing values\n",
    "from sklearn.metrics import r2_score, mean_squared_error # For comparing among models\n",
    "from sklearn.ensemble import RandomForestClassifier # For selecting features\n",
    "from sklearn.model_selection import GridSearchCV # For selecting the best model\n",
    "from sklearn.model_selection import train_test_split # For spliting train/test\n",
    "from sklearn.preprocessing import MinMaxScaler # For min max normalization\n",
    "plt.style.use('seaborn') #set same style for all plots\n",
    "import time #Calculate time processing for each model\n",
    "pd.options.mode.chained_assignment = None  # suppress chained assignment warning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Exploratory & Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Height</th>\n",
       "      <th>Weight</th>\n",
       "      <th>QRS_duration</th>\n",
       "      <th>P-R_interval</th>\n",
       "      <th>Q-T_interval</th>\n",
       "      <th>T_interval</th>\n",
       "      <th>P_interval</th>\n",
       "      <th>QRS</th>\n",
       "      <th>...</th>\n",
       "      <th>V6_Amp_Q</th>\n",
       "      <th>V6_Amp_R</th>\n",
       "      <th>V6_Amp_S</th>\n",
       "      <th>V6_Amp_R'</th>\n",
       "      <th>V6_Amp_S'</th>\n",
       "      <th>V6_Amp_P</th>\n",
       "      <th>V6_Amp_T</th>\n",
       "      <th>V6_QRSA</th>\n",
       "      <th>V6_QRSTA</th>\n",
       "      <th>cardiac_arrhythmia</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>190</td>\n",
       "      <td>80</td>\n",
       "      <td>91</td>\n",
       "      <td>193</td>\n",
       "      <td>371</td>\n",
       "      <td>174</td>\n",
       "      <td>121</td>\n",
       "      <td>-16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>-0.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2.9</td>\n",
       "      <td>23.3</td>\n",
       "      <td>49.4</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>56</td>\n",
       "      <td>1</td>\n",
       "      <td>165</td>\n",
       "      <td>64</td>\n",
       "      <td>81</td>\n",
       "      <td>174</td>\n",
       "      <td>401</td>\n",
       "      <td>149</td>\n",
       "      <td>39</td>\n",
       "      <td>25</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>2.1</td>\n",
       "      <td>20.4</td>\n",
       "      <td>38.8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>95</td>\n",
       "      <td>138</td>\n",
       "      <td>163</td>\n",
       "      <td>386</td>\n",
       "      <td>185</td>\n",
       "      <td>102</td>\n",
       "      <td>96</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.5</td>\n",
       "      <td>-2.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.3</td>\n",
       "      <td>3.4</td>\n",
       "      <td>12.3</td>\n",
       "      <td>49.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "      <td>175</td>\n",
       "      <td>94</td>\n",
       "      <td>100</td>\n",
       "      <td>202</td>\n",
       "      <td>380</td>\n",
       "      <td>179</td>\n",
       "      <td>143</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.2</td>\n",
       "      <td>-2.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4</td>\n",
       "      <td>2.6</td>\n",
       "      <td>34.6</td>\n",
       "      <td>61.6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>190</td>\n",
       "      <td>80</td>\n",
       "      <td>88</td>\n",
       "      <td>181</td>\n",
       "      <td>360</td>\n",
       "      <td>177</td>\n",
       "      <td>103</td>\n",
       "      <td>-16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.1</td>\n",
       "      <td>-3.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.1</td>\n",
       "      <td>3.9</td>\n",
       "      <td>25.4</td>\n",
       "      <td>62.8</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>169</td>\n",
       "      <td>51</td>\n",
       "      <td>100</td>\n",
       "      <td>167</td>\n",
       "      <td>321</td>\n",
       "      <td>174</td>\n",
       "      <td>91</td>\n",
       "      <td>107</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.6</td>\n",
       "      <td>12.2</td>\n",
       "      <td>-2.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2.2</td>\n",
       "      <td>13.5</td>\n",
       "      <td>31.1</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>160</td>\n",
       "      <td>52</td>\n",
       "      <td>77</td>\n",
       "      <td>129</td>\n",
       "      <td>377</td>\n",
       "      <td>133</td>\n",
       "      <td>77</td>\n",
       "      <td>77</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>14.3</td>\n",
       "      <td>20.5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>162</td>\n",
       "      <td>54</td>\n",
       "      <td>78</td>\n",
       "      <td>0</td>\n",
       "      <td>376</td>\n",
       "      <td>157</td>\n",
       "      <td>70</td>\n",
       "      <td>67</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.2</td>\n",
       "      <td>-1.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.5</td>\n",
       "      <td>15.8</td>\n",
       "      <td>19.8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "      <td>168</td>\n",
       "      <td>56</td>\n",
       "      <td>84</td>\n",
       "      <td>118</td>\n",
       "      <td>354</td>\n",
       "      <td>160</td>\n",
       "      <td>63</td>\n",
       "      <td>61</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>-1.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>2.1</td>\n",
       "      <td>12.5</td>\n",
       "      <td>30.9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>167</td>\n",
       "      <td>67</td>\n",
       "      <td>89</td>\n",
       "      <td>130</td>\n",
       "      <td>383</td>\n",
       "      <td>156</td>\n",
       "      <td>73</td>\n",
       "      <td>85</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.6</td>\n",
       "      <td>10.8</td>\n",
       "      <td>-1.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.9</td>\n",
       "      <td>20.1</td>\n",
       "      <td>25.1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 280 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age  Sex  Height  Weight  QRS_duration  P-R_interval  Q-T_interval  \\\n",
       "0   75    0     190      80            91           193           371   \n",
       "1   56    1     165      64            81           174           401   \n",
       "2   54    0     172      95           138           163           386   \n",
       "3   55    0     175      94           100           202           380   \n",
       "4   75    0     190      80            88           181           360   \n",
       "5   13    0     169      51           100           167           321   \n",
       "6   40    1     160      52            77           129           377   \n",
       "7   49    1     162      54            78             0           376   \n",
       "8   44    0     168      56            84           118           354   \n",
       "9   50    1     167      67            89           130           383   \n",
       "\n",
       "   T_interval  P_interval  QRS         ...         V6_Amp_Q V6_Amp_R V6_Amp_S  \\\n",
       "0         174         121  -16         ...              0.0      9.0     -0.9   \n",
       "1         149          39   25         ...              0.0      8.5      0.0   \n",
       "2         185         102   96         ...              0.0      9.5     -2.4   \n",
       "3         179         143   28         ...              0.0     12.2     -2.2   \n",
       "4         177         103  -16         ...              0.0     13.1     -3.6   \n",
       "5         174          91  107         ...             -0.6     12.2     -2.8   \n",
       "6         133          77   77         ...              0.0      6.5      0.0   \n",
       "7         157          70   67         ...              0.0      8.2     -1.9   \n",
       "8         160          63   61         ...              0.0      7.0     -1.3   \n",
       "9         156          73   85         ...             -0.6     10.8     -1.7   \n",
       "\n",
       "  V6_Amp_R' V6_Amp_S'  V6_Amp_P  V6_Amp_T  V6_QRSA  V6_QRSTA  \\\n",
       "0       0.0         0       0.9       2.9     23.3      49.4   \n",
       "1       0.0         0       0.2       2.1     20.4      38.8   \n",
       "2       0.0         0       0.3       3.4     12.3      49.0   \n",
       "3       0.0         0       0.4       2.6     34.6      61.6   \n",
       "4       0.0         0      -0.1       3.9     25.4      62.8   \n",
       "5       0.0         0       0.9       2.2     13.5      31.1   \n",
       "6       0.0         0       0.4       1.0     14.3      20.5   \n",
       "7       0.0         0       0.1       0.5     15.8      19.8   \n",
       "8       0.0         0       0.6       2.1     12.5      30.9   \n",
       "9       0.0         0       0.8       0.9     20.1      25.1   \n",
       "\n",
       "   cardiac_arrhythmia  \n",
       "0                   8  \n",
       "1                   6  \n",
       "2                  10  \n",
       "3                   1  \n",
       "4                   7  \n",
       "5                  14  \n",
       "6                   1  \n",
       "7                   1  \n",
       "8                   1  \n",
       "9                  10  \n",
       "\n",
       "[10 rows x 280 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"cardiac_arrhythmia.csv\")\n",
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Height</th>\n",
       "      <th>Weight</th>\n",
       "      <th>QRS_duration</th>\n",
       "      <th>P-R_interval</th>\n",
       "      <th>Q-T_interval</th>\n",
       "      <th>T_interval</th>\n",
       "      <th>P_interval</th>\n",
       "      <th>QRS</th>\n",
       "      <th>...</th>\n",
       "      <th>V6_Amp_Q</th>\n",
       "      <th>V6_Amp_R</th>\n",
       "      <th>V6_Amp_S</th>\n",
       "      <th>V6_Amp_R'</th>\n",
       "      <th>V6_Amp_S'</th>\n",
       "      <th>V6_Amp_P</th>\n",
       "      <th>V6_Amp_T</th>\n",
       "      <th>V6_QRSA</th>\n",
       "      <th>V6_QRSTA</th>\n",
       "      <th>cardiac_arrhythmia</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.0</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "      <td>452.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>46.471239</td>\n",
       "      <td>0.550885</td>\n",
       "      <td>166.188053</td>\n",
       "      <td>68.170354</td>\n",
       "      <td>88.920354</td>\n",
       "      <td>155.152655</td>\n",
       "      <td>367.207965</td>\n",
       "      <td>169.949115</td>\n",
       "      <td>90.004425</td>\n",
       "      <td>33.676991</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.278982</td>\n",
       "      <td>9.048009</td>\n",
       "      <td>-1.457301</td>\n",
       "      <td>0.003982</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.514823</td>\n",
       "      <td>1.222345</td>\n",
       "      <td>19.326106</td>\n",
       "      <td>29.473230</td>\n",
       "      <td>3.880531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>16.466631</td>\n",
       "      <td>0.497955</td>\n",
       "      <td>37.170340</td>\n",
       "      <td>16.590803</td>\n",
       "      <td>15.364394</td>\n",
       "      <td>44.842283</td>\n",
       "      <td>33.385421</td>\n",
       "      <td>35.633072</td>\n",
       "      <td>25.826643</td>\n",
       "      <td>45.431434</td>\n",
       "      <td>...</td>\n",
       "      <td>0.548876</td>\n",
       "      <td>3.472862</td>\n",
       "      <td>2.002430</td>\n",
       "      <td>0.050118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.347531</td>\n",
       "      <td>1.426052</td>\n",
       "      <td>13.503922</td>\n",
       "      <td>18.493927</td>\n",
       "      <td>4.407097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>105.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>232.000000</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-172.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-4.100000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-28.600000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.800000</td>\n",
       "      <td>-6.000000</td>\n",
       "      <td>-44.200000</td>\n",
       "      <td>-38.600000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>36.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>160.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>142.000000</td>\n",
       "      <td>350.000000</td>\n",
       "      <td>148.000000</td>\n",
       "      <td>79.000000</td>\n",
       "      <td>3.750000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.425000</td>\n",
       "      <td>6.600000</td>\n",
       "      <td>-2.100000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>11.450000</td>\n",
       "      <td>17.550000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>47.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>164.000000</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>86.000000</td>\n",
       "      <td>157.000000</td>\n",
       "      <td>367.000000</td>\n",
       "      <td>162.000000</td>\n",
       "      <td>91.000000</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.800000</td>\n",
       "      <td>-1.100000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>1.350000</td>\n",
       "      <td>18.100000</td>\n",
       "      <td>27.900000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>58.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>79.000000</td>\n",
       "      <td>94.000000</td>\n",
       "      <td>175.000000</td>\n",
       "      <td>384.000000</td>\n",
       "      <td>179.000000</td>\n",
       "      <td>102.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>2.100000</td>\n",
       "      <td>25.825000</td>\n",
       "      <td>41.125000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>83.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>176.000000</td>\n",
       "      <td>188.000000</td>\n",
       "      <td>524.000000</td>\n",
       "      <td>509.000000</td>\n",
       "      <td>381.000000</td>\n",
       "      <td>205.000000</td>\n",
       "      <td>169.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.600000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.400000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>88.800000</td>\n",
       "      <td>115.900000</td>\n",
       "      <td>16.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 275 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              Age         Sex      Height      Weight  QRS_duration  \\\n",
       "count  452.000000  452.000000  452.000000  452.000000    452.000000   \n",
       "mean    46.471239    0.550885  166.188053   68.170354     88.920354   \n",
       "std     16.466631    0.497955   37.170340   16.590803     15.364394   \n",
       "min      0.000000    0.000000  105.000000    6.000000     55.000000   \n",
       "25%     36.000000    0.000000  160.000000   59.000000     80.000000   \n",
       "50%     47.000000    1.000000  164.000000   68.000000     86.000000   \n",
       "75%     58.000000    1.000000  170.000000   79.000000     94.000000   \n",
       "max     83.000000    1.000000  780.000000  176.000000    188.000000   \n",
       "\n",
       "       P-R_interval  Q-T_interval  T_interval  P_interval         QRS  \\\n",
       "count    452.000000    452.000000  452.000000  452.000000  452.000000   \n",
       "mean     155.152655    367.207965  169.949115   90.004425   33.676991   \n",
       "std       44.842283     33.385421   35.633072   25.826643   45.431434   \n",
       "min        0.000000    232.000000  108.000000    0.000000 -172.000000   \n",
       "25%      142.000000    350.000000  148.000000   79.000000    3.750000   \n",
       "50%      157.000000    367.000000  162.000000   91.000000   40.000000   \n",
       "75%      175.000000    384.000000  179.000000  102.000000   66.000000   \n",
       "max      524.000000    509.000000  381.000000  205.000000  169.000000   \n",
       "\n",
       "              ...            V6_Amp_Q    V6_Amp_R    V6_Amp_S   V6_Amp_R'  \\\n",
       "count         ...          452.000000  452.000000  452.000000  452.000000   \n",
       "mean          ...           -0.278982    9.048009   -1.457301    0.003982   \n",
       "std           ...            0.548876    3.472862    2.002430    0.050118   \n",
       "min           ...           -4.100000    0.000000  -28.600000    0.000000   \n",
       "25%           ...           -0.425000    6.600000   -2.100000    0.000000   \n",
       "50%           ...            0.000000    8.800000   -1.100000    0.000000   \n",
       "75%           ...            0.000000   11.200000    0.000000    0.000000   \n",
       "max           ...            0.000000   23.600000    0.000000    0.800000   \n",
       "\n",
       "       V6_Amp_S'    V6_Amp_P    V6_Amp_T     V6_QRSA    V6_QRSTA  \\\n",
       "count      452.0  452.000000  452.000000  452.000000  452.000000   \n",
       "mean         0.0    0.514823    1.222345   19.326106   29.473230   \n",
       "std          0.0    0.347531    1.426052   13.503922   18.493927   \n",
       "min          0.0   -0.800000   -6.000000  -44.200000  -38.600000   \n",
       "25%          0.0    0.400000    0.500000   11.450000   17.550000   \n",
       "50%          0.0    0.500000    1.350000   18.100000   27.900000   \n",
       "75%          0.0    0.700000    2.100000   25.825000   41.125000   \n",
       "max          0.0    2.400000    6.000000   88.800000  115.900000   \n",
       "\n",
       "       cardiac_arrhythmia  \n",
       "count          452.000000  \n",
       "mean             3.880531  \n",
       "std              4.407097  \n",
       "min              1.000000  \n",
       "25%              1.000000  \n",
       "50%              1.000000  \n",
       "75%              6.000000  \n",
       "max             16.000000  \n",
       "\n",
       "[8 rows x 275 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8       2\n",
       "7       3\n",
       "14      4\n",
       "15      5\n",
       "9       9\n",
       "5      13\n",
       "4      15\n",
       "3      15\n",
       "16     22\n",
       "6      25\n",
       "2      44\n",
       "10     50\n",
       "1     245\n",
       "Name: cardiac_arrhythmia, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['cardiac_arrhythmia'].value_counts().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "J                     376\n",
       "P                      22\n",
       "T                       8\n",
       "Heart_Rate              1\n",
       "QRST                    1\n",
       "cardiac_arrhythmia      0\n",
       "V1_S'_wave              0\n",
       "V1_Intrinsic_def        0\n",
       "V1_Ragged_R             0\n",
       "V1_DD_R                 0\n",
       "V1_Ragged_P             0\n",
       "V1_S_wave               0\n",
       "V1_DD_P                 0\n",
       "V1_Ragged_T             0\n",
       "V1_DD_T                 0\n",
       "V2_Q_wave               0\n",
       "V2_R_wave               0\n",
       "V2_S_wave               0\n",
       "V2_R'_wave              0\n",
       "V2_S'_wave              0\n",
       "V1_R'_wave              0\n",
       "AVF_DD_T                0\n",
       "V1_R_wave               0\n",
       "V1_Q_wave               0\n",
       "V2_Ragged_R             0\n",
       "AVF_Ragged_T            0\n",
       "AVF_DD_P                0\n",
       "AVF_Ragged_P            0\n",
       "AVF_DD_R                0\n",
       "AVF_Ragged_R            0\n",
       "                     ... \n",
       "AVR_Amp_T               0\n",
       "AVR_QRSA                0\n",
       "AVR_QRSTA               0\n",
       "AVL_Amp_JJ              0\n",
       "AVL_Amp_R               0\n",
       "DII_Amp_T               0\n",
       "AVL_Amp_S               0\n",
       "AVL_Amp_R'              0\n",
       "AVL_Amp_S'              0\n",
       "AVL_Amp_P               0\n",
       "AVL_Amp_T               0\n",
       "AVL_QRSA                0\n",
       "AVR_Amp_R'              0\n",
       "AVR_Amp_S               0\n",
       "AVR_Amp_R               0\n",
       "AVR_Amp_Q               0\n",
       "AVR_Amp_JJ              0\n",
       "DIII_QRSTA              0\n",
       "DIII_QRSA               0\n",
       "DIII_Amp_T              0\n",
       "DIII_Amp_P              0\n",
       "DIII_Amp_S'             0\n",
       "DIII_Amp_R'             0\n",
       "DIII_Amp_S              0\n",
       "DIII_Amp_R              0\n",
       "DIII_Amp_Q              0\n",
       "DIII_Amp_JJ             0\n",
       "DII_QRSTA               0\n",
       "DII_QRSA                0\n",
       "Age                     0\n",
       "Length: 280, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.replace('?', np.NaN)\n",
    "data.isnull().sum().sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At first glance, the data have 452 observations and 280 variables. \n",
    "More than 50% of the observations have negative diagnosis (cardiac_arrhythmia = 1)\n",
    "Among the predictor variables, QRST & Heart_Rate have 1 missing values, T has 8, P has 22, and J has 376 out of 452 observations. Since more than 75% of J are missing, we'll drop this variable from the data set."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Drop or Impute Missing Values?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66      2\n",
       "91      4\n",
       "200     2\n",
       "212     2\n",
       "238     1\n",
       "360     2\n",
       "372     2\n",
       "412    16\n",
       "Name: cardiac_arrhythmia, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['cardiac_arrhythmia'][data['T'].isna() == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2142427e710>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAD8CAYAAAC4uSVNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt8lNW5L/Df3AjJJEAuwyWJECywUGwPiN2AbVMo4qUobGk51dr6affelvZs7bHt0ba2KO5arZet2HqpiretHttDaz1Wth90u9V07w+bWirbXZFF9RjSJBiGEEJIIJdJzh/JJDOT9z7vO/Ouye/7DzDzrrWe91nvu5jMrDwTGBoaAhERqSWY7wCIiMg+Lt5ERAri4k1EpCAu3kRECuLiTUSkoHAuBonHu7La0lJeXoKOjh63wlEac5GO+RjDXKQrhHzEYmUBveeUeOUdDofyHYJvMBfpmI8xzEW6Qs+HEos3ERGl4+JNRKQgLt5ERAri4k1EpCAu3kRECsrJVsF82r2vDTt2NaL1SA+qq0qwdkUdlp05w1E/2197D0e7egEAFWVF2LhqnmFfbo3tJNZcjOtmbq3288wrB9CwtwX9iSFEQgFcsKIOGz451/EYXuZK9fnX6geApb6zaavXh5hdDtnU4fi8kv21xLsRDgUwMDiEmqqoYT/5mkMrAlaqCgohlgG4XUq5UggxHcAjAMoBhABcKaV836h9tvu8Y7EyxONdttvt3teGh154Z9zjm9Ytsj3pWv0Y9eXW2JnMcuHVuF6NY6efZ145gFf3NI87dvXSWlyxZoHtMVYvrdXsz41c5WoeUsViZXjxjfc8v+bN+s6mrd0+jM4r9V4x60+rn3zMYaas9nkLIa4HsA3A5JGH7gDwjJSyHsAPASx0I0gv7NjVqPP4QVf6MerLrbHtytW4XudWq5+GvS2axzbsbXU0hl47N3Kl+vwbXfNmfWfT1m4fVs/LrD+tfvI1h1ZZedvkfQAbADw18u9PAHhbCPEvABoB/E+zDsrLS7LeMB+Lldlu09qu/dtVh9q7bfWn149RX26NrcWovZfjejGOnX76E9o/wPUnBh3lpD8xaHlsu3I1D16Na3TNm/WdTVu7fZidV/I5s/60+snXHFplunhLKX8thKhLeagOQIeU8jwhxI0AvgvgRqM+sv0VVadvm1RXlqA53j3u8VmVUVv96fVj1JdbY2cyy4VX43o1jp1+IqGA5gIeCQUd5SQSCmou4G7kKlfzkCoWK8vJNW/WdzZt7fZhdF6p94pZf1r95GMOMxn9J+Fkt0k7gBdG/v5bAOc46CMnkh+SjH98jiv9GPXl1th25Wpcr3Or1U/94hrNY+sXVzsaQ6+dG7lSff6NrnmzvrNpa7cPq+dl1p9WP/maQ6uc7Db5NwCfxfDbKPUArH0ykQfJDxV27DqIQ+3dmFUZxdoVc2x/2JA8Pm23yZQibFypv9vErbHtytW4bufWSj/JDyUb9raiPzGISCiIC1bMMd1tYjTGvJqpnuRK9fnX68dK39m0NepDzJ4G2XTM0Xml9tdy5ATCwSASg4OorirV7Sdfc2iV1d0mdQB+IaVcLoSYg+EPMKMAOgF8UUrZYdQ+X7tNChFzkY75GMNcpCuEfBjtNrH0yltK2Qhg+cjfDwJY40pkRETkCH/DkohIQVy8iYgUxMWbiEhBXLyJiBRU8IWpyF/yUegnHwW0cmk0rvYeVFf6Jy6V+HVujXDxppzJLPTTHO8e/bdXN4pbY+Yjdiv8GpdKVM0h3zahnMlHoZ98FNDKJb/GpRJVc8jFm3Km9Yh+oR+/j5mP2K3wa1wqUTWHXLwpZ6qrSjQfn1UZ9f2Y+YjdCr/GpRJVc8jFm3ImH4V+8lFAK5f8GpdKVM0hP7CknMlHoZ98FNDKJb/GpRJVc2ipMFW2WJjKPcxFOuZjDHORrhDykdXXoBERkf9w8SYiUhAXbyIiBXHxJiJSEBdvIiIFcfEmIlKQpX3eQohlAG6XUq5MeeyLAK6RUq7wKLY0Tqt+qVgtjDhv5NxEqbJoungLIa4H8GUA3SmPLQbwtwB09yC6qeGtZkdVv1StFjbRcd7IqYl07Vh52+R9ABuS/xBCVAL4CYBrvQoq0/ZX/6z5uFnVL1WrhU10nDdyaiJdO6avvKWUvxZC1AGAECIE4FEA3wJw0uog5eUlCIdDTmNEU5v2b0kdau9GLFam2661Xb9amFE7v1M5divszluh58OOiZ6LQr3ntditbbIUwHwADwKYDOBMIcRWKaXhq/CODu2EWjV7RhkaDx0f9/isyqjhr79WV5agOT6+rKNZOz8rhF/5NWNn3iZCPqxiLgrvnjf6D8fWbhMp5e+llItGPri8DMA+s4XbDRtXz9d83Kzql6rVwiY6zhs5NZGuHSWqCtYvqcXx46dsV/1StVrYRMd5I6cm0rXDqoKKYS7SMR9jmIt0hZAPVhUkIiowXLyJiBTExZuISEFcvImIFMTFm4hIQVy8iYgUxMWbiEhBXLyJiBTExZuISEFcvImIFMTFm4hIQVy8iYgUxMWbiEhBXLyJiBTExZuISEFcvImIFMTFm4hIQVy8iYgUZOk7LIUQywDcLqVcKYRYDOBnABIAegFcKaVs8zBGIiLKYPrKWwhxPYBtACaPPHQvgGtGvkH+OQDf9Sw6IiLSZOVtk/cBbEj592VSyr0jfw8DOOV6VEREZMjSt8cLIeoA/EJKuTzlsXMBPAqgXkoZN2o/MJAYCodDWYZKRDTh6H57vKX3vDMJIb4A4AcA1pot3ADQ0dHjZJhRsVgZ4vGurPooFMxFOuZjDHORrhDyEYuV6T5ne/EWQnwJwCYAK6WUR7OIi4iIHLK1VVAIEQLwUwBlAJ4TQrwuhLjZk8iIiEiXpVfeUspGAMn3uys8i4aIiCzhL+kQESmIizcRkYK4eBMRKYiLNxGRgrh4ExEpiIs3EZGCuHgTESmIizcRkYK4eBMRKYiLNxGRgrh4ExEpiIs3EZGCuHgTESmIizcRkYK4eBMRKYiLNxGRgrh4ExEpiIs3EZGCHH17fC7t3teGnW/+AU0fdqG6qgRrV9QBAHbsakRLvBvhUAD9iSFEQgEMDA6hpiqKtSvqsOzMGWl97NjViNYjPaN9JJ83ei4zDivH6Z2D3ba797Vh+2vv4WhXLwCgoqwIG1fNw8Wf1v82aaexAOn5TM0jADz9skT3qYHRPgIAamLj8+xkbDvtrXjmlQNo2Nsyek3UL67BFWsWeBKLnT7Mjs18Xswuh2zqGDdPqddEdPLw7Zucm6ppxfhc/enjYtC6lpYsiI32P610EgCgo6tXc/712rbEuxEMAIkhjD63cdU8AMjL/ZZso3UdO5mX1P60zlNv/pL5PHaiz7PrHAACQ0NDpgcJIZYBuF1KuVIIMQ/AEwCGAPwJwN9LKQeN2sfjXeaDaNi9rw0PvfCOk6bYtG4Rlp05Q7ePTesWAYDuc5kTY+U4LU7aGp33dV9aijNqpxqOaTeWbFjJgdHYVtvricXKEI93ARheuF/d0zzumNVLa9MWcDdisdOH2bFuz0tqDF7MuRNe329m52l3XvRi1erTSo6dXuexWFlA7znTt02EENcD2AZg8shDdwP4oZTyUxh+EbbedkQW7djVmEXbg4Z97Nh10PA5K3FkHqd9jP22Rue9/dU/m47ppF/nfZrnwGhsq+2taNjbovN4q+ux2OnD7Fi35yU1Bi/m3Amv7zez87Q7L1byZmf+3LzOk6y8bfI+gA0Anhr591IAb4z8/SUA5wP4jVEH5eUlCIdDtoNrbe+x3SbpUHs3YrEy3T4OtXdD78eBZFuzODKP0+KkrdF5/6Wty3RMu7Fkw0oOjMa22t5Isn1/QntG+xODrs2nkz7MjnV7XlJj8GLOnfD6fjM7T7vzYuWtAjvz58Z1nsl08ZZS/loIUZfyUEBKmTy3LgCmP8N3dDi7gKorS9Ac73bUdlZlFPF4l24fsyqjAIZ0n0v+KG4UR+ZxWpy0NTrv02aUmY5pN5ZsWMmB0dhW2+tJfdskMvL5R6ZIKOjafDrpw+xYt+clNQYv5twJr+83s/O0Oy96sWr1aSXHTq9zowXfyW6T1Pe3ywAcc9CHJckPTJy1nWPYx9oVcwyfsxJH5nHax9hva3TeG1fPNx3TSb/O+zTPgdHYVttbUb+4RufxatdjsdOH2bFuz0tqDF7MuRNe329m52l3Xqzkzc78uXmdJznZbfKWEGKllPJ1ABcBeM3dkMYk3+Df+eZf8Je2LsyqjI4mYceug2g5cgLhYBADiUGEQ0EkBgdRXVWKtSvmjLZN/rlj10Ecau8e7SP1wwOj56z2YXYOdtomn0v7lH9KETaunIf6JbWOX6nqxZJ8LJnP1DwCertNSi3nwGhsNz+FT34o2bC3Ff2JQURCQdQvrh6328SNWOz0YXas1vNi9jTIpmPj5iltt0nxyG6Tk8NzE5tWjA0Zu030rqUl82Oj/U+NTgICI7tNNOZfr23LkRMIIoDEyKaH5DVqlhcv7rfUNlrXsZN5Se1P6zz15i+Zz84TfZ5c50lWd5vUAfiFlHK5EGIBgEcATALwLoCrpJQJo/ZOd5skpf5oPNExF+mYjzHMRbpCyIfRbhNLr7yllI0Alo/8/QCAT7sSGREROcLfsCQiUhAXbyIiBXHxJiJSEBdvIiIF+b4wVaGzU/xmIrFbmCgXRa/IfdkUoDIq4pXN3KtyLXHxzqPMgjapvxx4tKt39Dk/XjheysxLc7zbMBd2jyd/cDJvWm1Sf7sxtQ8nFThVupb4tkke5augjd/ZLUyUi6JX5D4vClBZ6cO4nXb/fryWuHjnUesRawVtJhq9vOjlwu7x5A9O5s3KPWPWh9sx5QsX7zyqrioxPWa4SM7EopcXvVzYPZ78wcm8WblnzPpwO6Z84eKdR/kqaON3dgsT5aLoFbnPiwJUVvowbqfdvx+vpdCWLVs8H6Snpy+rQaLRIvT09LkUjX/Uxkoxs6IEbUdPoutkH0KBwGgd4YopRbjygoXjPiTxWy5272vDwy+8g2de+TP+IA+jZHIEtbHSrPpMzUv3qX7UVJXi8vPma35gFI0WoTwaGctjdw+a3rgD139zE/7jladRV3c6Skuzi8eJz3/+Eqxf/zmEw8Z7Anp7e/HSSy9CiIV49NGH0NTUiIULz7Q11lNPPYFIJIy6utN8dW2YsTPPRm0+fsZ09A8MjuvDyb3iJCYvRaNFN+s9x90mebbszBm++xTbKi8/mbebl+Txvb29uGLPz0b+/Z2sYsiFo0fb8dvfPo9LLvlrx318+ctfcS+gHHNy/Xt9z6hyT3LxJseMPpl3evH39p7CrbfejA8//BADAwO4+upr8dxz23HiRBc6O4/hkksuxaWXfh5XX/01TJtWjt7eHtx002245ZYb0dXVhZqa2tG+rr76a7juuhtQXFyMu+76Cfr6enH8eCe+8pWrUF+/Ev/+77/D448/AgCYP1/guuu+j2Bw/DuJiUQCd955Kw4fbkNnZyeWLz8XV131Dfz4x1vQ2dmJ48c7cfnlX8bTTz+BSCSCdesuBQD84z/+BK2tw1/Nduutd+Huu2/H+edfhHPP/SQaGz/A/fdvRVVVDI2NH4zG8bvfNeC1115FZ2cn/u7vvo5PfrIeX/jCX+Ossz6G5ua/4Oyzz0F39wm8++47mD17DjZv/hF+/OMtWL36fKxa9Qls3vy9cbmiwsT3vMkxLz6Zf/75X2PmzGo89NDj+MEPboKU7+K8887HPffcjzvu2Ipf/vKZ0WPXrLkQTzzxBHbu3IG5cz+C++9/BOvXf25cnwcPNuKyy67A1q0P4Fvfuh7PPfd/MDAwgHvuuQN33rkV27b9E6ZPn47Dhw9rxnT4cBsWLfoo7r77PjzwwDY8//yvRp9buvQc/Pznj6GsrAx9fX144IFtuPDCtQCAtWvX4777HsasWdV4883dWLfuUrz00osAgB07XsDFF6/HlVf+Derq5uKrX70KABCLxXDvvQ/im9/89ug4H354CFdd9T9w//2P4Fe/+iUuvXQjHn74Sbz99n+iq2us5OnBgwd1c0WFh6+8ybHqKqOvkXKmqekgli8/FwBw+unzMGXKVDz44M/wxhuvoaQkioGBsS+FmD17+EOkDz74f1i2bAUAYNGis8a9z1xZWYUnn3wUO3b8XwABDAwMoLPzGMrKylBeXgEAo4unlilTpuDdd9/BH//4B0SjUfT19Y+LIfPvALBw4UIAQEVFJXp7T2HJkqXYuvVOdHQcxe9//x/YtOnvEY+n/4chxBmjMZ86dWpk/KmYOXMmAKC4uBhz554OAIhGS9HX1zvatqqqCg0N2zRzRYWHr7zJMS8+mZ8zZy7efXcfAKClpRl3330HzjrrY7jxxh/hM585D6lfHpJ8i2P27Dr86U//BQA4cGD/uEVr27af48IL12Lz5h/h7LPPAQCUl1fgxIkTOH68EwCwdeud2LfvT5ox/fM/v4jS0jLcdNMtuOyyL6G399RoHIHA2C0UDGbWzU//dyAQwPnnX4StW+/CX/3VcoTDYQQCQQwNDaYcM378gNaDGh577DHdXFHh4StvcsyLrzZbv34DbrvtH3D11V9DIpHApz71aWzf/ixefvklTJ06FaFQCH196TsINmzYiNtuuxnf+MbfYs6cOkQikbTnV61ajXvvvQtPPfU4pk+fgWPHjiEYDOLb3/4urrvuWgSDQSxYIHDGGYs0Y1q69OPYsuUGvP32XkyePBm1tafhyJG4o/P77GcvwYYNa/Hkk78AAJSXl6O/fwAPPPBTFBUVOeozadWqVdi8+cZxuZo0aVJW/ZI/WfoatGzxa9Dcw1ykUy0f8fhh3HLLTbj33gdd71u1XHitEPKR9degZRJCRAA8CaAOQALD32O531F0Bkare7X3oLrSv9W9qDA8/vgj2LPnzXGP33DDTaiu1v5mejtef/1VPPbYw/je9zZn3VehUqWinx84euUthFgP4Aop5X8XQqwB8HUp5fiP+Uc4eeWduYc4adO6RRN6Mgvh1YSbmI8xqufC7Xte9XwAxq+8nX5geQBAWAgRBDAFQL/J8bapVN2LiLLHe94epx9YnsDwWyb7AVQBuNjo4PLyEoTDIVsDtLbr7yGOxezX6S0kE/38MzEfY1TOhRf3vMr5MON08f4WgJ1Syu8LIU4D8K9CiI9KKU9pHdzRYa2MY6rqSv09xKr/KJSNQvhR0E3MxxjVc+H2Pa96PgDj/3ycvm3SAaBz5O9HAUQA2HtpbUKl6l5ElD3e8/Y4feV9D4DHhBC/AzAJwA1SSlerlXuxh5iI/Iv3vD3c560Y5iId8zGGuUhXCPnwYrcJERHlERdvIiIFcfEmIlIQF28iIgVx8SYiUpASJWEb3mrGszv3s1gNkQ12izyxKJRafL94e/klt0SFyu59w/tMPb5/24TFaojss3vf8D5Tj+8Xby++5Jao0Nm9b3ifqcf3i3d1VYnm49l8yS1RobN73/A+U4/vF28WqyGyz+59w/tMPb7/wHLZmTMwZcpkPLtTslgNkUV2izyxKJR6WJhKMcxFOuZjDHORrhDywcJUREQFhos3EZGCuHgTESmIizcRkYK4eBMRKcjxVkEhxPcBrMPwd1g+IKV81LWoRowWymnvQXXl+EI5bhXesfM4MPyrxC3xboRDAQwMDqGmKpo29u59bdj+2ns42tULAKgoK8LGVfPGxea0EJBR/7kqLpTNOPksgOSX4ktmcTi5tne++Qc0fdiF6qoSiNnlkE0dnp2n1fgyj3MaV77nLd/ja3G0VVAIsRLAdwCsB1AC4H9JKbfoHe9kq2BmoZykTesWjS5SRs9b7W/10lq8uqfZ8uNGNq1bBACa42TGZjf+pHebO3Hn03s0n9OL2axPu5zGnm1bLXa2g7k9tlO5urattrfLanxuxeV03tzaKpjP68aLrYIXAPgvAL8B8FsALzrsR5dZoRy3Cu807G219biRHbsO6o6TfN4sHrNCQNtf/bPuc3oxu11cKJsiRvksgOSX4ku5urattrfLanxuxZXvecv3+Hqcvm1SBWAOgIsBzAXwghBioZRS8xV2eXkJwuGQrQFa2/UL5cRiZabPW+2vPzFo63Ejh9q7YfQjRmpsduNPamrTfyWhF7NZn3Y5jT3btnqstvNibCdydW1bbW+X1fjciiubecvl+eaa08W7HcB+KWUfACmEOAUgBuCw1sEdHdYmMVV1ZQma4+Mrms2qjCIe7zJ93mp/kVBQc9HTe9zIcBGfIc1xMmOzG3/S7BllaDx0XPM5vZjN+rTLaezZttVi50djt8d2KlfXttX2dlmNz624nM6bW2+b5PO6MfrPwenbJv8G4EIhREAIUQ0giuEF3TVmhXLcKrxTv7ja1uNG1q6YoztO8nmzeMwKAW1cPV/3Ob2Y3S4ulE0Ro3wWQPJL8aVcXdtW29tlNT634sr3vOV7fD2hLVu22G50zTXXHLjvvvs+CuBWAFcA+I6UUvfN2J6ePtuD1MZKMbOiBG1HT6L7VD9qqkpx+XnzRz8gMHvean8XLZ9j6/FzxHS0HT2JrpN9iISCAIZQExsbOznOB63HcbIvAQComFKEKy9YmBab3fiTzvxIDFMmhzX714vZ7Q9VnMaebVst0WgRenr6PI/bTV5d2+3He9HV04eaqlJ8/Izp6B8Y9OQ8rcandZyTuJzOm51rw4vx3RCNFt2s9xwLUymGuUjHfIxhLtIVQj5YmIqIqMBw8SYiUhAXbyIiBXHxJiJSEBdvIiIF+f47LPPFSiEaqwWovBjba27F4IdzyTU/n3NqbNNKJwEAjp3oyzpON87Z6j1ndZyGt5rx7M79vpwHNzja522Xk33eqdzar2lVshDN8Z5+DAE43tOPPTKOmRUlqI2Vph2T3GsNACf7EuOOc3vsXOTCyvnnsh8jub42zOTinPWY5SIztpN9CZzsS2QdpxvnbOeeszLO7n1t+On2/8zLPLjJaJ833zbRYKUQjdUCVF6M7TW3YvDDueSan8/ZSqEoJ3G6cc7Z3HNa4/h5HtzCxVtD6xH9QjRmx2Qe58XYXnMrBj+cS675+ZyNrtkkJ3G6cc7Z3HNa4/h5HtzCxVtDdVWJ5uPDhaeMj8k8zouxveZWDH44l1zz8zkbXbNJTuJ045yzuee0xvHzPLiFi7cGK4VorBag8mJsr7kVgx/OJdf8fM5WCkU5idONc87mntMax8/z4BZ+YKnBSiEaqwWo3B47F7lwqxBPLgr6+O0DyzwXMTLMRWZs5aVFKJ4cRl//YFZxunHOdu45K+PUxkqxoK4CTYe68lqELFssTFVAmIt0zMcY5iJdIeSDhamIiAoMF28iIgVx8SYiUhAXbyIiBXHxJiJS0IQtTGVU4CZXhYWS47TEuxEOBTAwOITy0iIA44sFjcbU3oPqyhKI2eWQTR1KFN2xU3AoNRc1VdGcFkuyc7wX8ZK3/FwwzImstgoKIaYD2ANgjZRyv95xftsqmCxwk2nTukUAoPucmxOtF4OW1Utr8eqeZtPj3I7RDUa5Tv3P0igXRueld21YGdfp8dnE66VC2BrnptR82L0e/MKTrYJCiAiAhwCcdNpHvhgVrclVQRsrRYKSGva2WuzTf0V3si3ylXmsm+M6Pd6LeMlbhVioKpu3Te4C8HMA3zc7sLy8BOFwKIuhhv8XdUtru37RGr0fEQ61d+ckBi39iUFLx7kdoxuMcp2M1SwXZuel9ZyVcZ0en228XvLb/Oeb2TXmx3vGKkeLtxDiKwDiUsqdQgjTxbujw/pCpcXtHwerK0vQHB9fXWy4aM2Q7nO5iEFLJBS0tIC7HaMbjHKdjNUsF0bnpXdtWBnX6fHZxOslvm2SLjUfdq8HvzD6j8Xp2yZ/A2CNEOJ1AIsB/JMQYqbDvnLOqGhNrgraWCkSlFS/uNpin/4rupNtka/MY90c1+nxXsRL3irEQlWOXnlLKeuTfx9ZwL8upfzQraC8lvyAYseugzjU3o1ZlVGsXTEn7YMLo+fcjqHlyAmEg0EkBgcxrbQICACdJ/rSxp5XMzUtJjF7GmTTMU9jdIOVXOvlorqq1PF5WRnX6fFexEvesns9qCDrwlQpi7cyu01UxlykYz7GMBfpCiEfRrtNst7nLaVcmW0fRERkD3/DkohIQVy8iYgUxMWbiEhBXLyJiBTExZuISEFKVhVUsTqYijGroOGtZjy7c79pXrXyD8CVOXnmlQNo2NuC/sQQQgGgeHIEPacGsp5nu9eM1VzkSmr800onARhfLdNOH1YrPSaPvfyChTijdqon5+OH/Cr3BcQqVgdzM+ZC2LvqFqt5tVPB0e6cPPPKAdOKj07m2cuqiLlgJedmsblR6dGt889XfgvqC4hVrA6mYswqsJpXOxUc7c5Jw94W1/scbtNoqy+/XWNWcm4WmxuVHt06f7/lF1Bw8W49ol8dzK9UjFkFVvOqd5yVtmb6E+Y/VDqZZ7vXjN+uMSs5N4vNzjl5ff5+yy+g4OJdXVWi+fhwRUB/UjFmFVjNq95xVtqaiYR0f6p13Cdg/5rx2zVmJedmsdk5J6/P32/5BRRcvFWsDqZizCqwmlc7FRztzkn94hrX+xxuU2erL79dY1ZybhabG5Ue3Tp/v+UXAEJbtmzxfJCenr6sBolGi9DT0wcAqI2VYmZFCdqOnkT3qX7UVJXi8vPm+/bDSsDdmFNzMdHVxkqxoK4CTYe6DPOql/9zxPSs5+RjH6nEiZP9aD58AoNDQwgFAogWR5AYHMpqnu1eM1ZzkSuZ8ZeXFqF4chh9/YOWY7OTA61jN234KBZ/pNKT88lVfqPRopv1nlNut8lEx1ykYz7GMBfpCiEfBbXbhIiIuHgTESmJizcRkYK4eBMRKYiLNxGRghwVphJCRAA8BqAOQBGAW6SUL7gYl+tSiwdFQgHUL67BFWsWuDqGWeEavUI9Tov2WBkzF5zGkNlOzC7HWwfiONrVO3pMdPLwJdp9agAAUFFWhI2r5gEYKSrV3oPqyvFjps43AAQA1MSilmJLxtUS70Y4FMDA4BBqqqKmxayM2pkVU7Lb5umX5WhOQsEAVi6pwbVfXGq7GNTufW3Y/tp7ozmvKCvCkgUxyKblQ2H9AAAG4klEQVQOR9eUlTmtKCvCrKoo9jceRfIXVIsiIUQnh9HR1Ws5D07k6n7JxTiOtgoKIb4K4L9JKa8VQlQCeEtKOVvv+HxvFdQrHrR6aa1rC7hZ4Ro7xZFS22XyW5EupzHYzYcVyTHNikUZxeYkrk3rFgGAYTs7xZSctlkiYnhLxk1jTf3Pxs2CXV7MqdWxtWSuG7m6X1wuROf6VsHtADan/HvAYT85oVc8qGFvq2tjmBWusVMcKbVdNmPmgtMY7ObDiuSYZsWijGJzEteOXQdN29kppuS0jdnCndmn2wW7vJhTq2Nb66fR0/5zPY6jt02klCcAQAhRBuBXAH5odHx5eQnC4ZCToUbFYmWO2+oVD+pPDGbVb6rWdv3CNbFYme7zepLttCQfNxszF5zGYDcfViTHNCsWZRSbk7gOtXfD7EdLrTHNxnLSxkxqn3b6snJNeTGnVsfWk9ouV/dLrsZx/GUMQojTAPwGwANSyv9tdGxHR3aTmu3bJpFQQPOGjoSCrv0GVnVlCZrj4yuMzaqMIh7v0n1eT7JdptRcmI2ZC05jsJsPK5Jj6s23ldicxDVcnGjIsJ3WmGZjOWljJVaz68dqLJm8mFOrY2vJXDdydb+4OY7RYu/obRMhxAwALwP4rpTyMSd95JJe8aD6xdWujWFWuMZOcaTUdtmMmQtOY7CbD2uxDI9pVizKKDYnca1dMce0nZ1iSk7bLBExw/4y+3S7YJcXc2p1bGv91Hnaf67HcfrK+wYA5QA2CyGS731fJKU86U5Y7kp+KNmwtxX9iUFEQkHUL652dbdJ8oOIHbsO4lB7N2ZVRrF2xZzRxzOfnxqdBASAzhN9aX/PbJfNmLngNAatdmL2tPG7TYpHdpucHNltMqUIG1fOG9c2dczM+QaSu01KTWNLjavlyAmEg0EkBgdRXVU6evMZnateO60xzcYyaqO32+TFN97TvMa0Yk3+PW23yZQiLJkfg2w6ZvuasjqnFVOKMKsyiv2NHUiMbJgoioQQLR7ZbWIhD07k6n7J1TgsTKUY5iId8zGGuUhXCPlgYSoiogLDxZuISEFcvImIFMTFm4hIQVy8iYgU5PiXdHKp4a1mPLtz/7iN78EAMDgEhAKAyS/VobQ4givWLMB7LZ14/Y/NpsfrKYqEEA4FRrdpDT8WRN/AILQ27gQCSHs889/JbV5XrFkwrkhRf2LI8NwCAWBSOITe/oSzk7Ehs0BUtjLzkJQsPJWsB5O6jS2tPWD4W42hYACDQ0MoLy0CMFYArLc/kXYOoWAAicHsdlwFAJSXFaX17Va+wqHh+DJzlSy0JWaXQzZ1DF8z4SAGEoOm56wXPwB0dPUiqHHN6c1XxOA6Tc5BZoGpzOJZmfFl5i5zbLPcJudUb2617mEjqecxtbQoraCWVanXtVt8v1XQq2I3frNobgXe+eBovsPwjdVLaw2LS5F6rBTwKnR2i1MZbRX0/Stvr4rd+A0X7nRuFg0jfxguzOT9i0U/27HroGuvvn2/eLce8abYDflb8jcjqXAcau/WfOtlIjnU7l7tF99/YFldVZLvECgPIiHfX5pk06zK6IS/n4eLmLnD93eIV8Vu/GbR3Ip8h+ArbhYNI3+wUsCr0LlZnCq0ZcsW1zrT09PT53iQ2lgpFtRVoOlQF4739KU9Fxx5Kz8UCJi+k1ZaHMFXP3sGppUWoenDLsfvvBVFQphcFEL/wGDKY0EM6vw8GAgY/zsUDOAzZ9di07pFmFlRgrajJ9F1sg+RUBBDQ0OG5xYIDMeT7W4JK6LFYUyKBNPOOxuZeUiqmFKEKy9YiIuWz8HMihJ80HocJ/vG76bR/RRnRCgYGN1FUTw5jL7+4V0YwVAg7RxCwUDWP8oHMLybILVvt/IVDmnPfwDD98bHz5iO/oHB4WsmHASGhkzPWS/+4slhnOpLaF5zevNldJ0m56AmVorLz5uPZWfOQG2sdPQ67z7VrxlfZu4yxzbLbSgYAIb051brHjaSeh6nzSjF0c5TtteP5HVt9/3uaLToZr3nfL/bBCiMAjNuYS7SMR9jmIt0hZAPFqYiIiowXLyJiBTExZuISEFcvImIFMTFm4hIQTnZbUJERO7iK28iIgVx8SYiUhAXbyIiBXHxJiJSEBdvIiIFcfEmIlIQF28iIgVx8SYiUhAXbyIiBXHxJiJSEBdvIiIF+f7b44m8IIS4H8AnAEwCMA/AvpGn7pVSPp63wIgsYmEqmtCEEHUAXpdS1uU5FCJb+LYJEZGCuHgTESmIizcRkYK4eBMRKYiLNxGRgrh4ExEpiFsFiYgUxFfeREQK4uJNRKQgLt5ERAri4k1EpCAu3kRECuLiTUSkIC7eREQK+v9nuIOdUry0mwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2142785fa90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.plot(x='T', y = 'cardiac_arrhythmia',  style = 'o')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We cannot drop observations with missing T's because the outcome variable contains positive diagnosis.\n",
    "Since there's no linear relationship between T and the outcome, replace missing T's with the mean value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54     16\n",
       "59      1\n",
       "106     6\n",
       "108     2\n",
       "116    16\n",
       "133     1\n",
       "174    15\n",
       "177     1\n",
       "193     2\n",
       "204     3\n",
       "217    15\n",
       "219    16\n",
       "243    15\n",
       "253     3\n",
       "279     1\n",
       "284    10\n",
       "298    10\n",
       "300    15\n",
       "308     2\n",
       "310     1\n",
       "350     1\n",
       "420    15\n",
       "Name: cardiac_arrhythmia, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['cardiac_arrhythmia'][data['P'].isna() == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x21427d1b898>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAD8CAYAAAC4uSVNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt8XFW5N/Df3Jo2k7TNZXpJQhuw7SoUfVuKp61HY2spFyvtodpXEOUj5xysvgd9vbygooVyRJCLUJSLyP0AL/pW0Rfp4VM5HCCe8+lBRHo8UroqvqQxTUnTNE3TpM1lkvePZJKZyb6svWfvmb0mv+8/NDN7rfXsZ+29mMysPBMaHh4GERHpJVzoAIiIyDku3kREGuLiTUSkIS7eREQa4uJNRKShaD4GaW/vzmlLS0VFKTo7e70KR2vMRSbmYxxzkakY8pFIlIfMntPilXc0Gil0CIHBXGRiPsYxF5mKPR9aLN5ERJSJizcRkYa4eBMRaYiLNxGRhrh4ExFpKC9bBQvp1b1t2Lm7Ca1HelFTXYr1q+qx4qzZrvrZ8dLbONrdBwCoLC/B5jULLPvyamw3seZjXC9zq9rPUy/sR+OegxhIDiMWCeGCVfXY9MHTXY/hZ650n3+jfgAo9Z1LW7M+xLwKyOZO1+eV6u9gew+ikRAGh4ZRWx237KdQc6gipFJVUAixAsCtUsrVQohZAB4EUAEgAuAKKeWfrdrnus87kShHe3u343av7m3DA8++OeHxLRuWOJ50o36s+vJq7Gx2ufBrXL/GcdLPUy/sx4uvt0w4du3yOly+bpHjMdYurzPsz4tc5Wse0iUS5Xjulbd9v+bt+s6lrdM+rM4r/V6x68+on0LMYbac9nkLIa4F8BCAqaMP3QbgKSllA4BvA1jsRZB+2Lm7yeTxA570Y9WXV2M7la9x/c6tUT+New4aHtu4p9XVGGbtvMiV7vNvdc3b9Z1LW6d9qJ6XXX9G/RRqDlWpvG3yZwCbADwx+vNfA/iDEOJfADQB+J92HVRUlOa8YT6RKHfcprXD+K+rDnX0OOrPrB+rvrwa24hVez/H9WMcJ/0MJI1/gRtIDrnKyUBySHlsp/I1D36Na3XN2/WdS1unfdidV+o5u/6M+inUHKqyXbyllD8XQtSnPVQPoFNKeZ4Q4noAXwdwvVUfuf6Jqtu3TWqqStHS3jPh8blVcUf9mfVj1ZdXY2ezy4Vf4/o1jpN+YpGQ4QIei4Rd5SQWCRsu4F7kKl/zkC6RKM/LNW/Xdy5tnfZhdV7p94pdf0b9FGIOs1n9T8LNbpMOAM+O/vtXAM510UdepD4kmfj4fE/6serLq7Gdyte4fufWqJ+GpbWGxzYsrXE1hlk7L3Kl+/xbXfN2fefS1mkfqudl159RP4WaQ1Vudpv8G4CPYuRtlAYAap9MFEDqQ4Wduw/gUEcP5lbFsX7VfMcfNqSOz9htMr0Em1eb7zbxamyn8jWu17lV6Sf1oWTjnlYMJIcQi4Rxwar5trtNrMZYUDvDl1zpPv9m/aj0nUtbqz7EvJmQzcdcnVd6fwePnEA0HEZyaAg11WWm/RRqDlWp7japB/ATKeVKIcR8jHyAGQfQBeBTUspOq/aF2m1SjJiLTMzHOOYiUzHkw2q3idIrbyllE4CVo/8+AGCdJ5EREZEr/AtLIiINcfEmItIQF28iIg1x8SYi0lDRF6aiYClEoZ9CFNDKp7G4OnpRUxWcuHQS1Lm1wsWb8ia70E9Le8/Yz37dKF6NWYjYVQQ1Lp3omkO+bUJ5U4hCP4UooJVPQY1LJ7rmkIs35U3rEfNCP0EfsxCxqwhqXDrRNYdcvClvaqpLDR+fWxUP/JiFiF1FUOPSia455OJNeVOIQj+FKKCVT0GNSye65pAfWFLeFKLQTyEKaOVTUOPSia45VCpMlSsWpvIOc5GJ+RjHXGQqhnzk9DVoREQUPFy8iYg0xMWbiEhDXLyJiDTExZuISENcvImINKS0z1sIsQLArVLK1WmPfQrAF6WUq3yKLYPbql86Vgsjzhu5N1mqLNou3kKIawF8BkBP2mNLAfwdANM9iF5qfKPFVdUvXauFTXacN3JrMl07Km+b/BnAptQPQogqAN8D8GW/gsq248U/GT5uV/VL12phkx3njdyaTNeO7StvKeXPhRD1ACCEiAB4GMBXAJxUHaSiohTRaMRtjGhuM/4rqUMdPUgkyk3btXaYVwuzahd0Oseuwum8FXs+nJjsuSjWe96I09omywEsBHA/gKkAzhJCbJdSWr4K7+w0TqiqebPL0XTo+ITH51bFLf/8taaqFC3tE8s62rULsmL4k187TuZtMuRDFXNRfPe81f9wHO02kVL+Vkq5ZPSDy0sB7LVbuL2wee1Cw8ftqn7pWi1ssuO8kVuT6drRoqpgw7I6HD9+ynHVL12rhU12nDdyazJdO6wqqBnmIhPzMY65yFQM+WBVQSKiIsPFm4hIQ1y8iYg0xMWbiEhDXLyJiDTExZuISENcvImINMTFm4hIQ1y8iYg0xMWbiEhDXLyJiDTExZuISENcvImINMTFm4hIQ1y8iYg0xMWbiEhDXLyJiDTExZuISENK32EphFgB4FYp5WohxFIAPwSQBNAH4AopZZuPMRIRURbbV95CiGsBPARg6uhDdwP44ug3yD8D4Ou+RUdERIZU3jb5M4BNaT9fKqXcM/rvKIBTnkdFRESWlL49XghRD+AnUsqVaY99AMDDABqklO1W7QcHk8PRaCTHUImIJh3Tb49Xes87mxDikwC+BWC93cINAJ2dvW6GGZNIlKO9vTunPooFc5GJ+RjHXGQqhnwkEuWmzzlevIUQnwawBcBqKeXRHOIiIiKXHG0VFEJEAPwAQDmAZ4QQLwshbvQlMiIiMqX0yltK2QQg9X53pW/REBGREv6RDhGRhrh4ExFpiIs3EZGGuHgTEWmIizcRkYa4eBMRaYiLNxGRhrh4ExFpiIs3EZGGuHgTEWmIizcRkYa4eBMRaYiLNxGRhrh4ExFpiIs3EZGGuHgTEWmIizcRkYa4eBMRacjVt8fn06t727Drtd+h+d1u1FSXYv2qegDAzt1NONjeg2gkhIHkMGKREAaHhlFbHcf6VfVYcdbsjD527m5C65HesT5Sz1s9lx2HynFm5+C07at727DjpbdxtLsPAFBZXoLNaxbgYx82/zZpt7EAmflMzyMAPPlriZ5Tg2N9hADUJibm2c3YTtqreOqF/Wjcc3DsmmhYWovL1y3yJRYnfdgdm/28mFcB2dw5YZ7Sr4n41JHbNzU31TOn4eMNZ0yIwehaWrYoMdb/zLIpAIDO7j7D+Tdre7C9B+EQkBzG2HOb1ywAgILcb6k2Rtexm3lJ78/oPM3mL5XPYyf6fbvOASA0PDxse5AQYgWAW6WUq4UQCwA8BmAYwB8B/IOUcsiqfXt7t/0gBl7d24YHnn3TTVNs2bAEK86abdrHlg1LAMD0ueyJUTnOiJu2Vud9zaeX48y6GZZjOo0lFyo5sBpbtb2ZRKIc7e3dAEYW7hdfb5lwzNrldRkLuBexOOnD7liv5yU9Bj/m3A2/7ze783Q6L2axGvWpkmO313kiUR4ye872bRMhxLUAHgIwdfShOwF8W0r5IYy8CNvoOCJFO3c35dD2gGUfO3cfsHxOJY7s44yPcd7W6rx3vPgn2zHd9Ou+T/scWI2t2l5F456DJo+3eh6Lkz7sjvV6XtJj8GPO3fD7frM7T6fzopI3J/Pn5XWeovK2yZ8BbALwxOjPywG8Mvrv5wGcD+AXVh1UVJQiGo04Dq61o9dxm5RDHT1IJMpN+zjU0QOzXwdSbe3iyD7OiJu2Vuf9l7Zu2zGdxpILlRxYja3a3kqq/UDSeEYHkkOezaebPuyO9Xpe0mPwY87d8Pt+sztPp/Oi8laBk/nz4jrPZrt4Syl/LoSoT3soJKVMnVs3ANvf4Ts73V1ANVWlaGnvcdV2blUc7e3dpn3MrYoDGDZ9LvWruFUc2ccZcdPW6rxPm11uO6bTWHKhkgOrsVXbm0l/2yQ2+vlHtlgk7Nl8uunD7liv5yU9Bj/m3A2/7ze783Q6L2axGvWpkmO317nVgu9mt0n6+9vlAI656ENJ6gMTd23nW/axftV8y+dU4sg+zvgY522tznvz2oW2Y7rp132f9jmwGlu1vYqGpbUmj9d4HouTPuyO9Xpe0mPwY87d8Pt+sztPp/Oikjcn8+fldZ7iZrfJG0KI1VLKlwFcBOAlb0Mal3qDf9drf8Ff2roxtyo+loSduw/g4JETiIbDGEwOIRoJIzk0hJrqMqxfNX+sbeq/O3cfwKGOnrE+0j88sHpOtQ+7c3DSNvVcxqf800uwefUCNCyrc/1K1SyW1GOpfKbnETDbbVKmnAOrsb38FD71oWTjnlYMJIcQi4TRsLRmwm4TL2Jx0ofdsUbPi3kzIZuPTZinjN0m00Z3m5wcmZvEzGnYlLXbxOxaWrYwMdb/jPgUIDS628Rg/s3aHjxyAmGEkBzd9JC6Ru3y4sf9lt7G6Dp2My/p/Rmdp9n8pfLZdaLfl+s8RXW3ST2An0gpVwohFgF4EMAUAG8BuEpKmbRq73a3SUr6r8aTHXORifkYx1xkKoZ8WO02UXrlLaVsArBy9N/7AXzYk8iIiMgV/oUlEZGGuHgTEWmIizcRkYa4eBMRaSjwhamKnZPiN5OJ08JE+Sh6Rd7LpQCVVRGvXOZel2uJi3cBZRe0Sf/jwKPdfWPPBfHC8VN2Xlraeyxz4fR4CgY382bUJv2vG9P7cFOBU6driW+bFFChCtoEndPCRPkoekXe86MAlUof1u2M+w/itcTFu4Baj6gVtJlszPJilgunx1MwuJk3lXvGrg+vYyoULt4FVFNdanvMSJGcycUsL2a5cHo8BYObeVO5Z+z68DqmQuHiXUCFKmgTdE4LE+Wj6BV5z48CVCp9WLcz7j+I11Jk27Ztvg/S29uf0yDxeAl6e/s9iiY46hJlmFNZirajJ9F9sh+RUGisjnDl9BJcccHiCR+SBC0Xr+5tw4+ffRNPvfAn/E4eRunUGOoSZTn1mZ6XnlMDqK0uw2XnLTT8wCgeL0FFPDaex55eNL9yG6790hb8xwtPor7+DJSV5RaPG5/4xMXYuPHjiEat9wT09fXh+eefgxCL8fDDD6C5uQmLF5/laKwnnngMsVgU9fWnBerasONknq3avP/MWRgYHJrQh5t7xU1MforHS240e467TQpsxVmzA/cptio/P5l3mpfU8X19fbj89R+O/vy1nGLIh6NHO/CrX/0SF1/8N677+MxnPutdQHnm5vr3+57R5Z7k4k2uWX0y7/bi7+s7hZtvvhHvvvsuBgcHcfXVX8Yzz+zAiRPd6Oo6hosvvgSXXPIJXH315zBzZgX6+npxww234Kabrkd3dzdqa+vG+rr66s/hmmuuw7Rp03DHHd9Df38fjh/vwmc/exUaGlbj3//9N3j00QcBAAsXClxzzTcRDk98JzGZTOL222/G4cNt6OrqwsqVH8BVV30B3/3uNnR1deH48S5cdtln8OSTjyEWi2HDhksAAN///vfQ2jry1Ww333wH7rzzVpx//kX4wAc+iKamd3DvvdtRXZ1AU9M7Y3H85jeNeOmlF9HV1YW///vP44MfbMAnP/k3OPvs96Gl5S8455xz0dNzAm+99SbmzZuPrVu/g+9+dxvWrj0fa9b8NbZu/caEXFFx4nve5Jofn8z/8pc/x5w5NXjggUfxrW/dACnfwnnnnY+77roXt922HT/96VNjx65bdyEee+wx7Nq1E6ef/h7ce++D2Ljx4xP6PHCgCZdeejm2b78PX/nKtXjmmf+DwcFB3HXXbbj99u146KF/wqxZs3D48GHDmA4fbsOSJe/FnXfeg/vuewi//OXPxp5bvvxc/OhHj6C8vBz9/f24776HcOGF6wEA69dvxD33/Bhz59bgtddexYYNl+D5558DAOzc+Sw+9rGNuOKKv0V9/em48sqrAACJRAJ3330/vvSlr46N8+67h3DVVf8D9977IH72s5/ikks248c/fhx/+MN/ort7vOTpgQMHTHNFxYevvMm1mmqrr5Fyp7n5AFau/AAA4IwzFmD69Bm4//4f4pVXXkJpaRyDg+NfCjFv3siHSO+88/+wYsUqAMCSJWdPeJ+5qqoajz/+MHbu/L8AQhgcHERX1zGUl5ejoqISAMYWTyPTp0/HW2+9id///neIx+Po7x+YEEP2vwFg8eLFAIDKyir09Z3CsmXLsX377ejsPIrf/vY/sGXLP6C9PfN/GEKcORbzqVOnRsefgTlz5gAApk2bhtNPPwMAEI+Xob+/b6xtdXU1GhsfMswVFR++8ibX/Phkfv780/HWW3sBAAcPtuDOO2/D2We/D9df/x185CPnIf3LQ1JvccybV48//vG/AAD79++bsGg99NCPcOGF67F163dwzjnnAgAqKipx4sQJHD/eBQDYvv127N37R8OY/vmfn0NZWTluuOEmXHrpp9HXd2osjlBo/BYKh7Pr5mf+HAqFcP75F2H79jvwV3+1EtFoFKFQGMPDQ2nHTBw/ZPSggUceecQ0V1R8+MqbXPPjq802btyEW275R1x99eeQTCbxoQ99GDt2PI1f//p5zJgxA5FIBP39mTsINm3ajFtuuRFf+MLfYf78esRisYzn16xZi7vvvgNPPPEoZs2ajWPHjiEcDuOrX/06rrnmywiHw1i0SODMM5cYxrR8+fuxbdt1+MMf9mDq1KmoqzsNR460uzq/j370YmzatB6PP/4TAEBFRQUGBgZx330/QElJias+U9asWYOtW6+fkKspU6bk1C8Fk9LXoOWKX4PmHeYik275aG8/jJtuugF3332/533rlgu/FUM+cv4atGxCiBiAxwHUA0hi5Hss97mKzsJYda+OXtRUBbe6FxWHRx99EK+//tqEx6+77gbU1Bh/M70TL7/8Ih555Mf4xje25txXsdKlol8QuHrlLYTYCOByKeV/F0KsA/B5KeXEj/lHuXnlnb2HOGXLhiWTejKL4dWEl5iPcbrnwut7Xvd8ANavvN1+YLkfQFQIEQYwHcCAzfGO6VTdi4hyx3veGbcfWJ7AyFsm+wBUA/iY1cEVFaWIRiOOBmjtMN9DnEg4r9NbTCb7+WdjPsbpnAs/7nmd82HH7eL9FQC7pJTfFEKcBuBfhRDvlVKeMjq4s1OtjGO6mirzPcS6/yqUi2L4VdBLzMc43XPh9T2vez4A6//5uH3bpBNA1+i/jwKIAXD20tqGTtW9iCh3vOedcfvK+y4AjwghfgNgCoDrpJSeViv3Yw8xEQUX73lnuM9bM8xFJuZjHHORqRjy4cduEyIiKiAu3kREGuLiTUSkIS7eREQa4uJNRKQhLUrCNr7Rgqd37WOxGiIHnBZ5YlEovQR+8fbzS26JipXT+4b3mX4C/7YJi9UQOef0vuF9pp/AL95+fMktUbFzet/wPtNP4BfvmupSw8dz+ZJbomLn9L7hfaafwC/eLFZD5JzT+4b3mX4C/4HlirNmY/r0qXh6l2SxGiJFTos8sSiUfliYSjPMRSbmYxxzkakY8sHCVERERYaLNxGRhrh4ExFpiIs3EZGGuHgTEWnI9VZBIcQ3AWzAyHdY3ielfNizqEaNFcrp6EVN1cRCOV4V3nHyODDyp8QH23sQjYQwODSM2up4xtiv7m3DjpfextHuPgBAZXkJNq9ZMCE2t4WArPrPV3GhXMYpZAGkoBRfsovDzbW967XfofndbtRUl0LMq4Bs7vTtPFXjyz7ObVyFnrdCj2/E1VZBIcRqAF8DsBFAKYD/JaXcZna8m62C2YVyUrZsWDK2SFk9r9rf2uV1ePH1FuXHrWzZsAQADMfJjs1p/ClvtXTh9idfN3zOLGa7Pp1yG3uubY042Q7m9dhu5evaVm3vlGp8XsXldt682ipYyOvGj62CFwD4LwC/APArAM+57MeUXaEcrwrvNO5pdfS4lZ27D5iOk3reLh67QkA7XvyT6XNmMXtdXCiXIkaFLIAUlOJL+bq2Vds7pRqfV3EVet4KPb4Zt2+bVAOYD+BjAE4H8KwQYrGU0vAVdkVFKaLRiKMBWjvMC+UkEuW2z6v2N5AccvS4lUMdPbD6FSM9NqfxpzS3mb+SMIvZrk+n3Maea1szqu38GNuNfF3bqu2dUo3Pq7hymbd8nm++uV28OwDsk1L2A5BCiFMAEgAOGx3c2ak2ielqqkrR0j6xotncqjja27ttn1ftLxYJGy56Zo9bGSniM2w4TnZsTuNPmTe7HE2Hjhs+ZxazXZ9OuY0917ZGnPxq7PXYbuXr2lZt75RqfF7F5XbevHrbpJDXjdX/HNy+bfJvAC4UQoSEEDUA4hhZ0D1jVyjHq8I7DUtrHD1uZf2q+abjpJ63i8euENDmtQtNnzOL2eviQrkUMSpkAaSgFF/K17Wt2t4p1fi8iqvQ81bo8c1Etm3b5rjRF7/4xf333HPPewHcDOByAF+TUpq+Gdvb2+94kLpEGeZUlqLt6En0nBpAbXUZLjtv4dgHBHbPq/Z30cr5jh4/V8xC29GT6D7Zj1gkDGAYtYnxsVPjvNN6HCf7kwCAyukluOKCxRmxOY0/5az3JDB9atSwf7OYvf5QxW3subY1Eo+XoLe33/e4veTXtd1xvA/dvf2orS7D+8+chYHBIV/OUzU+o+PcxOV23pxcG36M74V4vORGs+dYmEozzEUm5mMcc5GpGPLBwlREREWGizcRkYa4eBMRaYiLNxGRhrh4ExFpKPDfYVkoKoVoVAtQ+TG237yKIQjnkm9BPuf02GaWTQEAHDvRn3OcXpyz6j2nOk7jGy14ete+QM6DF1zt83bKzT7vdF7t11SVKkRzvHcAwwCO9w7gddmOOZWlqEuUZRyT2msNACf7kxOO83rsfORC5fzz2Y+VfF8bdvJxzmbscpEd28n+JE72J3OO04tzdnLPqYzz6t42/GDHfxZkHrxktc+bb5sYUClEo1qAyo+x/eZVDEE4l3wL8jmrFIpyE6cX55zLPWc0TpDnwStcvA20HjEvRGN3TPZxfoztN69iCMK55FuQz9nqmk1xE6cX55zLPWc0TpDnwStcvA3UVJcaPj5SeMr6mOzj/Bjbb17FEIRzybcgn7PVNZviJk4vzjmXe85onCDPg1e4eBtQKUSjWoDKj7H95lUMQTiXfAvyOasUinITpxfnnMs9ZzROkOfBK/zA0oBKIRrVAlRej52PXHhViCcfBX2C9oFlgYsYWeYiO7aKshJMmxpF/8BQTnF6cc5O7jmVceoSZVhUX4nmQ90FLUKWKxamKiLMRSbmYxxzkakY8sHCVERERYaLNxGRhrh4ExFpiIs3EZGGuHgTEWlo0hamsipwk6/CQqlxDrb3IBoJYXBoGBVlJQAmFgsai6mjFzVVpRDzKiCbO7UouuOk4FB6Lmqr43ktluTkeD/iJX8FuWCYGzltFRRCzALwOoB1Usp9ZscFbatgqsBNti0blgCA6XNeTrRZDEbWLq/Di6+32B7ndYxesMp1+v8srXJhdV5m14bKuG6PzyVePxXD1jgvpefD6fUQFL5sFRRCxAA8AOCk2z4KxapoTb4K2qgUCUpp3NOq2Gfwiu7kWuQr+1gvx3V7vB/xkr+KsVBVLm+b3AHgRwC+aXdgRUUpotFIDkON/F/UK60d5kVrzH5FONTRk5cYjAwkh5SO8zpGL1jlOhWrXS7szsvoOZVx3R6fa7x+Ctr8F5rdNRbEe0aVq8VbCPFZAO1Syl1CCNvFu7NTfaEy4vWvgzVVpWhpn1hdbKRozbDpc/mIwUgsElZawL2O0QtWuU7FapcLq/MyuzZUxnV7fC7x+olvm2RKz4fT6yEorP7H4vZtk78FsE4I8TKApQD+SQgxx2VfeWdVtCZfBW1UigSlNCytUewzeEV3ci3ylX2sl+O6Pd6PeMlfxVioytUrbyllQ+rfowv456WU73oVlN9SH1Ds3H0Ahzp6MLcqjvWr5md8cGH1nNcxHDxyAtFwGMmhIcwsKwFCQNeJ/oyxF9TOyIhJzJsJ2XzM1xi9oJJrs1zUVJe5Pi+Vcd0e70e85C+n14MOci5MlbZ4a7PbRGfMRSbmYxxzkakY8mG12yTnfd5SytW59kFERM7wLyyJiDTExZuISENcvImINMTFm4hIQ1y8iYg0pGVVQR2rg+kYsw4a32jB07v22ebVKP8APJmTp17Yj8Y9BzGQHEYkBEybGkPvqcGc59npNaOai3xJj39m2RQAE6tlOulDtdJj6tjLLliMM+tm+HI+Qcivdl9ArGN1MC9jLoa9q15RzauTCo5O5+SpF/bbVnx0M89+VkXMB5Wc28XmRaVHr86/UPktqi8g1rE6mI4x60A1r04qODqdk8Y9Bz3vc6RNk6O+gnaNqeTcLjYvKj16df5Byy+g4eLdesS8OlhQ6RizDlTzanacSls7A0n7XyrdzLPTayZo15hKzu1ic3JOfp9/0PILaLh411SXGj4+UhEwmHSMWQeqeTU7TqWtnVjE9Lda130Czq+ZoF1jKjm3i83JOfl9/kHLL6Dh4q1jdTAdY9aBal6dVHB0OicNS2s973OkTb2jvoJ2jank3C42Lyo9enX+QcsvAES2bdvm+yC9vf05DRKPl6C3tx8AUJcow5zKUrQdPYmeUwOorS7DZectDOyHlYC3MafnYrKrS5RhUX0lmg91W+bVLP/nilk5z8n73lOFEycH0HL4BIaGhxEJhRCfFkNyaDineXZ6zajmIl+y468oK8G0qVH0Dwwpx+YkB0bHbtn0Xix9T5Uv55Ov/MbjJTeaPafdbpPJjrnIxHyMYy4yFUM+imq3CRERcfEmItISF28iIg1x8SYi0hAXbyIiDbkqTCWEiAF4BEA9gBIAN0kpn/UwLs+lFw+KRUJoWFqLy9ct8nQMu8I1ZoV63BbtURkzH9zGkN1OzKvAG/vbcbS7b+yY+NSRS7Tn1CAAoLK8BJvXLAAwWlSqoxc1VRPHTJ9vAAgBqE3ElWJLxXWwvQfRSAiDQ8OorY7bFrOyamdXTMlpmyd/LcdyEgmHsHpZLb78qeWOi0G9urcNO156eyznleUlWLbWowwFAAAHEklEQVQoAdnc6eqaUpnTyvISzK2OY1/TUaT+QLUkFkF8ahSd3X3KeXAjX/dLPsZxtVVQCHElgP8mpfyyEKIKwBtSynlmxxd6q6BZ8aC1y+s8W8DtCtc4KY6U3i5b0Ip0uY3BaT5UpMa0KxZlFZubuLZsWAIAlu2cFFNy22aZSOAN2W4ba/r/bLws2OXHnKqObSR73cjX/eJxITrPtwruALA17edBl/3khVnxoMY9rZ6NYVe4xklxpPR2uYyZD25jcJoPFakx7YpFWcXmJq6duw/YtnNSTMltG7uFO7tPrwt2+TGnqmOr9dPka//5HsfV2yZSyhMAIIQoB/AzAN+2Or6iohTRaMTNUGMSiXLXbc2KBw0kh3LqN11rh3nhmkSi3PR5M6l2RlKP242ZD25jcJoPFakx7YpFWcXmJq5DHT2w+9XSaEy7sdy0sZPep5O+VK4pP+ZUdWwz6e3ydb/kaxzXX8YghDgNwC8A3Cel/N9Wx3Z25japub5tEouEDG/oWCTs2V9g1VSVoqV9YoWxuVVxtLd3mz5vJtUuW3ou7MbMB7cxOM2HitSYZvOtEpubuEaKEw1btjMa024sN21UYrW7flRjyebHnKqObSR73cjX/eLlOFaLvau3TYQQswH8GsDXpZSPuOkjn8yKBzUsrfFsDLvCNU6KI6W3y2XMfHAbg9N8qMUyMqZdsSir2NzEtX7VfNt2ToopuW2zTCQs+8vu0+uCXX7MqerYav3U+9p/vsdx+8r7OgAVALYKIVLvfV8kpTzpTVjeSn0o2binFQPJIcQiYTQsrfF0t0nqg4iduw/gUEcP5lbFsX7V/LHHs5+fEZ8ChICuE/0Z/85ul8uY+eA2BqN2Yt7MibtNpo3uNjk5uttkegk2r14woW36mNnzDaR2m5TZxpYe18EjJxANh5EcGkJNddnYzWd1rmbtjMa0G8uqjdluk+deedvwGjOKNfXvjN0m00uwbGECsvmY42tKdU4rp5dgblUc+5o6kRzdMFESiyA+bXS3iUIe3MjX/ZKvcViYSjPMRSbmYxxzkakY8sHCVERERYaLNxGRhrh4ExFpiIs3EZGGuHgTEWnI9R/p5FPjGy14ete+CRvfwyFgaBiIhACbP6pD2bQYLl+3CG8f7MLLv2+xPd5MSSyCaCQ0tk1r5LEw+geHYLRxJxRCxuPZP6e2eV2+btGEIkUDyWHLcwuFgCnRCPoGku5OxoHsAlG5ys5DSqrwVKoeTPo2toz2gOVfNUbCIQwND6OirATAeAGwvoFkxjlEwiEkh3LbcRUCUFFektG3V/mKRkbiy85VqtCWmFcB2dw5cs1EwxhMDtmes1n8ANDZ3YewwTVnNl8xi+s0NQfZBaayi2dlx5edu+yx7XKbmlOzuTW6h62kn8eMspKMglqq0q9rrwR+q6BfxW6CZsnplXjznaOFDiMw1i6vsywuRfpRKeBV7JwWp7LaKhj4V95+FbsJGi7cmbwsGkbBMFKYyf8Xi0G2c/cBz159B37xbj3iT7EbCrbUX0ZS8TjU0WP41stkcqjDu9ovgf/Asqa6tNAhUAHEIoG/NMmhuVXxSX8/jxQx80bg7xC/it0EzZLTKwsdQqB4WTSMgkGlgFex87I4VWTbtm2edWamt7ff9SB1iTIsqq9E86FuHO/tz3guPPpWfiQUsn0nrWxaDFd+9EzMLCtB87vdrt95K4lFMLUkgoHBobTHwhgy+X0wFLL+ORIO4SPn1GHLhiWYU1mKtqMn0X2yH7FIGMPDw5bnFgqNxJPrbgkV8WlRTImFM847F9l5SKmcXoIrLliMi1bOx5zKUrzTehwn+yfupjH9FGdUJBwa20UxbWoU/QMjuzDCkVDGOUTCoZx/lQ9hZDdBet9e5SsaMZ7/EEbujfefOQsDg0Mj10w0DAwP256zWfzTpkZxqj9peM2ZzZfVdZqag9pEGS47byFWnDUbdYmyseu859SAYXzZucse2y63kXAIGDafW6N72Er6eZw2uwxHu045Xj9S17XT97vj8ZIbzZ4L/G4ToDgKzHiFucjEfIxjLjIVQz5YmIqIqMhw8SYi0hAXbyIiDXHxJiLSEBdvIiIN5WW3CREReYuvvImINMTFm4hIQ1y8iYg0xMWbiEhDXLyJiDTExZuISENcvImINMTFm4hIQ1y8iYg0xMWbiEhDXLyJiDQU+G+PJ/KDEKIewH4AewEMA5gCoBXAlVLKlgKGRqSEizdNZq1SyqWpH4QQ3wdwO4DLChcSkRq+bUI07iUAZxc6CCIVXLyJAAghYgA+AWB3oWMhUsG3TWgyqxFC7Bn9dwmA3wL4RgHjIVLGxZsms4z3vIl0wrdNiIg0xMWbiEhD/A5LIiIN8ZU3EZGGuHgTEWmIizcRkYa4eBMRaYiLNxGRhrh4ExFpiIs3EZGG/j/lMYz365TSAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21427dfa898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.plot(x='P', y = 'cardiac_arrhythmia',  style = 'o')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similarly, we cannot drop observations with missing P because the outcome variable contains positive diagnosis.\n",
    "Since there is no linear relationship between P and the outcome, replace missing P with the mean value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    7\n",
       "Name: cardiac_arrhythmia, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['cardiac_arrhythmia'][data['Heart_Rate'].isna() == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x21427da0da0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAD8CAYAAAC4uSVNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucXVV99/HPmTPDJHMJmcsBMonJgJAFRG0wWojVGAw3jYRCm0cpyqNVGlvRx8sDVisSHu+iXFrBCwgoUrV4KxIp8lAwtq+UCiWlclmIJYnJhDCZDMlkJpl7/5g5M+ec2fezz2VPvu9/yJy911q//Vt7L87ss+d3UuPj44iISLLUVDoAEREJT4u3iEgCafEWEUkgLd4iIgmkxVtEJIFqyzFId3dfUY+0tLQ00Ns7EFc4iaZc5FM+pikX+WZDPjKZ5pTbtkS8866tTVc6hKqhXORTPqYpF/lmez4SsXiLiEg+Ld4iIgmkxVtEJIG0eIuIJJAWbxGRBCrLo4KV9MhTe9i0ZRtdewfoaG9g7cpOTj/12Ej93P3Qc+zrGwSgtbme9Wee6NlXXGNHibUc48aZ26D93PXAs2zeuovh0XHq0inOXdnJRa8/PvIYpcxV0uffqR8gUN/FtHXrwyxuwe7ojXxc2f52dfdTm04xMjbOwvZGz34qNYdBpIJUFTTGnA580Vq72hhzDHAL0AKkgUuttb/zal/sc96ZTDPd3X2h2z3y1B6+cc+TM17fsG5Z6El36serr7jGLuSXi1KNW6pxwvRz1wPP8uBjO2fsu2bFIi45e2noMdasWOTYXxy5Ktc85Mpkmrn3l8+V/Jz367uYtmH78Dqu3GvFrz+nfioxh4WKes7bGHMlcCswZ/KlLwF3WWtXAZ8ETo4jyFLYtGWby+vbY+nHq6+4xg6rXOOWOrdO/Wzeustx381buyKN4dYujlwlff69znm/votpG7aPoMfl159TP5Waw6CC3Db5HXARcOfkz38EPGGM+f/ANuD/+HXQ0tJQ9APzmUxz6DZdPc5/XbW7pz9Uf279ePUV19hOvNqXctxSjBOmn+FR51/ghkfHIuVkeHQs8NhhlWseSjWu1znv13cxbcP24Xdc2W1+/Tn1U6k5DMp38bbW/sgY05nzUifQa609yxjzKeBjwKe8+ij2T1Sj3jbpaGtgZ3f/jNcXtDWG6s+tH6++4hq7kF8uSjVuqcYJ009dOuW4gNelayLlpC5d47iAx5Grcs1DrkymuSznvF/fxbQN24fXceVeK379OfVTiTks5PU/iShPm/QA90z++2fAayL0URbZD0lmvr4kln68+opr7LDKNW6pc+vUz6rlCx33XbW8I9IYbu3iyFXS59/rnPfru5i2YfsIelx+/Tn1U6k5DCrK0yb/AryFidsoq4Bgn0xUQPZDhU1btrO7p58FbY2sXbkk9IcN2f3znjaZV8/61e5Pm8Q1dljlGjfu3AbpJ/uh5OatXQyPjlGXruHclUt8nzbxGuPEhUeXJFdJn3+3foL0XUxbrz7M4vnYHS9FOq7c/nbtPUhtTQ2jY2N0tDe59lOpOQwq6NMmncD3rbVnGGOWMPEBZiOwH/gza22vV/tKPW0yGykX+ZSPacpFvtmQD6+nTQK987bWbgPOmPz3duDsWCITEZFI9BeWIiIJpMVbRCSBtHiLiCSQFm8RkQSa9YWppLpUotBPJQpoldNUXD0DdLRVT1xJUq1z60WLt5RNYaGfnd39Uz+X6kKJa8xKxB5EtcaVJEnNoW6bSNlUotBPJQpolVO1xpUkSc2hFm8pm6697oV+qn3MSsQeRLXGlSRJzaEWbymbjvYGx9cXtDVW/ZiViD2Iao0rSZKaQy3eUjaVKPRTiQJa5VStcSVJUnOoDyylbCpR6KcSBbTKqVrjSpKk5jBQYapiqTBVfJSLfMrHNOUi32zIR1FfgyYiItVHi7eISAJp8RYRSSAt3iIiCaTFW0QkgbR4i4gkUKDnvI0xpwNftNauznntz4APWGtXlii2PFGrfiWxWpho3iS6I6XKou/ibYy5Engn0J/z2nLgPYDrM4hx2vz4zkhVv5JaLexIp3mTqI6kcyfIbZPfARdlfzDGtAFfAD5UqqAK3f3gbx1f96v6ldRqYUc6zZtEdSSdO77vvK21PzLGdAIYY9LAt4APA4eCDtLS0kBtbTpqjOzY4/xXUrt7+slkml3bdfW4Vwvzalftkhx7EGHnbbbnI4wjPRez9Zp3Era2yQrgJOBrwBzgVGPMDdZaz3fhvb3OCQ1q8bHNbNt9YMbrC9oaPf/8taOtgZ3dM8s6+rWrZrPhT379hJm3IyEfQSkXs++a9/ofTqinTay1/26tXTb5weXbgaf8Fu44rF9zkuPrflW/klot7EineZOojqRzJxFVBVedtogDBw6HrvqV1GphRzrNm0R1JJ07qiqYMMpFPuVjmnKRbzbkQ1UFRURmGS3eIiIJpMVbRCSBtHiLiCSQFm8RkQTS4i0ikkBavEVEEkiLt4hIAmnxFhFJIC3eIiIJpMVbRCSBtHiLiCSQFm8RkQTS4i0ikkBavEVEEkiLt4hIAmnxFhFJIC3eIiIJFOg7LI0xpwNftNauNsYsB/4OGAUGgUuttXtKGKOIiBTwfedtjLkSuBWYM/nSjcAHJr9B/sfAx0oWnYiIOApy2+R3wEU5P7/dWrt18t+1wOHYoxIREU+Bvj3eGNMJfN9ae0bOa68DvgWsstZ2e7UfGRkdr61NFxmqiMgRx/Xb4wPd8y5kjHkb8DfAWr+FG6C3dyDKMFMymWa6u/uK6mO2UC7yKR/TlIt8syEfmUyz67bQi7cx5h3ABmC1tXZfEXGJiEhEoR4VNMakgb8FmoEfG2MeNsZcU5LIRETEVaB33tbabUD2fndryaIREZFA9Ec6IiIJpMVbRCSBtHiLiCSQFm8RkQTS4i0ikkBavEVEEkiLt4hIAmnxFhFJIC3eIiIJpMVbRCSBtHiLiCSQFm8RkQTS4i0ikkBavEVEEkiLt4hIAmnxFhFJIC3eIiIJpMVbRCSBIn17fDk98tQe7v/1o+x4oY+O9gbWruwEYNOWbezq7qc2nWJ4dJy6dIqRsXEWtjeydmUnp596bF4fm7Zso2vvwFQf2e1e2wrjCLKf2zGEbfvIU3u4+6Hn2Nc3CEBrcz3rzzyRt77R/duko8YC+fnMzSPAd39h6T88MtVHCliYmZnnKGOHaR/EXQ88y+atu6bOiVXLF3LJ2UtLEkuYPvz2LdxuFrdgd/TOmKfcc6JxzsTlm52b9vlz+ZNVJ8yIwelcOm1pZqr/+U1HAdDbN+g4/25td3X3U5OC0XGmtq0/80SAilxv2TZO53GUecntz+k43eYvm8+XDg6V7DwHSI2Pj/vuZIw5HfiitXa1MeZE4A5gHPgN8H5r7ZhX++7uPv9BHDzy1B6+cc+TUZqyYd0yTj/1WNc+NqxbBuC6rXBiguznJEpbr+O+4h0rOGXR0Z5jho2lGEFy4DV20PZuMplmurv7gImF+8HHds7YZ82KRXkLeByxhOnDb9+45yU3hlLMeRSlvt78jjPsvLjF6tRnkBxHPc8zmeaU2zbf2ybGmCuBW4E5ky9dB3zSWvsGJt6EXRA6ooA2bdlWRNvtnn1s2rLdc1uQOAr3c94nfFuv4777wd/6jhml3+h9+ufAa+yg7YPYvHWXy+tdsccSpg+/feOel9wYSjHnUZT6evM7zrDzEiRvYeYvzvM8K8htk98BFwF3Tv68Avjl5L/vA84BfuLVQUtLA7W16dDBdfUMhG6Ttbunn0ym2bWP3T39uP06kG3rF0fhfk6itPU67t/v6fMdM2wsxQiSA6+xg7b3km0/POo8o8OjY7HNZ5Q+/PaNe15yYyjFnEdR6uvN7zjDzkuQWwVh5i+O87yQ7+Jtrf2RMaYz56WUtTZ7bH2A7+/wvb3RTqCOtgZ2dvdHarugrZHu7j7XPha0NQLjrtuyv4p7xVG4n5Mobb2O+2XHNvuOGTaWYgTJgdfYQdu7yb1tUjf5+UehunRNbPMZpQ+/feOel9wYSjHnUZT6evM7zrDz4harU59Bchz1PPda8KM8bZJ7f7sZeClCH4FkPzCJ1naJZx9rVy7x3BYkjsL9nPcJ39bruNevOcl3zCj9Ru/TPwdeYwdtH8Sq5QtdXu+IPZYwffjtG/e85MZQijmPotTXm99xhp2XIHkLM39xnudZUZ42edwYs9pa+zDwZuCheEOalr3Bf/+vf8/v9/SxoK1xKgmbtmxn196D1NbUMDI6Rm26htGxMTram1i7cslU2+x/N23Zzu6e/qk+cj888NoWtA+/YwjTNrst71P+efWsX30iq05bFPmdqlss2dey+czNI7g9bdIUOAdeY8f5KXz2Q8nNW7sYHh2jLl3DquUdM542iSOWMH347eu03Syej93x0ox5ynvaZO7k0yaHJuYmM38uFxU8beJ2Lp12Umaq/6Mbj4LU5NMmDvPv1nbX3oPUkGJ08qGH7Dnql5dSXG+5bZzO4yjzktuf03G6zV82n/sPDpXkPM8K+rRJJ/B9a+0ZxpilwC3AUcDTwGXW2lGv9lGfNsnK/dX4SKdc5FM+pikX+WZDPryeNgn0zttauw04Y/LfzwJvjCUyERGJRH9hKSKSQFq8RUQSSIu3iEgCafEWEUmgqi9MNduFKX5zJAlbmKgcRa8kfsUUoPIq4lXM3CflXNLiXUGFBW1y/zhwX9/g1LZqPHFKqTAvO7v7PXMRdn+pDlHmzalN7l835vYRpQJnks4l3TapoEoVtKl2YQsTlaPolcSvFAWogvTh3c65/2o8l7R4V1DX3mAFbY40bnlxy0XY/aU6RJm3INeMXx9xx1QpWrwrqKO9wXefiSI5Rxa3vLjlIuz+Uh2izFuQa8avj7hjqhQt3hVUqYI21S5sYaJyFL2S+JWiAFWQPrzbOfdfjedSeuPGjSUfZGBgqKhBGhvrGRgYiima6rEo08RxrQ3s2XeIvkNDpFOpqTrCrfPqufTck2d8SFJtuXjkqT18854nueuB3/KofZGGOXUsyjQV1WduXvoPD7OwvYmLzzrJ8QOjxsZ6WhrrpvPYP8COX36JKz+4gX974Lt0dp5AU1Nx8UTxp396Phdc8CfU1no/EzA4OMh9992LMSfzrW99gx07tnHyyaeGGuvOO++grq6Wzs6XVdW54SfMPHu1ee0pxzA8MjajjyjXSpSYSqmxsf4at2162qTCTj/12Kr7FDuoUn4yHzYv2f0HBwe55LG/m/z5o0XFUA779vXws5/9lPPP/+PIfbzzne+KL6Ayi3L+l/qaSco1qcVbIvP6ZD7qyT84eJjPfe4aXnjhBUZGRrj88g/x4x/fzcGDfezf/xLnn38hF174p1x++V8wf34Lg4MDXH315/nMZz5FX18fCxcumurr8sv/giuu+ARz587ly1/+AkNDgxw4sJ93vesyVq1azb/+66+4/fZbADjpJMMVV3ycmpqZdxJHR0e59trP8eKLe9i/fz9nnPE6LrvsL/nsZzeyf/9+DhzYz8UXv5PvfvcO6urqWLfuQgC+8pUv0NU18dVsn/vcl7nuui9yzjlv5nWvez3btj3PTTfdQHt7hm3bnp+K41e/2sxDDz3I/v37ee9738frX7+Kt73tj3nFK17Fzp2/59Wvfg39/Qd5+uknWbx4CVdd9Wk++9mNrFlzDmee+UdcddVfz8iVzE665y2RleKT+Z/+9Eccd1wH3/jG7fzN31yNtU9z1lnncP31N/GlL93AD35w19S+Z599HnfccQf337+J449/OTfddAsXXPAnM/rcvn0bb3/7Jdxww818+MNX8uMf/wMjIyNcf/2XuPbaG7j11u9wzDHH8OKLLzrG9OKLe1i27JVcd91XufnmW/npT384tW3Fitfw9a/fRnNzM0NDQ9x8862cd95aANauvYCvfvWbLFjQwa9//Qjr1l3IfffdC8CmTffw1rdewKWX/jmdncfz7ndfBkAmk+HGG7/GBz/4kalxXnhhN5dd9lfcdNMt/PCHP+DCC9fzzW9+myee+E/6+qZLnm7fvt01VzL76J23RNbR7vU1UtHs2LGdM854HQAnnHAi8+Ydzde+9nf88pcP0dDQyMjI9JdCLF488SHS88//N6efvhKAZcteMeM+c1tbO9/+9rfYtOkfgRQjIyPs3/8Szc3NtLS0Akwtnk7mzZvH008/yX/8x6M0NjYyNDQ8I4bCfwOcfPLJALS2tjE4eJjTTlvBDTdcS2/vPv793/+NDRveT3d3/v8wjDllKubDhw9Pjn80xx13HABz587l+ONPAKCxsYmhocGptu3t7WzefKtjrmT20TtviawUn8wvWXI8Tz/9FAC7du3kuuu+xCte8So+9alP86Y3nUXul4dkb3EsXtzJb37zXwA8++wzMxatW2/9Ouedt5arrvo0r371awBoaWnl4MGDHDiwH4AbbriWp576jWNMP//5vTQ1NXP11Z/h7W9/B4ODh6fiSKWmL6GamsK6+fk/p1Ipzjnnzdxww5f5wz88g9raWlKpGsbHx3L2mTl+yulFB7fddptrrmT20TtviawUX212wQUX8fnP/z8uv/wvGB0d5Q1veCN33/09fvGL+zj66KNJp9MMDeU/QXDRRev5/Oev4S//8j0sWdJJXV1d3vYzz1zDjTd+mTvvvJ1jjjmWl156iZqaGj7ykY9xxRUfoqamhqVLDaecsswxphUrXsvGjZ/giSe2MmfOHBYtehl793ZHOr63vOV8LrpoLd/+9vcBaGlpYXh4hJtv/lvq6+sj9Zl15plnctVVn5qRq6OOOqqofqU6BfoatGLpa9Dio1zkS1o+urtf5DOfuZobb/xa7H0nLRelNhvyUfTXoBUyxtQB3wY6gVEmvsfymUjReZiq7tUzQEdb9Vb3ktnh9ttv4bHHfj3j9U984mo6Opy/mT6Mhx9+kNtu+yZ//ddXFd3XbJWUin7VINI7b2PMBcAl1tr/ZYw5G3iftXbmx/yTorzzLnyGOGvDumVH9GTOhncTcVI+piU9F3Ff80nPB3i/8476geWzQK0xpgaYBwz77B9akqp7iUjxdM2HE/UDy4NM3DJ5BmgH3uq1c0tLA7W16VADdPW4P0OcyYSv0zubHOnHX0j5mJbkXJTimk9yPvxEXbw/DNxvrf24MeZlwD8bY15prT3stHNvb7Ayjrk62tyfIU76r0LFmA2/CsZJ+ZiW9FzEfc0nPR/g/T+fqLdNeoH9k//eB9QB4d5a+0hSdS8RKZ6u+XCivvO+HrjNGPMr4CjgE9baWKuVl+IZYhGpXrrmw9Fz3gmjXORTPqYpF/lmQz5K8bSJiIhUkBZvEZEE0uItIpJAWrxFRBJIi7eISAIloiTs5sd38r37n1GxGpEQwhZ5UlGoZKn6xbuUX3IrMluFvW50nSVP1d82UbEakfDCXje6zpKn6hfvUnzJrchsF/a60XWWPFW/eHe0Nzi+XsyX3IrMdmGvG11nyVP1i7eK1YiEF/a60XWWPFX/geXppx7LvHlz+N79VsVqRAIKW+RJRaGSR4WpEka5yKd8TFMu8s2GfKgwlYjILKPFW0QkgbR4i4gkkBZvEZEE0uItIpJAkR8VNMZ8HFjHxHdY3myt/VZsUU2aKpTTM0BH28xCOXEV3gnzOkz8KfGu7n5q0ylGxsZZ2N6YN/YjT+3h7oeeY1/fIACtzfWsP/PEGbFFLQTk1X+5igsVM04lCyBVS/ElvziinNv3//pRdrzQR0d7A2ZxC3ZHb8mOM2h8hftFjavS81bp8Z1EelTQGLMa+ChwAdAA/F9r7Ua3/aM8KlhYKCdrw7plU4uU1/ag/a1ZsYgHH9sZ+HUvG9YtA3AcpzC2sPFnPb1zP9d+9zHHbW4x+/UZVtTYi23rJMzjYHGPHVW5zu2g7cMKGl9ccUWdt7geFazkeVOKRwXPBf4L+AnwM+DeiP248iuUE1fhnc1bu0K97mXTlu2u42S3+8XjVwjo7gd/67rNLea4iwsVU8SokgWQqqX4UrnO7aDtwwoaX1xxVXreKj2+m6i3TdqBJcBbgeOBe4wxJ1trHd9ht7Q0UFubDjVAV497oZxMptl3e9D+hkfHQr3uZXdPP16/YuTGFjb+rB173N9JuMXs12dYUWMvtq2boO1KMXYU5Tq3g7YPK2h8ccVVzLyV83jLLeri3QM8Y60dAqwx5jCQAV502rm3N9gk5upoa2Bn98yKZgvaGunu7vPdHrS/unSN46Ln9rqXiSI+447jFMYWNv6sxcc2s233AcdtbjH79RlW1NiLbeskzK/GcY8dVbnO7aDtwwoaX1xxRZ23uG6bVPK88fqfQ9TbJv8CnGeMSRljOoBGJhb02PgVyomr8M6q5R2hXveyduUS13Gy2/3i8SsEtH7NSa7b3GKOu7hQMUWMKlkAqVqKL5Xr3A7aPqyg8cUVV6XnrdLju0lv3LgxdKMPfOADz371q199JfA54BLgo9Za15uxAwNDoQdZlGniuNYG9uw7RP/hYRa2N3HxWSdNfUDgtz1of28+Y0mo119jjmHPvkP0HRqiLl0DjLMwMz12dpznuw5waGgUgNZ59Vx67sl5sYWNP+vUl2eYN6fWsX+3mOP+UCVq7MW2ddLYWM/AwFDJ445Tqc7tngOD9A0MsbC9ideecgzDI2MlOc6g8TntFyWuqPMW5twoxfhxaGysv8ZtmwpTJYxykU/5mKZc5JsN+VBhKhGRWUaLt4hIAmnxFhFJIC3eIiIJpMVbRCSBqv47LCslSCGaoAWoSjF2qcUVQzUcS7lV8zHnxja/6SgAXjo4VHSccRxz0Gsu6DibH9/J9+5/pirnIQ6RnvMOK8pz3rniel4zqGwhmgMDw4wDBwaGecx2c1xrA4syTXn7ZJ+1Bjg0NDpjv7jHLkcughx/OfvxUu5zw085jtmNXy4KYzs0NMqhodGi44zjmMNcc0HGeeSpPfzt3f9ZkXmIk9dz3rpt4iBIIZqgBahKMXapxRVDNRxLuVXzMQcpFBUlzjiOuZhrzmmcap6HuGjxdtC1170Qjd8+hfuVYuxSiyuGajiWcqvmY/Y6Z7OixBnHMRdzzTmNU83zEBct3g462hscX58oPOW9T+F+pRi71OKKoRqOpdyq+Zi9ztmsKHHGcczFXHNO41TzPMRFi7eDIIVoghagKsXYpRZXDNVwLOVWzcccpFBUlDjjOOZirjmncap5HuKiDywdBClEE7QAVdxjlyMXcRXiKUdBn2r7wLLCRYw8c1EYW0tTPXPn1DI0PFZUnHEcc5hrLsg4izJNLO1sZcfuvooWISuWClPNIspFPuVjmnKRbzbkQ4WpRERmGS3eIiIJpMVbRCSBtHiLiCSQFm8RkQQ6YgtTeRW4KVdhoew4u7r7qU2nGBkbp6WpHphZLGgqpp4BOtoaMItbsDt6E1F0J0zBodxcLGxvLGuxpDD7lyJeKa1qLhgWRVGPChpjjgEeA8621j7jtl+1PSqYLXBTaMO6ZQCu2+KcaLcYnKxZsYgHH9vpu1/cMcbBK9e5/7P0yoXXcbmdG0HGjbp/MfGW0mx4NC5OufkIez5Ui5I8KmiMqQO+ARyK2keleBWtKVdBmyBFgrI2b+0K2Gf1Fd0ptshX4b5xjht1/1LEK6U1GwtVFXPb5MvA14GP++3Y0tJAbW26iKEm/i8al64e96I1br8i7O7pL0sMToZHxwLtF3eMcfDKdTZWv1z4HZfTtiDjRt2/2HhLqdrmv9L8zrFqvGaCirR4G2PeBXRba+83xvgu3r29wRcqJ3H/OtjR1sDO7pnVxSaK1oy7bitHDE7q0jWBFvC4Y4yDV66zsfrlwuu43M6NIONG3b+YeEtJt03y5eYj7PlQLbz+xxL1tsmfA2cbYx4GlgPfMcYcF7GvsvMqWlOugjZBigRlrVreEbDP6iu6U2yRr8J94xw36v6liFdKazYWqor0zttauyr778kF/H3W2hfiCqrUsh9QbNqynd09/Sxoa2TtyiV5H1x4bYs7hl17D1JbU8Po2Bjzm+ohBfsPDuWNfeLCo/NiMovnY3e8VNIY4xAk12656GhvinxcQcaNun8p4pXSCns+JEHRhalyFu/EPG2SZMpFPuVjmnKRbzbkw+tpk6Kf87bWri62DxERCUd/YSkikkBavEVEEkiLt4hIAmnxFhFJIC3eIiIJlMiqgkmsDpbEmJNg8+M7+d79z/jm1Sn/QCxzctcDz7J56y6GR8dJp2DunDoGDo8UPc9hz5mguSiX3PjnNx0FzKyWGaaPoJUes/tefO7JnLLo6JIcTzXkN3FfQJzE6mBxxjwbnl2NS9C8hqngGHZO7nrgWd+Kj1HmuZRVEcshSM79Youj0mNcx1+p/M6qLyBOYnWwJMacBEHzGqaCY9g52bx1V+x9TrTZFqqvajvHguTcL7Y4Kj3GdfzVll9I4OLdtde9Oli1SmLMSRA0r277BWnrZ3jU/5fKKPMc9pyptnMsSM79YgtzTKU+/mrLLyRw8e5ob3B8faIiYHVKYsxJEDSvbvsFaeunLu36W23kPiH8OVNt51iQnPvFFuaYSn381ZZfSODincTqYEmMOQmC5jVMBcewc7Jq+cLY+5xo0xmqr2o7x4Lk3C+2OCo9xnX81ZZfgPTGjRtLPsjAwFBRgzQ21jMwMATAokwTx7U2sGffIfoPD7OwvYmLzzqpaj+shHhjzs3FkW5Rpomlna3s2N3nmVe3/L/GHFP0nLzq5W0cPDTMzhcPMjY+TjqVonFuHaNj40XNc9hzJmguyqUw/pameubOqWVoeCxwbGFy4LTvhoteyfKXt5XkeMqV38bG+mvctiXuaZMjnXKRT/mYplzkmw35mFVPm4iIiBZvEZFE0uItIpJAWrxFRBJIi7eISAJFKkxljKkDbgM6gXrgM9bae2KMK3a5xYPq0ilWLV/IJWcvjXUMv8I1boV6ohbtCTJmOUSNobCdWdzC4892s69vcGqfxjkTp2j/4REAWpvrWX/micBkUameATraZo6ZO98AKWBhpjFQbNm4dnX3U5tOMTI2zsL2Rt9iVl7t/IophW3z3V/YqZyka1KsPm0hH/qzFaGLQT3y1B67irNtAAAJK0lEQVTufui5qZy3Ntdz2tIMdkdvpHMqyJy2NtezoL2RZ7btI/sHqvV1aRrn1NLbNxg4D1GU63opxziRHhU0xrwb+ANr7YeMMW3A49baxW77V/pRQbfiQWtWLIptAfcrXBOmOFJuu0LVVqQragxh8xFEdky/YlFesUWJa8O6ZQCe7cIUU4ra5jST4XHb7Rtr7v9s4izYVYo5DTq2k8J1o1zXS8yF6GJ/VPBu4Kqcn0ci9lMWbsWDNm/tim0Mv8I1YYoj5bYrZsxyiBpD2HwEkR3Tr1iUV2xR4tq0ZbtvuzDFlKK28Vu4C/uMu2BXKeY06NjB+tlW0v7LPU6k2ybW2oMAxphm4IfAJ732b2lpoLY2HWWoKZlMc+S2bsWDhkfHiuo3V1ePe+GaTKbZdbubbDsn2df9xiyHqDGEzUcQ2TH9ikV5xRYlrt09/fj9auk0pt9YUdr4ye0zTF9BzqlSzGnQsd3ktivX9VKucSJ/GYMx5mXAT4CbrbV/77Vvb29xk1rsbZO6dMrxgq5L18T2F1gdbQ3s7J5ZYWxBWyPd3X2u291k2xXKzYXfmOUQNYaw+QgiO6bbfAeJLUpcE8WJxj3bOY3pN1aUNkFi9Tt/gsZSqBRzGnRsJ4XrRrmulzjH8VrsI902McYcC/wC+Ji19rYofZSTW/GgVcs7YhvDr3BNmOJIue2KGbMcosYQNh/BYpkY069YlFdsUeJau3KJb7swxZSitjnNZDz7K+wz7oJdpZjToGMH66ezpP2Xe5yo77w/AbQAVxljsve+32ytPRRPWPHKfii5eWsXw6Nj1KVrWLW8I9anTbIfRGzasp3dPf0saGtk7colU68Xbj+68ShIwf6DQ3n/LmxXzJjlEDUGp3Zm8fyZT5vMnXza5NDk0ybz6lm/+sQZbXPHLJxvyD5t0uQbW25cu/YepLamhtGxMTram6YuPq9jdWvnNKbfWF5t3J42ufeXzzmeY06xZv+d97TJvHpOOymD3fFS6HMq6Jy2zqtnQVsjz2zrZXTygYn6ujSNcyefNgmQhyjKdb2UaxwVpkoY5SKf8jFNucg3G/KhwlQiIrOMFm8RkQTS4i0ikkBavEVEEkiLt4hIAkX+I51y2vz4Tr53/zMzHnyvScHYOKRT4PNHdTTNreOSs5fy3K79PPwfO333d1Nfl6Y2nZp6TGvitRqGRsZwenAnlSLv9cKfs495XXL20hlFioZHxz2PLZWCo2rTDA6PRjuYEAoLRBWrMA9Z2cJT2XowuY+x5bUHz79qTNekGBsfp6WpHpguADY4PJp3DOmaFKNjxT1xlQJamuvz+o4rX7XpifgKc5UttGUWt2B39E6cM7U1jIyO+R6zW/wAvX2D1Dicc27zVedxnmbnoLDAVGHxrML4CnNXOLZfbrNz6ja3Ttewl9zjOLqpPq+gVlC553Vcqv5RwVIVu6k2y45v5cnn91U6jKqxZsUiz+JSkjxBCnjNdmGLU3k9Klj177xLVeym2mjhzhdn0TCpDhOFmUr/ZrGabdqyPbZ331W/eHftLU2xG6lu2b+MlNljd0+/462XI8nunvhqv1T9B5Yd7Q2VDkEqoC5d9aemhLSgrfGIv54nipjFo+qvkFIVu6k2y45vrXQIVSXOomFSHYIU8Jrt4ixOld64cWNsnbkZGBiKPMiiTBNLO1vZsbuPAwNDedtqJm/lp1Mp3ztpTXPrePdbTmF+Uz07XuiLfOetvi7NnPo0wyNjOa/VMOby+2Aq5f1zuibFm169iA3rlnFcawN79h2i79AQdekaxsfHPY8tlZqIp9inJYJonFvLUXU1ecddjMI8ZLXOq+fSc0/mzWcs4bjWBp7vOsChoZlP07h+ijMpXZOaeopi7pxahoYnnsKoSafyjiFdkyr6V/kUE08T5PYdV75q087zn2Li2njtKccwPDI2cc7U1sD4uO8xu8U/d04th4dGHc85t/nyOk+zc7Aw08TFZ53E6acey6JM09R53n942DG+wtwVju2X23RNCsbd59bpGvaSexwvO7aJffsPh14/sud12PvdjY3117htq/qnTWB2FJiJi3KRT/mYplzkmw35UGEqEZFZRou3iEgCafEWEUkgLd4iIgmkxVtEJIHK8rSJiIjES++8RUQSSIu3iEgCafEWEUkgLd4iIgmkxVtEJIG0eIuIJJAWbxGRBNLiLSKSQFq8RUQSSIu3iEgCafEWEUkgLd4iIglUW+kARAoZYzqBh621nQWvj1tr/b6+0q/v44FPWmvf47PfOPCfkz+mgPnAPwF/Za2d+aWa0+1uBzZaa7cXE6eIHy3ecqRZArw8yI7W2uXZfxtj5gG/Ac4B7vNodibg+qWxInHR4i2JYoxJA9cCq4E0cIe19npjTC3wNeAVwLHAE8DFk//+J2AvcGjy5xOMMTdZa98fYuh2oAHYNxnHZ4E1QCvQBbwNeDfQAfzcGPMG4ATg+sl2e4EN1trnIx+8SA4t3lKtOowxWx1evwzAWvtqY0w9cL8x5lEmbm0MWWtXGmNqgH8G3gI8BhjgPGvtNmPMaiZua/gu3JPj1wHHAE8DH7TWPmKMORE4GXidtXbMGPMd4B3W2i8YY943OW4fcCtwvrV2hzHmXOAW4KzoKRGZpsVbqlVX7m0LmLoPfRaw3BjzpsmXm4BXWmtvNsb0GGPez8TCetLkNoAXrbXbwgaQHd8Y82HgXcA/Tr7+nDHmo8B7jTEGWAn8rqD5UiZuz9wzsQsA88LGIOJGT5tI0qSBK621yycX1zOA24wx64C7gAHgdmAzE+/GYeJ2SWTW2uuBF5i4XYMxZgXwCyaunx8CP8kZKzfO/86JcwXw+mLiEMmlxVuS5p+By4wxdcaYJuBfmFjAzwL+wVp7O/ASEx8cph3ajxDtN86PAO8xxrwKeCMTT8N8HXgWeGvOWNn+nwFaJ+99A/w58PcRxhVxpMVbkubrwG+Bx4FHgduttQ8zcT/5YmPMfwF3A/8KHO/Q/mlgvjHmzjCDWmufBL4DfAX4AfAHk2M9PBlHdqx7gZ8z8cHleuArxpgngP8NeD6eKBKGvoBYRCSB9IGlHJGMMS8HfuSy+b3W2kfLGY9IWHrnLSKSQLrnLSKSQFq8RUQSSIu3iEgCafEWEUmg/wH7PkZEcG+cKAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21427dc24e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.plot(x='Heart_Rate', y = 'cardiac_arrhythmia',  style = 'o')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similarly, we cannot drop observations with missing Heart_Rate because the outcome variable contains positive diagnosis.\n",
    "Since there is no linear relationship between Heart_Rate and the outcome, replace missing Heart_Rate with the mean value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "241    1\n",
       "Name: cardiac_arrhythmia, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['cardiac_arrhythmia'][data['QRST'].isna() == True]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can drop this observation from the data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.drop(data[data['QRST'].isna() == True].index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filling the rest of the missing values with missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = Imputer(missing_values = np.nan, strategy = 'mean', axis = 1).fit_transform(data)\n",
    "data = pd.DataFrame(data = z, columns = data.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### X's and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = data.cardiac_arrhythmia.values.ravel()\n",
    "X_raw = data.drop(['J','cardiac_arrhythmia'], 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check to see if there's any other missing data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(X_raw.isnull().sum() > 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparison Table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will compare different algorithms using R-square, MSE, and running time. \n",
    "Additionally, since this is a healthcare problem and negative diagnosis (y=1) is much more prevalent than positive diagnosis, we'll also look at the accuracy, accuracy_positive (how the model accurately predicts each type of diagnosis when y > 1), and true_positive (how the model can predict people with any positive diagnosis)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(index=None, columns=['model','train_Rsquare', 'test_Rsquare', 'train_MSE','test_MSE', 'accuracy', 'accuracy_positive', 'true_positive', 'duration'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a function to calculate the results matrix for each model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def result(X_train, y_train, X_test, y_test, y_test_simple, name, grid_search, duration):\n",
    "    train_Rsquare = grid_search.score(X_train, y_train)\n",
    "    test_Rsquare = grid_search.score(X_test, y_test)\n",
    "    train_MSE = mean_squared_error(y_train, grid_search.predict(X_train))\n",
    "    test_MSE = mean_squared_error(y_test, grid_search.predict(X_test))\n",
    "    accuracy = sum(y_test == grid_search.predict(X_test))/y_test.shape[0] \n",
    "    accuracy_positive = sum(y_test[np.where(y_test>1)] == grid_search.predict(X_test[np.where(y_test>1)]))/y_test[np.where(y_test>1)].shape[0]\n",
    "    y_predict = grid_search.predict(X_test)\n",
    "    y_predict[y_predict == 1] = 0 # normal\n",
    "    y_predict[y_predict >0] = 1 # not-normal\n",
    "    true_positive = sum(y_test_simple[np.where(y_test_simple>0)] == y_predict[np.where(y_test_simple>0)])/y_test[np.where(y_test_simple>0)].shape[0] \n",
    "    results = pd.Series({'model':name,'train_Rsquare':train_Rsquare, 'test_Rsquare':test_Rsquare, \n",
    "                                    'train_MSE':train_MSE,'test_MSE':test_MSE, 'accuracy': accuracy, \n",
    "                                    'accuracy_positive': accuracy_positive, 'true_positive':true_positive, 'duration': duration})\n",
    "    return(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of train (338, 278), shape of test (113, 278)\n"
     ]
    }
   ],
   "source": [
    "X_train1, X_test1, y_train1, y_test1 = train_test_split(X_raw, Y, random_state = 100)\n",
    "\n",
    "print('Shape of train {}, shape of test {}'.format(X_train1.shape, X_test1.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the data set includes both nominal (0,1) and continous variables, we can re-scale the data for quicker conversion."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "MinMax = MinMaxScaler(feature_range= (0,1))\n",
    "X_train1 = MinMax.fit_transform(X_train1)\n",
    "X_test1 = MinMax.transform(X_test1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To calculate true positive (y = 0 means normal, y = 1 means not normal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_simple1 = np.array(y_test1)\n",
    "y_test_simple1[y_test_simple1 == 1] = 0 # normal\n",
    "y_test_simple1[y_test_simple1 >0] = 1 # not-normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### KNN Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         model  train_Rsquare  test_Rsquare  train_MSE  \\\n",
       "0  KNN Classifier All Features            1.0      0.663717        0.0   \n",
       "\n",
       "    test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0  23.460177  0.663717           0.181818       0.227273  96.224313  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "start_time = time.time()\n",
    "param_grid_knn1 = {'weights':['distance', 'uniform'], 'n_neighbors':range(2,100)}\n",
    "grid_search_knn1 = GridSearchCV(KNeighborsClassifier(), param_grid_knn1, cv=5)\n",
    "grid_search_knn1.fit(X_train1, y_train1)\n",
    "duration_knn1 = time.time() - start_time\n",
    "name = 'KNN Classifier All Features'   \n",
    "results = results.append(result(X_train1, y_train1, X_test1, y_test1, y_test_simple1, name, grid_search_knn1,duration_knn1),ignore_index=True )  \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Logistics Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.14497</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              model  train_Rsquare  test_Rsquare  train_MSE  \\\n",
       "0       KNN Classifier All Features       1.000000      0.663717    0.00000   \n",
       "1  Logistic Regression All Features       0.926036      0.743363    6.14497   \n",
       "\n",
       "    test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1  15.221239  0.743363           0.500000       0.727273   3.183996  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "start_time = time.time()\n",
    "param_grid_logit1 = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000] }\n",
    "\n",
    "grid_search_logit1 = GridSearchCV(LogisticRegression(penalty='l2'), param_grid_logit1, cv=5)\n",
    "grid_search_logit1.fit(X_train1, y_train1)\n",
    "\n",
    "duration_logit1 = time.time() - start_time\n",
    "name = 'Logistic Regression All Features'\n",
    "results = results.append(result(X_train1, y_train1, X_test1, y_test1, y_test_simple1, name, grid_search_logit1, duration_logit1),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Linear Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              model  train_Rsquare  test_Rsquare  train_MSE  \\\n",
       "0       KNN Classifier All Features       1.000000      0.663717   0.000000   \n",
       "1  Logistic Regression All Features       0.926036      0.743363   6.144970   \n",
       "2           Linear SVC All Features       0.899408      0.752212   6.887574   \n",
       "\n",
       "    test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2  13.159292  0.752212           0.500000       0.727273  52.810225  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "start_time = time.time()\n",
    "param_grid_svc1 = {'C': [0.001, 0.01, 0.1, 0.5, 1, 10, 50, 100, 1000], 'max_iter':[1000,10000] }\n",
    "\n",
    "grid_search_svc1 = GridSearchCV(LinearSVC(random_state=0), param_grid_svc1, cv=5)\n",
    "grid_search_svc1.fit(X_train1, y_train1)\n",
    "duration_svc1 = time.time() - start_time\n",
    "name = 'Linear SVC All Features'\n",
    "results = results.append(result(X_train1, y_train1, X_test1, y_test1, y_test_simple1, name, grid_search_svc1, duration_svc1),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Kernelized Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     model  train_Rsquare  test_Rsquare  \\\n",
       "0              KNN Classifier All Features       1.000000      0.663717   \n",
       "1         Logistic Regression All Features       0.926036      0.743363   \n",
       "2                  Linear SVC All Features       0.899408      0.752212   \n",
       "3  Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "\n",
       "   train_MSE   test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0   0.000000  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1   6.144970  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2   6.887574  13.159292  0.752212           0.500000       0.727273  52.810225  \n",
       "3   3.612426  18.991150  0.734513           0.500000       0.750000  33.635989  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "start_time = time.time()\n",
    "param_grid_rbf1 = {'C':[0.001, 0.01, 0.1, 0.5, 1, 10, 50, 100, 1000], 'gamma':[0.001, 0.01, 0.1, 0.5, 1, 10]}\n",
    "\n",
    "grid_search_rbf1 = GridSearchCV(SVC(kernel = 'rbf'), param_grid_rbf1, cv=5)\n",
    "grid_search_rbf1.fit(X_train1, y_train1)\n",
    "duration_rbf1 = time.time() - start_time\n",
    "name = 'Linear SVC with RBF Kernel All Features'\n",
    "results = results.append(result(X_train1, y_train1, X_test1, y_test1, y_test_simple1, name, grid_search_rbf1, duration_rbf1),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     model  train_Rsquare  test_Rsquare  \\\n",
       "0              KNN Classifier All Features       1.000000      0.663717   \n",
       "1         Logistic Regression All Features       0.926036      0.743363   \n",
       "2                  Linear SVC All Features       0.899408      0.752212   \n",
       "3  Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4               Decision Tree All Features       0.798817      0.654867   \n",
       "\n",
       "   train_MSE   test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0   0.000000  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1   6.144970  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2   6.887574  13.159292  0.752212           0.500000       0.727273  52.810225  \n",
       "3   3.612426  18.991150  0.734513           0.500000       0.750000  33.635989  \n",
       "4  13.192308  18.601770  0.654867           0.318182       0.659091   1.069025  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "start_time = time.time()\n",
    "param_grid_tree1 = {'max_features':[None,'auto', 'log2'], 'max_depth':[5,10,15,20,50]}\n",
    "\n",
    "grid_search_tree1 = GridSearchCV(DecisionTreeClassifier(random_state = 10), param_grid_tree1, cv=5)\n",
    "grid_search_tree1.fit(X_train1, y_train1)\n",
    "duration_tree1 = time.time() - start_time\n",
    "name = 'Decision Tree All Features'\n",
    "results = results.append(result(X_train1, y_train1, X_test1, y_test1, y_test_simple1, name, grid_search_tree1, duration_tree1),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "###### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=3.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest All Features</td>\n",
       "      <td>0.739645</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>18.002959</td>\n",
       "      <td>18.938053</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.340909</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>51.342942</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     model  train_Rsquare  test_Rsquare  \\\n",
       "0              KNN Classifier All Features       1.000000      0.663717   \n",
       "1         Logistic Regression All Features       0.926036      0.743363   \n",
       "2                  Linear SVC All Features       0.899408      0.752212   \n",
       "3  Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4               Decision Tree All Features       0.798817      0.654867   \n",
       "5               Random Forest All Features       0.739645      0.716814   \n",
       "\n",
       "   train_MSE   test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0   0.000000  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1   6.144970  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2   6.887574  13.159292  0.752212           0.500000       0.727273  52.810225  \n",
       "3   3.612426  18.991150  0.734513           0.500000       0.750000  33.635989  \n",
       "4  13.192308  18.601770  0.654867           0.318182       0.659091   1.069025  \n",
       "5  18.002959  18.938053  0.716814           0.340909       0.454545  51.342942  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint as sp_randint\n",
    "start_time = time.time()\n",
    "#Tuning ridge on new dataset\n",
    "param_grid_forest1 = {\"max_depth\": [3, 5],\n",
    "              \"max_features\": sp_randint(1, 40),\n",
    "              \"min_samples_split\": sp_randint(2, 30),\n",
    "              \"min_samples_leaf\": sp_randint(1, 20),\n",
    "              \"bootstrap\": [True, False]}\n",
    "random_search_forest1 = RandomizedSearchCV(RandomForestClassifier(n_estimators=1000), param_distributions=param_grid_forest1,\n",
    "                                   n_iter=20, random_state=0,n_jobs=-1)\n",
    "random_search_forest1.fit(X_train1, y_train1)\n",
    "duration_forest1 = time.time() - start_time\n",
    "name = 'Random Forest All Features'\n",
    "results = results.append(result(X_train1, y_train1, X_test1, y_test1, y_test_simple1, name, random_search_forest1, duration_forest1),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Feature Selection using Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the data set includes both nominal (0,1) and continous variables, we need to re-scale the data before running random forest for feature selection. The reason is that random forest tends to give higher weights to variables with bigger range."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "MinMax = MinMaxScaler(feature_range= (0,1))\n",
    "X_scaled = pd.DataFrame(MinMax.fit_transform(X_raw))\n",
    "X_scaled.columns = X_raw.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 2 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=-1,\n",
       "            oob_score=False, random_state=10, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'max_features': ['auto', 'log2'], 'max_depth': [2, 5, 10, 15, 20, 50]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid = {'max_features':['auto', 'log2'], 'max_depth':[2,5,10,15,20,50]}\n",
    "rf_reg = RandomForestClassifier(n_estimators= 500, n_jobs = -1, random_state = 10)\n",
    "grid_search = GridSearchCV(rf_reg, param_grid, cv=5)\n",
    "grid_search.fit(X_scaled, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = grid_search.best_estimator_.fit(X_scaled, Y)\n",
    "r = pd.DataFrame(data = rf.feature_importances_*1000)\n",
    "dictionary = dict(zip(X_scaled.columns, rf.feature_importances_*1000))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select independent variables in the top 15% with highest feature importances => we've reduced from 278 variables to 42 variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(451, 42)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d={k: v for k, v in dictionary.items() if v > np.percentile(r, 85)}\n",
    "\n",
    "columns=[]\n",
    "for key in d.keys():\n",
    "  columns.append(key)\n",
    "X=X_scaled[columns]\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['QRS_duration', 'Q-T_interval', 'Heart_Rate', 'AVF_Q_wave', 'V1_S_wave',\n",
       "       'V1_R'_wave', 'V1_Intrinsic_def', 'V3_R_wave', 'V3_S_wave',\n",
       "       'V3_Intrinsic_def', 'DI_Amp_T', 'DI_QRSTA', 'DII_Amp_T', 'DII_QRSTA',\n",
       "       'AVR_Amp_T', 'AVR_QRSTA', 'AVL_Amp_T', 'AVF_Amp_Q', 'AVF_Amp_T',\n",
       "       'V1_Amp_R'', 'V1_Amp_P', 'V1_QRSA', 'V2_Amp_JJ', 'V2_Amp_S',\n",
       "       'V2_Amp_R'', 'V2_Amp_T', 'V2_QRSA', 'V3_Amp_JJ', 'V3_Amp_Q', 'V3_Amp_R',\n",
       "       'V3_Amp_S', 'V3_Amp_T', 'V3_QRSA', 'V3_QRSTA', 'V4_Amp_JJ', 'V4_Amp_T',\n",
       "       'V4_QRSTA', 'V5_Amp_JJ', 'V5_Amp_T', 'V5_QRSTA', 'V6_Amp_T',\n",
       "       'V6_QRSTA'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Split data into train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of train (338, 42), shape of test (113, 42)\n"
     ]
    }
   ],
   "source": [
    "X_train2, X_test2, y_train2, y_test2 = train_test_split(X, Y, random_state = 100)\n",
    "print('Shape of train {}, shape of test {}'.format(X_train2.shape, X_test2.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "MinMax = MinMaxScaler(feature_range= (0,1))\n",
    "X_train2 = MinMax.fit_transform(X_train2)\n",
    "X_test2 = MinMax.transform(X_test2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create a function to calculate the results matrix for each model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To calculate true_positive:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_simple2 = np.array(y_test2)\n",
    "y_test_simple2[y_test_simple2 == 1] = 0 # normal\n",
    "y_test_simple2[y_test_simple2 >0] = 1 # not-normal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### KNN Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest All Features</td>\n",
       "      <td>0.739645</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>18.002959</td>\n",
       "      <td>18.938053</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.340909</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>51.342942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNN Classifier Top 15% Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.752212</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>15.490877</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     model  train_Rsquare  test_Rsquare  \\\n",
       "0              KNN Classifier All Features       1.000000      0.663717   \n",
       "1         Logistic Regression All Features       0.926036      0.743363   \n",
       "2                  Linear SVC All Features       0.899408      0.752212   \n",
       "3  Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4               Decision Tree All Features       0.798817      0.654867   \n",
       "5               Random Forest All Features       0.739645      0.716814   \n",
       "6          KNN Classifier Top 15% Features       1.000000      0.752212   \n",
       "\n",
       "   train_MSE   test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0   0.000000  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1   6.144970  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2   6.887574  13.159292  0.752212           0.500000       0.727273  52.810225  \n",
       "3   3.612426  18.991150  0.734513           0.500000       0.750000  33.635989  \n",
       "4  13.192308  18.601770  0.654867           0.318182       0.659091   1.069025  \n",
       "5  18.002959  18.938053  0.716814           0.340909       0.454545  51.342942  \n",
       "6   0.000000  18.752212  0.752212           0.454545       0.613636  15.490877  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_time_knn2 = time.time()\n",
    "param_grid_knn2 = {'weights':['distance', 'uniform'], 'n_neighbors':range(3,100)}\n",
    "grid_search_knn2 = GridSearchCV(KNeighborsClassifier(), param_grid_knn2, cv=5)\n",
    "grid_search_knn2.fit(X_train2, y_train2)\n",
    "duration_knn2 = time.time() - start_time_knn2\n",
    "name = 'KNN Classifier Top 15% Features'   \n",
    "results = results.append(result(X_train2, y_train2, X_test2, y_test2, y_test_simple2, name, grid_search_knn2,duration_knn2),ignore_index=True )  \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Logistics Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest All Features</td>\n",
       "      <td>0.739645</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>18.002959</td>\n",
       "      <td>18.938053</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.340909</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>51.342942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNN Classifier Top 15% Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.752212</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>15.490877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Logistic Regression Top 15% Features</td>\n",
       "      <td>0.760355</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>15.757396</td>\n",
       "      <td>18.566372</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>0.715414</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     model  train_Rsquare  test_Rsquare  \\\n",
       "0              KNN Classifier All Features       1.000000      0.663717   \n",
       "1         Logistic Regression All Features       0.926036      0.743363   \n",
       "2                  Linear SVC All Features       0.899408      0.752212   \n",
       "3  Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4               Decision Tree All Features       0.798817      0.654867   \n",
       "5               Random Forest All Features       0.739645      0.716814   \n",
       "6          KNN Classifier Top 15% Features       1.000000      0.752212   \n",
       "7     Logistic Regression Top 15% Features       0.760355      0.752212   \n",
       "\n",
       "   train_MSE   test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0   0.000000  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1   6.144970  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2   6.887574  13.159292  0.752212           0.500000       0.727273  52.810225  \n",
       "3   3.612426  18.991150  0.734513           0.500000       0.750000  33.635989  \n",
       "4  13.192308  18.601770  0.654867           0.318182       0.659091   1.069025  \n",
       "5  18.002959  18.938053  0.716814           0.340909       0.454545  51.342942  \n",
       "6   0.000000  18.752212  0.752212           0.454545       0.613636  15.490877  \n",
       "7  15.757396  18.566372  0.752212           0.454545       0.613636   0.715414  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_time_logit2 = time.time()\n",
    "param_grid_logit2 = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000] }\n",
    "\n",
    "grid_search_logit2 = GridSearchCV(LogisticRegression(penalty='l2'), param_grid_logit2, cv=5)\n",
    "grid_search_logit2.fit(X_train2, y_train2)\n",
    "\n",
    "duration_logit2 = time.time() - start_time_logit2\n",
    "name = 'Logistic Regression Top 15% Features'\n",
    "results = results.append(result(X_train2, y_train2, X_test2, y_test2, y_test_simple2, name, grid_search_logit2, duration_logit2),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Linear Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest All Features</td>\n",
       "      <td>0.739645</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>18.002959</td>\n",
       "      <td>18.938053</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.340909</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>51.342942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNN Classifier Top 15% Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.752212</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>15.490877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Logistic Regression Top 15% Features</td>\n",
       "      <td>0.760355</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>15.757396</td>\n",
       "      <td>18.566372</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>0.715414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Linear SVC Top 15% Features</td>\n",
       "      <td>0.813609</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>11.292899</td>\n",
       "      <td>13.840708</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>20.257206</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     model  train_Rsquare  test_Rsquare  \\\n",
       "0              KNN Classifier All Features       1.000000      0.663717   \n",
       "1         Logistic Regression All Features       0.926036      0.743363   \n",
       "2                  Linear SVC All Features       0.899408      0.752212   \n",
       "3  Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4               Decision Tree All Features       0.798817      0.654867   \n",
       "5               Random Forest All Features       0.739645      0.716814   \n",
       "6          KNN Classifier Top 15% Features       1.000000      0.752212   \n",
       "7     Logistic Regression Top 15% Features       0.760355      0.752212   \n",
       "8              Linear SVC Top 15% Features       0.813609      0.752212   \n",
       "\n",
       "   train_MSE   test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0   0.000000  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1   6.144970  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2   6.887574  13.159292  0.752212           0.500000       0.727273  52.810225  \n",
       "3   3.612426  18.991150  0.734513           0.500000       0.750000  33.635989  \n",
       "4  13.192308  18.601770  0.654867           0.318182       0.659091   1.069025  \n",
       "5  18.002959  18.938053  0.716814           0.340909       0.454545  51.342942  \n",
       "6   0.000000  18.752212  0.752212           0.454545       0.613636  15.490877  \n",
       "7  15.757396  18.566372  0.752212           0.454545       0.613636   0.715414  \n",
       "8  11.292899  13.840708  0.752212           0.545455       0.795455  20.257206  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_time_svc2 = time.time()\n",
    "param_grid_svc2 = {'C': [0.001, 0.01, 0.1, 0.5, 1, 10, 50, 100, 1000], 'max_iter':[1000,10000] }\n",
    "\n",
    "grid_search_svc2 = GridSearchCV(LinearSVC(random_state=0), param_grid_svc2, cv=5)\n",
    "grid_search_svc2.fit(X_train2, y_train2)\n",
    "duration_svc2 = time.time() - start_time_svc2\n",
    "name = 'Linear SVC Top 15% Features'\n",
    "results = results.append(result(X_train2, y_train2, X_test2, y_test2, y_test_simple2, name, grid_search_svc2, duration_svc2),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Kernelized Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest All Features</td>\n",
       "      <td>0.739645</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>18.002959</td>\n",
       "      <td>18.938053</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.340909</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>51.342942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNN Classifier Top 15% Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.752212</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>15.490877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Logistic Regression Top 15% Features</td>\n",
       "      <td>0.760355</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>15.757396</td>\n",
       "      <td>18.566372</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>0.715414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Linear SVC Top 15% Features</td>\n",
       "      <td>0.813609</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>11.292899</td>\n",
       "      <td>13.840708</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>20.257206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Linear SVC with RBF Kernel Top 15% Features</td>\n",
       "      <td>0.914201</td>\n",
       "      <td>0.823009</td>\n",
       "      <td>7.505917</td>\n",
       "      <td>16.176991</td>\n",
       "      <td>0.823009</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>6.158037</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         model  train_Rsquare  test_Rsquare  \\\n",
       "0                  KNN Classifier All Features       1.000000      0.663717   \n",
       "1             Logistic Regression All Features       0.926036      0.743363   \n",
       "2                      Linear SVC All Features       0.899408      0.752212   \n",
       "3      Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4                   Decision Tree All Features       0.798817      0.654867   \n",
       "5                   Random Forest All Features       0.739645      0.716814   \n",
       "6              KNN Classifier Top 15% Features       1.000000      0.752212   \n",
       "7         Logistic Regression Top 15% Features       0.760355      0.752212   \n",
       "8                  Linear SVC Top 15% Features       0.813609      0.752212   \n",
       "9  Linear SVC with RBF Kernel Top 15% Features       0.914201      0.823009   \n",
       "\n",
       "   train_MSE   test_MSE  accuracy  accuracy_positive  true_positive   duration  \n",
       "0   0.000000  23.460177  0.663717           0.181818       0.227273  96.224313  \n",
       "1   6.144970  15.221239  0.743363           0.500000       0.727273   3.183996  \n",
       "2   6.887574  13.159292  0.752212           0.500000       0.727273  52.810225  \n",
       "3   3.612426  18.991150  0.734513           0.500000       0.750000  33.635989  \n",
       "4  13.192308  18.601770  0.654867           0.318182       0.659091   1.069025  \n",
       "5  18.002959  18.938053  0.716814           0.340909       0.454545  51.342942  \n",
       "6   0.000000  18.752212  0.752212           0.454545       0.613636  15.490877  \n",
       "7  15.757396  18.566372  0.752212           0.454545       0.613636   0.715414  \n",
       "8  11.292899  13.840708  0.752212           0.545455       0.795455  20.257206  \n",
       "9   7.505917  16.176991  0.823009           0.636364       0.795455   6.158037  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_time_rbf2 = time.time()\n",
    "param_grid_rbf2 = {'C':[0.001, 0.01, 0.1, 0.5, 1, 10, 50, 100, 1000], 'gamma':[0.001, 0.01, 0.1, 0.5, 1, 10]}\n",
    "\n",
    "grid_search_rbf2 = GridSearchCV(SVC(kernel = 'rbf'), param_grid_rbf2, cv=5)\n",
    "grid_search_rbf2.fit(X_train2, y_train2)\n",
    "duration_rbf2 = time.time() - start_time_rbf2\n",
    "name = 'Linear SVC with RBF Kernel Top 15% Features'\n",
    "results = results.append(result(X_train2, y_train2, X_test2, y_test2, y_test_simple2, name, grid_search_rbf2, duration_rbf2),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=5.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest All Features</td>\n",
       "      <td>0.739645</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>18.002959</td>\n",
       "      <td>18.938053</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.340909</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>51.342942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNN Classifier Top 15% Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.752212</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>15.490877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Logistic Regression Top 15% Features</td>\n",
       "      <td>0.760355</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>15.757396</td>\n",
       "      <td>18.566372</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>0.715414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Linear SVC Top 15% Features</td>\n",
       "      <td>0.813609</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>11.292899</td>\n",
       "      <td>13.840708</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>20.257206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Linear SVC with RBF Kernel Top 15% Features</td>\n",
       "      <td>0.914201</td>\n",
       "      <td>0.823009</td>\n",
       "      <td>7.505917</td>\n",
       "      <td>16.176991</td>\n",
       "      <td>0.823009</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>6.158037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Decision Tree Top 15% Features</td>\n",
       "      <td>0.979290</td>\n",
       "      <td>0.672566</td>\n",
       "      <td>0.843195</td>\n",
       "      <td>18.530973</td>\n",
       "      <td>0.672566</td>\n",
       "      <td>0.431818</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>0.462417</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          model  train_Rsquare  test_Rsquare  \\\n",
       "0                   KNN Classifier All Features       1.000000      0.663717   \n",
       "1              Logistic Regression All Features       0.926036      0.743363   \n",
       "2                       Linear SVC All Features       0.899408      0.752212   \n",
       "3       Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4                    Decision Tree All Features       0.798817      0.654867   \n",
       "5                    Random Forest All Features       0.739645      0.716814   \n",
       "6               KNN Classifier Top 15% Features       1.000000      0.752212   \n",
       "7          Logistic Regression Top 15% Features       0.760355      0.752212   \n",
       "8                   Linear SVC Top 15% Features       0.813609      0.752212   \n",
       "9   Linear SVC with RBF Kernel Top 15% Features       0.914201      0.823009   \n",
       "10               Decision Tree Top 15% Features       0.979290      0.672566   \n",
       "\n",
       "    train_MSE   test_MSE  accuracy  accuracy_positive  true_positive  \\\n",
       "0    0.000000  23.460177  0.663717           0.181818       0.227273   \n",
       "1    6.144970  15.221239  0.743363           0.500000       0.727273   \n",
       "2    6.887574  13.159292  0.752212           0.500000       0.727273   \n",
       "3    3.612426  18.991150  0.734513           0.500000       0.750000   \n",
       "4   13.192308  18.601770  0.654867           0.318182       0.659091   \n",
       "5   18.002959  18.938053  0.716814           0.340909       0.454545   \n",
       "6    0.000000  18.752212  0.752212           0.454545       0.613636   \n",
       "7   15.757396  18.566372  0.752212           0.454545       0.613636   \n",
       "8   11.292899  13.840708  0.752212           0.545455       0.795455   \n",
       "9    7.505917  16.176991  0.823009           0.636364       0.795455   \n",
       "10   0.843195  18.530973  0.672566           0.431818       0.818182   \n",
       "\n",
       "     duration  \n",
       "0   96.224313  \n",
       "1    3.183996  \n",
       "2   52.810225  \n",
       "3   33.635989  \n",
       "4    1.069025  \n",
       "5   51.342942  \n",
       "6   15.490877  \n",
       "7    0.715414  \n",
       "8   20.257206  \n",
       "9    6.158037  \n",
       "10   0.462417  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_time_tree2 = time.time()\n",
    "param_grid_tree2 = {'max_features':[None,'auto', 'log2'], 'max_depth':[5,10,15,20,50]}\n",
    "\n",
    "grid_search_tree2 = GridSearchCV(DecisionTreeClassifier(random_state = 10), param_grid_tree2, cv=5)\n",
    "grid_search_tree2.fit(X_train2, y_train2)\n",
    "duration_tree2 = time.time() - start_time_tree2\n",
    "name = 'Decision Tree Top 15% Features'\n",
    "results = results.append(result(X_train2, y_train2, X_test2, y_test2, y_test_simple2, name, grid_search_tree2, duration_tree2),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thaoduong252\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:605: Warning: The least populated class in y has only 1 members, which is too few. The minimum number of members in any class cannot be less than n_splits=3.\n",
      "  % (min_groups, self.n_splits)), Warning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>train_Rsquare</th>\n",
       "      <th>test_Rsquare</th>\n",
       "      <th>train_MSE</th>\n",
       "      <th>test_MSE</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>accuracy_positive</th>\n",
       "      <th>true_positive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KNN Classifier All Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.460177</td>\n",
       "      <td>0.663717</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>96.224313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Logistic Regression All Features</td>\n",
       "      <td>0.926036</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>6.144970</td>\n",
       "      <td>15.221239</td>\n",
       "      <td>0.743363</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>3.183996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Linear SVC All Features</td>\n",
       "      <td>0.899408</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>6.887574</td>\n",
       "      <td>13.159292</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>52.810225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Linear SVC with RBF Kernel All Features</td>\n",
       "      <td>0.967456</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>3.612426</td>\n",
       "      <td>18.991150</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>33.635989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Decision Tree All Features</td>\n",
       "      <td>0.798817</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>13.192308</td>\n",
       "      <td>18.601770</td>\n",
       "      <td>0.654867</td>\n",
       "      <td>0.318182</td>\n",
       "      <td>0.659091</td>\n",
       "      <td>1.069025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Random Forest All Features</td>\n",
       "      <td>0.739645</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>18.002959</td>\n",
       "      <td>18.938053</td>\n",
       "      <td>0.716814</td>\n",
       "      <td>0.340909</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>51.342942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>KNN Classifier Top 15% Features</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18.752212</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>15.490877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Logistic Regression Top 15% Features</td>\n",
       "      <td>0.760355</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>15.757396</td>\n",
       "      <td>18.566372</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.454545</td>\n",
       "      <td>0.613636</td>\n",
       "      <td>0.715414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Linear SVC Top 15% Features</td>\n",
       "      <td>0.813609</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>11.292899</td>\n",
       "      <td>13.840708</td>\n",
       "      <td>0.752212</td>\n",
       "      <td>0.545455</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>20.257206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Linear SVC with RBF Kernel Top 15% Features</td>\n",
       "      <td>0.914201</td>\n",
       "      <td>0.823009</td>\n",
       "      <td>7.505917</td>\n",
       "      <td>16.176991</td>\n",
       "      <td>0.823009</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>6.158037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Decision Tree Top 15% Features</td>\n",
       "      <td>0.979290</td>\n",
       "      <td>0.672566</td>\n",
       "      <td>0.843195</td>\n",
       "      <td>18.530973</td>\n",
       "      <td>0.672566</td>\n",
       "      <td>0.431818</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>0.462417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Random Forest Top 15% Features</td>\n",
       "      <td>0.772189</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>17.133136</td>\n",
       "      <td>17.477876</td>\n",
       "      <td>0.734513</td>\n",
       "      <td>0.386364</td>\n",
       "      <td>0.568182</td>\n",
       "      <td>57.157247</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          model  train_Rsquare  test_Rsquare  \\\n",
       "0                   KNN Classifier All Features       1.000000      0.663717   \n",
       "1              Logistic Regression All Features       0.926036      0.743363   \n",
       "2                       Linear SVC All Features       0.899408      0.752212   \n",
       "3       Linear SVC with RBF Kernel All Features       0.967456      0.734513   \n",
       "4                    Decision Tree All Features       0.798817      0.654867   \n",
       "5                    Random Forest All Features       0.739645      0.716814   \n",
       "6               KNN Classifier Top 15% Features       1.000000      0.752212   \n",
       "7          Logistic Regression Top 15% Features       0.760355      0.752212   \n",
       "8                   Linear SVC Top 15% Features       0.813609      0.752212   \n",
       "9   Linear SVC with RBF Kernel Top 15% Features       0.914201      0.823009   \n",
       "10               Decision Tree Top 15% Features       0.979290      0.672566   \n",
       "11               Random Forest Top 15% Features       0.772189      0.734513   \n",
       "\n",
       "    train_MSE   test_MSE  accuracy  accuracy_positive  true_positive  \\\n",
       "0    0.000000  23.460177  0.663717           0.181818       0.227273   \n",
       "1    6.144970  15.221239  0.743363           0.500000       0.727273   \n",
       "2    6.887574  13.159292  0.752212           0.500000       0.727273   \n",
       "3    3.612426  18.991150  0.734513           0.500000       0.750000   \n",
       "4   13.192308  18.601770  0.654867           0.318182       0.659091   \n",
       "5   18.002959  18.938053  0.716814           0.340909       0.454545   \n",
       "6    0.000000  18.752212  0.752212           0.454545       0.613636   \n",
       "7   15.757396  18.566372  0.752212           0.454545       0.613636   \n",
       "8   11.292899  13.840708  0.752212           0.545455       0.795455   \n",
       "9    7.505917  16.176991  0.823009           0.636364       0.795455   \n",
       "10   0.843195  18.530973  0.672566           0.431818       0.818182   \n",
       "11  17.133136  17.477876  0.734513           0.386364       0.568182   \n",
       "\n",
       "     duration  \n",
       "0   96.224313  \n",
       "1    3.183996  \n",
       "2   52.810225  \n",
       "3   33.635989  \n",
       "4    1.069025  \n",
       "5   51.342942  \n",
       "6   15.490877  \n",
       "7    0.715414  \n",
       "8   20.257206  \n",
       "9    6.158037  \n",
       "10   0.462417  \n",
       "11  57.157247  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_time_forest2 = time.time()\n",
    "#Tuning ridge on new dataset\n",
    "param_grid_forest2 = {\"max_depth\": [3, 5],\n",
    "              \"max_features\": sp_randint(1, 40),\n",
    "              \"min_samples_split\": sp_randint(2, 30),\n",
    "              \"min_samples_leaf\": sp_randint(1, 20),\n",
    "              \"bootstrap\": [True, False]}\n",
    "random_search_forest2 = RandomizedSearchCV(RandomForestClassifier(n_estimators=1000), param_distributions=param_grid_forest2,\n",
    "                                   n_iter=20, random_state=0,n_jobs=-1)\n",
    "random_search_forest2.fit(X_train2, y_train2)\n",
    "duration_forest2 = time.time() - start_time_forest2\n",
    "name = 'Random Forest Top 15% Features'\n",
    "results = results.append(result(X_train2, y_train2, X_test2, y_test2, y_test_simple2, name, random_search_forest2, duration_forest2),ignore_index=True ) \n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion\n",
    "1. Features Selection helps reduce running time\n",
    "2. Models with Feature Selection perform better than those with all features\n",
    "3. Decision Tree Top 15% Features has highest true_positive and takes least amount of time to run. SVC with RBF Kernel Top 15% Features has highest accuracy and accuracy positive.\n",
    "=> If you want to have a good diagnosis whethere one is normal or has ANY of the cardiac arrhythmia, you should use Decision Tree.\n",
    "=> If you want to have the best diagnosis of one's cardiac arrhythmia case, you should use Linear SVC with RBF Kernel."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}