Adv_Classification_Models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Dataset For Classification Task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dataset_2 = pd.read_csv('Twitter-Absolute-Sigma-500.data', header = None)   #Reading Data For classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 140707 entries, 0 to 140706\n",
      "Data columns (total 78 columns):\n",
      "0     140707 non-null int64\n",
      "1     140707 non-null int64\n",
      "2     140707 non-null int64\n",
      "3     140707 non-null int64\n",
      "4     140707 non-null int64\n",
      "5     140707 non-null int64\n",
      "6     140707 non-null int64\n",
      "7     140707 non-null int64\n",
      "8     140707 non-null int64\n",
      "9     140707 non-null int64\n",
      "10    140707 non-null int64\n",
      "11    140707 non-null int64\n",
      "12    140707 non-null int64\n",
      "13    140707 non-null int64\n",
      "14    140707 non-null float64\n",
      "15    140707 non-null float64\n",
      "16    140707 non-null float64\n",
      "17    140707 non-null float64\n",
      "18    140707 non-null float64\n",
      "19    140707 non-null float64\n",
      "20    140707 non-null float64\n",
      "21    140707 non-null float64\n",
      "22    140707 non-null float64\n",
      "23    140707 non-null float64\n",
      "24    140707 non-null float64\n",
      "25    140707 non-null float64\n",
      "26    140707 non-null float64\n",
      "27    140707 non-null float64\n",
      "28    140707 non-null int64\n",
      "29    140707 non-null int64\n",
      "30    140707 non-null int64\n",
      "31    140707 non-null int64\n",
      "32    140707 non-null int64\n",
      "33    140707 non-null int64\n",
      "34    140707 non-null int64\n",
      "35    140707 non-null float64\n",
      "36    140707 non-null float64\n",
      "37    140707 non-null float64\n",
      "38    140707 non-null float64\n",
      "39    140707 non-null float64\n",
      "40    140707 non-null float64\n",
      "41    140707 non-null float64\n",
      "42    140707 non-null float64\n",
      "43    140707 non-null float64\n",
      "44    140707 non-null float64\n",
      "45    140707 non-null float64\n",
      "46    140707 non-null float64\n",
      "47    140707 non-null float64\n",
      "48    140707 non-null float64\n",
      "49    140707 non-null float64\n",
      "50    140707 non-null float64\n",
      "51    140707 non-null float64\n",
      "52    140707 non-null float64\n",
      "53    140707 non-null float64\n",
      "54    140707 non-null float64\n",
      "55    140707 non-null float64\n",
      "56    140707 non-null int64\n",
      "57    140707 non-null int64\n",
      "58    140707 non-null int64\n",
      "59    140707 non-null int64\n",
      "60    140707 non-null int64\n",
      "61    140707 non-null int64\n",
      "62    140707 non-null int64\n",
      "63    140707 non-null float64\n",
      "64    140707 non-null float64\n",
      "65    140707 non-null float64\n",
      "66    140707 non-null float64\n",
      "67    140707 non-null float64\n",
      "68    140707 non-null float64\n",
      "69    140707 non-null float64\n",
      "70    140707 non-null int64\n",
      "71    140707 non-null int64\n",
      "72    140707 non-null int64\n",
      "73    140707 non-null int64\n",
      "74    140707 non-null int64\n",
      "75    140707 non-null int64\n",
      "76    140707 non-null int64\n",
      "77    140707 non-null float64\n",
      "dtypes: float64(43), int64(35)\n",
      "memory usage: 83.7 MB\n"
     ]
    }
   ],
   "source": [
    "Dataset_2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>68</th>\n",
       "      <th>69</th>\n",
       "      <th>70</th>\n",
       "      <th>71</th>\n",
       "      <th>72</th>\n",
       "      <th>73</th>\n",
       "      <th>74</th>\n",
       "      <th>75</th>\n",
       "      <th>76</th>\n",
       "      <th>77</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>889</td>\n",
       "      <td>939</td>\n",
       "      <td>960</td>\n",
       "      <td>805</td>\n",
       "      <td>805</td>\n",
       "      <td>1143</td>\n",
       "      <td>1121</td>\n",
       "      <td>549</td>\n",
       "      <td>613</td>\n",
       "      <td>587</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>889</td>\n",
       "      <td>939</td>\n",
       "      <td>960</td>\n",
       "      <td>805</td>\n",
       "      <td>805</td>\n",
       "      <td>1143</td>\n",
       "      <td>1121</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>542</td>\n",
       "      <td>473</td>\n",
       "      <td>504</td>\n",
       "      <td>626</td>\n",
       "      <td>647</td>\n",
       "      <td>795</td>\n",
       "      <td>832</td>\n",
       "      <td>366</td>\n",
       "      <td>288</td>\n",
       "      <td>318</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>542</td>\n",
       "      <td>473</td>\n",
       "      <td>504</td>\n",
       "      <td>626</td>\n",
       "      <td>647</td>\n",
       "      <td>795</td>\n",
       "      <td>832</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>92</td>\n",
       "      <td>99</td>\n",
       "      <td>196</td>\n",
       "      <td>100</td>\n",
       "      <td>184</td>\n",
       "      <td>79</td>\n",
       "      <td>162</td>\n",
       "      <td>66</td>\n",
       "      <td>59</td>\n",
       "      <td>118</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>92</td>\n",
       "      <td>99</td>\n",
       "      <td>196</td>\n",
       "      <td>100</td>\n",
       "      <td>184</td>\n",
       "      <td>79</td>\n",
       "      <td>162</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>90</td>\n",
       "      <td>87</td>\n",
       "      <td>92</td>\n",
       "      <td>344</td>\n",
       "      <td>184</td>\n",
       "      <td>848</td>\n",
       "      <td>184</td>\n",
       "      <td>83</td>\n",
       "      <td>78</td>\n",
       "      <td>76</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>90</td>\n",
       "      <td>87</td>\n",
       "      <td>92</td>\n",
       "      <td>344</td>\n",
       "      <td>184</td>\n",
       "      <td>848</td>\n",
       "      <td>184</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>169</td>\n",
       "      <td>98</td>\n",
       "      <td>101</td>\n",
       "      <td>90</td>\n",
       "      <td>96</td>\n",
       "      <td>95</td>\n",
       "      <td>185</td>\n",
       "      <td>141</td>\n",
       "      <td>68</td>\n",
       "      <td>85</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>169</td>\n",
       "      <td>98</td>\n",
       "      <td>101</td>\n",
       "      <td>90</td>\n",
       "      <td>96</td>\n",
       "      <td>95</td>\n",
       "      <td>185</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0    1    2    3    4     5     6    7    8    9  ...    68   69   70  \\\n",
       "0  889  939  960  805  805  1143  1121  549  613  587 ...   1.0  1.0  889   \n",
       "1  542  473  504  626  647   795   832  366  288  318 ...   1.0  1.0  542   \n",
       "2   92   99  196  100  184    79   162   66   59  118 ...   1.0  1.0   92   \n",
       "3   90   87   92  344  184   848   184   83   78   76 ...   1.0  1.0   90   \n",
       "4  169   98  101   90   96    95   185  141   68   85 ...   1.0  1.0  169   \n",
       "\n",
       "    71   72   73   74    75    76   77  \n",
       "0  939  960  805  805  1143  1121  1.0  \n",
       "1  473  504  626  647   795   832  1.0  \n",
       "2   99  196  100  184    79   162  0.0  \n",
       "3   87   92  344  184   848   184  1.0  \n",
       "4   98  101   90   96    95   185  1.0  \n",
       "\n",
       "[5 rows x 78 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Dataset_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Dataset.columns\n",
    "Dataset_2[77].unique().astype(int)   #Dependent variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Dataset_2.isnull().sum().sum()    #No Missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>68</th>\n",
       "      <th>69</th>\n",
       "      <th>70</th>\n",
       "      <th>71</th>\n",
       "      <th>72</th>\n",
       "      <th>73</th>\n",
       "      <th>74</th>\n",
       "      <th>75</th>\n",
       "      <th>76</th>\n",
       "      <th>77</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "      <td>140707.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>172.279823</td>\n",
       "      <td>155.150625</td>\n",
       "      <td>165.464476</td>\n",
       "      <td>176.820549</td>\n",
       "      <td>186.937700</td>\n",
       "      <td>216.209208</td>\n",
       "      <td>243.866510</td>\n",
       "      <td>87.050154</td>\n",
       "      <td>78.639236</td>\n",
       "      <td>84.269574</td>\n",
       "      <td>...</td>\n",
       "      <td>1.113444</td>\n",
       "      <td>1.196131</td>\n",
       "      <td>172.838807</td>\n",
       "      <td>155.630878</td>\n",
       "      <td>165.938674</td>\n",
       "      <td>177.314810</td>\n",
       "      <td>187.463794</td>\n",
       "      <td>216.776294</td>\n",
       "      <td>244.479194</td>\n",
       "      <td>0.197396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>509.872276</td>\n",
       "      <td>471.573236</td>\n",
       "      <td>495.360236</td>\n",
       "      <td>528.351277</td>\n",
       "      <td>560.331281</td>\n",
       "      <td>632.188378</td>\n",
       "      <td>707.402192</td>\n",
       "      <td>234.731748</td>\n",
       "      <td>218.448179</td>\n",
       "      <td>233.536510</td>\n",
       "      <td>...</td>\n",
       "      <td>1.374287</td>\n",
       "      <td>1.826150</td>\n",
       "      <td>510.937549</td>\n",
       "      <td>472.462733</td>\n",
       "      <td>496.233557</td>\n",
       "      <td>529.286514</td>\n",
       "      <td>561.309487</td>\n",
       "      <td>633.203935</td>\n",
       "      <td>708.436795</td>\n",
       "      <td>0.398035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>22.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>33.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>33.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>125.000000</td>\n",
       "      <td>112.000000</td>\n",
       "      <td>119.000000</td>\n",
       "      <td>126.000000</td>\n",
       "      <td>133.000000</td>\n",
       "      <td>161.000000</td>\n",
       "      <td>186.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>67.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.100000</td>\n",
       "      <td>1.119048</td>\n",
       "      <td>126.000000</td>\n",
       "      <td>113.000000</td>\n",
       "      <td>119.000000</td>\n",
       "      <td>127.000000</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>162.000000</td>\n",
       "      <td>187.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>24210.000000</td>\n",
       "      <td>22899.000000</td>\n",
       "      <td>20495.000000</td>\n",
       "      <td>27007.000000</td>\n",
       "      <td>30957.000000</td>\n",
       "      <td>28603.000000</td>\n",
       "      <td>37505.000000</td>\n",
       "      <td>15105.000000</td>\n",
       "      <td>15730.000000</td>\n",
       "      <td>16389.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>185.666672</td>\n",
       "      <td>295.000000</td>\n",
       "      <td>24301.000000</td>\n",
       "      <td>22980.000000</td>\n",
       "      <td>20495.000000</td>\n",
       "      <td>27071.000000</td>\n",
       "      <td>31028.000000</td>\n",
       "      <td>28697.000000</td>\n",
       "      <td>37505.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0              1              2              3   \\\n",
       "count  140707.000000  140707.000000  140707.000000  140707.000000   \n",
       "mean      172.279823     155.150625     165.464476     176.820549   \n",
       "std       509.872276     471.573236     495.360236     528.351277   \n",
       "min         0.000000       0.000000       0.000000       0.000000   \n",
       "25%         3.000000       2.000000       3.000000       3.000000   \n",
       "50%        22.000000      19.000000      20.000000      22.000000   \n",
       "75%       125.000000     112.000000     119.000000     126.000000   \n",
       "max     24210.000000   22899.000000   20495.000000   27007.000000   \n",
       "\n",
       "                  4              5              6              7   \\\n",
       "count  140707.000000  140707.000000  140707.000000  140707.000000   \n",
       "mean      186.937700     216.209208     243.866510      87.050154   \n",
       "std       560.331281     632.188378     707.402192     234.731748   \n",
       "min         0.000000       0.000000       0.000000       0.000000   \n",
       "25%         3.000000       4.000000       5.000000       2.000000   \n",
       "50%        23.000000      28.000000      33.000000      13.000000   \n",
       "75%       133.000000     161.000000     186.000000      70.000000   \n",
       "max     30957.000000   28603.000000   37505.000000   15105.000000   \n",
       "\n",
       "                  8              9       ...                   68  \\\n",
       "count  140707.000000  140707.000000      ...        140707.000000   \n",
       "mean       78.639236      84.269574      ...             1.113444   \n",
       "std       218.448179     233.536510      ...             1.374287   \n",
       "min         0.000000       0.000000      ...             0.000000   \n",
       "25%         2.000000       2.000000      ...             1.000000   \n",
       "50%        11.000000      13.000000      ...             1.000000   \n",
       "75%        64.000000      67.000000      ...             1.100000   \n",
       "max     15730.000000   16389.000000      ...           185.666672   \n",
       "\n",
       "                  69             70             71             72  \\\n",
       "count  140707.000000  140707.000000  140707.000000  140707.000000   \n",
       "mean        1.196131     172.838807     155.630878     165.938674   \n",
       "std         1.826150     510.937549     472.462733     496.233557   \n",
       "min         0.000000       0.000000       0.000000       0.000000   \n",
       "25%         1.000000       3.000000       2.000000       3.000000   \n",
       "50%         1.000000      22.000000      19.000000      21.000000   \n",
       "75%         1.119048     126.000000     113.000000     119.000000   \n",
       "max       295.000000   24301.000000   22980.000000   20495.000000   \n",
       "\n",
       "                  73             74             75             76  \\\n",
       "count  140707.000000  140707.000000  140707.000000  140707.000000   \n",
       "mean      177.314810     187.463794     216.776294     244.479194   \n",
       "std       529.286514     561.309487     633.203935     708.436795   \n",
       "min         0.000000       0.000000       0.000000       0.000000   \n",
       "25%         3.000000       3.000000       4.000000       6.000000   \n",
       "50%        22.000000      23.000000      28.000000      33.000000   \n",
       "75%       127.000000     134.000000     162.000000     187.000000   \n",
       "max     27071.000000   31028.000000   28697.000000   37505.000000   \n",
       "\n",
       "                  77  \n",
       "count  140707.000000  \n",
       "mean        0.197396  \n",
       "std         0.398035  \n",
       "min         0.000000  \n",
       "25%         0.000000  \n",
       "50%         0.000000  \n",
       "75%         0.000000  \n",
       "max         1.000000  \n",
       "\n",
       "[8 rows x 78 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Dataset_2.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr = Dataset_2.corr()    #calculate correlation matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x152a14335c0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATcAAAD/CAYAAACU7pFfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXm8VVX1wL8LcMxZc0gsnC3NUBHtlxYOKWmimZZDhZoRaWmWKWS/n1r5y7TSJvtJikOKgoqKhCiKolioCKgYIoPMo8yDgOj6/bH3OXffyznvnnfPu++9e9/6fj7nc/c9ezh7n/fOvmudvdZeoqoYhmHUG+1augOGYRjVwCY3wzDqEpvcDMOoS2xyMwyjLrHJzTCMusQmN8Mw6hKb3AzDqEtyTW4i0l1EJovIVBHp01SdMgzDyItUasQrIu2Bd4AvA3OAV4FzVfU/Tdc9wzCMysgjuXUFpqrqdFXdADwInN403TIMw8hHhxx19wRmB9/nAEeVqWO+XoZRXaSlO9BayCO5Jd3ETSYvEeklImNFZGy/fv1yXM4wDCM7eSS3OcBewfeOwLzSQqraD4hmNZPcDMNoFvJIbq8C+4vI3iKyOXAOMKRcpSnHnJzjkoZhGNmoeLUUQEROAW4F2gP9VfWGMlVMcjOM6mLv3Dy5JrcKsMnNMKqLTW6eZvdQiNTSKcecbCqqYRhVwyQ3w6gvTHLz5JbcRKS9iIwXkaFN0SHDMIymoCnU0suBSVkLJ6mlpqIahtHU5F0t7QjcA9wA/ERVv1qmiqmlhlFdTC315JXcbgWuAj5qgr4YhmE0GRVPbiLyVWCRqr5WplyR+1U5tdTUU8MwmoI8Wx79Bvg2sBHYEtgOGKyq32qgmqmlhlFdTC31VCy5qWpfVe2oqp1wrlcjy0xsQPkFhVByMynOMIxKsW3GDcOoS8yI1zDqC1NLPa3C/SqLWmoqqmEYjcEkN8OoL0xy8+SNfnWFiLwlIhNF5AER2bKpOmYYhpGHPHZuewKXAV1U9RDcnm7nlKuXVy01Vy3DMLKQx85tT2AM8DlgJfAY8CdVfbqBaqaWGkZ1MbXUk8fObS7wO2AWMB9YUWZiMwzDaDbyqKU74uKU7g18AviYiGxixJvH/aqcWmoqqmEYaeRRS88Guqvqd/337wBHq+olDVQztdQwqouppZ48q6WzgKNFZGsREeAEMuzrVi3JzSQ4wzBC8rxzexl4GBgHvOnbsqjLhmG0CsyI1zDqC1NLPTXnfmW7iRiGkQXbFcQwjLrE1FLDqC9MLfWUldxEpL+ILBKRicG5m0XkbRF5Q0QeFZEdsl6wudVSU1ENo21SVnITkS8Cq4F7vQ8pInISbufdjSLyWwBVvTrD9UxyM4zqYpKbp6zkpqovAEtLzj2tqhv91zFAxyr0zTAMo2KaYkHhIuDJtMzmcr+y3UQMwwjJtKAgIp2AoZFaGpy/BugCnKnZViZMLTWM6mJqqadDpRVFpCfwVeCEjBObYRhGs1GRWioi3YGrgR6qurYxdVuDWmrqqWHUP1lWSx8AugG7AAuBa4G+wBbAEl9sjKr2znA9k/AMo7qYWurJslp6rqruoaqb+SDMd6rqfqq6l6p29keWiQ1ofZKbSXGGUZ+Y+5VhGHWJuV8ZRn1haqmnIvcrf/5HIjLZh/a7KesFW7NaaiqqYdQPlbpfHQdcA5yqqutFZFdVXZTheia5GUZ1McnNU5H7FfAD4EZVXe/LZJnYDMMwmo1KFxQOAI4VkZdFZJSIHJm1Yq2opaaiGkZtU5H7lX//NhK4HDgSGAjsk+SpICK9gF4At99++xG9evVqqr4bhrEpppZ6KnW/mgMM9pPZKyLyEc7Id3FpQVXtRyFwjL1zMwyjWahULX0MOB5ARA4ANgfey1Kx1tRS203EMGqTSt2v/gH0BzoDG4ArVXVkhuuZ5GYY1cXUUo8Z8RpGfWGTm6fuQ/s1pVpq6qlh1A4muRlGfWGSmyeL+9VeIvKciEzyrlaX+/M7icgIEZniP3fMcsF6kdxMijOM1k0WU5CNwE9VdZyIbAu8JiIjgAuAZ1X1RhHpA/TBbWBpGEYNM+WYkxM1rP1HP1VTUmGj1VIReRz4iz+6qep8EdkDeF5VDyxT3dRSw6guuSegKcd+JXlye/HJmprcGrWg4D0VDgNeBnZT1fkA/nPXLG3Uo1pqKqpRV4gkHzVGZslNRLYBRgE3qOpgEVmuqjsE+ctUdZP3buZ+ZRjNSu5ZaOpxpyVOCvs990RNzXCZ3K9EZDPgEeB+VR3sTy8UkT0CtTRxZxBzvzKMGqMGpbQksqyWCnAnMElV/xBkDQF6+nRP4PEsF1wx+An3OWQYK4YMc+nHhrLisaFxOi4bpqN6g5+I08seeIRlDzwCwPKHCpdfPujRTdLLBz1aSA8cXMhPS/v2iuqF7QZloz4su38Qy+4fVP4mGEYrRtq3SzxqjSzuV8cALwJvAh/50z/HvXcbBHwSmAWcraql+76VYpKbYVSX3GLXtO5fT3xO9x3+SE2JdGXVUlUdTfoNO6Fpu2MYRovTVtTSpmb1yBcAWPHEcFY8MTw+F52PPgFWjXiukH56ZPwZpVcOf5aVw58FYPkjQzYpG5UBWPnkCFY+OcKlhz1dyA/S4fUi1XfViOfi80nthm2EYxr51lRGvjW1zN0wjNaHtGuXeNQa5n5lGPVFbrFreo9zE5/TfYY8UFMiXcXuV0H+lSKiIrJLlgvGL+ofGRJLW+GL+KKFgYRFguUDB8cv85cNeIhlAx5yaf9SH2Bp//vidPyyPyi79B8DC/nBAkDYRnS9pf3vi9uL6pfWi9pbevcAlt49ADCHe6N2qZcFhYrdr1T1PyKyF/Bl3IKCYRj1gNTeRJZExe5XqjpCRB4GfoUzA+miquV24zW11DCqS27VccbZPROf004P3dNg2yLSHfgj0B64Q1VvLMn/CXAxTmBaDFykqjOD/O2AScCjqvrDXIMgh/uViPQA5qrq641po5ydW2g/Ftq5RSpsqM6GKmqanVukMob2aqH6maSKFl0vqLf0ngcK+WXs3Gw3EaNmkXbJR0NVRNoDfwW+AnwGOFdEPlNSbDxOCDoUeBgoDeb+K5wXVJOQeXLz7lePAD/GzbzXAP+ToV4vERkrImP79etXrrhhGC2MdGifeJShKzBVVaer6gbgQeD0sICqPqeqa/3XMUDH+JoiRwC7AU/TRGQN7bcZMBR4SlX/ICKfBZ4Foo52BOYBXVV1QQNNmVpqGNUlt1o681u9Ep/TT93XL7VtETkL6K6qF/vv3waOSlMvReQvwAJV/bWItMOFCv02zna2S7OopUnuV6r6pqruqqqdVLUTLtTf4WUmNqCgahappSXpiFBNjOsF7ldFrlFpq6w+XbQiG7pZJbhqQYoaHLRbtOJaoVpqKqrRKmnXLvEItTB/hLtgJE18iZOkiHwL6ALc7E9dAgxT1dlNOYyK3a9UdVhQZga2oGAYrYHcktusCy9JfE4/eddtDUlunweuU9WT/fe+AKr6m5JyJwJ/Br6kqov8ufuBY3Hzyza4UKG3qWqfPOPI634VlemUpxOGYbQiKjMFeRXYX0T2BuYC5wDnFTUrchhwO059jXcRUtXzgzIX4ASlXBMbtID7VezK9OwoVj07Kj4Xnw9droJ05O4Ull057OnY9anIHSohHbpfRS5S0fk4HbplJbltpZSN2gvdr6Ye34Opx/dw6W6nxmUt8LPR2qnEiFdVNwI/BJ7CmXMMUtW3ROSX3rICnBq6DfCQiEwQkSEpzTXNOMz9yjDqitxq6ezeVyQ+p3v93y1tw/1KRDqLyBg/A48Vka7V765hGNWmXhzns/Q4cr/6NHA0cKk3zrsJuF5VO+Ps3UoN8hJZ8+9X3OeYsawZMzY+F56Py4bpKP+ll1nz0ssArH7xX6x+8V/u/OgxhbIJ6TWjxxSlI1aPeqmiekVt+D6Eu5uEqmbkbwrFKuqcS68EYF6f65jX5zoAZixZjmG0KG0thkJcoRD96kqgv6oOFJFzgdNU9byGa5taahhVJvcsNOeyqxOf045/+m1NzXB5ol/9GLhZRGYDvwP6ZmmjnqNfhWVD+7hwj7oiGzyfH9ruhXvGRdKqYTQnbUktBYrdr1R1JfAD4ApV3Qu4Amfom1TP3K8Mo5ZoJ8lHjVGR+5U/twLYQVXVezGsUNXtyjRlaqlhVJfcs9C8n/134nP6iZt/VVMzXJ7oV/OAL/n08cCULBdsK2ppUUSvwK4u3A49yRUttO0L1VKzgzOajRT3q1ojT/Srlbi9mzoA64BLVPW1Mtczyc0wqktu6Wp+3+sTn9M9fnNtTUlued2vjmja7hiG0eK0L7u9UU3Q7LJmW1RLw9XScDeReOPOoGwWtdRUVKOqtFU7t5yYWmoY1SX3LLTguhsTn9Pdr+tTUzNclgWFLUXkFRF53btfXe/P3y8ik0Vkooj09yuqhmHUOnViCpJFLV0PHK+qnwM6A91F5GjgfuAg4LPAVrjAD2Vp62ppaMQbbXJZqVpqu4kY1UDat088ao1GqaUisjUwGviBqr4cnL8C2EVVrynThKmlhlFdcotYC2+8JfE53a3PFXmjX30RuBU4FDhHVR8O8j4J3AHshZsnTlHVGTmGkW1BQUTai8gEYBEwomRi2wy39/nwtPqGYdQOIpJ4lKmTJfrVLOACYACbci9ws9+goyturslFpslNVT/0u390BLqKyCFB9m3AC6r6YlLdUvertqiWhqpmUVyIhFgQlaqlpp4aTUYFof3IFv1qhqq+QcFe1l3OTYIdVHWEL7c6iJJV+TAq2BXkWmCNqv7Opw8DzlTVj8pUBVNLDaPa5FZLF/3+L4nP6a4//WGTRL8SkbuBoZFaKiJn4N7ZbwD2Bp4B+qjqh3nGkWW19OMisoNPbwWcCLwtIhcDJwPnZpzYgLazoBDuChLu9FEUeSthV5Bwi/RKJTeT4ow8pC0oNFX0qwQ64ALEXAkcCeyDU19zUdZDAdgDuMfr1O1we6MPFZGNwEzg314fH6yqv8zbIcMwWpiU92uq2g9I29pnDm4xICKKZZyFOcB4VZ3uLi+P4TbGTdxpKCtmxGsY9UVutfS9v/498Tnd5dLvNaSWdgDewQVVnouLhnWeqr6VUPZuitXS9sA44ERVXSwidwFjVfWvecZh7lfNoJaGqmZk2waFxYU0FTbcyrwpxmQYmahgQSFL9CsROVJE5gBnA7eLyFu+7oc4lfRZEXkTN0H/PfcwTHIzjLoit+S25Pa7Ep/Tnb9/YU25KeRxvxIRuUFE3vGRsS6rfncNw6g67dsnHzVGHverC3AvEA/yhncPZrlgW1FLwxXQ1c+PjssmblbZBO5XjembYTRIW9wVJHS/Av6Me2E4tRHXM7XUMKpL7llo6d0DEp/TnS44r6ZmuDzuV/sC3/T2Lk+KyP7V7KhhGM1EnUhuedyvtgDWqWoX3MpG/6S6bdb96onhceyElUOfistG6mdUprRsuIPI2lcKu7Y3Zd9MRTUaQjq0TzxqjYrdr3DuEt1VdYYPIrNcVbcvU93UUsOoLrlFrGUPPJL4nO547tdrSnyr2P0KeAwX9QpcFKx3qtVJwzCaD2kniUetkUUt3QN4TkTewFkdj1DVocCNwNe90d1vsM0qi9XScLPKYLV02f2D4nQY+i9SS4tWS8eMrUrf0uoZBlA379zMiNcw6ovcs9DyR4YkPqc7fL1HTc1w5n5VJcktdKkq2mY82M8tcZvx0P2qSnZuWeoZbRdp3y7xqDWy7ApiGEZbogajyydhaqlh1Be5VceVQ59KfE63++rJ9amWekPe8SIy1H/fW0ReFpEpIjJQRDbP0k5bVEvTthmP8pc98EisorYWtdRU1DZMnYT2yyy5ichPgC7Adqr6VREZhNug8kER+T/gdVX9W5lmTHIzjOqSX3Ib/myy5Nb9hLzRr7bABYI5AlgCfNPbyW6Gi3x1OO5V2b2q+pu848jqftURONV3AG+0ezwQhea6Bzgjb2cMw2h5KllQyBj96rvAMlXdD7gF+K0/fzawhap+FjfxfV9EOuUdR1a19FbgKgpRa3bGeSRs9N/nAHtmaajNqKUPPR4HYA5VzXKbVa4c9nScv+alOIJii6mlYVmjjdCuXfLRMGWjX/nv9/j0w8AJXlBS4GN+N9+tcIFiVuYdRlm1VES+iguQeomIdMPtmHkh8G8/AyMiewHD/MxbWr8X0Avg9ttvP6JXr16lRQzDaDpyq6Wrnx+dOCls0+2YXNGvRGSiLzPHf58GHAWsAP6B26J8a+AKH68hF1lMQb4A9BCRU4Atge1wktwOItLBS2+pwSBKgkrYOzfDaO2keCOEgoqnXzAJZYl+lVamK/Ah8AlgR+BFEXkmChhTKWVlTVXtq6odVbUTcA4wUlXPB54DzvLFegKPZ7lgm1FLBw6Ow/gVxVAI3K+i/DAo88onR8T5q0e9VJW+NcV9M+qXtNB+qtpPVbsERyhdZYl+FZfxKuj2wFLgPGC4qn6gqouAl3CLl/nG0cjNKrsBV/rV0n1wevVOwHjgW6q6vkwTJrkZRnXJrZau+fcric/pxz7fNVf0KxG5FPisqvYWkXNwwdy/ISJXAwcBF+HU0leBc3x0+opplCmyqj6vql/16emq2lVV91PVszNMbECys/iKIcNYMWRYnI7LhumELblDiSfcvjspHb7gL7I1S0s3ol60SLDs/kGxZBZKPCPenBKXLScdTT2+R2LZon3gonsV3Iui/DDttzoPFy2K6vm/QdhuWC/cLr3ovgbjN+qMKkW/wsUh3VlEpgI/Afr4838FtgEm4ia2u/JObGDuV4ZhlFDp9kaqOgwYVnLuf4L0OpzZR2m91Unn82LuV4ZRX+RWS9eOHZ/4nG7d5bCaclOo2P0qOP9nEVmdtZ1oh4yVw56ObbpWPz863vMs3EFj1bOjCmnvwrRqxHOF9NMjYxuySHUKy0ZlwL3Uj17sh7ZkYTqsF+/S8eyouB9hfrhIEC0CrBz6VLyl+DMTp/LMRBc7pzEv7ad2OzXOX3rPA3E6vC/xluRp9y0cf8K9Wj3qpXixIhx/uIARtZdl/KHabdQ+0qFD4lFrNOad2+U4XTpGRLoAOzRpjwzDaFna0maV3v3qHuAG4Cd+tbQ98AxuGXeKqm6T4XqmlhpGdck9C62bOCnxOd3ykE/X1AxXqfsVuJWRIao6vzEXjFfsglW4ZQMeYtmAh4ryN0lHq6LBCmm4m0bo1rS0/31xOs4PrhHamhWlE1yjlva/L24vzI/aCttYevcAlt49AGgaW7JQRY1WbMP7Eq7OFuWHaW9LF96r0pXTpDGFK8TxvfdthWNuaHxGjdJWJLcU96tewCCgm6puFJHVaZKbuV8ZRrOSX3J7+51kye2gA2pqhssyuf0G+DawkYL71Xp/rPPFPglMj3xNG8DUUsOoLrknoPVTpiU+p1vsv29NTW6Vul/tqKq7q2onf35thokNSDHiLTHMLS0LycaoRSpTqIqFBraR2laizsZlA1WrnBHv0n8MTL5GGSPeStXSOZdeucn4w/tSdN+CcYT3LWkcqUa8oYoaxXcIjXgT1N0sYzJqjMp2BWl1VOx+VXI+VS0twSQ3w6guuaWrDdNnJD6nm+/TqaYkNzPiNYz6Iv/kNnN28uT2qb1qanJrdlkzUS0tSUeEq6V5fUuLVmTT/EkTViSL1OCg3aTdPcJrNIVaOr/v9ZuMHyjyw03yyS1S7cv4lkZGx5Csooa7m6T5ltpuIvVFvYT2M8nNMOqL3NLVB/MWJD6nm31i9/qU3BKiX50gIuNEZIKIjBaRTAsKkSvP6pEvJLv4BC5XSS5FofvVyidHFFyfQneohHSq+1WCG1VqvbRreOkndL+aenyPeIeP0F6tMVLOtEXLNhk/UOQOFd6X0vywn6Vlo3Rj3N2S7k801nhMXzylwTGZFFcD1En0qzzuV38DzlfVzsAA4BdN2THDMFoGkXaJR/l60l1EJovIVBHpk5C/hQ8DOtWHBe0U5PX15yeLSJP8AuZxv5oMfEdVXxaRvsC2qvrzMk2ZWmoY1SW3iLVq1arE53TbbbdtaLPK9rjNKr+M23H3VeBcVf1PUOYS4NBgs8qvqeo3fZSsB3DbjX8C59Z5gKp+mGccedyvLgaGicgcnJHvjUkVS1n7ymvxZ1q6tCzAmjFj3ee/X4mDFa8ZPYY1o8fE5+OyCenSenF+mA6iTSXWC/PDtG8j3G0jVMvC3T1CFXXOD38GuIWDaPFg+uKCKhqqcNH4w/uyZszY+HzqfUsaR5Z60f0Oyhbd1+C+hf2M3M+mdjs1Hms0zmisEaHabdQ8eaJfnQ48qKrrVfVdYKpvLxcVuV95yW0w8Fsvuf0MODCKfFNS39yvDKP5aCnJLU/0q+uAMap6nz9/J/Ckqj5MDiqKfiUi/wQOUtVIfBkIDE+qbNGvDKM+qGL0qyx1G01F7lc4MXJ7ETnAF/syJXu9pfHBnLnxZ1q6tGxavQ3TZ7Bh+gyXnjk7LrthxqxN0htmzCpKN1Q2bK+onr/WJmWjPgT9mXn+95h5/veAYlUzyb0s3LgzVPeijSg3GV/Ut5mzi9KNKRunZ83JXi9hzABTjzstTs9e5mLppu30EtrShZtjhoGrjdZJFaNfZanbaPJEv/oa8Evce7hlwEUZ4gya5GYY1aWl1NI80a8OxllcRAsKzwL7511QMCNew6gvck9uS9euS3xOd9p6ywbb9q+ubgXaA/1V9QYR+SUwVlWHiMiWuMjyh+EktnMigUhErsGF9tsI/FhVn8w7jmb3qfhgwUL3OW8BH8xbEJ+Lz/vPTdJR2bBeqKIG6tUHc+dvks6k+ob1orLh9RLaLSobXGPhyjUsXLkGgHlXXxuXTVplTVv1DdXEqA+pfQvzw3Q0/rnzC+mSe5hYL+l+J4wZYPHq9+P0rAsvAYpXjle/+K84P0yHKvj6KdMAd6/C+2U0P6rJR/l6OkxVD1DVfVX1Bn/uf1R1iE+v82FA9/NhQacHdW/w9Q5siokNTHIzjHojt+S2ePX7ic/px7fZqqbcFDKFtBGRGcAq4ENgo6p2EZGbgdOADcA04EJVXV6tjhqG0Tx89NFH5QvVAI1RS49T1c6q2sV/HwEcoqqH4l4k9s3SSNoqXLwi2YhVv/XT3mX9tHfd+WD1LlJxwvPhSmZUpzRdtBoalZ0yLW4vKT9sIywb7sIR+mFO73FunI4MXpf+Y2C8EWZRqL1AhUtaqS0af9qqb8L4U+93wviK6iWMGYo3/4x9awMj3iW337XJmKFk5dSvDBeFOQx8Vo3m48OPNPGoNbK6X80Auqjqeyn5XwPOUtXzyzRVe3fIMGqL3Krj3GXJq6V77pi+WtoaySq5KfC0iLzmDflKuQjI9BIwlBjy2muFklIorSWlQymnKD9FiovLZqkX9CFKh9tzh5LbkjvujdORZBJGpgp393j/9YmF8TdCqspSNpTikuqFEnE5iTfcBy+SPN+77U7eu+1OoMQ9K4hMFu4ysvbVcQAsvPEWFt54CwCLb/lrnG+7iTQfH+pHiUetkXVy+4KqHg58BbhURL4YZfgl3I3A/UkVRaSXiIwVkbH9+vVLKmIYRiviI9XEo9Zo9GqpiFwHrFbV34lIT6A3cIKqrs1QvfbukGHUFrlVx2mLliU+p/vuumN9qaUi8jER2TZKAycBE0WkO3A10CPjxAak2E9lsddKKLth1pzYvi3Ndi1JnS3n4hWm02zpilRmfz7sT9ouJEkbXs657GrmXHY1UKKKhrZ7CfZ4RbZrjRhHUTrBJi6sl+Ueh7Z70SJI6FIWqp8zv9O7UC/Y6WTd2+8AxW5b4Uaa4c4qpqJWl3pRS7PsCrIPEDlEdgAGeMvjqcAWwBKfN0ZVeye1EWCSm2FUl9zS1eQF7yU+pwfuvktNSW5l7dy8FfHnEs5n2lbcMIzaohbNPpJodverVatWxZ9p6dKyafU2LlzMxoWLXXpxwUolOhemw7KhW1dS2bC9omuklU24xrpJk1k3aTIA778Z+w4XuVetHTve5Y97nffHvQ7AgusKe36+s3BJnA7HF49/8XuFfobjT7gXpWWjdHiPE+tlGP+6iYUNYd6f8KYbZ+BSFq2EQvGGmLN7XxGn569YDTibuMguLgyCPePsnoV637sMgJnnXczM8zbZQtDIiaomHrWGuV8ZRn2RW3V8Y3Zy9KtD96qt6FcVu1/58z8CfogzBfmnql5VpX4ahtFMfNjW3a9E5DjcppWHqurBwO+yNNKUammeeg2VrbReWDZc1S1S/QJ1Lqne1IVLC/nBBo5N2bemvm/hymm58YfpGUsKrsiRce/S/vfF6dBtK3TxijYBDV21pn3lLKZ95SyM/FTDzk1EdhKRESIyxX/umFKupy8zxZuaRefPFZE3ReQNERkuIruUvWal7lciMgi3zfAzWQbnMbXUMKpLbtVx7LtzE5/TLnvvWXHbInITsFRVb/Rh/3ZU1atLyuwEjAW64OaK14AjcFrjPOAzqvqeb2utql7X0DXzuF8dABzr4w+OEpEjszSUFoi4XADfpq7XUNlK64Vllw96NN5SPHSpCiWQKH/5Q4/HLkyhTVhahKnWdt9C96sosPPygYNZPnDwJvnhluOhzV+00BI69Yd2d0muYaG9XrjwEUqHRuP58KOPEo+chJGv7gHOSChzMjBCVZeq6jLc5hzdcRO2AB/z0bK2I8M25JneueHcr+aJyK7ACBF529fdETgaOBIYJCL7aIkoWBL9iuMyXtAwjJbho+qYguymqvMBVHW+n0tK2ROYHXyfA+ypqh+IyA+AN4E1wBTg0nIXrNj9CjgRuFFVn/fnpwFHq+ri9NqmlhpGlcmtlr70zszE5/SYAzt9n/ToV4jIM8DuCVWvAe5R1R2CsstUtei9mw8RuoWq/tp//29gLfAnXHS9XsB04M/AgqhcGhW7XwGPAcf78wcAmwOJWyKFtBm1NNjPLYpyBcXRr6L8tL3fQremVq2WhtGt/FhDtbwo3++UAsULJrFamuIml7TTSZFammK7Z65ajSdNLS0T/QpVPVFVD0k4HgcWisgeAP5zUcKl06JgdfbtT/Oa4SDgv8qNI8s7t92A0SLyOvAKzuTE/b4/AAAVl0lEQVRjONAf2McHWn0Q6FmqkhqGUXtUaVeQIUC0+tkTeDyhzFPASSKyo19NPcmfmwt8RkQ+7stlCiVqRryGUV/kVktHvjU18Tk9/uD98qyW7oyTuD4JzALOVtWlItIF6B1Eqr8I+LmvdoOq3uXP9wYuBz4AZgIXqOoSGqDZ3a/aoloarSBC8Tbb0SpiPaml0U4eS+95IN7Jo1FqabhaGuxYkrR1fJbV0rQ+G+lUY5txVV2iqieo6v7+c6k/Pzaa2Pz3/j461n7RxObP/5+qflpVD1XV08pNbGCSm2HUGzXlIlVNMkluIrKDiDwsIm+LyCQR+XxWi2PDMIyWIKta+kdguKoehNv+aBLQB3hWVfcHnvXfy9Im1dLAiDcybAVYdv+gTcrWuloajTVcLY02n4RGrpYGammlRrzlxmTUL1k2q9wOeB0oMtAVkclAN2+QtwfwvKoeWOZ6ppYaRnUxtdSTRXLbB1gM3CUi40XkDm/vVmRxDCRZHBuGYbQIWSa3DsDhwN9U9TCc+0MmFRQ2jX7VZtTS0F80UDWLfEt9fhjar+bVUq9qFvmWBvnh+MuqpSm+pXFYwiC+Q6Vqqamn9UsWtXR3XHyETv77sbjJbT9MLTWM1oappZ6ykpuqLgBmi0g0cZ0A/IdsFseb0GYkt+CFerjTR7SIAIXFhaKyPmIU1OauIJHkGUaxKnI5SwjgDAUptcjOLcX9qikXFNLGZNQ+WXcF+RFwv4hsjnNcvRA3MQ4Ske/iLY6r00XDMIzGY0a8hlFfmFrqMferFlRLo5fraWVXj3qpKn1rcrU0UDtjtfT+QQU7vsC2L6ksFFTwNLU0UkXDdGN3Ban0f8GoTUxyM4z6wiQ3T7NLboZhGM1Bxb6lQd6VIqJZotFAwaUmjI5Umi4tm1YvtHMKVZikdKjCpJWN2grPh9cI84vS3i4rtNGa3fuKOOjw9MXL4rKh+9HKoU+5z+HPxipaGLQ5XC2tdExpZbPei7Bs0piBoqhTs5etBIpV7RVDhhXGnLIaHO2cEqqM839R2Gh1ardT4/Sim/4EFNsHLrnjXpbccS9QiKSV9V40VLY0bdQOWaNf3QO8qKp3+BXTrVV1uYjsBdwBHAQcEUbHSsHUUsOoLqaWerJsM74d8EXgTgBV3aCqUcDJW4CrsEnLMIxWRsW+pSLSA5irqq83VLnU/SoytPxgwUI+WLAQKDbADIMWJ6XDskXqbOCqU06dLVcWAhUmi8rs2wvVnYUr17Bw5RoA5v/3DXHZIrXTp9f8+5VCeszYOD9U/RKvF9zDxqjzqffCt9VQvdI+ACxe/X6cnv29ywBY/eK/WP3iv9yYAvWzKB24l0WbUc7rcx3z+lwHFK9YTuv+9cL1bvmru8bIF+LNMcPV2TCYc977Fqbn972e+X2vx6gNsrhfdQHG4ML7vSwifwQ24KS5k1R1RVLQ5hRMwjOM6mJqqSeL5DYHmKOq0c/swzhH+r2B1/3E1hEY5/1QDcMwWpxKfUvHqequqtrJO9TPAQ73ZRuk0tW7pLKhwWeowhXtIJGwkllUtkx6w/QZhV0ownx/3TAdll3x2NA4zF24WeWMs3vG6aX/GAhk2xUksW9p408qWzL+OB2ugIZjKnffgntcZJjrV0Onnng6U088HSBexYRiI+ZotRgKBsvhammoiiYZ2IZxGmZ840JmfONC15/AaLhRf9My6bS+Ga2TrKulnXGrorFvqQ93H+XPwNRSw2gNmFrqyWTnpqoTfBDWQ1X1jHBi8/mdMkxsQIrkNnd+4WVv+II7TCdJbll+gQPJJpR4Giobnk+tF6ajPgT9WTn0qVgyKYp+5SUNgOmnnQMUbzMeulyte+vt5H5WII1mktwac9+CdBh0OpI8Q7uzUOoKJbdQon1/nFuXWnTTn2I7tsW3/i3OL+dGFdoVzur5g8R+5pXiw74t+v1fEvtmtB7MQ8EwjLrEfEsNo74wtdSTJ7RfZxEZIyITvB1b1yxtJdmrNYmdWzkbtFD1bYRtU6pNWIKtXHiNta+OY+2r44Bi27ZQFYvSc3/6C+b+9BcArJs4qdBuuKCSwXatwb6l2bkl2LaljSntGqFtXmTHFrqUhYsksy68JE6vfeW1OL1+8lSgeJPLyIYNitX5JLV05nkXM/O8i4vyU8fUiPsWpkNbuvDvGLp7mYraeqjY/QoYBNyiqk+KyCnAVararUxTJrkZRnUxyc1TdifewP3qAnDuV8AGEVFgO19se2BelfpoGIbRaPKE9vsxcLOIzAZ+B/TNcsFoI8FVq1alpkvLVqNeQ2UrrReWXT/t3XhTxXWTJsdlo1VBKKig6yZOitPRahzAhFkFNakp+9bU923d2+8UxvfmW4BTOSO18/0Jbxbyg/Scy66O0zOWOHflot09vB0gwMxv9SrUu/RKd+47vZn5nd5AsYr67tfOzz2mpPSS2++KXbuW3j2g0PdzLir07ZKfunOB3Z3RMlTqfrUSJ62NUtVHROQbQC9VPTGhfi+gF8Dtt99+RK9evUqLGIbRdJha6skT2u8YYAdVVRERYIWqbpfeEmDv3Ayj2tjk5skT2m8e8CV/7nhgSpYLLl27Lv5MS5eWDdNL1qxjyRqXbs1qadqe/knpcPxvzC6s0oUGva1ZLQ1XFhtT7+35hdXwSM1b2v++ePUxdNsKAzvPuuiHQIkqeua3effMb8fn844pUS3tdzdL+t3t0sHOI6G716zv/miTvk3vcS7Te5yL0bxU7H4FHAz8EbcosQ64RFVfS23EYZKbYVQXk9w8FbtfqepoVT1CVT+nqkdlmNiAthP9asXgJ+ItxUN7rTAocZSf5mQf2se15uhXoftVNNbQpSzMD9OhW1pkK1fklhfY4CW5hoX2amnSWFP+L4TXC20wkzZ4KA0SHUrpRvNg7leGYdQl5n5lGPWFqaWeLDEUDvQuVtGxUkR+LCI3e3esN0TkURHZIcsF24xaOmRYHPVpxRPD47Lhi/E4PyhbpJYGbk2tWi0NxhfthBKq5WH0q7BsqK5HNnFFammwUJG0S0dLqqVFbmsVqqXmqlVdsqyWTlbVzqraGTgCWAs8CowADlHVQ4F3yGjEaxiG0Rw0Si0VkZOAa1X1CyXnvwacparnJ9eMMbXUMKqLqaWexi4onAM8kHD+IuDJLA20GbU0WAEN1a9ws8YoP1UtrZXV0kDtXP38aDfOYHePotXSQC0NV0sT1dJwtTRh08lWuVrqdxZprFpqKmrTk1ly87uBzAMOVtWFwflrgC7AmZrQmLlfGUazYpKbpzGT2+nApap6UnCuJ9AbOEFV12ZoxtRSw6guNrl5GqOWnkugkopId+BqoEfGiQ0wtTQ04o3cdmp+tTRUSxOMeMPoWBWrpa1stbRaamnYNyMfWd2vtgZmA/uo6gp/biqwBbDEFxujqr3LNGWSm2FUF5PcPGbEaxj1hU1unmZ3v2qLammoahb5lvr85YMejVW3WldLI1Uzzbc0NGIua8Rbi2pphUa85fpmNB6T3AyjvjDJzVOx+5XP+5GITBaRt0TkpiwXbDOSW+B+tOrpkXHZ8OV6KNlE0k0YKapm7NzCnT685BlKo9F9SCsLBSl1w6w5caDo1ux+1dySm0lxjadsgBhVnQx0BhCR9sBc4FEROQ44HThUVdeLyK5V7alhGEYjqNj9SkQGAf1U9ZlGXM/UUsOoLqaWevK4Xx0AHCsiL4vIKBE5MksDbV4tDbakjtXSsGyoqr30clX61txqadFOKGlqqVfBa2VXkErV0miL/Kbqm5FO5snNu1/1AB7ypzoAOwJHAz8DBvlAMaX1evmI9GP79evXBF02DMMoT8XuVyIyHLhRVZ/336cBR6vq4vRWTC01jCpjaqmnYvcr4DFc1CtE5ABc8Jj3yjXSVtTSxbf+jcW3/g0o3gkkDLq86Pd/2aTs0nsKtzh0VWrNamnU93Csi276UzzWaJylZcOgyyuHPQ24AM9RkOcN02fE+VHQaigEfg4DX5eqqHnHlJReP2Ua66dMc30LdikJg26ve+vtOD8q09SrpaaiZiOP+9XmQH/cSuoG4EpVHZneCmCSm2FUG5PcPFmjX61V1Z2jic2f26Cq31LVQ1T18AwTG4CIyPdxf4DUo1yZvPmtpQ27Rv31sxVcw4hQ1WY9gLF5y+TNby1t2DXqr5+t5Rp2qIX2MwyjPrHJzTCMuqQlJrcsxm7lyuTNby1t2DWat422dI02T3PvCmIYhtEsmFpqGEZdYpObYRh1Sdktj/IiIgfhtkbaE2fEOw8YoqqTfH5XQFX1VRH5DNAdeFtVh6W1aRiGUY6qvnMTkatxblsPAnP86Y643UUexAWY+Qpukh0BHAU8D5wIPKWqN4jIUcAkVV0pIlsBfYDDgf8A/6uqK0RkX+BrwF7ARmAK8IAGRsetERHZVVUXtXQ/6oXWcD9FZGdVXVK+pFF1qmlEB7wDbJZwfnPcBPQm0B7YGlgJbOfztwLe8Om3gA4+3Q+4FTgGuBYYDFyGmxh/AfwLuA24ATf5dWtpQ0Lf7yeBnUqOnYEZuJ1VdgLG+THs20A7HYDvA8OBN4DXfdu9gc38fbwKt0vLlsAFwBDgJmAb4IfALr6t/YAXgOXAy8Bn/fl2wEXAP337r+F+iLr5/Pa+D78CvlDSv1/gNi+Nvm/mzw0B/hfY2p/fB+e692vfr78DE3E7znQqdy/9Z677CWwP3Ai8jYvgtgSY5M/t4MvsDvwN+Ktv/zr/PzsI2MOXje5nF2A6MBWYCXzJj+2XuP/hFcBiYAxwQdCP7YDfAP8Azivp421A95I+3+n/9gOA3Vr6f7s1H9WW3N4GTlbVmSXnPwU8DaxV1cP8ufFR2n+foKqdRWSSqn7anxunqoeHZXAPW2dV/dD7wA5T1W4i8kngcVU9TES2A/ripMYnVXVA0MZtODV5uP++PfAH4EjcA3eFqi4UkS7AzbidiPviHs6uuAm8F+muLwIMBXbD/dOHdMRJtOrLPQJ8A1iA26RgoKrOC/r6AG4yuodiSbgn7oEWnA/wVsCBuId1EHAa7kE9XFUP9m39E7hDVR8VkW7ADeo2Ib3L9/MZ4Czcj86LuBi1jwOfw02irwDfBkap6k+ivw9A9DcSkd/jJoW7gDOAnVX1OyLygh/f9sC3fP4g4CTgfODKhu6lqu4hIh/luZ8i8hQwErhHVRf4c7v7e3miqn7Z73zzT+BjwHnA/b6d03HaxT6q+llf9zngKnWvVw7ATT5zgUf9vfyGb+dB3KQ7V1V/LiKP4H7ox+B+VD7ATXLrE+7nHX4sfwfOBL6kqmek3CujmjMn7v3ZVJx00c8fw/257jiJIfo1b1fyCzXOpx8CLvTpu4AuPn0A8Crul3QLf25H4LWgnYn+8xHcr+wZOCnikaDOuOha/vsdOIniU8AVwGP+/Cs4Ffpc3ARylj9/AvBv4EPcw/JcwvE+7oEdjpeQfN13g3TYh2Nxv9oLfP1e/vzkBu71O8AEnxZfV4Lvb4T1gVdL6r8Rfgbnx/jPLXCT5RtBXgf/Nx3s88cD44P8CXjJPeqDT4dlZpVcb3y5e+nL5bqfZe7l5Az9nICT+iKtYkxJ/pvA6yXnXo3+13HvlYn+ZkGZa4CXcD8Kpf+bpWUnpI3BDq2+b6n/Qx4NfB0nCRwNtPd5W6TU2YWCmrQ9cDcwDTcZfoAT/0fhpIjL/YPbz/+zRRPhx4EXmuofKMMDORHYP2U8s/1nR9xk/QdgW2B6UGZcQr32uB+Bu/z3McDZFP8QtAO+6e/NhOB8/5K2Xsep63fj1MKfAz8GPglciJOIwKmh+/r04dE99N//Ez2UJW1f6+/nFP+3+Zr/e08q7UNwjQNw0vF7FH6w9vN/y7L3Mu/9xGkOVxGodjjp+mrgmbC/Pv3rkrbeAH7k2zkep7LeCnwRuB6nZv4LOMaXPw33HjmqH02gk8K/pz/XE6fKzsRJoj8BfurvrYR9qPbzW8tHi3cgc0fdP+/ngCMoedcAHIybOA9KqZv7HwgnnZ2Em1xmAmf4818CxvrrH5hy/TNKvp+Gm6gWBOcezHAPOgEDgUU4Se0dnx4I7I2TOrdJqLcvMNqnL8BNhO8Bq/ALM8D2Pv94YJZv+13gKH/+47h3d/cRvAcKrnEx7ofnbj95RMduPn934FmfPgGY7P8ux+Ak6Sl+LGc05l5Wej9xUv5vcT+Iy4Clvj+/BXbyZX6Zcj/3Ax726W7+/o/HSWvDcJLhZsChOIl/OTAaOCC4l5f59E04Nbj0Gt39Pbm25Ph4cD/vbennsjUfLd6BZhlkE/wD4SbWp3Aq9kHAH/0/7VvAf/kyB/kHd5vS65Tm496LHZKWn1Tfp4/Cvevb2U8MVwKnBPldgSN9+jO4SftUCipqmH8wbkI/peR6n09o45Ske+vLNPiQZXkIce8l26XkHeP7cFID9Y/FvctKLFPahr+P0YS+NW4iG4qb3LYPyoSLXNcDT0RlSvI3aSPhGkX1/fnLgL0aGFeD+XakH23e/UpELlTVuyrNj8rgJMtLcb/+nYHLVfVxnz8OJ9E0lH8XbjUzMV9VDxeRayk2nemKU89PxE28HWjAtCYhv6i+OtObctc4qnT4wHG4d2RpHB/lq2oPERnSUBlgd1Xt6sf+PX/fHsVJzk+o6o0i8kpJmUtwu0OfhJtAzmyoDdxiyOdUdaOI9APW4CTIE/z5M0XkrZIya4GHozK4RZvUNhLyi+r7a6zw9abhFise0mCr/nL5RgO09Oza0gcl788amx+Vwakk2/jvnXCq6uX++/i8+f6zQdOZvPkZrzEep5p2w6nk3YD5Pv2lcvnBeBtsI7i3r1KQpD8GvBnek7QyGfInBfnjSv6e0XvWBsvkzQ/uRTvcpHsnzlxkOO61ybbl8lv6+WnNR9U9FFoDIvJGWhawW7n8LG0Aq1V1NYCqzvDmFQ97sxfBLaLkyQfYqKofAmtFZJqqrvTl3/emEZozP8s1jsAt4lwD/ExVJ4jI+6o6yt+nBvM95dpoJyI74h5qUS+pqOoaEdno2yhXplz+24FU/rqIdFHVsd6M4wN/jYllykzJme+7pB/hFiaeFpHNKKzK/w63gNJQ/scxkmnp2bU5DmAhTtX7VMnRCecO1mB+xjZG4uztwut2AO6lYNpQcb7/3qDpTN78LNcIvkcrlX8hQbotl99QGZwx7nTcgsZ0nJoK7l3lhCxlMuQ3uAofjDm1TN58f43xSffG521VLr+ln63WfLR4B5plkE6cPyYlb0C5/IxtdIweoIT8L+TN958Nms7kzc9yjYTzp+Lc4NLufYP5Wcv4clsDe+cpU5pPA6vwWcvkycevoDbQ3wbz7Ug/2vyCgmEY9YlteWQYRl1ik5thGHWJTW6GYdQlNrkZhlGX2ORmGEZd8v9u/xR2eZDVCAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "#Developing correlation plot\n",
    "\n",
    "# Generate a mask for the upper triangle\n",
    "mask = np.zeros_like(corr, dtype=np.bool)\n",
    "mask[np.triu_indices_from(mask)] = True\n",
    "\n",
    "# Generate a custom diverging colormap\n",
    "cmap = sns.diverging_palette(220, 10, as_cmap=True)\n",
    "\n",
    "# Draw the heatmap with the mask and correct aspect ratio\n",
    "sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0,\n",
    "            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = Dataset_2[77]\n",
    "X_org = Dataset_2.drop([77], axis = 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.19739600730596205"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.sum()/y.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Above value shows that this dataset is biased. Which means accuracy may not a be good evaluation function. For the rest of this project we consider ``AUC`` as a scoring function.** "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Taking Random sample with 10% data and pre-processing the data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "_,sample_data, _, sample_target = train_test_split(X_org, y, shuffle = True, test_size = 0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_unscaled , X_text_unscaled, y_train, y_test = train_test_split(sample_data, sample_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train_unscaled)\n",
    "X_test = scaler.transform(X_text_unscaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MinMaxScaler rescales the data set such that all feature values are in the range [0,1] . So that it is standardized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Voting Classifier KNN and Linear SVC (Hard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=20, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn=KNeighborsClassifier(n_neighbors=20)\n",
    "knn.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=10, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "svc_lin=LinearSVC(C=10)\n",
    "svc_lin.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNeighborsClassifier 0.9502558271745309\n",
      "LinearSVC 0.9587833996588971\n",
      "VotingClassifier 0.9536668561682774\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Amit\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "voting_clf = VotingClassifier(estimators=[ ('knn', knn), ('svc', svc_lin)], voting='hard')\n",
    "voting_clf.fit(X_train, y_train)\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "for clf in ( knn, svc_lin, voting_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=4,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "dtree=DecisionTreeClassifier(max_depth=4)\n",
    "dtree.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=100, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lrclf=LogisticRegression(penalty='l2',C=100)\n",
    "lrclf.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Voting Classifier Decision Tree and Logistic regression (Soft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeClassifier 0.9536668561682774\n",
      "LogisticRegression 0.9573621375781695\n",
      "VotingClassifier 0.9567936327458784\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Amit\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "voting_clf = VotingClassifier(estimators=[ ('Decision Tree',dtree ), ('Logistic Regression', lrclf)], voting='soft')\n",
    "voting_clf.fit(X_train, y_train)\n",
    "\n",
    "for clf in ( dtree, lrclf, voting_clf):\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging Classifier Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.96\n"
     ]
    }
   ],
   "source": [
    "bag_tree = BaggingClassifier(dtree, n_estimators=500, max_samples=100, bootstrap=True, n_jobs=-1, random_state=0)\n",
    "\n",
    "bag_tree.fit(X_train, y_train)\n",
    "y_pred = bag_tree.predict(X_test)\n",
    "print('Accuracy score: {:.2f}'.format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.97\n",
      "Test score: 0.96\n"
     ]
    }
   ],
   "source": [
    "bag_tree.fit(X_train, y_train)\n",
    "print('Train score: {:.2f}'.format(bag_tree.score(X_train, y_train)))\n",
    "print('Test score: {:.2f}'.format(bag_tree.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging Classifier Logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.94\n"
     ]
    }
   ],
   "source": [
    "bag_log = BaggingClassifier(lrclf, n_estimators=500, max_samples=100, bootstrap=True, n_jobs=-1, random_state=0)\n",
    "\n",
    "bag_log.fit(X_train, y_train)\n",
    "y_pred = bag_log.predict(X_test)\n",
    "print('Accuracy score: {:.2f}'.format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.95\n",
      "Test score: 0.94\n"
     ]
    }
   ],
   "source": [
    "bag_log.fit(X_train, y_train)\n",
    "print('Train score: {:.2f}'.format(bag_log.score(X_train, y_train)))\n",
    "print('Test score: {:.2f}'.format(bag_log.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pasting KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.93\n"
     ]
    }
   ],
   "source": [
    "past_knn = BaggingClassifier(knn, n_estimators=500, max_samples=100, bootstrap=False, n_jobs=-1, random_state=0)\n",
    "\n",
    "past_knn.fit(X_train, y_train)\n",
    "y_pred = past_knn.predict(X_test)\n",
    "print('Accuracy score: {:.2f}'.format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.93\n",
      "Test score: 0.93\n"
     ]
    }
   ],
   "source": [
    "past_knn.fit(X_train, y_train)\n",
    "print('Train score: {:.2f}'.format(past_knn.score(X_train, y_train)))\n",
    "print('Test score: {:.2f}'.format(past_knn.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pasting Linear SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.96\n"
     ]
    }
   ],
   "source": [
    "past_svc = BaggingClassifier(svc_lin, n_estimators=500, max_samples=100, bootstrap=False, n_jobs=-1, random_state=0)\n",
    "\n",
    "past_svc.fit(X_train, y_train)\n",
    "y_pred = past_svc.predict(X_test)\n",
    "print('Accuracy score: {:.2f}'.format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train score: 0.97\n",
      "Test score: 0.96\n"
     ]
    }
   ],
   "source": [
    "past_svc.fit(X_train, y_train)\n",
    "print('Train score: {:.2f}'.format(past_svc.score(X_train, y_train)))\n",
    "print('Test score: {:.2f}'.format(past_svc.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ADA Boosting Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "          base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=4,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "          learning_rate=0.5, n_estimators=200, random_state=0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "\n",
    "ada_clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=4), n_estimators=200, algorithm=\"SAMME.R\", learning_rate=0.5, random_state=0)\n",
    "ada_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.96\n"
     ]
    }
   ],
   "source": [
    "ada_clf.fit(X_train, y_train)\n",
    "y_pred = ada_clf.predict(X_test)\n",
    "print('Accuracy score: {:.2f}'.format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ADA Boosting Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "          base_estimator=LogisticRegression(C=100, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "          learning_rate=0.5, n_estimators=200, random_state=0)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada_log = AdaBoostClassifier(LogisticRegression(C=100), n_estimators=200, algorithm=\"SAMME.R\", learning_rate=0.5, random_state=0)\n",
    "ada_log.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.95\n"
     ]
    }
   ],
   "source": [
    "y_pred = ada_log.predict(X_test)\n",
    "print('Accuracy score: {:.2f}'.format(accuracy_score(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "              presort='auto', random_state=0, subsample=1.0, verbose=0,\n",
       "              warm_start=False)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "gbrt = GradientBoostingClassifier(random_state=0)\n",
    "gbrt.fit(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on training set: 0.982\n",
      "Accuracy on test set: 0.958\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy on training set: {:.3f}\".format(gbrt.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(gbrt.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA(n_components=0.95)\n",
    "X_reduced = pca.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca.n_components_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9578815647790832"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA(n_components = 8)\n",
    "X_reduced = pca.fit_transform(X_train)\n",
    "X_recovered = pca.inverse_transform(X_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10553, 77)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_recovered.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling all models after PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "knn = KNeighborsClassifier(n_neighbors=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=20, p=2,\n",
       "           weights='uniform')"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn.fit(X_recovered,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_knn_predict = knn.predict(X_test)\n",
    "y_knn_train_predict = knn.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train roc_auc_score: 0.95\n",
      "Test roc_auc_score: 0.93 \n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "print('Train roc_auc_score: %.2f'%roc_auc_score(y_knn_train_predict, y_train))\n",
    "print('Test roc_auc_score: %.2f '%roc_auc_score(y_knn_predict, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_report_table = [['knn', 'k = 20', knn.score(X_recovered, y_train), knn.score(X_test, y_test), roc_auc_score(y_knn_train_predict, y_train), roc_auc_score(y_knn_predict, y_test) ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "svc_lin = LinearSVC(C=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=10, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc_lin.fit(X_recovered,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "linsvc_predict = svc_lin.predict(X_test)\n",
    "linsvc_train_predict = svc_lin.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train roc_auc_score: 0.95\n",
      "Test roc_auc_score: 0.93 \n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "print('Train roc_auc_score: %.2f'%roc_auc_score(linsvc_train_predict, y_train))\n",
    "print('Test roc_auc_score: %.2f '%roc_auc_score(linsvc_predict, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_report_table = pca_report_table + [['Linear SVM', 'C = 10', svc_lin.score(X_recovered, y_train), svc_lin.score(X_test, y_test), roc_auc_score(linsvc_train_predict, y_train), roc_auc_score(linsvc_predict, y_test) ]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=100, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "log=LogisticRegression(penalty='l2', C=100)\n",
    "log.fit(X_recovered,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_predict = log.predict(X_test)\n",
    "log_train_predict = log.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train roc_auc_score: 0.95\n",
      "Test roc_auc_score: 0.93 \n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "print('Train roc_auc_score: %.2f'%roc_auc_score(log_train_predict, y_train))\n",
    "print('Test roc_auc_score: %.2f '%roc_auc_score(log_predict, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_report_table = pca_report_table + [['Logistic Regression', 'C = 100', log.score(X_recovered, y_train), log.score(X_test, y_test), roc_auc_score(log_train_predict, y_train), roc_auc_score(log_predict, y_test) ]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kernel SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=100, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " from sklearn.svm import SVC\n",
    "kernel = SVC(C=100)\n",
    "kernel.fit(X_recovered,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "kernel_predict = kernel.predict(X_test)\n",
    "kernel_train_predict = kernel.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train roc_auc_score: 0.95\n",
      "Test roc_auc_score: 0.93 \n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "print('Train roc_auc_score: %.2f'%roc_auc_score(kernel_train_predict, y_train))\n",
    "print('Test roc_auc_score: %.2f '%roc_auc_score(kernel_predict, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_report_table = pca_report_table + [['Kernel SVM', 'C = 100', kernel.score(X_recovered, y_train), kernel.score(X_test, y_test), roc_auc_score(kernel_train_predict, y_train), roc_auc_score(kernel_predict, y_test) ]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=4,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "tree= DecisionTreeClassifier(max_depth=4)\n",
    "tree.fit(X_recovered,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree_predict = tree.predict(X_test)\n",
    "tree_train_predict = tree.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train roc_auc_score: 0.90\n",
      "Test roc_auc_score: 0.90 \n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "print('Train roc_auc_score: %.2f'%roc_auc_score(tree_train_predict, y_train))\n",
    "print('Test roc_auc_score: %.2f '%roc_auc_score(tree_predict, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_report_table = pca_report_table + [['Decision Tree', 'D = 4', tree.score(X_recovered, y_train), tree.score(X_test, y_test), roc_auc_score(tree_train_predict, y_train), roc_auc_score(tree_predict, y_test) ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_report = pd.DataFrame(pca_report_table,columns = ['Model name', 'Model parameter', 'Train accuracy', 'Test accuracy', 'Train auc score', 'Test auc score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_report.index = pca_report['Model name']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following table contains results without applying PCA and is copied from the Project 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Model parameter</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "      <th>Train auc score</th>\n",
       "      <th>Test auc score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Model name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>knn</th>\n",
       "      <td>knn</td>\n",
       "      <td>k = 20</td>\n",
       "      <td>0.993724</td>\n",
       "      <td>0.988135</td>\n",
       "      <td>0.956491</td>\n",
       "      <td>0.960654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearSVC</th>\n",
       "      <td>LinearSVC</td>\n",
       "      <td>C = 10</td>\n",
       "      <td>0.992865</td>\n",
       "      <td>0.993405</td>\n",
       "      <td>0.959129</td>\n",
       "      <td>0.966063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>C = 100</td>\n",
       "      <td>0.967308</td>\n",
       "      <td>0.968732</td>\n",
       "      <td>0.958571</td>\n",
       "      <td>0.963571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kernalized SVM</th>\n",
       "      <td>Kernalized SVM</td>\n",
       "      <td>C = 100</td>\n",
       "      <td>0.991152</td>\n",
       "      <td>0.991533</td>\n",
       "      <td>0.957230</td>\n",
       "      <td>0.957388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Decision Tree</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>d = 4</td>\n",
       "      <td>0.988685</td>\n",
       "      <td>0.984661</td>\n",
       "      <td>0.956116</td>\n",
       "      <td>0.950319</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Model name Model parameter  Train accuracy  \\\n",
       "Model name                                                                 \n",
       "knn                                  knn          k = 20        0.993724   \n",
       "LinearSVC                      LinearSVC          C = 10        0.992865   \n",
       "Logistic Regression  Logistic Regression         C = 100        0.967308   \n",
       "Kernalized SVM            Kernalized SVM         C = 100        0.991152   \n",
       "Decision Tree              Decision Tree           d = 4        0.988685   \n",
       "\n",
       "                     Test accuracy  Train auc score  Test auc score  \n",
       "Model name                                                           \n",
       "knn                       0.988135         0.956491        0.960654  \n",
       "LinearSVC                 0.993405         0.959129        0.966063  \n",
       "Logistic Regression       0.968732         0.958571        0.963571  \n",
       "Kernalized SVM            0.991533         0.957230        0.957388  \n",
       "Decision Tree             0.984661         0.956116        0.950319  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following table summarizes results after applying PCA:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model name</th>\n",
       "      <th>Model parameter</th>\n",
       "      <th>Train accuracy</th>\n",
       "      <th>Test accuracy</th>\n",
       "      <th>Train auc score</th>\n",
       "      <th>Test auc score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Model name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>knn</th>\n",
       "      <td>knn</td>\n",
       "      <td>k = 20</td>\n",
       "      <td>0.961528</td>\n",
       "      <td>0.949972</td>\n",
       "      <td>0.947124</td>\n",
       "      <td>0.928344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Linear SVM</th>\n",
       "      <td>Linear SVM</td>\n",
       "      <td>C = 10</td>\n",
       "      <td>0.962001</td>\n",
       "      <td>0.949972</td>\n",
       "      <td>0.948638</td>\n",
       "      <td>0.931240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic Regression</th>\n",
       "      <td>Logistic Regression</td>\n",
       "      <td>C = 100</td>\n",
       "      <td>0.961907</td>\n",
       "      <td>0.950256</td>\n",
       "      <td>0.948727</td>\n",
       "      <td>0.932484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kernel SVM</th>\n",
       "      <td>Kernel SVM</td>\n",
       "      <td>C = 100</td>\n",
       "      <td>0.961243</td>\n",
       "      <td>0.948835</td>\n",
       "      <td>0.951082</td>\n",
       "      <td>0.931780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Decision Tree</th>\n",
       "      <td>Decision Tree</td>\n",
       "      <td>D = 4</td>\n",
       "      <td>0.964749</td>\n",
       "      <td>0.925526</td>\n",
       "      <td>0.904192</td>\n",
       "      <td>0.896027</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              Model name Model parameter  Train accuracy  \\\n",
       "Model name                                                                 \n",
       "knn                                  knn          k = 20        0.961528   \n",
       "Linear SVM                    Linear SVM          C = 10        0.962001   \n",
       "Logistic Regression  Logistic Regression         C = 100        0.961907   \n",
       "Kernel SVM                    Kernel SVM         C = 100        0.961243   \n",
       "Decision Tree              Decision Tree           D = 4        0.964749   \n",
       "\n",
       "                     Test accuracy  Train auc score  Test auc score  \n",
       "Model name                                                           \n",
       "knn                       0.949972         0.947124        0.928344  \n",
       "Linear SVM                0.949972         0.948638        0.931240  \n",
       "Logistic Regression       0.950256         0.948727        0.932484  \n",
       "Kernel SVM                0.948835         0.951082        0.931780  \n",
       "Decision Tree             0.925526         0.904192        0.896027  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have visualized the results from the above table, i.e. results after applying PCA.  \n",
    "As we can see the accuracy of models has been reduces a little bit, but the model training is much faster.   \n",
    "**Thus, PCA has not helped improve the accuracy of the models, but it certainly has reduced our efforts in training these models.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Learning\n",
    "Let's apply Deep learning model, we will use the original data instead of data obtained from the PCA for Deep learning model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\suraj\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "import numpy\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "numpy.random.seed(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train_org, X_test_org, y_train, y_test = train_test_split(X,Y, random_state = 0)\n",
    "\n",
    "scaler = MinMaxScaler(feature_range=(0, 1))\n",
    "X_train_scaled = scaler.fit_transform(X_train_org)\n",
    "X_test_scaled = scaler.transform(X_test_org)\n",
    "X_train =  pd.DataFrame(X_train_scaled)\n",
    "X_test = pd.DataFrame(X_test_scaled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1- Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create model\n",
    "model = Sequential()\n",
    "model.add(Dense(30, input_dim=77, kernel_initializer='normal', activation='relu'))\n",
    "model.add(Dense(8, activation='relu'))\n",
    "model.add(Dense(1, kernel_initializer='normal'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2- Compile Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile model\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3- Fit Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "10553/10553 [==============================] - 1s 77us/step - loss: 0.1713 - acc: 0.8940\n",
      "Epoch 2/10\n",
      "10553/10553 [==============================] - 1s 75us/step - loss: 0.1163 - acc: 0.9203\n",
      "Epoch 3/10\n",
      "10553/10553 [==============================] - 1s 75us/step - loss: 0.1143 - acc: 0.9085\n",
      "Epoch 4/10\n",
      "10553/10553 [==============================] - 1s 84us/step - loss: 0.1245 - acc: 0.9053\n",
      "Epoch 5/10\n",
      "10553/10553 [==============================] - 1s 81us/step - loss: 0.1705 - acc: 0.9022\n",
      "Epoch 6/10\n",
      "10553/10553 [==============================] - 1s 79us/step - loss: 0.1083 - acc: 0.9026\n",
      "Epoch 7/10\n",
      "10553/10553 [==============================] - 1s 75us/step - loss: 0.1152 - acc: 0.8977\n",
      "Epoch 8/10\n",
      "10553/10553 [==============================] - 1s 68us/step - loss: 0.1090 - acc: 0.8992\n",
      "Epoch 9/10\n",
      "10553/10553 [==============================] - 1s 66us/step - loss: 0.1143 - acc: 0.8986\n",
      "Epoch 10/10\n",
      "10553/10553 [==============================] - 1s 71us/step - loss: 0.1209 - acc: 0.9039\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2beff0e4080>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs = 10, batch_size=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 4- Evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3518/3518 [==============================] - 0s 32us/step\n",
      "\n",
      "acc: 90.59%\n"
     ]
    }
   ],
   "source": [
    "# evaluate the model\n",
    "scores = model.evaluate(X_test, y_test)\n",
    "print(\"\\n%s: %.2f%%\" % (model.metrics_names[1], scores[1]*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**To sum up, All the models give us a little bit better accuracy when using any kind of boosting or ensemble method.\n",
    "we can notice that all the models yield average an accuracy of 95% after applying PCA.\n",
    "And the deep learning model also gets us approximately 91% of accuracy.**"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}