Feature Selection in ML using Python. Part 3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Univariate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using ANOVA for Classification and Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.feature_selection import f_classif, f_regression\n",
    "from sklearn.feature_selection import SelectKBest, SelectPercentile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>var3</th>\n",
       "      <th>var15</th>\n",
       "      <th>imp_ent_var16_ult1</th>\n",
       "      <th>imp_op_var39_comer_ult1</th>\n",
       "      <th>imp_op_var39_comer_ult3</th>\n",
       "      <th>imp_op_var40_comer_ult1</th>\n",
       "      <th>imp_op_var40_comer_ult3</th>\n",
       "      <th>imp_op_var40_efect_ult1</th>\n",
       "      <th>imp_op_var40_efect_ult3</th>\n",
       "      <th>...</th>\n",
       "      <th>saldo_medio_var33_hace2</th>\n",
       "      <th>saldo_medio_var33_hace3</th>\n",
       "      <th>saldo_medio_var33_ult1</th>\n",
       "      <th>saldo_medio_var33_ult3</th>\n",
       "      <th>saldo_medio_var44_hace2</th>\n",
       "      <th>saldo_medio_var44_hace3</th>\n",
       "      <th>saldo_medio_var44_ult1</th>\n",
       "      <th>saldo_medio_var44_ult3</th>\n",
       "      <th>var38</th>\n",
       "      <th>TARGET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>39205.170000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49278.030000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67333.770000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>37</td>\n",
       "      <td>0.0</td>\n",
       "      <td>195.0</td>\n",
       "      <td>195.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64007.970000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>117310.979016</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 371 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  var3  var15  imp_ent_var16_ult1  imp_op_var39_comer_ult1  \\\n",
       "0   1     2     23                 0.0                      0.0   \n",
       "1   3     2     34                 0.0                      0.0   \n",
       "2   4     2     23                 0.0                      0.0   \n",
       "3   8     2     37                 0.0                    195.0   \n",
       "4  10     2     39                 0.0                      0.0   \n",
       "\n",
       "   imp_op_var39_comer_ult3  imp_op_var40_comer_ult1  imp_op_var40_comer_ult3  \\\n",
       "0                      0.0                      0.0                      0.0   \n",
       "1                      0.0                      0.0                      0.0   \n",
       "2                      0.0                      0.0                      0.0   \n",
       "3                    195.0                      0.0                      0.0   \n",
       "4                      0.0                      0.0                      0.0   \n",
       "\n",
       "   imp_op_var40_efect_ult1  imp_op_var40_efect_ult3  ...  \\\n",
       "0                      0.0                      0.0  ...   \n",
       "1                      0.0                      0.0  ...   \n",
       "2                      0.0                      0.0  ...   \n",
       "3                      0.0                      0.0  ...   \n",
       "4                      0.0                      0.0  ...   \n",
       "\n",
       "   saldo_medio_var33_hace2  saldo_medio_var33_hace3  saldo_medio_var33_ult1  \\\n",
       "0                      0.0                      0.0                     0.0   \n",
       "1                      0.0                      0.0                     0.0   \n",
       "2                      0.0                      0.0                     0.0   \n",
       "3                      0.0                      0.0                     0.0   \n",
       "4                      0.0                      0.0                     0.0   \n",
       "\n",
       "   saldo_medio_var33_ult3  saldo_medio_var44_hace2  saldo_medio_var44_hace3  \\\n",
       "0                     0.0                      0.0                      0.0   \n",
       "1                     0.0                      0.0                      0.0   \n",
       "2                     0.0                      0.0                      0.0   \n",
       "3                     0.0                      0.0                      0.0   \n",
       "4                     0.0                      0.0                      0.0   \n",
       "\n",
       "   saldo_medio_var44_ult1  saldo_medio_var44_ult3          var38  TARGET  \n",
       "0                     0.0                     0.0   39205.170000       0  \n",
       "1                     0.0                     0.0   49278.030000       0  \n",
       "2                     0.0                     0.0   67333.770000       0  \n",
       "3                     0.0                     0.0   64007.970000       0  \n",
       "4                     0.0                     0.0  117310.979016       0  \n",
       "\n",
       "[5 rows x 371 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('santander-train.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.drop(labels=['TARGET'], axis=1)\n",
    "y = data['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(76020, 370)\n",
      "(76020,)\n"
     ]
    }
   ],
   "source": [
    "print(X.shape)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Removing Constants, Quasi Constants and Duplicate Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VarianceThreshold(threshold=0.01)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "constant_filter = VarianceThreshold(threshold=0.01)\n",
    "constant_filter.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_filter = constant_filter.transform(X_train)\n",
    "X_test_filter = constant_filter.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(57015, 266)\n",
      "(19005, 266)\n"
     ]
    }
   ],
   "source": [
    "print(X_train_filter.shape)\n",
    "print(X_test_filter.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_T = X_train_filter.T\n",
    "X_test_T = X_test_filter.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "print(type(X_train_T))\n",
    "print(type(X_test_T))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_T = pd.DataFrame(X_train_T)\n",
    "X_test_T = pd.DataFrame(X_test_T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_T.duplicated().sum()  ## Number of duplicate Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "duplicated_features = X_train_T.duplicated()\n",
    "features_to_keep = [not index for index in duplicated_features]\n",
    "X_train_unique = X_train_T[features_to_keep].T\n",
    "X_test_unique = X_test_T[features_to_keep].T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(57015, 249)\n",
      "(19005, 249)\n"
     ]
    }
   ],
   "source": [
    "print(X_train_unique.shape)\n",
    "print(X_test_unique.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tuple'>\n"
     ]
    }
   ],
   "source": [
    "### Applying f_classif returns a numpy array having F-score and p-value(if p-values are less than 0.05 them fearures are important) \n",
    "sel = f_classif(X_train_unique, y_train)\n",
    "print(type(sel))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2.87445436e+00, 6.61551963e-01, 5.69066991e+02, 8.45840584e-02,\n",
       "        2.98673532e+00, 1.80142083e-01, 3.11060896e-01, 9.16203203e-02,\n",
       "        1.96847973e+01, 2.06837460e+01, 1.17844827e-01, 2.78692450e+00,\n",
       "        2.97153529e-01, 6.24094678e+01, 2.72882366e+01, 4.72952767e+01,\n",
       "        6.58597969e+01, 2.89970388e+01, 4.44738929e+01, 1.62339577e-03,\n",
       "        3.48002488e-03, 6.99224120e+01, 1.12816058e+03, 1.35000746e+02,\n",
       "        4.64759690e+01, 7.83118930e+01, 6.24878127e+01, 8.14105190e+01,\n",
       "        6.04505747e+01, 6.18504997e+01, 8.31363281e+01, 2.00300671e+01,\n",
       "        5.00157370e+01, 5.65610322e+01, 3.52431627e+01, 2.47750627e+01,\n",
       "        3.50171071e+01, 2.59684586e+01, 1.36259980e+03, 1.75909706e-01,\n",
       "        5.81971279e-01, 6.63406360e+01, 4.48999028e-03, 6.28529059e+01,\n",
       "        6.20711799e-02, 5.10970458e+00, 3.84053909e+02, 5.18016481e+01,\n",
       "        1.10885914e+03, 1.34670129e+02, 4.64759690e+01, 2.56281679e+01,\n",
       "        6.22875200e+01, 7.65039021e+01, 6.01925481e+01, 6.18259115e+01,\n",
       "        1.76951125e+01, 1.75614089e+01, 7.87351151e+01, 6.89579720e-02,\n",
       "        2.04969799e+00, 9.56672199e-02, 1.03743670e-02, 8.38423522e+00,\n",
       "        5.98846834e+00, 4.99453299e+01, 5.65486133e+01, 2.47746369e+01,\n",
       "        2.72654199e+01, 3.81986506e-01, 1.02305864e-01, 1.92805375e-01,\n",
       "        3.03256954e-01, 4.33728026e+00, 2.33318494e-01, 7.89394009e+00,\n",
       "        7.14822494e+00, 3.90435120e+00, 2.82273420e-01, 7.10723985e+00,\n",
       "        6.40798895e+00, 5.07918232e+01, 1.15597335e+03, 1.07139273e+00,\n",
       "        8.11009506e-01, 2.49934635e-01, 1.26308799e+00, 3.51453509e+02,\n",
       "        6.72818865e-03, 3.30953301e-01, 4.49856908e+01, 6.61924560e-02,\n",
       "        5.19765005e+00, 4.77563800e+01, 1.06382700e+01, 1.11398912e+03,\n",
       "        2.62454008e+00, 2.23978034e+00, 1.99224813e+00, 1.08155660e+01,\n",
       "        7.89968296e-02, 9.04259888e-01, 2.67860250e+01, 3.66289460e+01,\n",
       "        1.23908861e+01, 4.17687438e-02, 4.89930057e+01, 1.20999209e+00,\n",
       "        2.47439148e-02, 2.73901864e-01, 2.59642792e+01, 2.67317807e+00,\n",
       "        2.71008180e+00, 7.70481887e+01, 7.03148596e-02, 9.15354659e-03,\n",
       "        5.22676006e-01, 4.17687438e-02, 3.43336919e+00, 6.22881632e+00,\n",
       "        3.34007969e+01, 2.36145416e-01, 6.32685194e+02, 4.17687438e-02,\n",
       "        2.42795484e+00, 4.94912206e-02, 7.20218362e-01, 1.80262489e-03,\n",
       "        2.08743910e-01, 4.17687438e-02, 1.25310811e-01, 8.35390140e-02,\n",
       "        1.67084134e-01, 4.17687438e-02, 1.46281481e+00, 2.42795484e+00,\n",
       "        4.94912206e-02, 7.20218362e-01, 1.46281481e+00, 4.17687438e-02,\n",
       "        2.42368479e+01, 4.31242537e+00, 5.61759024e-02, 3.18273155e-01,\n",
       "        3.65396551e-01, 1.30266888e-01, 7.65548219e-02, 4.77611562e-01,\n",
       "        3.70847184e-01, 1.06779395e-01, 4.15560052e-01, 6.76364389e+00,\n",
       "        4.17687438e-02, 5.55590932e-01, 5.42353914e+00, 7.40428109e-02,\n",
       "        7.50103946e-02, 4.55923569e-02, 1.53574879e-01, 9.04249832e-02,\n",
       "        4.17687438e-02, 4.17935356e-02, 1.57695623e-01, 6.86812166e+00,\n",
       "        4.66376522e+00, 2.78197042e+00, 4.39263782e+00, 7.20518466e+00,\n",
       "        2.68866534e+01, 8.87335331e-01, 3.73668113e+01, 2.77759433e+00,\n",
       "        5.24401940e-01, 1.94583501e+00, 3.20432956e-01, 9.40676027e-01,\n",
       "        4.30537468e+00, 7.52064849e+00, 6.69521087e-01, 2.49804524e+01,\n",
       "        7.87499760e+00, 1.36211617e+01, 2.25187875e+00, 1.32922873e+03,\n",
       "        4.19653701e+01, 5.70723274e+01, 6.20433818e+01, 1.56562279e+01,\n",
       "        5.31144778e+00, 1.08980663e+00, 6.83033765e-01, 2.48317419e-01,\n",
       "        5.08404559e-01, 2.01578597e+01, 1.67660526e+01, 1.43190494e+00,\n",
       "        1.09979832e+00, 3.36879597e+01, 2.60105917e+01, 3.59767497e+01,\n",
       "        2.75875741e+01, 1.18267210e+01, 3.78378849e+00, 1.08862805e+00,\n",
       "        1.58041225e+01, 6.13620599e+00, 7.21702036e-01, 8.85088369e-01,\n",
       "        6.45554551e-01, 5.82154240e+00, 1.48438586e+00, 2.22412217e+01,\n",
       "        1.29031049e+01, 1.35857485e+01, 1.92126918e+01, 1.55238561e+00,\n",
       "        5.14150442e-01, 7.52999141e-01, 6.40223409e-01, 1.68347399e+01,\n",
       "        7.51138275e+00, 2.43563539e+01, 2.46829302e+01, 2.88758809e+01,\n",
       "        1.05128900e+01, 3.60147822e+01, 3.37906270e+01, 8.15207217e+00,\n",
       "        2.70992960e+00, 8.98997310e+00, 8.95396963e+00, 4.17687438e-02,\n",
       "        4.17687438e-02, 8.62896709e-02, 5.25256112e-02, 9.97084208e-03,\n",
       "        9.04099815e-04, 8.18168572e-02, 7.54759239e-02, 1.08823883e-01,\n",
       "        6.44341357e-01, 3.11731333e-01, 5.10842882e-01, 5.65368390e-01,\n",
       "        1.07258527e+00, 3.24056433e-01, 6.05883880e-01, 4.91466062e-01,\n",
       "        1.89277032e+01]),\n",
       " array([9.00009337e-002, 4.16015973e-001, 3.69026568e-125, 7.71179808e-001,\n",
       "        8.39546731e-002, 6.71252767e-001, 5.77032096e-001, 7.62128377e-001,\n",
       "        9.14957990e-006, 5.42852930e-006, 7.31385072e-001, 9.50419544e-002,\n",
       "        5.85674562e-001, 2.83935108e-015, 1.75889041e-007, 6.16860216e-012,\n",
       "        4.93740408e-016, 7.27749421e-008, 2.60116459e-011, 9.67860962e-001,\n",
       "        9.52958897e-001, 6.30575816e-017, 6.23639753e-245, 3.57880015e-031,\n",
       "        9.36689441e-012, 9.04483083e-019, 2.72873461e-015, 1.88915635e-019,\n",
       "        7.67075322e-015, 3.77014891e-015, 7.89967423e-020, 7.63813985e-006,\n",
       "        1.54266132e-012, 5.52771915e-014, 2.92679938e-009, 6.46130132e-007,\n",
       "        3.28682354e-009, 3.48146781e-007, 8.61444362e-295, 6.74913779e-001,\n",
       "        4.45543835e-001, 3.86966559e-016, 9.46576055e-001, 2.26747869e-015,\n",
       "        8.03252703e-001, 2.37961259e-002, 3.11504034e-085, 6.21545315e-013,\n",
       "        8.11360158e-241, 4.22555952e-031, 9.36689441e-012, 4.15237914e-007,\n",
       "        3.02052472e-015, 2.25636963e-018, 8.74395471e-015, 3.81746996e-015,\n",
       "        2.59687670e-005, 2.78595155e-005, 7.30272010e-019, 7.92861005e-001,\n",
       "        1.52241868e-001, 7.57093564e-001, 9.18872416e-001, 3.78633138e-003,\n",
       "        1.44026967e-002, 1.59897011e-012, 5.56270594e-014, 6.46272820e-007,\n",
       "        1.77975891e-007, 5.36544473e-001, 7.49081032e-001, 6.60593578e-001,\n",
       "        5.81849928e-001, 3.72908863e-002, 6.29075913e-001, 4.96175817e-003,\n",
       "        7.50595917e-003, 4.81660334e-002, 5.95217064e-001, 7.67948763e-003,\n",
       "        1.13634654e-002, 1.03914221e-012, 7.39429384e-251, 3.00635463e-001,\n",
       "        3.67825762e-001, 6.17123036e-001, 2.61072116e-001, 3.51735521e-078,\n",
       "        9.34626695e-001, 5.65100559e-001, 2.00325934e-011, 7.96964373e-001,\n",
       "        2.26211209e-002, 4.87661343e-012, 1.10836294e-003, 6.53991600e-242,\n",
       "        1.05228521e-001, 1.34505621e-001, 1.58111473e-001, 1.00711187e-003,\n",
       "        7.78662527e-001, 3.41646072e-001, 2.28043734e-007, 1.43782472e-009,\n",
       "        4.31770768e-004, 8.38061953e-001, 2.59703819e-012, 2.71338334e-001,\n",
       "        8.75007412e-001, 6.00728332e-001, 3.48900952e-007, 1.02057577e-001,\n",
       "        9.97213899e-002, 1.71348074e-018, 7.90879854e-001, 9.23779693e-001,\n",
       "        4.69705367e-001, 8.38061953e-001, 6.38967978e-002, 1.25715755e-002,\n",
       "        7.53826525e-009, 6.27005934e-001, 7.48030932e-139, 8.38061953e-001,\n",
       "        1.19193859e-001, 8.23951761e-001, 3.96075855e-001, 9.66134275e-001,\n",
       "        6.47755442e-001, 8.38061953e-001, 7.23345685e-001, 7.72558580e-001,\n",
       "        6.82717874e-001, 8.38061953e-001, 2.26487800e-001, 1.19193859e-001,\n",
       "        8.23951761e-001, 3.96075855e-001, 2.26487800e-001, 8.38061953e-001,\n",
       "        8.54243201e-007, 3.78394513e-002, 8.12646345e-001, 5.72649487e-001,\n",
       "        5.45526704e-001, 7.18156818e-001, 7.82022671e-001, 4.89508991e-001,\n",
       "        5.42545268e-001, 7.43842169e-001, 5.19163441e-001, 9.30576038e-003,\n",
       "        8.38061953e-001, 4.56045271e-001, 1.98705261e-002, 7.85539897e-001,\n",
       "        7.84177637e-001, 8.30919294e-001, 6.95143684e-001, 7.63638668e-001,\n",
       "        8.38061953e-001, 8.38014567e-001, 6.91288509e-001, 8.77701036e-003,\n",
       "        3.08097227e-002, 9.53363094e-002, 3.60988260e-002, 7.27142713e-003,\n",
       "        2.16478652e-007, 3.46204422e-001, 9.85062967e-010, 9.55971474e-002,\n",
       "        4.68972928e-001, 1.63041810e-001, 5.71349798e-001, 3.32108160e-001,\n",
       "        3.79965999e-002, 6.10147482e-003, 4.13222047e-001, 5.80856862e-007,\n",
       "        5.01399283e-003, 2.23858382e-004, 1.33457829e-001, 1.04662790e-287,\n",
       "        9.36582083e-011, 4.26325572e-014, 3.41873156e-015, 7.60539130e-005,\n",
       "        2.11893187e-002, 2.96518483e-001, 4.08547470e-001, 6.18264257e-001,\n",
       "        4.75833245e-001, 7.14461805e-006, 4.23412651e-005, 2.31458778e-001,\n",
       "        2.94314815e-001, 6.50415369e-009, 3.40634451e-007, 2.00884792e-009,\n",
       "        1.50677865e-007, 5.84284512e-004, 5.17562709e-002, 2.96779812e-001,\n",
       "        7.03357917e-005, 1.32469949e-002, 3.95589745e-001, 3.46815779e-001,\n",
       "        4.21709892e-001, 1.58341912e-002, 2.23094614e-001, 2.41023810e-006,\n",
       "        3.28313314e-004, 2.28120287e-004, 1.17142124e-005, 2.12788725e-001,\n",
       "        4.73350598e-001, 3.85531996e-001, 4.23633243e-001, 4.08365205e-005,\n",
       "        6.13293725e-003, 8.02875879e-007, 6.77752633e-007, 7.74697706e-008,\n",
       "        1.18612819e-003, 1.97004255e-009, 6.16998418e-009, 4.30275053e-003,\n",
       "        9.97309043e-002, 2.71582014e-003, 2.76985776e-003, 8.38061953e-001,\n",
       "        8.38061953e-001, 7.68949222e-001, 8.18726212e-001, 9.20460505e-001,\n",
       "        9.76012725e-001, 7.74851235e-001, 7.83525584e-001, 7.41488333e-001,\n",
       "        4.22146433e-001, 5.76621901e-001, 4.74777133e-001, 4.52108391e-001,\n",
       "        3.00366617e-001, 5.69182257e-001, 4.36345055e-001, 4.83276917e-001,\n",
       "        1.36002301e-005]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[9.00009337e-002 4.16015973e-001 3.69026568e-125 7.71179808e-001\n",
      " 8.39546731e-002 6.71252767e-001 5.77032096e-001 7.62128377e-001\n",
      " 9.14957990e-006 5.42852930e-006 7.31385072e-001 9.50419544e-002\n",
      " 5.85674562e-001 2.83935108e-015 1.75889041e-007 6.16860216e-012\n",
      " 4.93740408e-016 7.27749421e-008 2.60116459e-011 9.67860962e-001\n",
      " 9.52958897e-001 6.30575816e-017 6.23639753e-245 3.57880015e-031\n",
      " 9.36689441e-012 9.04483083e-019 2.72873461e-015 1.88915635e-019\n",
      " 7.67075322e-015 3.77014891e-015 7.89967423e-020 7.63813985e-006\n",
      " 1.54266132e-012 5.52771915e-014 2.92679938e-009 6.46130132e-007\n",
      " 3.28682354e-009 3.48146781e-007 8.61444362e-295 6.74913779e-001\n",
      " 4.45543835e-001 3.86966559e-016 9.46576055e-001 2.26747869e-015\n",
      " 8.03252703e-001 2.37961259e-002 3.11504034e-085 6.21545315e-013\n",
      " 8.11360158e-241 4.22555952e-031 9.36689441e-012 4.15237914e-007\n",
      " 3.02052472e-015 2.25636963e-018 8.74395471e-015 3.81746996e-015\n",
      " 2.59687670e-005 2.78595155e-005 7.30272010e-019 7.92861005e-001\n",
      " 1.52241868e-001 7.57093564e-001 9.18872416e-001 3.78633138e-003\n",
      " 1.44026967e-002 1.59897011e-012 5.56270594e-014 6.46272820e-007\n",
      " 1.77975891e-007 5.36544473e-001 7.49081032e-001 6.60593578e-001\n",
      " 5.81849928e-001 3.72908863e-002 6.29075913e-001 4.96175817e-003\n",
      " 7.50595917e-003 4.81660334e-002 5.95217064e-001 7.67948763e-003\n",
      " 1.13634654e-002 1.03914221e-012 7.39429384e-251 3.00635463e-001\n",
      " 3.67825762e-001 6.17123036e-001 2.61072116e-001 3.51735521e-078\n",
      " 9.34626695e-001 5.65100559e-001 2.00325934e-011 7.96964373e-001\n",
      " 2.26211209e-002 4.87661343e-012 1.10836294e-003 6.53991600e-242\n",
      " 1.05228521e-001 1.34505621e-001 1.58111473e-001 1.00711187e-003\n",
      " 7.78662527e-001 3.41646072e-001 2.28043734e-007 1.43782472e-009\n",
      " 4.31770768e-004 8.38061953e-001 2.59703819e-012 2.71338334e-001\n",
      " 8.75007412e-001 6.00728332e-001 3.48900952e-007 1.02057577e-001\n",
      " 9.97213899e-002 1.71348074e-018 7.90879854e-001 9.23779693e-001\n",
      " 4.69705367e-001 8.38061953e-001 6.38967978e-002 1.25715755e-002\n",
      " 7.53826525e-009 6.27005934e-001 7.48030932e-139 8.38061953e-001\n",
      " 1.19193859e-001 8.23951761e-001 3.96075855e-001 9.66134275e-001\n",
      " 6.47755442e-001 8.38061953e-001 7.23345685e-001 7.72558580e-001\n",
      " 6.82717874e-001 8.38061953e-001 2.26487800e-001 1.19193859e-001\n",
      " 8.23951761e-001 3.96075855e-001 2.26487800e-001 8.38061953e-001\n",
      " 8.54243201e-007 3.78394513e-002 8.12646345e-001 5.72649487e-001\n",
      " 5.45526704e-001 7.18156818e-001 7.82022671e-001 4.89508991e-001\n",
      " 5.42545268e-001 7.43842169e-001 5.19163441e-001 9.30576038e-003\n",
      " 8.38061953e-001 4.56045271e-001 1.98705261e-002 7.85539897e-001\n",
      " 7.84177637e-001 8.30919294e-001 6.95143684e-001 7.63638668e-001\n",
      " 8.38061953e-001 8.38014567e-001 6.91288509e-001 8.77701036e-003\n",
      " 3.08097227e-002 9.53363094e-002 3.60988260e-002 7.27142713e-003\n",
      " 2.16478652e-007 3.46204422e-001 9.85062967e-010 9.55971474e-002\n",
      " 4.68972928e-001 1.63041810e-001 5.71349798e-001 3.32108160e-001\n",
      " 3.79965999e-002 6.10147482e-003 4.13222047e-001 5.80856862e-007\n",
      " 5.01399283e-003 2.23858382e-004 1.33457829e-001 1.04662790e-287\n",
      " 9.36582083e-011 4.26325572e-014 3.41873156e-015 7.60539130e-005\n",
      " 2.11893187e-002 2.96518483e-001 4.08547470e-001 6.18264257e-001\n",
      " 4.75833245e-001 7.14461805e-006 4.23412651e-005 2.31458778e-001\n",
      " 2.94314815e-001 6.50415369e-009 3.40634451e-007 2.00884792e-009\n",
      " 1.50677865e-007 5.84284512e-004 5.17562709e-002 2.96779812e-001\n",
      " 7.03357917e-005 1.32469949e-002 3.95589745e-001 3.46815779e-001\n",
      " 4.21709892e-001 1.58341912e-002 2.23094614e-001 2.41023810e-006\n",
      " 3.28313314e-004 2.28120287e-004 1.17142124e-005 2.12788725e-001\n",
      " 4.73350598e-001 3.85531996e-001 4.23633243e-001 4.08365205e-005\n",
      " 6.13293725e-003 8.02875879e-007 6.77752633e-007 7.74697706e-008\n",
      " 1.18612819e-003 1.97004255e-009 6.16998418e-009 4.30275053e-003\n",
      " 9.97309043e-002 2.71582014e-003 2.76985776e-003 8.38061953e-001\n",
      " 8.38061953e-001 7.68949222e-001 8.18726212e-001 9.20460505e-001\n",
      " 9.76012725e-001 7.74851235e-001 7.83525584e-001 7.41488333e-001\n",
      " 4.22146433e-001 5.76621901e-001 4.74777133e-001 4.52108391e-001\n",
      " 3.00366617e-001 5.69182257e-001 4.36345055e-001 4.83276917e-001\n",
      " 1.36002301e-005]\n"
     ]
    }
   ],
   "source": [
    "### Second array has p-values\n",
    "p_values = sel[1]\n",
    "print(p_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "249\n"
     ]
    }
   ],
   "source": [
    "print(len(p_values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_values = pd.Series(p_values, index=X_train_unique.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_values.sort_values(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "p_values.plot.bar(figsize=(12, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_values = p_values[p_values<0.05]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(p_values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_p = X_train_unique[p_values.index]\n",
    "X_test_p = X_test_unique[p_values.index]\n",
    "print(X_train_p.shape)\n",
    "print(X_test_p.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "model = RandomForestClassifier(n_estimators=1000, max_depth= 4)\n",
    "model.fit(X_train_p, y_train)\n",
    "y_pred = model.predict(X_test_p)\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "model = RandomForestClassifier(n_estimators=1000, max_depth= 4)\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Similarly we can do for Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}