galicia1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['device_data.csv', 'PAGE.csv', 'CONTENT_CATEGORY.csv', 'conversiones.csv', 'sampleSubmission.csv', 'pageviews_complemento.csv', 'SITE_ID.csv', 'CONTENT_CATEGORY_BOTTOM.csv', 'CONTENT_CATEGORY_TOP.csv', 'README.md', 'pageviews.csv']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sys\n",
    "np.set_printoptions(threshold=20)\n",
    "pd.set_option('display.max_columns', 20)\n",
    "pd.set_option('display.max_rows', 20)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"./data/\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "data = pd.concat([\n",
    "       pd.read_csv(\"./data/pageviews.csv\", parse_dates=[\"FEC_EVENT\"]),\n",
    "       pd.read_csv(\"./data/pageviews_complemento.csv\", parse_dates=[\"FEC_EVENT\"])\n",
    "], sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FEC_EVENT</th>\n",
       "      <th>PAGE</th>\n",
       "      <th>CONTENT_CATEGORY</th>\n",
       "      <th>CONTENT_CATEGORY_TOP</th>\n",
       "      <th>CONTENT_CATEGORY_BOTTOM</th>\n",
       "      <th>SITE_ID</th>\n",
       "      <th>ON_SITE_SEARCH_TERM</th>\n",
       "      <th>USER_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2018-03-30 07:35:48</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2018-03-30 07:35:52</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018-03-30 07:36:11</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018-03-30 07:36:16</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018-03-30 07:41:38</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            FEC_EVENT  PAGE  CONTENT_CATEGORY  CONTENT_CATEGORY_TOP  \\\n",
       "0 2018-03-30 07:35:48     1                 1                     1   \n",
       "1 2018-03-30 07:35:52     2                 2                     2   \n",
       "2 2018-03-30 07:36:11     3                 2                     2   \n",
       "3 2018-03-30 07:36:16     4                 2                     2   \n",
       "4 2018-03-30 07:41:38     5                 2                     2   \n",
       "\n",
       "   CONTENT_CATEGORY_BOTTOM  SITE_ID  ON_SITE_SEARCH_TERM  USER_ID  \n",
       "0                        1        1                    1        0  \n",
       "1                        2        2                    1        0  \n",
       "2                        2        3                    1        0  \n",
       "3                        2        3                    1        0  \n",
       "4                        2        2                    1        0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benchmark Kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "haciendo PAGE\n",
      "haciendo CONTENT_CATEGORY\n",
      "haciendo CONTENT_CATEGORY_TOP\n",
      "haciendo CONTENT_CATEGORY_BOTTOM\n",
      "haciendo SITE_ID\n",
      "haciendo ON_SITE_SEARCH_TERM\n"
     ]
    }
   ],
   "source": [
    "X_test = []\n",
    "for c in data.drop([\"USER_ID\", \"FEC_EVENT\"], axis=1).columns:\n",
    "    print(\"haciendo\", c)\n",
    "    temp = pd.crosstab(data.USER_ID, data[c])\n",
    "    temp.columns = [c + \"_\" + str(v) for v in temp.columns]\n",
    "    X_test.append(temp.apply(lambda x: x / x.sum(), axis=1))\n",
    "X_test = pd.concat(X_test, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PAGE_1</th>\n",
       "      <th>PAGE_2</th>\n",
       "      <th>PAGE_3</th>\n",
       "      <th>PAGE_4</th>\n",
       "      <th>PAGE_5</th>\n",
       "      <th>PAGE_6</th>\n",
       "      <th>PAGE_7</th>\n",
       "      <th>PAGE_8</th>\n",
       "      <th>PAGE_9</th>\n",
       "      <th>PAGE_10</th>\n",
       "      <th>...</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_285</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_286</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_287</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_288</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_289</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_290</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_291</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_292</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_293</th>\n",
       "      <th>ON_SITE_SEARCH_TERM_295</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>USER_ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.028300</td>\n",
       "      <td>0.128440</td>\n",
       "      <td>0.064842</td>\n",
       "      <td>0.043384</td>\n",
       "      <td>0.016949</td>\n",
       "      <td>0.005287</td>\n",
       "      <td>0.000155</td>\n",
       "      <td>0.000155</td>\n",
       "      <td>0.044472</td>\n",
       "      <td>0.114601</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.103742</td>\n",
       "      <td>0.157271</td>\n",
       "      <td>0.065846</td>\n",
       "      <td>0.013738</td>\n",
       "      <td>0.006158</td>\n",
       "      <td>0.003316</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.054950</td>\n",
       "      <td>0.128375</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.032517</td>\n",
       "      <td>0.168074</td>\n",
       "      <td>0.114443</td>\n",
       "      <td>0.025760</td>\n",
       "      <td>0.026605</td>\n",
       "      <td>0.002956</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000422</td>\n",
       "      <td>0.027872</td>\n",
       "      <td>0.059966</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.005168</td>\n",
       "      <td>0.192765</td>\n",
       "      <td>0.145736</td>\n",
       "      <td>0.034625</td>\n",
       "      <td>0.040827</td>\n",
       "      <td>0.012920</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.010628</td>\n",
       "      <td>0.120451</td>\n",
       "      <td>0.099839</td>\n",
       "      <td>0.008696</td>\n",
       "      <td>0.046699</td>\n",
       "      <td>0.018035</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000322</td>\n",
       "      <td>0.000966</td>\n",
       "      <td>0.176490</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2174 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           PAGE_1    PAGE_2    PAGE_3    PAGE_4    PAGE_5    PAGE_6    PAGE_7  \\\n",
       "USER_ID                                                                         \n",
       "0        0.028300  0.128440  0.064842  0.043384  0.016949  0.005287  0.000155   \n",
       "1        0.103742  0.157271  0.065846  0.013738  0.006158  0.003316  0.000000   \n",
       "2        0.032517  0.168074  0.114443  0.025760  0.026605  0.002956  0.000000   \n",
       "3        0.005168  0.192765  0.145736  0.034625  0.040827  0.012920  0.000000   \n",
       "4        0.010628  0.120451  0.099839  0.008696  0.046699  0.018035  0.000000   \n",
       "\n",
       "           PAGE_8    PAGE_9   PAGE_10  ...  ON_SITE_SEARCH_TERM_285  \\\n",
       "USER_ID                                ...                            \n",
       "0        0.000155  0.044472  0.114601  ...                      0.0   \n",
       "1        0.000000  0.054950  0.128375  ...                      0.0   \n",
       "2        0.000422  0.027872  0.059966  ...                      0.0   \n",
       "3        0.000000  0.000000  0.000000  ...                      0.0   \n",
       "4        0.000322  0.000966  0.176490  ...                      0.0   \n",
       "\n",
       "         ON_SITE_SEARCH_TERM_286  ON_SITE_SEARCH_TERM_287  \\\n",
       "USER_ID                                                     \n",
       "0                            0.0                      0.0   \n",
       "1                            0.0                      0.0   \n",
       "2                            0.0                      0.0   \n",
       "3                            0.0                      0.0   \n",
       "4                            0.0                      0.0   \n",
       "\n",
       "         ON_SITE_SEARCH_TERM_288  ON_SITE_SEARCH_TERM_289  \\\n",
       "USER_ID                                                     \n",
       "0                            0.0                      0.0   \n",
       "1                            0.0                      0.0   \n",
       "2                            0.0                      0.0   \n",
       "3                            0.0                      0.0   \n",
       "4                            0.0                      0.0   \n",
       "\n",
       "         ON_SITE_SEARCH_TERM_290  ON_SITE_SEARCH_TERM_291  \\\n",
       "USER_ID                                                     \n",
       "0                            0.0                      0.0   \n",
       "1                            0.0                      0.0   \n",
       "2                            0.0                      0.0   \n",
       "3                            0.0                      0.0   \n",
       "4                            0.0                      0.0   \n",
       "\n",
       "         ON_SITE_SEARCH_TERM_292  ON_SITE_SEARCH_TERM_293  \\\n",
       "USER_ID                                                     \n",
       "0                            0.0                      0.0   \n",
       "1                            0.0                      0.0   \n",
       "2                            0.0                      0.0   \n",
       "3                            0.0                      0.0   \n",
       "4                            0.0                      0.0   \n",
       "\n",
       "         ON_SITE_SEARCH_TERM_295  \n",
       "USER_ID                           \n",
       "0                            0.0  \n",
       "1                            0.0  \n",
       "2                            0.0  \n",
       "3                            0.0  \n",
       "4                            0.0  \n",
       "\n",
       "[5 rows x 2174 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11676, 2174)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9586378479241177"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.iloc[0,0:65].sum() # la suma de las frecuencias debe ser 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "haciendo PAGE\n",
      "haciendo CONTENT_CATEGORY\n",
      "haciendo CONTENT_CATEGORY_TOP\n",
      "haciendo CONTENT_CATEGORY_BOTTOM\n",
      "haciendo SITE_ID\n",
      "haciendo ON_SITE_SEARCH_TERM\n"
     ]
    }
   ],
   "source": [
    "data = data[data.FEC_EVENT.dt.month < 10]\n",
    "X_train = []\n",
    "for c in data.drop([\"USER_ID\", \"FEC_EVENT\"], axis=1).columns:\n",
    "    print(\"haciendo\", c)\n",
    "    temp = pd.crosstab(data.USER_ID, data[c])\n",
    "    temp.columns = [c + \"_\" + str(v) for v in temp.columns]\n",
    "    X_train.append(temp.apply(lambda x: x / x.sum(), axis=1))\n",
    "X_train = pd.concat(X_train, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "USER_ID\n",
       "0        0.001221\n",
       "1        0.001224\n",
       "2        0.005242\n",
       "3        0.007722\n",
       "4        0.020178\n",
       "           ...   \n",
       "11671    0.034161\n",
       "11672    0.000000\n",
       "11673    0.016129\n",
       "11674    0.000000\n",
       "11675    0.000000\n",
       "Name: PAGE_57, Length: 11529, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.iloc[:,56]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9723127035830619"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.iloc[0,0:65].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = list(set(X_train.columns).intersection(set(X_test.columns)))\n",
    "X_train = X_train[features]\n",
    "X_test = X_test[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prev = pd.read_csv(\"./data/conversiones.csv\")\n",
    "y_train = pd.Series(0, index=X_train.index)\n",
    "idx = set(y_prev[y_prev.mes >= 10].USER_ID.unique()).intersection(set(X_train.index))\n",
    "y_train.loc[list(idx)] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11529, 2026) (11676, 2026)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import model_selection\n",
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "fi = [] # objeto vacío para guardar la importancia de las variables de los modelos entrenados\n",
    "test_probs = [] # objeto vacío para guardar las probabilidades estimadas\n",
    "i = 0 \n",
    "for train_idx, valid_idx in model_selection.KFold(n_splits=10, shuffle=True).split(X_train):\n",
    "    i += 1\n",
    "    Xt = X_train.iloc[train_idx]\n",
    "    yt = y_train.loc[X_train.index].iloc[train_idx]\n",
    "\n",
    "    Xv = X_train.iloc[valid_idx]\n",
    "    yv = y_train.loc[X_train.index].iloc[valid_idx]\n",
    "\n",
    "    learner = LGBMClassifier(n_estimators=10000) # lightgbm with max 10.000 trees\n",
    "    learner.fit(Xt, yt,  early_stopping_rounds=10, eval_metric=\"auc\",\n",
    "                eval_set=[(Xt, yt), (Xv, yv)],verbose=False)\n",
    "    \n",
    "    test_probs.append(pd.Series(learner.predict_proba(X_test)[:, -1],\n",
    "                                index=X_test.index, name=\"fold_\" + str(i)))\n",
    "    fi.append(pd.Series(learner.feature_importances_ / learner.feature_importances_.sum(), index=Xt.columns))\n",
    "\n",
    "test_probs = pd.concat(test_probs, axis=1).mean(axis=1) # mean of probabilities\n",
    "test_probs.index.name=\"USER_ID\"\n",
    "test_probs.name=\"SCORE\"\n",
    "test_probs.to_csv(\"respuesta\", header=True, compression=\"zip\")\n",
    "fi = pd.concat(fi, axis=1).mean(axis=1) # Variables as explanation of model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lado B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experimental models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train set and Test set\n",
    "\n",
    "We split the test set, so we use last three months to test (Given that original test set is not given to competitors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "haciendo PAGE\n",
      "haciendo CONTENT_CATEGORY\n",
      "haciendo CONTENT_CATEGORY_TOP\n",
      "haciendo CONTENT_CATEGORY_BOTTOM\n",
      "haciendo SITE_ID\n",
      "haciendo ON_SITE_SEARCH_TERM\n"
     ]
    }
   ],
   "source": [
    "data = data[data.FEC_EVENT.dt.month < 10]\n",
    "X_test2 = []\n",
    "for c in data.drop([\"USER_ID\", \"FEC_EVENT\"], axis=1).columns:\n",
    "    print(\"haciendo\", c)\n",
    "    temp = pd.crosstab(data.USER_ID, data[c])\n",
    "    temp.columns = [c + \"_\" + str(v) for v in temp.columns]\n",
    "    X_test2.append(temp.apply(lambda x: x / x.sum(), axis=1))\n",
    "X_test2 = pd.concat(X_test2, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "haciendo PAGE\n",
      "haciendo CONTENT_CATEGORY\n",
      "haciendo CONTENT_CATEGORY_TOP\n",
      "haciendo CONTENT_CATEGORY_BOTTOM\n",
      "haciendo SITE_ID\n",
      "haciendo ON_SITE_SEARCH_TERM\n"
     ]
    }
   ],
   "source": [
    "data = data[(data.FEC_EVENT.dt.month < 7)]\n",
    "X_train2 = []\n",
    "for c in data.drop([\"USER_ID\", \"FEC_EVENT\"], axis=1).columns:\n",
    "    print(\"haciendo\", c)\n",
    "    temp = pd.crosstab(data.USER_ID, data[c])\n",
    "    temp.columns = [c + \"_\" + str(v) for v in temp.columns]\n",
    "    X_train2.append(temp.apply(lambda x: x / x.sum(), axis=1))\n",
    "X_train2 = pd.concat(X_train2, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_prev2 = pd.read_csv(\"./data/conversiones.csv\")\n",
    "y_test2 = pd.Series(0, index=X_test2.index)\n",
    "idx = set(y_prev2[y_prev2.mes >= 10].USER_ID.unique()).intersection(set(X_test2.index))\n",
    "y_test2.loc[list(idx)] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train2 = pd.Series(0, index=X_train2.index)\n",
    "idx = set(y_prev2[(y_prev2.mes < 10) & (y_prev2.mes > 6)].USER_ID.unique()).intersection(set(X_train2.index))\n",
    "y_train2.loc[list(idx)] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11314, 1890) (11314,) (11529, 2026) (11529,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train2.shape, y_train2.shape, X_test2.shape, y_test2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = list(set(X_train2.columns).intersection(set(X_test2.columns)))\n",
    "X_train2 = X_train2[features]\n",
    "X_test2 = X_test2[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11314, 1890)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train2.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Some feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca2 = PCA(0.99,random_state=23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCA(copy=True, iterated_power='auto', n_components=0.99, random_state=23,\n",
       "    svd_solver='auto', tol=0.0, whiten=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca2.fit(X_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_pca2 = pd.DataFrame(pca2.transform(X_train2))\n",
    "X_test_pca2 = pd.DataFrame(pca2.transform(X_test2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11314, 43) (11529, 43)\n"
     ]
    }
   ],
   "source": [
    "print(X_train_pca2.shape, X_test_pca2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For training and validating\n",
    "\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "\n",
    "selected_features = SelectKBest(chi2,k=200)\n",
    "selected_features.fit(X_train2, y_train2)\n",
    "X_train_skb2 = pd.DataFrame(selected_features.transform(X_train2))\n",
    "X_test_skb2 = pd.DataFrame(selected_features.transform(X_test2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>P_VALUES</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FEAT_N</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1239</th>\n",
       "      <td>0.00005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1026</th>\n",
       "      <td>0.00064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.00272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1019</th>\n",
       "      <td>0.00312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1431</th>\n",
       "      <td>0.02693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1462</th>\n",
       "      <td>0.99974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1215</th>\n",
       "      <td>0.99976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1299</th>\n",
       "      <td>0.99981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960</th>\n",
       "      <td>0.99983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>0.99999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1890 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        P_VALUES\n",
       "FEAT_N          \n",
       "1239     0.00005\n",
       "1026     0.00064\n",
       "41       0.00272\n",
       "1019     0.00312\n",
       "1431     0.02693\n",
       "...          ...\n",
       "1462     0.99974\n",
       "1215     0.99976\n",
       "1299     0.99981\n",
       "960      0.99983\n",
       "190      0.99999\n",
       "\n",
       "[1890 rows x 1 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_values = pd.DataFrame(selected_features.pvalues_.round(5))\n",
    "p_values.index.name=\"FEAT_N\"\n",
    "p_values.columns = [\"P_VALUES\"]\n",
    "p_values.sort_values(by=['P_VALUES'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SCORE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FEAT_N</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1239</th>\n",
       "      <td>16.40521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1026</th>\n",
       "      <td>11.64837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>8.98474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1019</th>\n",
       "      <td>8.73733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>971</th>\n",
       "      <td>4.89520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1424</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>956</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>800</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1462</th>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1890 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           SCORE\n",
       "FEAT_N          \n",
       "1239    16.40521\n",
       "1026    11.64837\n",
       "41       8.98474\n",
       "1019     8.73733\n",
       "971      4.89520\n",
       "...          ...\n",
       "1424     0.00000\n",
       "960      0.00000\n",
       "956      0.00000\n",
       "800      0.00000\n",
       "1462     0.00000\n",
       "\n",
       "[1890 rows x 1 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(selected_features.scores_.round(5))\n",
    "df.index.name=\"FEAT_N\"\n",
    "df.columns = [\"SCORE\"]\n",
    "df.sort_values(by=['SCORE'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For testing\n",
    "\n",
    "selected_features = SelectKBest(chi2,k=200)\n",
    "selected_features.fit(X_train, y_train)\n",
    "X_train_skb_compl = pd.DataFrame(selected_features.transform(X_train))\n",
    "X_test_skb_compl = pd.DataFrame(selected_features.transform(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "dtc = DecisionTreeClassifier(random_state=47)\n",
    "\n",
    "params = {\n",
    "    'criterion' : ['gini','entropy'],\n",
    "    'splitter' : ['best','random'],\n",
    "    'max_depth' : [None,3,4,5,8,10],\n",
    "    'max_features' : [None, 10, 25]\n",
    "}\n",
    "\n",
    "gs = GridSearchCV(dtc, params, scoring='roc_auc', cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs2 = gs.fit(X_train2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7929365377576045"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=4,\n",
       "                       max_features=None, max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=47, splitter='best')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs2.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_arbol1 = DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=4,\n",
    "            max_features=None, max_leaf_nodes=None,\n",
    "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
    "            min_samples_leaf=1, min_samples_split=2,\n",
    "            min_weight_fraction_leaf=0.0, presort=False, random_state=47,\n",
    "            splitter='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor2 = opt_arbol1.fit(X_train2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediccion2 = predictor2.predict_proba(X_test2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8068575687787587"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "scores = roc_auc_score(y_test2,prediccion2[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the same with CV = 2\n",
    "\n",
    "gs2_no_cv = GridSearchCV(dtc,params,scoring='roc_auc',cv=2)\n",
    "gs2_no_cv.fit(X_train2,y_train2)\n",
    "pred_no_cv = gs2_no_cv.best_estimator_\n",
    "predijo = pred_no_cv.predict_proba(X_test2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8068575687787587"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_no_cv = roc_auc_score(y_test2,predijo[:,1])\n",
    "scores_no_cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_pca = gs.fit(X_train_pca2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7178605582963131"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_pca.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictor_pca = gs_pca.best_estimator_\n",
    "pred_pca = predictor_pca.predict_proba(X_test_pca2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7383593018050242"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_pca = roc_auc_score(y_test2,pred_pca[:,1])\n",
    "scores_pca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs_skb = gs.fit(X_train_skb2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7904701654101584"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs_skb.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=3,\n",
       "                       max_features=None, max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=47, splitter='best')"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictor_skb = gs_skb.best_estimator_\n",
    "predictor_skb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "predic_skb = predictor_skb.predict_proba(X_test_skb2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8234152386314861"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_skb_tree = roc_auc_score(y_test2,predic_skb[:,1])\n",
    "scores_skb_tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DTR - Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "arbol1 = predictor_skb.fit(X_train_skb_compl,y_train)\n",
    "sol_arbol1 = arbol1.predict_proba(X_test_skb_compl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "resultado_arbol1 = pd.Series(sol_arbol1[:,1])\n",
    "resultado_arbol1.index.name=\"USER_ID\" # Renombramos las columnas\n",
    "resultado_arbol1.name=\"SCORE\" # Renombramos las columnas\n",
    "resultado_arbol1.to_csv(\"respuesta_arbol_1\", header=True, compression=\"zip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DTR 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimization through GridSearchCV\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "pg = [\n",
    "    {\"splitter\":[\"best\",\"random\"],\n",
    "     \"max_depth\":[None,1,2,3,4,5,7,10], \n",
    "     \"max_features\":[10,25], \n",
    "     \"max_leaf_nodes\":[10,50,100,150,200]}\n",
    "]\n",
    "\n",
    "dtc2 = arbol1\n",
    "\n",
    "opt = GridSearchCV(dtc2, pg, scoring=\"roc_auc\",n_jobs=-1,cv=StratifiedKFold(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=None, shuffle=False),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=DecisionTreeClassifier(class_weight=None,\n",
       "                                              criterion='entropy', max_depth=3,\n",
       "                                              max_features=None,\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=1,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              presort=False, random_state=47,\n",
       "                                              splitter='best'),\n",
       "             iid='warn', n_jobs=-1,\n",
       "             param_grid=[{'max_depth': [None, 1, 2, 3, 4, 5, 7, 10],\n",
       "                          'max_features': [10, 25],\n",
       "                          'max_leaf_nodes': [10, 50, 100, 150, 200],\n",
       "                          'splitter': ['best', 'random']}],\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt.fit(X_train2, y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6986078770421436"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=None,\n",
       "                       max_features=25, max_leaf_nodes=10,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=47, splitter='best')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimised_decision_tree = opt.best_estimator_\n",
    "optimised_decision_tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol = optimised_decision_tree.predict_proba(X_test2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6385312198566504"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = roc_auc_score(y_test2,sol[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_arbol2 = DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=None,\n",
    "            max_features=25, max_leaf_nodes=10, min_impurity_decrease=0.0,\n",
    "            min_impurity_split=None, min_samples_leaf=1,\n",
    "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
    "            presort=False, random_state=47, splitter='best')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_pca = opt.fit(X_train_pca2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7157811656206726"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt_pca.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_tree_2 = opt_pca.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_tree_2 = pca_tree_2.predict_proba(X_test_pca2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.711334591758087"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = roc_auc_score(y_test2,res_tree_2[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt_skb = opt.fit(X_train_skb2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7835440680555065"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt_skb.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "skb_tree_2 = opt_skb.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol_skb_2 = skb_tree_2.predict_proba(X_test_skb2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8165546699404171"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = roc_auc_score(y_test2,sol_skb_2[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DTR2 - solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=4,\n",
       "                       max_features=25, max_leaf_nodes=10,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=47, splitter='best')"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arbol2 = skb_tree_2\n",
    "skb_tree_2.fit(X_train_skb_compl,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol_a2 = skb_tree_2.predict_proba(X_test_skb_compl)\n",
    "resultado_arbol2 = pd.Series(sol_a2[:,1])\n",
    "resultado_arbol2.index.name=\"USER_ID\" # Renombramos las columnas\n",
    "resultado_arbol2.name=\"SCORE\" # Renombramos las columnas\n",
    "resultado_arbol2.to_csv(\"respuesta_arbol_2\", header=True, compression=\"zip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ADABOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None, learning_rate=1.0,\n",
       "                   n_estimators=8, random_state=23)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "clf = AdaBoostClassifier(n_estimators = 8,random_state=23)\n",
    "clf.fit(X_train2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "probab = clf.predict_proba(X_test2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8264840335436348"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = roc_auc_score(y_test2,probab[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7574733584710601"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_pca = clf.fit(X_train_pca2,y_train2)\n",
    "probab_pca = clf_pca.predict_proba(X_test_pca2)\n",
    "scores = roc_auc_score(y_test2,probab_pca[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8264840335436348"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_skb = clf.fit(X_train_skb2,y_train2)\n",
    "probab_skb = clf_skb.predict_proba(X_test_skb2)\n",
    "scores = roc_auc_score(y_test2,probab_skb[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ADABOOST - solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None, learning_rate=1.0,\n",
       "                   n_estimators=8, random_state=23)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada_final = AdaBoostClassifier(n_estimators = 8,random_state=23)\n",
    "ada_final.fit(X_train_skb_compl,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol_ada_final = ada_final.predict_proba(X_test_skb_compl)\n",
    "resultado_ada = pd.Series(sol_ada_final[:,1])\n",
    "resultado_ada.index.name=\"USER_ID\" # Renombramos las columnas\n",
    "resultado_ada.name=\"SCORE\" # Renombramos las columnas\n",
    "resultado_ada.to_csv(\"respuesta_ada\", header=True, compression=\"zip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RandomForest()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "\n",
    "param_grid = {\n",
    "    \"n_estimators\" : [200,500,1000],\n",
    "    \"max_depth\" : [3,4],\n",
    "    \"max_features\" : [50,100,200],\n",
    "}\n",
    "\n",
    "random_forest = RandomForestClassifier(random_state=47,class_weight=\"balanced\",criterion='gini')\n",
    "\n",
    "clasificador = GridSearchCV(random_forest,param_grid,scoring=\"roc_auc\",cv=StratifiedKFold(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "forest2 = clasificador.fit(X_train2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.83349526827768 RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
      "                       criterion='gini', max_depth=4, max_features=100,\n",
      "                       max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
      "                       min_impurity_split=None, min_samples_leaf=1,\n",
      "                       min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "                       n_estimators=1000, n_jobs=None, oob_score=False,\n",
      "                       random_state=47, verbose=0, warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "print(forest2.best_score_,forest2.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_for2 = forest2.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8565312706243972"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sol_rf2 = clf_for2.predict_proba(X_test2)\n",
    "scores = roc_auc_score(y_test2,sol_rf2[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_opt = RandomForestClassifier(random_state=47,class_weight=\"balanced\",criterion='gini',n_estimators=200,max_features=100,max_depth=4)\n",
    "rf_opt_pca = RandomForestClassifier(random_state=47,class_weight=\"balanced\",criterion='gini',n_estimators=200,max_depth=4)\n",
    "rf_opt_skb = RandomForestClassifier(random_state=47,class_weight=\"balanced\",criterion='gini',n_estimators=200,max_depth=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "                       criterion='gini', max_depth=4, max_features='auto',\n",
       "                       max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "                       min_impurity_split=None, min_samples_leaf=1,\n",
       "                       min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                       n_estimators=200, n_jobs=None, oob_score=False,\n",
       "                       random_state=47, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_opt_pca.fit(X_train_pca2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8060365158556902"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sol_rf22 = rf_opt_pca.predict_proba(X_test_pca2)\n",
    "scores22 = roc_auc_score(y_test2,sol_rf22[:,1])\n",
    "scores22"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "                       criterion='gini', max_depth=4, max_features='auto',\n",
       "                       max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "                       min_impurity_split=None, min_samples_leaf=1,\n",
       "                       min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                       n_estimators=200, n_jobs=None, oob_score=False,\n",
       "                       random_state=47, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_opt_skb.fit(X_train_skb2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8553940730963249"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sol_rf_skb = rf_opt_skb.predict_proba(X_test_skb2)\n",
    "scores = roc_auc_score(y_test2,sol_rf_skb[:,1])\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RF - solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight='balanced',\n",
       "                       criterion='gini', max_depth=4, max_features='auto',\n",
       "                       max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "                       min_impurity_split=None, min_samples_leaf=1,\n",
       "                       min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "                       n_estimators=200, n_jobs=None, oob_score=False,\n",
       "                       random_state=47, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf_final = RandomForestClassifier(random_state=47,class_weight=\"balanced\",criterion='gini',n_estimators=200,max_depth=4)\n",
    "rf_final.fit(X_train_skb_compl,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_final_sol = rf_final.predict_proba(X_test_skb_compl)\n",
    "resultado_random = pd.Series(rf_final_sol[:,1])\n",
    "resultado_random.index.name=\"USER_ID\" # Renombramos las columnas\n",
    "resultado_random.name=\"SCORE\" # Renombramos las columnas\n",
    "resultado_random.to_csv(\"respuesta_random\", header=True, compression=\"zip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## CATBOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = CatBoostClassifier(random_seed=47,verbose=0)\n",
    "\n",
    "grid_cat = {\n",
    "    \"iterations\": [50,100,200],\n",
    "    \"max_depth\": [None,2,4,6],\n",
    "}\n",
    "\n",
    "cat_clf = GridSearchCV(cat,grid_cat,cv=StratifiedKFold(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=None, shuffle=False),\n",
       "             error_score='raise-deprecating',\n",
       "             estimator=<catboost.core.CatBoostClassifier object at 0x7f76b89e9f10>,\n",
       "             iid='warn', n_jobs=None,\n",
       "             param_grid={'iterations': [50, 100, 200],\n",
       "                         'max_depth': [None, 2, 4, 6]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_clf.fit(X_train2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x7f76b8fcb490>"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.848346356029593"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mod_cat2 = cat_clf.best_estimator_\n",
    "sol_cat2 = mod_cat2.predict_proba(X_test2)\n",
    "sco = roc_auc_score(y_test2, sol_cat2[:,1])\n",
    "sco"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8466754048727807"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "mod_cat_nogs = CatBoostClassifier(iterations=220,random_seed=47,max_depth=None,verbose=0)\n",
    "mod_cat_nogs.fit(X_train2,y_train2)\n",
    "sol_cat_nogs = mod_cat_nogs.predict_proba(X_test2)\n",
    "sco_cat_nogs = roc_auc_score(y_test2,sol_cat_nogs[:,1])\n",
    "sco_cat_nogs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8512671168073548"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mod_cat_skb = CatBoostClassifier(iterations=220,random_seed=47,max_depth=None,verbose=0)\n",
    "mod_cat_skb.fit(X_train_skb2,y_train2)\n",
    "sol_cat_skb = mod_cat_skb.predict_proba(X_test_skb2)\n",
    "sco_cat_skb = roc_auc_score(y_test2,sol_cat_skb[:,1])\n",
    "sco_cat_skb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CATBOOST - solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x7f76b8067950>"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_final = CatBoostClassifier(iterations=220,random_seed=47,max_depth=None,verbose=0)\n",
    "cat_final.fit(X_train_skb_compl,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol_cat = cat_final.predict_proba(X_test_skb_compl)\n",
    "resultado_cat = pd.Series(sol_cat[:,1])\n",
    "resultado_cat.index.name=\"USER_ID\" # Renombramos las columnas\n",
    "resultado_cat.name=\"SCORE\" # Renombramos las columnas\n",
    "resultado_cat.to_csv(\"respuesta_cat\", header=True, compression=\"zip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "xgb_clf = XGBClassifier(verbosity=0,max_depth=2,sub_sample=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "              learning_rate=0.1, max_delta_step=0, max_depth=2,\n",
       "              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "              nthread=None, objective='binary:logistic', random_state=0,\n",
       "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "              silent=None, sub_sample=0.5, subsample=1, verbosity=0)"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_clf.fit(X_train_skb2, y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8535749724238829"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sol_xgb = xgb_clf.predict_proba(X_test_skb2)\n",
    "sco_xgb = roc_auc_score(y_test2,sol_xgb[:,1])\n",
    "sco_xgb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Modelo Final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "              learning_rate=0.1, max_delta_step=0, max_depth=2,\n",
       "              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "              nthread=None, objective='binary:logistic', random_state=0,\n",
       "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "              silent=None, sub_sample=0.5, subsample=1, verbosity=0)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_final = XGBClassifier(verbosity=0,max_depth=2,sub_sample=0.5)\n",
    "xgb_final.fit(X_train_skb_compl,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol_xgb = xgb_final.predict_proba(X_test_skb_compl)\n",
    "resultado_xgb = pd.Series(sol_xgb[:,1])\n",
    "resultado_xgb.index.name=\"USER_ID\" # Renombramos las columnas\n",
    "resultado_xgb.name=\"SCORE\" # Renombramos las columnas\n",
    "resultado_xgb.to_csv(\"respuesta_xgb\", header=True, compression=\"zip\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "svm = SVC(class_weight=\"balanced\",probability=True,verbose=1,random_state=99,gamma=\"auto\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibSVM]"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=99, shrinking=True, tol=0.001,\n",
       "    verbose=1)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm.fit(X_train_skb2,y_train2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6048455506685189"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_prueba = svm.predict_proba(X_test_skb2)\n",
    "scoring_svm = roc_auc_score(y_test2, svm_prueba[:,1])\n",
    "scoring_svm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans \n",
    "\n",
    "kmeans = KMeans(n_clusters=2,random_state=44,verbose=0)\n",
    "X_train_km = kmeans.fit_transform(X_train_skb2)\n",
    "y_kmeans = kmeans.predict(X_test_skb2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5032292902207474"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scoring = roc_auc_score(y_test2,y_kmeans)\n",
    "scoring"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# STACKING SOLUTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "models = [\n",
    "    xgb_final,\n",
    "    cat_final,\n",
    "    ada_final,\n",
    "    rf_final,\n",
    "    LGBMClassifier()\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task:         [classification]\n",
      "n_classes:    [2]\n",
      "metric:       [log_loss]\n",
      "mode:         [oof_pred]\n",
      "n_models:     [5]\n",
      "\n",
      "model  0:     [XGBClassifier]\n",
      "    ----\n",
      "    MEAN:     [0.11679379] + [0.00297508]\n",
      "    FULL:     [0.11679388]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model  1:     [CatBoostClassifier]\n",
      "    ----\n",
      "    MEAN:     [0.12067422] + [0.00363037]\n",
      "    FULL:     [0.12067456]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model  2:     [AdaBoostClassifier]\n",
      "    ----\n",
      "    MEAN:     [0.45627183] + [0.00659728]\n",
      "    FULL:     [0.45627252]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model  3:     [RandomForestClassifier]\n",
      "    ----\n",
      "    MEAN:     [0.42819883] + [0.00452144]\n",
      "    FULL:     [0.42819862]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "model  4:     [LGBMClassifier]\n",
      "    ----\n",
      "    MEAN:     [0.14252408] + [0.00788062]\n",
      "    FULL:     [0.14252473]\n",
      "\n",
      "    Fitting on full train set...\n",
      "\n",
      "Result was saved to [./[2019.11.23].[19.06.55].075918.ef1c1a.npy]\n"
     ]
    }
   ],
   "source": [
    "from vecstack import stacking\n",
    "\n",
    "S_train, S_test = stacking(models,                                             # list of models\n",
    "                               X_train_skb_compl, y_train, X_test_skb_compl,   # data\n",
    "                               regression=False,                               # classification task (if you need \n",
    "                                                                               #     regression - set to True)\n",
    "                               mode='oof_pred',                                # mode: oof for train set, fit on full \n",
    "                                                                               #     train and predict test set once\n",
    "                               needs_proba=True,                               # predict probabilities (if you need \n",
    "                                                                               #     class labels - set to False) \n",
    "                               save_dir='.',                                   # save result and log in current dir \n",
    "                                                                               #     (to disable saving - set to None)\n",
    "                               n_folds=5,                                      # number of folds\n",
    "                               stratified=True,                                # stratified split for folds\n",
    "                               shuffle=True,                                   # shuffle the data\n",
    "                               random_state=0,                                 # ensure reproducibility\n",
    "                               verbose=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "              learning_rate=0.1, max_delta_step=0, max_depth=2,\n",
       "              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "              nthread=None, objective='binary:logistic', random_state=0,\n",
       "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "              silent=None, sub_sample=0.5, subsample=1, verbosity=0)"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_of_models = xgb_final\n",
    "model_of_models.fit(S_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "sol_final = model_of_models.predict_proba(S_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "resultado_final = pd.Series(sol_final[:,1])\n",
    "resultado_final.index.name=\"USER_ID\" # Renombramos las columnas\n",
    "resultado_final.name=\"SCORE\" # Renombramos las columnas\n",
    "resultado_final.to_csv(\"respuesta_final\", header=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}