Train & test clean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(120000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 120 seconds\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 4\n",
    "%autosave 120"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.io import *\n",
    "from fastai.structured import *\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from pandas_summary import DataFrameSummary\n",
    "from IPython.display import display\n",
    "from sklearn import metrics\n",
    "import feather\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_clean = pd.read_csv('train_clean.csv')\n",
    "test_clean = pd.read_csv('test_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_old = feather.read_dataframe('train_df')\n",
    "test_old = feather.read_dataframe('test_df')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [train_old, test_old]:\n",
    "    for f in ['purchase_date_max','purchase_date_min','purchase_date_max_old',\\\n",
    "                     'purchase_date_min_old', 'first_active_month']:\n",
    "        df[f] = df[f].astype(np.int64) * 1e-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_clean0 = train_clean.fillna(0)\n",
    "train_clean0.to_feather('train_clean0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display_all(df):\n",
    "    with pd.option_context(\"display.max_rows\", 150, \"display.max_columns\", 100): \n",
    "        display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>2017-06-01</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>2016-08-01</td>\n",
       "      <td>2017-09-01</td>\n",
       "      <td>2017-11-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>C_ID_92a2005557</td>\n",
       "      <td>C_ID_3d0044924f</td>\n",
       "      <td>C_ID_d639edf6cd</td>\n",
       "      <td>C_ID_186d6a6901</td>\n",
       "      <td>C_ID_cdbd2c0db2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>-0.820283</td>\n",
       "      <td>0.392913</td>\n",
       "      <td>0.688056</td>\n",
       "      <td>0.142495</td>\n",
       "      <td>-0.159749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2016</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>elapsed_time</th>\n",
       "      <td>245</td>\n",
       "      <td>396</td>\n",
       "      <td>549</td>\n",
       "      <td>153</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2_1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2_2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2_3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_transactions_count</th>\n",
       "      <td>247</td>\n",
       "      <td>339</td>\n",
       "      <td>41</td>\n",
       "      <td>77</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_1_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_1_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0855457</td>\n",
       "      <td>0</td>\n",
       "      <td>0.155844</td>\n",
       "      <td>0.09375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_3_A_mean</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_5.0_mean</th>\n",
       "      <td>0.0121457</td>\n",
       "      <td>0</td>\n",
       "      <td>0.902439</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0078125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_3_C_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.19174</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0909091</td>\n",
       "      <td>0.03125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_subsector_id_nunique</th>\n",
       "      <td>21</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_merchant_id_nunique</th>\n",
       "      <td>93</td>\n",
       "      <td>141</td>\n",
       "      <td>13</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_1.0_mean</th>\n",
       "      <td>0.987854</td>\n",
       "      <td>0.914454</td>\n",
       "      <td>0.097561</td>\n",
       "      <td>0.155844</td>\n",
       "      <td>0.078125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_state_id_nunique</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_4.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.688312</td>\n",
       "      <td>0.820312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_sum</th>\n",
       "      <td>-157.397</td>\n",
       "      <td>-208.883</td>\n",
       "      <td>-27.8284</td>\n",
       "      <td>-49.4914</td>\n",
       "      <td>-69.0405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_mean</th>\n",
       "      <td>-0.637235</td>\n",
       "      <td>-0.616175</td>\n",
       "      <td>-0.678742</td>\n",
       "      <td>-0.642745</td>\n",
       "      <td>-0.539379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_max</th>\n",
       "      <td>2.25839</td>\n",
       "      <td>4.6303</td>\n",
       "      <td>-0.145847</td>\n",
       "      <td>1.4456</td>\n",
       "      <td>6.99262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_min</th>\n",
       "      <td>-0.739395</td>\n",
       "      <td>-0.7424</td>\n",
       "      <td>-0.730138</td>\n",
       "      <td>-0.740897</td>\n",
       "      <td>-0.746156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_std</th>\n",
       "      <td>0.216518</td>\n",
       "      <td>0.355553</td>\n",
       "      <td>0.0892296</td>\n",
       "      <td>0.261624</td>\n",
       "      <td>0.737087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>501</td>\n",
       "      <td>0</td>\n",
       "      <td>84</td>\n",
       "      <td>144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>1.47788</td>\n",
       "      <td>0</td>\n",
       "      <td>1.09091</td>\n",
       "      <td>1.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_max</th>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_min</th>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_std</th>\n",
       "      <td>0</td>\n",
       "      <td>1.35063</td>\n",
       "      <td>0</td>\n",
       "      <td>0.588974</td>\n",
       "      <td>1.00393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_merchant_category_id_nunique</th>\n",
       "      <td>41</td>\n",
       "      <td>57</td>\n",
       "      <td>8</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_mean</th>\n",
       "      <td>7.97976</td>\n",
       "      <td>6.14454</td>\n",
       "      <td>4.63415</td>\n",
       "      <td>7.74026</td>\n",
       "      <td>5.55469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_max</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_min</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_std</th>\n",
       "      <td>3.52857</td>\n",
       "      <td>3.85918</td>\n",
       "      <td>3.32984</td>\n",
       "      <td>3.9048</td>\n",
       "      <td>5.04126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_month_lag_min</th>\n",
       "      <td>-8</td>\n",
       "      <td>-12</td>\n",
       "      <td>-13</td>\n",
       "      <td>-5</td>\n",
       "      <td>-3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_month_lag_max</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_3.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_2.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_date_ptp</th>\n",
       "      <td>2.0978e+07</td>\n",
       "      <td>3.37177e+07</td>\n",
       "      <td>3.56356e+07</td>\n",
       "      <td>1.33753e+07</td>\n",
       "      <td>9.40564e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_date_min</th>\n",
       "      <td>1.49857e+09</td>\n",
       "      <td>1.48372e+09</td>\n",
       "      <td>1.48412e+09</td>\n",
       "      <td>1.50644e+09</td>\n",
       "      <td>1.51044e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_date_max</th>\n",
       "      <td>1.51955e+09</td>\n",
       "      <td>1.51744e+09</td>\n",
       "      <td>1.51976e+09</td>\n",
       "      <td>1.51982e+09</td>\n",
       "      <td>1.51985e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_3_B_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.80236</td>\n",
       "      <td>0</td>\n",
       "      <td>0.883117</td>\n",
       "      <td>0.96875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_city_id_nunique</th>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_transactions_count</th>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_1_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_1_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_3_A_mean</th>\n",
       "      <td>0.692308</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_5.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_3_C_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_subsector_id_nunique</th>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_merchant_id_nunique</th>\n",
       "      <td>12</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_1.0_mean</th>\n",
       "      <td>1</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_state_id_nunique</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_4.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_sum</th>\n",
       "      <td>-8.57172</td>\n",
       "      <td>-1.12289</td>\n",
       "      <td>-1.33897</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20.3528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_mean</th>\n",
       "      <td>-0.659363</td>\n",
       "      <td>-0.102081</td>\n",
       "      <td>-0.669484</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.07056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_max</th>\n",
       "      <td>-0.431922</td>\n",
       "      <td>1.94284</td>\n",
       "      <td>-0.637515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.19304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_min</th>\n",
       "      <td>-0.737892</td>\n",
       "      <td>-0.740897</td>\n",
       "      <td>-0.701453</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.512945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_std</th>\n",
       "      <td>0.0988505</td>\n",
       "      <td>0.785906</td>\n",
       "      <td>0.0452109</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.18495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_sum</th>\n",
       "      <td>4</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_mean</th>\n",
       "      <td>0.307692</td>\n",
       "      <td>3.81818</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_max</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_min</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_std</th>\n",
       "      <td>0.480384</td>\n",
       "      <td>3.48764</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.02495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_merchant_category_id_nunique</th>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_mean</th>\n",
       "      <td>9.53846</td>\n",
       "      <td>8.54545</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_max</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_min</th>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_std</th>\n",
       "      <td>1.6641</td>\n",
       "      <td>2.6968</td>\n",
       "      <td>1.41421</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.547723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_month_lag_min</th>\n",
       "      <td>-7</td>\n",
       "      <td>-10</td>\n",
       "      <td>-12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_month_lag_max</th>\n",
       "      <td>-2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_3.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_2.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_date_ptp</th>\n",
       "      <td>1.42545e+07</td>\n",
       "      <td>2.58908e+07</td>\n",
       "      <td>4.92288e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.27433e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_date_min</th>\n",
       "      <td>1.50013e+09</td>\n",
       "      <td>1.48858e+09</td>\n",
       "      <td>1.48788e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.51648e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_date_max</th>\n",
       "      <td>1.51439e+09</td>\n",
       "      <td>1.51447e+09</td>\n",
       "      <td>1.4928e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.51976e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_3_B_mean</th>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_city_id_nunique</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_transactions_count</th>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_1_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_1_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.0555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_3_A_mean</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_5.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_3_C_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0277778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_subsector_id_nunique</th>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_merchant_id_nunique</th>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_1.0_mean</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_state_id_nunique</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_4.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.694444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_sum</th>\n",
       "      <td>-13.24</td>\n",
       "      <td>-4.355</td>\n",
       "      <td>-0.7</td>\n",
       "      <td>-4.656</td>\n",
       "      <td>-19.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_mean</th>\n",
       "      <td>-0.5757</td>\n",
       "      <td>-0.726</td>\n",
       "      <td>-0.7</td>\n",
       "      <td>-0.665</td>\n",
       "      <td>-0.5537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_max</th>\n",
       "      <td>-0.2961</td>\n",
       "      <td>-0.7017</td>\n",
       "      <td>-0.7</td>\n",
       "      <td>-0.567</td>\n",
       "      <td>0.451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_min</th>\n",
       "      <td>-0.7246</td>\n",
       "      <td>-0.7393</td>\n",
       "      <td>-0.7</td>\n",
       "      <td>-0.7344</td>\n",
       "      <td>-0.7393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_std</th>\n",
       "      <td>0.1357</td>\n",
       "      <td>0.01438</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0659</td>\n",
       "      <td>0.2239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.972222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_max</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_min</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_std</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.755929</td>\n",
       "      <td>0.376913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_merchant_category_id_nunique</th>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_mean</th>\n",
       "      <td>3.47826</td>\n",
       "      <td>2.5</td>\n",
       "      <td>4</td>\n",
       "      <td>3.71429</td>\n",
       "      <td>3.55556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_max</th>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_min</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_std</th>\n",
       "      <td>0.510754</td>\n",
       "      <td>0.547723</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.48795</td>\n",
       "      <td>0.503953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_month_lag_min</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_month_lag_max</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_3.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.194444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_2.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_date_ptp</th>\n",
       "      <td>4.74231e+06</td>\n",
       "      <td>4.88763e+06</td>\n",
       "      <td>0</td>\n",
       "      <td>3.62550e+06</td>\n",
       "      <td>4.94968e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_date_min</th>\n",
       "      <td>1.52026e+09</td>\n",
       "      <td>1.5175e+09</td>\n",
       "      <td>1.52494e+09</td>\n",
       "      <td>1.52042e+09</td>\n",
       "      <td>1.51999e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_date_max</th>\n",
       "      <td>1.525e+09</td>\n",
       "      <td>1.52239e+09</td>\n",
       "      <td>1.52494e+09</td>\n",
       "      <td>1.52405e+09</td>\n",
       "      <td>1.52494e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_3_B_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.944444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_city_id_nunique</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>outliers</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                         0                1  \\\n",
       "first_active_month                              2017-06-01       2017-01-01   \n",
       "card_id                                    C_ID_92a2005557  C_ID_3d0044924f   \n",
       "feature_3                                                1                0   \n",
       "target                                           -0.820283         0.392913   \n",
       "month                                                    6                1   \n",
       "year                                                  2017             2017   \n",
       "elapsed_time                                           245              396   \n",
       "feature_1_1                                              0                0   \n",
       "feature_1_2                                              0                0   \n",
       "feature_1_3                                              0                0   \n",
       "feature_1_4                                              0                1   \n",
       "feature_1_5                                              1                0   \n",
       "feature_2_1                                              0                1   \n",
       "feature_2_2                                              1                0   \n",
       "feature_2_3                                              0                0   \n",
       "authorized_transactions_count                          247              339   \n",
       "authorized_category_1_sum                                0               29   \n",
       "authorized_category_1_mean                               0        0.0855457   \n",
       "authorized_category_3_A_mean                             1                0   \n",
       "authorized_category_2_5.0_mean                   0.0121457                0   \n",
       "authorized_category_3_C_mean                             0          0.19174   \n",
       "authorized_subsector_id_nunique                         21               24   \n",
       "authorized_merchant_id_nunique                          93              141   \n",
       "authorized_category_2_1.0_mean                    0.987854         0.914454   \n",
       "authorized_state_id_nunique                              3                3   \n",
       "authorized_category_2_4.0_mean                           0                0   \n",
       "authorized_purchase_amount_sum                    -157.397         -208.883   \n",
       "authorized_purchase_amount_mean                  -0.637235        -0.616175   \n",
       "authorized_purchase_amount_max                     2.25839           4.6303   \n",
       "authorized_purchase_amount_min                   -0.739395          -0.7424   \n",
       "authorized_purchase_amount_std                    0.216518         0.355553   \n",
       "authorized_installments_sum                              0              501   \n",
       "authorized_installments_mean                             0          1.47788   \n",
       "authorized_installments_max                              0               10   \n",
       "authorized_installments_min                              0               -1   \n",
       "authorized_installments_std                              0          1.35063   \n",
       "authorized_merchant_category_id_nunique                 41               57   \n",
       "authorized_purchase_month_mean                     7.97976          6.14454   \n",
       "authorized_purchase_month_max                           12               12   \n",
       "authorized_purchase_month_min                            1                1   \n",
       "authorized_purchase_month_std                      3.52857          3.85918   \n",
       "authorized_month_lag_min                                -8              -12   \n",
       "authorized_month_lag_max                                 0                0   \n",
       "authorized_category_2_3.0_mean                           0                0   \n",
       "authorized_category_2_2.0_mean                           0                0   \n",
       "authorized_purchase_date_ptp                    2.0978e+07      3.37177e+07   \n",
       "authorized_purchase_date_min                   1.49857e+09      1.48372e+09   \n",
       "authorized_purchase_date_max                   1.51955e+09      1.51744e+09   \n",
       "authorized_category_3_B_mean                             0          0.80236   \n",
       "authorized_city_id_nunique                               7                9   \n",
       "unauthorized_transactions_count                         13               11   \n",
       "unauthorized_category_1_sum                              0                2   \n",
       "unauthorized_category_1_mean                             0         0.181818   \n",
       "unauthorized_category_3_A_mean                    0.692308                0   \n",
       "unauthorized_category_2_5.0_mean                         0                0   \n",
       "unauthorized_category_3_C_mean                           0         0.636364   \n",
       "unauthorized_subsector_id_nunique                        7                9   \n",
       "unauthorized_merchant_id_nunique                        12                9   \n",
       "unauthorized_category_2_1.0_mean                         1         0.818182   \n",
       "unauthorized_state_id_nunique                            1                2   \n",
       "unauthorized_category_2_4.0_mean                         0                0   \n",
       "unauthorized_purchase_amount_sum                  -8.57172         -1.12289   \n",
       "unauthorized_purchase_amount_mean                -0.659363        -0.102081   \n",
       "unauthorized_purchase_amount_max                 -0.431922          1.94284   \n",
       "unauthorized_purchase_amount_min                 -0.737892        -0.740897   \n",
       "unauthorized_purchase_amount_std                 0.0988505         0.785906   \n",
       "unauthorized_installments_sum                            4               42   \n",
       "unauthorized_installments_mean                    0.307692          3.81818   \n",
       "unauthorized_installments_max                            1               10   \n",
       "unauthorized_installments_min                            0                1   \n",
       "unauthorized_installments_std                     0.480384          3.48764   \n",
       "unauthorized_merchant_category_id_nunique               10                9   \n",
       "unauthorized_purchase_month_mean                   9.53846          8.54545   \n",
       "unauthorized_purchase_month_max                         12               12   \n",
       "unauthorized_purchase_month_min                          7                3   \n",
       "unauthorized_purchase_month_std                     1.6641           2.6968   \n",
       "unauthorized_month_lag_min                              -7              -10   \n",
       "unauthorized_month_lag_max                              -2               -1   \n",
       "unauthorized_category_2_3.0_mean                         0                0   \n",
       "unauthorized_category_2_2.0_mean                         0                0   \n",
       "unauthorized_purchase_date_ptp                 1.42545e+07      2.58908e+07   \n",
       "unauthorized_purchase_date_min                 1.50013e+09      1.48858e+09   \n",
       "unauthorized_purchase_date_max                 1.51439e+09      1.51447e+09   \n",
       "unauthorized_category_3_B_mean                    0.307692         0.363636   \n",
       "unauthorized_city_id_nunique                             2                2   \n",
       "new_transactions_count                                  23                6   \n",
       "new_category_1_sum                                       0                0   \n",
       "new_category_1_mean                                      0                0   \n",
       "new_category_3_A_mean                                    1                0   \n",
       "new_category_2_5.0_mean                                  0                0   \n",
       "new_category_3_C_mean                                    0                0   \n",
       "new_subsector_id_nunique                                10                4   \n",
       "new_merchant_id_nunique                                 23                6   \n",
       "new_category_2_1.0_mean                                  1                1   \n",
       "new_state_id_nunique                                     1                1   \n",
       "new_category_2_4.0_mean                                  0                0   \n",
       "new_purchase_amount_sum                             -13.24           -4.355   \n",
       "new_purchase_amount_mean                           -0.5757           -0.726   \n",
       "new_purchase_amount_max                            -0.2961          -0.7017   \n",
       "new_purchase_amount_min                            -0.7246          -0.7393   \n",
       "new_purchase_amount_std                             0.1357          0.01438   \n",
       "new_installments_sum                                     0                6   \n",
       "new_installments_mean                                    0                1   \n",
       "new_installments_max                                     0                1   \n",
       "new_installments_min                                     0                1   \n",
       "new_installments_std                                     0                0   \n",
       "new_merchant_category_id_nunique                        14                5   \n",
       "new_purchase_month_mean                            3.47826              2.5   \n",
       "new_purchase_month_max                                   4                3   \n",
       "new_purchase_month_min                                   3                2   \n",
       "new_purchase_month_std                            0.510754         0.547723   \n",
       "new_month_lag_min                                        1                1   \n",
       "new_month_lag_max                                        2                2   \n",
       "new_category_2_3.0_mean                                  0                0   \n",
       "new_category_2_2.0_mean                                  0                0   \n",
       "new_purchase_date_ptp                          4.74231e+06      4.88763e+06   \n",
       "new_purchase_date_min                          1.52026e+09       1.5175e+09   \n",
       "new_purchase_date_max                            1.525e+09      1.52239e+09   \n",
       "new_category_3_B_mean                                    0                1   \n",
       "new_city_id_nunique                                      3                1   \n",
       "outliers                                                 0                0   \n",
       "\n",
       "                                                         2                3  \\\n",
       "first_active_month                              2016-08-01       2017-09-01   \n",
       "card_id                                    C_ID_d639edf6cd  C_ID_186d6a6901   \n",
       "feature_3                                                0                0   \n",
       "target                                            0.688056         0.142495   \n",
       "month                                                    8                9   \n",
       "year                                                  2016             2017   \n",
       "elapsed_time                                           549              153   \n",
       "feature_1_1                                              0                0   \n",
       "feature_1_2                                              1                0   \n",
       "feature_1_3                                              0                0   \n",
       "feature_1_4                                              0                1   \n",
       "feature_1_5                                              0                0   \n",
       "feature_2_1                                              0                0   \n",
       "feature_2_2                                              1                0   \n",
       "feature_2_3                                              0                1   \n",
       "authorized_transactions_count                           41               77   \n",
       "authorized_category_1_sum                                0               12   \n",
       "authorized_category_1_mean                               0         0.155844   \n",
       "authorized_category_3_A_mean                             1                0   \n",
       "authorized_category_2_5.0_mean                    0.902439                0   \n",
       "authorized_category_3_C_mean                             0        0.0909091   \n",
       "authorized_subsector_id_nunique                          7               13   \n",
       "authorized_merchant_id_nunique                          13               50   \n",
       "authorized_category_2_1.0_mean                    0.097561         0.155844   \n",
       "authorized_state_id_nunique                              2                5   \n",
       "authorized_category_2_4.0_mean                           0         0.688312   \n",
       "authorized_purchase_amount_sum                    -27.8284         -49.4914   \n",
       "authorized_purchase_amount_mean                  -0.678742        -0.642745   \n",
       "authorized_purchase_amount_max                   -0.145847           1.4456   \n",
       "authorized_purchase_amount_min                   -0.730138        -0.740897   \n",
       "authorized_purchase_amount_std                   0.0892296         0.261624   \n",
       "authorized_installments_sum                              0               84   \n",
       "authorized_installments_mean                             0          1.09091   \n",
       "authorized_installments_max                              0                3   \n",
       "authorized_installments_min                              0               -1   \n",
       "authorized_installments_std                              0         0.588974   \n",
       "authorized_merchant_category_id_nunique                  8               25   \n",
       "authorized_purchase_month_mean                     4.63415          7.74026   \n",
       "authorized_purchase_month_max                           12               12   \n",
       "authorized_purchase_month_min                            1                1   \n",
       "authorized_purchase_month_std                      3.32984           3.9048   \n",
       "authorized_month_lag_min                               -13               -5   \n",
       "authorized_month_lag_max                                 0                0   \n",
       "authorized_category_2_3.0_mean                           0                0   \n",
       "authorized_category_2_2.0_mean                           0                0   \n",
       "authorized_purchase_date_ptp                   3.56356e+07      1.33753e+07   \n",
       "authorized_purchase_date_min                   1.48412e+09      1.50644e+09   \n",
       "authorized_purchase_date_max                   1.51976e+09      1.51982e+09   \n",
       "authorized_category_3_B_mean                             0         0.883117   \n",
       "authorized_city_id_nunique                               5                7   \n",
       "unauthorized_transactions_count                          2              NaN   \n",
       "unauthorized_category_1_sum                              0              NaN   \n",
       "unauthorized_category_1_mean                             0              NaN   \n",
       "unauthorized_category_3_A_mean                           1              NaN   \n",
       "unauthorized_category_2_5.0_mean                         1              NaN   \n",
       "unauthorized_category_3_C_mean                           0              NaN   \n",
       "unauthorized_subsector_id_nunique                        1              NaN   \n",
       "unauthorized_merchant_id_nunique                         1              NaN   \n",
       "unauthorized_category_2_1.0_mean                         0              NaN   \n",
       "unauthorized_state_id_nunique                            1              NaN   \n",
       "unauthorized_category_2_4.0_mean                         0              NaN   \n",
       "unauthorized_purchase_amount_sum                  -1.33897              NaN   \n",
       "unauthorized_purchase_amount_mean                -0.669484              NaN   \n",
       "unauthorized_purchase_amount_max                 -0.637515              NaN   \n",
       "unauthorized_purchase_amount_min                 -0.701453              NaN   \n",
       "unauthorized_purchase_amount_std                 0.0452109              NaN   \n",
       "unauthorized_installments_sum                            0              NaN   \n",
       "unauthorized_installments_mean                           0              NaN   \n",
       "unauthorized_installments_max                            0              NaN   \n",
       "unauthorized_installments_min                            0              NaN   \n",
       "unauthorized_installments_std                            0              NaN   \n",
       "unauthorized_merchant_category_id_nunique                1              NaN   \n",
       "unauthorized_purchase_month_mean                         3              NaN   \n",
       "unauthorized_purchase_month_max                          4              NaN   \n",
       "unauthorized_purchase_month_min                          2              NaN   \n",
       "unauthorized_purchase_month_std                    1.41421              NaN   \n",
       "unauthorized_month_lag_min                             -12              NaN   \n",
       "unauthorized_month_lag_max                             -10              NaN   \n",
       "unauthorized_category_2_3.0_mean                         0              NaN   \n",
       "unauthorized_category_2_2.0_mean                         0              NaN   \n",
       "unauthorized_purchase_date_ptp                 4.92288e+06              NaN   \n",
       "unauthorized_purchase_date_min                 1.48788e+09              NaN   \n",
       "unauthorized_purchase_date_max                  1.4928e+09              NaN   \n",
       "unauthorized_category_3_B_mean                           0              NaN   \n",
       "unauthorized_city_id_nunique                             1              NaN   \n",
       "new_transactions_count                                   1                7   \n",
       "new_category_1_sum                                       0                1   \n",
       "new_category_1_mean                                      0         0.142857   \n",
       "new_category_3_A_mean                                    1                0   \n",
       "new_category_2_5.0_mean                                  1                0   \n",
       "new_category_3_C_mean                                    0                0   \n",
       "new_subsector_id_nunique                                 1                5   \n",
       "new_merchant_id_nunique                                  1                7   \n",
       "new_category_2_1.0_mean                                  0                0   \n",
       "new_state_id_nunique                                     1                2   \n",
       "new_category_2_4.0_mean                                  0         0.857143   \n",
       "new_purchase_amount_sum                               -0.7           -4.656   \n",
       "new_purchase_amount_mean                              -0.7           -0.665   \n",
       "new_purchase_amount_max                               -0.7           -0.567   \n",
       "new_purchase_amount_min                               -0.7          -0.7344   \n",
       "new_purchase_amount_std                                NaN           0.0659   \n",
       "new_installments_sum                                     0                5   \n",
       "new_installments_mean                                    0         0.714286   \n",
       "new_installments_max                                     0                1   \n",
       "new_installments_min                                     0               -1   \n",
       "new_installments_std                                   NaN         0.755929   \n",
       "new_merchant_category_id_nunique                         1                6   \n",
       "new_purchase_month_mean                                  4          3.71429   \n",
       "new_purchase_month_max                                   4                4   \n",
       "new_purchase_month_min                                   4                3   \n",
       "new_purchase_month_std                                 NaN          0.48795   \n",
       "new_month_lag_min                                        2                1   \n",
       "new_month_lag_max                                        2                2   \n",
       "new_category_2_3.0_mean                                  0                0   \n",
       "new_category_2_2.0_mean                                  0                0   \n",
       "new_purchase_date_ptp                                    0      3.62550e+06   \n",
       "new_purchase_date_min                          1.52494e+09      1.52042e+09   \n",
       "new_purchase_date_max                          1.52494e+09      1.52405e+09   \n",
       "new_category_3_B_mean                                    0         0.857143   \n",
       "new_city_id_nunique                                      1                2   \n",
       "outliers                                                 0                0   \n",
       "\n",
       "                                                         4  \n",
       "first_active_month                              2017-11-01  \n",
       "card_id                                    C_ID_cdbd2c0db2  \n",
       "feature_3                                                0  \n",
       "target                                           -0.159749  \n",
       "month                                                   11  \n",
       "year                                                  2017  \n",
       "elapsed_time                                            92  \n",
       "feature_1_1                                              1  \n",
       "feature_1_2                                              0  \n",
       "feature_1_3                                              0  \n",
       "feature_1_4                                              0  \n",
       "feature_1_5                                              0  \n",
       "feature_2_1                                              0  \n",
       "feature_2_2                                              0  \n",
       "feature_2_3                                              1  \n",
       "authorized_transactions_count                          128  \n",
       "authorized_category_1_sum                               12  \n",
       "authorized_category_1_mean                         0.09375  \n",
       "authorized_category_3_A_mean                             0  \n",
       "authorized_category_2_5.0_mean                   0.0078125  \n",
       "authorized_category_3_C_mean                       0.03125  \n",
       "authorized_subsector_id_nunique                         17  \n",
       "authorized_merchant_id_nunique                          65  \n",
       "authorized_category_2_1.0_mean                    0.078125  \n",
       "authorized_state_id_nunique                              6  \n",
       "authorized_category_2_4.0_mean                    0.820312  \n",
       "authorized_purchase_amount_sum                    -69.0405  \n",
       "authorized_purchase_amount_mean                  -0.539379  \n",
       "authorized_purchase_amount_max                     6.99262  \n",
       "authorized_purchase_amount_min                   -0.746156  \n",
       "authorized_purchase_amount_std                    0.737087  \n",
       "authorized_installments_sum                            144  \n",
       "authorized_installments_mean                         1.125  \n",
       "authorized_installments_max                             12  \n",
       "authorized_installments_min                              1  \n",
       "authorized_installments_std                        1.00393  \n",
       "authorized_merchant_category_id_nunique                 26  \n",
       "authorized_purchase_month_mean                     5.55469  \n",
       "authorized_purchase_month_max                           12  \n",
       "authorized_purchase_month_min                            1  \n",
       "authorized_purchase_month_std                      5.04126  \n",
       "authorized_month_lag_min                                -3  \n",
       "authorized_month_lag_max                                 0  \n",
       "authorized_category_2_3.0_mean                           0  \n",
       "authorized_category_2_2.0_mean                           0  \n",
       "authorized_purchase_date_ptp                   9.40564e+06  \n",
       "authorized_purchase_date_min                   1.51044e+09  \n",
       "authorized_purchase_date_max                   1.51985e+09  \n",
       "authorized_category_3_B_mean                       0.96875  \n",
       "authorized_city_id_nunique                               6  \n",
       "unauthorized_transactions_count                          5  \n",
       "unauthorized_category_1_sum                              3  \n",
       "unauthorized_category_1_mean                           0.6  \n",
       "unauthorized_category_3_A_mean                           0  \n",
       "unauthorized_category_2_5.0_mean                         0  \n",
       "unauthorized_category_3_C_mean                         0.6  \n",
       "unauthorized_subsector_id_nunique                        2  \n",
       "unauthorized_merchant_id_nunique                         2  \n",
       "unauthorized_category_2_1.0_mean                         0  \n",
       "unauthorized_state_id_nunique                            2  \n",
       "unauthorized_category_2_4.0_mean                       0.4  \n",
       "unauthorized_purchase_amount_sum                   20.3528  \n",
       "unauthorized_purchase_amount_mean                  4.07056  \n",
       "unauthorized_purchase_amount_max                   7.19304  \n",
       "unauthorized_purchase_amount_min                 -0.512945  \n",
       "unauthorized_purchase_amount_std                   4.18495  \n",
       "unauthorized_installments_sum                           38  \n",
       "unauthorized_installments_mean                         7.6  \n",
       "unauthorized_installments_max                           12  \n",
       "unauthorized_installments_min                            1  \n",
       "unauthorized_installments_std                      6.02495  \n",
       "unauthorized_merchant_category_id_nunique                2  \n",
       "unauthorized_purchase_month_mean                       1.6  \n",
       "unauthorized_purchase_month_max                          2  \n",
       "unauthorized_purchase_month_min                          1  \n",
       "unauthorized_purchase_month_std                   0.547723  \n",
       "unauthorized_month_lag_min                              -1  \n",
       "unauthorized_month_lag_max                               0  \n",
       "unauthorized_category_2_3.0_mean                         0  \n",
       "unauthorized_category_2_2.0_mean                         0  \n",
       "unauthorized_purchase_date_ptp                 3.27433e+06  \n",
       "unauthorized_purchase_date_min                 1.51648e+09  \n",
       "unauthorized_purchase_date_max                 1.51976e+09  \n",
       "unauthorized_category_3_B_mean                         0.4  \n",
       "unauthorized_city_id_nunique                             2  \n",
       "new_transactions_count                                  36  \n",
       "new_category_1_sum                                       2  \n",
       "new_category_1_mean                              0.0555556  \n",
       "new_category_3_A_mean                                    0  \n",
       "new_category_2_5.0_mean                                  0  \n",
       "new_category_3_C_mean                            0.0277778  \n",
       "new_subsector_id_nunique                                10  \n",
       "new_merchant_id_nunique                                 36  \n",
       "new_category_2_1.0_mean                          0.0555556  \n",
       "new_state_id_nunique                                     5  \n",
       "new_category_2_4.0_mean                           0.694444  \n",
       "new_purchase_amount_sum                             -19.92  \n",
       "new_purchase_amount_mean                           -0.5537  \n",
       "new_purchase_amount_max                              0.451  \n",
       "new_purchase_amount_min                            -0.7393  \n",
       "new_purchase_amount_std                             0.2239  \n",
       "new_installments_sum                                    35  \n",
       "new_installments_mean                             0.972222  \n",
       "new_installments_max                                     2  \n",
       "new_installments_min                                    -1  \n",
       "new_installments_std                              0.376913  \n",
       "new_merchant_category_id_nunique                        17  \n",
       "new_purchase_month_mean                            3.55556  \n",
       "new_purchase_month_max                                   4  \n",
       "new_purchase_month_min                                   3  \n",
       "new_purchase_month_std                            0.503953  \n",
       "new_month_lag_min                                        1  \n",
       "new_month_lag_max                                        2  \n",
       "new_category_2_3.0_mean                           0.194444  \n",
       "new_category_2_2.0_mean                                  0  \n",
       "new_purchase_date_ptp                          4.94968e+06  \n",
       "new_purchase_date_min                          1.51999e+09  \n",
       "new_purchase_date_max                          1.52494e+09  \n",
       "new_category_3_B_mean                             0.944444  \n",
       "new_city_id_nunique                                      5  \n",
       "outliers                                                 0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_all(train_clean.head().T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>2017-04-01</td>\n",
       "      <td>2017-01-01</td>\n",
       "      <td>2017-08-01</td>\n",
       "      <td>2017-12-01</td>\n",
       "      <td>2015-12-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>C_ID_0ab67a22ab</td>\n",
       "      <td>C_ID_130fd0cbdd</td>\n",
       "      <td>C_ID_b709037bc5</td>\n",
       "      <td>C_ID_d27d835a9f</td>\n",
       "      <td>C_ID_2b5e3df5c2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>elapsed_time</th>\n",
       "      <td>306</td>\n",
       "      <td>396</td>\n",
       "      <td>184</td>\n",
       "      <td>62</td>\n",
       "      <td>793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1_5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2_1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2_2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2_3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_transactions_count</th>\n",
       "      <td>44</td>\n",
       "      <td>77</td>\n",
       "      <td>9</td>\n",
       "      <td>26</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_1_sum</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_1_mean</th>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.012987</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_3_A_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_5.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_3_C_mean</th>\n",
       "      <td>0.204545</td>\n",
       "      <td>0.025974</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.269231</td>\n",
       "      <td>0.0574713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_subsector_id_nunique</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_merchant_id_nunique</th>\n",
       "      <td>23</td>\n",
       "      <td>27</td>\n",
       "      <td>7</td>\n",
       "      <td>23</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_1.0_mean</th>\n",
       "      <td>0.772727</td>\n",
       "      <td>0</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>1</td>\n",
       "      <td>0.045977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_state_id_nunique</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_4.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.974026</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.908046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_sum</th>\n",
       "      <td>-28.0385</td>\n",
       "      <td>-48.4046</td>\n",
       "      <td>-0.894543</td>\n",
       "      <td>-13.6907</td>\n",
       "      <td>40.7014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_mean</th>\n",
       "      <td>-0.637239</td>\n",
       "      <td>-0.628632</td>\n",
       "      <td>-0.0993937</td>\n",
       "      <td>-0.526566</td>\n",
       "      <td>0.467832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_max</th>\n",
       "      <td>0.235676</td>\n",
       "      <td>0.318818</td>\n",
       "      <td>0.410134</td>\n",
       "      <td>0.0879652</td>\n",
       "      <td>15.7823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_min</th>\n",
       "      <td>-0.743902</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>-0.536537</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>-0.745405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_amount_std</th>\n",
       "      <td>0.169498</td>\n",
       "      <td>0.155571</td>\n",
       "      <td>0.398568</td>\n",
       "      <td>0.219162</td>\n",
       "      <td>3.08036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_sum</th>\n",
       "      <td>63</td>\n",
       "      <td>82</td>\n",
       "      <td>26</td>\n",
       "      <td>38</td>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_mean</th>\n",
       "      <td>1.43182</td>\n",
       "      <td>1.06494</td>\n",
       "      <td>2.88889</td>\n",
       "      <td>1.46154</td>\n",
       "      <td>1.10345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_max</th>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_min</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_installments_std</th>\n",
       "      <td>1.14927</td>\n",
       "      <td>0.408388</td>\n",
       "      <td>2.08833</td>\n",
       "      <td>1.47596</td>\n",
       "      <td>0.483074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_merchant_category_id_nunique</th>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_mean</th>\n",
       "      <td>8.52273</td>\n",
       "      <td>3.25974</td>\n",
       "      <td>9.88889</td>\n",
       "      <td>7.69231</td>\n",
       "      <td>5.35632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_max</th>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_min</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_month_std</th>\n",
       "      <td>2.57437</td>\n",
       "      <td>1.35144</td>\n",
       "      <td>3.21887</td>\n",
       "      <td>5.1363</td>\n",
       "      <td>3.38937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_month_lag_min</th>\n",
       "      <td>-8</td>\n",
       "      <td>-13</td>\n",
       "      <td>-6</td>\n",
       "      <td>-2</td>\n",
       "      <td>-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_month_lag_max</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_3.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.012987</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.045977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_2_2.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_date_ptp</th>\n",
       "      <td>2.31794e+07</td>\n",
       "      <td>3.46679e+07</td>\n",
       "      <td>1.39249e+07</td>\n",
       "      <td>6.73485e+06</td>\n",
       "      <td>3.61741e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_date_min</th>\n",
       "      <td>1.49133e+09</td>\n",
       "      <td>1.48432e+09</td>\n",
       "      <td>1.50367e+09</td>\n",
       "      <td>1.51239e+09</td>\n",
       "      <td>1.48355e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_purchase_date_max</th>\n",
       "      <td>1.51451e+09</td>\n",
       "      <td>1.51899e+09</td>\n",
       "      <td>1.5176e+09</td>\n",
       "      <td>1.51913e+09</td>\n",
       "      <td>1.51973e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_category_3_B_mean</th>\n",
       "      <td>0.795455</td>\n",
       "      <td>0.974026</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.653846</td>\n",
       "      <td>0.942529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_city_id_nunique</th>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_transactions_count</th>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_1_sum</th>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_1_mean</th>\n",
       "      <td>0.541667</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_3_A_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_5.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_3_C_mean</th>\n",
       "      <td>0.541667</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0434783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_subsector_id_nunique</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_merchant_id_nunique</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_1.0_mean</th>\n",
       "      <td>0.458333</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_state_id_nunique</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_4.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.956522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_sum</th>\n",
       "      <td>-12.6952</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>5.42338</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-15.562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_mean</th>\n",
       "      <td>-0.528967</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>1.35585</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.67661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_max</th>\n",
       "      <td>0.0600009</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>2.52587</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.361448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_min</th>\n",
       "      <td>-0.728876</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>0.214624</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.746757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_amount_std</th>\n",
       "      <td>0.214614</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.953347</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.232976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_sum</th>\n",
       "      <td>78</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_mean</th>\n",
       "      <td>3.25</td>\n",
       "      <td>1</td>\n",
       "      <td>4.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.04348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_max</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_min</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_installments_std</th>\n",
       "      <td>2.77018</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.35085</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.208514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_merchant_category_id_nunique</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_mean</th>\n",
       "      <td>8.08333</td>\n",
       "      <td>5</td>\n",
       "      <td>1.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.82609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_max</th>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_min</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_month_std</th>\n",
       "      <td>2.24416</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.57735</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.40553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_month_lag_min</th>\n",
       "      <td>-8</td>\n",
       "      <td>-9</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_month_lag_max</th>\n",
       "      <td>0</td>\n",
       "      <td>-9</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_3.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0434783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_2_2.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_date_ptp</th>\n",
       "      <td>1.98829e+07</td>\n",
       "      <td>0</td>\n",
       "      <td>1.91209e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.15479e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_date_min</th>\n",
       "      <td>1.4933e+09</td>\n",
       "      <td>1.49408e+09</td>\n",
       "      <td>1.51568e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.48344e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_purchase_date_max</th>\n",
       "      <td>1.51319e+09</td>\n",
       "      <td>1.49408e+09</td>\n",
       "      <td>1.51759e+09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.51499e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_category_3_B_mean</th>\n",
       "      <td>0.458333</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.956522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unauthorized_city_id_nunique</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_transactions_count</th>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_1_sum</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_1_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_3_A_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_5.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_3_C_mean</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_subsector_id_nunique</th>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_merchant_id_nunique</th>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_1.0_mean</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_state_id_nunique</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_4.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_sum</th>\n",
       "      <td>-1.777</td>\n",
       "      <td>-5.945</td>\n",
       "      <td>0.1797</td>\n",
       "      <td>-5.746</td>\n",
       "      <td>12.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_mean</th>\n",
       "      <td>-0.5923</td>\n",
       "      <td>-0.6606</td>\n",
       "      <td>0.08984</td>\n",
       "      <td>-0.574</td>\n",
       "      <td>2.012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_max</th>\n",
       "      <td>-0.3833</td>\n",
       "      <td>-0.5063</td>\n",
       "      <td>0.9043</td>\n",
       "      <td>-0.448</td>\n",
       "      <td>14.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_min</th>\n",
       "      <td>-0.722</td>\n",
       "      <td>-0.7407</td>\n",
       "      <td>-0.7246</td>\n",
       "      <td>-0.672</td>\n",
       "      <td>-0.704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_amount_std</th>\n",
       "      <td>0.1829</td>\n",
       "      <td>0.07117</td>\n",
       "      <td>1.152</td>\n",
       "      <td>0.0732</td>\n",
       "      <td>6.027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_sum</th>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_mean</th>\n",
       "      <td>1.66667</td>\n",
       "      <td>1.22222</td>\n",
       "      <td>5.5</td>\n",
       "      <td>2.9</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_max</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_min</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_installments_std</th>\n",
       "      <td>1.1547</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>6.36396</td>\n",
       "      <td>3.3483</td>\n",
       "      <td>0.983192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_merchant_category_id_nunique</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_mean</th>\n",
       "      <td>2</td>\n",
       "      <td>3.44444</td>\n",
       "      <td>3</td>\n",
       "      <td>3.3</td>\n",
       "      <td>3.16667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_max</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_min</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_month_std</th>\n",
       "      <td>0</td>\n",
       "      <td>0.527046</td>\n",
       "      <td>0</td>\n",
       "      <td>0.483046</td>\n",
       "      <td>0.408248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_month_lag_min</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_month_lag_max</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_3.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_2_2.0_mean</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_date_ptp</th>\n",
       "      <td>2.19386e+06</td>\n",
       "      <td>4.16769e+06</td>\n",
       "      <td>1.03075e+06</td>\n",
       "      <td>3.83855e+06</td>\n",
       "      <td>3.40269e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_date_min</th>\n",
       "      <td>1.51765e+09</td>\n",
       "      <td>1.52008e+09</td>\n",
       "      <td>1.51992e+09</td>\n",
       "      <td>1.52016e+09</td>\n",
       "      <td>1.52013e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_purchase_date_max</th>\n",
       "      <td>1.51984e+09</td>\n",
       "      <td>1.52425e+09</td>\n",
       "      <td>1.52095e+09</td>\n",
       "      <td>1.524e+09</td>\n",
       "      <td>1.52354e+09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_category_3_B_mean</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_city_id_nunique</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                         0                1  \\\n",
       "first_active_month                              2017-04-01       2017-01-01   \n",
       "card_id                                    C_ID_0ab67a22ab  C_ID_130fd0cbdd   \n",
       "feature_3                                                1                0   \n",
       "month                                                    4                1   \n",
       "year                                                  2017             2017   \n",
       "elapsed_time                                           306              396   \n",
       "feature_1_1                                              0                0   \n",
       "feature_1_2                                              0                1   \n",
       "feature_1_3                                              1                0   \n",
       "feature_1_4                                              0                0   \n",
       "feature_1_5                                              0                0   \n",
       "feature_2_1                                              0                0   \n",
       "feature_2_2                                              0                0   \n",
       "feature_2_3                                              1                1   \n",
       "authorized_transactions_count                           44               77   \n",
       "authorized_category_1_sum                               10                1   \n",
       "authorized_category_1_mean                        0.227273         0.012987   \n",
       "authorized_category_3_A_mean                             0                0   \n",
       "authorized_category_2_5.0_mean                           0                0   \n",
       "authorized_category_3_C_mean                      0.204545         0.025974   \n",
       "authorized_subsector_id_nunique                         12               12   \n",
       "authorized_merchant_id_nunique                          23               27   \n",
       "authorized_category_2_1.0_mean                    0.772727                0   \n",
       "authorized_state_id_nunique                              3                3   \n",
       "authorized_category_2_4.0_mean                           0         0.974026   \n",
       "authorized_purchase_amount_sum                    -28.0385         -48.4046   \n",
       "authorized_purchase_amount_mean                  -0.637239        -0.628632   \n",
       "authorized_purchase_amount_max                    0.235676         0.318818   \n",
       "authorized_purchase_amount_min                   -0.743902        -0.731881   \n",
       "authorized_purchase_amount_std                    0.169498         0.155571   \n",
       "authorized_installments_sum                             63               82   \n",
       "authorized_installments_mean                       1.43182          1.06494   \n",
       "authorized_installments_max                              6                4   \n",
       "authorized_installments_min                              1                1   \n",
       "authorized_installments_std                        1.14927         0.408388   \n",
       "authorized_merchant_category_id_nunique                 15               16   \n",
       "authorized_purchase_month_mean                     8.52273          3.25974   \n",
       "authorized_purchase_month_max                           12                5   \n",
       "authorized_purchase_month_min                            4                1   \n",
       "authorized_purchase_month_std                      2.57437          1.35144   \n",
       "authorized_month_lag_min                                -8              -13   \n",
       "authorized_month_lag_max                                 0                0   \n",
       "authorized_category_2_3.0_mean                           0         0.012987   \n",
       "authorized_category_2_2.0_mean                           0                0   \n",
       "authorized_purchase_date_ptp                   2.31794e+07      3.46679e+07   \n",
       "authorized_purchase_date_min                   1.49133e+09      1.48432e+09   \n",
       "authorized_purchase_date_max                   1.51451e+09      1.51899e+09   \n",
       "authorized_category_3_B_mean                      0.795455         0.974026   \n",
       "authorized_city_id_nunique                               7                4   \n",
       "unauthorized_transactions_count                         24                1   \n",
       "unauthorized_category_1_sum                             13                1   \n",
       "unauthorized_category_1_mean                      0.541667                1   \n",
       "unauthorized_category_3_A_mean                           0                0   \n",
       "unauthorized_category_2_5.0_mean                         0                0   \n",
       "unauthorized_category_3_C_mean                    0.541667                0   \n",
       "unauthorized_subsector_id_nunique                        7                1   \n",
       "unauthorized_merchant_id_nunique                         7                1   \n",
       "unauthorized_category_2_1.0_mean                  0.458333                0   \n",
       "unauthorized_state_id_nunique                            2                1   \n",
       "unauthorized_category_2_4.0_mean                         0                0   \n",
       "unauthorized_purchase_amount_sum                  -12.6952        -0.731881   \n",
       "unauthorized_purchase_amount_mean                -0.528967        -0.731881   \n",
       "unauthorized_purchase_amount_max                 0.0600009        -0.731881   \n",
       "unauthorized_purchase_amount_min                 -0.728876        -0.731881   \n",
       "unauthorized_purchase_amount_std                  0.214614              NaN   \n",
       "unauthorized_installments_sum                           78                1   \n",
       "unauthorized_installments_mean                        3.25                1   \n",
       "unauthorized_installments_max                           12                1   \n",
       "unauthorized_installments_min                            1                1   \n",
       "unauthorized_installments_std                      2.77018              NaN   \n",
       "unauthorized_merchant_category_id_nunique                7                1   \n",
       "unauthorized_purchase_month_mean                   8.08333                5   \n",
       "unauthorized_purchase_month_max                         12                5   \n",
       "unauthorized_purchase_month_min                          4                5   \n",
       "unauthorized_purchase_month_std                    2.24416              NaN   \n",
       "unauthorized_month_lag_min                              -8               -9   \n",
       "unauthorized_month_lag_max                               0               -9   \n",
       "unauthorized_category_2_3.0_mean                         0                0   \n",
       "unauthorized_category_2_2.0_mean                         0                0   \n",
       "unauthorized_purchase_date_ptp                 1.98829e+07                0   \n",
       "unauthorized_purchase_date_min                  1.4933e+09      1.49408e+09   \n",
       "unauthorized_purchase_date_max                 1.51319e+09      1.49408e+09   \n",
       "unauthorized_category_3_B_mean                    0.458333                1   \n",
       "unauthorized_city_id_nunique                             4                1   \n",
       "new_transactions_count                                   3                9   \n",
       "new_category_1_sum                                       0                2   \n",
       "new_category_1_mean                                      0         0.222222   \n",
       "new_category_3_A_mean                                    0                0   \n",
       "new_category_2_5.0_mean                                  0                0   \n",
       "new_category_3_C_mean                             0.333333         0.111111   \n",
       "new_subsector_id_nunique                                 3                6   \n",
       "new_merchant_id_nunique                                  3                9   \n",
       "new_category_2_1.0_mean                                  1                0   \n",
       "new_state_id_nunique                                     1                2   \n",
       "new_category_2_4.0_mean                                  0         0.777778   \n",
       "new_purchase_amount_sum                             -1.777           -5.945   \n",
       "new_purchase_amount_mean                           -0.5923          -0.6606   \n",
       "new_purchase_amount_max                            -0.3833          -0.5063   \n",
       "new_purchase_amount_min                             -0.722          -0.7407   \n",
       "new_purchase_amount_std                             0.1829          0.07117   \n",
       "new_installments_sum                                     5               11   \n",
       "new_installments_mean                              1.66667          1.22222   \n",
       "new_installments_max                                     3                3   \n",
       "new_installments_min                                     1                1   \n",
       "new_installments_std                                1.1547         0.666667   \n",
       "new_merchant_category_id_nunique                         3                8   \n",
       "new_purchase_month_mean                                  2          3.44444   \n",
       "new_purchase_month_max                                   2                4   \n",
       "new_purchase_month_min                                   2                3   \n",
       "new_purchase_month_std                                   0         0.527046   \n",
       "new_month_lag_min                                        2                1   \n",
       "new_month_lag_max                                        2                2   \n",
       "new_category_2_3.0_mean                                  0                0   \n",
       "new_category_2_2.0_mean                                  0                0   \n",
       "new_purchase_date_ptp                          2.19386e+06      4.16769e+06   \n",
       "new_purchase_date_min                          1.51765e+09      1.52008e+09   \n",
       "new_purchase_date_max                          1.51984e+09      1.52425e+09   \n",
       "new_category_3_B_mean                             0.666667         0.888889   \n",
       "new_city_id_nunique                                      3                2   \n",
       "\n",
       "                                                         2                3  \\\n",
       "first_active_month                              2017-08-01       2017-12-01   \n",
       "card_id                                    C_ID_b709037bc5  C_ID_d27d835a9f   \n",
       "feature_3                                                1                0   \n",
       "month                                                    8               12   \n",
       "year                                                  2017             2017   \n",
       "elapsed_time                                           184               62   \n",
       "feature_1_1                                              0                0   \n",
       "feature_1_2                                              0                1   \n",
       "feature_1_3                                              0                0   \n",
       "feature_1_4                                              0                0   \n",
       "feature_1_5                                              1                0   \n",
       "feature_2_1                                              1                1   \n",
       "feature_2_2                                              0                0   \n",
       "feature_2_3                                              0                0   \n",
       "authorized_transactions_count                            9               26   \n",
       "authorized_category_1_sum                                1                0   \n",
       "authorized_category_1_mean                        0.111111                0   \n",
       "authorized_category_3_A_mean                             0                0   \n",
       "authorized_category_2_5.0_mean                    0.777778                0   \n",
       "authorized_category_3_C_mean                      0.777778         0.269231   \n",
       "authorized_subsector_id_nunique                          5               11   \n",
       "authorized_merchant_id_nunique                           7               23   \n",
       "authorized_category_2_1.0_mean                    0.111111                1   \n",
       "authorized_state_id_nunique                              4                1   \n",
       "authorized_category_2_4.0_mean                           0                0   \n",
       "authorized_purchase_amount_sum                   -0.894543         -13.6907   \n",
       "authorized_purchase_amount_mean                 -0.0993937        -0.526566   \n",
       "authorized_purchase_amount_max                    0.410134        0.0879652   \n",
       "authorized_purchase_amount_min                   -0.536537        -0.731881   \n",
       "authorized_purchase_amount_std                    0.398568         0.219162   \n",
       "authorized_installments_sum                             26               38   \n",
       "authorized_installments_mean                       2.88889          1.46154   \n",
       "authorized_installments_max                              6                6   \n",
       "authorized_installments_min                             -1               -1   \n",
       "authorized_installments_std                        2.08833          1.47596   \n",
       "authorized_merchant_category_id_nunique                  6               18   \n",
       "authorized_purchase_month_mean                     9.88889          7.69231   \n",
       "authorized_purchase_month_max                           12               12   \n",
       "authorized_purchase_month_min                            2                1   \n",
       "authorized_purchase_month_std                      3.21887           5.1363   \n",
       "authorized_month_lag_min                                -6               -2   \n",
       "authorized_month_lag_max                                 0                0   \n",
       "authorized_category_2_3.0_mean                           0                0   \n",
       "authorized_category_2_2.0_mean                           0                0   \n",
       "authorized_purchase_date_ptp                   1.39249e+07      6.73485e+06   \n",
       "authorized_purchase_date_min                   1.50367e+09      1.51239e+09   \n",
       "authorized_purchase_date_max                    1.5176e+09      1.51913e+09   \n",
       "authorized_category_3_B_mean                      0.111111         0.653846   \n",
       "authorized_city_id_nunique                               4                1   \n",
       "unauthorized_transactions_count                          4              NaN   \n",
       "unauthorized_category_1_sum                              0              NaN   \n",
       "unauthorized_category_1_mean                             0              NaN   \n",
       "unauthorized_category_3_A_mean                           0              NaN   \n",
       "unauthorized_category_2_5.0_mean                         1              NaN   \n",
       "unauthorized_category_3_C_mean                         0.5              NaN   \n",
       "unauthorized_subsector_id_nunique                        1              NaN   \n",
       "unauthorized_merchant_id_nunique                         2              NaN   \n",
       "unauthorized_category_2_1.0_mean                         0              NaN   \n",
       "unauthorized_state_id_nunique                            1              NaN   \n",
       "unauthorized_category_2_4.0_mean                         0              NaN   \n",
       "unauthorized_purchase_amount_sum                   5.42338              NaN   \n",
       "unauthorized_purchase_amount_mean                  1.35585              NaN   \n",
       "unauthorized_purchase_amount_max                   2.52587              NaN   \n",
       "unauthorized_purchase_amount_min                  0.214624              NaN   \n",
       "unauthorized_purchase_amount_std                  0.953347              NaN   \n",
       "unauthorized_installments_sum                           18              NaN   \n",
       "unauthorized_installments_mean                         4.5              NaN   \n",
       "unauthorized_installments_max                           10              NaN   \n",
       "unauthorized_installments_min                           -1              NaN   \n",
       "unauthorized_installments_std                      6.35085              NaN   \n",
       "unauthorized_merchant_category_id_nunique                2              NaN   \n",
       "unauthorized_purchase_month_mean                       1.5              NaN   \n",
       "unauthorized_purchase_month_max                          2              NaN   \n",
       "unauthorized_purchase_month_min                          1              NaN   \n",
       "unauthorized_purchase_month_std                    0.57735              NaN   \n",
       "unauthorized_month_lag_min                              -1              NaN   \n",
       "unauthorized_month_lag_max                               0              NaN   \n",
       "unauthorized_category_2_3.0_mean                         0              NaN   \n",
       "unauthorized_category_2_2.0_mean                         0              NaN   \n",
       "unauthorized_purchase_date_ptp                 1.91209e+06              NaN   \n",
       "unauthorized_purchase_date_min                 1.51568e+09              NaN   \n",
       "unauthorized_purchase_date_max                 1.51759e+09              NaN   \n",
       "unauthorized_category_3_B_mean                           0              NaN   \n",
       "unauthorized_city_id_nunique                             1              NaN   \n",
       "new_transactions_count                                   2               10   \n",
       "new_category_1_sum                                       1                1   \n",
       "new_category_1_mean                                    0.5              0.1   \n",
       "new_category_3_A_mean                                    0                0   \n",
       "new_category_2_5.0_mean                                0.5              0.1   \n",
       "new_category_3_C_mean                                  0.5              0.5   \n",
       "new_subsector_id_nunique                                 2                8   \n",
       "new_merchant_id_nunique                                  2               10   \n",
       "new_category_2_1.0_mean                                  0              0.8   \n",
       "new_state_id_nunique                                     2                3   \n",
       "new_category_2_4.0_mean                                  0                0   \n",
       "new_purchase_amount_sum                             0.1797           -5.746   \n",
       "new_purchase_amount_mean                           0.08984           -0.574   \n",
       "new_purchase_amount_max                             0.9043           -0.448   \n",
       "new_purchase_amount_min                            -0.7246           -0.672   \n",
       "new_purchase_amount_std                              1.152           0.0732   \n",
       "new_installments_sum                                    11               29   \n",
       "new_installments_mean                                  5.5              2.9   \n",
       "new_installments_max                                    10               12   \n",
       "new_installments_min                                     1                1   \n",
       "new_installments_std                               6.36396           3.3483   \n",
       "new_merchant_category_id_nunique                         2               10   \n",
       "new_purchase_month_mean                                  3              3.3   \n",
       "new_purchase_month_max                                   3                4   \n",
       "new_purchase_month_min                                   3                3   \n",
       "new_purchase_month_std                                   0         0.483046   \n",
       "new_month_lag_min                                        1                1   \n",
       "new_month_lag_max                                        1                2   \n",
       "new_category_2_3.0_mean                                  0                0   \n",
       "new_category_2_2.0_mean                                  0                0   \n",
       "new_purchase_date_ptp                          1.03075e+06      3.83855e+06   \n",
       "new_purchase_date_min                          1.51992e+09      1.52016e+09   \n",
       "new_purchase_date_max                          1.52095e+09        1.524e+09   \n",
       "new_category_3_B_mean                                  0.5              0.5   \n",
       "new_city_id_nunique                                      2                3   \n",
       "\n",
       "                                                         4  \n",
       "first_active_month                              2015-12-01  \n",
       "card_id                                    C_ID_2b5e3df5c2  \n",
       "feature_3                                                1  \n",
       "month                                                   12  \n",
       "year                                                  2015  \n",
       "elapsed_time                                           793  \n",
       "feature_1_1                                              0  \n",
       "feature_1_2                                              0  \n",
       "feature_1_3                                              0  \n",
       "feature_1_4                                              0  \n",
       "feature_1_5                                              1  \n",
       "feature_2_1                                              1  \n",
       "feature_2_2                                              0  \n",
       "feature_2_3                                              0  \n",
       "authorized_transactions_count                           87  \n",
       "authorized_category_1_sum                                0  \n",
       "authorized_category_1_mean                               0  \n",
       "authorized_category_3_A_mean                             0  \n",
       "authorized_category_2_5.0_mean                           0  \n",
       "authorized_category_3_C_mean                     0.0574713  \n",
       "authorized_subsector_id_nunique                         15  \n",
       "authorized_merchant_id_nunique                          45  \n",
       "authorized_category_2_1.0_mean                    0.045977  \n",
       "authorized_state_id_nunique                              3  \n",
       "authorized_category_2_4.0_mean                    0.908046  \n",
       "authorized_purchase_amount_sum                     40.7014  \n",
       "authorized_purchase_amount_mean                   0.467832  \n",
       "authorized_purchase_amount_max                     15.7823  \n",
       "authorized_purchase_amount_min                   -0.745405  \n",
       "authorized_purchase_amount_std                     3.08036  \n",
       "authorized_installments_sum                             96  \n",
       "authorized_installments_mean                       1.10345  \n",
       "authorized_installments_max                              4  \n",
       "authorized_installments_min                              1  \n",
       "authorized_installments_std                       0.483074  \n",
       "authorized_merchant_category_id_nunique                 30  \n",
       "authorized_purchase_month_mean                     5.35632  \n",
       "authorized_purchase_month_max                           12  \n",
       "authorized_purchase_month_min                            1  \n",
       "authorized_purchase_month_std                      3.38937  \n",
       "authorized_month_lag_min                               -13  \n",
       "authorized_month_lag_max                                 0  \n",
       "authorized_category_2_3.0_mean                    0.045977  \n",
       "authorized_category_2_2.0_mean                           0  \n",
       "authorized_purchase_date_ptp                   3.61741e+07  \n",
       "authorized_purchase_date_min                   1.48355e+09  \n",
       "authorized_purchase_date_max                   1.51973e+09  \n",
       "authorized_category_3_B_mean                      0.942529  \n",
       "authorized_city_id_nunique                               4  \n",
       "unauthorized_transactions_count                         23  \n",
       "unauthorized_category_1_sum                              0  \n",
       "unauthorized_category_1_mean                             0  \n",
       "unauthorized_category_3_A_mean                           0  \n",
       "unauthorized_category_2_5.0_mean                         0  \n",
       "unauthorized_category_3_C_mean                   0.0434783  \n",
       "unauthorized_subsector_id_nunique                        4  \n",
       "unauthorized_merchant_id_nunique                         5  \n",
       "unauthorized_category_2_1.0_mean                         0  \n",
       "unauthorized_state_id_nunique                            3  \n",
       "unauthorized_category_2_4.0_mean                  0.956522  \n",
       "unauthorized_purchase_amount_sum                   -15.562  \n",
       "unauthorized_purchase_amount_mean                 -0.67661  \n",
       "unauthorized_purchase_amount_max                  0.361448  \n",
       "unauthorized_purchase_amount_min                 -0.746757  \n",
       "unauthorized_purchase_amount_std                  0.232976  \n",
       "unauthorized_installments_sum                           24  \n",
       "unauthorized_installments_mean                     1.04348  \n",
       "unauthorized_installments_max                            2  \n",
       "unauthorized_installments_min                            1  \n",
       "unauthorized_installments_std                     0.208514  \n",
       "unauthorized_merchant_category_id_nunique                4  \n",
       "unauthorized_purchase_month_mean                   2.82609  \n",
       "unauthorized_purchase_month_max                         12  \n",
       "unauthorized_purchase_month_min                          1  \n",
       "unauthorized_purchase_month_std                    2.40553  \n",
       "unauthorized_month_lag_min                             -13  \n",
       "unauthorized_month_lag_max                              -1  \n",
       "unauthorized_category_2_3.0_mean                 0.0434783  \n",
       "unauthorized_category_2_2.0_mean                         0  \n",
       "unauthorized_purchase_date_ptp                 3.15479e+07  \n",
       "unauthorized_purchase_date_min                 1.48344e+09  \n",
       "unauthorized_purchase_date_max                 1.51499e+09  \n",
       "unauthorized_category_3_B_mean                    0.956522  \n",
       "unauthorized_city_id_nunique                             3  \n",
       "new_transactions_count                                   6  \n",
       "new_category_1_sum                                       0  \n",
       "new_category_1_mean                                      0  \n",
       "new_category_3_A_mean                                    0  \n",
       "new_category_2_5.0_mean                                  0  \n",
       "new_category_3_C_mean                             0.166667  \n",
       "new_subsector_id_nunique                                 4  \n",
       "new_merchant_id_nunique                                  6  \n",
       "new_category_2_1.0_mean                                  0  \n",
       "new_state_id_nunique                                     2  \n",
       "new_category_2_4.0_mean                           0.833333  \n",
       "new_purchase_amount_sum                              12.06  \n",
       "new_purchase_amount_mean                             2.012  \n",
       "new_purchase_amount_max                              14.28  \n",
       "new_purchase_amount_min                             -0.704  \n",
       "new_purchase_amount_std                              6.027  \n",
       "new_installments_sum                                     5  \n",
       "new_installments_mean                             0.833333  \n",
       "new_installments_max                                     2  \n",
       "new_installments_min                                    -1  \n",
       "new_installments_std                              0.983192  \n",
       "new_merchant_category_id_nunique                         5  \n",
       "new_purchase_month_mean                            3.16667  \n",
       "new_purchase_month_max                                   4  \n",
       "new_purchase_month_min                                   3  \n",
       "new_purchase_month_std                            0.408248  \n",
       "new_month_lag_min                                        1  \n",
       "new_month_lag_max                                        2  \n",
       "new_category_2_3.0_mean                           0.166667  \n",
       "new_category_2_2.0_mean                                  0  \n",
       "new_purchase_date_ptp                          3.40269e+06  \n",
       "new_purchase_date_min                          1.52013e+09  \n",
       "new_purchase_date_max                          1.52354e+09  \n",
       "new_category_3_B_mean                             0.666667  \n",
       "new_city_id_nunique                                      2  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_all(test_clean.head().T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_score(m, x_trn, y_trn, x_val, y_val):\n",
    "    res = [rmse(m.predict(x_trn), y_trn), rmse(m.predict(x_val), y_val),\n",
    "                m.score(x_trn, y_trn), m.score(x_val, y_val)]\n",
    "    if hasattr(m, 'oob_score_'): res.append(m.oob_score_)\n",
    "    print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(preds, target): \n",
    "    return np.sqrt(mean_squared_error(preds, target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_args = dict({\n",
    "    \"n_estimators\": 20, \n",
    "    \"min_samples_leaf\":3, \n",
    "    \"max_features\": 0.9, \n",
    "    \"n_jobs\":-1, \n",
    "    \"oob_score\":True\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = train_clean['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24230040"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_clean.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201917, 120)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_clean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_clean.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_datepart(train_df, 'first_active_month')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "card_id                                       object\n",
       "feature_3                                      int64\n",
       "month                                          int64\n",
       "year                                           int64\n",
       "elapsed_time                                   int64\n",
       "feature_1_1                                    int64\n",
       "feature_1_2                                    int64\n",
       "feature_1_3                                    int64\n",
       "feature_1_4                                    int64\n",
       "feature_1_5                                    int64\n",
       "feature_2_1                                    int64\n",
       "feature_2_2                                    int64\n",
       "feature_2_3                                    int64\n",
       "authorized_transactions_count                  int64\n",
       "authorized_category_1_sum                    float64\n",
       "authorized_category_1_mean                   float64\n",
       "authorized_category_3_A_mean                 float64\n",
       "authorized_category_2_5.0_mean               float64\n",
       "authorized_category_3_C_mean                 float64\n",
       "authorized_subsector_id_nunique                int64\n",
       "authorized_merchant_id_nunique                 int64\n",
       "authorized_category_2_1.0_mean               float64\n",
       "authorized_state_id_nunique                    int64\n",
       "authorized_category_2_4.0_mean               float64\n",
       "authorized_purchase_amount_sum               float64\n",
       "authorized_purchase_amount_mean              float64\n",
       "authorized_purchase_amount_max               float64\n",
       "authorized_purchase_amount_min               float64\n",
       "authorized_purchase_amount_std               float64\n",
       "authorized_installments_sum                    int64\n",
       "authorized_installments_mean                 float64\n",
       "authorized_installments_max                    int64\n",
       "authorized_installments_min                    int64\n",
       "authorized_installments_std                  float64\n",
       "authorized_merchant_category_id_nunique        int64\n",
       "authorized_purchase_month_mean               float64\n",
       "authorized_purchase_month_max                  int64\n",
       "authorized_purchase_month_min                  int64\n",
       "authorized_purchase_month_std                float64\n",
       "authorized_month_lag_min                       int64\n",
       "authorized_month_lag_max                       int64\n",
       "authorized_category_2_3.0_mean               float64\n",
       "authorized_category_2_2.0_mean               float64\n",
       "authorized_purchase_date_ptp                 float64\n",
       "authorized_purchase_date_min                 float64\n",
       "authorized_purchase_date_max                 float64\n",
       "authorized_category_3_B_mean                 float64\n",
       "authorized_city_id_nunique                     int64\n",
       "unauthorized_transactions_count              float64\n",
       "unauthorized_category_1_sum                  float64\n",
       "unauthorized_category_1_mean                 float64\n",
       "unauthorized_category_3_A_mean               float64\n",
       "unauthorized_category_2_5.0_mean             float64\n",
       "unauthorized_category_3_C_mean               float64\n",
       "unauthorized_subsector_id_nunique            float64\n",
       "unauthorized_merchant_id_nunique             float64\n",
       "unauthorized_category_2_1.0_mean             float64\n",
       "unauthorized_state_id_nunique                float64\n",
       "unauthorized_category_2_4.0_mean             float64\n",
       "unauthorized_purchase_amount_sum             float64\n",
       "unauthorized_purchase_amount_mean            float64\n",
       "unauthorized_purchase_amount_max             float64\n",
       "unauthorized_purchase_amount_min             float64\n",
       "unauthorized_purchase_amount_std             float64\n",
       "unauthorized_installments_sum                float64\n",
       "unauthorized_installments_mean               float64\n",
       "unauthorized_installments_max                float64\n",
       "unauthorized_installments_min                float64\n",
       "unauthorized_installments_std                float64\n",
       "unauthorized_merchant_category_id_nunique    float64\n",
       "unauthorized_purchase_month_mean             float64\n",
       "unauthorized_purchase_month_max              float64\n",
       "unauthorized_purchase_month_min              float64\n",
       "unauthorized_purchase_month_std              float64\n",
       "unauthorized_month_lag_min                   float64\n",
       "unauthorized_month_lag_max                   float64\n",
       "unauthorized_category_2_3.0_mean             float64\n",
       "unauthorized_category_2_2.0_mean             float64\n",
       "unauthorized_purchase_date_ptp               float64\n",
       "unauthorized_purchase_date_min               float64\n",
       "unauthorized_purchase_date_max               float64\n",
       "unauthorized_category_3_B_mean               float64\n",
       "unauthorized_city_id_nunique                 float64\n",
       "new_transactions_count                       float64\n",
       "new_category_1_sum                           float64\n",
       "new_category_1_mean                          float64\n",
       "new_category_3_A_mean                        float64\n",
       "new_category_2_5.0_mean                      float64\n",
       "new_category_3_C_mean                        float64\n",
       "new_subsector_id_nunique                     float64\n",
       "new_merchant_id_nunique                      float64\n",
       "new_category_2_1.0_mean                      float64\n",
       "new_state_id_nunique                         float64\n",
       "new_category_2_4.0_mean                      float64\n",
       "new_purchase_amount_sum                      float64\n",
       "new_purchase_amount_mean                     float64\n",
       "new_purchase_amount_max                      float64\n",
       "new_purchase_amount_min                      float64\n",
       "new_purchase_amount_std                      float64\n",
       "new_installments_sum                         float64\n",
       "new_installments_mean                        float64\n",
       "new_installments_max                         float64\n",
       "new_installments_min                         float64\n",
       "new_installments_std                         float64\n",
       "new_merchant_category_id_nunique             float64\n",
       "new_purchase_month_mean                      float64\n",
       "new_purchase_month_max                       float64\n",
       "new_purchase_month_min                       float64\n",
       "new_purchase_month_std                       float64\n",
       "new_month_lag_min                            float64\n",
       "new_month_lag_max                            float64\n",
       "new_category_2_3.0_mean                      float64\n",
       "new_category_2_2.0_mean                      float64\n",
       "new_purchase_date_ptp                        float64\n",
       "new_purchase_date_min                        float64\n",
       "new_purchase_date_max                        float64\n",
       "new_category_3_B_mean                        float64\n",
       "new_city_id_nunique                          float64\n",
       "outliers                                       int64\n",
       "first_active_monthYear                         int64\n",
       "first_active_monthMonth                        int64\n",
       "first_active_monthWeek                         int64\n",
       "first_active_monthDay                          int64\n",
       "first_active_monthDayofweek                    int64\n",
       "first_active_monthDayofyear                    int64\n",
       "first_active_monthIs_month_end                  bool\n",
       "first_active_monthIs_month_start                bool\n",
       "first_active_monthIs_quarter_end                bool\n",
       "first_active_monthIs_quarter_start              bool\n",
       "first_active_monthIs_year_end                   bool\n",
       "first_active_monthIs_year_start                 bool\n",
       "first_active_monthElapsed                      int64\n",
       "dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_all(train_df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card_id</th>\n",
       "      <th>feature_3</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "      <th>elapsed_time</th>\n",
       "      <th>feature_1_1</th>\n",
       "      <th>feature_1_2</th>\n",
       "      <th>feature_1_3</th>\n",
       "      <th>feature_1_4</th>\n",
       "      <th>feature_1_5</th>\n",
       "      <th>...</th>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <th>first_active_monthDayofweek</th>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <th>first_active_monthIs_month_end</th>\n",
       "      <th>first_active_monthIs_month_start</th>\n",
       "      <th>first_active_monthIs_quarter_end</th>\n",
       "      <th>first_active_monthIs_quarter_start</th>\n",
       "      <th>first_active_monthIs_year_end</th>\n",
       "      <th>first_active_monthIs_year_start</th>\n",
       "      <th>first_active_monthElapsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C_ID_92a2005557</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>2017</td>\n",
       "      <td>245</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>152</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1496275200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C_ID_3d0044924f</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2017</td>\n",
       "      <td>396</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>1483228800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C_ID_d639edf6cd</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>2016</td>\n",
       "      <td>549</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>214</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1470009600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C_ID_186d6a6901</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2017</td>\n",
       "      <td>153</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>244</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1504224000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C_ID_cdbd2c0db2</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>2017</td>\n",
       "      <td>92</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>305</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1509494400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 132 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           card_id  feature_3  month  year  elapsed_time  feature_1_1  \\\n",
       "0  C_ID_92a2005557          1      6  2017           245            0   \n",
       "1  C_ID_3d0044924f          0      1  2017           396            0   \n",
       "2  C_ID_d639edf6cd          0      8  2016           549            0   \n",
       "3  C_ID_186d6a6901          0      9  2017           153            0   \n",
       "4  C_ID_cdbd2c0db2          0     11  2017            92            1   \n",
       "\n",
       "   feature_1_2  feature_1_3  feature_1_4  feature_1_5  \\\n",
       "0            0            0            0            1   \n",
       "1            0            0            1            0   \n",
       "2            1            0            0            0   \n",
       "3            0            0            1            0   \n",
       "4            0            0            0            0   \n",
       "\n",
       "             ...              first_active_monthDay  \\\n",
       "0            ...                                  1   \n",
       "1            ...                                  1   \n",
       "2            ...                                  1   \n",
       "3            ...                                  1   \n",
       "4            ...                                  1   \n",
       "\n",
       "   first_active_monthDayofweek  first_active_monthDayofyear  \\\n",
       "0                            3                          152   \n",
       "1                            6                            1   \n",
       "2                            0                          214   \n",
       "3                            4                          244   \n",
       "4                            2                          305   \n",
       "\n",
       "   first_active_monthIs_month_end  first_active_monthIs_month_start  \\\n",
       "0                           False                              True   \n",
       "1                           False                              True   \n",
       "2                           False                              True   \n",
       "3                           False                              True   \n",
       "4                           False                              True   \n",
       "\n",
       "   first_active_monthIs_quarter_end  first_active_monthIs_quarter_start  \\\n",
       "0                             False                               False   \n",
       "1                             False                                True   \n",
       "2                             False                               False   \n",
       "3                             False                               False   \n",
       "4                             False                               False   \n",
       "\n",
       "   first_active_monthIs_year_end  first_active_monthIs_year_start  \\\n",
       "0                          False                            False   \n",
       "1                          False                             True   \n",
       "2                          False                            False   \n",
       "3                          False                            False   \n",
       "4                          False                            False   \n",
       "\n",
       "   first_active_monthElapsed  \n",
       "0                 1496275200  \n",
       "1                 1483228800  \n",
       "2                 1470009600  \n",
       "3                 1504224000  \n",
       "4                 1509494400  \n",
       "\n",
       "[5 rows x 132 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.drop('card_id', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2 = train_df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df2.drop('outliers', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cats(train_df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_old.drop('card_id', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hot_ext = feather.read_dataframe('df_hot_ext')\n",
    "df_hot_test_ext = feather.read_dataframe('df_hot_test_ext')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "for df in [df_hot_ext, df_hot_test_ext]:\n",
    "    for f in ['purchase_date_amax','purchase_date_amin','purchase_date_amax_old',\\\n",
    "                     'purchase_date_amin_old']:\n",
    "        df[f] = df[f].astype(np.int64) * 1e-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_hot_ext.drop('is_outlier', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>counts</th>\n",
       "      <th>uniques</th>\n",
       "      <th>missing</th>\n",
       "      <th>missing_perc</th>\n",
       "      <th>types</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>month_diff_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>11.7556</td>\n",
       "      <td>2.39709</td>\n",
       "      <td>10</td>\n",
       "      <td>10.5543</td>\n",
       "      <td>10.6744</td>\n",
       "      <td>11.6591</td>\n",
       "      <td>23</td>\n",
       "      <td>201917</td>\n",
       "      <td>32343</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>6.37936</td>\n",
       "      <td>19.5725</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1786</td>\n",
       "      <td>201917</td>\n",
       "      <td>357</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.893353</td>\n",
       "      <td>0.114446</td>\n",
       "      <td>0.0304878</td>\n",
       "      <td>0.852941</td>\n",
       "      <td>0.926829</td>\n",
       "      <td>0.972516</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>11356</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.173108</td>\n",
       "      <td>1.45139</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.634209</td>\n",
       "      <td>-0.51302</td>\n",
       "      <td>-0.251063</td>\n",
       "      <td>76.6829</td>\n",
       "      <td>201917</td>\n",
       "      <td>43853</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-3.84099</td>\n",
       "      <td>2.12349</td>\n",
       "      <td>-12.4524</td>\n",
       "      <td>-5.52062</td>\n",
       "      <td>-3.57143</td>\n",
       "      <td>-2.03497</td>\n",
       "      <td>-0.0149254</td>\n",
       "      <td>201917</td>\n",
       "      <td>54905</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amax_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.51585</td>\n",
       "      <td>0.00645884</td>\n",
       "      <td>1.48594</td>\n",
       "      <td>1.51475</td>\n",
       "      <td>1.5189</td>\n",
       "      <td>1.51967</td>\n",
       "      <td>1.51986</td>\n",
       "      <td>201917</td>\n",
       "      <td>186997</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.10108</td>\n",
       "      <td>0.208051</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0909091</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>10370</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amin_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.49475</td>\n",
       "      <td>0.0102942</td>\n",
       "      <td>1.48323</td>\n",
       "      <td>1.48434</td>\n",
       "      <td>1.49262</td>\n",
       "      <td>1.50419</td>\n",
       "      <td>1.51476</td>\n",
       "      <td>201917</td>\n",
       "      <td>193984</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.729993</td>\n",
       "      <td>0.0918995</td>\n",
       "      <td>-0.746908</td>\n",
       "      <td>-0.743902</td>\n",
       "      <td>-0.740296</td>\n",
       "      <td>-0.731881</td>\n",
       "      <td>28.5548</td>\n",
       "      <td>201917</td>\n",
       "      <td>7432</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hist_first_buy</th>\n",
       "      <td>201917</td>\n",
       "      <td>118.748</td>\n",
       "      <td>224.136</td>\n",
       "      <td>-331</td>\n",
       "      <td>10</td>\n",
       "      <td>22</td>\n",
       "      <td>123</td>\n",
       "      <td>1995</td>\n",
       "      <td>201917</td>\n",
       "      <td>1905</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_ptp_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>243.678</td>\n",
       "      <td>118.292</td>\n",
       "      <td>1</td>\n",
       "      <td>138</td>\n",
       "      <td>233</td>\n",
       "      <td>361</td>\n",
       "      <td>423</td>\n",
       "      <td>201917</td>\n",
       "      <td>422</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>2.45943</td>\n",
       "      <td>1115.17</td>\n",
       "      <td>-0.745371</td>\n",
       "      <td>-0.679118</td>\n",
       "      <td>-0.621435</td>\n",
       "      <td>-0.501713</td>\n",
       "      <td>500883</td>\n",
       "      <td>201917</td>\n",
       "      <td>201640</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_max_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>66.6717</td>\n",
       "      <td>13448.5</td>\n",
       "      <td>-0.745315</td>\n",
       "      <td>-0.347203</td>\n",
       "      <td>0.123548</td>\n",
       "      <td>1.27716</td>\n",
       "      <td>6.0106e+06</td>\n",
       "      <td>201917</td>\n",
       "      <td>74899</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.276675</td>\n",
       "      <td>0.138688</td>\n",
       "      <td>0</td>\n",
       "      <td>0.186047</td>\n",
       "      <td>0.269663</td>\n",
       "      <td>0.356522</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>15633</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-400.169</td>\n",
       "      <td>610.571</td>\n",
       "      <td>-14172</td>\n",
       "      <td>-471</td>\n",
       "      <td>-179</td>\n",
       "      <td>-66</td>\n",
       "      <td>-1</td>\n",
       "      <td>201917</td>\n",
       "      <td>4267</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>inverse_avg_transactions_per_day_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>6.07849</td>\n",
       "      <td>6.93816</td>\n",
       "      <td>0.0869565</td>\n",
       "      <td>2</td>\n",
       "      <td>3.76147</td>\n",
       "      <td>7.36</td>\n",
       "      <td>135.667</td>\n",
       "      <td>201917</td>\n",
       "      <td>45200</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_since_last_transaction</th>\n",
       "      <td>201917</td>\n",
       "      <td>303.57</td>\n",
       "      <td>69.9628</td>\n",
       "      <td>256</td>\n",
       "      <td>263</td>\n",
       "      <td>277</td>\n",
       "      <td>305</td>\n",
       "      <td>679</td>\n",
       "      <td>201917</td>\n",
       "      <td>407</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_date_amax</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.353809</td>\n",
       "      <td>3.3431</td>\n",
       "      <td>-9.22337</td>\n",
       "      <td>1.51688</td>\n",
       "      <td>1.52278</td>\n",
       "      <td>1.52451</td>\n",
       "      <td>1.52513</td>\n",
       "      <td>201917</td>\n",
       "      <td>173712</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_std_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>12.6537</td>\n",
       "      <td>3865.24</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0766826</td>\n",
       "      <td>0.160484</td>\n",
       "      <td>0.361742</td>\n",
       "      <td>1.73511e+06</td>\n",
       "      <td>201917</td>\n",
       "      <td>201633</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_mean_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.863678</td>\n",
       "      <td>1.13581</td>\n",
       "      <td>-0.692308</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.34426</td>\n",
       "      <td>79.3077</td>\n",
       "      <td>201917</td>\n",
       "      <td>21136</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_var_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.217643</td>\n",
       "      <td>0.0521406</td>\n",
       "      <td>0</td>\n",
       "      <td>0.204545</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.250065</td>\n",
       "      <td>0.5</td>\n",
       "      <td>201917</td>\n",
       "      <td>78101</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_mean</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.537137</td>\n",
       "      <td>0.525156</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.687445</td>\n",
       "      <td>-0.641737</td>\n",
       "      <td>-0.552641</td>\n",
       "      <td>44.1824</td>\n",
       "      <td>201917</td>\n",
       "      <td>146245</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_min</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.667412</td>\n",
       "      <td>0.409663</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>-0.734887</td>\n",
       "      <td>-0.723466</td>\n",
       "      <td>-0.699725</td>\n",
       "      <td>44.1824</td>\n",
       "      <td>201917</td>\n",
       "      <td>15978</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_std_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.459549</td>\n",
       "      <td>0.0803638</td>\n",
       "      <td>0</td>\n",
       "      <td>0.452267</td>\n",
       "      <td>0.485071</td>\n",
       "      <td>0.500065</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>201917</td>\n",
       "      <td>66347</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount_sum</th>\n",
       "      <td>201917</td>\n",
       "      <td>-3.62723</td>\n",
       "      <td>4.16273</td>\n",
       "      <td>-70.8135</td>\n",
       "      <td>-4.82878</td>\n",
       "      <td>-2.68625</td>\n",
       "      <td>-1.3437</td>\n",
       "      <td>109.086</td>\n",
       "      <td>201917</td>\n",
       "      <td>149496</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_std_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>2.40891</td>\n",
       "      <td>1.17382</td>\n",
       "      <td>0.122169</td>\n",
       "      <td>1.37663</td>\n",
       "      <td>2.2734</td>\n",
       "      <td>3.44656</td>\n",
       "      <td>7</td>\n",
       "      <td>201917</td>\n",
       "      <td>160944</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_since_last_transaction_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>363.735</td>\n",
       "      <td>74.7394</td>\n",
       "      <td>317</td>\n",
       "      <td>319</td>\n",
       "      <td>328</td>\n",
       "      <td>376</td>\n",
       "      <td>710</td>\n",
       "      <td>201917</td>\n",
       "      <td>393</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id_nunique_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>33.7136</td>\n",
       "      <td>30.6869</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>25</td>\n",
       "      <td>44</td>\n",
       "      <td>398</td>\n",
       "      <td>201917</td>\n",
       "      <td>313</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_nunique_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>18.278</td>\n",
       "      <td>11.4705</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>24</td>\n",
       "      <td>95</td>\n",
       "      <td>201917</td>\n",
       "      <td>92</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_first_buy</th>\n",
       "      <td>201917</td>\n",
       "      <td>372.072</td>\n",
       "      <td>270.356</td>\n",
       "      <td>16</td>\n",
       "      <td>189</td>\n",
       "      <td>294</td>\n",
       "      <td>460</td>\n",
       "      <td>2351</td>\n",
       "      <td>201917</td>\n",
       "      <td>1898</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_mean</th>\n",
       "      <td>201917</td>\n",
       "      <td>11.6172</td>\n",
       "      <td>2.29955</td>\n",
       "      <td>10</td>\n",
       "      <td>10.5</td>\n",
       "      <td>10.7857</td>\n",
       "      <td>11.25</td>\n",
       "      <td>23</td>\n",
       "      <td>201917</td>\n",
       "      <td>2783</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_sum_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>81.559</td>\n",
       "      <td>99.2434</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>48</td>\n",
       "      <td>100</td>\n",
       "      <td>2537</td>\n",
       "      <td>201917</td>\n",
       "      <td>973</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthDayofyear</th>\n",
       "      <td>201917</td>\n",
       "      <td>194.482</td>\n",
       "      <td>101.796</td>\n",
       "      <td>1</td>\n",
       "      <td>121</td>\n",
       "      <td>213</td>\n",
       "      <td>275</td>\n",
       "      <td>336</td>\n",
       "      <td>201917</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id_nunique_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>4.90163</td>\n",
       "      <td>3.49626</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>67</td>\n",
       "      <td>201917</td>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <td>201917</td>\n",
       "      <td>30.9446</td>\n",
       "      <td>13.8454</td>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "      <td>31</td>\n",
       "      <td>44</td>\n",
       "      <td>53</td>\n",
       "      <td>201917</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>elapsed_time</th>\n",
       "      <td>201917</td>\n",
       "      <td>726.979</td>\n",
       "      <td>293.71</td>\n",
       "      <td>345</td>\n",
       "      <td>498</td>\n",
       "      <td>651</td>\n",
       "      <td>833</td>\n",
       "      <td>2629</td>\n",
       "      <td>201917</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_mean</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.288241</td>\n",
       "      <td>0.271316</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.4375</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>579</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_mean</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.699745</td>\n",
       "      <td>1.08214</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1</td>\n",
       "      <td>34.6207</td>\n",
       "      <td>201917</td>\n",
       "      <td>1104</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id_nunique_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>2.73086</td>\n",
       "      <td>1.51666</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>201917</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id_total</th>\n",
       "      <td>201917</td>\n",
       "      <td>96.107</td>\n",
       "      <td>104.283</td>\n",
       "      <td>3</td>\n",
       "      <td>36</td>\n",
       "      <td>64</td>\n",
       "      <td>114</td>\n",
       "      <td>2915</td>\n",
       "      <td>201917</td>\n",
       "      <td>1034</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_month</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.48444e+09</td>\n",
       "      <td>2.53766e+07</td>\n",
       "      <td>1.32011e+09</td>\n",
       "      <td>1.47528e+09</td>\n",
       "      <td>1.491e+09</td>\n",
       "      <td>1.50422e+09</td>\n",
       "      <td>1.51744e+09</td>\n",
       "      <td>201917</td>\n",
       "      <td>75</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_mean</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.0438083</td>\n",
       "      <td>0.155868</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>234</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_1</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.0109302</td>\n",
       "      <td>0.0012591</td>\n",
       "      <td>0.00805849</td>\n",
       "      <td>0.0104794</td>\n",
       "      <td>0.0106099</td>\n",
       "      <td>0.0107116</td>\n",
       "      <td>0.0131446</td>\n",
       "      <td>201917</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_sum</th>\n",
       "      <td>201917</td>\n",
       "      <td>4.23266</td>\n",
       "      <td>7.81939</td>\n",
       "      <td>-6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1004</td>\n",
       "      <td>201917</td>\n",
       "      <td>107</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_var</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.211563</td>\n",
       "      <td>0.121516</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.242647</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.5</td>\n",
       "      <td>201917</td>\n",
       "      <td>4582</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_max_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>4.25316</td>\n",
       "      <td>22.9412</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>999</td>\n",
       "      <td>201917</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_max_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-0.166058</td>\n",
       "      <td>0.636972</td>\n",
       "      <td>-11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201917</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id_nunique</th>\n",
       "      <td>201917</td>\n",
       "      <td>5.19377</td>\n",
       "      <td>4.03915</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>40</td>\n",
       "      <td>201917</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_min_old</th>\n",
       "      <td>201917</td>\n",
       "      <td>-7.87778</td>\n",
       "      <td>3.84331</td>\n",
       "      <td>-13</td>\n",
       "      <td>-12</td>\n",
       "      <td>-8</td>\n",
       "      <td>-4</td>\n",
       "      <td>-1</td>\n",
       "      <td>201917</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff_std</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.410666</td>\n",
       "      <td>0.207164</td>\n",
       "      <td>0</td>\n",
       "      <td>0.408248</td>\n",
       "      <td>0.492592</td>\n",
       "      <td>0.516398</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>201917</td>\n",
       "      <td>3401</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_sum</th>\n",
       "      <td>201917</td>\n",
       "      <td>9.67641</td>\n",
       "      <td>9.57903</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>12</td>\n",
       "      <td>185</td>\n",
       "      <td>201917</td>\n",
       "      <td>126</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_std</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.394333</td>\n",
       "      <td>0.979057</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.408248</td>\n",
       "      <td>185.477</td>\n",
       "      <td>201917</td>\n",
       "      <td>7102</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>feature_2</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.0109302</td>\n",
       "      <td>0.00195247</td>\n",
       "      <td>0.00875212</td>\n",
       "      <td>0.00875212</td>\n",
       "      <td>0.0113848</td>\n",
       "      <td>0.0113848</td>\n",
       "      <td>0.0141664</td>\n",
       "      <td>201917</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend_sum</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.96555</td>\n",
       "      <td>2.59564</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>45</td>\n",
       "      <td>201917</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag_std</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.421683</td>\n",
       "      <td>0.203212</td>\n",
       "      <td>0</td>\n",
       "      <td>0.440959</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.516398</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>201917</td>\n",
       "      <td>1473</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments_var</th>\n",
       "      <td>201917</td>\n",
       "      <td>1.11405</td>\n",
       "      <td>76.6517</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>34401.7</td>\n",
       "      <td>201917</td>\n",
       "      <td>8167</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id_nunique</th>\n",
       "      <td>201917</td>\n",
       "      <td>4.40462</td>\n",
       "      <td>2.89102</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>201917</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>first_active_monthIs_quarter_start</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id_nunique</th>\n",
       "      <td>201917</td>\n",
       "      <td>2.27582</td>\n",
       "      <td>1.57974</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>28</td>\n",
       "      <td>201917</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is_outlier</th>\n",
       "      <td>201917</td>\n",
       "      <td>0.00338753</td>\n",
       "      <td>0.058104</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>201917</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0%</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>75 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       count         mean          std  \\\n",
       "month_diff_mean_old                   201917      11.7556      2.39709   \n",
       "category_1_sum_old                    201917      6.37936      19.5725   \n",
       "authorized_flag_mean_old              201917     0.893353     0.114446   \n",
       "purchase_amount_max                   201917    -0.173108      1.45139   \n",
       "month_lag_mean_old                    201917     -3.84099      2.12349   \n",
       "purchase_date_amax_old                201917      1.51585   0.00645884   \n",
       "category_1_mean_old                   201917      0.10108     0.208051   \n",
       "purchase_date_amin_old                201917      1.49475    0.0102942   \n",
       "purchase_amount_min_old               201917    -0.729993    0.0918995   \n",
       "hist_first_buy                        201917      118.748      224.136   \n",
       "purchase_date_ptp_old                 201917      243.678      118.292   \n",
       "purchase_amount_mean_old              201917      2.45943      1115.17   \n",
       "purchase_amount_max_old               201917      66.6717      13448.5   \n",
       "purchased_on_weekend_mean_old         201917     0.276675     0.138688   \n",
       "month_lag_sum_old                     201917     -400.169      610.571   \n",
       "inverse_avg_transactions_per_day_old  201917      6.07849      6.93816   \n",
       "days_since_last_transaction           201917       303.57      69.9628   \n",
       "purchase_date_amax                    201917     0.353809       3.3431   \n",
       "purchase_amount_std_old               201917      12.6537      3865.24   \n",
       "installments_mean_old                 201917     0.863678      1.13581   \n",
       "month_diff_var_old                    201917     0.217643    0.0521406   \n",
       "purchase_amount_mean                  201917    -0.537137     0.525156   \n",
       "purchase_amount_min                   201917    -0.667412     0.409663   \n",
       "month_diff_std_old                    201917     0.459549    0.0803638   \n",
       "purchase_amount_sum                   201917     -3.62723      4.16273   \n",
       "month_lag_std_old                     201917      2.40891      1.17382   \n",
       "days_since_last_transaction_old       201917      363.735      74.7394   \n",
       "merchant_id_nunique_old               201917      33.7136      30.6869   \n",
       "merchant_category_id_nunique_old      201917       18.278      11.4705   \n",
       "new_first_buy                         201917      372.072      270.356   \n",
       "...                                      ...          ...          ...   \n",
       "month_diff_mean                       201917      11.6172      2.29955   \n",
       "authorized_flag_sum_old               201917       81.559      99.2434   \n",
       "first_active_monthDayofyear           201917      194.482      101.796   \n",
       "city_id_nunique_old                   201917      4.90163      3.49626   \n",
       "first_active_monthWeek                201917      30.9446      13.8454   \n",
       "elapsed_time                          201917      726.979       293.71   \n",
       "purchased_on_weekend_mean             201917     0.288241     0.271316   \n",
       "installments_mean                     201917     0.699745      1.08214   \n",
       "state_id_nunique_old                  201917      2.73086      1.51666   \n",
       "card_id_total                         201917       96.107      104.283   \n",
       "first_active_month                    201917  1.48444e+09  2.53766e+07   \n",
       "category_1_mean                       201917    0.0438083     0.155868   \n",
       "feature_1                             201917    0.0109302    0.0012591   \n",
       "installments_sum                      201917      4.23266      7.81939   \n",
       "month_diff_var                        201917     0.211563     0.121516   \n",
       "installments_max_old                  201917      4.25316      22.9412   \n",
       "month_lag_max_old                     201917    -0.166058     0.636972   \n",
       "merchant_category_id_nunique          201917      5.19377      4.03915   \n",
       "month_lag_min_old                     201917     -7.87778      3.84331   \n",
       "month_diff_std                        201917     0.410666     0.207164   \n",
       "month_lag_sum                         201917      9.67641      9.57903   \n",
       "installments_std                      201917     0.394333     0.979057   \n",
       "feature_2                             201917    0.0109302   0.00195247   \n",
       "purchased_on_weekend_sum              201917      1.96555      2.59564   \n",
       "month_lag_std                         201917     0.421683     0.203212   \n",
       "installments_var                      201917      1.11405      76.6517   \n",
       "subsector_id_nunique                  201917      4.40462      2.89102   \n",
       "first_active_monthIs_quarter_start       NaN          NaN          NaN   \n",
       "city_id_nunique                       201917      2.27582      1.57974   \n",
       "is_outlier                            201917   0.00338753     0.058104   \n",
       "\n",
       "                                              min          25%        50%  \\\n",
       "month_diff_mean_old                            10      10.5543    10.6744   \n",
       "category_1_sum_old                              0            0          0   \n",
       "authorized_flag_mean_old                0.0304878     0.852941   0.926829   \n",
       "purchase_amount_max                     -0.746893    -0.634209   -0.51302   \n",
       "month_lag_mean_old                       -12.4524     -5.52062   -3.57143   \n",
       "purchase_date_amax_old                    1.48594      1.51475     1.5189   \n",
       "category_1_mean_old                             0            0          0   \n",
       "purchase_date_amin_old                    1.48323      1.48434    1.49262   \n",
       "purchase_amount_min_old                 -0.746908    -0.743902  -0.740296   \n",
       "hist_first_buy                               -331           10         22   \n",
       "purchase_date_ptp_old                           1          138        233   \n",
       "purchase_amount_mean_old                -0.745371    -0.679118  -0.621435   \n",
       "purchase_amount_max_old                 -0.745315    -0.347203   0.123548   \n",
       "purchased_on_weekend_mean_old                   0     0.186047   0.269663   \n",
       "month_lag_sum_old                          -14172         -471       -179   \n",
       "inverse_avg_transactions_per_day_old    0.0869565            2    3.76147   \n",
       "days_since_last_transaction                   256          263        277   \n",
       "purchase_date_amax                       -9.22337      1.51688    1.52278   \n",
       "purchase_amount_std_old                         0    0.0766826   0.160484   \n",
       "installments_mean_old                   -0.692308            0          1   \n",
       "month_diff_var_old                              0     0.204545   0.235294   \n",
       "purchase_amount_mean                    -0.746893    -0.687445  -0.641737   \n",
       "purchase_amount_min                     -0.746893    -0.734887  -0.723466   \n",
       "month_diff_std_old                              0     0.452267   0.485071   \n",
       "purchase_amount_sum                      -70.8135     -4.82878   -2.68625   \n",
       "month_lag_std_old                        0.122169      1.37663     2.2734   \n",
       "days_since_last_transaction_old               317          319        328   \n",
       "merchant_id_nunique_old                         1           13         25   \n",
       "merchant_category_id_nunique_old                1           10         16   \n",
       "new_first_buy                                  16          189        294   \n",
       "...                                           ...          ...        ...   \n",
       "month_diff_mean                                10         10.5    10.7857   \n",
       "authorized_flag_sum_old                         2           23         48   \n",
       "first_active_monthDayofyear                     1          121        213   \n",
       "city_id_nunique_old                             1            3          4   \n",
       "first_active_monthWeek                          1           22         31   \n",
       "elapsed_time                                  345          498        651   \n",
       "purchased_on_weekend_mean                       0            0       0.25   \n",
       "installments_mean                              -1            0   0.333333   \n",
       "state_id_nunique_old                            1            2          2   \n",
       "card_id_total                                   3           36         64   \n",
       "first_active_month                    1.32011e+09  1.47528e+09  1.491e+09   \n",
       "category_1_mean                                 0            0          0   \n",
       "feature_1                              0.00805849    0.0104794  0.0106099   \n",
       "installments_sum                               -6            0          1   \n",
       "month_diff_var                                  0     0.166667   0.242647   \n",
       "installments_max_old                            0            0          2   \n",
       "month_lag_max_old                             -11            0          0   \n",
       "merchant_category_id_nunique                    1            2          4   \n",
       "month_lag_min_old                             -13          -12         -8   \n",
       "month_diff_std                                  0     0.408248   0.492592   \n",
       "month_lag_sum                                   1            4          7   \n",
       "installments_std                                0            0          0   \n",
       "feature_2                              0.00875212   0.00875212  0.0113848   \n",
       "purchased_on_weekend_sum                        0            0          1   \n",
       "month_lag_std                                   0     0.440959        0.5   \n",
       "installments_var                                0            0          0   \n",
       "subsector_id_nunique                            1            2          4   \n",
       "first_active_monthIs_quarter_start            NaN          NaN        NaN   \n",
       "city_id_nunique                                 1            1          2   \n",
       "is_outlier                                      0            0          0   \n",
       "\n",
       "                                              75%          max  counts  \\\n",
       "month_diff_mean_old                       11.6591           23  201917   \n",
       "category_1_sum_old                              5         1786  201917   \n",
       "authorized_flag_mean_old                 0.972516            1  201917   \n",
       "purchase_amount_max                     -0.251063      76.6829  201917   \n",
       "month_lag_mean_old                       -2.03497   -0.0149254  201917   \n",
       "purchase_date_amax_old                    1.51967      1.51986  201917   \n",
       "category_1_mean_old                     0.0909091            1  201917   \n",
       "purchase_date_amin_old                    1.50419      1.51476  201917   \n",
       "purchase_amount_min_old                 -0.731881      28.5548  201917   \n",
       "hist_first_buy                                123         1995  201917   \n",
       "purchase_date_ptp_old                         361          423  201917   \n",
       "purchase_amount_mean_old                -0.501713       500883  201917   \n",
       "purchase_amount_max_old                   1.27716   6.0106e+06  201917   \n",
       "purchased_on_weekend_mean_old            0.356522            1  201917   \n",
       "month_lag_sum_old                             -66           -1  201917   \n",
       "inverse_avg_transactions_per_day_old         7.36      135.667  201917   \n",
       "days_since_last_transaction                   305          679  201917   \n",
       "purchase_date_amax                        1.52451      1.52513  201917   \n",
       "purchase_amount_std_old                  0.361742  1.73511e+06  201917   \n",
       "installments_mean_old                     1.34426      79.3077  201917   \n",
       "month_diff_var_old                       0.250065          0.5  201917   \n",
       "purchase_amount_mean                    -0.552641      44.1824  201917   \n",
       "purchase_amount_min                     -0.699725      44.1824  201917   \n",
       "month_diff_std_old                       0.500065     0.707107  201917   \n",
       "purchase_amount_sum                       -1.3437      109.086  201917   \n",
       "month_lag_std_old                         3.44656            7  201917   \n",
       "days_since_last_transaction_old               376          710  201917   \n",
       "merchant_id_nunique_old                        44          398  201917   \n",
       "merchant_category_id_nunique_old               24           95  201917   \n",
       "new_first_buy                                 460         2351  201917   \n",
       "...                                           ...          ...     ...   \n",
       "month_diff_mean                             11.25           23  201917   \n",
       "authorized_flag_sum_old                       100         2537  201917   \n",
       "first_active_monthDayofyear                   275          336  201917   \n",
       "city_id_nunique_old                             6           67  201917   \n",
       "first_active_monthWeek                         44           53  201917   \n",
       "elapsed_time                                  833         2629  201917   \n",
       "purchased_on_weekend_mean                  0.4375            1  201917   \n",
       "installments_mean                               1      34.6207  201917   \n",
       "state_id_nunique_old                            3           20  201917   \n",
       "card_id_total                                 114         2915  201917   \n",
       "first_active_month                    1.50422e+09  1.51744e+09  201917   \n",
       "category_1_mean                                 0            1  201917   \n",
       "feature_1                               0.0107116    0.0131446  201917   \n",
       "installments_sum                                6         1004  201917   \n",
       "month_diff_var                           0.266667          0.5  201917   \n",
       "installments_max_old                            6          999  201917   \n",
       "month_lag_max_old                               0            0  201917   \n",
       "merchant_category_id_nunique                    7           40  201917   \n",
       "month_lag_min_old                              -4           -1  201917   \n",
       "month_diff_std                           0.516398     0.707107  201917   \n",
       "month_lag_sum                                  12          185  201917   \n",
       "installments_std                         0.408248      185.477  201917   \n",
       "feature_2                               0.0113848    0.0141664  201917   \n",
       "purchased_on_weekend_sum                        3           45  201917   \n",
       "month_lag_std                            0.516398     0.707107  201917   \n",
       "installments_var                         0.166667      34401.7  201917   \n",
       "subsector_id_nunique                            6           24  201917   \n",
       "first_active_monthIs_quarter_start            NaN          NaN  201917   \n",
       "city_id_nunique                                 3           28  201917   \n",
       "is_outlier                                      0            1  201917   \n",
       "\n",
       "                                     uniques missing missing_perc    types  \n",
       "month_diff_mean_old                    32343       0           0%  numeric  \n",
       "category_1_sum_old                       357       0           0%  numeric  \n",
       "authorized_flag_mean_old               11356       0           0%  numeric  \n",
       "purchase_amount_max                    43853       0           0%  numeric  \n",
       "month_lag_mean_old                     54905       0           0%  numeric  \n",
       "purchase_date_amax_old                186997       0           0%  numeric  \n",
       "category_1_mean_old                    10370       0           0%  numeric  \n",
       "purchase_date_amin_old                193984       0           0%  numeric  \n",
       "purchase_amount_min_old                 7432       0           0%  numeric  \n",
       "hist_first_buy                          1905       0           0%  numeric  \n",
       "purchase_date_ptp_old                    422       0           0%  numeric  \n",
       "purchase_amount_mean_old              201640       0           0%  numeric  \n",
       "purchase_amount_max_old                74899       0           0%  numeric  \n",
       "purchased_on_weekend_mean_old          15633       0           0%  numeric  \n",
       "month_lag_sum_old                       4267       0           0%  numeric  \n",
       "inverse_avg_transactions_per_day_old   45200       0           0%  numeric  \n",
       "days_since_last_transaction              407       0           0%  numeric  \n",
       "purchase_date_amax                    173712       0           0%  numeric  \n",
       "purchase_amount_std_old               201633       0           0%  numeric  \n",
       "installments_mean_old                  21136       0           0%  numeric  \n",
       "month_diff_var_old                     78101       0           0%  numeric  \n",
       "purchase_amount_mean                  146245       0           0%  numeric  \n",
       "purchase_amount_min                    15978       0           0%  numeric  \n",
       "month_diff_std_old                     66347       0           0%  numeric  \n",
       "purchase_amount_sum                   149496       0           0%  numeric  \n",
       "month_lag_std_old                     160944       0           0%  numeric  \n",
       "days_since_last_transaction_old          393       0           0%  numeric  \n",
       "merchant_id_nunique_old                  313       0           0%  numeric  \n",
       "merchant_category_id_nunique_old          92       0           0%  numeric  \n",
       "new_first_buy                           1898       0           0%  numeric  \n",
       "...                                      ...     ...          ...      ...  \n",
       "month_diff_mean                         2783       0           0%  numeric  \n",
       "authorized_flag_sum_old                  973       0           0%  numeric  \n",
       "first_active_monthDayofyear               22       0           0%  numeric  \n",
       "city_id_nunique_old                       58       0           0%  numeric  \n",
       "first_active_monthWeek                    21       0           0%  numeric  \n",
       "elapsed_time                              75       0           0%  numeric  \n",
       "purchased_on_weekend_mean                579       0           0%  numeric  \n",
       "installments_mean                       1104       0           0%  numeric  \n",
       "state_id_nunique_old                      20       0           0%  numeric  \n",
       "card_id_total                           1034       0           0%  numeric  \n",
       "first_active_month                        75       0           0%  numeric  \n",
       "category_1_mean                          234       0           0%  numeric  \n",
       "feature_1                                  5       0           0%  numeric  \n",
       "installments_sum                         107       0           0%  numeric  \n",
       "month_diff_var                          4582       0           0%  numeric  \n",
       "installments_max_old                      14       0           0%  numeric  \n",
       "month_lag_max_old                         12       0           0%  numeric  \n",
       "merchant_category_id_nunique              39       0           0%  numeric  \n",
       "month_lag_min_old                         13       0           0%  numeric  \n",
       "month_diff_std                          3401       0           0%  numeric  \n",
       "month_lag_sum                            126       0           0%  numeric  \n",
       "installments_std                        7102       0           0%  numeric  \n",
       "feature_2                                  3       0           0%  numeric  \n",
       "purchased_on_weekend_sum                  39       0           0%  numeric  \n",
       "month_lag_std                           1473       0           0%  numeric  \n",
       "installments_var                        8167       0           0%  numeric  \n",
       "subsector_id_nunique                      24       0           0%  numeric  \n",
       "first_active_monthIs_quarter_start         2       0           0%     bool  \n",
       "city_id_nunique                           26       0           0%  numeric  \n",
       "is_outlier                                 2       0           0%     bool  \n",
       "\n",
       "[75 rows x 13 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrameSummary(df_hot_ext).summary().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chandrasekhar/anaconda3/envs/fastai/lib/python3.6/site-packages/sklearn/ensemble/forest.py:732: UserWarning: Some inputs do not have OOB scores. This probably means too few trees were used to compute any reliable oob estimates.\n",
      "  warn(\"Some inputs do not have OOB scores. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.303248285113751, 2.303248285113751, 0.6421925248673591, 0.6421925248673591, -0.023896062366220994]\n"
     ]
    }
   ],
   "source": [
    "m = RandomForestRegressor(**rf_args)\n",
    "m.fit(df_hot_ext, target)\n",
    "m.score(df_hot_ext, target)\n",
    "print_score(m, df_hot_ext, target.values, df_hot_ext, target.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Base model - 0.8348232234871118, 0.8348232234871118, 0.9529936055779922, 0.9529936055779922, 0.8147359464393436\n",
    "- Base model with outliers removed - 2.299946409218127, 2.299946409218127, 0.6432176762133195, 0.6432176762133195, -0.02364908316679548\n",
    "- Base model with df_hot_ext - 2.1145840168101624, 2.1145840168101624, 0.6984094099753392, 0.6984094099753393, 0.1985261186716828\n",
    "- Base model with df_hot_ext ouliers removed - 2.303248285113751, 2.303248285113751, 0.6421925248673591, 0.6421925248673591, -0.023896062366220994"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "unauthorized_purchase_amount_std             51988\n",
       "unauthorized_purchase_month_std              51988\n",
       "unauthorized_installments_std                51988\n",
       "new_installments_std                         48718\n",
       "new_purchase_month_std                       48718\n",
       "new_purchase_amount_std                      48718\n",
       "unauthorized_month_lag_max                   30959\n",
       "unauthorized_month_lag_min                   30959\n",
       "unauthorized_category_2_3.0_mean             30959\n",
       "unauthorized_installments_sum                30959\n",
       "unauthorized_category_2_2.0_mean             30959\n",
       "unauthorized_purchase_month_mean             30959\n",
       "unauthorized_merchant_category_id_nunique    30959\n",
       "unauthorized_purchase_date_ptp               30959\n",
       "unauthorized_installments_min                30959\n",
       "unauthorized_purchase_date_min               30959\n",
       "unauthorized_installments_max                30959\n",
       "unauthorized_installments_mean               30959\n",
       "unauthorized_purchase_month_min              30959\n",
       "unauthorized_category_2_4.0_mean             30959\n",
       "unauthorized_category_3_B_mean               30959\n",
       "unauthorized_purchase_amount_min             30959\n",
       "unauthorized_purchase_amount_max             30959\n",
       "unauthorized_purchase_amount_mean            30959\n",
       "unauthorized_purchase_amount_sum             30959\n",
       "unauthorized_state_id_nunique                30959\n",
       "unauthorized_category_2_1.0_mean             30959\n",
       "unauthorized_merchant_id_nunique             30959\n",
       "unauthorized_subsector_id_nunique            30959\n",
       "unauthorized_category_3_C_mean               30959\n",
       "unauthorized_category_2_5.0_mean             30959\n",
       "unauthorized_category_3_A_mean               30959\n",
       "unauthorized_category_1_mean                 30959\n",
       "unauthorized_category_1_sum                  30959\n",
       "unauthorized_transactions_count              30959\n",
       "unauthorized_purchase_date_max               30959\n",
       "unauthorized_purchase_month_max              30959\n",
       "unauthorized_city_id_nunique                 30959\n",
       "new_month_lag_min                            21931\n",
       "new_installments_min                         21931\n",
       "new_merchant_category_id_nunique             21931\n",
       "new_purchase_month_mean                      21931\n",
       "new_city_id_nunique                          21931\n",
       "new_purchase_month_max                       21931\n",
       "new_purchase_month_min                       21931\n",
       "new_month_lag_max                            21931\n",
       "new_installments_mean                        21931\n",
       "new_transactions_count                       21931\n",
       "new_category_2_2.0_mean                      21931\n",
       "new_purchase_date_ptp                        21931\n",
       "new_purchase_date_min                        21931\n",
       "new_purchase_date_max                        21931\n",
       "new_category_3_B_mean                        21931\n",
       "new_installments_max                         21931\n",
       "new_category_2_3.0_mean                      21931\n",
       "new_installments_sum                         21931\n",
       "new_merchant_id_nunique                      21931\n",
       "new_purchase_amount_min                      21931\n",
       "new_category_1_sum                           21931\n",
       "new_category_1_mean                          21931\n",
       "new_category_2_5.0_mean                      21931\n",
       "new_category_3_C_mean                        21931\n",
       "new_subsector_id_nunique                     21931\n",
       "new_category_3_A_mean                        21931\n",
       "new_category_2_1.0_mean                      21931\n",
       "new_state_id_nunique                         21931\n",
       "new_category_2_4.0_mean                      21931\n",
       "new_purchase_amount_sum                      21931\n",
       "new_purchase_amount_mean                     21931\n",
       "new_purchase_amount_max                      21931\n",
       "authorized_category_2_5.0_mean                   0\n",
       "authorized_transactions_count                    0\n",
       "authorized_category_1_sum                        0\n",
       "authorized_category_1_mean                       0\n",
       "authorized_category_3_A_mean                     0\n",
       "authorized_subsector_id_nunique                  0\n",
       "authorized_category_3_C_mean                     0\n",
       "authorized_category_2_1.0_mean                   0\n",
       "authorized_merchant_id_nunique                   0\n",
       "feature_2_2                                      0\n",
       "feature_2_3                                      0\n",
       "year                                             0\n",
       "feature_2_1                                      0\n",
       "feature_1_5                                      0\n",
       "feature_1_4                                      0\n",
       "feature_1_3                                      0\n",
       "feature_1_2                                      0\n",
       "feature_1_1                                      0\n",
       "elapsed_time                                     0\n",
       "month                                            0\n",
       "feature_3                                        0\n",
       "card_id                                          0\n",
       "authorized_category_2_4.0_mean                   0\n",
       "authorized_state_id_nunique                      0\n",
       "outliers                                         0\n",
       "authorized_purchase_amount_sum                   0\n",
       "authorized_purchase_month_min                    0\n",
       "authorized_city_id_nunique                       0\n",
       "authorized_category_3_B_mean                     0\n",
       "authorized_purchase_date_max                     0\n",
       "authorized_purchase_date_min                     0\n",
       "authorized_purchase_date_ptp                     0\n",
       "authorized_category_2_2.0_mean                   0\n",
       "authorized_category_2_3.0_mean                   0\n",
       "authorized_month_lag_max                         0\n",
       "authorized_month_lag_min                         0\n",
       "authorized_purchase_month_std                    0\n",
       "authorized_purchase_month_max                    0\n",
       "authorized_purchase_amount_mean                  0\n",
       "authorized_purchase_month_mean                   0\n",
       "authorized_merchant_category_id_nunique          0\n",
       "authorized_installments_std                      0\n",
       "authorized_installments_min                      0\n",
       "authorized_installments_max                      0\n",
       "authorized_installments_mean                     0\n",
       "authorized_installments_sum                      0\n",
       "authorized_purchase_amount_std                   0\n",
       "authorized_purchase_amount_min                   0\n",
       "authorized_purchase_amount_max                   0\n",
       "first_active_month                               0\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_all(train_clean.isna().sum().sort_values(ascending=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_clean0 = train_clean.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "outliers                                     0\n",
       "new_city_id_nunique                          0\n",
       "authorized_installments_mean                 0\n",
       "authorized_installments_max                  0\n",
       "authorized_installments_min                  0\n",
       "authorized_installments_std                  0\n",
       "authorized_merchant_category_id_nunique      0\n",
       "authorized_purchase_month_mean               0\n",
       "authorized_purchase_month_max                0\n",
       "authorized_purchase_month_min                0\n",
       "authorized_purchase_month_std                0\n",
       "authorized_month_lag_min                     0\n",
       "authorized_month_lag_max                     0\n",
       "authorized_category_2_3.0_mean               0\n",
       "authorized_category_2_2.0_mean               0\n",
       "authorized_purchase_date_ptp                 0\n",
       "authorized_purchase_date_min                 0\n",
       "authorized_purchase_date_max                 0\n",
       "authorized_category_3_B_mean                 0\n",
       "authorized_city_id_nunique                   0\n",
       "unauthorized_transactions_count              0\n",
       "unauthorized_category_1_sum                  0\n",
       "unauthorized_category_1_mean                 0\n",
       "unauthorized_category_3_A_mean               0\n",
       "unauthorized_category_2_5.0_mean             0\n",
       "unauthorized_category_3_C_mean               0\n",
       "unauthorized_subsector_id_nunique            0\n",
       "unauthorized_merchant_id_nunique             0\n",
       "unauthorized_category_2_1.0_mean             0\n",
       "authorized_installments_sum                  0\n",
       "                                            ..\n",
       "new_category_3_C_mean                        0\n",
       "unauthorized_purchase_amount_mean            0\n",
       "unauthorized_purchase_month_std              0\n",
       "unauthorized_purchase_amount_max             0\n",
       "unauthorized_purchase_amount_min             0\n",
       "unauthorized_purchase_amount_std             0\n",
       "unauthorized_installments_sum                0\n",
       "unauthorized_installments_mean               0\n",
       "unauthorized_installments_max                0\n",
       "unauthorized_installments_min                0\n",
       "unauthorized_installments_std                0\n",
       "unauthorized_merchant_category_id_nunique    0\n",
       "unauthorized_purchase_month_mean             0\n",
       "unauthorized_purchase_month_max              0\n",
       "unauthorized_purchase_month_min              0\n",
       "unauthorized_month_lag_min                   0\n",
       "new_category_2_5.0_mean                      0\n",
       "unauthorized_month_lag_max                   0\n",
       "unauthorized_category_2_3.0_mean             0\n",
       "unauthorized_category_2_2.0_mean             0\n",
       "unauthorized_purchase_date_ptp               0\n",
       "unauthorized_purchase_date_min               0\n",
       "unauthorized_purchase_date_max               0\n",
       "unauthorized_category_3_B_mean               0\n",
       "unauthorized_city_id_nunique                 0\n",
       "new_transactions_count                       0\n",
       "new_category_1_sum                           0\n",
       "new_category_1_mean                          0\n",
       "new_category_3_A_mean                        0\n",
       "first_active_month                           0\n",
       "Length: 120, dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_clean0.isna().sum().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_clean0.to_feather('train_clean0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}