TS Fresh.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsfresh.examples.robot_execution_failures import download_robot_execution_failures, \\\n",
    "    load_robot_execution_failures\n",
    "download_robot_execution_failures()\n",
    "timeseries, y = load_robot_execution_failures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsfresh import extract_features\n",
    "from tsfresh import extract_relevant_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_filtered_direct.to_feather('adf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import feather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "adf = feather.read_dataframe('adf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>F_x__abs_energy</th>\n",
       "      <th>F_x__range_count__max_1__min_-1</th>\n",
       "      <th>F_y__abs_energy</th>\n",
       "      <th>T_y__standard_deviation</th>\n",
       "      <th>T_y__variance</th>\n",
       "      <th>F_x__fft_coefficient__coeff_1__attr_\"abs\"</th>\n",
       "      <th>T_y__fft_coefficient__coeff_1__attr_\"abs\"</th>\n",
       "      <th>T_y__abs_energy</th>\n",
       "      <th>F_z__standard_deviation</th>\n",
       "      <th>...</th>\n",
       "      <th>T_z__large_standard_deviation__r_0.35000000000000003</th>\n",
       "      <th>T_z__quantile__q_0.9</th>\n",
       "      <th>F_z__agg_linear_trend__f_agg_\"max\"__chunk_len_5__attr_\"intercept\"</th>\n",
       "      <th>T_x__agg_autocorrelation__f_agg_\"mean\"</th>\n",
       "      <th>F_y__change_quantiles__f_agg_\"var\"__isabs_False__qh_1.0__ql_0.8</th>\n",
       "      <th>T_x__spkt_welch_density__coeff_5</th>\n",
       "      <th>T_y__agg_linear_trend__f_agg_\"min\"__chunk_len_5__attr_\"intercept\"</th>\n",
       "      <th>F_y__change_quantiles__f_agg_\"var\"__isabs_False__qh_0.6__ql_0.2</th>\n",
       "      <th>F_z__agg_linear_trend__f_agg_\"max\"__chunk_len_10__attr_\"intercept\"</th>\n",
       "      <th>F_z__change_quantiles__f_agg_\"var\"__isabs_False__qh_0.8__ql_0.4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.471405</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.165352</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.203698</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.833333</td>\n",
       "      <td>-0.095589</td>\n",
       "      <td>0.209184</td>\n",
       "      <td>0.037795</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>63.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>25.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>2.054805</td>\n",
       "      <td>4.222222</td>\n",
       "      <td>0.624118</td>\n",
       "      <td>6.020261</td>\n",
       "      <td>90.0</td>\n",
       "      <td>4.333846</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.666667</td>\n",
       "      <td>-0.054604</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.319311</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>70.0</td>\n",
       "      <td>2.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>12.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>1.768867</td>\n",
       "      <td>3.128889</td>\n",
       "      <td>2.203858</td>\n",
       "      <td>8.235442</td>\n",
       "      <td>103.0</td>\n",
       "      <td>4.616877</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>67.333333</td>\n",
       "      <td>-0.061050</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.102780</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>68.0</td>\n",
       "      <td>8.187500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>16.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>2.669998</td>\n",
       "      <td>7.128889</td>\n",
       "      <td>0.844394</td>\n",
       "      <td>12.067855</td>\n",
       "      <td>124.0</td>\n",
       "      <td>3.833188</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>63.666667</td>\n",
       "      <td>-0.133794</td>\n",
       "      <td>1.555556</td>\n",
       "      <td>56.910262</td>\n",
       "      <td>-3.166667</td>\n",
       "      <td>1.360000</td>\n",
       "      <td>66.0</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>17.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>2.039608</td>\n",
       "      <td>4.160000</td>\n",
       "      <td>2.730599</td>\n",
       "      <td>6.445330</td>\n",
       "      <td>180.0</td>\n",
       "      <td>4.841487</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>64.333333</td>\n",
       "      <td>-0.106108</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>22.841805</td>\n",
       "      <td>-4.166667</td>\n",
       "      <td>1.040000</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 625 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  F_x__abs_energy  F_x__range_count__max_1__min_-1  F_y__abs_energy  \\\n",
       "0   1             14.0                             15.0             13.0   \n",
       "1   2             25.0                             13.0             76.0   \n",
       "2   3             12.0                             14.0             40.0   \n",
       "3   4             16.0                             10.0             60.0   \n",
       "4   5             17.0                             13.0             46.0   \n",
       "\n",
       "   T_y__standard_deviation  T_y__variance  \\\n",
       "0                 0.471405       0.222222   \n",
       "1                 2.054805       4.222222   \n",
       "2                 1.768867       3.128889   \n",
       "3                 2.669998       7.128889   \n",
       "4                 2.039608       4.160000   \n",
       "\n",
       "   F_x__fft_coefficient__coeff_1__attr_\"abs\"  \\\n",
       "0                                   1.000000   \n",
       "1                                   0.624118   \n",
       "2                                   2.203858   \n",
       "3                                   0.844394   \n",
       "4                                   2.730599   \n",
       "\n",
       "   T_y__fft_coefficient__coeff_1__attr_\"abs\"  T_y__abs_energy  \\\n",
       "0                                   1.165352             10.0   \n",
       "1                                   6.020261             90.0   \n",
       "2                                   8.235442            103.0   \n",
       "3                                  12.067855            124.0   \n",
       "4                                   6.445330            180.0   \n",
       "\n",
       "   F_z__standard_deviation  \\\n",
       "0                 1.203698   \n",
       "1                 4.333846   \n",
       "2                 4.616877   \n",
       "3                 3.833188   \n",
       "4                 4.841487   \n",
       "\n",
       "                                ...                                 \\\n",
       "0                               ...                                  \n",
       "1                               ...                                  \n",
       "2                               ...                                  \n",
       "3                               ...                                  \n",
       "4                               ...                                  \n",
       "\n",
       "   T_z__large_standard_deviation__r_0.35000000000000003  T_z__quantile__q_0.9  \\\n",
       "0                                                0.0                      0.0   \n",
       "1                                                1.0                      0.0   \n",
       "2                                                1.0                      0.0   \n",
       "3                                                0.0                      0.0   \n",
       "4                                                0.0                      0.6   \n",
       "\n",
       "   F_z__agg_linear_trend__f_agg_\"max\"__chunk_len_5__attr_\"intercept\"  \\\n",
       "0                                          62.833333                   \n",
       "1                                          64.666667                   \n",
       "2                                          67.333333                   \n",
       "3                                          63.666667                   \n",
       "4                                          64.333333                   \n",
       "\n",
       "   T_x__agg_autocorrelation__f_agg_\"mean\"  \\\n",
       "0                               -0.095589   \n",
       "1                               -0.054604   \n",
       "2                               -0.061050   \n",
       "3                               -0.133794   \n",
       "4                               -0.106108   \n",
       "\n",
       "   F_y__change_quantiles__f_agg_\"var\"__isabs_False__qh_1.0__ql_0.8  \\\n",
       "0                                           0.209184                 \n",
       "1                                           0.000000                 \n",
       "2                                           0.000000                 \n",
       "3                                           1.555556                 \n",
       "4                                           0.000000                 \n",
       "\n",
       "   T_x__spkt_welch_density__coeff_5  \\\n",
       "0                          0.037795   \n",
       "1                          0.319311   \n",
       "2                          9.102780   \n",
       "3                         56.910262   \n",
       "4                         22.841805   \n",
       "\n",
       "   T_y__agg_linear_trend__f_agg_\"min\"__chunk_len_5__attr_\"intercept\"  \\\n",
       "0                                          -1.000000                   \n",
       "1                                          -1.000000                   \n",
       "2                                          -3.000000                   \n",
       "3                                          -3.166667                   \n",
       "4                                          -4.166667                   \n",
       "\n",
       "   F_y__change_quantiles__f_agg_\"var\"__isabs_False__qh_0.6__ql_0.2  \\\n",
       "0                                           0.000000                 \n",
       "1                                           0.222222                 \n",
       "2                                           0.250000                 \n",
       "3                                           1.360000                 \n",
       "4                                           1.040000                 \n",
       "\n",
       "   F_z__agg_linear_trend__f_agg_\"max\"__chunk_len_10__attr_\"intercept\"  \\\n",
       "0                                               63.0                    \n",
       "1                                               70.0                    \n",
       "2                                               68.0                    \n",
       "3                                               66.0                    \n",
       "4                                               67.0                    \n",
       "\n",
       "   F_z__change_quantiles__f_agg_\"var\"__isabs_False__qh_0.8__ql_0.4  \n",
       "0                                           0.000000                \n",
       "1                                           2.666667                \n",
       "2                                           8.187500                \n",
       "3                                           1.000000                \n",
       "4                                           0.000000                \n",
       "\n",
       "[5 rows x 625 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist = feather.read_dataframe('new_hist_trans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>city_id</th>\n",
       "      <th>category_1</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>...</th>\n",
       "      <th>purchase_Dayofyear</th>\n",
       "      <th>purchase_Is_month_end</th>\n",
       "      <th>purchase_Is_month_start</th>\n",
       "      <th>purchase_Is_quarter_end</th>\n",
       "      <th>purchase_Is_quarter_start</th>\n",
       "      <th>purchase_Is_year_end</th>\n",
       "      <th>purchase_Is_year_start</th>\n",
       "      <th>purchase_Elapsed</th>\n",
       "      <th>purchased_on_weekend</th>\n",
       "      <th>month_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>107</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_b0c793002c</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.557574</td>\n",
       "      <td>...</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520780256</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>140</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_88920c89e8</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.569580</td>\n",
       "      <td>...</td>\n",
       "      <td>78</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521485617</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>330</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>507</td>\n",
       "      <td>M_ID_ad5237ef6b</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.551037</td>\n",
       "      <td>...</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1524751724</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>661</td>\n",
       "      <td>M_ID_9e84cda3b1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.671925</td>\n",
       "      <td>...</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520415801</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>166</td>\n",
       "      <td>M_ID_3c86fa3831</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.659904</td>\n",
       "      <td>...</td>\n",
       "      <td>81</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521752873</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   authorized_flag          card_id city_id  category_1  installments  \\\n",
       "0                1  C_ID_415bb3a509     107           0             1   \n",
       "1                1  C_ID_415bb3a509     140           0             1   \n",
       "2                1  C_ID_415bb3a509     330           0             1   \n",
       "3                1  C_ID_415bb3a509      -1           1             1   \n",
       "4                1  C_ID_ef55cf8d4b      -1           1             1   \n",
       "\n",
       "  category_3 merchant_category_id      merchant_id  month_lag  \\\n",
       "0          B                  307  M_ID_b0c793002c          1   \n",
       "1          B                  307  M_ID_88920c89e8          1   \n",
       "2          B                  507  M_ID_ad5237ef6b          2   \n",
       "3          B                  661  M_ID_9e84cda3b1          1   \n",
       "4          B                  166  M_ID_3c86fa3831          1   \n",
       "\n",
       "   purchase_amount     ...     purchase_Dayofyear  purchase_Is_month_end  \\\n",
       "0        -0.557574     ...                     70                  False   \n",
       "1        -0.569580     ...                     78                  False   \n",
       "2        -0.551037     ...                    116                  False   \n",
       "3        -0.671925     ...                     66                  False   \n",
       "4        -0.659904     ...                     81                  False   \n",
       "\n",
       "  purchase_Is_month_start purchase_Is_quarter_end purchase_Is_quarter_start  \\\n",
       "0                   False                   False                     False   \n",
       "1                   False                   False                     False   \n",
       "2                   False                   False                     False   \n",
       "3                   False                   False                     False   \n",
       "4                   False                   False                     False   \n",
       "\n",
       "  purchase_Is_year_end  purchase_Is_year_start  purchase_Elapsed  \\\n",
       "0                False                   False        1520780256   \n",
       "1                False                   False        1521485617   \n",
       "2                False                   False        1524751724   \n",
       "3                False                   False        1520415801   \n",
       "4                False                   False        1521752873   \n",
       "\n",
       "   purchased_on_weekend  month_diff  \n",
       "0                     1          11  \n",
       "1                     0          10  \n",
       "2                     0          10  \n",
       "3                     0          11  \n",
       "4                     0          10  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "hist_copy = new_hist[:20].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>city_id</th>\n",
       "      <th>category_1</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>...</th>\n",
       "      <th>purchase_Dayofyear</th>\n",
       "      <th>purchase_Is_month_end</th>\n",
       "      <th>purchase_Is_month_start</th>\n",
       "      <th>purchase_Is_quarter_end</th>\n",
       "      <th>purchase_Is_quarter_start</th>\n",
       "      <th>purchase_Is_year_end</th>\n",
       "      <th>purchase_Is_year_start</th>\n",
       "      <th>purchase_Elapsed</th>\n",
       "      <th>purchased_on_weekend</th>\n",
       "      <th>month_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>107</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_b0c793002c</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.557574</td>\n",
       "      <td>...</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520780256</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>140</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_88920c89e8</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.569580</td>\n",
       "      <td>...</td>\n",
       "      <td>78</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521485617</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>330</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>507</td>\n",
       "      <td>M_ID_ad5237ef6b</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.551037</td>\n",
       "      <td>...</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1524751724</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>661</td>\n",
       "      <td>M_ID_9e84cda3b1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.671925</td>\n",
       "      <td>...</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520415801</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>166</td>\n",
       "      <td>M_ID_3c86fa3831</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.659904</td>\n",
       "      <td>...</td>\n",
       "      <td>81</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521752873</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>231</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>367</td>\n",
       "      <td>M_ID_8874615e00</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.633007</td>\n",
       "      <td>...</td>\n",
       "      <td>92</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1522673608</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>333</td>\n",
       "      <td>M_ID_6d061b5ddc</td>\n",
       "      <td>1</td>\n",
       "      <td>5.263697</td>\n",
       "      <td>...</td>\n",
       "      <td>87</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1522266619</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>231</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_df1e022f41</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.553787</td>\n",
       "      <td>...</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1522915612</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>278</td>\n",
       "      <td>M_ID_d15eae0468</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.596643</td>\n",
       "      <td>...</td>\n",
       "      <td>97</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1523126260</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>437</td>\n",
       "      <td>M_ID_5f9bffd028</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.607191</td>\n",
       "      <td>...</td>\n",
       "      <td>76</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521310241</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>A</td>\n",
       "      <td>45</td>\n",
       "      <td>M_ID_3ffd43b4cd</td>\n",
       "      <td>1</td>\n",
       "      <td>4.452265</td>\n",
       "      <td>...</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1522490140</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>108</td>\n",
       "      <td>M_ID_e6f5213fbf</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.605959</td>\n",
       "      <td>...</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520772821</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>278</td>\n",
       "      <td>M_ID_aa97bc87f6</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.634209</td>\n",
       "      <td>...</td>\n",
       "      <td>73</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521036463</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>157</td>\n",
       "      <td>M_ID_28fbc8c74d</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.492058</td>\n",
       "      <td>...</td>\n",
       "      <td>104</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1523698065</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>302</td>\n",
       "      <td>M_ID_b9f9332438</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.665539</td>\n",
       "      <td>...</td>\n",
       "      <td>82</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521840953</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>231</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>80</td>\n",
       "      <td>M_ID_ad15049b64</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.646245</td>\n",
       "      <td>...</td>\n",
       "      <td>108</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1524055308</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>307</td>\n",
       "      <td>M_ID_51757498fd</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.671775</td>\n",
       "      <td>...</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1520771936</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>560</td>\n",
       "      <td>M_ID_a1183a4e5e</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.690228</td>\n",
       "      <td>...</td>\n",
       "      <td>112</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1524409227</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>231</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>C</td>\n",
       "      <td>333</td>\n",
       "      <td>M_ID_f74d9cbbd0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.751371</td>\n",
       "      <td>...</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1519914412</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>231</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>369</td>\n",
       "      <td>M_ID_f0bf60c061</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.343296</td>\n",
       "      <td>...</td>\n",
       "      <td>81</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1521712643</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    authorized_flag          card_id city_id  category_1  installments  \\\n",
       "0                 1  C_ID_415bb3a509     107           0             1   \n",
       "1                 1  C_ID_415bb3a509     140           0             1   \n",
       "2                 1  C_ID_415bb3a509     330           0             1   \n",
       "3                 1  C_ID_415bb3a509      -1           1             1   \n",
       "4                 1  C_ID_ef55cf8d4b      -1           1             1   \n",
       "5                 1  C_ID_ef55cf8d4b     231           0             1   \n",
       "6                 1  C_ID_ef55cf8d4b      69           0             1   \n",
       "7                 1  C_ID_ef55cf8d4b     231           0             1   \n",
       "8                 1  C_ID_ef55cf8d4b      69           0             1   \n",
       "9                 1  C_ID_ef55cf8d4b      69           0             1   \n",
       "10                1  C_ID_ef55cf8d4b      69           0            -1   \n",
       "11                1  C_ID_ef55cf8d4b      69           0             1   \n",
       "12                1  C_ID_ef55cf8d4b      69           0             1   \n",
       "13                1  C_ID_ef55cf8d4b      69           0             1   \n",
       "14                1  C_ID_ef55cf8d4b      -1           1             1   \n",
       "15                1  C_ID_ef55cf8d4b     231           0             1   \n",
       "16                1  C_ID_ef55cf8d4b      69           0             1   \n",
       "17                1  C_ID_ef55cf8d4b      69           0             1   \n",
       "18                1  C_ID_ef55cf8d4b     231           0             3   \n",
       "19                1  C_ID_ef55cf8d4b     231           0             1   \n",
       "\n",
       "   category_3 merchant_category_id      merchant_id  month_lag  \\\n",
       "0           B                  307  M_ID_b0c793002c          1   \n",
       "1           B                  307  M_ID_88920c89e8          1   \n",
       "2           B                  507  M_ID_ad5237ef6b          2   \n",
       "3           B                  661  M_ID_9e84cda3b1          1   \n",
       "4           B                  166  M_ID_3c86fa3831          1   \n",
       "5           B                  367  M_ID_8874615e00          2   \n",
       "6           B                  333  M_ID_6d061b5ddc          1   \n",
       "7           B                  307  M_ID_df1e022f41          2   \n",
       "8           B                  278  M_ID_d15eae0468          2   \n",
       "9           B                  437  M_ID_5f9bffd028          1   \n",
       "10          A                   45  M_ID_3ffd43b4cd          1   \n",
       "11          B                  108  M_ID_e6f5213fbf          1   \n",
       "12          B                  278  M_ID_aa97bc87f6          1   \n",
       "13          B                  157  M_ID_28fbc8c74d          2   \n",
       "14          B                  302  M_ID_b9f9332438          1   \n",
       "15          B                   80  M_ID_ad15049b64          2   \n",
       "16          B                  307  M_ID_51757498fd          1   \n",
       "17          B                  560  M_ID_a1183a4e5e          2   \n",
       "18          C                  333  M_ID_f74d9cbbd0          1   \n",
       "19          B                  369  M_ID_f0bf60c061          1   \n",
       "\n",
       "    purchase_amount     ...     purchase_Dayofyear  purchase_Is_month_end  \\\n",
       "0         -0.557574     ...                     70                  False   \n",
       "1         -0.569580     ...                     78                  False   \n",
       "2         -0.551037     ...                    116                  False   \n",
       "3         -0.671925     ...                     66                  False   \n",
       "4         -0.659904     ...                     81                  False   \n",
       "5         -0.633007     ...                     92                  False   \n",
       "6          5.263697     ...                     87                  False   \n",
       "7         -0.553787     ...                     95                  False   \n",
       "8         -0.596643     ...                     97                  False   \n",
       "9         -0.607191     ...                     76                  False   \n",
       "10         4.452265     ...                     90                   True   \n",
       "11        -0.605959     ...                     70                  False   \n",
       "12        -0.634209     ...                     73                  False   \n",
       "13        -0.492058     ...                    104                  False   \n",
       "14        -0.665539     ...                     82                  False   \n",
       "15        -0.646245     ...                    108                  False   \n",
       "16        -0.671775     ...                     70                  False   \n",
       "17        -0.690228     ...                    112                  False   \n",
       "18         0.751371     ...                     60                  False   \n",
       "19        -0.343296     ...                     81                  False   \n",
       "\n",
       "   purchase_Is_month_start purchase_Is_quarter_end purchase_Is_quarter_start  \\\n",
       "0                    False                   False                     False   \n",
       "1                    False                   False                     False   \n",
       "2                    False                   False                     False   \n",
       "3                    False                   False                     False   \n",
       "4                    False                   False                     False   \n",
       "5                    False                   False                     False   \n",
       "6                    False                   False                     False   \n",
       "7                    False                   False                     False   \n",
       "8                    False                   False                     False   \n",
       "9                    False                   False                     False   \n",
       "10                   False                    True                     False   \n",
       "11                   False                   False                     False   \n",
       "12                   False                   False                     False   \n",
       "13                   False                   False                     False   \n",
       "14                   False                   False                     False   \n",
       "15                   False                   False                     False   \n",
       "16                   False                   False                     False   \n",
       "17                   False                   False                     False   \n",
       "18                    True                   False                     False   \n",
       "19                   False                   False                     False   \n",
       "\n",
       "   purchase_Is_year_end  purchase_Is_year_start  purchase_Elapsed  \\\n",
       "0                 False                   False        1520780256   \n",
       "1                 False                   False        1521485617   \n",
       "2                 False                   False        1524751724   \n",
       "3                 False                   False        1520415801   \n",
       "4                 False                   False        1521752873   \n",
       "5                 False                   False        1522673608   \n",
       "6                 False                   False        1522266619   \n",
       "7                 False                   False        1522915612   \n",
       "8                 False                   False        1523126260   \n",
       "9                 False                   False        1521310241   \n",
       "10                False                   False        1522490140   \n",
       "11                False                   False        1520772821   \n",
       "12                False                   False        1521036463   \n",
       "13                False                   False        1523698065   \n",
       "14                False                   False        1521840953   \n",
       "15                False                   False        1524055308   \n",
       "16                False                   False        1520771936   \n",
       "17                False                   False        1524409227   \n",
       "18                False                   False        1519914412   \n",
       "19                False                   False        1521712643   \n",
       "\n",
       "    purchased_on_weekend  month_diff  \n",
       "0                      1          11  \n",
       "1                      0          10  \n",
       "2                      0          10  \n",
       "3                      0          11  \n",
       "4                      0          10  \n",
       "5                      0          11  \n",
       "6                      0          10  \n",
       "7                      0          11  \n",
       "8                      1          11  \n",
       "9                      1          11  \n",
       "10                     1          10  \n",
       "11                     1          11  \n",
       "12                     0          11  \n",
       "13                     1          11  \n",
       "14                     0          10  \n",
       "15                     0          10  \n",
       "16                     1          11  \n",
       "17                     1          10  \n",
       "18                     0          11  \n",
       "19                     0          10  \n",
       "\n",
       "[20 rows x 29 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>card_id</th>\n",
       "      <th>category_2</th>\n",
       "      <th colspan=\"2\" halign=\"left\">purchase_amount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C_ID_415bb3a509</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-2.350116</td>\n",
       "      <td>-0.587529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C_ID_ef55cf8d4b</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.667491</td>\n",
       "      <td>0.166718</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           card_id category_2 purchase_amount          \n",
       "                                          sum      mean\n",
       "0  C_ID_415bb3a509        1.0       -2.350116 -0.587529\n",
       "1  C_ID_ef55cf8d4b        1.0        2.667491  0.166718"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_copy.groupby(['card_id', 'category_2']).agg({'purchase_amount': ['sum', 'mean']}).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.io import *\n",
    "from fastai.structured import *\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from pandas_summary import DataFrameSummary\n",
    "from IPython.display import display\n",
    "from sklearn import metrics\n",
    "import feather"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_filtered_direct = extract_relevant_features(hist_copy, y,\n",
    "                                                     column_id='purchase_date', column_sort='time')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "??extract_relevant_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def successive_aggregates(df, field1, field2):\n",
    "    t = df.groupby(['card_id', field1])[field2].mean()\n",
    "    u = pd.DataFrame(t).reset_index().groupby('card_id')[field2].agg(['mean', 'min', 'max', 'std'])\n",
    "    u.columns = [field1 + '_' + field2 + '_' + col for col in u.columns.values]\n",
    "    u.reset_index(inplace=True)\n",
    "    return u"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card_id</th>\n",
       "      <th>category_1_purchase_amount_mean</th>\n",
       "      <th>category_1_purchase_amount_min</th>\n",
       "      <th>category_1_purchase_amount_max</th>\n",
       "      <th>category_1_purchase_amount_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C_ID_00007093c1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C_ID_0001238066</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C_ID_0001506ef0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C_ID_0001793786</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C_ID_000183fdda</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>C_ID_0002709b5a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>C_ID_000298032a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>C_ID_0002ba3c2e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>C_ID_0002c7c2c1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>C_ID_00032df08f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>C_ID_0003754056</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>C_ID_000377f6a0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>C_ID_0003be3c83</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>C_ID_0003f41435</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>C_ID_00042d509c</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>C_ID_0004587331</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>C_ID_0004725b87</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>C_ID_0004888ddd</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>C_ID_0004b68c49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>C_ID_0004c2a5ab</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>C_ID_00057b99fe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>C_ID_000599daf9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>C_ID_0005f16cc8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>C_ID_0006152db8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>C_ID_000616f4a8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>C_ID_000664aa02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>C_ID_00079fec55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>C_ID_0007a60a33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>C_ID_0007fe3157</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>C_ID_000844d8f1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289971</th>\n",
       "      <td>C_ID_fffaed41f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289972</th>\n",
       "      <td>C_ID_fffb79fb56</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289973</th>\n",
       "      <td>C_ID_fffba72dc5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289974</th>\n",
       "      <td>C_ID_fffbdf036b</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289975</th>\n",
       "      <td>C_ID_fffbee5c24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289976</th>\n",
       "      <td>C_ID_fffc96bf24</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289977</th>\n",
       "      <td>C_ID_fffcb74f49</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289978</th>\n",
       "      <td>C_ID_fffcf66e12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289979</th>\n",
       "      <td>C_ID_fffd070e0e</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289980</th>\n",
       "      <td>C_ID_fffd1207a8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289981</th>\n",
       "      <td>C_ID_fffd24dcf3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289982</th>\n",
       "      <td>C_ID_fffd337484</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289983</th>\n",
       "      <td>C_ID_fffd345048</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289984</th>\n",
       "      <td>C_ID_fffd351e7f</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289985</th>\n",
       "      <td>C_ID_fffd93c8bf</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289986</th>\n",
       "      <td>C_ID_fffd943b91</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289987</th>\n",
       "      <td>C_ID_fffdad8aa0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289988</th>\n",
       "      <td>C_ID_fffde15ab6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289989</th>\n",
       "      <td>C_ID_fffdf34798</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289990</th>\n",
       "      <td>C_ID_fffe29a04a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289991</th>\n",
       "      <td>C_ID_fffe78b232</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289992</th>\n",
       "      <td>C_ID_fffe866457</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289993</th>\n",
       "      <td>C_ID_fffea6de74</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289994</th>\n",
       "      <td>C_ID_fffeced303</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289995</th>\n",
       "      <td>C_ID_fffeed3a89</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289996</th>\n",
       "      <td>C_ID_ffff1d9928</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289997</th>\n",
       "      <td>C_ID_ffff579d3a</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289998</th>\n",
       "      <td>C_ID_ffff756266</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289999</th>\n",
       "      <td>C_ID_ffff828181</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290000</th>\n",
       "      <td>C_ID_fffffd5772</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>290001 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                card_id  category_1_purchase_amount_mean  \\\n",
       "0       C_ID_00007093c1                              NaN   \n",
       "1       C_ID_0001238066                              NaN   \n",
       "2       C_ID_0001506ef0                              NaN   \n",
       "3       C_ID_0001793786                              NaN   \n",
       "4       C_ID_000183fdda                              NaN   \n",
       "5       C_ID_0002709b5a                              NaN   \n",
       "6       C_ID_000298032a                              NaN   \n",
       "7       C_ID_0002ba3c2e                              NaN   \n",
       "8       C_ID_0002c7c2c1                              NaN   \n",
       "9       C_ID_00032df08f                              NaN   \n",
       "10      C_ID_0003754056                              NaN   \n",
       "11      C_ID_000377f6a0                              NaN   \n",
       "12      C_ID_0003be3c83                              NaN   \n",
       "13      C_ID_0003f41435                              NaN   \n",
       "14      C_ID_00042d509c                              NaN   \n",
       "15      C_ID_0004587331                              NaN   \n",
       "16      C_ID_0004725b87                              NaN   \n",
       "17      C_ID_0004888ddd                              NaN   \n",
       "18      C_ID_0004b68c49                              NaN   \n",
       "19      C_ID_0004c2a5ab                              NaN   \n",
       "20      C_ID_00057b99fe                              NaN   \n",
       "21      C_ID_000599daf9                              NaN   \n",
       "22      C_ID_0005f16cc8                              NaN   \n",
       "23      C_ID_0006152db8                              NaN   \n",
       "24      C_ID_000616f4a8                              NaN   \n",
       "25      C_ID_000664aa02                              NaN   \n",
       "26      C_ID_00079fec55                              NaN   \n",
       "27      C_ID_0007a60a33                              NaN   \n",
       "28      C_ID_0007fe3157                              NaN   \n",
       "29      C_ID_000844d8f1                              NaN   \n",
       "...                 ...                              ...   \n",
       "289971  C_ID_fffaed41f0                              NaN   \n",
       "289972  C_ID_fffb79fb56                              NaN   \n",
       "289973  C_ID_fffba72dc5                              NaN   \n",
       "289974  C_ID_fffbdf036b                              NaN   \n",
       "289975  C_ID_fffbee5c24                              NaN   \n",
       "289976  C_ID_fffc96bf24                              NaN   \n",
       "289977  C_ID_fffcb74f49                              NaN   \n",
       "289978  C_ID_fffcf66e12                              NaN   \n",
       "289979  C_ID_fffd070e0e                              NaN   \n",
       "289980  C_ID_fffd1207a8                              NaN   \n",
       "289981  C_ID_fffd24dcf3                              NaN   \n",
       "289982  C_ID_fffd337484                              NaN   \n",
       "289983  C_ID_fffd345048                              NaN   \n",
       "289984  C_ID_fffd351e7f                              NaN   \n",
       "289985  C_ID_fffd93c8bf                              NaN   \n",
       "289986  C_ID_fffd943b91                              NaN   \n",
       "289987  C_ID_fffdad8aa0                              NaN   \n",
       "289988  C_ID_fffde15ab6                              NaN   \n",
       "289989  C_ID_fffdf34798                              NaN   \n",
       "289990  C_ID_fffe29a04a                              NaN   \n",
       "289991  C_ID_fffe78b232                              NaN   \n",
       "289992  C_ID_fffe866457                              NaN   \n",
       "289993  C_ID_fffea6de74                              NaN   \n",
       "289994  C_ID_fffeced303                              NaN   \n",
       "289995  C_ID_fffeed3a89                              NaN   \n",
       "289996  C_ID_ffff1d9928                              NaN   \n",
       "289997  C_ID_ffff579d3a                              NaN   \n",
       "289998  C_ID_ffff756266                              NaN   \n",
       "289999  C_ID_ffff828181                              NaN   \n",
       "290000  C_ID_fffffd5772                              NaN   \n",
       "\n",
       "        category_1_purchase_amount_min  category_1_purchase_amount_max  \\\n",
       "0                                  NaN                             NaN   \n",
       "1                                  NaN                             NaN   \n",
       "2                                  NaN                             NaN   \n",
       "3                                  NaN                             NaN   \n",
       "4                                  NaN                             NaN   \n",
       "5                                  NaN                             NaN   \n",
       "6                                  NaN                             NaN   \n",
       "7                                  NaN                             NaN   \n",
       "8                                  NaN                             NaN   \n",
       "9                                  NaN                             NaN   \n",
       "10                                 NaN                             NaN   \n",
       "11                                 NaN                             NaN   \n",
       "12                                 NaN                             NaN   \n",
       "13                                 NaN                             NaN   \n",
       "14                                 NaN                             NaN   \n",
       "15                                 NaN                             NaN   \n",
       "16                                 NaN                             NaN   \n",
       "17                                 NaN                             NaN   \n",
       "18                                 NaN                             NaN   \n",
       "19                                 NaN                             NaN   \n",
       "20                                 NaN                             NaN   \n",
       "21                                 NaN                             NaN   \n",
       "22                                 NaN                             NaN   \n",
       "23                                 NaN                             NaN   \n",
       "24                                 NaN                             NaN   \n",
       "25                                 NaN                             NaN   \n",
       "26                                 NaN                             NaN   \n",
       "27                                 NaN                             NaN   \n",
       "28                                 NaN                             NaN   \n",
       "29                                 NaN                             NaN   \n",
       "...                                ...                             ...   \n",
       "289971                             NaN                             NaN   \n",
       "289972                             NaN                             NaN   \n",
       "289973                             NaN                             NaN   \n",
       "289974                             NaN                             NaN   \n",
       "289975                             NaN                             NaN   \n",
       "289976                             NaN                             NaN   \n",
       "289977                             NaN                             NaN   \n",
       "289978                             NaN                             NaN   \n",
       "289979                             NaN                             NaN   \n",
       "289980                             NaN                             NaN   \n",
       "289981                             NaN                             NaN   \n",
       "289982                             NaN                             NaN   \n",
       "289983                             NaN                             NaN   \n",
       "289984                             NaN                             NaN   \n",
       "289985                             NaN                             NaN   \n",
       "289986                             NaN                             NaN   \n",
       "289987                             NaN                             NaN   \n",
       "289988                             NaN                             NaN   \n",
       "289989                             NaN                             NaN   \n",
       "289990                             NaN                             NaN   \n",
       "289991                             NaN                             NaN   \n",
       "289992                             NaN                             NaN   \n",
       "289993                             NaN                             NaN   \n",
       "289994                             NaN                             NaN   \n",
       "289995                             NaN                             NaN   \n",
       "289996                             NaN                             NaN   \n",
       "289997                             NaN                             NaN   \n",
       "289998                             NaN                             NaN   \n",
       "289999                             NaN                             NaN   \n",
       "290000                             NaN                             NaN   \n",
       "\n",
       "        category_1_purchase_amount_std  \n",
       "0                                  NaN  \n",
       "1                                  NaN  \n",
       "2                                  NaN  \n",
       "3                                  NaN  \n",
       "4                                  NaN  \n",
       "5                                  NaN  \n",
       "6                                  NaN  \n",
       "7                                  NaN  \n",
       "8                                  NaN  \n",
       "9                                  NaN  \n",
       "10                                 NaN  \n",
       "11                                 NaN  \n",
       "12                                 NaN  \n",
       "13                                 NaN  \n",
       "14                                 NaN  \n",
       "15                                 NaN  \n",
       "16                                 NaN  \n",
       "17                                 NaN  \n",
       "18                                 NaN  \n",
       "19                                 NaN  \n",
       "20                                 NaN  \n",
       "21                                 NaN  \n",
       "22                                 NaN  \n",
       "23                                 NaN  \n",
       "24                                 NaN  \n",
       "25                                 NaN  \n",
       "26                                 NaN  \n",
       "27                                 NaN  \n",
       "28                                 NaN  \n",
       "29                                 NaN  \n",
       "...                                ...  \n",
       "289971                             NaN  \n",
       "289972                             NaN  \n",
       "289973                             NaN  \n",
       "289974                             NaN  \n",
       "289975                             NaN  \n",
       "289976                             NaN  \n",
       "289977                             NaN  \n",
       "289978                             NaN  \n",
       "289979                             NaN  \n",
       "289980                             NaN  \n",
       "289981                             NaN  \n",
       "289982                             NaN  \n",
       "289983                             NaN  \n",
       "289984                             NaN  \n",
       "289985                             NaN  \n",
       "289986                             NaN  \n",
       "289987                             NaN  \n",
       "289988                             NaN  \n",
       "289989                             NaN  \n",
       "289990                             NaN  \n",
       "289991                             NaN  \n",
       "289992                             NaN  \n",
       "289993                             NaN  \n",
       "289994                             NaN  \n",
       "289995                             NaN  \n",
       "289996                             NaN  \n",
       "289997                             NaN  \n",
       "289998                             NaN  \n",
       "289999                             NaN  \n",
       "290000                             NaN  \n",
       "\n",
       "[290001 rows x 5 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "successive_aggregates(hist_copy, 'category_1', 'purchase_amount')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1963031, 29)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cats(new_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag                       int64\n",
       "card_id                            category\n",
       "city_id                            category\n",
       "category_1                            int64\n",
       "installments                          int64\n",
       "category_3                         category\n",
       "merchant_category_id               category\n",
       "merchant_id                        category\n",
       "month_lag                             int64\n",
       "purchase_amount                     float64\n",
       "purchase_date                datetime64[ns]\n",
       "category_2                          float64\n",
       "state_id                           category\n",
       "subsector_id                       category\n",
       "purchase_Year                      category\n",
       "purchase_Month                     category\n",
       "purchase_Week                         int64\n",
       "purchase_Day                          int64\n",
       "purchase_Dayofweek                    int64\n",
       "purchase_Dayofyear                    int64\n",
       "purchase_Is_month_end                  bool\n",
       "purchase_Is_month_start                bool\n",
       "purchase_Is_quarter_end                bool\n",
       "purchase_Is_quarter_start              bool\n",
       "purchase_Is_year_end                   bool\n",
       "purchase_Is_year_start                 bool\n",
       "purchase_Elapsed                      int64\n",
       "purchased_on_weekend                  int64\n",
       "month_diff                            int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag                       int64\n",
       "card_id                            category\n",
       "city_id                            category\n",
       "category_1                            int64\n",
       "installments                          int64\n",
       "category_3                           object\n",
       "merchant_category_id               category\n",
       "merchant_id                        category\n",
       "month_lag                             int64\n",
       "purchase_amount                     float64\n",
       "purchase_date                datetime64[ns]\n",
       "category_2                          float64\n",
       "state_id                           category\n",
       "subsector_id                       category\n",
       "purchase_Year                      category\n",
       "purchase_Month                     category\n",
       "purchase_Week                         int64\n",
       "purchase_Day                          int64\n",
       "purchase_Dayofweek                    int64\n",
       "purchase_Dayofyear                    int64\n",
       "purchase_Is_month_end                  bool\n",
       "purchase_Is_month_start                bool\n",
       "purchase_Is_quarter_end                bool\n",
       "purchase_Is_quarter_start              bool\n",
       "purchase_Is_year_end                   bool\n",
       "purchase_Is_year_start                 bool\n",
       "purchase_Elapsed                      int64\n",
       "purchased_on_weekend                  int64\n",
       "month_diff                            int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist_copy.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist = pd.read_csv('data/elo/new_merchant_transactions.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "add_datepart(new_hist, 'purchase_date', drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1963031, 27)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_extra_cols(df):\n",
    "    df['purchased_on_weekend'] = (df.purchase_Dayofweek >=5).astype(int)\n",
    "    df['month_diff'] = ((datetime.datetime.today() - df['purchase_date']).dt.days)//30\n",
    "    df['month_diff'] += df['month_lag']\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag                      object\n",
       "card_id                              object\n",
       "city_id                               int64\n",
       "category_1                           object\n",
       "installments                          int64\n",
       "category_3                           object\n",
       "merchant_category_id                  int64\n",
       "merchant_id                          object\n",
       "month_lag                             int64\n",
       "purchase_amount                     float64\n",
       "purchase_date                datetime64[ns]\n",
       "category_2                          float64\n",
       "state_id                              int64\n",
       "subsector_id                          int64\n",
       "purchase_Year                         int64\n",
       "purchase_Month                        int64\n",
       "purchase_Week                         int64\n",
       "purchase_Day                          int64\n",
       "purchase_Dayofweek                    int64\n",
       "purchase_Dayofyear                    int64\n",
       "purchase_Is_month_end                  bool\n",
       "purchase_Is_month_start                bool\n",
       "purchase_Is_quarter_end                bool\n",
       "purchase_Is_quarter_start              bool\n",
       "purchase_Is_year_end                   bool\n",
       "purchase_Is_year_start                 bool\n",
       "purchase_Elapsed                      int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist = add_extra_cols(new_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorized_flag</th>\n",
       "      <th>card_id</th>\n",
       "      <th>city_id</th>\n",
       "      <th>category_1</th>\n",
       "      <th>installments</th>\n",
       "      <th>category_3</th>\n",
       "      <th>merchant_category_id</th>\n",
       "      <th>merchant_id</th>\n",
       "      <th>month_lag</th>\n",
       "      <th>purchase_amount</th>\n",
       "      <th>...</th>\n",
       "      <th>purchase_Dayofyear</th>\n",
       "      <th>purchase_Is_month_end</th>\n",
       "      <th>purchase_Is_month_start</th>\n",
       "      <th>purchase_Is_quarter_end</th>\n",
       "      <th>purchase_Is_quarter_start</th>\n",
       "      <th>purchase_Is_year_end</th>\n",
       "      <th>purchase_Is_year_start</th>\n",
       "      <th>purchase_Elapsed</th>\n",
       "      <th>purchased_on_weekend</th>\n",
       "      <th>month_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>...</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>1.96303e+06</td>\n",
       "      <td>1.96303e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>134.387</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.682964</td>\n",
       "      <td>NaN</td>\n",
       "      <td>430.97</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.47652</td>\n",
       "      <td>-0.550969</td>\n",
       "      <td>...</td>\n",
       "      <td>109.884</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.51935e+09</td>\n",
       "      <td>0.307438</td>\n",
       "      <td>11.8304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>101.515</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.58407</td>\n",
       "      <td>NaN</td>\n",
       "      <td>246.339</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.499448</td>\n",
       "      <td>0.694004</td>\n",
       "      <td>...</td>\n",
       "      <td>75.1097</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.6753e+06</td>\n",
       "      <td>0.461433</td>\n",
       "      <td>2.55716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.746893</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.48834e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>69</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>278</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.716629</td>\n",
       "      <td>...</td>\n",
       "      <td>71</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.51967e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>110</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>367</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.674841</td>\n",
       "      <td>...</td>\n",
       "      <td>90</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.52146e+09</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>212</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>683</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.581616</td>\n",
       "      <td>...</td>\n",
       "      <td>112</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.52322e+09</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>347</td>\n",
       "      <td>NaN</td>\n",
       "      <td>999</td>\n",
       "      <td>NaN</td>\n",
       "      <td>891</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>263.157</td>\n",
       "      <td>...</td>\n",
       "      <td>365</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.52513e+09</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>counts</th>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1907109</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1936815</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>...</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "      <td>1963031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uniques</th>\n",
       "      <td>1</td>\n",
       "      <td>290001</td>\n",
       "      <td>308</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>314</td>\n",
       "      <td>226129</td>\n",
       "      <td>2</td>\n",
       "      <td>75190</td>\n",
       "      <td>...</td>\n",
       "      <td>365</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1667025</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>55922</td>\n",
       "      <td>0</td>\n",
       "      <td>26216</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing_perc</th>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>2.85%</td>\n",
       "      <td>0%</td>\n",
       "      <td>1.34%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>...</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>types</th>\n",
       "      <td>constant</td>\n",
       "      <td>categorical</td>\n",
       "      <td>numeric</td>\n",
       "      <td>bool</td>\n",
       "      <td>numeric</td>\n",
       "      <td>categorical</td>\n",
       "      <td>numeric</td>\n",
       "      <td>categorical</td>\n",
       "      <td>bool</td>\n",
       "      <td>numeric</td>\n",
       "      <td>...</td>\n",
       "      <td>numeric</td>\n",
       "      <td>bool</td>\n",
       "      <td>bool</td>\n",
       "      <td>bool</td>\n",
       "      <td>bool</td>\n",
       "      <td>bool</td>\n",
       "      <td>bool</td>\n",
       "      <td>numeric</td>\n",
       "      <td>bool</td>\n",
       "      <td>numeric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             authorized_flag      card_id      city_id category_1  \\\n",
       "count                    NaN          NaN  1.96303e+06        NaN   \n",
       "mean                     NaN          NaN      134.387        NaN   \n",
       "std                      NaN          NaN      101.515        NaN   \n",
       "min                      NaN          NaN           -1        NaN   \n",
       "25%                      NaN          NaN           69        NaN   \n",
       "50%                      NaN          NaN          110        NaN   \n",
       "75%                      NaN          NaN          212        NaN   \n",
       "max                      NaN          NaN          347        NaN   \n",
       "counts               1963031      1963031      1963031    1963031   \n",
       "uniques                    1       290001          308          2   \n",
       "missing                    0            0            0          0   \n",
       "missing_perc              0%           0%           0%         0%   \n",
       "types               constant  categorical      numeric       bool   \n",
       "\n",
       "             installments   category_3 merchant_category_id  merchant_id  \\\n",
       "count         1.96303e+06          NaN          1.96303e+06          NaN   \n",
       "mean             0.682964          NaN               430.97          NaN   \n",
       "std               1.58407          NaN              246.339          NaN   \n",
       "min                    -1          NaN                   -1          NaN   \n",
       "25%                     0          NaN                  278          NaN   \n",
       "50%                     1          NaN                  367          NaN   \n",
       "75%                     1          NaN                  683          NaN   \n",
       "max                   999          NaN                  891          NaN   \n",
       "counts            1963031      1907109              1963031      1936815   \n",
       "uniques                15            3                  314       226129   \n",
       "missing                 0        55922                    0        26216   \n",
       "missing_perc           0%        2.85%                   0%        1.34%   \n",
       "types             numeric  categorical              numeric  categorical   \n",
       "\n",
       "                month_lag purchase_amount     ...      purchase_Dayofyear  \\\n",
       "count         1.96303e+06     1.96303e+06     ...             1.96303e+06   \n",
       "mean              1.47652       -0.550969     ...                 109.884   \n",
       "std              0.499448        0.694004     ...                 75.1097   \n",
       "min                     1       -0.746893     ...                       1   \n",
       "25%                     1       -0.716629     ...                      71   \n",
       "50%                     1       -0.674841     ...                      90   \n",
       "75%                     2       -0.581616     ...                     112   \n",
       "max                     2         263.157     ...                     365   \n",
       "counts            1963031         1963031     ...                 1963031   \n",
       "uniques                 2           75190     ...                     365   \n",
       "missing                 0               0     ...                       0   \n",
       "missing_perc           0%              0%     ...                      0%   \n",
       "types                bool         numeric     ...                 numeric   \n",
       "\n",
       "             purchase_Is_month_end purchase_Is_month_start  \\\n",
       "count                          NaN                     NaN   \n",
       "mean                           NaN                     NaN   \n",
       "std                            NaN                     NaN   \n",
       "min                            NaN                     NaN   \n",
       "25%                            NaN                     NaN   \n",
       "50%                            NaN                     NaN   \n",
       "75%                            NaN                     NaN   \n",
       "max                            NaN                     NaN   \n",
       "counts                     1963031                 1963031   \n",
       "uniques                          2                       2   \n",
       "missing                          0                       0   \n",
       "missing_perc                    0%                      0%   \n",
       "types                         bool                    bool   \n",
       "\n",
       "             purchase_Is_quarter_end purchase_Is_quarter_start  \\\n",
       "count                            NaN                       NaN   \n",
       "mean                             NaN                       NaN   \n",
       "std                              NaN                       NaN   \n",
       "min                              NaN                       NaN   \n",
       "25%                              NaN                       NaN   \n",
       "50%                              NaN                       NaN   \n",
       "75%                              NaN                       NaN   \n",
       "max                              NaN                       NaN   \n",
       "counts                       1963031                   1963031   \n",
       "uniques                            2                         2   \n",
       "missing                            0                         0   \n",
       "missing_perc                      0%                        0%   \n",
       "types                           bool                      bool   \n",
       "\n",
       "             purchase_Is_year_end purchase_Is_year_start purchase_Elapsed  \\\n",
       "count                         NaN                    NaN      1.96303e+06   \n",
       "mean                          NaN                    NaN      1.51935e+09   \n",
       "std                           NaN                    NaN       6.6753e+06   \n",
       "min                           NaN                    NaN      1.48834e+09   \n",
       "25%                           NaN                    NaN      1.51967e+09   \n",
       "50%                           NaN                    NaN      1.52146e+09   \n",
       "75%                           NaN                    NaN      1.52322e+09   \n",
       "max                           NaN                    NaN      1.52513e+09   \n",
       "counts                    1963031                1963031          1963031   \n",
       "uniques                         2                      2          1667025   \n",
       "missing                         0                      0                0   \n",
       "missing_perc                   0%                     0%               0%   \n",
       "types                        bool                   bool          numeric   \n",
       "\n",
       "             purchased_on_weekend   month_diff  \n",
       "count                 1.96303e+06  1.96303e+06  \n",
       "mean                     0.307438      11.8304  \n",
       "std                      0.461433      2.55716  \n",
       "min                             0           10  \n",
       "25%                             0           10  \n",
       "50%                             0           11  \n",
       "75%                             1           12  \n",
       "max                             1           23  \n",
       "counts                    1963031      1963031  \n",
       "uniques                         2           14  \n",
       "missing                         0            0  \n",
       "missing_perc                   0%           0%  \n",
       "types                        bool      numeric  \n",
       "\n",
       "[13 rows x 29 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrameSummary(new_hist).summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cats(new_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "authorized_flag                    category\n",
       "card_id                            category\n",
       "city_id                               int64\n",
       "category_1                         category\n",
       "installments                          int64\n",
       "category_3                         category\n",
       "merchant_category_id                  int64\n",
       "merchant_id                        category\n",
       "month_lag                             int64\n",
       "purchase_amount                     float64\n",
       "purchase_date                datetime64[ns]\n",
       "category_2                          float64\n",
       "state_id                              int64\n",
       "subsector_id                          int64\n",
       "purchase_Year                         int64\n",
       "purchase_Month                        int64\n",
       "purchase_Week                         int64\n",
       "purchase_Day                          int64\n",
       "purchase_Dayofweek                    int64\n",
       "purchase_Dayofyear                    int64\n",
       "purchase_Is_month_end                  bool\n",
       "purchase_Is_month_start                bool\n",
       "purchase_Is_quarter_end                bool\n",
       "purchase_Is_quarter_start              bool\n",
       "purchase_Is_year_end                   bool\n",
       "purchase_Is_year_start                 bool\n",
       "purchase_Elapsed                      int64\n",
       "purchased_on_weekend                  int64\n",
       "month_diff                            int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist_prc, y, nas = proc_df(new_hist, ignore_flds=['purchase_date'], max_n_cat=7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "??proc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "purchase_date                datetime64[ns]\n",
       "card_id                               int64\n",
       "city_id                               int64\n",
       "installments                          int64\n",
       "merchant_category_id                  int64\n",
       "merchant_id                           int64\n",
       "month_lag                             int64\n",
       "purchase_amount                     float64\n",
       "category_2                          float64\n",
       "state_id                              int64\n",
       "subsector_id                          int64\n",
       "purchase_Year                         int64\n",
       "purchase_Month                        int64\n",
       "purchase_Week                         int64\n",
       "purchase_Day                          int64\n",
       "purchase_Dayofweek                    int64\n",
       "purchase_Dayofyear                    int64\n",
       "purchase_Is_month_end                  bool\n",
       "purchase_Is_month_start                bool\n",
       "purchase_Is_quarter_end                bool\n",
       "purchase_Is_quarter_start              bool\n",
       "purchase_Is_year_end                   bool\n",
       "purchase_Is_year_start                 bool\n",
       "purchase_Elapsed                      int64\n",
       "purchased_on_weekend                  int64\n",
       "month_diff                            int64\n",
       "category_2_na                          bool\n",
       "authorized_flag_Y                     uint8\n",
       "authorized_flag_nan                   uint8\n",
       "category_1_N                          uint8\n",
       "category_1_Y                          uint8\n",
       "category_1_nan                        uint8\n",
       "category_3_A                          uint8\n",
       "category_3_B                          uint8\n",
       "category_3_C                          uint8\n",
       "category_3_nan                        uint8\n",
       "dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist_prc.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>purchase_date</th>\n",
       "      <td>2018-03-11 14:57:36</td>\n",
       "      <td>2018-03-19 18:53:37</td>\n",
       "      <td>2018-04-26 14:08:44</td>\n",
       "      <td>2018-03-07 09:43:21</td>\n",
       "      <td>2018-03-22 21:07:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_id</th>\n",
       "      <td>74151</td>\n",
       "      <td>74151</td>\n",
       "      <td>74151</td>\n",
       "      <td>74151</td>\n",
       "      <td>271039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city_id</th>\n",
       "      <td>107</td>\n",
       "      <td>140</td>\n",
       "      <td>330</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>installments</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_category_id</th>\n",
       "      <td>307</td>\n",
       "      <td>307</td>\n",
       "      <td>507</td>\n",
       "      <td>661</td>\n",
       "      <td>166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>merchant_id</th>\n",
       "      <td>156250</td>\n",
       "      <td>120794</td>\n",
       "      <td>153186</td>\n",
       "      <td>140183</td>\n",
       "      <td>53390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_lag</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_amount</th>\n",
       "      <td>-0.557574</td>\n",
       "      <td>-0.56958</td>\n",
       "      <td>-0.551037</td>\n",
       "      <td>-0.671925</td>\n",
       "      <td>-0.659904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state_id</th>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsector_id</th>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>14</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Year</th>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Month</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Week</th>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Day</th>\n",
       "      <td>11</td>\n",
       "      <td>19</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Dayofweek</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Dayofyear</th>\n",
       "      <td>70</td>\n",
       "      <td>78</td>\n",
       "      <td>116</td>\n",
       "      <td>66</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_month_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_month_start</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_quarter_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_quarter_start</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_year_end</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Is_year_start</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchase_Elapsed</th>\n",
       "      <td>1520780256</td>\n",
       "      <td>1521485617</td>\n",
       "      <td>1524751724</td>\n",
       "      <td>1520415801</td>\n",
       "      <td>1521752873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>purchased_on_weekend</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_diff</th>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_2_na</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_Y</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>authorized_flag_nan</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_N</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_Y</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_1_nan</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_3_A</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_3_B</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_3_C</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category_3_nan</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             0                    1  \\\n",
       "purchase_date              2018-03-11 14:57:36  2018-03-19 18:53:37   \n",
       "card_id                                  74151                74151   \n",
       "city_id                                    107                  140   \n",
       "installments                                 1                    1   \n",
       "merchant_category_id                       307                  307   \n",
       "merchant_id                             156250               120794   \n",
       "month_lag                                    1                    1   \n",
       "purchase_amount                      -0.557574             -0.56958   \n",
       "category_2                                   1                    1   \n",
       "state_id                                     9                    9   \n",
       "subsector_id                                19                   19   \n",
       "purchase_Year                             2018                 2018   \n",
       "purchase_Month                               3                    3   \n",
       "purchase_Week                               10                   12   \n",
       "purchase_Day                                11                   19   \n",
       "purchase_Dayofweek                           6                    0   \n",
       "purchase_Dayofyear                          70                   78   \n",
       "purchase_Is_month_end                    False                False   \n",
       "purchase_Is_month_start                  False                False   \n",
       "purchase_Is_quarter_end                  False                False   \n",
       "purchase_Is_quarter_start                False                False   \n",
       "purchase_Is_year_end                     False                False   \n",
       "purchase_Is_year_start                   False                False   \n",
       "purchase_Elapsed                    1520780256           1521485617   \n",
       "purchased_on_weekend                         1                    0   \n",
       "month_diff                                  11                   11   \n",
       "category_2_na                            False                False   \n",
       "authorized_flag_Y                            1                    1   \n",
       "authorized_flag_nan                          0                    0   \n",
       "category_1_N                                 1                    1   \n",
       "category_1_Y                                 0                    0   \n",
       "category_1_nan                               0                    0   \n",
       "category_3_A                                 0                    0   \n",
       "category_3_B                                 1                    1   \n",
       "category_3_C                                 0                    0   \n",
       "category_3_nan                               0                    0   \n",
       "\n",
       "                                             2                    3  \\\n",
       "purchase_date              2018-04-26 14:08:44  2018-03-07 09:43:21   \n",
       "card_id                                  74151                74151   \n",
       "city_id                                    330                   -1   \n",
       "installments                                 1                    1   \n",
       "merchant_category_id                       507                  661   \n",
       "merchant_id                             153186               140183   \n",
       "month_lag                                    2                    1   \n",
       "purchase_amount                      -0.551037            -0.671925   \n",
       "category_2                                   1                    1   \n",
       "state_id                                     9                   -1   \n",
       "subsector_id                                14                    8   \n",
       "purchase_Year                             2018                 2018   \n",
       "purchase_Month                               4                    3   \n",
       "purchase_Week                               17                   10   \n",
       "purchase_Day                                26                    7   \n",
       "purchase_Dayofweek                           3                    2   \n",
       "purchase_Dayofyear                         116                   66   \n",
       "purchase_Is_month_end                    False                False   \n",
       "purchase_Is_month_start                  False                False   \n",
       "purchase_Is_quarter_end                  False                False   \n",
       "purchase_Is_quarter_start                False                False   \n",
       "purchase_Is_year_end                     False                False   \n",
       "purchase_Is_year_start                   False                False   \n",
       "purchase_Elapsed                    1524751724           1520415801   \n",
       "purchased_on_weekend                         0                    0   \n",
       "month_diff                                  10                   11   \n",
       "category_2_na                            False                 True   \n",
       "authorized_flag_Y                            1                    1   \n",
       "authorized_flag_nan                          0                    0   \n",
       "category_1_N                                 1                    0   \n",
       "category_1_Y                                 0                    1   \n",
       "category_1_nan                               0                    0   \n",
       "category_3_A                                 0                    0   \n",
       "category_3_B                                 1                    1   \n",
       "category_3_C                                 0                    0   \n",
       "category_3_nan                               0                    0   \n",
       "\n",
       "                                             4  \n",
       "purchase_date              2018-03-22 21:07:53  \n",
       "card_id                                 271039  \n",
       "city_id                                     -1  \n",
       "installments                                 1  \n",
       "merchant_category_id                       166  \n",
       "merchant_id                              53390  \n",
       "month_lag                                    1  \n",
       "purchase_amount                      -0.659904  \n",
       "category_2                                   1  \n",
       "state_id                                    -1  \n",
       "subsector_id                                29  \n",
       "purchase_Year                             2018  \n",
       "purchase_Month                               3  \n",
       "purchase_Week                               12  \n",
       "purchase_Day                                22  \n",
       "purchase_Dayofweek                           3  \n",
       "purchase_Dayofyear                          81  \n",
       "purchase_Is_month_end                    False  \n",
       "purchase_Is_month_start                  False  \n",
       "purchase_Is_quarter_end                  False  \n",
       "purchase_Is_quarter_start                False  \n",
       "purchase_Is_year_end                     False  \n",
       "purchase_Is_year_start                   False  \n",
       "purchase_Elapsed                    1521752873  \n",
       "purchased_on_weekend                         0  \n",
       "month_diff                                  10  \n",
       "category_2_na                             True  \n",
       "authorized_flag_Y                            1  \n",
       "authorized_flag_nan                          0  \n",
       "category_1_N                                 0  \n",
       "category_1_Y                                 1  \n",
       "category_1_nan                               0  \n",
       "category_3_A                                 0  \n",
       "category_3_B                                 1  \n",
       "category_3_C                                 0  \n",
       "category_3_nan                               0  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist_prc.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1963031, 36)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_hist_prc.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_hist_prc.to_feather('new_hist_prc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'category_2': 1.0}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'dict' object has no attribute 'columns'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-10cce8bb9070>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mapply_cats\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_hist_prc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnas\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/private/competitions/fastai/structured.py\u001b[0m in \u001b[0;36mapply_cats\u001b[0;34m(df, trn)\u001b[0m\n\u001b[1;32m    201\u001b[0m     \"\"\"\n\u001b[1;32m    202\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 203\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'category'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    204\u001b[0m             \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCategorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcategories\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mordered\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'columns'"
     ]
    }
   ],
   "source": [
    "apply_cats(new_hist_prc, nas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "??proc_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "??train_cats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = feather.read_dataframe('train_df_proc_new_m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first_active_month</th>\n",
       "      <th>card_id</th>\n",
       "      <th>feature_1</th>\n",
       "      <th>feature_2</th>\n",
       "      <th>feature_3</th>\n",
       "      <th>target</th>\n",
       "      <th>first_active_monthYear</th>\n",
       "      <th>first_active_monthMonth</th>\n",
       "      <th>first_active_monthWeek</th>\n",
       "      <th>first_active_monthDay</th>\n",
       "      <th>...</th>\n",
       "      <th>installments_total</th>\n",
       "      <th>category_1_Y_sum_total</th>\n",
       "      <th>category_2_sum_total</th>\n",
       "      <th>category_3_A_total</th>\n",
       "      <th>category_3_B_total</th>\n",
       "      <th>category_3_C_total</th>\n",
       "      <th>category_3_sum_total_all</th>\n",
       "      <th>hist_first_buy</th>\n",
       "      <th>new_first_buy</th>\n",
       "      <th>outliers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>...</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>201917</td>\n",
       "      <td>179986</td>\n",
       "      <td>201917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0109302</td>\n",
       "      <td>0.0109302</td>\n",
       "      <td>0.0109302</td>\n",
       "      <td>-0.393636</td>\n",
       "      <td>2016.51</td>\n",
       "      <td>7.37874</td>\n",
       "      <td>30.9446</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>64.1608</td>\n",
       "      <td>6.1902</td>\n",
       "      <td>209.184</td>\n",
       "      <td>53.6095</td>\n",
       "      <td>40.1053</td>\n",
       "      <td>6.30438</td>\n",
       "      <td>100.019</td>\n",
       "      <td>118.748</td>\n",
       "      <td>381.585</td>\n",
       "      <td>0.0109302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0012591</td>\n",
       "      <td>0.00195247</td>\n",
       "      <td>0.000567402</td>\n",
       "      <td>3.8505</td>\n",
       "      <td>0.788199</td>\n",
       "      <td>3.34072</td>\n",
       "      <td>13.8454</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>107.095</td>\n",
       "      <td>18.2701</td>\n",
       "      <td>307.071</td>\n",
       "      <td>99.0326</td>\n",
       "      <td>75.8319</td>\n",
       "      <td>11.1427</td>\n",
       "      <td>109.814</td>\n",
       "      <td>224.136</td>\n",
       "      <td>284.896</td>\n",
       "      <td>0.103975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00805849</td>\n",
       "      <td>0.00875212</td>\n",
       "      <td>0.0102828</td>\n",
       "      <td>-33.2193</td>\n",
       "      <td>2011</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>-331</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0104794</td>\n",
       "      <td>0.00875212</td>\n",
       "      <td>0.0102828</td>\n",
       "      <td>-0.88311</td>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>10</td>\n",
       "      <td>182</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0106099</td>\n",
       "      <td>0.0113848</td>\n",
       "      <td>0.0114275</td>\n",
       "      <td>-0.0234369</td>\n",
       "      <td>2017</td>\n",
       "      <td>8</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>107</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>64</td>\n",
       "      <td>22</td>\n",
       "      <td>294</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0107116</td>\n",
       "      <td>0.0113848</td>\n",
       "      <td>0.0114275</td>\n",
       "      <td>0.765453</td>\n",
       "      <td>2017</td>\n",
       "      <td>10</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>87</td>\n",
       "      <td>5</td>\n",
       "      <td>238</td>\n",
       "      <td>67</td>\n",
       "      <td>48</td>\n",
       "      <td>9</td>\n",
       "      <td>124</td>\n",
       "      <td>123</td>\n",
       "      <td>489</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0131446</td>\n",
       "      <td>0.0141664</td>\n",
       "      <td>0.0114275</td>\n",
       "      <td>17.9651</td>\n",
       "      <td>2018</td>\n",
       "      <td>12</td>\n",
       "      <td>53</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2452</td>\n",
       "      <td>720</td>\n",
       "      <td>6175</td>\n",
       "      <td>2913</td>\n",
       "      <td>1394</td>\n",
       "      <td>518</td>\n",
       "      <td>2915</td>\n",
       "      <td>1995</td>\n",
       "      <td>2351</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>counts</th>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>201917</td>\n",
       "      <td>...</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>179986</td>\n",
       "      <td>201917</td>\n",
       "      <td>179986</td>\n",
       "      <td>201917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uniques</th>\n",
       "      <td>75</td>\n",
       "      <td>201917</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>197110</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>21</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1049</td>\n",
       "      <td>324</td>\n",
       "      <td>2538</td>\n",
       "      <td>971</td>\n",
       "      <td>789</td>\n",
       "      <td>164</td>\n",
       "      <td>1034</td>\n",
       "      <td>1905</td>\n",
       "      <td>1898</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>21931</td>\n",
       "      <td>21931</td>\n",
       "      <td>21931</td>\n",
       "      <td>21931</td>\n",
       "      <td>21931</td>\n",
       "      <td>21931</td>\n",
       "      <td>21931</td>\n",
       "      <td>0</td>\n",
       "      <td>21931</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>missing_perc</th>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "      <td>...</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>0%</td>\n",
       "      <td>10.86%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>types</th>\n",
       "      <td>date</td>\n",
       "      <td>unique</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>bool</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>constant</td>\n",
       "      <td>...</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>numeric</td>\n",
       "      <td>bool</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13 rows × 339 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             first_active_month card_id   feature_1   feature_2    feature_3  \\\n",
       "count                       NaN     NaN      201917      201917       201917   \n",
       "mean                        NaN     NaN   0.0109302   0.0109302    0.0109302   \n",
       "std                         NaN     NaN   0.0012591  0.00195247  0.000567402   \n",
       "min                         NaN     NaN  0.00805849  0.00875212    0.0102828   \n",
       "25%                         NaN     NaN   0.0104794  0.00875212    0.0102828   \n",
       "50%                         NaN     NaN   0.0106099   0.0113848    0.0114275   \n",
       "75%                         NaN     NaN   0.0107116   0.0113848    0.0114275   \n",
       "max                         NaN     NaN   0.0131446   0.0141664    0.0114275   \n",
       "counts                   201917  201917      201917      201917       201917   \n",
       "uniques                      75  201917           5           3            2   \n",
       "missing                       0       0           0           0            0   \n",
       "missing_perc                 0%      0%          0%          0%           0%   \n",
       "types                      date  unique     numeric     numeric         bool   \n",
       "\n",
       "                 target first_active_monthYear first_active_monthMonth  \\\n",
       "count            201917                 201917                  201917   \n",
       "mean          -0.393636                2016.51                 7.37874   \n",
       "std              3.8505               0.788199                 3.34072   \n",
       "min            -33.2193                   2011                       1   \n",
       "25%            -0.88311                   2016                       5   \n",
       "50%          -0.0234369                   2017                       8   \n",
       "75%            0.765453                   2017                      10   \n",
       "max             17.9651                   2018                      12   \n",
       "counts           201917                 201917                  201917   \n",
       "uniques          197110                      8                      12   \n",
       "missing               0                      0                       0   \n",
       "missing_perc         0%                     0%                      0%   \n",
       "types           numeric                numeric                 numeric   \n",
       "\n",
       "             first_active_monthWeek first_active_monthDay    ...      \\\n",
       "count                        201917                201917    ...       \n",
       "mean                        30.9446                     1    ...       \n",
       "std                         13.8454                     0    ...       \n",
       "min                               1                     1    ...       \n",
       "25%                              22                     1    ...       \n",
       "50%                              31                     1    ...       \n",
       "75%                              44                     1    ...       \n",
       "max                              53                     1    ...       \n",
       "counts                       201917                201917    ...       \n",
       "uniques                          21                     1    ...       \n",
       "missing                           0                     0    ...       \n",
       "missing_perc                     0%                    0%    ...       \n",
       "types                       numeric              constant    ...       \n",
       "\n",
       "             installments_total category_1_Y_sum_total category_2_sum_total  \\\n",
       "count                    179986                 179986               179986   \n",
       "mean                    64.1608                 6.1902              209.184   \n",
       "std                     107.095                18.2701              307.071   \n",
       "min                          -5                      0                    3   \n",
       "25%                           1                      0                   50   \n",
       "50%                          20                      0                  107   \n",
       "75%                          87                      5                  238   \n",
       "max                        2452                    720                 6175   \n",
       "counts                   179986                 179986               179986   \n",
       "uniques                    1049                    324                 2538   \n",
       "missing                   21931                  21931                21931   \n",
       "missing_perc             10.86%                 10.86%               10.86%   \n",
       "types                   numeric                numeric              numeric   \n",
       "\n",
       "             category_3_A_total category_3_B_total category_3_C_total  \\\n",
       "count                    179986             179986             179986   \n",
       "mean                    53.6095            40.1053            6.30438   \n",
       "std                     99.0326            75.8319            11.1427   \n",
       "min                           0                  0                  0   \n",
       "25%                           1                  0                  0   \n",
       "50%                           7                  8                  1   \n",
       "75%                          67                 48                  9   \n",
       "max                        2913               1394                518   \n",
       "counts                   179986             179986             179986   \n",
       "uniques                     971                789                164   \n",
       "missing                   21931              21931              21931   \n",
       "missing_perc             10.86%             10.86%             10.86%   \n",
       "types                   numeric            numeric            numeric   \n",
       "\n",
       "             category_3_sum_total_all hist_first_buy new_first_buy   outliers  \n",
       "count                          179986         201917        179986     201917  \n",
       "mean                          100.019        118.748       381.585  0.0109302  \n",
       "std                           109.814        224.136       284.896   0.103975  \n",
       "min                                 3           -331            16          0  \n",
       "25%                                33             10           182          0  \n",
       "50%                                64             22           294          0  \n",
       "75%                               124            123           489          0  \n",
       "max                              2915           1995          2351          1  \n",
       "counts                         179986         201917        179986     201917  \n",
       "uniques                          1034           1905          1898          2  \n",
       "missing                         21931              0         21931          0  \n",
       "missing_perc                   10.86%             0%        10.86%         0%  \n",
       "types                         numeric        numeric       numeric       bool  \n",
       "\n",
       "[13 rows x 339 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DataFrameSummary(t).summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "th = pd.get_dummies(t, columns=['feature_1', 'feature_2', 'feature_3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201917, 339)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201917, 346)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "th.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}