generate_lag_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from itertools import product\n",
    "import seaborn as sns\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.sparse \n",
    "import sklearn\n",
    "\n",
    "from utils import *\n",
    "\n",
    "%matplotlib inline\n",
    "data_path = 'data/'\n",
    "seed=1204\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy 1.13.3\n",
      "pandas 0.20.3\n",
      "scipy 0.19.1\n",
      "sklearn 0.19.1\n"
     ]
    }
   ],
   "source": [
    "for p in [np, pd, scipy, sklearn]:\n",
    "    print (p.__name__, p.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating new_sales.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sales    = pd.read_csv(os.path.join(data_path, 'sales_train.csv.gz'))\n",
    "items           = pd.read_csv(os.path.join(data_path, 'items.csv'),encoding ='ISO-8859-1')\n",
    "item_categories = pd.read_csv(os.path.join(data_path, 'item_categories.csv'),encoding ='ISO-8859-1')\n",
    "shops           = pd.read_csv(os.path.join(data_path, 'shops.csv'),encoding ='ISO-8859-1')\n",
    "test            = pd.read_csv(os.path.join(data_path, 'test.csv.gz'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sales = sales[sales.item_cnt_day<=1000] # there is only 1 item\n",
    "sales = sales[sales.item_price<100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\pandas\\core\\groupby.py:4036: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "# Create \"grid\" with columns\n",
    "index_cols = ['shop_id', 'item_id', 'date_block_num']\n",
    "\n",
    "# For every month we create a grid from all shops/items combinations from that month\n",
    "grid = [] \n",
    "for block_num in sales['date_block_num'].unique():\n",
    "    cur_shops = sales.loc[sales['date_block_num'] == block_num, 'shop_id'].unique()\n",
    "    cur_items = sales.loc[sales['date_block_num'] == block_num, 'item_id'].unique()\n",
    "    grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])),dtype='int32'))\n",
    "\n",
    "# Turn the grid into a dataframe\n",
    "grid = pd.DataFrame(np.vstack(grid), columns = index_cols,dtype=np.int32)\n",
    "\n",
    "# Groupby data to get shop-item-month aggregates to get rid of duplicates\n",
    "gb = sales.groupby(index_cols,as_index=False).agg({'item_cnt_day':{'target':'sum'}})\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('shop_id', '') ('item_id', '') ('date_block_num', '')\n",
      " ('item_cnt_day', 'target')]\n"
     ]
    }
   ],
   "source": [
    "# Fix column names\n",
    "print(gb.columns.values)\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>1</td>\n",
       "      <td>31.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   shop_id  item_id  date_block_num  target\n",
       "0        0       30               1    31.0\n",
       "1        0       31               1    11.0\n",
       "2        0       32               0     6.0\n",
       "3        0       32               1    10.0\n",
       "4        0       33               0     3.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Join it to the grid\n",
    "all_data = pd.merge(grid, gb, how='left', on=index_cols).fillna(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# merge with category id\n",
    "all_data =pd.merge(all_data,items,on=['item_id'],how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\pandas\\core\\groupby.py:4036: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Same as above but with shop-month aggregates\n",
    "gb = sales.groupby(['shop_id', 'date_block_num'],as_index=False).agg({'item_cnt_day':{'shop_block_target_sum':'sum','shop_block_target_mean':np.mean}})\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]\n",
    "all_data = pd.merge(all_data, gb, how='left', on=['shop_id', 'date_block_num']).fillna(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\pandas\\core\\groupby.py:4036: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "# Same as above but with item-month aggregates\n",
    "gb = sales.groupby(['item_id', 'date_block_num'],as_index=False).agg({'item_cnt_day':{'item_block_target_sum':'sum','item_block_target_mean':np.mean}})\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]\n",
    "all_data = pd.merge(all_data, gb, how='left', on=['item_id', 'date_block_num']).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\qtran\\AppData\\Local\\Continuum\\Miniconda3\\lib\\site-packages\\pandas\\core\\groupby.py:4036: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "# Same as above but with item category-month aggregates\n",
    "sales =pd.merge(sales,items,on=['item_id'],how='left')\n",
    "gb = sales.groupby(['item_category_id', 'date_block_num'],as_index=False).agg({'item_cnt_day':{'item_cat_block_target_sum':'sum','item_cat_block_target_mean':np.mean}})\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]\n",
    "all_data = pd.merge(all_data, gb, how='left', on=['item_category_id', 'date_block_num']).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# Downcast dtypes from 64 to 32 bit to save memory\n",
    "all_data = downcast_dtypes(all_data)\n",
    "del grid, gb \n",
    "# gc.collect();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10913804, 12)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data.to_csv(data_path+'new_sales.csv',index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate lag feature new_sales_lag_after12.pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = pd.read_csv(data_path+'new_sales.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['item_block_target_mean', 'item_block_target_sum', 'item_cat_block_target_mean', 'item_cat_block_target_sum', 'shop_block_target_mean', 'shop_block_target_sum', 'target']\n"
     ]
    }
   ],
   "source": [
    "index_cols = ['shop_id', 'item_id', 'date_block_num','item_category_id']\n",
    "cols_to_rename = list(all_data.columns.difference(index_cols))\n",
    "for i in ['item_name']:\n",
    "    cols_to_rename.remove(i)\n",
    "print(cols_to_rename)\n",
    "cols_gb_item = [i for i in cols_to_rename if 'item_block' in i]\n",
    "cols_gb_shop = [i for i in cols_to_rename if 'shop_block' in i]\n",
    "cols_gb_cat = [i for i in cols_to_rename if 'item_cat' in i]\n",
    "cols_gb_all = ['target']\n",
    "cols_gb_key=[['item_id'],['shop_id'],['item_category_id'],['shop_id','item_id']]\n",
    "cols_gb_value = [cols_gb_item,cols_gb_shop,cols_gb_cat,cols_gb_all]\n",
    "print(cols_gb_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "shift_range = [1,2,3,5,12]\n",
    "for month_shift in shift_range:\n",
    "    for k,v in zip(cols_gb_key,cols_gb_value): \n",
    "        index_col = ['date_block_num'] + k\n",
    "        train_shift = all_data[index_col + v].copy().drop_duplicates()\n",
    "\n",
    "        train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift\n",
    "\n",
    "        foo = lambda x: '{}_lag_{}'.format(x, month_shift) if x in v else x\n",
    "        train_shift = train_shift.rename(columns=foo)\n",
    "        all_data = pd.merge(all_data, train_shift, on=index_col, how='left').fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "      <th>item_name</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>shop_block_target_sum</th>\n",
       "      <th>shop_block_target_mean</th>\n",
       "      <th>item_block_target_sum</th>\n",
       "      <th>item_block_target_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>item_cat_block_target_mean_lag_5</th>\n",
       "      <th>item_cat_block_target_sum_lag_5</th>\n",
       "      <th>target_lag_5</th>\n",
       "      <th>item_block_target_mean_lag_12</th>\n",
       "      <th>item_block_target_sum_lag_12</th>\n",
       "      <th>shop_block_target_mean_lag_12</th>\n",
       "      <th>shop_block_target_sum_lag_12</th>\n",
       "      <th>item_cat_block_target_mean_lag_12</th>\n",
       "      <th>item_cat_block_target_sum_lag_12</th>\n",
       "      <th>target_lag_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>59</td>\n",
       "      <td>22154</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>SCENE 2012 (BD)</td>\n",
       "      <td>37</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.092041</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>59</td>\n",
       "      <td>2552</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DEEP PURPLE The House Of Blue Light LP</td>\n",
       "      <td>58</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.092041</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>59</td>\n",
       "      <td>2554</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DEEP PURPLE Who Do You Think We Are LP</td>\n",
       "      <td>58</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.092041</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>59</td>\n",
       "      <td>2555</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DEEP PURPLE 30 Very Best Of 2CD (Firm).</td>\n",
       "      <td>56</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.092041</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>59</td>\n",
       "      <td>2564</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>DEEP PURPLE Perihelion: Live In Concert DVD (C...</td>\n",
       "      <td>59</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.092041</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   shop_id  item_id  date_block_num  target  \\\n",
       "0       59    22154               0     1.0   \n",
       "1       59     2552               0     0.0   \n",
       "2       59     2554               0     0.0   \n",
       "3       59     2555               0     0.0   \n",
       "4       59     2564               0     0.0   \n",
       "\n",
       "                                           item_name  item_category_id  \\\n",
       "0                                    SCENE 2012 (BD)                37   \n",
       "1             DEEP PURPLE The House Of Blue Light LP                58   \n",
       "2             DEEP PURPLE Who Do You Think We Are LP                58   \n",
       "3            DEEP PURPLE 30 Very Best Of 2CD (Firm).                56   \n",
       "4  DEEP PURPLE Perihelion: Live In Concert DVD (C...                59   \n",
       "\n",
       "   shop_block_target_sum  shop_block_target_mean  item_block_target_sum  \\\n",
       "0                 2017.0                1.092041                   18.0   \n",
       "1                 2017.0                1.092041                    0.0   \n",
       "2                 2017.0                1.092041                    1.0   \n",
       "3                 2017.0                1.092041                    2.0   \n",
       "4                 2017.0                1.092041                    5.0   \n",
       "\n",
       "   item_block_target_mean      ...        item_cat_block_target_mean_lag_5  \\\n",
       "0                     1.0      ...                                     0.0   \n",
       "1                     0.0      ...                                     0.0   \n",
       "2                     1.0      ...                                     0.0   \n",
       "3                     1.0      ...                                     0.0   \n",
       "4                     1.0      ...                                     0.0   \n",
       "\n",
       "   item_cat_block_target_sum_lag_5  target_lag_5  \\\n",
       "0                              0.0           0.0   \n",
       "1                              0.0           0.0   \n",
       "2                              0.0           0.0   \n",
       "3                              0.0           0.0   \n",
       "4                              0.0           0.0   \n",
       "\n",
       "   item_block_target_mean_lag_12  item_block_target_sum_lag_12  \\\n",
       "0                            0.0                           0.0   \n",
       "1                            0.0                           0.0   \n",
       "2                            0.0                           0.0   \n",
       "3                            0.0                           0.0   \n",
       "4                            0.0                           0.0   \n",
       "\n",
       "   shop_block_target_mean_lag_12  shop_block_target_sum_lag_12  \\\n",
       "0                            0.0                           0.0   \n",
       "1                            0.0                           0.0   \n",
       "2                            0.0                           0.0   \n",
       "3                            0.0                           0.0   \n",
       "4                            0.0                           0.0   \n",
       "\n",
       "   item_cat_block_target_mean_lag_12  item_cat_block_target_sum_lag_12  \\\n",
       "0                                0.0                               0.0   \n",
       "1                                0.0                               0.0   \n",
       "2                                0.0                               0.0   \n",
       "3                                0.0                               0.0   \n",
       "4                                0.0                               0.0   \n",
       "\n",
       "   target_lag_12  \n",
       "0            0.0  \n",
       "1            0.0  \n",
       "2            0.0  \n",
       "3            0.0  \n",
       "4            0.0  \n",
       "\n",
       "[5 rows x 47 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10913804, 47)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# all_data.to_csv(data_path+'new_sales_lag.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = all_data[all_data['date_block_num'] >= 12] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "      <th>item_name</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>shop_block_target_sum</th>\n",
       "      <th>shop_block_target_mean</th>\n",
       "      <th>item_block_target_sum</th>\n",
       "      <th>item_block_target_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>item_cat_block_target_mean_lag_5</th>\n",
       "      <th>item_cat_block_target_sum_lag_5</th>\n",
       "      <th>target_lag_5</th>\n",
       "      <th>item_block_target_mean_lag_12</th>\n",
       "      <th>item_block_target_sum_lag_12</th>\n",
       "      <th>shop_block_target_mean_lag_12</th>\n",
       "      <th>shop_block_target_sum_lag_12</th>\n",
       "      <th>item_cat_block_target_mean_lag_12</th>\n",
       "      <th>item_cat_block_target_sum_lag_12</th>\n",
       "      <th>target_lag_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4488710</th>\n",
       "      <td>54</td>\n",
       "      <td>10297</td>\n",
       "      <td>12</td>\n",
       "      <td>4.0</td>\n",
       "      <td>THE POOR I AM 1-2 (BD)</td>\n",
       "      <td>37</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.051357</td>\n",
       "      <td>8680.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.053414</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488711</th>\n",
       "      <td>54</td>\n",
       "      <td>10296</td>\n",
       "      <td>12</td>\n",
       "      <td>3.0</td>\n",
       "      <td>THE POOR I AM 1-2 (3D BD)</td>\n",
       "      <td>38</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.023669</td>\n",
       "      <td>1384.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.019387</td>\n",
       "      <td>1630.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488712</th>\n",
       "      <td>54</td>\n",
       "      <td>10298</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>DESPICABLE ME 2</td>\n",
       "      <td>40</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>182.0</td>\n",
       "      <td>1.181818</td>\n",
       "      <td>...</td>\n",
       "      <td>1.102380</td>\n",
       "      <td>24130.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.145903</td>\n",
       "      <td>33489.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488713</th>\n",
       "      <td>54</td>\n",
       "      <td>10300</td>\n",
       "      <td>12</td>\n",
       "      <td>3.0</td>\n",
       "      <td>THE POOR I AM 2 (BD)</td>\n",
       "      <td>37</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.962963</td>\n",
       "      <td>...</td>\n",
       "      <td>1.051357</td>\n",
       "      <td>8680.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.053414</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488714</th>\n",
       "      <td>54</td>\n",
       "      <td>10284</td>\n",
       "      <td>12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Vysotsky Vladimir Thank you that the live (mp3...</td>\n",
       "      <td>57</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.005348</td>\n",
       "      <td>1316.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.003914</td>\n",
       "      <td>1539.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         shop_id  item_id  date_block_num  target  \\\n",
       "4488710       54    10297              12     4.0   \n",
       "4488711       54    10296              12     3.0   \n",
       "4488712       54    10298              12    14.0   \n",
       "4488713       54    10300              12     3.0   \n",
       "4488714       54    10284              12     1.0   \n",
       "\n",
       "                                                 item_name  item_category_id  \\\n",
       "4488710                             THE POOR I AM 1-2 (BD)                37   \n",
       "4488711                          THE POOR I AM 1-2 (3D BD)                38   \n",
       "4488712                                    DESPICABLE ME 2                40   \n",
       "4488713                               THE POOR I AM 2 (BD)                37   \n",
       "4488714  Vysotsky Vladimir Thank you that the live (mp3...                57   \n",
       "\n",
       "         shop_block_target_sum  shop_block_target_mean  item_block_target_sum  \\\n",
       "4488710                 8198.0                1.295717                   23.0   \n",
       "4488711                 8198.0                1.295717                   17.0   \n",
       "4488712                 8198.0                1.295717                  182.0   \n",
       "4488713                 8198.0                1.295717                   26.0   \n",
       "4488714                 8198.0                1.295717                    3.0   \n",
       "\n",
       "         item_block_target_mean      ...        \\\n",
       "4488710                1.000000      ...         \n",
       "4488711                1.000000      ...         \n",
       "4488712                1.181818      ...         \n",
       "4488713                0.962963      ...         \n",
       "4488714                1.000000      ...         \n",
       "\n",
       "         item_cat_block_target_mean_lag_5  item_cat_block_target_sum_lag_5  \\\n",
       "4488710                          1.051357                           8680.0   \n",
       "4488711                          1.023669                           1384.0   \n",
       "4488712                          1.102380                          24130.0   \n",
       "4488713                          1.051357                           8680.0   \n",
       "4488714                          1.005348                           1316.0   \n",
       "\n",
       "         target_lag_5  item_block_target_mean_lag_12  \\\n",
       "4488710           0.0                            0.0   \n",
       "4488711           0.0                            0.0   \n",
       "4488712           0.0                            0.0   \n",
       "4488713           0.0                            0.0   \n",
       "4488714           0.0                            0.0   \n",
       "\n",
       "         item_block_target_sum_lag_12  shop_block_target_mean_lag_12  \\\n",
       "4488710                           0.0                       1.247143   \n",
       "4488711                           0.0                       1.247143   \n",
       "4488712                           0.0                       1.247143   \n",
       "4488713                           0.0                       1.247143   \n",
       "4488714                           0.0                       1.247143   \n",
       "\n",
       "         shop_block_target_sum_lag_12  item_cat_block_target_mean_lag_12  \\\n",
       "4488710                        9386.0                           1.053414   \n",
       "4488711                        9386.0                           1.019387   \n",
       "4488712                        9386.0                           1.145903   \n",
       "4488713                        9386.0                           1.053414   \n",
       "4488714                        9386.0                           1.003914   \n",
       "\n",
       "         item_cat_block_target_sum_lag_12  target_lag_12  \n",
       "4488710                            6094.0            0.0  \n",
       "4488711                            1630.0            0.0  \n",
       "4488712                           33489.0            0.0  \n",
       "4488713                            6094.0            0.0  \n",
       "4488714                            1539.0            0.0  \n",
       "\n",
       "[5 rows x 47 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# all_data.to_csv(data_path+'new_sales_lag_after12.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add boolean holiday features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/quantran/anaconda3/envs/xgb/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "all_data['December'] = all_data.date_block_num.apply(lambda x: 1 if x ==23 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/quantran/anaconda3/envs/xgb/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "/home/quantran/anaconda3/envs/xgb/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "/home/quantran/anaconda3/envs/xgb/lib/python3.6/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "/home/quantran/anaconda3/envs/xgb/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "all_data['Newyear_Xmas'] = all_data.date_block_num.apply(lambda x: 1 if x in [12,24] else 0)\n",
    "all_data['Valentine_MenDay'] = all_data.date_block_num.apply(lambda x: 1 if x in [13,25] else 0)\n",
    "all_data['WomenDay'] = all_data.date_block_num.apply(lambda x: 1 if x in [14,26] else 0)\n",
    "all_data['Easter_Labor'] = all_data.date_block_num.apply(lambda x: 1 if x in [15,27] else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data.to_pickle(data_path+'new_sales_lag_after12.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = pd.read_pickle(data_path+'new_sales_lag_after12.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "      <th>item_name</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>shop_block_target_sum</th>\n",
       "      <th>shop_block_target_mean</th>\n",
       "      <th>item_block_target_sum</th>\n",
       "      <th>item_block_target_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>shop_block_target_mean_lag_12</th>\n",
       "      <th>shop_block_target_sum_lag_12</th>\n",
       "      <th>item_cat_block_target_mean_lag_12</th>\n",
       "      <th>item_cat_block_target_sum_lag_12</th>\n",
       "      <th>target_lag_12</th>\n",
       "      <th>December</th>\n",
       "      <th>Newyear_Xmas</th>\n",
       "      <th>Valentine_MenDay</th>\n",
       "      <th>WomenDay</th>\n",
       "      <th>Easter_Labor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4488710</th>\n",
       "      <td>54</td>\n",
       "      <td>10297</td>\n",
       "      <td>12</td>\n",
       "      <td>4.0</td>\n",
       "      <td>THE POOR I AM 1-2 (BD)</td>\n",
       "      <td>37</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.053414</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488711</th>\n",
       "      <td>54</td>\n",
       "      <td>10296</td>\n",
       "      <td>12</td>\n",
       "      <td>3.0</td>\n",
       "      <td>THE POOR I AM 1-2 (3D BD)</td>\n",
       "      <td>38</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.019387</td>\n",
       "      <td>1630.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488712</th>\n",
       "      <td>54</td>\n",
       "      <td>10298</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>DESPICABLE ME 2</td>\n",
       "      <td>40</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>182.0</td>\n",
       "      <td>1.181818</td>\n",
       "      <td>...</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.145903</td>\n",
       "      <td>33489.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488713</th>\n",
       "      <td>54</td>\n",
       "      <td>10300</td>\n",
       "      <td>12</td>\n",
       "      <td>3.0</td>\n",
       "      <td>THE POOR I AM 2 (BD)</td>\n",
       "      <td>37</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.962963</td>\n",
       "      <td>...</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.053414</td>\n",
       "      <td>6094.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4488714</th>\n",
       "      <td>54</td>\n",
       "      <td>10284</td>\n",
       "      <td>12</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Vysotsky Vladimir Thank you that the live (mp3...</td>\n",
       "      <td>57</td>\n",
       "      <td>8198.0</td>\n",
       "      <td>1.295717</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.247143</td>\n",
       "      <td>9386.0</td>\n",
       "      <td>1.003914</td>\n",
       "      <td>1539.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 52 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         shop_id  item_id  date_block_num  target  \\\n",
       "4488710       54    10297              12     4.0   \n",
       "4488711       54    10296              12     3.0   \n",
       "4488712       54    10298              12    14.0   \n",
       "4488713       54    10300              12     3.0   \n",
       "4488714       54    10284              12     1.0   \n",
       "\n",
       "                                                 item_name  item_category_id  \\\n",
       "4488710                             THE POOR I AM 1-2 (BD)                37   \n",
       "4488711                          THE POOR I AM 1-2 (3D BD)                38   \n",
       "4488712                                    DESPICABLE ME 2                40   \n",
       "4488713                               THE POOR I AM 2 (BD)                37   \n",
       "4488714  Vysotsky Vladimir Thank you that the live (mp3...                57   \n",
       "\n",
       "         shop_block_target_sum  shop_block_target_mean  item_block_target_sum  \\\n",
       "4488710                 8198.0                1.295717                   23.0   \n",
       "4488711                 8198.0                1.295717                   17.0   \n",
       "4488712                 8198.0                1.295717                  182.0   \n",
       "4488713                 8198.0                1.295717                   26.0   \n",
       "4488714                 8198.0                1.295717                    3.0   \n",
       "\n",
       "         item_block_target_mean      ...       shop_block_target_mean_lag_12  \\\n",
       "4488710                1.000000      ...                            1.247143   \n",
       "4488711                1.000000      ...                            1.247143   \n",
       "4488712                1.181818      ...                            1.247143   \n",
       "4488713                0.962963      ...                            1.247143   \n",
       "4488714                1.000000      ...                            1.247143   \n",
       "\n",
       "         shop_block_target_sum_lag_12  item_cat_block_target_mean_lag_12  \\\n",
       "4488710                        9386.0                           1.053414   \n",
       "4488711                        9386.0                           1.019387   \n",
       "4488712                        9386.0                           1.145903   \n",
       "4488713                        9386.0                           1.053414   \n",
       "4488714                        9386.0                           1.003914   \n",
       "\n",
       "         item_cat_block_target_sum_lag_12  target_lag_12  December  \\\n",
       "4488710                            6094.0            0.0         0   \n",
       "4488711                            1630.0            0.0         0   \n",
       "4488712                           33489.0            0.0         0   \n",
       "4488713                            6094.0            0.0         0   \n",
       "4488714                            1539.0            0.0         0   \n",
       "\n",
       "         Newyear_Xmas  Valentine_MenDay  WomenDay  Easter_Labor  \n",
       "4488710             1                 0         0             0  \n",
       "4488711             1                 0         0             0  \n",
       "4488712             1                 0         0             0  \n",
       "4488713             1                 0         0             0  \n",
       "4488714             1                 0         0             0  \n",
       "\n",
       "[5 rows x 52 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6425094, 52)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add months feature (For Neural network only. This is not useful for Gradient Boosting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # convert dbn to month\n",
    "# months = [i for i in range(12)]\n",
    "# month_list = [[j for j in range(i,34,12) if j >=12 ] for i in months]\n",
    "# print(month_list)\n",
    "\n",
    "\n",
    "# def get_month_from_dbn(date_block_num):\n",
    "#     result=[]\n",
    "#     for i,dbn in enumerate(date_block_num):\n",
    "#         for j,m in enumerate(month_list):\n",
    "#             if dbn in m:\n",
    "#                 result.append(j+1)\n",
    "#                 break\n",
    "#     result = np.array(result)\n",
    "#     return result\n",
    "\n",
    "# all_data = pd.read_pickle(data_path+'new_sales_lag_after12.pickle')\n",
    "# all_data['month'] = get_month_from_dbn(all_data.date_block_num)\n",
    "\n",
    "\n",
    "# all_data.to_pickle(data_path+'new_sales_lag_after12_month.pickle')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}