competitive-data-science-predict-future-sales/predict_future_sales.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "from fastai.imports import *\n",
    "from fastai.transforms import *\n",
    "from fastai.conv_learner import *\n",
    "from fastai.model import *\n",
    "from fastai.dataset import *\n",
    "from fastai.sgdr import *\n",
    "from fastai.plots import *\n",
    "from fastai.dataset import *\n",
    "from fastai.structured import *\n",
    "from fastai.column_data import *\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "\n",
    "from math import ceil\n",
    "\n",
    "from itertools import product\n",
    "import gc\n",
    "from dateutil import parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = \"data/\"\n",
    "sz=224\n",
    "arch=resnext101_64\n",
    "batch_size=5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_vars = [\n",
    "    'date_block_num',\n",
    "    'item_id',\n",
    "    'month',\n",
    "    'shop_id',\n",
    "    'year',\n",
    "    'item_category_id',\n",
    "    'item_category_name'\n",
    "]\n",
    "\n",
    "dep = 'target'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_categories = pd.read_csv(f'{PATH}item_categories.csv')\n",
    "items = pd.read_csv(f'{PATH}items.csv')\n",
    "shops = pd.read_csv(f'{PATH}shops.csv')\n",
    "sales = pd.read_csv(f'{PATH}sales_train.csv')\n",
    "sales_test = pd.read_csv(f'{PATH}test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_category_name</th>\n",
       "      <th>item_category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>PC - Гарнитуры/Наушники</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Аксессуары - PS2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Аксессуары - PS3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Аксессуары - PS4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Аксессуары - PSP</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        item_category_name  item_category_id\n",
       "0  PC - Гарнитуры/Наушники                 0\n",
       "1         Аксессуары - PS2                 1\n",
       "2         Аксессуары - PS3                 2\n",
       "3         Аксессуары - PS4                 3\n",
       "4         Аксессуары - PSP                 4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_name</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>! ВО ВЛАСТИ НАВАЖДЕНИЯ (ПЛАСТ.)         D</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>!ABBYY FineReader 12 Professional Edition Full...</td>\n",
       "      <td>1</td>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>***В ЛУЧАХ СЛАВЫ   (UNV)                    D</td>\n",
       "      <td>2</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>***ГОЛУБАЯ ВОЛНА  (Univ)                      D</td>\n",
       "      <td>3</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>***КОРОБКА (СТЕКЛО)                       D</td>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           item_name  item_id  \\\n",
       "0          ! ВО ВЛАСТИ НАВАЖДЕНИЯ (ПЛАСТ.)         D        0   \n",
       "1  !ABBYY FineReader 12 Professional Edition Full...        1   \n",
       "2      ***В ЛУЧАХ СЛАВЫ   (UNV)                    D        2   \n",
       "3    ***ГОЛУБАЯ ВОЛНА  (Univ)                      D        3   \n",
       "4        ***КОРОБКА (СТЕКЛО)                       D        4   \n",
       "\n",
       "   item_category_id  \n",
       "0                40  \n",
       "1                76  \n",
       "2                40  \n",
       "3                40  \n",
       "4                40  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_name</th>\n",
       "      <th>shop_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>!Якутск Орджоникидзе, 56 фран</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>!Якутск ТЦ \"Центральный\" фран</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Адыгея ТЦ \"Мега\"</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Балашиха ТРК \"Октябрь-Киномир\"</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Волжский ТЦ \"Волга Молл\"</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        shop_name  shop_id\n",
       "0   !Якутск Орджоникидзе, 56 фран        0\n",
       "1   !Якутск ТЦ \"Центральный\" фран        1\n",
       "2                Адыгея ТЦ \"Мега\"        2\n",
       "3  Балашиха ТРК \"Октябрь-Киномир\"        3\n",
       "4        Волжский ТЦ \"Волга Молл\"        4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_price</th>\n",
       "      <th>item_cnt_day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>22154</td>\n",
       "      <td>999.00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2552</td>\n",
       "      <td>899.00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2552</td>\n",
       "      <td>899.00</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>06.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2554</td>\n",
       "      <td>1709.05</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2555</td>\n",
       "      <td>1099.00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date  date_block_num  shop_id  item_id  item_price  item_cnt_day\n",
       "0  02.01.2013               0       59    22154      999.00           1.0\n",
       "1  03.01.2013               0       25     2552      899.00           1.0\n",
       "2  05.01.2013               0       25     2552      899.00          -1.0\n",
       "3  06.01.2013               0       25     2554     1709.05           1.0\n",
       "4  15.01.2013               0       25     2555     1099.00           1.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5268</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  shop_id  item_id\n",
       "0   0        5     5037\n",
       "1   1        5     5320\n",
       "2   2        5     5233\n",
       "3   3        5     5232\n",
       "4   4        5     5268"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(item_categories.head())\n",
    "display(items.head())\n",
    "display(shops.head())\n",
    "display(sales.head())\n",
    "display(sales_test.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_cols  = ['shop_id', 'item_id', 'date_block_num']\n",
    "\n",
    "# For every month we create a grid from all shops/items combinations from that month\n",
    "grid = [] \n",
    "for block_num in sales['date_block_num'].unique():\n",
    "    cur_shops = sales[sales['date_block_num']==block_num]['shop_id'].unique()\n",
    "    cur_items = sales[sales['date_block_num']==block_num]['item_id'].unique()\n",
    "    grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])),dtype='int32'))\n",
    "\n",
    "#turn the grid into pandas dataframe\n",
    "grid = pd.DataFrame(np.vstack(grid), columns = index_cols,dtype=np.int32)\n",
    "\n",
    "#get aggregated values for (shop_id, item_id, month)\n",
    "gb = sales.groupby(index_cols,as_index=False).agg({'item_cnt_day':{'target':'sum'},'item_price': np.mean})\n",
    "\n",
    "#fix column names\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]\n",
    "#join aggregated data to the grid\n",
    "# all_data = pd.merge(grid,gb,how='left',on=index_cols).fillna(0)\n",
    "all_data = pd.merge(grid,gb,how='left',on=index_cols)\n",
    "#sort the data\n",
    "all_data.sort_values(['date_block_num','shop_id','item_id'],inplace=True)\n",
    "\n",
    "# del grid,cur_shops,cur_items,gb\n",
    "# del sales\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def downcast_dtypes(df):\n",
    "    '''\n",
    "        Changes column types in the dataframe: \n",
    "                \n",
    "                `float64` type to `float32`\n",
    "                `int64`   type to `int32`\n",
    "    '''\n",
    "    \n",
    "    # Select columns to downcast\n",
    "    float_cols = [c for c in df if df[c].dtype == \"float64\"]\n",
    "    int_cols =   [c for c in df if df[c].dtype == \"int64\"]\n",
    "    \n",
    "    # Downcast\n",
    "    df[float_cols] = df[float_cols].astype(np.float32)\n",
    "    df[int_cols]   = df[int_cols].astype(np.int32)\n",
    "    \n",
    "    return df\n",
    "\n",
    "all_data.fillna(0.0,inplace=True)\n",
    "all_data['target'] = all_data['target'].clip(0.0,40.0)\n",
    "all_data['month'] = all_data['date_block_num'] % 12\n",
    "all_data['year'] = all_data['date_block_num'] // 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sales_test['date_block_num'] = 34\n",
    "sales_test['month'] = sales_test['date_block_num']%12\n",
    "sales_test['year'] = sales_test['date_block_num']//12\n",
    "\n",
    "all_data = pd.concat([all_data,sales_test])\n",
    "\n",
    "# del sales_test\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = all_data.merge(items, on=\"item_id\")\n",
    "all_data.drop('item_name',axis = 1,inplace=True)\n",
    "all_data.sort_values(['date_block_num','shop_id','item_id'],inplace=True)\n",
    "\n",
    "all_data = downcast_dtypes(all_data)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lag_variable = ['target']\n",
    "lags = [1 ,2 ,3 ,4, 5, 12]\n",
    "for lag in lags:\n",
    "    sales_new_df = all_data[['date_block_num','shop_id','item_id']+lag_variable].copy()\n",
    "    sales_new_df.date_block_num+=lag\n",
    "    sales_new_df.columns = ['date_block_num','shop_id','item_id']+ [lag_feat+'_lag_'+str(lag) for lag_feat in lag_variable]\n",
    "    all_data = pd.merge(all_data, sales_new_df,on=['date_block_num','shop_id','item_id'] ,how='left')\n",
    "    del sales_new_df\n",
    "    gc.collect()\n",
    "\n",
    "for feat in all_data.columns:\n",
    "    if 'target' in feat:\n",
    "        all_data[feat]=all_data[feat].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "all_data['avg_sales_per_shop_id'] = 0.0\n",
    "all_data['avg_sales_per_item_id'] = 0.0\n",
    "all_data['avg_sales_per_item_cat_id'] = 0.0\n",
    "\n",
    "avg_shop_sales_per_month = all_data.groupby(['shop_id','date_block_num'],as_index=False).agg({'target':'mean'})\n",
    "avg_item_sales_per_month = all_data.groupby(['item_id','date_block_num'],as_index=False).agg({'target':'mean'})\n",
    "avg_item_cat_sales_per_month = all_data.groupby(['item_category_id','date_block_num', ],as_index=False).agg({'target':'mean'})\n",
    "\n",
    "#lets calculate  6 lags\n",
    "lag_variable = ['item_id','shop_id','item_category_id']\n",
    "lags = [1 ,2 ,3 ,4, 5, 12]\n",
    "curr = 0\n",
    "for lag in lags:\n",
    "    diff = lag - curr\n",
    "    curr = lag\n",
    "    avg_shop_sales_per_month.date_block_num+=diff\n",
    "    avg_item_cat_sales_per_month.date_block_num+=diff\n",
    "    avg_item_sales_per_month.date_block_num+=diff\n",
    "    \n",
    "    avg_shop_sales_per_month.columns = ['shop_id','date_block_num']+ [lag_variable[1]+'_lag_'+str(lag)]\n",
    "    avg_item_sales_per_month.columns = ['item_id','date_block_num']+ [lag_variable[0]+'_lag_'+str(lag)]\n",
    "    avg_item_cat_sales_per_month.columns = ['item_category_id','date_block_num']+ [lag_variable[2]+'_lag_'+str(lag)]\n",
    "    \n",
    "    all_data = pd.merge(all_data, avg_shop_sales_per_month,on=['date_block_num','shop_id'] ,how='left')\n",
    "    all_data = pd.merge(all_data, avg_item_sales_per_month,on=['date_block_num','item_id'] ,how='left')\n",
    "    all_data = pd.merge(all_data, avg_item_cat_sales_per_month,on=['date_block_num','item_category_id'] ,how='left')\n",
    "    \n",
    "    gc.collect()\n",
    "del avg_item_cat_sales_per_month,avg_item_sales_per_month,avg_shop_sales_per_month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.drop(columns=['mean'],inplace=True)\n",
    "all_data.sort_values(['date_block_num','shop_id','item_id'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.fillna(0,inplace=True)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.to_pickle(PATH + 'all_data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = pd.read_pickle(PATH + 'all_data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding category names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_cat = list(item_categories.item_category_name)\n",
    "for ind in range(0,1):\n",
    "    l_cat[ind] = 'PC Headsets / Headphones'\n",
    "for ind in range(1,8):\n",
    "    l_cat[ind] = 'Access'\n",
    "l_cat[8] = 'Tickets (figure)'\n",
    "l_cat[9] = 'Delivery of goods'\n",
    "for ind in range(10,18):\n",
    "    l_cat[ind] = 'Consoles'\n",
    "for ind in range(18,25):\n",
    "    l_cat[ind] = 'Consoles Games'\n",
    "l_cat[25] = 'Accessories for games'\n",
    "for ind in range(26,28):\n",
    "    l_cat[ind] = 'phone games'\n",
    "for ind in range(28,32):\n",
    "    l_cat[ind] = 'CD games'\n",
    "for ind in range(32,37):\n",
    "    l_cat[ind] = 'Card'\n",
    "for ind in range(37,43):\n",
    "    l_cat[ind] = 'Movie'\n",
    "for ind in range(43,55):\n",
    "    l_cat[ind] = 'Books'\n",
    "for ind in range(55,61):\n",
    "    l_cat[ind] = 'Music'\n",
    "for ind in range(61,73):\n",
    "    l_cat[ind] = 'Gifts'\n",
    "for ind in range(73,79):\n",
    "    l_cat[ind] = 'Soft'\n",
    "for ind in range(79,81):\n",
    "    l_cat[ind] = 'Office'\n",
    "for ind in range(81,83):\n",
    "    l_cat[ind] = 'Clean'\n",
    "l_cat[83] = 'Elements of a food'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data['item_category_name'] = all_data.item_category_id.apply(\n",
    "    lambda x: l_cat[x]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_list = list(all_data.item_category_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cat in cat_vars:\n",
    "    all_data[cat] = all_data[cat].astype('category')\n",
    "all_data['date_block_num'] = all_data['date_block_num'].cat.as_ordered()\n",
    "all_data['month'] = all_data['month'].cat.as_ordered()\n",
    "all_data['year'] = all_data['year'].cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train = all_data[(all_data['date_block_num']>12)&(all_data['date_block_num']<33)].drop(['target','ID'], axis = 1)\n",
    "# y_train = all_data[(all_data['date_block_num']>12)&(all_data['date_block_num']<33)]['target']\n",
    "X_train = all_data[all_data['date_block_num']<33].drop(['target','ID'], axis = 1)\n",
    "y_train = all_data[all_data['date_block_num']<33]['target']\n",
    "X_val =  all_data[all_data['date_block_num']==33].drop(['target','ID'], axis = 1)\n",
    "y_val = all_data[all_data['date_block_num'] == 33]['target']\n",
    "X_test = all_data[(all_data['date_block_num']==34)].drop(['target'],axis=1)\n",
    "del all_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = X_test.drop('ID', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.to_pickle(PATH + 'X_train.pkl')\n",
    "y_train.to_pickle(PATH + 'y_train.pkl')\n",
    "X_val.to_pickle(PATH + 'X_val.pkl')\n",
    "y_val.to_pickle(PATH + 'y_val.pkl')\n",
    "X_test.to_pickle(PATH + 'X_test.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.head().columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.read_pickle(PATH + 'X_train.pkl')\n",
    "y_train = pd.read_pickle(PATH + 'y_train.pkl')\n",
    "X_val = pd.read_pickle(PATH + 'X_val.pkl')\n",
    "y_val = pd.read_pickle(PATH + 'y_val.pkl')\n",
    "X_test = pd.read_pickle(PATH + 'X_test.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Buffer dtype mismatch, expected 'Python object' but got 'long'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;31mValueError\u001b[0m: Buffer dtype mismatch, expected 'Python object' but got 'long'"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: 'pandas._libs.lib.is_bool_array'\n",
      "ValueError: Buffer dtype mismatch, expected 'Python object' but got 'long'\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Buffer dtype mismatch, expected 'Python object' but got 'long'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;31mValueError\u001b[0m: Buffer dtype mismatch, expected 'Python object' but got 'long'"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: 'pandas._libs.lib.is_bool_array'\n",
      "ValueError: Buffer dtype mismatch, expected 'Python object' but got 'long'\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Buffer dtype mismatch, expected 'Python object' but got 'long'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;31mValueError\u001b[0m: Buffer dtype mismatch, expected 'Python object' but got 'long'"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: 'pandas._libs.lib.is_bool_array'\n",
      "ValueError: Buffer dtype mismatch, expected 'Python object' but got 'long'\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Buffer dtype mismatch, expected 'Python object' but got 'long'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;31mValueError\u001b[0m: Buffer dtype mismatch, expected 'Python object' but got 'long'"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: 'pandas._libs.lib.is_bool_array'\n",
      "ValueError: Buffer dtype mismatch, expected 'Python object' but got 'long'\n"
     ]
    }
   ],
   "source": [
    "orig_len = len(X_test)\n",
    "X_test = X_test.merge(sales_test, on=['item_id', 'shop_id'], how='inner')\n",
    "X_test = X_test.set_index('ID').sort_index().reset_index().drop('ID', axis=1)\n",
    "assert len(X_test) == orig_len  # sanity check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(y_pred, targ):\n",
    "    return np.sqrt(((targ - y_pred)**2).mean())\n",
    "\n",
    "def rmse_expm1(y_pred, targ):\n",
    "    return rmse(y_pred=np.expm1(y_pred), targ=np.expm1(targ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## _Deep Learning Approach_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train['target'] = y_train\n",
    "X_val['target'] = y_val\n",
    "X_test['target'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['date_block_num', 'item_id', 'month', 'shop_id', 'year',\n",
       "       'item_category_id', 'target_lag_1', 'target_lag_2', 'target_lag_3',\n",
       "       'target_lag_4', 'target_lag_5', 'target_lag_12',\n",
       "       'avg_sales_per_shop_id', 'avg_sales_per_item_id',\n",
       "       'avg_sales_per_item_cat_id', 'shop_id_lag_1', 'item_id_lag_1',\n",
       "       'item_category_id_lag_1', 'shop_id_lag_2', 'item_id_lag_2',\n",
       "       'item_category_id_lag_2', 'shop_id_lag_3', 'item_id_lag_3',\n",
       "       'item_category_id_lag_3', 'shop_id_lag_4', 'item_id_lag_4',\n",
       "       'item_category_id_lag_4', 'shop_id_lag_5', 'item_id_lag_5',\n",
       "       'item_category_id_lag_5', 'shop_id_lag_12', 'item_id_lag_12',\n",
       "       'item_category_id_lag_12', 'item_category_name', 'target'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>item_id</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>month</th>\n",
       "      <th>year</th>\n",
       "      <th>target</th>\n",
       "      <th>target_lag_1</th>\n",
       "      <th>target_lag_2</th>\n",
       "      <th>target_lag_3</th>\n",
       "      <th>target_lag_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5037</td>\n",
       "      <td>5</td>\n",
       "      <td>34</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5320</td>\n",
       "      <td>5</td>\n",
       "      <td>34</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5233</td>\n",
       "      <td>5</td>\n",
       "      <td>34</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5232</td>\n",
       "      <td>5</td>\n",
       "      <td>34</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5268</td>\n",
       "      <td>5</td>\n",
       "      <td>34</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   item_id  shop_id date_block_num month year  target  target_lag_1  \\\n",
       "0     5037        5             34    10    2       0           0.0   \n",
       "1     5320        5             34    10    2       0           0.0   \n",
       "2     5233        5             34    10    2       0           1.0   \n",
       "3     5232        5             34    10    2       0           0.0   \n",
       "4     5268        5             34    10    2       0           0.0   \n",
       "\n",
       "   target_lag_2  target_lag_3  target_lag_12  \n",
       "0           1.0           3.0            1.0  \n",
       "1           0.0           0.0            0.0  \n",
       "2           3.0           1.0            0.0  \n",
       "3           0.0           1.0            0.0  \n",
       "4           0.0           0.0            0.0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()[['item_id', 'shop_id', 'date_block_num', 'month', 'year', 'target', 'target_lag_1', 'target_lag_2', 'target_lag_3', 'target_lag_12']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For faster prototyping\n",
    "X_train = X_train[X_train['date_block_num'] > 12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "trn_df, trn_y, nas, mapper = proc_df(X_train, 'target', do_scale=True)\n",
    "val_df, val_y, nas, mapper = proc_df(\n",
    "    X_val,\n",
    "    dep,\n",
    "    do_scale=True,\n",
    "    mapper=mapper,\n",
    ")\n",
    "test_df, _, nas, mapper = proc_df(\n",
    "    X_test,\n",
    "    dep,\n",
    "    do_scale=True,\n",
    "    mapper=mapper,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reduces volatility in the loss function\n",
    "trn_y = np.log1p(trn_y)\n",
    "val_y = np.log1p(val_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "trn_df.to_pickle(PATH + 'trn_df.pkl')\n",
    "val_df.to_pickle(PATH + 'val_df.pkl')\n",
    "test_df.to_pickle(PATH + 'test_df.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>item_id</th>\n",
       "      <th>month</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>year</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>target_lag_1</th>\n",
       "      <th>target_lag_2</th>\n",
       "      <th>target_lag_3</th>\n",
       "      <th>target_lag_4</th>\n",
       "      <th>...</th>\n",
       "      <th>shop_id_lag_4</th>\n",
       "      <th>item_id_lag_4</th>\n",
       "      <th>item_category_id_lag_4</th>\n",
       "      <th>shop_id_lag_5</th>\n",
       "      <th>item_id_lag_5</th>\n",
       "      <th>item_category_id_lag_5</th>\n",
       "      <th>shop_id_lag_12</th>\n",
       "      <th>item_id_lag_12</th>\n",
       "      <th>item_category_id_lag_12</th>\n",
       "      <th>item_category_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4836102</th>\n",
       "      <td>14</td>\n",
       "      <td>31</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>41</td>\n",
       "      <td>-0.207089</td>\n",
       "      <td>-0.206903</td>\n",
       "      <td>-0.203934</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.852644</td>\n",
       "      <td>0.180835</td>\n",
       "      <td>-0.116117</td>\n",
       "      <td>-0.785319</td>\n",
       "      <td>-0.023856</td>\n",
       "      <td>-0.136686</td>\n",
       "      <td>-0.974826</td>\n",
       "      <td>14.200087</td>\n",
       "      <td>-0.081953</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4836103</th>\n",
       "      <td>14</td>\n",
       "      <td>32</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>-0.207089</td>\n",
       "      <td>-0.206903</td>\n",
       "      <td>-0.203934</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.852644</td>\n",
       "      <td>0.200010</td>\n",
       "      <td>-0.228357</td>\n",
       "      <td>-0.785319</td>\n",
       "      <td>0.091835</td>\n",
       "      <td>-0.188312</td>\n",
       "      <td>-0.974826</td>\n",
       "      <td>11.874555</td>\n",
       "      <td>-0.184191</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4836104</th>\n",
       "      <td>14</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>41</td>\n",
       "      <td>0.490813</td>\n",
       "      <td>-0.206903</td>\n",
       "      <td>-0.203934</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.852644</td>\n",
       "      <td>0.832791</td>\n",
       "      <td>-0.116117</td>\n",
       "      <td>-0.785319</td>\n",
       "      <td>0.863108</td>\n",
       "      <td>-0.136686</td>\n",
       "      <td>-0.974826</td>\n",
       "      <td>3.853442</td>\n",
       "      <td>-0.081953</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4836105</th>\n",
       "      <td>14</td>\n",
       "      <td>34</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>0.490813</td>\n",
       "      <td>0.466910</td>\n",
       "      <td>1.115581</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.852644</td>\n",
       "      <td>0.353411</td>\n",
       "      <td>-0.228357</td>\n",
       "      <td>-0.785319</td>\n",
       "      <td>0.554599</td>\n",
       "      <td>-0.188312</td>\n",
       "      <td>-0.974826</td>\n",
       "      <td>0.522808</td>\n",
       "      <td>-0.184191</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4836106</th>\n",
       "      <td>14</td>\n",
       "      <td>35</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>41</td>\n",
       "      <td>-0.207089</td>\n",
       "      <td>-0.206903</td>\n",
       "      <td>-0.203934</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.852644</td>\n",
       "      <td>-0.241019</td>\n",
       "      <td>-0.116117</td>\n",
       "      <td>-0.785319</td>\n",
       "      <td>-0.100983</td>\n",
       "      <td>-0.136686</td>\n",
       "      <td>-0.974826</td>\n",
       "      <td>-0.029013</td>\n",
       "      <td>-0.081953</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         date_block_num  item_id  month  shop_id  year  item_category_id  \\\n",
       "4836102              14       31      2        3     2                41   \n",
       "4836103              14       32      2        3     2                38   \n",
       "4836104              14       33      2        3     2                41   \n",
       "4836105              14       34      2        3     2                38   \n",
       "4836106              14       35      2        3     2                41   \n",
       "\n",
       "         target_lag_1  target_lag_2  target_lag_3  target_lag_4  \\\n",
       "4836102     -0.207089     -0.206903     -0.203934     -0.201037   \n",
       "4836103     -0.207089     -0.206903     -0.203934     -0.201037   \n",
       "4836104      0.490813     -0.206903     -0.203934     -0.201037   \n",
       "4836105      0.490813      0.466910      1.115581     -0.201037   \n",
       "4836106     -0.207089     -0.206903     -0.203934     -0.201037   \n",
       "\n",
       "                ...          shop_id_lag_4  item_id_lag_4  \\\n",
       "4836102         ...              -0.852644       0.180835   \n",
       "4836103         ...              -0.852644       0.200010   \n",
       "4836104         ...              -0.852644       0.832791   \n",
       "4836105         ...              -0.852644       0.353411   \n",
       "4836106         ...              -0.852644      -0.241019   \n",
       "\n",
       "         item_category_id_lag_4  shop_id_lag_5  item_id_lag_5  \\\n",
       "4836102               -0.116117      -0.785319      -0.023856   \n",
       "4836103               -0.228357      -0.785319       0.091835   \n",
       "4836104               -0.116117      -0.785319       0.863108   \n",
       "4836105               -0.228357      -0.785319       0.554599   \n",
       "4836106               -0.116117      -0.785319      -0.100983   \n",
       "\n",
       "         item_category_id_lag_5  shop_id_lag_12  item_id_lag_12  \\\n",
       "4836102               -0.136686       -0.974826       14.200087   \n",
       "4836103               -0.188312       -0.974826       11.874555   \n",
       "4836104               -0.136686       -0.974826        3.853442   \n",
       "4836105               -0.188312       -0.974826        0.522808   \n",
       "4836106               -0.136686       -0.974826       -0.029013   \n",
       "\n",
       "         item_category_id_lag_12  item_category_name  \n",
       "4836102                -0.081953                  12  \n",
       "4836103                -0.184191                  12  \n",
       "4836104                -0.081953                  12  \n",
       "4836105                -0.184191                  12  \n",
       "4836106                -0.081953                  12  \n",
       "\n",
       "[5 rows x 34 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>item_id</th>\n",
       "      <th>month</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>year</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>target_lag_1</th>\n",
       "      <th>target_lag_2</th>\n",
       "      <th>target_lag_3</th>\n",
       "      <th>target_lag_4</th>\n",
       "      <th>...</th>\n",
       "      <th>shop_id_lag_4</th>\n",
       "      <th>item_id_lag_4</th>\n",
       "      <th>item_category_id_lag_4</th>\n",
       "      <th>shop_id_lag_5</th>\n",
       "      <th>item_id_lag_5</th>\n",
       "      <th>item_category_id_lag_5</th>\n",
       "      <th>shop_id_lag_12</th>\n",
       "      <th>item_id_lag_12</th>\n",
       "      <th>item_category_id_lag_12</th>\n",
       "      <th>item_category_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35</td>\n",
       "      <td>5037</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>-0.207089</td>\n",
       "      <td>0.466910</td>\n",
       "      <td>1.775339</td>\n",
       "      <td>0.450108</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.495571</td>\n",
       "      <td>0.828332</td>\n",
       "      <td>0.178719</td>\n",
       "      <td>-0.502948</td>\n",
       "      <td>1.844238</td>\n",
       "      <td>0.234431</td>\n",
       "      <td>-0.30674</td>\n",
       "      <td>0.932732</td>\n",
       "      <td>0.359401</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35</td>\n",
       "      <td>5320</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>56</td>\n",
       "      <td>-0.207089</td>\n",
       "      <td>-0.206903</td>\n",
       "      <td>-0.203934</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.495571</td>\n",
       "      <td>-0.279370</td>\n",
       "      <td>-0.161931</td>\n",
       "      <td>-0.502948</td>\n",
       "      <td>-0.274519</td>\n",
       "      <td>-0.179152</td>\n",
       "      <td>-0.30674</td>\n",
       "      <td>-0.245800</td>\n",
       "      <td>-0.121941</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35</td>\n",
       "      <td>5233</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>0.490813</td>\n",
       "      <td>1.814536</td>\n",
       "      <td>0.455823</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.495571</td>\n",
       "      <td>0.479611</td>\n",
       "      <td>0.178719</td>\n",
       "      <td>-0.502948</td>\n",
       "      <td>2.126740</td>\n",
       "      <td>0.234431</td>\n",
       "      <td>-0.30674</td>\n",
       "      <td>-0.245800</td>\n",
       "      <td>0.359401</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35</td>\n",
       "      <td>5232</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>-0.207089</td>\n",
       "      <td>-0.206903</td>\n",
       "      <td>0.455823</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.495571</td>\n",
       "      <td>-0.279370</td>\n",
       "      <td>0.136655</td>\n",
       "      <td>-0.502948</td>\n",
       "      <td>-0.274519</td>\n",
       "      <td>0.188583</td>\n",
       "      <td>-0.30674</td>\n",
       "      <td>-0.245800</td>\n",
       "      <td>0.418859</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35</td>\n",
       "      <td>5268</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>-0.207089</td>\n",
       "      <td>-0.206903</td>\n",
       "      <td>-0.203934</td>\n",
       "      <td>-0.201037</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.495571</td>\n",
       "      <td>-0.279370</td>\n",
       "      <td>0.884385</td>\n",
       "      <td>-0.502948</td>\n",
       "      <td>-0.274519</td>\n",
       "      <td>1.045059</td>\n",
       "      <td>-0.30674</td>\n",
       "      <td>-0.245800</td>\n",
       "      <td>3.306020</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   date_block_num  item_id  month  shop_id  year  item_category_id  \\\n",
       "0              35     5037     11        5     3                20   \n",
       "1              35     5320     11        5     3                56   \n",
       "2              35     5233     11        5     3                20   \n",
       "3              35     5232     11        5     3                24   \n",
       "4              35     5268     11        5     3                21   \n",
       "\n",
       "   target_lag_1  target_lag_2  target_lag_3  target_lag_4         ...          \\\n",
       "0     -0.207089      0.466910      1.775339      0.450108         ...           \n",
       "1     -0.207089     -0.206903     -0.203934     -0.201037         ...           \n",
       "2      0.490813      1.814536      0.455823     -0.201037         ...           \n",
       "3     -0.207089     -0.206903      0.455823     -0.201037         ...           \n",
       "4     -0.207089     -0.206903     -0.203934     -0.201037         ...           \n",
       "\n",
       "   shop_id_lag_4  item_id_lag_4  item_category_id_lag_4  shop_id_lag_5  \\\n",
       "0      -0.495571       0.828332                0.178719      -0.502948   \n",
       "1      -0.495571      -0.279370               -0.161931      -0.502948   \n",
       "2      -0.495571       0.479611                0.178719      -0.502948   \n",
       "3      -0.495571      -0.279370                0.136655      -0.502948   \n",
       "4      -0.495571      -0.279370                0.884385      -0.502948   \n",
       "\n",
       "   item_id_lag_5  item_category_id_lag_5  shop_id_lag_12  item_id_lag_12  \\\n",
       "0       1.844238                0.234431        -0.30674        0.932732   \n",
       "1      -0.274519               -0.179152        -0.30674       -0.245800   \n",
       "2       2.126740                0.234431        -0.30674       -0.245800   \n",
       "3      -0.274519                0.188583        -0.30674       -0.245800   \n",
       "4      -0.274519                1.045059        -0.30674       -0.245800   \n",
       "\n",
       "   item_category_id_lag_12  item_category_name  \n",
       "0                 0.359401                   8  \n",
       "1                -0.121941                  13  \n",
       "2                 0.359401                   8  \n",
       "3                 0.418859                   8  \n",
       "4                 3.306020                   8  \n",
       "\n",
       "[5 rows x 34 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = ColumnarModelData.from_data_frames(\n",
    "    path=PATH,\n",
    "    trn_df=trn_df,\n",
    "    trn_y=trn_y,\n",
    "    val_df=val_df,\n",
    "    val_y=val_y,\n",
    "    cat_flds=cat_vars,\n",
    "    bs=128,\n",
    "    test_df=test_df,\n",
    "    is_reg=True,  # is regression\n",
    "    is_multi=False, \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_sz = [(c, len(X_train[c].cat.categories)+1) for c in cat_vars]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('date_block_num', 36),\n",
       " ('item_id', 22171),\n",
       " ('month', 13),\n",
       " ('shop_id', 61),\n",
       " ('year', 4),\n",
       " ('item_category_id', 85),\n",
       " ('item_category_name', 19)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_sz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_szs = [(c, min(50, (c+1)//2)) for _,c in cat_sz]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(36, 18), (22171, 50), (13, 7), (61, 31), (4, 2), (85, 43), (19, 10)]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_szs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## _First Attempt - No Regularization_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "m = md.get_learner(\n",
    "    emb_szs=emb_szs,\n",
    "    n_cont=len(trn_df.columns) - len(cat_vars),\n",
    "    emb_drop=.02,\n",
    "    out_sz=1,\n",
    "    szs=[1000, 500, 500],\n",
    "    drops=[.001, .005, .005],\n",
    "    use_bn=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.crit = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "m.lr_find(1e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "m.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-5\n",
    "m.fit(lrs=lr, n_cycle=2, metrics=[rmse, rmse_expm1], cycle_len=1, cycle_mult=2, wds=1e-6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## _New Attempt - Adding Regularization_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above model seems to be converging on _something_, but it's overfitting\n",
    "far too much. Here we will increase dropout in the last linear layer\n",
    "and include weight decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "m = md.get_learner(\n",
    "    emb_szs=emb_szs,\n",
    "    n_cont=len(trn_df.columns) - len(cat_vars),\n",
    "    emb_drop=.05,\n",
    "    out_sz=1,\n",
    "    szs=[1000, 500, 500],\n",
    "    drops=[.001, .005, .01],\n",
    "    use_bn=True,\n",
    "    y_range=(np.log1p(0), np.log1p(20))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "??md.get_learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.crit = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "m.lr_find(1e-6, wds=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "m.sched.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interestingly, adding weight decay made the plot much more noisy. I'm not\n",
    "sure what causes this, but it could be an intersting thing to investigate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lr = 1e-5\n",
    "m.fit(lrs=lr, n_cycle=2, metrics=[rmse, rmse_expm1], cycle_len=1, cycle_mult=2, wds=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lr = 1e-5\n",
    "m.fit(lrs=lr, n_cycle=2, metrics=[rmse, rmse_expm1], wds=1e-3, cycle_len=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### _Simplifying the Model_\n",
    "\n",
    "Here we will run the same code as above, except with less regularization and more\n",
    "dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "111"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/fastai/fastai/column_data.py:101: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  for o in self.lins: kaiming_normal(o.weight.data)\n",
      "/home/paperspace/fastai/fastai/column_data.py:103: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  kaiming_normal(self.outp.weight.data)\n"
     ]
    }
   ],
   "source": [
    "m = md.get_learner(\n",
    "    emb_szs=emb_szs,\n",
    "    n_cont=len(trn_df.columns) - len(cat_vars),\n",
    "    emb_drop=.1,\n",
    "    out_sz=1,\n",
    "    szs=[1000, 500],\n",
    "    drops=[.1, .15],\n",
    "    use_bn=True,\n",
    "    y_range=(np.log1p(0), np.log1p(20))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.crit = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "m.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "845d0ab6768149518816909ebf9ec17a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   rmse       rmse_expm1             \n",
      "    0      0.082594   0.091674   0.279879   0.873397  \n",
      "    1      0.085406   0.091155   0.27896    0.871969               \n",
      "    2      0.085814   0.091124   0.278873   0.873061               \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.091123535837760089, 0.27887298227514706, 0.87306084084035918]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = 1e-4\n",
    "m.fit(lrs=lr, n_cycle=2, metrics=[rmse, rmse_expm1], cycle_len=1, cycle_mult=2, wds=3e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the competition emphasizes that answers range from 0 to 20, the above stats aren't truly\n",
    "representative of what we can expect from this model. Let's add some clipping."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96388239"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  8%|▊         | 3576/45622 [00:50<09:47, 71.52it/s, loss=0.107] "
     ]
    }
   ],
   "source": [
    "validation_predictions = m.predict()\n",
    "np.sqrt((\n",
    "    (np.expm1(val_y.reshape(len(val_y), 1)).clip(0,20) - np.expm1(validation_predictions).clip(0,20))**2\n",
    ").mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make Predictions on the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(y_train)\n",
    "\n",
    "result = m.predict(is_test=True)\n",
    "predictions = np.expm1(result).clip(0, 20)\n",
    "# predictions = result.clip(0,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "??np.mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.save('final_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.load('final_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Boost Regressor\n",
    "\n",
    "This is the most common methodology that I see in this Kaggle competition. Gradient boosting is\n",
    "a well-established method for tabular data. It's historically been a great approach\n",
    "in Kaggle competitions.\n",
    "\n",
    "From peeking at other kernels, people have actually gotten worse results with\n",
    "using the entire dataset with gradient boosting. The goal of the deep \n",
    "learning approach is to capture the entire dataset and come up with better generalization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train['item_category_name'] = X_train['item_category_name'].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test['item_category_name'] = X_test['item_category_name'].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val['item_category_name'] = X_val['item_category_name'].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for var in cat_vars:\n",
    "    if var == 'item_category_name':\n",
    "        continue # hack\n",
    "    X_train[var] = X_train[var].astype('int32')\n",
    "    X_val[var] = X_val[var].astype('int32')\n",
    "    X_test[var] = X_test[var].astype('int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost.sklearn import XGBRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "params = {\n",
    "    'objective': 'reg:linear',\n",
    "    'max_depth':4 ,\n",
    "    'learning_rate': 1,\n",
    "    'silent': 1.0,\n",
    "    'n_estimators': 30,\n",
    "    'min_child_weight':10\n",
    "}\n",
    "\n",
    "bst = XGBRegressor(**params).fit(X_train, y_train,eval_metric='rmse')\n",
    "preds = bst.predict(X_val)\n",
    "sqrt(mean_squared_error(y_val, preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost.sklearn import XGBRegressor\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "params = {\n",
    "    'objective': 'reg:linear',\n",
    "    'max_depth':4 ,\n",
    "    'learning_rate': 1,\n",
    "    'silent': 1.0,\n",
    "    'n_estimators': 30,\n",
    "    'min_child_weight':10\n",
    "}\n",
    "\n",
    "X = pd.concat([X_train,X_val])\n",
    "y = np.concatenate([y_train,y_val])\n",
    "del X_train,X_val,y_train,y_val\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bst = XGBRegressor(**params).fit(X, y,eval_metric='rmse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = bst.predict(X_test)\n",
    "predictions = np.clip(predictions,0.0,20.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Submit It!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.30741],\n",
       "       [ 0.04753],\n",
       "       [ 0.55086],\n",
       "       ..., \n",
       "       [ 0.04521],\n",
       "       [ 0.04091],\n",
       "       [ 0.03766]], dtype=float32)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "final_dataframe = pd.read_csv(f'{PATH}test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataframe['item_cnt_month'] = predictions\n",
    "final_dataframe = final_dataframe[['ID', 'item_cnt_month']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>item_cnt_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.307406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.047529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.550857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.213405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.183690</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  item_cnt_month\n",
       "0   0        0.307406\n",
       "1   1        0.047529\n",
       "2   2        0.550857\n",
       "3   3        0.213405\n",
       "4   4        0.183690"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_dataframe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "214200"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(final_dataframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0.00438,   0.00484,   0.00857, ...,  18.78045,  18.80826,  18.83339], dtype=float32)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "214200"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(final_dataframe['ID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "SUBM = 'sub/'\n",
    "os.makedirs(SUBM, exist_ok=True)\n",
    "final_dataframe.to_csv(f'{SUBM}subm.gz', compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='sub/subm.gz' target='_blank'>sub/subm.gz</a><br>"
      ],
      "text/plain": [
       "/home/paperspace/kaggle/competitive-data-science-predict-future-sales/sub/subm.gz"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FileLink(f'{SUBM}subm.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}