Lightgbm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1018,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 100)\n",
    "from lightgbm import LGBMRegressor\n",
    "import time\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1019,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv('Train.csv',parse_dates=['Date'])\n",
    "test=pd.read_csv('Test.csv',parse_dates=['Date'])\n",
    "sub=pd.read_excel('sample_submission.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic Understanding of Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1020,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Product</th>\n",
       "      <th>Product_Brand</th>\n",
       "      <th>Item_Category</th>\n",
       "      <th>Subcategory_1</th>\n",
       "      <th>Subcategory_2</th>\n",
       "      <th>Item_Rating</th>\n",
       "      <th>Date</th>\n",
       "      <th>Selling_Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>P-2610</td>\n",
       "      <td>B-659</td>\n",
       "      <td>bags wallets belts</td>\n",
       "      <td>bags</td>\n",
       "      <td>hand bags</td>\n",
       "      <td>4.3</td>\n",
       "      <td>2017-02-03</td>\n",
       "      <td>291.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>P-2453</td>\n",
       "      <td>B-3078</td>\n",
       "      <td>clothing</td>\n",
       "      <td>women s clothing</td>\n",
       "      <td>western wear</td>\n",
       "      <td>3.1</td>\n",
       "      <td>2015-07-01</td>\n",
       "      <td>897.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P-6802</td>\n",
       "      <td>B-1810</td>\n",
       "      <td>home decor festive needs</td>\n",
       "      <td>showpieces</td>\n",
       "      <td>ethnic</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2019-01-12</td>\n",
       "      <td>792.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>P-4452</td>\n",
       "      <td>B-3078</td>\n",
       "      <td>beauty and personal care</td>\n",
       "      <td>eye care</td>\n",
       "      <td>h2o plus eye care</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2014-12-12</td>\n",
       "      <td>837.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>P-8454</td>\n",
       "      <td>B-3078</td>\n",
       "      <td>clothing</td>\n",
       "      <td>men s clothing</td>\n",
       "      <td>t shirts</td>\n",
       "      <td>4.3</td>\n",
       "      <td>2013-12-12</td>\n",
       "      <td>470.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Product Product_Brand             Item_Category     Subcategory_1  \\\n",
       "0  P-2610         B-659        bags wallets belts              bags   \n",
       "1  P-2453        B-3078                  clothing  women s clothing   \n",
       "2  P-6802        B-1810  home decor festive needs        showpieces   \n",
       "3  P-4452        B-3078  beauty and personal care          eye care   \n",
       "4  P-8454        B-3078                  clothing    men s clothing   \n",
       "\n",
       "       Subcategory_2  Item_Rating       Date  Selling_Price  \n",
       "0          hand bags          4.3 2017-02-03          291.0  \n",
       "1       western wear          3.1 2015-07-01          897.0  \n",
       "2             ethnic          3.5 2019-01-12          792.0  \n",
       "3  h2o plus eye care          4.0 2014-12-12          837.0  \n",
       "4           t shirts          4.3 2013-12-12          470.0  "
      ]
     },
     "execution_count": 1020,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1021,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Product</th>\n",
       "      <th>Product_Brand</th>\n",
       "      <th>Item_Category</th>\n",
       "      <th>Subcategory_1</th>\n",
       "      <th>Subcategory_2</th>\n",
       "      <th>Item_Rating</th>\n",
       "      <th>Date</th>\n",
       "      <th>Selling_Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2447</th>\n",
       "      <td>P-8870</td>\n",
       "      <td>B-2292</td>\n",
       "      <td>clothing</td>\n",
       "      <td>kids clothing</td>\n",
       "      <td>girls wear</td>\n",
       "      <td>2.3</td>\n",
       "      <td>2017-11-03</td>\n",
       "      <td>741.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2448</th>\n",
       "      <td>P-10672</td>\n",
       "      <td>B-3078</td>\n",
       "      <td>footwear</td>\n",
       "      <td>women s footwear</td>\n",
       "      <td>casual shoes</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2016-04-12</td>\n",
       "      <td>1590.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2449</th>\n",
       "      <td>P-2134</td>\n",
       "      <td>B-479</td>\n",
       "      <td>clothing</td>\n",
       "      <td>men s clothing</td>\n",
       "      <td>t shirts</td>\n",
       "      <td>1.9</td>\n",
       "      <td>2019-04-19</td>\n",
       "      <td>995.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2450</th>\n",
       "      <td>P-724</td>\n",
       "      <td>B-133</td>\n",
       "      <td>automotive</td>\n",
       "      <td>accessories spare parts</td>\n",
       "      <td>car interior exterior</td>\n",
       "      <td>2.7</td>\n",
       "      <td>2014-01-12</td>\n",
       "      <td>1598.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2451</th>\n",
       "      <td>P-1154</td>\n",
       "      <td>B-174</td>\n",
       "      <td>bags wallets belts</td>\n",
       "      <td>bags</td>\n",
       "      <td>pouches and potlis</td>\n",
       "      <td>4.1</td>\n",
       "      <td>2019-03-03</td>\n",
       "      <td>397.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Product Product_Brand       Item_Category            Subcategory_1  \\\n",
       "2447   P-8870        B-2292            clothing            kids clothing   \n",
       "2448  P-10672        B-3078            footwear         women s footwear   \n",
       "2449   P-2134         B-479            clothing           men s clothing   \n",
       "2450    P-724         B-133          automotive  accessories spare parts   \n",
       "2451   P-1154         B-174  bags wallets belts                     bags   \n",
       "\n",
       "              Subcategory_2  Item_Rating       Date  Selling_Price  \n",
       "2447             girls wear          2.3 2017-11-03          741.0  \n",
       "2448           casual shoes          1.9 2016-04-12         1590.0  \n",
       "2449               t shirts          1.9 2019-04-19          995.0  \n",
       "2450  car interior exterior          2.7 2014-01-12         1598.0  \n",
       "2451     pouches and potlis          4.1 2019-03-03          397.0  "
      ]
     },
     "execution_count": 1021,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1022,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Product</th>\n",
       "      <th>Product_Brand</th>\n",
       "      <th>Item_Category</th>\n",
       "      <th>Subcategory_1</th>\n",
       "      <th>Subcategory_2</th>\n",
       "      <th>Item_Rating</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>P-11284</td>\n",
       "      <td>B-2984</td>\n",
       "      <td>computers</td>\n",
       "      <td>network components</td>\n",
       "      <td>routers</td>\n",
       "      <td>4.3</td>\n",
       "      <td>2018-01-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>P-6580</td>\n",
       "      <td>B-1732</td>\n",
       "      <td>jewellery</td>\n",
       "      <td>bangles bracelets armlets</td>\n",
       "      <td>bracelets</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2012-12-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P-5843</td>\n",
       "      <td>B-3078</td>\n",
       "      <td>clothing</td>\n",
       "      <td>women s clothing</td>\n",
       "      <td>western wear</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2014-01-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>P-5334</td>\n",
       "      <td>B-1421</td>\n",
       "      <td>jewellery</td>\n",
       "      <td>necklaces chains</td>\n",
       "      <td>necklaces</td>\n",
       "      <td>3.9</td>\n",
       "      <td>2019-01-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>P-5586</td>\n",
       "      <td>B-3078</td>\n",
       "      <td>clothing</td>\n",
       "      <td>women s clothing</td>\n",
       "      <td>western wear</td>\n",
       "      <td>1.4</td>\n",
       "      <td>2017-01-12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Product Product_Brand Item_Category              Subcategory_1  \\\n",
       "0  P-11284        B-2984     computers         network components   \n",
       "1   P-6580        B-1732     jewellery  bangles bracelets armlets   \n",
       "2   P-5843        B-3078      clothing           women s clothing   \n",
       "3   P-5334        B-1421     jewellery           necklaces chains   \n",
       "4   P-5586        B-3078      clothing           women s clothing   \n",
       "\n",
       "  Subcategory_2  Item_Rating       Date  \n",
       "0       routers          4.3 2018-01-12  \n",
       "1     bracelets          3.0 2012-12-20  \n",
       "2  western wear          1.5 2014-01-12  \n",
       "3     necklaces          3.9 2019-01-12  \n",
       "4  western wear          1.4 2017-01-12  "
      ]
     },
     "execution_count": 1022,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1023,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Product</th>\n",
       "      <th>Product_Brand</th>\n",
       "      <th>Item_Category</th>\n",
       "      <th>Subcategory_1</th>\n",
       "      <th>Subcategory_2</th>\n",
       "      <th>Item_Rating</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1046</th>\n",
       "      <td>P-9758</td>\n",
       "      <td>B-2543</td>\n",
       "      <td>sports fitness</td>\n",
       "      <td>outdoor adventure</td>\n",
       "      <td>camping hiking</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2014-02-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1047</th>\n",
       "      <td>P-11898</td>\n",
       "      <td>B-3197</td>\n",
       "      <td>jewellery</td>\n",
       "      <td>necklaces chains</td>\n",
       "      <td>necklaces</td>\n",
       "      <td>4.7</td>\n",
       "      <td>2019-01-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048</th>\n",
       "      <td>P-904</td>\n",
       "      <td>B-133</td>\n",
       "      <td>automotive</td>\n",
       "      <td>accessories spare parts</td>\n",
       "      <td>car interior exterior</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2014-01-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1049</th>\n",
       "      <td>P-1714</td>\n",
       "      <td>B-358</td>\n",
       "      <td>kitchen dining</td>\n",
       "      <td>lighting</td>\n",
       "      <td>bulbs</td>\n",
       "      <td>2.4</td>\n",
       "      <td>2013-06-23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1050</th>\n",
       "      <td>P-620</td>\n",
       "      <td>B-133</td>\n",
       "      <td>automotive</td>\n",
       "      <td>accessories spare parts</td>\n",
       "      <td>car interior exterior</td>\n",
       "      <td>3.1</td>\n",
       "      <td>2012-01-12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Product Product_Brand   Item_Category            Subcategory_1  \\\n",
       "1046   P-9758        B-2543  sports fitness        outdoor adventure   \n",
       "1047  P-11898        B-3197       jewellery         necklaces chains   \n",
       "1048    P-904         B-133      automotive  accessories spare parts   \n",
       "1049   P-1714         B-358  kitchen dining                 lighting   \n",
       "1050    P-620         B-133      automotive  accessories spare parts   \n",
       "\n",
       "              Subcategory_2  Item_Rating       Date  \n",
       "1046         camping hiking          2.0 2014-02-28  \n",
       "1047              necklaces          4.7 2019-01-12  \n",
       "1048  car interior exterior          3.5 2014-01-12  \n",
       "1049                  bulbs          2.4 2013-06-23  \n",
       "1050  car interior exterior          3.1 2012-01-12  "
      ]
     },
     "execution_count": 1023,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1024,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Product                  object\n",
       "Product_Brand            object\n",
       "Item_Category            object\n",
       "Subcategory_1            object\n",
       "Subcategory_2            object\n",
       "Item_Rating             float64\n",
       "Date             datetime64[ns]\n",
       "Selling_Price           float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 1024,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1025,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Item_Rating</th>\n",
       "      <th>Selling_Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2452.000000</td>\n",
       "      <td>2452.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.078467</td>\n",
       "      <td>2494.375612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.187137</td>\n",
       "      <td>7115.256516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>33.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>371.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.100000</td>\n",
       "      <td>596.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.100000</td>\n",
       "      <td>1195.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>116289.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Item_Rating  Selling_Price\n",
       "count  2452.000000    2452.000000\n",
       "mean      3.078467    2494.375612\n",
       "std       1.187137    7115.256516\n",
       "min       1.000000      33.000000\n",
       "25%       2.000000     371.000000\n",
       "50%       3.100000     596.000000\n",
       "75%       4.100000    1195.250000\n",
       "max       5.000000  116289.000000"
      ]
     },
     "execution_count": 1025,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1026,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ABetta\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "train['train_or_test']='train'\n",
    "test['train_or_test']='test'\n",
    "df=pd.concat([train,test])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Time Based Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1027,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_date_featues(df):\n",
    "\n",
    "    df['Month'] = pd.to_datetime(df['Date']).dt.month\n",
    "\n",
    "    df['DayOfyear'] = pd.to_datetime(df['Date']).dt.dayofyear\n",
    "\n",
    "    df['Quarter'] = pd.to_datetime(df['Date']).dt.quarter \n",
    "\n",
    "    df['Is_month_start'] = pd.to_datetime(df['Date']).dt.is_month_start\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1028,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=create_date_featues(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Replaced Unknown as NAN , created a boolean feat which return True for all NAN values. Replacing NAN of Subcategory_1 didnt give useful results ,so filled back with unknown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1029,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Subcategory_1']=df['Subcategory_1'].replace('unknown', np.nan)\n",
    "df['Subcategory_2']=df['Subcategory_2'].replace('unknown', np.nan)\n",
    "\n",
    "cols_with_missing = [col for col in df.columns\n",
    "                     if df[col].isnull().any()]\n",
    "for col in cols_with_missing:\n",
    "    df[col + '_was_missing'] = df[col].isnull()\n",
    "\n",
    "df['Subcategory_1'].fillna('unknown',inplace=True)\n",
    "#df['Subcategory_1'].fillna(df['Item_Category'],inplace=True)\n",
    "#df['Subcategory_2'].fillna(df['Subcategory_1'],inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Replaced Majority of NAN of sub_category_2 with mode grouped by with sub_cat_1 ,remaining few NA values with unknown as it is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1030,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(inplace=True)\n",
    "def fast_mode(df, key_cols, value_col):\n",
    "    \"\"\" \n",
    "    Calculate a column mode, by group, ignoring null values. \n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    df : pandas.DataFrame\n",
    "        DataFrame over which to calcualate the mode. \n",
    "    key_cols : list of str\n",
    "        Columns to groupby for calculation of mode.\n",
    "    value_col : str\n",
    "        Column for which to calculate the mode. \n",
    "\n",
    "    Return\n",
    "    ------ \n",
    "    pandas.DataFrame\n",
    "        One row for the mode of value_col per key_cols group. If ties, \n",
    "        returns the one which is sorted first. \n",
    "    \"\"\"\n",
    "    return (df.groupby(key_cols + [value_col]).size() \n",
    "              .to_frame('counts').reset_index() \n",
    "              .sort_values('counts', ascending=False) \n",
    "              .drop_duplicates(subset=key_cols)).drop(columns='counts')\n",
    "\n",
    "\n",
    "df.loc[df.Subcategory_2.isnull(), 'Subcategory_2'] = df.Subcategory_1.map(fast_mode(df, ['Subcategory_1'], 'Subcategory_2').set_index('Subcategory_1').Subcategory_2)\n",
    "del df['index']\n",
    "df['Subcategory_2'].fillna('unknown',inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interaction Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1031,
   "metadata": {},
   "outputs": [],
   "source": [
    "interaction1=df['Product_Brand']+'_'+df['Subcategory_1']\n",
    "interaction2=df['Product_Brand']+'_'+df['Subcategory_2']\n",
    "\n",
    "interaction3=df['Item_Category']+'_'+df['Subcategory_1']\n",
    "interaction4=df['Item_Category']+'_'+df['Subcategory_2']\n",
    "\n",
    "interaction5=df['Subcategory_1']+'_'+df['Subcategory_2']\n",
    "\n",
    "\n",
    "df['PB_S1']=interaction1\n",
    "df['PB_S2']=interaction2\n",
    "df['IC_S1']=interaction3\n",
    "df['IC_S2']=interaction4\n",
    "df['S1_S2']=interaction5\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1032,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "\n",
    "for col in ['Item_Category', 'Product','Subcategory_1', 'PB_S2', 'IC_S1','S1_S2']:\n",
    "    df[col]= le.fit_transform(df[col]) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Frequency Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1033,
   "metadata": {},
   "outputs": [],
   "source": [
    "fe_pol = (df.groupby('Product_Brand').size()) / len(df)\n",
    "df['Product_Brand_fe'] = df['Product_Brand'].apply(lambda x : fe_pol[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1034,
   "metadata": {},
   "outputs": [],
   "source": [
    "fe_pol = (df.groupby('Subcategory_1').size()) / len(df)\n",
    "df['Subcategory_1_fe'] = df['Subcategory_1'].apply(lambda x : fe_pol[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1035,
   "metadata": {},
   "outputs": [],
   "source": [
    "fe_pol = (df.groupby('Subcategory_2').size()) / len(df)\n",
    "df['Subcategory_2_fe'] = df['Subcategory_2'].apply(lambda x : fe_pol[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1036,
   "metadata": {},
   "outputs": [],
   "source": [
    "fe_pol = (df.groupby('PB_S2').size()) / len(df)\n",
    "df['PB_S2_fe'] = df['PB_S2'].apply(lambda x : fe_pol[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Binary Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1037,
   "metadata": {},
   "outputs": [],
   "source": [
    "import category_encoders as ce\n",
    "encoder=ce.BinaryEncoder(cols=['Product_Brand'])\n",
    "dfbin=encoder.fit_transform(df['Product_Brand'])\n",
    "df=pd.concat([df,dfbin],axis=1)\n",
    "del df['Product_Brand']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1038,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder=ce.BinaryEncoder(cols=['Subcategory_2'])\n",
    "dfbin=encoder.fit_transform(df['Subcategory_2'])\n",
    "df=pd.concat([df,dfbin],axis=1)\n",
    "del df['Subcategory_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1039,
   "metadata": {},
   "outputs": [],
   "source": [
    "import category_encoders as ce\n",
    "encoder=ce.BinaryEncoder(cols=['PB_S1'])\n",
    "dfbin=encoder.fit_transform(df['PB_S1'])\n",
    "df=pd.concat([df,dfbin],axis=1)\n",
    "del df['PB_S1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1040,
   "metadata": {},
   "outputs": [],
   "source": [
    "import category_encoders as ce\n",
    "encoder=ce.BinaryEncoder(cols=['PB_S2'])\n",
    "dfbin=encoder.fit_transform(df['PB_S2'])\n",
    "df=pd.concat([df,dfbin],axis=1)\n",
    "del df['PB_S2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1041,
   "metadata": {},
   "outputs": [],
   "source": [
    "import category_encoders as ce\n",
    "encoder=ce.BinaryEncoder(cols=['IC_S2'])\n",
    "dfbin=encoder.fit_transform(df['IC_S2'])\n",
    "df=pd.concat([df,dfbin],axis=1)\n",
    "del df['IC_S2']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting back train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1042,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ABetta\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3940: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    }
   ],
   "source": [
    "train=df.loc[df.train_or_test.isin(['train'])]\n",
    "test=df.loc[df.train_or_test.isin(['test'])]\n",
    "train.drop(columns={'train_or_test'},axis=1,inplace=True)\n",
    "test.drop(columns={'train_or_test'},axis=1,inplace=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log transforming target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1043,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ABetta\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "train['Selling_Price']=np.log(train['Selling_Price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1044,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Date', 'Item_Category', 'Item_Rating', 'Product', 'Selling_Price',\n",
       "       'Subcategory_1', 'Month', 'DayOfyear', 'Quarter', 'Is_month_start',\n",
       "       'Selling_Price_was_missing', 'Subcategory_1_was_missing',\n",
       "       'Subcategory_2_was_missing', 'IC_S1', 'S1_S2', 'Product_Brand_fe',\n",
       "       'Subcategory_1_fe', 'Subcategory_2_fe', 'PB_S2_fe', 'Product_Brand_0',\n",
       "       'Product_Brand_1', 'Product_Brand_2', 'Product_Brand_3',\n",
       "       'Product_Brand_4', 'Product_Brand_5', 'Product_Brand_6',\n",
       "       'Product_Brand_7', 'Product_Brand_8', 'Product_Brand_9',\n",
       "       'Product_Brand_10', 'Product_Brand_11', 'Subcategory_2_0',\n",
       "       'Subcategory_2_1', 'Subcategory_2_2', 'Subcategory_2_3',\n",
       "       'Subcategory_2_4', 'Subcategory_2_5', 'Subcategory_2_6',\n",
       "       'Subcategory_2_7', 'Subcategory_2_8', 'Subcategory_2_9', 'PB_S1_0',\n",
       "       'PB_S1_1', 'PB_S1_2', 'PB_S1_3', 'PB_S1_4', 'PB_S1_5', 'PB_S1_6',\n",
       "       'PB_S1_7', 'PB_S1_8', 'PB_S1_9', 'PB_S1_10', 'PB_S1_11', 'PB_S2_0',\n",
       "       'PB_S2_1', 'PB_S2_2', 'PB_S2_3', 'PB_S2_4', 'PB_S2_5', 'PB_S2_6',\n",
       "       'PB_S2_7', 'PB_S2_8', 'PB_S2_9', 'PB_S2_10', 'PB_S2_11', 'IC_S2_0',\n",
       "       'IC_S2_1', 'IC_S2_2', 'IC_S2_3', 'IC_S2_4', 'IC_S2_5', 'IC_S2_6',\n",
       "       'IC_S2_7', 'IC_S2_8', 'IC_S2_9'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 1044,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Here i have droped features with low importance too"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1045,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=train.drop(columns={'Selling_Price','Date','Selling_Price_was_missing','Subcategory_2_0','PB_S1_0','Product_Brand_1','PB_S2_0','Product_Brand_0','IC_S2_0'},axis=1)\n",
    "y=train['Selling_Price']\n",
    "test=test.drop(columns={'Selling_Price','Date','Selling_Price_was_missing','Subcategory_2_0','PB_S1_0','Product_Brand_1','PB_S2_0','Product_Brand_0','IC_S2_0'},axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1046,
   "metadata": {},
   "outputs": [],
   "source": [
    "col=x.columns\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "st=StandardScaler()\n",
    "st.fit(x)\n",
    "x=st.transform(x)\n",
    "test=st.transform(test)\n",
    "x=pd.DataFrame(x,columns=col)\n",
    "test=pd.DataFrame(test,columns=col)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Since we have log transformed target variable, calculating rmse will give rmsle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 941,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "def rmse(y, y_pred):\n",
    "    return np.sqrt(mean_squared_error(y,y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kfold cross validating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1048,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds\n",
      "[200]\ttraining's l2: 0.149351\tvalid_1's l2: 0.460698\n",
      "[400]\ttraining's l2: 0.0805028\tvalid_1's l2: 0.454747\n",
      "Early stopping, best iteration is:\n",
      "[321]\ttraining's l2: 0.101046\tvalid_1's l2: 0.453721\n",
      "1  err_lgbm:  0.6735882994837199\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[200]\ttraining's l2: 0.155561\tvalid_1's l2: 0.349065\n",
      "[400]\ttraining's l2: 0.0754913\tvalid_1's l2: 0.343882\n",
      "Early stopping, best iteration is:\n",
      "[325]\ttraining's l2: 0.0954031\tvalid_1's l2: 0.343256\n",
      "2  err_lgbm:  0.5858808382218244\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[200]\ttraining's l2: 0.148606\tvalid_1's l2: 0.424671\n",
      "Early stopping, best iteration is:\n",
      "[278]\ttraining's l2: 0.107175\tvalid_1's l2: 0.42208\n",
      "3  err_lgbm:  0.6496770246450424\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[200]\ttraining's l2: 0.146011\tvalid_1's l2: 0.433763\n",
      "[400]\ttraining's l2: 0.0675067\tvalid_1's l2: 0.425623\n",
      "Early stopping, best iteration is:\n",
      "[352]\ttraining's l2: 0.0784692\tvalid_1's l2: 0.424229\n",
      "4  err_lgbm:  0.6513288281362372\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[200]\ttraining's l2: 0.146145\tvalid_1's l2: 0.376406\n",
      "[400]\ttraining's l2: 0.0740176\tvalid_1's l2: 0.371343\n",
      "[600]\ttraining's l2: 0.046688\tvalid_1's l2: 0.3694\n",
      "[800]\ttraining's l2: 0.0327893\tvalid_1's l2: 0.368562\n",
      "[1000]\ttraining's l2: 0.0256159\tvalid_1's l2: 0.367801\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[1000]\ttraining's l2: 0.0256159\tvalid_1's l2: 0.367801\n",
      "5  err_lgbm:  0.6064663611851395\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6333882703343926"
      ]
     },
     "execution_count": 1048,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "err = []\n",
    "y_pred_tot_lgbm = []\n",
    "\n",
    "\n",
    "fold = KFold(n_splits=5, shuffle=True, random_state=2020)\n",
    "i = 1\n",
    "\n",
    "for train_index, test_index in fold.split(x, y):\n",
    "    x_train, x_val = x.iloc[train_index], x.iloc[test_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[test_index]\n",
    "    m = LGBMRegressor(n_estimators=1000, num_leaves=127, max_depth=8,min_child_samples=4,\n",
    "                      learning_rate=0.02, colsample_bytree=0.4, reg_alpha=0.5, reg_lambda=2)\n",
    "    m.fit(x_train, y_train,eval_set=[(x_train,y_train),(x_val, y_val)],early_stopping_rounds=100,verbose=200,#eval_metric=rmsle(y_val,pred_y)\n",
    "         )\n",
    "    pred_y = m.predict(x_val)\n",
    "    print(i, \" err_lgbm: \",  rmse(y_val,pred_y))\n",
    "    err.append(rmse(y_val,pred_y))\n",
    "    pred_test = m.predict(test)\n",
    "    i = i + 1\n",
    "    y_pred_tot_lgbm.append(pred_test)\n",
    "sum(err)/5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1049,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.633"
      ]
     },
     "execution_count": 1049,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "0.633"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1050,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_imp = pd.DataFrame(m.feature_importances_, index=x.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1051,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Subcategory_1_was_missing        5\n",
       "PB_S1_1                         60\n",
       "Subcategory_2_1                 85\n",
       "IC_S2_1                        116\n",
       "Subcategory_2_was_missing      124\n",
       "Subcategory_2_6                126\n",
       "IC_S2_2                        140\n",
       "Subcategory_2_2                150\n",
       "Is_month_start                 154\n",
       "PB_S2_5                        157\n",
       "PB_S2_3                        171\n",
       "Product_Brand_2                172\n",
       "Product_Brand_9                185\n",
       "IC_S2_8                        189\n",
       "Subcategory_2_7                204\n",
       "PB_S1_3                        205\n",
       "PB_S2_1                        213\n",
       "PB_S1_6                        223\n",
       "PB_S1_2                        224\n",
       "IC_S2_4                        227\n",
       "IC_S2_9                        241\n",
       "PB_S1_7                        249\n",
       "PB_S2_4                        262\n",
       "Product_Brand_11               265\n",
       "Product_Brand_3                269\n",
       "Subcategory_2_9                272\n",
       "Subcategory_2_5                275\n",
       "Product_Brand_7                281\n",
       "IC_S2_3                        287\n",
       "Product_Brand_10               291\n",
       "Subcategory_2_3                295\n",
       "IC_S2_7                        295\n",
       "Product_Brand_6                298\n",
       "Product_Brand_4                299\n",
       "PB_S2_2                        299\n",
       "PB_S1_5                        303\n",
       "PB_S1_11                       309\n",
       "PB_S2_11                       321\n",
       "PB_S2_10                       327\n",
       "Subcategory_2_4                336\n",
       "IC_S2_5                        353\n",
       "PB_S1_10                       355\n",
       "IC_S2_6                        357\n",
       "Product_Brand_8                361\n",
       "Product_Brand_5                376\n",
       "PB_S1_9                        379\n",
       "PB_S1_4                        384\n",
       "PB_S1_8                        384\n",
       "PB_S2_6                        412\n",
       "Subcategory_2_8                425\n",
       "PB_S2_9                        436\n",
       "PB_S2_8                        451\n",
       "PB_S2_7                        454\n",
       "Quarter                        570\n",
       "Product_Brand_fe              3021\n",
       "Item_Category                 3198\n",
       "PB_S2_fe                      3296\n",
       "Month                         3589\n",
       "Subcategory_1_fe              3713\n",
       "Subcategory_1                 4364\n",
       "IC_S1                         4449\n",
       "Subcategory_2_fe              4881\n",
       "S1_S2                         4894\n",
       "DayOfyear                     7680\n",
       "Item_Rating                   9559\n",
       "Product                      11573\n",
       "Name: 0, dtype: int32"
      ]
     },
     "execution_count": 1051,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_imp[0].sort_values(ascending=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Model on All train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 948,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = LGBMRegressor(n_estimators=325, num_leaves=127, max_depth=8,min_child_samples=4,\n",
    "                      learning_rate=0.02, colsample_bytree=0.4, reg_alpha=0.5, reg_lambda=2,random_state=2020)\n",
    "m.fit(x,y)\n",
    "lgbpred=m.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 949,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sub['Selling_Price']=np.exp(lgbpred)\n",
    "sub['Selling_Price']=lgbpred\n",
    "#sub.to_excel('lgbm.xlsx',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 707,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgb=pd.read_excel('lgbm.xlsx')\n",
    "rf=pd.read_excel('randomforest.xlsx') # Random Forest Code is in another file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Blending lgbm and Rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 708,
   "metadata": {},
   "outputs": [],
   "source": [
    "average=np.exp(0.45*lgb['Selling_Price']+0.55*rf['Selling_Price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 710,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub['Selling_Price']=average\n",
    "sub.to_excel('blending.xlsx',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kfold validating blending accuracy of LGBM and RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 666,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1  err_xgb:  0.6744392062002934\n",
      "2  err_xgb:  0.5830066895429288\n",
      "3  err_xgb:  0.6439375050185021\n",
      "4  err_xgb:  0.6536547390285575\n",
      "5  err_xgb:  0.6097304120455125\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6329537103671589"
      ]
     },
     "execution_count": 666,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "err = []\n",
    "y_pred_tot_xgb = []\n",
    "\n",
    "\n",
    "\n",
    "fold = KFold(n_splits=5, shuffle=True, random_state=2020)\n",
    "i = 1\n",
    "\n",
    "for train_index, test_index in fold.split(x, y):\n",
    "    x_train, x_val = x.iloc[train_index], x.iloc[test_index]\n",
    "    y_train, y_val = y.iloc[train_index], y.iloc[test_index]\n",
    "    \n",
    "    m = RandomForestRegressor(bootstrap=True, max_features=0.45, min_samples_leaf=1, min_samples_split=5, n_estimators=100,random_state=2020)\n",
    "    m.fit(x_train, y_train)\n",
    "    rfpred = m.predict(x_val)*0.999\n",
    "    \n",
    "    m = LGBMRegressor(n_estimators=442, num_leaves=127, max_depth=8,min_child_samples=4,\n",
    "                      learning_rate=0.02, colsample_bytree=0.4, reg_alpha=0.5, reg_lambda=2,random_state=2020)\n",
    "    m.fit(x_train, y_train)\n",
    "    lgbpred=m.predict(x_val)*0.999\n",
    "    average=0.55*rfpred+0.45*lgbpred\n",
    "    \n",
    "    print(i, \" err_xgb: \",  rmse(y_val,average))\n",
    "    err.append(rmse(y_val,average))\n",
    "    #pred_test = m.predict(test)\n",
    "    i = i + 1\n",
    "    #y_pred_tot_xgb.append(pred_test)\n",
    "(err[0]+err[1]+err[2]+err[3]+err[4])/5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "0.6326"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}