Black Friday Sales Prediction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Black Friday Sales Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "import xgboost as xgb\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Product_ID</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>City_Category</th>\n",
       "      <th>Stay_In_Current_City_Years</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Product_Category_1</th>\n",
       "      <th>Product_Category_2</th>\n",
       "      <th>Product_Category_3</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00069042</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00248942</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00087842</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>P00085442</td>\n",
       "      <td>F</td>\n",
       "      <td>0-17</td>\n",
       "      <td>10</td>\n",
       "      <td>A</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>14.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000002</td>\n",
       "      <td>P00285442</td>\n",
       "      <td>M</td>\n",
       "      <td>55+</td>\n",
       "      <td>16</td>\n",
       "      <td>C</td>\n",
       "      <td>4+</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7969</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User_ID Product_ID Gender   Age  Occupation City_Category  \\\n",
       "0  1000001  P00069042      F  0-17          10             A   \n",
       "1  1000001  P00248942      F  0-17          10             A   \n",
       "2  1000001  P00087842      F  0-17          10             A   \n",
       "3  1000001  P00085442      F  0-17          10             A   \n",
       "4  1000002  P00285442      M   55+          16             C   \n",
       "\n",
       "  Stay_In_Current_City_Years  Marital_Status  Product_Category_1  \\\n",
       "0                          2               0                   3   \n",
       "1                          2               0                   1   \n",
       "2                          2               0                  12   \n",
       "3                          2               0                  12   \n",
       "4                         4+               0                   8   \n",
       "\n",
       "   Product_Category_2  Product_Category_3  Purchase  \n",
       "0                 NaN                 NaN      8370  \n",
       "1                 6.0                14.0     15200  \n",
       "2                 NaN                 NaN      1422  \n",
       "3                14.0                 NaN      1057  \n",
       "4                 NaN                 NaN      7969  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('/DATA SETS/Black Friday/train.csv')\n",
    "test = pd.read_csv('/DATA SETS/Black Friday/test.csv')\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Taking an Overview of the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 550068 entries, 0 to 550067\n",
      "Data columns (total 12 columns):\n",
      "User_ID                       550068 non-null int64\n",
      "Product_ID                    550068 non-null object\n",
      "Gender                        550068 non-null object\n",
      "Age                           550068 non-null object\n",
      "Occupation                    550068 non-null int64\n",
      "City_Category                 550068 non-null object\n",
      "Stay_In_Current_City_Years    550068 non-null object\n",
      "Marital_Status                550068 non-null int64\n",
      "Product_Category_1            550068 non-null int64\n",
      "Product_Category_2            376430 non-null float64\n",
      "Product_Category_3            166821 non-null float64\n",
      "Purchase                      550068 non-null int64\n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 50.4+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Marital_Status</th>\n",
       "      <th>Product_Category_1</th>\n",
       "      <th>Product_Category_2</th>\n",
       "      <th>Product_Category_3</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.500680e+05</td>\n",
       "      <td>550068.000000</td>\n",
       "      <td>550068.000000</td>\n",
       "      <td>550068.000000</td>\n",
       "      <td>376430.000000</td>\n",
       "      <td>166821.000000</td>\n",
       "      <td>550068.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.003029e+06</td>\n",
       "      <td>8.076707</td>\n",
       "      <td>0.409653</td>\n",
       "      <td>5.404270</td>\n",
       "      <td>9.842329</td>\n",
       "      <td>12.668243</td>\n",
       "      <td>9263.968713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.727592e+03</td>\n",
       "      <td>6.522660</td>\n",
       "      <td>0.491770</td>\n",
       "      <td>3.936211</td>\n",
       "      <td>5.086590</td>\n",
       "      <td>4.125338</td>\n",
       "      <td>5023.065394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000001e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>12.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.001516e+06</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>5823.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.003077e+06</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>8047.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.004478e+06</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>12054.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.006040e+06</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>23961.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            User_ID     Occupation  Marital_Status  Product_Category_1  \\\n",
       "count  5.500680e+05  550068.000000   550068.000000       550068.000000   \n",
       "mean   1.003029e+06       8.076707        0.409653            5.404270   \n",
       "std    1.727592e+03       6.522660        0.491770            3.936211   \n",
       "min    1.000001e+06       0.000000        0.000000            1.000000   \n",
       "25%    1.001516e+06       2.000000        0.000000            1.000000   \n",
       "50%    1.003077e+06       7.000000        0.000000            5.000000   \n",
       "75%    1.004478e+06      14.000000        1.000000            8.000000   \n",
       "max    1.006040e+06      20.000000        1.000000           20.000000   \n",
       "\n",
       "       Product_Category_2  Product_Category_3       Purchase  \n",
       "count       376430.000000       166821.000000  550068.000000  \n",
       "mean             9.842329           12.668243    9263.968713  \n",
       "std              5.086590            4.125338    5023.065394  \n",
       "min              2.000000            3.000000      12.000000  \n",
       "25%              5.000000            9.000000    5823.000000  \n",
       "50%              9.000000           14.000000    8047.000000  \n",
       "75%             15.000000           16.000000   12054.000000  \n",
       "max             18.000000           18.000000   23961.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The number of unique Users are: 5891\n",
      "The number of unique Products are: 3631\n"
     ]
    }
   ],
   "source": [
    "#Checking for unique values\n",
    "\n",
    "print('The number of unique Users are:',train['User_ID'].nunique())\n",
    "print('The number of unique Products are:',train['Product_ID'].nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a copy of the data to work on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For Train Data\n",
    "data = train.copy()\n",
    "data = data.drop(columns = ['Marital_Status'])\n",
    "\n",
    "#For Test Data\n",
    "data2 = test.copy()\n",
    "data2 = data2.drop(columns = ['Marital_Status'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dealing with the Null values in our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For Train Data\n",
    "\n",
    "# First do One Hot encoding for Product Category 1\n",
    "df_oneHot = pd.get_dummies(data, columns = ['Product_Category_1'], prefix = ['P'])\n",
    "#Fill NaN values with Zeros\n",
    "df_oneHot = df_oneHot.fillna(0)\n",
    "for i in range(1, 15):\n",
    "    df_oneHot.loc[df_oneHot.Product_Category_2 == i,'P_'+ str(i)]= 1\n",
    "    df_oneHot.loc[df_oneHot.Product_Category_3 == i,'P_'+ str(i)]= 1\n",
    "\n",
    "#For Test Data    \n",
    "\n",
    "# First do One Hot encoding for Product Category 1\n",
    "df2_oneHot = pd.get_dummies(data2, columns = ['Product_Category_1'], prefix = ['P'])\n",
    "#Fill NaN values with Zeros\n",
    "df2_oneHot = df2_oneHot.fillna(0)\n",
    "for i in range(1, 15):\n",
    "    df2_oneHot.loc[df2_oneHot.Product_Category_2 == i,'P_'+ str(i)]= 1\n",
    "    df2_oneHot.loc[df2_oneHot.Product_Category_3 == i,'P_'+ str(i)]= 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Two Columns missing in test data, so we add it manually. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2_oneHot['P_19'] = np.zeros([233599,1])\n",
    "df2_oneHot['P_20'] = np.zeros([233599,1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop unnecessary columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For train data\n",
    "df_oneHot = df_oneHot.drop(columns = ['Product_Category_2', 'Product_Category_3'])\n",
    "\n",
    "#For test data\n",
    "df2_oneHot = df2_oneHot.drop(columns = ['Product_Category_2', 'Product_Category_3'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making Product ID usable for the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For train Data\n",
    "df_oneHot['Product_ID'] = df_oneHot['Product_ID'].str.replace('P00', '')\n",
    "\n",
    "#For Test data\n",
    "df2_oneHot['Product_ID'] = df2_oneHot['Product_ID'].str.replace('P00', '')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Separarting Target and Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = data.Purchase\n",
    "df_oneHot = df_oneHot.drop(columns = ['Purchase'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode Gender Column "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For train Data\n",
    "df_oneHot = df_oneHot.replace({'Gender': {'M': 1, 'F':0}})\n",
    "\n",
    "#For test Data\n",
    "df2_oneHot = df2_oneHot.replace({'Gender': {'M': 1, 'F':0}})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding rest of the categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For train Data\n",
    "data_df_onehot = pd.get_dummies(df_oneHot, columns=['Age',\"Occupation\", \n",
    "                                               'City_Category','Stay_In_Current_City_Years'], \n",
    "                                 prefix = ['Age',\"Occupation\", \n",
    "                                               'City','Stay'])\n",
    "\n",
    "\n",
    "#For test Data\n",
    "data2_df_onehot = pd.get_dummies(df2_oneHot, columns=['Age',\"Occupation\", \n",
    "                                               'City_Category','Stay_In_Current_City_Years'], \n",
    "                                 prefix = ['Age',\"Occupation\", \n",
    "                                               'City','Stay'])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scaling Data for effective model  building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "\n",
    "#For Train Data\n",
    "data_df_onehot['Product_ID'] = scaler.fit_transform(data_df_onehot['Product_ID'].values.reshape(-1, 1))\n",
    "data_df_onehot['User_ID'] = scaler.transform(data_df_onehot['User_ID'].values.reshape(-1, 1))\n",
    "\n",
    "#For Test Data\n",
    "data2_df_onehot['Product_ID'] = scaler.transform(data2_df_onehot['Product_ID'].values.reshape(-1, 1))\n",
    "data2_df_onehot['User_ID'] = scaler.transform(data2_df_onehot['User_ID'].values.reshape(-1, 1))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting Data into test and train dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2_df_onehot.shape\n",
    "train_data, test_data, train_labels, test_labels = train_test_split(data_df_onehot, target, test_size=0.2, random_state=42)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Converting Data into DMatrix for XGBoost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#For Train Data\n",
    "dtrain = xgb.DMatrix(train_data, label=train_labels)\n",
    "dtest = xgb.DMatrix(test_data, label=test_labels)\n",
    "\n",
    "#For Test Data\n",
    "dpred = xgb.DMatrix(data2_df_onehot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initializing the basic parameter dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    # Parameters that we are going to tune.\n",
    "    'max_depth':6,\n",
    "    'min_child_weight': 1,\n",
    "    'eta':.3,\n",
    "    'subsample': 1,\n",
    "    'colsample_bytree': 1,\n",
    "    # Other parameters\n",
    "    'objective':'reg:squarederror',\n",
    "}\n",
    "\n",
    "\n",
    "params['eval_metric'] = \"rmse\"\n",
    "num_boost_round = 999"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tTest-rmse:7753.65625\n",
      "Will train until Test-rmse hasn't improved in 10 rounds.\n",
      "[1]\tTest-rmse:5930.35205\n",
      "[2]\tTest-rmse:4769.00244\n",
      "[3]\tTest-rmse:4027.15063\n",
      "[4]\tTest-rmse:3586.85400\n",
      "[5]\tTest-rmse:3348.09839\n",
      "[6]\tTest-rmse:3185.80688\n",
      "[7]\tTest-rmse:3085.66431\n",
      "[8]\tTest-rmse:3035.15942\n",
      "[9]\tTest-rmse:3002.64429\n",
      "[10]\tTest-rmse:2969.64380\n",
      "[11]\tTest-rmse:2954.43018\n",
      "[12]\tTest-rmse:2940.31592\n",
      "[13]\tTest-rmse:2928.56982\n",
      "[14]\tTest-rmse:2923.85156\n",
      "[15]\tTest-rmse:2912.66113\n",
      "[16]\tTest-rmse:2908.83594\n",
      "[17]\tTest-rmse:2900.41846\n",
      "[18]\tTest-rmse:2896.76709\n",
      "[19]\tTest-rmse:2874.67773\n",
      "[20]\tTest-rmse:2870.76929\n",
      "[21]\tTest-rmse:2866.29443\n",
      "[22]\tTest-rmse:2861.17773\n",
      "[23]\tTest-rmse:2860.06372\n",
      "[24]\tTest-rmse:2851.43481\n",
      "[25]\tTest-rmse:2850.10791\n",
      "[26]\tTest-rmse:2842.15869\n",
      "[27]\tTest-rmse:2836.49390\n",
      "[28]\tTest-rmse:2832.87061\n",
      "[29]\tTest-rmse:2831.68188\n",
      "[30]\tTest-rmse:2825.16406\n",
      "[31]\tTest-rmse:2821.50781\n",
      "[32]\tTest-rmse:2818.22144\n",
      "[33]\tTest-rmse:2813.50757\n",
      "[34]\tTest-rmse:2809.32959\n",
      "[35]\tTest-rmse:2808.22412\n",
      "[36]\tTest-rmse:2807.19849\n",
      "[37]\tTest-rmse:2805.14819\n",
      "[38]\tTest-rmse:2802.42456\n",
      "[39]\tTest-rmse:2799.29272\n",
      "[40]\tTest-rmse:2798.17041\n",
      "[41]\tTest-rmse:2794.95410\n",
      "[42]\tTest-rmse:2789.10693\n",
      "[43]\tTest-rmse:2787.27661\n",
      "[44]\tTest-rmse:2784.07519\n",
      "[45]\tTest-rmse:2782.86694\n",
      "[46]\tTest-rmse:2778.72363\n",
      "[47]\tTest-rmse:2775.02637\n",
      "[48]\tTest-rmse:2773.46045\n",
      "[49]\tTest-rmse:2770.43774\n",
      "[50]\tTest-rmse:2767.57105\n",
      "[51]\tTest-rmse:2765.29077\n",
      "[52]\tTest-rmse:2762.44702\n",
      "[53]\tTest-rmse:2759.14038\n",
      "[54]\tTest-rmse:2757.89111\n",
      "[55]\tTest-rmse:2755.83862\n",
      "[56]\tTest-rmse:2754.56812\n",
      "[57]\tTest-rmse:2753.99536\n",
      "[58]\tTest-rmse:2752.09277\n",
      "[59]\tTest-rmse:2751.00317\n",
      "[60]\tTest-rmse:2748.97217\n",
      "[61]\tTest-rmse:2748.70776\n",
      "[62]\tTest-rmse:2746.70776\n",
      "[63]\tTest-rmse:2742.81592\n",
      "[64]\tTest-rmse:2741.16211\n",
      "[65]\tTest-rmse:2739.90820\n",
      "[66]\tTest-rmse:2738.18628\n",
      "[67]\tTest-rmse:2735.71948\n",
      "[68]\tTest-rmse:2730.12793\n",
      "[69]\tTest-rmse:2724.97290\n",
      "[70]\tTest-rmse:2723.82324\n",
      "[71]\tTest-rmse:2721.86328\n",
      "[72]\tTest-rmse:2721.00220\n",
      "[73]\tTest-rmse:2718.66309\n",
      "[74]\tTest-rmse:2716.28247\n",
      "[75]\tTest-rmse:2712.99170\n",
      "[76]\tTest-rmse:2711.07056\n",
      "[77]\tTest-rmse:2709.59180\n",
      "[78]\tTest-rmse:2708.97656\n",
      "[79]\tTest-rmse:2708.40820\n",
      "[80]\tTest-rmse:2706.07642\n",
      "[81]\tTest-rmse:2704.33447\n",
      "[82]\tTest-rmse:2701.89380\n",
      "[83]\tTest-rmse:2700.85254\n",
      "[84]\tTest-rmse:2699.56055\n",
      "[85]\tTest-rmse:2696.41528\n",
      "[86]\tTest-rmse:2696.00928\n",
      "[87]\tTest-rmse:2695.26269\n",
      "[88]\tTest-rmse:2692.47876\n",
      "[89]\tTest-rmse:2691.28491\n",
      "[90]\tTest-rmse:2689.32739\n",
      "[91]\tTest-rmse:2688.14307\n",
      "[92]\tTest-rmse:2684.49683\n",
      "[93]\tTest-rmse:2683.07056\n",
      "[94]\tTest-rmse:2682.53833\n",
      "[95]\tTest-rmse:2681.57397\n",
      "[96]\tTest-rmse:2680.80444\n",
      "[97]\tTest-rmse:2679.69849\n",
      "[98]\tTest-rmse:2677.56104\n",
      "[99]\tTest-rmse:2676.17578\n",
      "[100]\tTest-rmse:2675.58301\n",
      "[101]\tTest-rmse:2673.49902\n",
      "[102]\tTest-rmse:2672.21118\n",
      "[103]\tTest-rmse:2671.58643\n",
      "[104]\tTest-rmse:2670.30493\n",
      "[105]\tTest-rmse:2669.42407\n",
      "[106]\tTest-rmse:2669.00855\n",
      "[107]\tTest-rmse:2668.00342\n",
      "[108]\tTest-rmse:2667.54541\n",
      "[109]\tTest-rmse:2666.71777\n",
      "[110]\tTest-rmse:2665.51343\n",
      "[111]\tTest-rmse:2664.85400\n",
      "[112]\tTest-rmse:2664.83643\n",
      "[113]\tTest-rmse:2664.97876\n",
      "[114]\tTest-rmse:2663.59985\n",
      "[115]\tTest-rmse:2660.82373\n",
      "[116]\tTest-rmse:2659.51758\n",
      "[117]\tTest-rmse:2658.59644\n",
      "[118]\tTest-rmse:2656.52783\n",
      "[119]\tTest-rmse:2654.00073\n",
      "[120]\tTest-rmse:2653.00562\n",
      "[121]\tTest-rmse:2650.12671\n",
      "[122]\tTest-rmse:2649.46387\n",
      "[123]\tTest-rmse:2648.99829\n",
      "[124]\tTest-rmse:2648.41675\n",
      "[125]\tTest-rmse:2647.36743\n",
      "[126]\tTest-rmse:2645.26807\n",
      "[127]\tTest-rmse:2644.27295\n",
      "[128]\tTest-rmse:2643.74731\n",
      "[129]\tTest-rmse:2643.20923\n",
      "[130]\tTest-rmse:2641.60400\n",
      "[131]\tTest-rmse:2640.45801\n",
      "[132]\tTest-rmse:2640.48560\n",
      "[133]\tTest-rmse:2639.57080\n",
      "[134]\tTest-rmse:2639.06128\n",
      "[135]\tTest-rmse:2638.40210\n",
      "[136]\tTest-rmse:2637.84521\n",
      "[137]\tTest-rmse:2636.94409\n",
      "[138]\tTest-rmse:2634.89160\n",
      "[139]\tTest-rmse:2633.40112\n",
      "[140]\tTest-rmse:2632.32446\n",
      "[141]\tTest-rmse:2632.15186\n",
      "[142]\tTest-rmse:2630.91138\n",
      "[143]\tTest-rmse:2630.42822\n",
      "[144]\tTest-rmse:2630.38965\n",
      "[145]\tTest-rmse:2629.87695\n",
      "[146]\tTest-rmse:2629.37256\n",
      "[147]\tTest-rmse:2627.49731\n",
      "[148]\tTest-rmse:2627.45386\n",
      "[149]\tTest-rmse:2626.59204\n",
      "[150]\tTest-rmse:2626.05347\n",
      "[151]\tTest-rmse:2625.51929\n",
      "[152]\tTest-rmse:2625.58667\n",
      "[153]\tTest-rmse:2625.22583\n",
      "[154]\tTest-rmse:2625.08911\n",
      "[155]\tTest-rmse:2624.69287\n",
      "[156]\tTest-rmse:2623.99048\n",
      "[157]\tTest-rmse:2623.36011\n",
      "[158]\tTest-rmse:2623.02124\n",
      "[159]\tTest-rmse:2622.30200\n",
      "[160]\tTest-rmse:2621.41602\n",
      "[161]\tTest-rmse:2621.47388\n",
      "[162]\tTest-rmse:2620.24878\n",
      "[163]\tTest-rmse:2619.44653\n",
      "[164]\tTest-rmse:2618.38184\n",
      "[165]\tTest-rmse:2617.83325\n",
      "[166]\tTest-rmse:2617.81323\n",
      "[167]\tTest-rmse:2616.63086\n",
      "[168]\tTest-rmse:2616.26025\n",
      "[169]\tTest-rmse:2616.06763\n",
      "[170]\tTest-rmse:2615.53564\n",
      "[171]\tTest-rmse:2614.89624\n",
      "[172]\tTest-rmse:2614.53516\n",
      "[173]\tTest-rmse:2614.24780\n",
      "[174]\tTest-rmse:2613.55151\n",
      "[175]\tTest-rmse:2612.36108\n",
      "[176]\tTest-rmse:2611.42310\n",
      "[177]\tTest-rmse:2610.50830\n",
      "[178]\tTest-rmse:2610.18921\n",
      "[179]\tTest-rmse:2609.84448\n",
      "[180]\tTest-rmse:2608.69165\n",
      "[181]\tTest-rmse:2608.74805\n",
      "[182]\tTest-rmse:2607.91528\n",
      "[183]\tTest-rmse:2606.49976\n",
      "[184]\tTest-rmse:2605.71509\n",
      "[185]\tTest-rmse:2605.00635\n",
      "[186]\tTest-rmse:2604.02051\n",
      "[187]\tTest-rmse:2602.94897\n",
      "[188]\tTest-rmse:2602.05371\n",
      "[189]\tTest-rmse:2601.69263\n",
      "[190]\tTest-rmse:2601.04736\n",
      "[191]\tTest-rmse:2600.66992\n",
      "[192]\tTest-rmse:2599.76392\n",
      "[193]\tTest-rmse:2599.51318\n",
      "[194]\tTest-rmse:2598.99414\n",
      "[195]\tTest-rmse:2598.41675\n",
      "[196]\tTest-rmse:2598.04736\n",
      "[197]\tTest-rmse:2597.94214\n",
      "[198]\tTest-rmse:2597.25122\n",
      "[199]\tTest-rmse:2596.58545\n",
      "[200]\tTest-rmse:2596.33643\n",
      "[201]\tTest-rmse:2595.64575\n",
      "[202]\tTest-rmse:2595.49585\n",
      "[203]\tTest-rmse:2595.40308\n",
      "[204]\tTest-rmse:2595.22656\n",
      "[205]\tTest-rmse:2594.96753\n",
      "[206]\tTest-rmse:2594.61450\n",
      "[207]\tTest-rmse:2594.18262\n",
      "[208]\tTest-rmse:2593.13062\n",
      "[209]\tTest-rmse:2592.25586\n",
      "[210]\tTest-rmse:2591.36255\n",
      "[211]\tTest-rmse:2590.53906\n",
      "[212]\tTest-rmse:2590.35986\n",
      "[213]\tTest-rmse:2590.22730\n",
      "[214]\tTest-rmse:2589.92114\n",
      "[215]\tTest-rmse:2589.25098\n",
      "[216]\tTest-rmse:2588.76318\n",
      "[217]\tTest-rmse:2588.53296\n",
      "[218]\tTest-rmse:2588.39136\n",
      "[219]\tTest-rmse:2588.04175\n",
      "[220]\tTest-rmse:2587.85889\n",
      "[221]\tTest-rmse:2587.22778\n",
      "[222]\tTest-rmse:2586.46143\n",
      "[223]\tTest-rmse:2585.67017\n",
      "[224]\tTest-rmse:2585.17895\n",
      "[225]\tTest-rmse:2584.63355\n",
      "[226]\tTest-rmse:2583.70142\n",
      "[227]\tTest-rmse:2583.26880\n",
      "[228]\tTest-rmse:2582.83325\n",
      "[229]\tTest-rmse:2582.41284\n",
      "[230]\tTest-rmse:2581.37207\n",
      "[231]\tTest-rmse:2580.08081\n",
      "[232]\tTest-rmse:2579.69824\n",
      "[233]\tTest-rmse:2578.97900\n",
      "[234]\tTest-rmse:2578.65112\n",
      "[235]\tTest-rmse:2578.70703\n",
      "[236]\tTest-rmse:2578.14160\n",
      "[237]\tTest-rmse:2578.09521\n",
      "[238]\tTest-rmse:2577.92432\n",
      "[239]\tTest-rmse:2577.51856\n",
      "[240]\tTest-rmse:2577.33813\n",
      "[241]\tTest-rmse:2576.86841\n",
      "[242]\tTest-rmse:2576.51489\n",
      "[243]\tTest-rmse:2576.51416\n",
      "[244]\tTest-rmse:2576.14819\n",
      "[245]\tTest-rmse:2575.68433\n",
      "[246]\tTest-rmse:2575.50855\n",
      "[247]\tTest-rmse:2575.31128\n",
      "[248]\tTest-rmse:2574.96167\n",
      "[249]\tTest-rmse:2574.87402\n",
      "[250]\tTest-rmse:2574.42529\n",
      "[251]\tTest-rmse:2574.40186\n",
      "[252]\tTest-rmse:2573.95459\n",
      "[253]\tTest-rmse:2573.83276\n",
      "[254]\tTest-rmse:2573.70190\n",
      "[255]\tTest-rmse:2573.12183\n",
      "[256]\tTest-rmse:2572.52563\n",
      "[257]\tTest-rmse:2571.81006\n",
      "[258]\tTest-rmse:2571.38330\n",
      "[259]\tTest-rmse:2571.08862\n",
      "[260]\tTest-rmse:2570.52930\n",
      "[261]\tTest-rmse:2570.22070\n",
      "[262]\tTest-rmse:2569.87598\n",
      "[263]\tTest-rmse:2569.36572\n",
      "[264]\tTest-rmse:2569.34009\n",
      "[265]\tTest-rmse:2569.12427\n",
      "[266]\tTest-rmse:2568.73828\n",
      "[267]\tTest-rmse:2568.38501\n",
      "[268]\tTest-rmse:2568.43164\n",
      "[269]\tTest-rmse:2568.37646\n",
      "[270]\tTest-rmse:2567.81250\n",
      "[271]\tTest-rmse:2567.47827\n",
      "[272]\tTest-rmse:2567.02881\n",
      "[273]\tTest-rmse:2566.95239\n",
      "[274]\tTest-rmse:2566.80933\n",
      "[275]\tTest-rmse:2566.41113\n",
      "[276]\tTest-rmse:2566.10693\n",
      "[277]\tTest-rmse:2565.90820\n",
      "[278]\tTest-rmse:2565.81152\n",
      "[279]\tTest-rmse:2565.68530\n",
      "[280]\tTest-rmse:2565.68335\n",
      "[281]\tTest-rmse:2564.94434\n",
      "[282]\tTest-rmse:2564.87817\n",
      "[283]\tTest-rmse:2564.51978\n",
      "[284]\tTest-rmse:2564.39990\n",
      "[285]\tTest-rmse:2564.00806\n",
      "[286]\tTest-rmse:2563.85864\n",
      "[287]\tTest-rmse:2563.23242\n",
      "[288]\tTest-rmse:2563.20703\n",
      "[289]\tTest-rmse:2562.96069\n",
      "[290]\tTest-rmse:2562.79834\n",
      "[291]\tTest-rmse:2562.69873\n",
      "[292]\tTest-rmse:2562.53052\n",
      "[293]\tTest-rmse:2562.29224\n",
      "[294]\tTest-rmse:2562.03076\n",
      "[295]\tTest-rmse:2561.68433\n",
      "[296]\tTest-rmse:2561.42310\n",
      "[297]\tTest-rmse:2560.94531\n",
      "[298]\tTest-rmse:2560.81323\n",
      "[299]\tTest-rmse:2560.48511\n",
      "[300]\tTest-rmse:2560.34521\n",
      "[301]\tTest-rmse:2560.00684\n",
      "[302]\tTest-rmse:2559.89209\n",
      "[303]\tTest-rmse:2559.67871\n",
      "[304]\tTest-rmse:2559.75098\n",
      "[305]\tTest-rmse:2559.54614\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[306]\tTest-rmse:2559.48071\n",
      "[307]\tTest-rmse:2559.15845\n",
      "[308]\tTest-rmse:2559.02417\n",
      "[309]\tTest-rmse:2558.67456\n",
      "[310]\tTest-rmse:2557.94775\n",
      "[311]\tTest-rmse:2557.97192\n",
      "[312]\tTest-rmse:2557.73975\n",
      "[313]\tTest-rmse:2557.64819\n",
      "[314]\tTest-rmse:2557.68970\n",
      "[315]\tTest-rmse:2557.48755\n",
      "[316]\tTest-rmse:2557.52270\n",
      "[317]\tTest-rmse:2557.03369\n",
      "[318]\tTest-rmse:2556.60034\n",
      "[319]\tTest-rmse:2556.49951\n",
      "[320]\tTest-rmse:2556.16455\n",
      "[321]\tTest-rmse:2555.59863\n",
      "[322]\tTest-rmse:2555.44800\n",
      "[323]\tTest-rmse:2555.47778\n",
      "[324]\tTest-rmse:2554.99536\n",
      "[325]\tTest-rmse:2554.66772\n",
      "[326]\tTest-rmse:2554.58569\n",
      "[327]\tTest-rmse:2554.47852\n",
      "[328]\tTest-rmse:2554.38208\n",
      "[329]\tTest-rmse:2554.16553\n",
      "[330]\tTest-rmse:2554.01733\n",
      "[331]\tTest-rmse:2553.97876\n",
      "[332]\tTest-rmse:2553.73218\n",
      "[333]\tTest-rmse:2553.72876\n",
      "[334]\tTest-rmse:2553.51392\n",
      "[335]\tTest-rmse:2553.28442\n",
      "[336]\tTest-rmse:2552.99512\n",
      "[337]\tTest-rmse:2552.59375\n",
      "[338]\tTest-rmse:2552.42895\n",
      "[339]\tTest-rmse:2552.25830\n",
      "[340]\tTest-rmse:2552.02319\n",
      "[341]\tTest-rmse:2551.92651\n",
      "[342]\tTest-rmse:2551.81445\n",
      "[343]\tTest-rmse:2550.98364\n",
      "[344]\tTest-rmse:2550.90161\n",
      "[345]\tTest-rmse:2550.76904\n",
      "[346]\tTest-rmse:2550.41162\n",
      "[347]\tTest-rmse:2550.31445\n",
      "[348]\tTest-rmse:2550.13989\n",
      "[349]\tTest-rmse:2549.82080\n",
      "[350]\tTest-rmse:2549.94019\n",
      "[351]\tTest-rmse:2549.88379\n",
      "[352]\tTest-rmse:2549.71533\n",
      "[353]\tTest-rmse:2549.23096\n",
      "[354]\tTest-rmse:2549.12720\n",
      "[355]\tTest-rmse:2548.99121\n",
      "[356]\tTest-rmse:2548.41431\n",
      "[357]\tTest-rmse:2548.05200\n",
      "[358]\tTest-rmse:2547.80640\n",
      "[359]\tTest-rmse:2547.68921\n",
      "[360]\tTest-rmse:2547.52686\n",
      "[361]\tTest-rmse:2546.78442\n",
      "[362]\tTest-rmse:2546.64478\n",
      "[363]\tTest-rmse:2546.48242\n",
      "[364]\tTest-rmse:2546.18408\n",
      "[365]\tTest-rmse:2546.02954\n",
      "[366]\tTest-rmse:2545.98389\n",
      "[367]\tTest-rmse:2545.89575\n",
      "[368]\tTest-rmse:2545.82471\n",
      "[369]\tTest-rmse:2545.61279\n",
      "[370]\tTest-rmse:2545.14380\n",
      "[371]\tTest-rmse:2544.88867\n",
      "[372]\tTest-rmse:2544.68213\n",
      "[373]\tTest-rmse:2544.71216\n",
      "[374]\tTest-rmse:2544.71631\n",
      "[375]\tTest-rmse:2544.57251\n",
      "[376]\tTest-rmse:2544.57446\n",
      "[377]\tTest-rmse:2544.40796\n",
      "[378]\tTest-rmse:2544.36890\n",
      "[379]\tTest-rmse:2544.17798\n",
      "[380]\tTest-rmse:2543.92798\n",
      "[381]\tTest-rmse:2543.60815\n",
      "[382]\tTest-rmse:2543.57227\n",
      "[383]\tTest-rmse:2543.36792\n",
      "[384]\tTest-rmse:2543.34424\n",
      "[385]\tTest-rmse:2543.24268\n",
      "[386]\tTest-rmse:2543.06665\n",
      "[387]\tTest-rmse:2542.99487\n",
      "[388]\tTest-rmse:2542.73804\n",
      "[389]\tTest-rmse:2542.46289\n",
      "[390]\tTest-rmse:2542.30908\n",
      "[391]\tTest-rmse:2541.89136\n",
      "[392]\tTest-rmse:2541.87329\n",
      "[393]\tTest-rmse:2541.46582\n",
      "[394]\tTest-rmse:2540.94189\n",
      "[395]\tTest-rmse:2540.92383\n",
      "[396]\tTest-rmse:2540.86743\n",
      "[397]\tTest-rmse:2540.84985\n",
      "[398]\tTest-rmse:2540.88843\n",
      "[399]\tTest-rmse:2540.65063\n",
      "[400]\tTest-rmse:2540.43066\n",
      "[401]\tTest-rmse:2540.25269\n",
      "[402]\tTest-rmse:2540.35767\n",
      "[403]\tTest-rmse:2540.16455\n",
      "[404]\tTest-rmse:2539.88110\n",
      "[405]\tTest-rmse:2539.77075\n",
      "[406]\tTest-rmse:2539.48682\n",
      "[407]\tTest-rmse:2539.06445\n",
      "[408]\tTest-rmse:2538.90137\n",
      "[409]\tTest-rmse:2538.74634\n",
      "[410]\tTest-rmse:2538.59888\n",
      "[411]\tTest-rmse:2538.65527\n",
      "[412]\tTest-rmse:2538.43652\n",
      "[413]\tTest-rmse:2538.35132\n",
      "[414]\tTest-rmse:2538.43335\n",
      "[415]\tTest-rmse:2538.42529\n",
      "[416]\tTest-rmse:2538.40015\n",
      "[417]\tTest-rmse:2538.43555\n",
      "[418]\tTest-rmse:2538.36328\n",
      "[419]\tTest-rmse:2538.19727\n",
      "[420]\tTest-rmse:2538.13013\n",
      "[421]\tTest-rmse:2537.94897\n",
      "[422]\tTest-rmse:2537.95972\n",
      "[423]\tTest-rmse:2537.85352\n",
      "[424]\tTest-rmse:2537.89062\n",
      "[425]\tTest-rmse:2537.88672\n",
      "[426]\tTest-rmse:2537.83252\n",
      "[427]\tTest-rmse:2537.68018\n",
      "[428]\tTest-rmse:2537.59570\n",
      "[429]\tTest-rmse:2537.63428\n",
      "[430]\tTest-rmse:2537.56836\n",
      "[431]\tTest-rmse:2537.52344\n",
      "[432]\tTest-rmse:2537.42407\n",
      "[433]\tTest-rmse:2537.17188\n",
      "[434]\tTest-rmse:2537.13818\n",
      "[435]\tTest-rmse:2536.80469\n",
      "[436]\tTest-rmse:2536.68286\n",
      "[437]\tTest-rmse:2536.67993\n",
      "[438]\tTest-rmse:2536.66748\n",
      "[439]\tTest-rmse:2536.64551\n",
      "[440]\tTest-rmse:2536.33081\n",
      "[441]\tTest-rmse:2536.17725\n",
      "[442]\tTest-rmse:2536.13696\n",
      "[443]\tTest-rmse:2535.99072\n",
      "[444]\tTest-rmse:2535.46436\n",
      "[445]\tTest-rmse:2535.42358\n",
      "[446]\tTest-rmse:2535.30957\n",
      "[447]\tTest-rmse:2534.97168\n",
      "[448]\tTest-rmse:2534.87476\n",
      "[449]\tTest-rmse:2534.66235\n",
      "[450]\tTest-rmse:2534.55981\n",
      "[451]\tTest-rmse:2534.62061\n",
      "[452]\tTest-rmse:2534.62500\n",
      "[453]\tTest-rmse:2534.58301\n",
      "[454]\tTest-rmse:2534.51001\n",
      "[455]\tTest-rmse:2534.43384\n",
      "[456]\tTest-rmse:2534.27930\n",
      "[457]\tTest-rmse:2534.21436\n",
      "[458]\tTest-rmse:2534.07983\n",
      "[459]\tTest-rmse:2534.04004\n",
      "[460]\tTest-rmse:2534.05200\n",
      "[461]\tTest-rmse:2534.15601\n",
      "[462]\tTest-rmse:2533.97339\n",
      "[463]\tTest-rmse:2533.80395\n",
      "[464]\tTest-rmse:2533.76685\n",
      "[465]\tTest-rmse:2533.76416\n",
      "[466]\tTest-rmse:2533.57275\n",
      "[467]\tTest-rmse:2533.58569\n",
      "[468]\tTest-rmse:2533.51904\n",
      "[469]\tTest-rmse:2533.38769\n",
      "[470]\tTest-rmse:2533.47363\n",
      "[471]\tTest-rmse:2533.49145\n",
      "[472]\tTest-rmse:2533.40454\n",
      "[473]\tTest-rmse:2533.42407\n",
      "[474]\tTest-rmse:2533.28809\n",
      "[475]\tTest-rmse:2533.01440\n",
      "[476]\tTest-rmse:2532.91650\n",
      "[477]\tTest-rmse:2532.44751\n",
      "[478]\tTest-rmse:2532.09058\n",
      "[479]\tTest-rmse:2531.93823\n",
      "[480]\tTest-rmse:2531.74829\n",
      "[481]\tTest-rmse:2531.71265\n",
      "[482]\tTest-rmse:2531.60742\n",
      "[483]\tTest-rmse:2531.68774\n",
      "[484]\tTest-rmse:2531.49512\n",
      "[485]\tTest-rmse:2531.27295\n",
      "[486]\tTest-rmse:2531.24170\n",
      "[487]\tTest-rmse:2531.06128\n",
      "[488]\tTest-rmse:2530.86133\n",
      "[489]\tTest-rmse:2530.87158\n",
      "[490]\tTest-rmse:2530.90308\n",
      "[491]\tTest-rmse:2530.53271\n",
      "[492]\tTest-rmse:2530.48682\n",
      "[493]\tTest-rmse:2530.19019\n",
      "[494]\tTest-rmse:2529.80054\n",
      "[495]\tTest-rmse:2529.59448\n",
      "[496]\tTest-rmse:2529.51099\n",
      "[497]\tTest-rmse:2529.33594\n",
      "[498]\tTest-rmse:2529.21973\n",
      "[499]\tTest-rmse:2529.03955\n",
      "[500]\tTest-rmse:2528.95435\n",
      "[501]\tTest-rmse:2528.93433\n",
      "[502]\tTest-rmse:2528.96240\n",
      "[503]\tTest-rmse:2528.76514\n",
      "[504]\tTest-rmse:2528.61255\n",
      "[505]\tTest-rmse:2528.47290\n",
      "[506]\tTest-rmse:2528.36426\n",
      "[507]\tTest-rmse:2528.26025\n",
      "[508]\tTest-rmse:2528.18945\n",
      "[509]\tTest-rmse:2527.82617\n",
      "[510]\tTest-rmse:2527.68799\n",
      "[511]\tTest-rmse:2527.50684\n",
      "[512]\tTest-rmse:2527.50146\n",
      "[513]\tTest-rmse:2527.13159\n",
      "[514]\tTest-rmse:2527.13623\n",
      "[515]\tTest-rmse:2527.14795\n",
      "[516]\tTest-rmse:2527.06641\n",
      "[517]\tTest-rmse:2526.96411\n",
      "[518]\tTest-rmse:2527.01392\n",
      "[519]\tTest-rmse:2526.92969\n",
      "[520]\tTest-rmse:2526.92895\n",
      "[521]\tTest-rmse:2526.90527\n",
      "[522]\tTest-rmse:2526.77710\n",
      "[523]\tTest-rmse:2526.67432\n",
      "[524]\tTest-rmse:2526.56030\n",
      "[525]\tTest-rmse:2526.43042\n",
      "[526]\tTest-rmse:2526.34863\n",
      "[527]\tTest-rmse:2526.37061\n",
      "[528]\tTest-rmse:2526.28516\n",
      "[529]\tTest-rmse:2526.18823\n",
      "[530]\tTest-rmse:2526.21362\n",
      "[531]\tTest-rmse:2526.05591\n",
      "[532]\tTest-rmse:2526.06934\n",
      "[533]\tTest-rmse:2525.99512\n",
      "[534]\tTest-rmse:2525.96191\n",
      "[535]\tTest-rmse:2525.96216\n",
      "[536]\tTest-rmse:2525.99902\n",
      "[537]\tTest-rmse:2525.96313\n",
      "[538]\tTest-rmse:2526.04102\n",
      "[539]\tTest-rmse:2525.89380\n",
      "[540]\tTest-rmse:2525.93896\n",
      "[541]\tTest-rmse:2525.88916\n",
      "[542]\tTest-rmse:2525.84521\n",
      "[543]\tTest-rmse:2525.78149\n",
      "[544]\tTest-rmse:2525.68140\n",
      "[545]\tTest-rmse:2525.66675\n",
      "[546]\tTest-rmse:2525.67529\n",
      "[547]\tTest-rmse:2525.60254\n",
      "[548]\tTest-rmse:2525.51880\n",
      "[549]\tTest-rmse:2525.54810\n",
      "[550]\tTest-rmse:2525.38086\n",
      "[551]\tTest-rmse:2525.34595\n",
      "[552]\tTest-rmse:2525.21606\n",
      "[553]\tTest-rmse:2525.11938\n",
      "[554]\tTest-rmse:2525.03101\n",
      "[555]\tTest-rmse:2524.95654\n",
      "[556]\tTest-rmse:2524.74634\n",
      "[557]\tTest-rmse:2524.74951\n",
      "[558]\tTest-rmse:2524.74048\n",
      "[559]\tTest-rmse:2524.59229\n",
      "[560]\tTest-rmse:2524.37158\n",
      "[561]\tTest-rmse:2524.13892\n",
      "[562]\tTest-rmse:2523.95093\n",
      "[563]\tTest-rmse:2523.96216\n",
      "[564]\tTest-rmse:2523.88501\n",
      "[565]\tTest-rmse:2523.77612\n",
      "[566]\tTest-rmse:2523.71313\n",
      "[567]\tTest-rmse:2523.71753\n",
      "[568]\tTest-rmse:2523.67017\n",
      "[569]\tTest-rmse:2523.42822\n",
      "[570]\tTest-rmse:2523.35620\n",
      "[571]\tTest-rmse:2523.28052\n",
      "[572]\tTest-rmse:2523.25415\n",
      "[573]\tTest-rmse:2523.21655\n",
      "[574]\tTest-rmse:2523.20630\n",
      "[575]\tTest-rmse:2523.20825\n",
      "[576]\tTest-rmse:2523.02319\n",
      "[577]\tTest-rmse:2522.69409\n",
      "[578]\tTest-rmse:2522.54907\n",
      "[579]\tTest-rmse:2522.33398\n",
      "[580]\tTest-rmse:2522.16577\n",
      "[581]\tTest-rmse:2522.17944\n",
      "[582]\tTest-rmse:2521.93872\n",
      "[583]\tTest-rmse:2521.90942\n",
      "[584]\tTest-rmse:2521.66821\n",
      "[585]\tTest-rmse:2521.50830\n",
      "[586]\tTest-rmse:2521.47485\n",
      "[587]\tTest-rmse:2521.46533\n",
      "[588]\tTest-rmse:2521.47730\n",
      "[589]\tTest-rmse:2521.38599\n",
      "[590]\tTest-rmse:2521.13062\n",
      "[591]\tTest-rmse:2521.03980\n",
      "[592]\tTest-rmse:2520.79663\n",
      "[593]\tTest-rmse:2520.75635\n",
      "[594]\tTest-rmse:2520.75391\n",
      "[595]\tTest-rmse:2520.64233\n",
      "[596]\tTest-rmse:2520.57422\n",
      "[597]\tTest-rmse:2520.46436\n",
      "[598]\tTest-rmse:2520.31641\n",
      "[599]\tTest-rmse:2520.23462\n",
      "[600]\tTest-rmse:2520.26489\n",
      "[601]\tTest-rmse:2520.34888\n",
      "[602]\tTest-rmse:2520.35449\n",
      "[603]\tTest-rmse:2520.29907\n",
      "[604]\tTest-rmse:2520.32739\n",
      "[605]\tTest-rmse:2520.24243\n",
      "[606]\tTest-rmse:2520.19800\n",
      "[607]\tTest-rmse:2520.14307\n",
      "[608]\tTest-rmse:2519.95996\n",
      "[609]\tTest-rmse:2519.92383\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[610]\tTest-rmse:2519.82324\n",
      "[611]\tTest-rmse:2519.79248\n",
      "[612]\tTest-rmse:2519.76343\n",
      "[613]\tTest-rmse:2519.65869\n",
      "[614]\tTest-rmse:2519.67969\n",
      "[615]\tTest-rmse:2519.63769\n",
      "[616]\tTest-rmse:2519.68408\n",
      "[617]\tTest-rmse:2519.59790\n",
      "[618]\tTest-rmse:2519.55786\n",
      "[619]\tTest-rmse:2519.51343\n",
      "[620]\tTest-rmse:2519.41382\n",
      "[621]\tTest-rmse:2519.51001\n",
      "[622]\tTest-rmse:2519.31274\n",
      "[623]\tTest-rmse:2519.27832\n",
      "[624]\tTest-rmse:2519.19580\n",
      "[625]\tTest-rmse:2519.07837\n",
      "[626]\tTest-rmse:2518.85132\n",
      "[627]\tTest-rmse:2518.57666\n",
      "[628]\tTest-rmse:2518.52319\n",
      "[629]\tTest-rmse:2518.55786\n",
      "[630]\tTest-rmse:2518.61035\n",
      "[631]\tTest-rmse:2518.42627\n",
      "[632]\tTest-rmse:2518.31812\n",
      "[633]\tTest-rmse:2518.18140\n",
      "[634]\tTest-rmse:2518.09668\n",
      "[635]\tTest-rmse:2518.03369\n",
      "[636]\tTest-rmse:2517.95850\n",
      "[637]\tTest-rmse:2517.93677\n",
      "[638]\tTest-rmse:2517.86108\n",
      "[639]\tTest-rmse:2517.57690\n",
      "[640]\tTest-rmse:2517.49805\n",
      "[641]\tTest-rmse:2517.46875\n",
      "[642]\tTest-rmse:2517.44141\n",
      "[643]\tTest-rmse:2517.33276\n",
      "[644]\tTest-rmse:2517.33325\n",
      "[645]\tTest-rmse:2517.30566\n",
      "[646]\tTest-rmse:2517.26758\n",
      "[647]\tTest-rmse:2517.30469\n",
      "[648]\tTest-rmse:2517.19507\n",
      "[649]\tTest-rmse:2516.96680\n",
      "[650]\tTest-rmse:2517.01514\n",
      "[651]\tTest-rmse:2516.90503\n",
      "[652]\tTest-rmse:2516.78345\n",
      "[653]\tTest-rmse:2516.82617\n",
      "[654]\tTest-rmse:2516.64917\n",
      "[655]\tTest-rmse:2516.64307\n",
      "[656]\tTest-rmse:2516.55762\n",
      "[657]\tTest-rmse:2516.50928\n",
      "[658]\tTest-rmse:2516.48950\n",
      "[659]\tTest-rmse:2516.50488\n",
      "[660]\tTest-rmse:2516.51196\n",
      "[661]\tTest-rmse:2516.48975\n",
      "[662]\tTest-rmse:2516.47510\n",
      "[663]\tTest-rmse:2516.51733\n",
      "[664]\tTest-rmse:2516.48169\n",
      "[665]\tTest-rmse:2516.41480\n",
      "[666]\tTest-rmse:2516.31201\n",
      "[667]\tTest-rmse:2516.29785\n",
      "[668]\tTest-rmse:2516.22070\n",
      "[669]\tTest-rmse:2516.24731\n",
      "[670]\tTest-rmse:2516.22485\n",
      "[671]\tTest-rmse:2516.10254\n",
      "[672]\tTest-rmse:2516.12231\n",
      "[673]\tTest-rmse:2516.11694\n",
      "[674]\tTest-rmse:2516.14429\n",
      "[675]\tTest-rmse:2516.05395\n",
      "[676]\tTest-rmse:2516.02832\n",
      "[677]\tTest-rmse:2515.86963\n",
      "[678]\tTest-rmse:2515.83594\n",
      "[679]\tTest-rmse:2515.86182\n",
      "[680]\tTest-rmse:2515.80884\n",
      "[681]\tTest-rmse:2515.61328\n",
      "[682]\tTest-rmse:2515.69116\n",
      "[683]\tTest-rmse:2515.69482\n",
      "[684]\tTest-rmse:2515.69531\n",
      "[685]\tTest-rmse:2515.65356\n",
      "[686]\tTest-rmse:2515.53882\n",
      "[687]\tTest-rmse:2515.67920\n",
      "[688]\tTest-rmse:2515.66699\n",
      "[689]\tTest-rmse:2515.57544\n",
      "[690]\tTest-rmse:2515.54394\n",
      "[691]\tTest-rmse:2515.48096\n",
      "[692]\tTest-rmse:2515.48926\n",
      "[693]\tTest-rmse:2515.50562\n",
      "[694]\tTest-rmse:2515.38330\n",
      "[695]\tTest-rmse:2515.45435\n",
      "[696]\tTest-rmse:2515.50269\n",
      "[697]\tTest-rmse:2515.57886\n",
      "[698]\tTest-rmse:2515.61816\n",
      "[699]\tTest-rmse:2515.55054\n",
      "[700]\tTest-rmse:2515.44922\n",
      "[701]\tTest-rmse:2515.52051\n",
      "[702]\tTest-rmse:2515.59033\n",
      "[703]\tTest-rmse:2515.47876\n",
      "[704]\tTest-rmse:2515.32422\n",
      "[705]\tTest-rmse:2515.29175\n",
      "[706]\tTest-rmse:2515.33398\n",
      "[707]\tTest-rmse:2515.29517\n",
      "[708]\tTest-rmse:2515.19873\n",
      "[709]\tTest-rmse:2515.09692\n",
      "[710]\tTest-rmse:2515.10230\n",
      "[711]\tTest-rmse:2515.07202\n",
      "[712]\tTest-rmse:2515.08423\n",
      "[713]\tTest-rmse:2514.93115\n",
      "[714]\tTest-rmse:2514.87793\n",
      "[715]\tTest-rmse:2514.78735\n",
      "[716]\tTest-rmse:2514.71069\n",
      "[717]\tTest-rmse:2514.74341\n",
      "[718]\tTest-rmse:2514.73291\n",
      "[719]\tTest-rmse:2514.69751\n",
      "[720]\tTest-rmse:2514.62573\n",
      "[721]\tTest-rmse:2514.77295\n",
      "[722]\tTest-rmse:2514.75439\n",
      "[723]\tTest-rmse:2514.78247\n",
      "[724]\tTest-rmse:2514.45019\n",
      "[725]\tTest-rmse:2514.47388\n",
      "[726]\tTest-rmse:2514.40747\n",
      "[727]\tTest-rmse:2514.40454\n",
      "[728]\tTest-rmse:2514.46094\n",
      "[729]\tTest-rmse:2514.42188\n",
      "[730]\tTest-rmse:2514.36353\n",
      "[731]\tTest-rmse:2514.23389\n",
      "[732]\tTest-rmse:2514.25269\n",
      "[733]\tTest-rmse:2514.31055\n",
      "[734]\tTest-rmse:2514.29248\n",
      "[735]\tTest-rmse:2514.19995\n",
      "[736]\tTest-rmse:2514.14770\n",
      "[737]\tTest-rmse:2514.03857\n",
      "[738]\tTest-rmse:2514.04663\n",
      "[739]\tTest-rmse:2514.00928\n",
      "[740]\tTest-rmse:2513.99976\n",
      "[741]\tTest-rmse:2513.91406\n",
      "[742]\tTest-rmse:2513.92285\n",
      "[743]\tTest-rmse:2514.00879\n",
      "[744]\tTest-rmse:2514.04028\n",
      "[745]\tTest-rmse:2514.08130\n",
      "[746]\tTest-rmse:2513.98828\n",
      "[747]\tTest-rmse:2514.05688\n",
      "[748]\tTest-rmse:2513.99316\n",
      "[749]\tTest-rmse:2513.90747\n",
      "[750]\tTest-rmse:2513.83520\n",
      "[751]\tTest-rmse:2513.85400\n",
      "[752]\tTest-rmse:2513.80151\n",
      "[753]\tTest-rmse:2513.82178\n",
      "[754]\tTest-rmse:2513.81250\n",
      "[755]\tTest-rmse:2513.78613\n",
      "[756]\tTest-rmse:2513.80518\n",
      "[757]\tTest-rmse:2513.71460\n",
      "[758]\tTest-rmse:2513.72949\n",
      "[759]\tTest-rmse:2513.74658\n",
      "[760]\tTest-rmse:2513.71606\n",
      "[761]\tTest-rmse:2513.50244\n",
      "[762]\tTest-rmse:2513.55347\n",
      "[763]\tTest-rmse:2513.49023\n",
      "[764]\tTest-rmse:2513.54761\n",
      "[765]\tTest-rmse:2513.55786\n",
      "[766]\tTest-rmse:2513.52246\n",
      "[767]\tTest-rmse:2513.40015\n",
      "[768]\tTest-rmse:2513.20288\n",
      "[769]\tTest-rmse:2513.01880\n",
      "[770]\tTest-rmse:2512.98462\n",
      "[771]\tTest-rmse:2512.75928\n",
      "[772]\tTest-rmse:2512.61914\n",
      "[773]\tTest-rmse:2512.51587\n",
      "[774]\tTest-rmse:2512.51001\n",
      "[775]\tTest-rmse:2512.48413\n",
      "[776]\tTest-rmse:2512.61255\n",
      "[777]\tTest-rmse:2512.59326\n",
      "[778]\tTest-rmse:2512.36914\n",
      "[779]\tTest-rmse:2512.44043\n",
      "[780]\tTest-rmse:2512.58838\n",
      "[781]\tTest-rmse:2512.57886\n",
      "[782]\tTest-rmse:2512.44800\n",
      "[783]\tTest-rmse:2512.14209\n",
      "[784]\tTest-rmse:2512.07764\n",
      "[785]\tTest-rmse:2512.07275\n",
      "[786]\tTest-rmse:2511.98731\n",
      "[787]\tTest-rmse:2511.99365\n",
      "[788]\tTest-rmse:2511.93701\n",
      "[789]\tTest-rmse:2511.78760\n",
      "[790]\tTest-rmse:2511.62207\n",
      "[791]\tTest-rmse:2511.59717\n",
      "[792]\tTest-rmse:2511.59497\n",
      "[793]\tTest-rmse:2511.45972\n",
      "[794]\tTest-rmse:2511.35571\n",
      "[795]\tTest-rmse:2511.32324\n",
      "[796]\tTest-rmse:2511.32764\n",
      "[797]\tTest-rmse:2511.35034\n",
      "[798]\tTest-rmse:2511.22070\n",
      "[799]\tTest-rmse:2511.21631\n",
      "[800]\tTest-rmse:2511.22339\n",
      "[801]\tTest-rmse:2511.19238\n",
      "[802]\tTest-rmse:2511.23975\n",
      "[803]\tTest-rmse:2511.26318\n",
      "[804]\tTest-rmse:2511.17432\n",
      "[805]\tTest-rmse:2511.09399\n",
      "[806]\tTest-rmse:2511.12085\n",
      "[807]\tTest-rmse:2511.02197\n",
      "[808]\tTest-rmse:2510.98340\n",
      "[809]\tTest-rmse:2510.89258\n",
      "[810]\tTest-rmse:2510.91406\n",
      "[811]\tTest-rmse:2511.02222\n",
      "[812]\tTest-rmse:2511.04028\n",
      "[813]\tTest-rmse:2511.00049\n",
      "[814]\tTest-rmse:2511.04102\n",
      "[815]\tTest-rmse:2511.13452\n",
      "[816]\tTest-rmse:2511.16455\n",
      "[817]\tTest-rmse:2511.13867\n",
      "[818]\tTest-rmse:2511.05640\n",
      "[819]\tTest-rmse:2511.06519\n",
      "Stopping. Best iteration:\n",
      "[809]\tTest-rmse:2510.89258\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = xgb.train(\n",
    "    params,\n",
    "    dtrain,\n",
    "    num_boost_round=num_boost_round,\n",
    "    evals=[(dtest, \"Test\")],\n",
    "    early_stopping_rounds=10\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Maximum Depth and Minimum Child Weight "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Takes a lot of time\n",
    "\n",
    "\n",
    "#Select a range of values for different parameters\n",
    "gridsearch_params = [\n",
    "    (max_depth, min_child_weight)\n",
    "    for max_depth in range(9,12)\n",
    "    for min_child_weight in range(5,8) #TRY GREATER VALUES > 60\n",
    "]\n",
    "#Initialize minimum rmse and the best parameters\n",
    "min_rmse  = float(\"Inf\")\n",
    "best_params = None\n",
    "for max_depth, min_child_weight in gridsearch_params:\n",
    "    print(\"CV with max_depth={}, min_child_weight={}\".format(\n",
    "                             max_depth,\n",
    "                             min_child_weight))\n",
    "    # Update our parameters\n",
    "    params['max_depth'] = max_depth\n",
    "    params['min_child_weight'] = min_child_weight\n",
    "    # Run CV\n",
    "    cv_results = xgb.cv(\n",
    "        params,\n",
    "        dtrain,\n",
    "        num_boost_round=num_boost_round,\n",
    "        seed=42,\n",
    "        nfold=5,\n",
    "        metrics={'rmse'},\n",
    "        early_stopping_rounds=5,\n",
    "        verbose_eval = True\n",
    "    )\n",
    "    # Update best RMSE\n",
    "    mean_rmse = cv_results['test-rmse-mean'].min()\n",
    "    boost_rounds = cv_results['test-rmse-mean'].argmin()\n",
    "    print(\"\\tRMSE {} for {} rounds\".format(mean_rmse, boost_rounds))\n",
    "    if mean_rmse < min_rmse:\n",
    "        min_rmse = mean_rmse\n",
    "        best_params = (max_depth,min_child_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['max_depth'] = 9\n",
    "params['min_child_weight'] = 7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Subsample and Colsample_bytree "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Takes a lot of time\n",
    "\n",
    "\n",
    "#Select a range of values for different parameters\n",
    "gridsearch_params = [(subsample, colsample)\n",
    "    for subsample in [i/10. for i in range(7,11)]\n",
    "    for colsample in [i/10. for i in range(7,11)]\n",
    "]\n",
    "#Initialize minimum rmse and the best parameters\n",
    "min_rmse = float(\"Inf\")\n",
    "best_params = None\n",
    "# We start by the largest values and go down to the smallest\n",
    "for subsample, colsample in reversed(gridsearch_params):\n",
    "    print(\"CV with subsample={}, colsample={}\".format(\n",
    "                             subsample,\n",
    "                             colsample))\n",
    "    # We update our parameters\n",
    "    params['subsample'] = subsample\n",
    "    params['colsample_bytree'] = colsample\n",
    "    # Run CV\n",
    "    cv_results = xgb.cv(\n",
    "        params,\n",
    "        dtrain,\n",
    "        num_boost_round=num_boost_round,\n",
    "        seed=42,\n",
    "        nfold=5,\n",
    "        metrics={'rmse'},\n",
    "        early_stopping_rounds=10\n",
    "    )\n",
    "    # Update best score\n",
    "    mean_rmse = cv_results['test-rmse-mean'].min()\n",
    "    boost_rounds = cv_results['test-rmse-mean'].argmin()\n",
    "    print(\"\\tRMSE {} for {} rounds\".format(mean_rmse, boost_rounds))\n",
    "    if mean_rmse < min_rmse:\n",
    "        min_rmse = mean_rmse\n",
    "        best_params = (subsample,colsample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['subsample'] = 1\n",
    "params['colsample_bytree'] = 0.7"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. ETA (Learning Rate) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Takes a lot of time\n",
    "\n",
    "\n",
    "min_rmse = float(\"Inf\")\n",
    "best_params = None\n",
    "for eta in [.3, .2, .1, .05, .01, .005]:\n",
    "    print(\"CV with eta={}\".format(eta))\n",
    "    # We update our parameters\n",
    "    params['eta'] = eta\n",
    "    # Run and time CV\n",
    "    cv_results = xgb.cv(\n",
    "            params,\n",
    "            dtrain,\n",
    "            num_boost_round=num_boost_round,\n",
    "            seed=42,\n",
    "            nfold=5,\n",
    "            metrics=['rmse'],\n",
    "            early_stopping_rounds=10\n",
    "          )\n",
    "    # Update best score\n",
    "    mean_rmse = cv_results['test-rmse-mean'].min()\n",
    "    boost_rounds = cv_results['test-rmse-mean'].argmin()\n",
    "    print(\"\\tRMSE {} for {} rounds\\n\".format(mean_rmse, boost_rounds))\n",
    "    if mean_rmse < min_rmse:\n",
    "        min_rmse = mean_rmse\n",
    "        best_params = eta\n",
    "print(\"Best params: {}\".format(best_params))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finalized Parameter Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'colsample_bytree': 0.7,\n",
    " 'eta': 0.2,\n",
    " 'eval_metric': 'rmse',\n",
    " 'max_depth': 9,\n",
    " 'min_child_weight': 66,\n",
    " 'objective': 'reg:squarederror',\n",
    " 'subsample': 1}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tTest-rmse:8656.07715\n",
      "Will train until Test-rmse hasn't improved in 10 rounds.\n",
      "[1]\tTest-rmse:7285.44141\n",
      "[2]\tTest-rmse:6153.22754\n",
      "[3]\tTest-rmse:5282.83594\n",
      "[4]\tTest-rmse:4647.38623\n",
      "[5]\tTest-rmse:4171.44238\n",
      "[6]\tTest-rmse:3781.28027\n",
      "[7]\tTest-rmse:3524.07202\n",
      "[8]\tTest-rmse:3331.38403\n",
      "[9]\tTest-rmse:3211.36353\n",
      "[10]\tTest-rmse:3126.53491\n",
      "[11]\tTest-rmse:3042.86768\n",
      "[12]\tTest-rmse:2976.89160\n",
      "[13]\tTest-rmse:2935.31445\n",
      "[14]\tTest-rmse:2901.69336\n",
      "[15]\tTest-rmse:2882.50024\n",
      "[16]\tTest-rmse:2871.74438\n",
      "[17]\tTest-rmse:2853.92310\n",
      "[18]\tTest-rmse:2847.78809\n",
      "[19]\tTest-rmse:2831.66284\n",
      "[20]\tTest-rmse:2820.27270\n",
      "[21]\tTest-rmse:2806.54492\n",
      "[22]\tTest-rmse:2801.78369\n",
      "[23]\tTest-rmse:2797.25732\n",
      "[24]\tTest-rmse:2792.64014\n",
      "[25]\tTest-rmse:2786.22461\n",
      "[26]\tTest-rmse:2782.07251\n",
      "[27]\tTest-rmse:2780.16797\n",
      "[28]\tTest-rmse:2777.12598\n",
      "[29]\tTest-rmse:2770.90771\n",
      "[30]\tTest-rmse:2764.07666\n",
      "[31]\tTest-rmse:2758.95459\n",
      "[32]\tTest-rmse:2756.15869\n",
      "[33]\tTest-rmse:2752.81396\n",
      "[34]\tTest-rmse:2751.79272\n",
      "[35]\tTest-rmse:2749.02319\n",
      "[36]\tTest-rmse:2743.32935\n",
      "[37]\tTest-rmse:2740.97461\n",
      "[38]\tTest-rmse:2736.45606\n",
      "[39]\tTest-rmse:2732.47534\n",
      "[40]\tTest-rmse:2729.35596\n",
      "[41]\tTest-rmse:2726.61304\n",
      "[42]\tTest-rmse:2721.52100\n",
      "[43]\tTest-rmse:2719.82617\n",
      "[44]\tTest-rmse:2717.49487\n",
      "[45]\tTest-rmse:2713.21265\n",
      "[46]\tTest-rmse:2707.69019\n",
      "[47]\tTest-rmse:2707.03149\n",
      "[48]\tTest-rmse:2705.64795\n",
      "[49]\tTest-rmse:2702.51660\n",
      "[50]\tTest-rmse:2696.35132\n",
      "[51]\tTest-rmse:2694.77612\n",
      "[52]\tTest-rmse:2693.31958\n",
      "[53]\tTest-rmse:2692.76099\n",
      "[54]\tTest-rmse:2690.99658\n",
      "[55]\tTest-rmse:2688.64648\n",
      "[56]\tTest-rmse:2687.94751\n",
      "[57]\tTest-rmse:2687.55981\n",
      "[58]\tTest-rmse:2685.36597\n",
      "[59]\tTest-rmse:2683.42529\n",
      "[60]\tTest-rmse:2678.71167\n",
      "[61]\tTest-rmse:2676.71069\n",
      "[62]\tTest-rmse:2676.42505\n",
      "[63]\tTest-rmse:2675.49023\n",
      "[64]\tTest-rmse:2671.60327\n",
      "[65]\tTest-rmse:2671.33789\n",
      "[66]\tTest-rmse:2669.71509\n",
      "[67]\tTest-rmse:2668.43042\n",
      "[68]\tTest-rmse:2664.37817\n",
      "[69]\tTest-rmse:2662.59326\n",
      "[70]\tTest-rmse:2661.82275\n",
      "[71]\tTest-rmse:2659.38330\n",
      "[72]\tTest-rmse:2656.93921\n",
      "[73]\tTest-rmse:2656.32568\n",
      "[74]\tTest-rmse:2652.12207\n",
      "[75]\tTest-rmse:2651.78882\n",
      "[76]\tTest-rmse:2649.38721\n",
      "[77]\tTest-rmse:2648.85547\n",
      "[78]\tTest-rmse:2647.69043\n",
      "[79]\tTest-rmse:2644.16821\n",
      "[80]\tTest-rmse:2642.48389\n",
      "[81]\tTest-rmse:2640.70093\n",
      "[82]\tTest-rmse:2639.56885\n",
      "[83]\tTest-rmse:2635.79663\n",
      "[84]\tTest-rmse:2631.68091\n",
      "[85]\tTest-rmse:2631.48144\n",
      "[86]\tTest-rmse:2630.56982\n",
      "[87]\tTest-rmse:2629.33813\n",
      "[88]\tTest-rmse:2627.79590\n",
      "[89]\tTest-rmse:2626.44238\n",
      "[90]\tTest-rmse:2626.09741\n",
      "[91]\tTest-rmse:2625.63501\n",
      "[92]\tTest-rmse:2624.78003\n",
      "[93]\tTest-rmse:2623.69385\n",
      "[94]\tTest-rmse:2622.47778\n",
      "[95]\tTest-rmse:2621.48560\n",
      "[96]\tTest-rmse:2621.04565\n",
      "[97]\tTest-rmse:2620.61572\n",
      "[98]\tTest-rmse:2619.23364\n",
      "[99]\tTest-rmse:2618.81860\n",
      "[100]\tTest-rmse:2616.44824\n",
      "[101]\tTest-rmse:2615.89600\n",
      "[102]\tTest-rmse:2615.71289\n",
      "[103]\tTest-rmse:2615.20923\n",
      "[104]\tTest-rmse:2614.13257\n",
      "[105]\tTest-rmse:2613.93481\n",
      "[106]\tTest-rmse:2612.37598\n",
      "[107]\tTest-rmse:2610.49658\n",
      "[108]\tTest-rmse:2609.81079\n",
      "[109]\tTest-rmse:2608.82690\n",
      "[110]\tTest-rmse:2608.31128\n",
      "[111]\tTest-rmse:2607.99072\n",
      "[112]\tTest-rmse:2606.56421\n",
      "[113]\tTest-rmse:2605.25049\n",
      "[114]\tTest-rmse:2604.43823\n",
      "[115]\tTest-rmse:2602.75513\n",
      "[116]\tTest-rmse:2602.36523\n",
      "[117]\tTest-rmse:2602.33765\n",
      "[118]\tTest-rmse:2601.76904\n",
      "[119]\tTest-rmse:2600.41089\n",
      "[120]\tTest-rmse:2599.45947\n",
      "[121]\tTest-rmse:2599.13843\n",
      "[122]\tTest-rmse:2597.19873\n",
      "[123]\tTest-rmse:2597.12671\n",
      "[124]\tTest-rmse:2596.53003\n",
      "[125]\tTest-rmse:2596.31323\n",
      "[126]\tTest-rmse:2596.04053\n",
      "[127]\tTest-rmse:2593.94775\n",
      "[128]\tTest-rmse:2593.26123\n",
      "[129]\tTest-rmse:2592.44775\n",
      "[130]\tTest-rmse:2591.69653\n",
      "[131]\tTest-rmse:2588.99365\n",
      "[132]\tTest-rmse:2588.73022\n",
      "[133]\tTest-rmse:2587.81348\n",
      "[134]\tTest-rmse:2586.19922\n",
      "[135]\tTest-rmse:2585.26025\n",
      "[136]\tTest-rmse:2584.24683\n",
      "[137]\tTest-rmse:2582.68628\n",
      "[138]\tTest-rmse:2581.77319\n",
      "[139]\tTest-rmse:2580.52173\n",
      "[140]\tTest-rmse:2579.38501\n",
      "[141]\tTest-rmse:2579.24097\n",
      "[142]\tTest-rmse:2578.52270\n",
      "[143]\tTest-rmse:2578.04248\n",
      "[144]\tTest-rmse:2577.96753\n",
      "[145]\tTest-rmse:2577.73340\n",
      "[146]\tTest-rmse:2576.89917\n",
      "[147]\tTest-rmse:2575.95508\n",
      "[148]\tTest-rmse:2574.66431\n",
      "[149]\tTest-rmse:2574.26514\n",
      "[150]\tTest-rmse:2574.12964\n",
      "[151]\tTest-rmse:2573.24951\n",
      "[152]\tTest-rmse:2573.10230\n",
      "[153]\tTest-rmse:2572.53882\n",
      "[154]\tTest-rmse:2571.43628\n",
      "[155]\tTest-rmse:2571.11523\n",
      "[156]\tTest-rmse:2571.02417\n",
      "[157]\tTest-rmse:2570.31763\n",
      "[158]\tTest-rmse:2569.88794\n",
      "[159]\tTest-rmse:2569.74951\n",
      "[160]\tTest-rmse:2569.13599\n",
      "[161]\tTest-rmse:2568.55786\n",
      "[162]\tTest-rmse:2568.12085\n",
      "[163]\tTest-rmse:2567.91577\n",
      "[164]\tTest-rmse:2567.81152\n",
      "[165]\tTest-rmse:2567.66309\n",
      "[166]\tTest-rmse:2567.57202\n",
      "[167]\tTest-rmse:2566.55835\n",
      "[168]\tTest-rmse:2566.26343\n",
      "[169]\tTest-rmse:2565.38696\n",
      "[170]\tTest-rmse:2564.66528\n",
      "[171]\tTest-rmse:2564.06372\n",
      "[172]\tTest-rmse:2563.63696\n",
      "[173]\tTest-rmse:2563.27979\n",
      "[174]\tTest-rmse:2562.24756\n",
      "[175]\tTest-rmse:2561.90454\n",
      "[176]\tTest-rmse:2561.52515\n",
      "[177]\tTest-rmse:2561.41821\n",
      "[178]\tTest-rmse:2560.96924\n",
      "[179]\tTest-rmse:2560.69849\n",
      "[180]\tTest-rmse:2559.83398\n",
      "[181]\tTest-rmse:2559.69043\n",
      "[182]\tTest-rmse:2559.50073\n",
      "[183]\tTest-rmse:2558.95508\n",
      "[184]\tTest-rmse:2558.51318\n",
      "[185]\tTest-rmse:2558.35352\n",
      "[186]\tTest-rmse:2558.26489\n",
      "[187]\tTest-rmse:2557.85059\n",
      "[188]\tTest-rmse:2557.44922\n",
      "[189]\tTest-rmse:2556.99902\n",
      "[190]\tTest-rmse:2556.72632\n",
      "[191]\tTest-rmse:2555.81689\n",
      "[192]\tTest-rmse:2555.38989\n",
      "[193]\tTest-rmse:2554.86084\n",
      "[194]\tTest-rmse:2554.61987\n",
      "[195]\tTest-rmse:2554.29394\n",
      "[196]\tTest-rmse:2554.21387\n",
      "[197]\tTest-rmse:2554.15356\n",
      "[198]\tTest-rmse:2553.66064\n",
      "[199]\tTest-rmse:2552.73071\n",
      "[200]\tTest-rmse:2552.55029\n",
      "[201]\tTest-rmse:2552.05200\n",
      "[202]\tTest-rmse:2551.30395\n",
      "[203]\tTest-rmse:2550.98853\n",
      "[204]\tTest-rmse:2550.98926\n",
      "[205]\tTest-rmse:2550.54761\n",
      "[206]\tTest-rmse:2550.09106\n",
      "[207]\tTest-rmse:2550.02002\n",
      "[208]\tTest-rmse:2549.84766\n",
      "[209]\tTest-rmse:2549.10352\n",
      "[210]\tTest-rmse:2548.77197\n",
      "[211]\tTest-rmse:2548.49487\n",
      "[212]\tTest-rmse:2548.13892\n",
      "[213]\tTest-rmse:2547.13281\n",
      "[214]\tTest-rmse:2546.82349\n",
      "[215]\tTest-rmse:2546.19141\n",
      "[216]\tTest-rmse:2545.49438\n",
      "[217]\tTest-rmse:2544.88672\n",
      "[218]\tTest-rmse:2544.85278\n",
      "[219]\tTest-rmse:2544.74292\n",
      "[220]\tTest-rmse:2544.55688\n",
      "[221]\tTest-rmse:2544.43384\n",
      "[222]\tTest-rmse:2544.33740\n",
      "[223]\tTest-rmse:2544.18384\n",
      "[224]\tTest-rmse:2544.08960\n",
      "[225]\tTest-rmse:2543.77612\n",
      "[226]\tTest-rmse:2543.41968\n",
      "[227]\tTest-rmse:2543.31299\n",
      "[228]\tTest-rmse:2543.21484\n",
      "[229]\tTest-rmse:2543.03882\n",
      "[230]\tTest-rmse:2543.03174\n",
      "[231]\tTest-rmse:2542.89941\n",
      "[232]\tTest-rmse:2542.90723\n",
      "[233]\tTest-rmse:2542.79736\n",
      "[234]\tTest-rmse:2542.53247\n",
      "[235]\tTest-rmse:2542.17969\n",
      "[236]\tTest-rmse:2541.97266\n",
      "[237]\tTest-rmse:2541.88525\n",
      "[238]\tTest-rmse:2541.68799\n",
      "[239]\tTest-rmse:2541.32910\n",
      "[240]\tTest-rmse:2541.03564\n",
      "[241]\tTest-rmse:2540.72974\n",
      "[242]\tTest-rmse:2540.01392\n",
      "[243]\tTest-rmse:2539.95093\n",
      "[244]\tTest-rmse:2539.72119\n",
      "[245]\tTest-rmse:2539.41821\n",
      "[246]\tTest-rmse:2539.33911\n",
      "[247]\tTest-rmse:2538.90771\n",
      "[248]\tTest-rmse:2538.19678\n",
      "[249]\tTest-rmse:2537.96802\n",
      "[250]\tTest-rmse:2537.90063\n",
      "[251]\tTest-rmse:2537.24243\n",
      "[252]\tTest-rmse:2537.22119\n",
      "[253]\tTest-rmse:2537.03271\n",
      "[254]\tTest-rmse:2536.75879\n",
      "[255]\tTest-rmse:2536.63013\n",
      "[256]\tTest-rmse:2536.38550\n",
      "[257]\tTest-rmse:2536.37744\n",
      "[258]\tTest-rmse:2536.21582\n",
      "[259]\tTest-rmse:2536.01245\n",
      "[260]\tTest-rmse:2535.72363\n",
      "[261]\tTest-rmse:2535.31348\n",
      "[262]\tTest-rmse:2534.79419\n",
      "[263]\tTest-rmse:2534.77148\n",
      "[264]\tTest-rmse:2534.68652\n",
      "[265]\tTest-rmse:2534.33179\n",
      "[266]\tTest-rmse:2533.79370\n",
      "[267]\tTest-rmse:2533.69043\n",
      "[268]\tTest-rmse:2533.57300\n",
      "[269]\tTest-rmse:2533.36743\n",
      "[270]\tTest-rmse:2533.24316\n",
      "[271]\tTest-rmse:2532.76782\n",
      "[272]\tTest-rmse:2532.71118\n",
      "[273]\tTest-rmse:2532.39917\n",
      "[274]\tTest-rmse:2532.35132\n",
      "[275]\tTest-rmse:2531.91724\n",
      "[276]\tTest-rmse:2531.75024\n",
      "[277]\tTest-rmse:2531.54321\n",
      "[278]\tTest-rmse:2530.72949\n",
      "[279]\tTest-rmse:2530.68066\n",
      "[280]\tTest-rmse:2530.55762\n",
      "[281]\tTest-rmse:2529.77808\n",
      "[282]\tTest-rmse:2529.54077\n",
      "[283]\tTest-rmse:2529.32568\n",
      "[284]\tTest-rmse:2529.27441\n",
      "[285]\tTest-rmse:2529.24512\n",
      "[286]\tTest-rmse:2529.12427\n",
      "[287]\tTest-rmse:2528.71655\n",
      "[288]\tTest-rmse:2528.61670\n",
      "[289]\tTest-rmse:2528.47730\n",
      "[290]\tTest-rmse:2528.01367\n",
      "[291]\tTest-rmse:2527.74976\n",
      "[292]\tTest-rmse:2527.73413\n",
      "[293]\tTest-rmse:2527.74902\n",
      "[294]\tTest-rmse:2527.62036\n",
      "[295]\tTest-rmse:2527.35547\n",
      "[296]\tTest-rmse:2527.19775\n",
      "[297]\tTest-rmse:2527.14722\n",
      "[298]\tTest-rmse:2526.72534\n",
      "[299]\tTest-rmse:2526.51050\n",
      "[300]\tTest-rmse:2525.99536\n",
      "[301]\tTest-rmse:2525.84009\n",
      "[302]\tTest-rmse:2525.79614\n",
      "[303]\tTest-rmse:2525.39673\n",
      "[304]\tTest-rmse:2525.34961\n",
      "[305]\tTest-rmse:2525.16675\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[306]\tTest-rmse:2525.01465\n",
      "[307]\tTest-rmse:2524.95337\n",
      "[308]\tTest-rmse:2524.62354\n",
      "[309]\tTest-rmse:2524.27734\n",
      "[310]\tTest-rmse:2523.99536\n",
      "[311]\tTest-rmse:2523.95264\n",
      "[312]\tTest-rmse:2523.97900\n",
      "[313]\tTest-rmse:2523.82324\n",
      "[314]\tTest-rmse:2523.56128\n",
      "[315]\tTest-rmse:2523.49268\n",
      "[316]\tTest-rmse:2523.16455\n",
      "[317]\tTest-rmse:2522.76099\n",
      "[318]\tTest-rmse:2522.65894\n",
      "[319]\tTest-rmse:2522.54590\n",
      "[320]\tTest-rmse:2522.55078\n",
      "[321]\tTest-rmse:2522.37451\n",
      "[322]\tTest-rmse:2522.27270\n",
      "[323]\tTest-rmse:2522.10254\n",
      "[324]\tTest-rmse:2522.05103\n",
      "[325]\tTest-rmse:2521.96387\n",
      "[326]\tTest-rmse:2521.80444\n",
      "[327]\tTest-rmse:2521.78589\n",
      "[328]\tTest-rmse:2521.52783\n",
      "[329]\tTest-rmse:2521.37305\n",
      "[330]\tTest-rmse:2521.34644\n",
      "[331]\tTest-rmse:2521.35498\n",
      "[332]\tTest-rmse:2521.26269\n",
      "[333]\tTest-rmse:2521.28247\n",
      "[334]\tTest-rmse:2521.19849\n",
      "[335]\tTest-rmse:2520.97290\n",
      "[336]\tTest-rmse:2520.91260\n",
      "[337]\tTest-rmse:2520.85254\n",
      "[338]\tTest-rmse:2520.80249\n",
      "[339]\tTest-rmse:2520.81763\n",
      "[340]\tTest-rmse:2520.79565\n",
      "[341]\tTest-rmse:2520.73926\n",
      "[342]\tTest-rmse:2520.72852\n",
      "[343]\tTest-rmse:2520.47803\n",
      "[344]\tTest-rmse:2520.43286\n",
      "[345]\tTest-rmse:2520.36645\n",
      "[346]\tTest-rmse:2520.12915\n",
      "[347]\tTest-rmse:2520.08301\n",
      "[348]\tTest-rmse:2519.98169\n",
      "[349]\tTest-rmse:2519.75855\n",
      "[350]\tTest-rmse:2519.55981\n",
      "[351]\tTest-rmse:2519.50488\n",
      "[352]\tTest-rmse:2519.44605\n",
      "[353]\tTest-rmse:2519.40527\n",
      "[354]\tTest-rmse:2519.28516\n",
      "[355]\tTest-rmse:2519.15088\n",
      "[356]\tTest-rmse:2518.95752\n",
      "[357]\tTest-rmse:2518.94751\n",
      "[358]\tTest-rmse:2518.77783\n",
      "[359]\tTest-rmse:2518.69605\n",
      "[360]\tTest-rmse:2518.55469\n",
      "[361]\tTest-rmse:2518.43750\n",
      "[362]\tTest-rmse:2518.37720\n",
      "[363]\tTest-rmse:2518.40186\n",
      "[364]\tTest-rmse:2518.20068\n",
      "[365]\tTest-rmse:2518.23291\n",
      "[366]\tTest-rmse:2518.22168\n",
      "[367]\tTest-rmse:2518.24976\n",
      "[368]\tTest-rmse:2517.89770\n",
      "[369]\tTest-rmse:2517.78931\n",
      "[370]\tTest-rmse:2517.52515\n",
      "[371]\tTest-rmse:2517.39404\n",
      "[372]\tTest-rmse:2517.24219\n",
      "[373]\tTest-rmse:2516.89356\n",
      "[374]\tTest-rmse:2516.72485\n",
      "[375]\tTest-rmse:2516.68164\n",
      "[376]\tTest-rmse:2516.63550\n",
      "[377]\tTest-rmse:2516.48828\n",
      "[378]\tTest-rmse:2516.48560\n",
      "[379]\tTest-rmse:2516.48340\n",
      "[380]\tTest-rmse:2516.51758\n",
      "[381]\tTest-rmse:2516.27002\n",
      "[382]\tTest-rmse:2515.96631\n",
      "[383]\tTest-rmse:2515.81616\n",
      "[384]\tTest-rmse:2515.54077\n",
      "[385]\tTest-rmse:2515.33643\n",
      "[386]\tTest-rmse:2515.29517\n",
      "[387]\tTest-rmse:2515.14502\n",
      "[388]\tTest-rmse:2514.98682\n",
      "[389]\tTest-rmse:2515.01221\n",
      "[390]\tTest-rmse:2514.99292\n",
      "[391]\tTest-rmse:2514.92358\n",
      "[392]\tTest-rmse:2514.64941\n",
      "[393]\tTest-rmse:2514.59131\n",
      "[394]\tTest-rmse:2514.52124\n",
      "[395]\tTest-rmse:2514.47168\n",
      "[396]\tTest-rmse:2514.40942\n",
      "[397]\tTest-rmse:2514.38013\n",
      "[398]\tTest-rmse:2514.36621\n",
      "[399]\tTest-rmse:2514.30298\n",
      "[400]\tTest-rmse:2514.41504\n",
      "[401]\tTest-rmse:2513.98901\n",
      "[402]\tTest-rmse:2513.55518\n",
      "[403]\tTest-rmse:2513.55591\n",
      "[404]\tTest-rmse:2513.49976\n",
      "[405]\tTest-rmse:2513.31372\n",
      "[406]\tTest-rmse:2513.29126\n",
      "[407]\tTest-rmse:2513.22705\n",
      "[408]\tTest-rmse:2513.15430\n",
      "[409]\tTest-rmse:2513.08057\n",
      "[410]\tTest-rmse:2513.01196\n",
      "[411]\tTest-rmse:2512.88110\n",
      "[412]\tTest-rmse:2512.79126\n",
      "[413]\tTest-rmse:2512.79272\n",
      "[414]\tTest-rmse:2512.59644\n",
      "[415]\tTest-rmse:2512.30884\n",
      "[416]\tTest-rmse:2511.86353\n",
      "[417]\tTest-rmse:2511.60547\n",
      "[418]\tTest-rmse:2511.49609\n",
      "[419]\tTest-rmse:2511.41821\n",
      "[420]\tTest-rmse:2511.45996\n",
      "[421]\tTest-rmse:2511.21436\n",
      "[422]\tTest-rmse:2511.01904\n",
      "[423]\tTest-rmse:2510.79346\n",
      "[424]\tTest-rmse:2510.79394\n",
      "[425]\tTest-rmse:2510.74145\n",
      "[426]\tTest-rmse:2510.61621\n",
      "[427]\tTest-rmse:2510.57861\n",
      "[428]\tTest-rmse:2510.55005\n",
      "[429]\tTest-rmse:2510.53857\n",
      "[430]\tTest-rmse:2510.41431\n",
      "[431]\tTest-rmse:2510.17212\n",
      "[432]\tTest-rmse:2509.95068\n",
      "[433]\tTest-rmse:2509.85107\n",
      "[434]\tTest-rmse:2509.74561\n",
      "[435]\tTest-rmse:2509.63769\n",
      "[436]\tTest-rmse:2509.61548\n",
      "[437]\tTest-rmse:2509.32788\n",
      "[438]\tTest-rmse:2508.89648\n",
      "[439]\tTest-rmse:2508.92651\n",
      "[440]\tTest-rmse:2508.90869\n",
      "[441]\tTest-rmse:2508.83569\n",
      "[442]\tTest-rmse:2508.79248\n",
      "[443]\tTest-rmse:2508.75122\n",
      "[444]\tTest-rmse:2508.71875\n",
      "[445]\tTest-rmse:2508.58691\n",
      "[446]\tTest-rmse:2508.46875\n",
      "[447]\tTest-rmse:2508.44824\n",
      "[448]\tTest-rmse:2508.44971\n",
      "[449]\tTest-rmse:2508.38452\n",
      "[450]\tTest-rmse:2508.31250\n",
      "[451]\tTest-rmse:2508.34253\n",
      "[452]\tTest-rmse:2508.28296\n",
      "[453]\tTest-rmse:2508.30420\n",
      "[454]\tTest-rmse:2508.10596\n",
      "[455]\tTest-rmse:2508.04883\n",
      "[456]\tTest-rmse:2508.00952\n",
      "[457]\tTest-rmse:2507.92212\n",
      "[458]\tTest-rmse:2507.75391\n",
      "[459]\tTest-rmse:2507.46509\n",
      "[460]\tTest-rmse:2507.36987\n",
      "[461]\tTest-rmse:2506.99487\n",
      "[462]\tTest-rmse:2506.92920\n",
      "[463]\tTest-rmse:2506.76025\n",
      "[464]\tTest-rmse:2506.69922\n",
      "[465]\tTest-rmse:2506.65845\n",
      "[466]\tTest-rmse:2506.38696\n",
      "[467]\tTest-rmse:2506.28662\n",
      "[468]\tTest-rmse:2506.01685\n",
      "[469]\tTest-rmse:2505.96997\n",
      "[470]\tTest-rmse:2505.94946\n",
      "[471]\tTest-rmse:2505.89233\n",
      "[472]\tTest-rmse:2505.85230\n",
      "[473]\tTest-rmse:2505.80518\n",
      "[474]\tTest-rmse:2505.39404\n",
      "[475]\tTest-rmse:2505.33276\n",
      "[476]\tTest-rmse:2505.10303\n",
      "[477]\tTest-rmse:2505.04590\n",
      "[478]\tTest-rmse:2505.02270\n",
      "[479]\tTest-rmse:2504.89770\n",
      "[480]\tTest-rmse:2504.74316\n",
      "[481]\tTest-rmse:2504.60205\n",
      "[482]\tTest-rmse:2504.59546\n",
      "[483]\tTest-rmse:2504.63281\n",
      "[484]\tTest-rmse:2504.16113\n",
      "[485]\tTest-rmse:2504.02222\n",
      "[486]\tTest-rmse:2504.08179\n",
      "[487]\tTest-rmse:2503.91943\n",
      "[488]\tTest-rmse:2503.77563\n",
      "[489]\tTest-rmse:2503.67603\n",
      "[490]\tTest-rmse:2503.62183\n",
      "[491]\tTest-rmse:2503.52319\n",
      "[492]\tTest-rmse:2503.46020\n",
      "[493]\tTest-rmse:2503.41528\n",
      "[494]\tTest-rmse:2503.39746\n",
      "[495]\tTest-rmse:2503.38501\n",
      "[496]\tTest-rmse:2503.34717\n",
      "[497]\tTest-rmse:2503.20923\n",
      "[498]\tTest-rmse:2503.13330\n",
      "[499]\tTest-rmse:2503.14551\n",
      "[500]\tTest-rmse:2503.12329\n",
      "[501]\tTest-rmse:2503.08520\n",
      "[502]\tTest-rmse:2502.95239\n",
      "[503]\tTest-rmse:2502.82349\n",
      "[504]\tTest-rmse:2502.75855\n",
      "[505]\tTest-rmse:2502.71094\n",
      "[506]\tTest-rmse:2502.67358\n",
      "[507]\tTest-rmse:2502.49756\n",
      "[508]\tTest-rmse:2502.51562\n",
      "[509]\tTest-rmse:2502.43604\n",
      "[510]\tTest-rmse:2502.44336\n",
      "[511]\tTest-rmse:2502.45532\n",
      "[512]\tTest-rmse:2502.37622\n",
      "[513]\tTest-rmse:2502.09521\n",
      "[514]\tTest-rmse:2501.99316\n",
      "[515]\tTest-rmse:2501.98413\n",
      "[516]\tTest-rmse:2502.03271\n",
      "[517]\tTest-rmse:2501.91699\n",
      "[518]\tTest-rmse:2501.82178\n",
      "[519]\tTest-rmse:2501.69531\n",
      "[520]\tTest-rmse:2501.69238\n",
      "[521]\tTest-rmse:2501.63184\n",
      "[522]\tTest-rmse:2501.59937\n",
      "[523]\tTest-rmse:2501.59033\n",
      "[524]\tTest-rmse:2501.54321\n",
      "[525]\tTest-rmse:2501.54541\n",
      "[526]\tTest-rmse:2501.45606\n",
      "[527]\tTest-rmse:2501.36231\n",
      "[528]\tTest-rmse:2501.31616\n",
      "[529]\tTest-rmse:2501.29785\n",
      "[530]\tTest-rmse:2501.22241\n",
      "[531]\tTest-rmse:2501.08301\n",
      "[532]\tTest-rmse:2500.98682\n",
      "[533]\tTest-rmse:2500.95410\n",
      "[534]\tTest-rmse:2500.95923\n",
      "[535]\tTest-rmse:2500.94214\n",
      "[536]\tTest-rmse:2500.93823\n",
      "[537]\tTest-rmse:2500.88013\n",
      "[538]\tTest-rmse:2500.83911\n",
      "[539]\tTest-rmse:2500.86255\n",
      "[540]\tTest-rmse:2500.75415\n",
      "[541]\tTest-rmse:2500.48877\n",
      "[542]\tTest-rmse:2500.51343\n",
      "[543]\tTest-rmse:2500.51514\n",
      "[544]\tTest-rmse:2500.47656\n",
      "[545]\tTest-rmse:2500.51807\n",
      "[546]\tTest-rmse:2500.47437\n",
      "[547]\tTest-rmse:2500.39990\n",
      "[548]\tTest-rmse:2500.36011\n",
      "[549]\tTest-rmse:2500.19043\n",
      "[550]\tTest-rmse:2500.18530\n",
      "[551]\tTest-rmse:2500.20898\n",
      "[552]\tTest-rmse:2500.12500\n",
      "[553]\tTest-rmse:2500.02319\n",
      "[554]\tTest-rmse:2500.02686\n",
      "[555]\tTest-rmse:2499.93433\n",
      "[556]\tTest-rmse:2499.85352\n",
      "[557]\tTest-rmse:2499.90454\n",
      "[558]\tTest-rmse:2499.79028\n",
      "[559]\tTest-rmse:2499.76269\n",
      "[560]\tTest-rmse:2499.70898\n",
      "[561]\tTest-rmse:2499.72461\n",
      "[562]\tTest-rmse:2499.72583\n",
      "[563]\tTest-rmse:2499.71777\n",
      "[564]\tTest-rmse:2499.61841\n",
      "[565]\tTest-rmse:2499.62427\n",
      "[566]\tTest-rmse:2499.62988\n",
      "[567]\tTest-rmse:2499.56616\n",
      "[568]\tTest-rmse:2499.54468\n",
      "[569]\tTest-rmse:2499.56104\n",
      "[570]\tTest-rmse:2499.52783\n",
      "[571]\tTest-rmse:2499.54126\n",
      "[572]\tTest-rmse:2499.53540\n",
      "[573]\tTest-rmse:2499.55273\n",
      "[574]\tTest-rmse:2499.62549\n",
      "[575]\tTest-rmse:2499.61694\n",
      "[576]\tTest-rmse:2499.59277\n",
      "[577]\tTest-rmse:2499.63159\n",
      "[578]\tTest-rmse:2499.62329\n",
      "[579]\tTest-rmse:2499.59937\n",
      "[580]\tTest-rmse:2499.64258\n",
      "Stopping. Best iteration:\n",
      "[570]\tTest-rmse:2499.52783\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = xgb.train(\n",
    "    params,\n",
    "    dtrain,\n",
    "    num_boost_round=num_boost_round,\n",
    "    evals=[(dtest, \"Test\")],\n",
    "    early_stopping_rounds=10\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predicting Values for Test data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict(dpred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing Data for Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Purchase</th>\n",
       "      <th>User_ID</th>\n",
       "      <th>Product_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14928.404297</td>\n",
       "      <td>1000004</td>\n",
       "      <td>P00128942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10792.964844</td>\n",
       "      <td>1000009</td>\n",
       "      <td>P00113442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7175.628418</td>\n",
       "      <td>1000010</td>\n",
       "      <td>P00288442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4489.056641</td>\n",
       "      <td>1000010</td>\n",
       "      <td>P00145342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2342.989990</td>\n",
       "      <td>1000011</td>\n",
       "      <td>P00053842</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Purchase  User_ID Product_ID\n",
       "0  14928.404297  1000004  P00128942\n",
       "1  10792.964844  1000009  P00113442\n",
       "2   7175.628418  1000010  P00288442\n",
       "3   4489.056641  1000010  P00145342\n",
       "4   2342.989990  1000011  P00053842"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.DataFrame(y_pred, columns = ['Purchase'])\n",
    "submission['User_ID'] = test['User_ID']\n",
    "submission['Product_ID'] = test['Product_ID']\n",
    "submission.head()\n",
    "\n",
    "submission.to_csv('XGboost_submission.csv') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}