Feature Added LGBoost.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# importing libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import cross_val_score\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil_Type32</th>\n",
       "      <th>Soil_Type33</th>\n",
       "      <th>Soil_Type34</th>\n",
       "      <th>Soil_Type35</th>\n",
       "      <th>Soil_Type36</th>\n",
       "      <th>Soil_Type37</th>\n",
       "      <th>Soil_Type38</th>\n",
       "      <th>Soil_Type39</th>\n",
       "      <th>Soil_Type40</th>\n",
       "      <th>Cover_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2596</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>221</td>\n",
       "      <td>232</td>\n",
       "      <td>148</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2590</td>\n",
       "      <td>56</td>\n",
       "      <td>2</td>\n",
       "      <td>212</td>\n",
       "      <td>-6</td>\n",
       "      <td>390</td>\n",
       "      <td>220</td>\n",
       "      <td>235</td>\n",
       "      <td>151</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2804</td>\n",
       "      <td>139</td>\n",
       "      <td>9</td>\n",
       "      <td>268</td>\n",
       "      <td>65</td>\n",
       "      <td>3180</td>\n",
       "      <td>234</td>\n",
       "      <td>238</td>\n",
       "      <td>135</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2785</td>\n",
       "      <td>155</td>\n",
       "      <td>18</td>\n",
       "      <td>242</td>\n",
       "      <td>118</td>\n",
       "      <td>3090</td>\n",
       "      <td>238</td>\n",
       "      <td>238</td>\n",
       "      <td>122</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>2595</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>153</td>\n",
       "      <td>-1</td>\n",
       "      <td>391</td>\n",
       "      <td>220</td>\n",
       "      <td>234</td>\n",
       "      <td>150</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>2579</td>\n",
       "      <td>132</td>\n",
       "      <td>6</td>\n",
       "      <td>300</td>\n",
       "      <td>-15</td>\n",
       "      <td>67</td>\n",
       "      <td>230</td>\n",
       "      <td>237</td>\n",
       "      <td>140</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>2606</td>\n",
       "      <td>45</td>\n",
       "      <td>7</td>\n",
       "      <td>270</td>\n",
       "      <td>5</td>\n",
       "      <td>633</td>\n",
       "      <td>222</td>\n",
       "      <td>225</td>\n",
       "      <td>138</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>2605</td>\n",
       "      <td>49</td>\n",
       "      <td>4</td>\n",
       "      <td>234</td>\n",
       "      <td>7</td>\n",
       "      <td>573</td>\n",
       "      <td>222</td>\n",
       "      <td>230</td>\n",
       "      <td>144</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>2617</td>\n",
       "      <td>45</td>\n",
       "      <td>9</td>\n",
       "      <td>240</td>\n",
       "      <td>56</td>\n",
       "      <td>666</td>\n",
       "      <td>223</td>\n",
       "      <td>221</td>\n",
       "      <td>133</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>2612</td>\n",
       "      <td>59</td>\n",
       "      <td>10</td>\n",
       "      <td>247</td>\n",
       "      <td>11</td>\n",
       "      <td>636</td>\n",
       "      <td>228</td>\n",
       "      <td>219</td>\n",
       "      <td>124</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  Elevation  Aspect  Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0   1       2596      51      3                               258   \n",
       "1   2       2590      56      2                               212   \n",
       "2   3       2804     139      9                               268   \n",
       "3   4       2785     155     18                               242   \n",
       "4   5       2595      45      2                               153   \n",
       "5   6       2579     132      6                               300   \n",
       "6   7       2606      45      7                               270   \n",
       "7   8       2605      49      4                               234   \n",
       "8   9       2617      45      9                               240   \n",
       "9  10       2612      59     10                               247   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                               0                              510   \n",
       "1                              -6                              390   \n",
       "2                              65                             3180   \n",
       "3                             118                             3090   \n",
       "4                              -1                              391   \n",
       "5                             -15                               67   \n",
       "6                               5                              633   \n",
       "7                               7                              573   \n",
       "8                              56                              666   \n",
       "9                              11                              636   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm     ...      Soil_Type32  \\\n",
       "0            221             232            148     ...                0   \n",
       "1            220             235            151     ...                0   \n",
       "2            234             238            135     ...                0   \n",
       "3            238             238            122     ...                0   \n",
       "4            220             234            150     ...                0   \n",
       "5            230             237            140     ...                0   \n",
       "6            222             225            138     ...                0   \n",
       "7            222             230            144     ...                0   \n",
       "8            223             221            133     ...                0   \n",
       "9            228             219            124     ...                0   \n",
       "\n",
       "   Soil_Type33  Soil_Type34  Soil_Type35  Soil_Type36  Soil_Type37  \\\n",
       "0            0            0            0            0            0   \n",
       "1            0            0            0            0            0   \n",
       "2            0            0            0            0            0   \n",
       "3            0            0            0            0            0   \n",
       "4            0            0            0            0            0   \n",
       "5            0            0            0            0            0   \n",
       "6            0            0            0            0            0   \n",
       "7            0            0            0            0            0   \n",
       "8            0            0            0            0            0   \n",
       "9            0            0            0            0            0   \n",
       "\n",
       "   Soil_Type38  Soil_Type39  Soil_Type40  Cover_Type  \n",
       "0            0            0            0           5  \n",
       "1            0            0            0           5  \n",
       "2            0            0            0           2  \n",
       "3            0            0            0           2  \n",
       "4            0            0            0           5  \n",
       "5            0            0            0           2  \n",
       "6            0            0            0           5  \n",
       "7            0            0            0           5  \n",
       "8            0            0            0           5  \n",
       "9            0            0            0           5  \n",
       "\n",
       "[10 rows x 56 columns]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Creation for train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train['Hillshade_mean'] = train['Hillshade_9am']+train['Hillshade_Noon']+train['Hillshade_3pm']\n",
    "train['slope_hyd'] = (train['Horizontal_Distance_To_Hydrology']**2+train['Vertical_Distance_To_Hydrology']**2)**0.5\n",
    "train.slope_hyd=train.slope_hyd.map(lambda x: 0 if np.isinf(x) else x) # remove infinite value if any\n",
    "train['log_elevation']= np.log(train.Elevation)\n",
    "train['Hillshade_9am_sq'] = train['Hillshade_9am']**2\n",
    "train['Hillshade_noon_sq'] = train['Hillshade_Noon']**2\n",
    "train['Hillshade_3pm_sq'] = train['Hillshade_3pm']**2\n",
    "\n",
    "train['cosine_slope'] = np.cos(train['Slope'])\n",
    "\n",
    "a = pd.cut(train['Aspect'],bins=[-1, 45, 90, 135, 180, 225, 270, 315, 360],\n",
    "           labels=['Aspect_1','Aspect_2','Aspect_3','Aspect_4','Aspect_5','Aspect_6','Aspect_7','Aspect_8'])\n",
    "b = pd.get_dummies(a)\n",
    "train.merge(b,right_index=True,left_index=True)\n",
    " \n",
    "\n",
    "train['interaction_9amnoon'] = train['Hillshade_9am']*train['Hillshade_Noon']\n",
    "train['interaction_noon3pm'] = train['Hillshade_3pm']*train['Hillshade_Noon']\n",
    "train['interaction_9am3pm'] = train['Hillshade_3pm']*train['Hillshade_9am']\n",
    "\n",
    "train['HF1'] = train['Horizontal_Distance_To_Hydrology']+train['Horizontal_Distance_To_Fire_Points']\n",
    "train['HF2'] = abs(train['Horizontal_Distance_To_Hydrology']-train['Horizontal_Distance_To_Fire_Points'])\n",
    "train['HR1'] = abs(train['Horizontal_Distance_To_Hydrology']+train['Horizontal_Distance_To_Roadways'])\n",
    "train['HR2'] = abs(train['Horizontal_Distance_To_Hydrology']-train['Horizontal_Distance_To_Roadways'])\n",
    "train['FR1'] = abs(train['Horizontal_Distance_To_Fire_Points']+train['Horizontal_Distance_To_Roadways'])\n",
    "train['FR2'] = abs(train['Horizontal_Distance_To_Fire_Points']-train['Horizontal_Distance_To_Roadways'])\n",
    "train['ele_vert'] = train.Elevation-train.Vertical_Distance_To_Hydrology\n",
    "\n",
    "train['slope_hyd'] = (train['Horizontal_Distance_To_Hydrology']**2+train['Vertical_Distance_To_Hydrology']**2)**0.5\n",
    "train.slope_hyd=train.slope_hyd.map(lambda x: 0 if np.isinf(x) else x) # remove infinite value if any\n",
    "\n",
    "#Mean distance to Amenities \n",
    "train['Mean_Amenities']=(train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology + train.Horizontal_Distance_To_Roadways) / 3 \n",
    "#Mean Distance to Fire and Water \n",
    "train['Mean_Fire_Hyd']=(train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology) / 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Creation for test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test['Hillshade_mean'] = test['Hillshade_9am']+test['Hillshade_Noon']+test['Hillshade_3pm']\n",
    "test['slope_hyd'] = (test['Horizontal_Distance_To_Hydrology']**2+test['Vertical_Distance_To_Hydrology']**2)**0.5\n",
    "test.slope_hyd=test.slope_hyd.map(lambda x: 0 if np.isinf(x) else x) # remove infinite value if any\n",
    "test['log_elevation']= np.log(test.Elevation)\n",
    "test['Hillshade_9am_sq'] = test['Hillshade_9am']**2\n",
    "test['Hillshade_noon_sq'] = test['Hillshade_Noon']**2\n",
    "test['Hillshade_3pm_sq'] = test['Hillshade_3pm']**2\n",
    "\n",
    "test['cosine_slope'] = np.cos(test['Slope'])\n",
    "\n",
    "a = pd.cut(test['Aspect'],bins=[-1, 45, 90, 135, 180, 225, 270, 315, 360],\n",
    "           labels=['Aspect_1','Aspect_2','Aspect_3','Aspect_4','Aspect_5','Aspect_6','Aspect_7','Aspect_8'])\n",
    "b = pd.get_dummies(a)\n",
    "test.merge(b,right_index=True,left_index=True)\n",
    "\n",
    "test['interaction_9amnoon'] = test['Hillshade_9am']*test['Hillshade_Noon']\n",
    "test['interaction_noon3pm'] = test['Hillshade_3pm']*test['Hillshade_Noon']\n",
    "test['interaction_9am3pm'] = test['Hillshade_3pm']*test['Hillshade_9am']\n",
    "\n",
    "test['HF1'] = test['Horizontal_Distance_To_Hydrology']+test['Horizontal_Distance_To_Fire_Points']\n",
    "test['HF2'] = abs(test['Horizontal_Distance_To_Hydrology']-test['Horizontal_Distance_To_Fire_Points'])\n",
    "test['HR1'] = abs(test['Horizontal_Distance_To_Hydrology']+test['Horizontal_Distance_To_Roadways'])\n",
    "test['HR2'] = abs(test['Horizontal_Distance_To_Hydrology']-test['Horizontal_Distance_To_Roadways'])\n",
    "test['FR1'] = abs(test['Horizontal_Distance_To_Fire_Points']+test['Horizontal_Distance_To_Roadways'])\n",
    "test['FR2'] = abs(test['Horizontal_Distance_To_Fire_Points']-test['Horizontal_Distance_To_Roadways'])\n",
    "test['ele_vert'] = test.Elevation-test.Vertical_Distance_To_Hydrology\n",
    "\n",
    "test['slope_hyd'] = (test['Horizontal_Distance_To_Hydrology']**2+test['Vertical_Distance_To_Hydrology']**2)**0.5\n",
    "test.slope_hyd=test.slope_hyd.map(lambda x: 0 if np.isinf(x) else x) # remove infinite value if any\n",
    "\n",
    "#Mean distance to Amenities \n",
    "test['Mean_Amenities']=(test.Horizontal_Distance_To_Fire_Points + test.Horizontal_Distance_To_Hydrology + test.Horizontal_Distance_To_Roadways) / 3 \n",
    "#Mean Distance to Fire and Water \n",
    "test['Mean_Fire_Hyd']=(test.Horizontal_Distance_To_Fire_Points + test.Horizontal_Distance_To_Hydrology) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# a = pd.cut(train['Aspect'],bins=[-1, 45, 90, 135, 180, 225, 270, 315, 360],\n",
    "#            labels=['Aspect_1','Aspect_2','Aspect_3','Aspect_4','Aspect_5','Aspect_6','Aspect_7','Aspect_8'])\n",
    "# b = pd.get_dummies(a)\n",
    "# train.merge(b,right_index=True,left_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = train.drop(['Id','Cover_Type'],1)\n",
    "Y_train = train['Cover_Type']\n",
    "X_test = test.drop(['Id'],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "X_train_tf = pd.DataFrame(scaler.fit_transform(X_train),index=X_train.index, columns=X_train.columns)\n",
    "X_test_tf = pd.DataFrame(scaler.transform(X_test),index=X_test.index, columns=X_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>Horizontal_Distance_To_Fire_Points</th>\n",
       "      <th>...</th>\n",
       "      <th>interaction_9am3pm</th>\n",
       "      <th>HF1</th>\n",
       "      <th>HF2</th>\n",
       "      <th>HR1</th>\n",
       "      <th>HR2</th>\n",
       "      <th>FR1</th>\n",
       "      <th>FR2</th>\n",
       "      <th>ele_vert</th>\n",
       "      <th>Mean_Amenities</th>\n",
       "      <th>Mean_Fire_Hyd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.367095</td>\n",
       "      <td>-0.959980</td>\n",
       "      <td>-1.597132</td>\n",
       "      <td>0.146639</td>\n",
       "      <td>-0.834074</td>\n",
       "      <td>-0.908681</td>\n",
       "      <td>0.271454</td>\n",
       "      <td>0.571653</td>\n",
       "      <td>0.281259</td>\n",
       "      <td>4.334805</td>\n",
       "      <td>...</td>\n",
       "      <td>0.690441</td>\n",
       "      <td>4.165194</td>\n",
       "      <td>4.390517</td>\n",
       "      <td>-0.848236</td>\n",
       "      <td>-0.959357</td>\n",
       "      <td>1.702247</td>\n",
       "      <td>5.505993</td>\n",
       "      <td>-0.246574</td>\n",
       "      <td>1.673476</td>\n",
       "      <td>4.165194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.381461</td>\n",
       "      <td>-0.914559</td>\n",
       "      <td>-1.715424</td>\n",
       "      <td>-0.072337</td>\n",
       "      <td>-0.932054</td>\n",
       "      <td>-0.999246</td>\n",
       "      <td>0.238732</td>\n",
       "      <td>0.703225</td>\n",
       "      <td>0.346627</td>\n",
       "      <td>4.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>0.760204</td>\n",
       "      <td>4.078395</td>\n",
       "      <td>4.383090</td>\n",
       "      <td>-0.968254</td>\n",
       "      <td>-1.016605</td>\n",
       "      <td>1.619137</td>\n",
       "      <td>5.580595</td>\n",
       "      <td>-0.246574</td>\n",
       "      <td>1.571055</td>\n",
       "      <td>4.078395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.130912</td>\n",
       "      <td>-0.160577</td>\n",
       "      <td>-0.887379</td>\n",
       "      <td>0.194243</td>\n",
       "      <td>0.227369</td>\n",
       "      <td>1.106379</td>\n",
       "      <td>0.696843</td>\n",
       "      <td>0.834797</td>\n",
       "      <td>-0.002005</td>\n",
       "      <td>4.191156</td>\n",
       "      <td>...</td>\n",
       "      <td>0.538108</td>\n",
       "      <td>4.036731</td>\n",
       "      <td>4.234547</td>\n",
       "      <td>1.089401</td>\n",
       "      <td>1.098462</td>\n",
       "      <td>2.902092</td>\n",
       "      <td>2.309390</td>\n",
       "      <td>0.098281</td>\n",
       "      <td>2.847589</td>\n",
       "      <td>4.036731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.085421</td>\n",
       "      <td>-0.015231</td>\n",
       "      <td>0.177250</td>\n",
       "      <td>0.070474</td>\n",
       "      <td>1.092853</td>\n",
       "      <td>1.038455</td>\n",
       "      <td>0.827731</td>\n",
       "      <td>0.834797</td>\n",
       "      <td>-0.285268</td>\n",
       "      <td>4.272981</td>\n",
       "      <td>...</td>\n",
       "      <td>0.190113</td>\n",
       "      <td>4.092283</td>\n",
       "      <td>4.342240</td>\n",
       "      <td>1.005533</td>\n",
       "      <td>1.048950</td>\n",
       "      <td>2.902092</td>\n",
       "      <td>2.512851</td>\n",
       "      <td>-0.075352</td>\n",
       "      <td>2.835485</td>\n",
       "      <td>4.092283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.369489</td>\n",
       "      <td>-1.014485</td>\n",
       "      <td>-1.715424</td>\n",
       "      <td>-0.353198</td>\n",
       "      <td>-0.850404</td>\n",
       "      <td>-0.998491</td>\n",
       "      <td>0.238732</td>\n",
       "      <td>0.659368</td>\n",
       "      <td>0.324838</td>\n",
       "      <td>4.237524</td>\n",
       "      <td>...</td>\n",
       "      <td>0.730228</td>\n",
       "      <td>3.981180</td>\n",
       "      <td>4.388660</td>\n",
       "      <td>-1.010188</td>\n",
       "      <td>-0.970188</td>\n",
       "      <td>1.594299</td>\n",
       "      <td>5.519557</td>\n",
       "      <td>-0.246574</td>\n",
       "      <td>1.519379</td>\n",
       "      <td>3.981180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.407798</td>\n",
       "      <td>-0.224166</td>\n",
       "      <td>-1.242255</td>\n",
       "      <td>0.346574</td>\n",
       "      <td>-1.079023</td>\n",
       "      <td>-1.243015</td>\n",
       "      <td>0.565954</td>\n",
       "      <td>0.790940</td>\n",
       "      <td>0.106943</td>\n",
       "      <td>4.109330</td>\n",
       "      <td>...</td>\n",
       "      <td>0.621224</td>\n",
       "      <td>3.986388</td>\n",
       "      <td>4.121283</td>\n",
       "      <td>-1.138159</td>\n",
       "      <td>-0.974056</td>\n",
       "      <td>1.372194</td>\n",
       "      <td>5.726409</td>\n",
       "      <td>-0.251397</td>\n",
       "      <td>1.371335</td>\n",
       "      <td>3.986388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.343152</td>\n",
       "      <td>-1.014485</td>\n",
       "      <td>-1.123963</td>\n",
       "      <td>0.203764</td>\n",
       "      <td>-0.752425</td>\n",
       "      <td>-0.815853</td>\n",
       "      <td>0.304176</td>\n",
       "      <td>0.264652</td>\n",
       "      <td>0.063364</td>\n",
       "      <td>4.313894</td>\n",
       "      <td>...</td>\n",
       "      <td>0.408121</td>\n",
       "      <td>4.155646</td>\n",
       "      <td>4.358023</td>\n",
       "      <td>-0.750631</td>\n",
       "      <td>-0.873486</td>\n",
       "      <td>1.750011</td>\n",
       "      <td>5.340963</td>\n",
       "      <td>-0.234516</td>\n",
       "      <td>1.725617</td>\n",
       "      <td>4.155646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.345547</td>\n",
       "      <td>-0.978148</td>\n",
       "      <td>-1.478839</td>\n",
       "      <td>0.032391</td>\n",
       "      <td>-0.719765</td>\n",
       "      <td>-0.861135</td>\n",
       "      <td>0.304176</td>\n",
       "      <td>0.483939</td>\n",
       "      <td>0.194101</td>\n",
       "      <td>4.288437</td>\n",
       "      <td>...</td>\n",
       "      <td>0.589613</td>\n",
       "      <td>4.100095</td>\n",
       "      <td>4.365450</td>\n",
       "      <td>-0.820039</td>\n",
       "      <td>-0.892053</td>\n",
       "      <td>1.707979</td>\n",
       "      <td>5.377134</td>\n",
       "      <td>-0.241751</td>\n",
       "      <td>1.667889</td>\n",
       "      <td>4.100095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.316816</td>\n",
       "      <td>-1.014485</td>\n",
       "      <td>-0.887379</td>\n",
       "      <td>0.060953</td>\n",
       "      <td>0.080400</td>\n",
       "      <td>-0.790947</td>\n",
       "      <td>0.336898</td>\n",
       "      <td>0.089223</td>\n",
       "      <td>-0.045584</td>\n",
       "      <td>4.302984</td>\n",
       "      <td>...</td>\n",
       "      <td>0.275000</td>\n",
       "      <td>4.119190</td>\n",
       "      <td>4.374734</td>\n",
       "      <td>-0.748462</td>\n",
       "      <td>-0.824748</td>\n",
       "      <td>1.760042</td>\n",
       "      <td>5.290098</td>\n",
       "      <td>-0.330979</td>\n",
       "      <td>1.721427</td>\n",
       "      <td>4.119190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.328787</td>\n",
       "      <td>-0.887307</td>\n",
       "      <td>-0.769087</td>\n",
       "      <td>0.094276</td>\n",
       "      <td>-0.654445</td>\n",
       "      <td>-0.813589</td>\n",
       "      <td>0.500509</td>\n",
       "      <td>0.001508</td>\n",
       "      <td>-0.241689</td>\n",
       "      <td>4.290256</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086014</td>\n",
       "      <td>4.113114</td>\n",
       "      <td>4.355238</td>\n",
       "      <td>-0.765091</td>\n",
       "      <td>-0.853372</td>\n",
       "      <td>1.739026</td>\n",
       "      <td>5.308183</td>\n",
       "      <td>-0.234516</td>\n",
       "      <td>1.704202</td>\n",
       "      <td>4.113114</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 73 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Elevation    Aspect     Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0  -0.367095 -0.959980 -1.597132                          0.146639   \n",
       "1  -0.381461 -0.914559 -1.715424                         -0.072337   \n",
       "2   0.130912 -0.160577 -0.887379                          0.194243   \n",
       "3   0.085421 -0.015231  0.177250                          0.070474   \n",
       "4  -0.369489 -1.014485 -1.715424                         -0.353198   \n",
       "5  -0.407798 -0.224166 -1.242255                          0.346574   \n",
       "6  -0.343152 -1.014485 -1.123963                          0.203764   \n",
       "7  -0.345547 -0.978148 -1.478839                          0.032391   \n",
       "8  -0.316816 -1.014485 -0.887379                          0.060953   \n",
       "9  -0.328787 -0.887307 -0.769087                          0.094276   \n",
       "\n",
       "   Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                       -0.834074                        -0.908681   \n",
       "1                       -0.932054                        -0.999246   \n",
       "2                        0.227369                         1.106379   \n",
       "3                        1.092853                         1.038455   \n",
       "4                       -0.850404                        -0.998491   \n",
       "5                       -1.079023                        -1.243015   \n",
       "6                       -0.752425                        -0.815853   \n",
       "7                       -0.719765                        -0.861135   \n",
       "8                        0.080400                        -0.790947   \n",
       "9                       -0.654445                        -0.813589   \n",
       "\n",
       "   Hillshade_9am  Hillshade_Noon  Hillshade_3pm  \\\n",
       "0       0.271454        0.571653       0.281259   \n",
       "1       0.238732        0.703225       0.346627   \n",
       "2       0.696843        0.834797      -0.002005   \n",
       "3       0.827731        0.834797      -0.285268   \n",
       "4       0.238732        0.659368       0.324838   \n",
       "5       0.565954        0.790940       0.106943   \n",
       "6       0.304176        0.264652       0.063364   \n",
       "7       0.304176        0.483939       0.194101   \n",
       "8       0.336898        0.089223      -0.045584   \n",
       "9       0.500509        0.001508      -0.241689   \n",
       "\n",
       "   Horizontal_Distance_To_Fire_Points      ...        interaction_9am3pm  \\\n",
       "0                            4.334805      ...                  0.690441   \n",
       "1                            4.285710      ...                  0.760204   \n",
       "2                            4.191156      ...                  0.538108   \n",
       "3                            4.272981      ...                  0.190113   \n",
       "4                            4.237524      ...                  0.730228   \n",
       "5                            4.109330      ...                  0.621224   \n",
       "6                            4.313894      ...                  0.408121   \n",
       "7                            4.288437      ...                  0.589613   \n",
       "8                            4.302984      ...                  0.275000   \n",
       "9                            4.290256      ...                  0.086014   \n",
       "\n",
       "        HF1       HF2       HR1       HR2       FR1       FR2  ele_vert  \\\n",
       "0  4.165194  4.390517 -0.848236 -0.959357  1.702247  5.505993 -0.246574   \n",
       "1  4.078395  4.383090 -0.968254 -1.016605  1.619137  5.580595 -0.246574   \n",
       "2  4.036731  4.234547  1.089401  1.098462  2.902092  2.309390  0.098281   \n",
       "3  4.092283  4.342240  1.005533  1.048950  2.902092  2.512851 -0.075352   \n",
       "4  3.981180  4.388660 -1.010188 -0.970188  1.594299  5.519557 -0.246574   \n",
       "5  3.986388  4.121283 -1.138159 -0.974056  1.372194  5.726409 -0.251397   \n",
       "6  4.155646  4.358023 -0.750631 -0.873486  1.750011  5.340963 -0.234516   \n",
       "7  4.100095  4.365450 -0.820039 -0.892053  1.707979  5.377134 -0.241751   \n",
       "8  4.119190  4.374734 -0.748462 -0.824748  1.760042  5.290098 -0.330979   \n",
       "9  4.113114  4.355238 -0.765091 -0.853372  1.739026  5.308183 -0.234516   \n",
       "\n",
       "   Mean_Amenities  Mean_Fire_Hyd  \n",
       "0        1.673476       4.165194  \n",
       "1        1.571055       4.078395  \n",
       "2        2.847589       4.036731  \n",
       "3        2.835485       4.092283  \n",
       "4        1.519379       3.981180  \n",
       "5        1.371335       3.986388  \n",
       "6        1.725617       4.155646  \n",
       "7        1.667889       4.100095  \n",
       "8        1.721427       4.119190  \n",
       "9        1.704202       4.113114  \n",
       "\n",
       "[10 rows x 73 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tf.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **************Start\n",
    "## Light GBM       boosting_type='gbdt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "m5=lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=63,\n",
    "                       n_estimators=2000, \n",
    "                       max_depth=50,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=0.7,\n",
    "                       min_split_gain=0.1,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "\n",
    "m5.fit(X_train_tf, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Y_pred=m5.predict(X_train_tf)\n",
    "accuracy=accuracy_score(Y_pred,Y_train)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Val with n_estimators=[200,300,400, 1000]\n",
    "\n",
    "Best 400\n",
    "Better 1000, but not a significant change\n",
    "\n",
    "Default = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.7473545 ,  0.75628307,  0.75529101,  0.77149471,  0.84887566])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result=cross_val_score(lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=63,\n",
    "                       n_estimators=1000, \n",
    "                       max_depth=50,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=0.7,\n",
    "                       min_split_gain=0.1,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "                       ,X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "\n",
    "result\n",
    "# Cross Val Outcome\n",
    "# n_estimators=200\n",
    "# array([ 0.71858466,  0.73478836,  0.74702381,  0.75429894,  0.82903439])\n",
    "# n_estimators=300\n",
    "# array([ 0.73247354,  0.74702381,  0.75231481,  0.76653439,  0.84325397])\n",
    "# n_estimators=400\n",
    "# array([ 0.74603175,  0.75330688,  0.75198413,  0.77149471,  0.84722222])\n",
    "# n_estimators=1000\n",
    "# array([ 0.7473545 ,  0.75628307,  0.75529101,  0.77149471,  0.84887566])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Val with min_split_gain= [0.0, 0.05, 0.2, 0.3]\n",
    "\n",
    "Best 0.05\n",
    "\n",
    "\n",
    "Default = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.74933862,  0.74966931,  0.75793651,  0.7744709 ,  0.84920635])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result=cross_val_score(lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=63,\n",
    "                       n_estimators=400, \n",
    "                       max_depth=50,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=0.7,\n",
    "                       min_split_gain=0.05,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "                       ,X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "\n",
    "result\n",
    "\n",
    "# Cross Val Outcome\n",
    "# min_split_gain=0.0\n",
    "# array([ 0.74933862,  0.74966931,  0.75793651,  0.7744709 ,  0.84920635])\n",
    "# min_split_gain=0.05\n",
    "# array([ 0.75066138,  0.75231481,  0.75165344,  0.77281746,  0.85019841])\n",
    "# min_split_gain=0.2\n",
    "# array([ 0.73115079,  0.74239418,  0.75727513,  0.76719577,  0.84093915])\n",
    "# min_split_gain=0.3\n",
    "# array([ 0.7228836 ,  0.73511905,  0.75330688,  0.75496032,  0.8369709 ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Val with colsample_bytree= [0.6, 0.7, 0.8, 0.9]\n",
    "Best 0.9 but very close to 0.7\n",
    "\n",
    "\n",
    "Default = 1 (all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.74834656,  0.75165344,  0.75760582,  0.77480159,  0.84722222])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result=cross_val_score(lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=63,\n",
    "                       n_estimators=400, \n",
    "                       max_depth=50,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=1.0,\n",
    "                       min_split_gain=0.05,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "                       ,X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "\n",
    "result\n",
    "\n",
    "# Cross Val Outcome\n",
    "# colsample_bytree= 1.0\n",
    "# array([ 0.74834656,  0.75165344,  0.75760582,  0.77480159,  0.84722222])\n",
    "# colsample_bytree= 0.9\n",
    "# array([ 0.75297619,  0.75628307,  0.75859788,  0.77513228,  0.85019841])\n",
    "# colsample_bytree= 0.8\n",
    "# array([ 0.74768519,  0.75066138,  0.76256614,  0.76917989,  0.84986772])\n",
    "# colsample_bytree= 0.7\n",
    "# array([ 0.75066138,  0.75231481,  0.75165344,  0.77281746,  0.85019841])\n",
    "# colsample_bytree= 0.6\n",
    "# array([ 0.74272487,  0.74537037,  0.75396825,  0.77414021,  0.8505291 ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Val with max_depth= [10, 20, 40, 50, 60]\n",
    "\n",
    "Best is achieved @ 20\n",
    "\n",
    "\n",
    "Default= -1 (No limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.74603175,  0.75      ,  0.75628307,  0.76752646,  0.84887566])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result=cross_val_score(lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=63,\n",
    "                       n_estimators=400, \n",
    "                       max_depth=20,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=0.9,\n",
    "                       min_split_gain=0.05,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "                       ,X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "\n",
    "result\n",
    "\n",
    "# Cross Val Outcome\n",
    "# max_depth=10\n",
    "# array([ 0.74603175,  0.75      ,  0.75628307,  0.76752646,  0.84887566])\n",
    "# max_depth=[20,40,50,60]\n",
    "# array([ 0.75297619,  0.75628307,  0.75859788,  0.77513228,  0.85019841])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Val with num_leaves= [20, 31, 40, 63]\n",
    "\n",
    "Default seems to work well\n",
    "\n",
    "Default= 31"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.7473545 ,  0.75033069,  0.76124339,  0.7718254 ,  0.84821429])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result=cross_val_score(lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=31,\n",
    "                       n_estimators=400, \n",
    "                       max_depth=20,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=0.9,\n",
    "                       min_split_gain=0.05,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "                       ,X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "\n",
    "result\n",
    "\n",
    "# Cross Val Outcome\n",
    "# num_leaves= [20,]\n",
    "# array([ 0.74537037,  0.75297619,  0.75958995,  0.77612434,  0.85185185])\n",
    "# num_leaves= [31,40,63]\n",
    "# array([ 0.75297619,  0.75628307,  0.75859788,  0.77513228,  0.85019841])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Val with bagging\n",
    "\n",
    "Bagging is not giving a better result than the current configuration\n",
    "\n",
    "\n",
    "Default is no bagging\n",
    "bagging_freq= None\n",
    "bagging_fraction=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.72982804,  0.75396825,  0.75628307,  0.77314815,  0.84259259])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "result=cross_val_score(lgb.LGBMClassifier(boosting_type='gbdt',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=31,\n",
    "                       n_estimators=400, \n",
    "                       max_depth=20,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=0.9,\n",
    "                       min_split_gain=0.05,\n",
    "                       bagging_freq=3,\n",
    "                       bagging_fraction=0.9,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "                       ,X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "\n",
    "result\n",
    "\n",
    "# Cross Val Outcome\n",
    "# Best so far\n",
    "# array([ 0.74537037,  0.75297619,  0.75958995,  0.77612434,  0.85185185])\n",
    "\n",
    "# bagging_freq=5\n",
    "# bagging_fraction=0.8\n",
    "# array([ 0.73710317,  0.73974868,  0.75396825,  0.75892857,  0.83399471])\n",
    "\n",
    "# bagging_freq=5\n",
    "# bagging_fraction=0.9\n",
    "# array([ 0.74503968,  0.75165344,  0.75661376,  0.77149471,  0.84457672])\n",
    "\n",
    "# bagging_freq=8\n",
    "# bagging_fraction=0.9\n",
    "# array([ 0.73875661,  0.74570106,  0.75595238,  0.77050265,  0.84292328])\n",
    "\n",
    "# bagging_freq=3\n",
    "# bagging_fraction=0.9\n",
    "# array([ 0.72982804,  0.75396825,  0.75628307,  0.77314815,  0.84259259])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **************Start\n",
    "## Light GBM       boosting_type='dart'\n",
    "\n",
    "When boosting_type='rf' then colsample_bytree=1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.68584656,  0.70899471,  0.73974868,  0.74041005,  0.8042328 ])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result=cross_val_score(lgb.LGBMClassifier(boosting_type='dart',\n",
    "                       objective='multiclass',\n",
    "                       num_leaves=31,\n",
    "                       n_estimators=400, \n",
    "                       max_depth=20,\n",
    "                       learning_rate=0.1,\n",
    "                       min_data_in_leaf=500,\n",
    "                       colsample_bytree=1.0,\n",
    "                       min_split_gain=0.05,\n",
    "                       classes=[1,2,3,4,5,6,7]\n",
    "                       )\n",
    "                       ,X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "\n",
    "result\n",
    "# Cross Val Outcome\n",
    "# boosting_type='gbdt'\n",
    "# array([ 0.74537037,  0.75297619,  0.75958995,  0.77612434,  0.85185185])\n",
    "# boosting_type='dart'\n",
    "# array([ 0.68584656,  0.70899471,  0.73974868,  0.74041005,  0.8042328 ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lgb.LGBMClassifier?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Previous Attempt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "d_train = lgb.Dataset(X_train_tf,label=Y_train_temp)\n",
    "params = {}\n",
    "params['learning_rate'] = 0.003\n",
    "params['device'] = 'cpu'\n",
    "params['boosting_type'] = 'dart'\n",
    "#params['boosting_type'] = 'rf'\n",
    "#params['boosting_type'] = 'gbdt'\n",
    "params['application'] = 'multiclass'\n",
    "params['num_class']= 7\n",
    "params['metric'] = 'multi_logloss'\n",
    "params['sub_feature'] = 0.5\n",
    "params['num_leaves'] = 30\n",
    "params['min_data'] = 100\n",
    "params['max_depth'] = 15\n",
    "clf = lgb.train(params, d_train, 500)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}