Feature Added XGBoost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Things to do\n",
    "##### Differernt Models\n",
    "#### 1. Gradient Boosting Classifier\n",
    "#### 2. Isolation classifier\n",
    "#### 3. Extra Tree classifier\n",
    "#### 4. Voting classifier\n",
    "##### Feature Selection/Engineering\n",
    "##### ANN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# importing libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Creation for train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['HF1'] = train['Horizontal_Distance_To_Hydrology']+train['Horizontal_Distance_To_Fire_Points']\n",
    "train['HF2'] = abs(train['Horizontal_Distance_To_Hydrology']-train['Horizontal_Distance_To_Fire_Points'])\n",
    "train['HR1'] = abs(train['Horizontal_Distance_To_Hydrology']+train['Horizontal_Distance_To_Roadways'])\n",
    "train['HR2'] = abs(train['Horizontal_Distance_To_Hydrology']-train['Horizontal_Distance_To_Roadways'])\n",
    "train['FR1'] = abs(train['Horizontal_Distance_To_Fire_Points']+train['Horizontal_Distance_To_Roadways'])\n",
    "train['FR2'] = abs(train['Horizontal_Distance_To_Fire_Points']-train['Horizontal_Distance_To_Roadways'])\n",
    "train['ele_vert'] = train.Elevation-train.Vertical_Distance_To_Hydrology\n",
    "\n",
    "train['slope_hyd'] = (train['Horizontal_Distance_To_Hydrology']**2+train['Vertical_Distance_To_Hydrology']**2)**0.5\n",
    "train.slope_hyd=train.slope_hyd.map(lambda x: 0 if np.isinf(x) else x) # remove infinite value if any\n",
    "\n",
    "#Mean distance to Amenities \n",
    "train['Mean_Amenities']=(train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology + train.Horizontal_Distance_To_Roadways) / 3 \n",
    "#Mean Distance to Fire and Water \n",
    "train['Mean_Fire_Hyd']=(train.Horizontal_Distance_To_Fire_Points + train.Horizontal_Distance_To_Hydrology) / 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Creation for test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['HF1'] = test['Horizontal_Distance_To_Hydrology']+test['Horizontal_Distance_To_Fire_Points']\n",
    "test['HF2'] = abs(test['Horizontal_Distance_To_Hydrology']-test['Horizontal_Distance_To_Fire_Points'])\n",
    "test['HR1'] = abs(test['Horizontal_Distance_To_Hydrology']+test['Horizontal_Distance_To_Roadways'])\n",
    "test['HR2'] = abs(test['Horizontal_Distance_To_Hydrology']-test['Horizontal_Distance_To_Roadways'])\n",
    "test['FR1'] = abs(test['Horizontal_Distance_To_Fire_Points']+test['Horizontal_Distance_To_Roadways'])\n",
    "test['FR2'] = abs(test['Horizontal_Distance_To_Fire_Points']-test['Horizontal_Distance_To_Roadways'])\n",
    "test['ele_vert'] = test.Elevation-test.Vertical_Distance_To_Hydrology\n",
    "\n",
    "test['slope_hyd'] = (test['Horizontal_Distance_To_Hydrology']**2+test['Vertical_Distance_To_Hydrology']**2)**0.5\n",
    "test.slope_hyd=test.slope_hyd.map(lambda x: 0 if np.isinf(x) else x) # remove infinite value if any\n",
    "\n",
    "#Mean distance to Amenities \n",
    "test['Mean_Amenities']=(test.Horizontal_Distance_To_Fire_Points + test.Horizontal_Distance_To_Hydrology + test.Horizontal_Distance_To_Roadways) / 3 \n",
    "#Mean Distance to Fire and Water \n",
    "test['Mean_Fire_Hyd']=(test.Horizontal_Distance_To_Fire_Points + test.Horizontal_Distance_To_Hydrology) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = train.drop(['Id','Cover_Type'],1)\n",
    "Y_train = train['Cover_Type']\n",
    "X_test = test.drop(['Id'],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "X_train_tf = pd.DataFrame(scaler.fit_transform(X_train),index=X_train.index, columns=X_train.columns)\n",
    "X_test_tf = pd.DataFrame(scaler.transform(X_test),index=X_test.index, columns=X_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Aspect</th>\n",
       "      <th>Slope</th>\n",
       "      <th>Horizontal_Distance_To_Hydrology</th>\n",
       "      <th>Vertical_Distance_To_Hydrology</th>\n",
       "      <th>Horizontal_Distance_To_Roadways</th>\n",
       "      <th>Hillshade_9am</th>\n",
       "      <th>Hillshade_Noon</th>\n",
       "      <th>Hillshade_3pm</th>\n",
       "      <th>Horizontal_Distance_To_Fire_Points</th>\n",
       "      <th>...</th>\n",
       "      <th>HF1</th>\n",
       "      <th>HF2</th>\n",
       "      <th>HR1</th>\n",
       "      <th>HR2</th>\n",
       "      <th>FR1</th>\n",
       "      <th>FR2</th>\n",
       "      <th>ele_vert</th>\n",
       "      <th>slope_hyd</th>\n",
       "      <th>Mean_Amenities</th>\n",
       "      <th>Mean_Fire_Hyd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.367095</td>\n",
       "      <td>-0.959980</td>\n",
       "      <td>-1.597132</td>\n",
       "      <td>0.146639</td>\n",
       "      <td>-0.834074</td>\n",
       "      <td>-0.908681</td>\n",
       "      <td>0.271454</td>\n",
       "      <td>0.571653</td>\n",
       "      <td>0.281259</td>\n",
       "      <td>4.334805</td>\n",
       "      <td>...</td>\n",
       "      <td>4.165194</td>\n",
       "      <td>4.390517</td>\n",
       "      <td>-0.848236</td>\n",
       "      <td>-0.959357</td>\n",
       "      <td>1.702247</td>\n",
       "      <td>5.505993</td>\n",
       "      <td>-0.246574</td>\n",
       "      <td>0.102333</td>\n",
       "      <td>1.673476</td>\n",
       "      <td>4.165194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.381461</td>\n",
       "      <td>-0.914559</td>\n",
       "      <td>-1.715424</td>\n",
       "      <td>-0.072337</td>\n",
       "      <td>-0.932054</td>\n",
       "      <td>-0.999246</td>\n",
       "      <td>0.238732</td>\n",
       "      <td>0.703225</td>\n",
       "      <td>0.346627</td>\n",
       "      <td>4.285710</td>\n",
       "      <td>...</td>\n",
       "      <td>4.078395</td>\n",
       "      <td>4.383090</td>\n",
       "      <td>-0.968254</td>\n",
       "      <td>-1.016605</td>\n",
       "      <td>1.619137</td>\n",
       "      <td>5.580595</td>\n",
       "      <td>-0.246574</td>\n",
       "      <td>-0.110745</td>\n",
       "      <td>1.571055</td>\n",
       "      <td>4.078395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.130912</td>\n",
       "      <td>-0.160577</td>\n",
       "      <td>-0.887379</td>\n",
       "      <td>0.194243</td>\n",
       "      <td>0.227369</td>\n",
       "      <td>1.106379</td>\n",
       "      <td>0.696843</td>\n",
       "      <td>0.834797</td>\n",
       "      <td>-0.002005</td>\n",
       "      <td>4.191156</td>\n",
       "      <td>...</td>\n",
       "      <td>4.036731</td>\n",
       "      <td>4.234547</td>\n",
       "      <td>1.089401</td>\n",
       "      <td>1.098462</td>\n",
       "      <td>2.902092</td>\n",
       "      <td>2.309390</td>\n",
       "      <td>0.098281</td>\n",
       "      <td>0.184798</td>\n",
       "      <td>2.847589</td>\n",
       "      <td>4.036731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.085421</td>\n",
       "      <td>-0.015231</td>\n",
       "      <td>0.177250</td>\n",
       "      <td>0.070474</td>\n",
       "      <td>1.092853</td>\n",
       "      <td>1.038455</td>\n",
       "      <td>0.827731</td>\n",
       "      <td>0.834797</td>\n",
       "      <td>-0.285268</td>\n",
       "      <td>4.272981</td>\n",
       "      <td>...</td>\n",
       "      <td>4.092283</td>\n",
       "      <td>4.342240</td>\n",
       "      <td>1.005533</td>\n",
       "      <td>1.048950</td>\n",
       "      <td>2.902092</td>\n",
       "      <td>2.512851</td>\n",
       "      <td>-0.075352</td>\n",
       "      <td>0.154476</td>\n",
       "      <td>2.835485</td>\n",
       "      <td>4.092283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.369489</td>\n",
       "      <td>-1.014485</td>\n",
       "      <td>-1.715424</td>\n",
       "      <td>-0.353198</td>\n",
       "      <td>-0.850404</td>\n",
       "      <td>-0.998491</td>\n",
       "      <td>0.238732</td>\n",
       "      <td>0.659368</td>\n",
       "      <td>0.324838</td>\n",
       "      <td>4.237524</td>\n",
       "      <td>...</td>\n",
       "      <td>3.981180</td>\n",
       "      <td>4.388660</td>\n",
       "      <td>-1.010188</td>\n",
       "      <td>-0.970188</td>\n",
       "      <td>1.594299</td>\n",
       "      <td>5.519557</td>\n",
       "      <td>-0.246574</td>\n",
       "      <td>-0.384926</td>\n",
       "      <td>1.519379</td>\n",
       "      <td>3.981180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.407798</td>\n",
       "      <td>-0.224166</td>\n",
       "      <td>-1.242255</td>\n",
       "      <td>0.346574</td>\n",
       "      <td>-1.079023</td>\n",
       "      <td>-1.243015</td>\n",
       "      <td>0.565954</td>\n",
       "      <td>0.790940</td>\n",
       "      <td>0.106943</td>\n",
       "      <td>4.109330</td>\n",
       "      <td>...</td>\n",
       "      <td>3.986388</td>\n",
       "      <td>4.121283</td>\n",
       "      <td>-1.138159</td>\n",
       "      <td>-0.974056</td>\n",
       "      <td>1.372194</td>\n",
       "      <td>5.726409</td>\n",
       "      <td>-0.251397</td>\n",
       "      <td>0.298982</td>\n",
       "      <td>1.371335</td>\n",
       "      <td>3.986388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>-0.343152</td>\n",
       "      <td>-1.014485</td>\n",
       "      <td>-1.123963</td>\n",
       "      <td>0.203764</td>\n",
       "      <td>-0.752425</td>\n",
       "      <td>-0.815853</td>\n",
       "      <td>0.304176</td>\n",
       "      <td>0.264652</td>\n",
       "      <td>0.063364</td>\n",
       "      <td>4.313894</td>\n",
       "      <td>...</td>\n",
       "      <td>4.155646</td>\n",
       "      <td>4.358023</td>\n",
       "      <td>-0.750631</td>\n",
       "      <td>-0.873486</td>\n",
       "      <td>1.750011</td>\n",
       "      <td>5.340963</td>\n",
       "      <td>-0.234516</td>\n",
       "      <td>0.158236</td>\n",
       "      <td>1.725617</td>\n",
       "      <td>4.155646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-0.345547</td>\n",
       "      <td>-0.978148</td>\n",
       "      <td>-1.478839</td>\n",
       "      <td>0.032391</td>\n",
       "      <td>-0.719765</td>\n",
       "      <td>-0.861135</td>\n",
       "      <td>0.304176</td>\n",
       "      <td>0.483939</td>\n",
       "      <td>0.194101</td>\n",
       "      <td>4.288437</td>\n",
       "      <td>...</td>\n",
       "      <td>4.100095</td>\n",
       "      <td>4.365450</td>\n",
       "      <td>-0.820039</td>\n",
       "      <td>-0.892053</td>\n",
       "      <td>1.707979</td>\n",
       "      <td>5.377134</td>\n",
       "      <td>-0.241751</td>\n",
       "      <td>-0.008558</td>\n",
       "      <td>1.667889</td>\n",
       "      <td>4.100095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-0.316816</td>\n",
       "      <td>-1.014485</td>\n",
       "      <td>-0.887379</td>\n",
       "      <td>0.060953</td>\n",
       "      <td>0.080400</td>\n",
       "      <td>-0.790947</td>\n",
       "      <td>0.336898</td>\n",
       "      <td>0.089223</td>\n",
       "      <td>-0.045584</td>\n",
       "      <td>4.302984</td>\n",
       "      <td>...</td>\n",
       "      <td>4.119190</td>\n",
       "      <td>4.374734</td>\n",
       "      <td>-0.748462</td>\n",
       "      <td>-0.824748</td>\n",
       "      <td>1.760042</td>\n",
       "      <td>5.290098</td>\n",
       "      <td>-0.330979</td>\n",
       "      <td>0.048718</td>\n",
       "      <td>1.721427</td>\n",
       "      <td>4.119190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-0.328787</td>\n",
       "      <td>-0.887307</td>\n",
       "      <td>-0.769087</td>\n",
       "      <td>0.094276</td>\n",
       "      <td>-0.654445</td>\n",
       "      <td>-0.813589</td>\n",
       "      <td>0.500509</td>\n",
       "      <td>0.001508</td>\n",
       "      <td>-0.241689</td>\n",
       "      <td>4.290256</td>\n",
       "      <td>...</td>\n",
       "      <td>4.113114</td>\n",
       "      <td>4.355238</td>\n",
       "      <td>-0.765091</td>\n",
       "      <td>-0.853372</td>\n",
       "      <td>1.739026</td>\n",
       "      <td>5.308183</td>\n",
       "      <td>-0.234516</td>\n",
       "      <td>0.052422</td>\n",
       "      <td>1.704202</td>\n",
       "      <td>4.113114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-0.328787</td>\n",
       "      <td>0.402639</td>\n",
       "      <td>-1.478839</td>\n",
       "      <td>-0.224668</td>\n",
       "      <td>-0.001250</td>\n",
       "      <td>-0.738873</td>\n",
       "      <td>0.173287</td>\n",
       "      <td>1.054084</td>\n",
       "      <td>0.564522</td>\n",
       "      <td>4.282982</td>\n",
       "      <td>...</td>\n",
       "      <td>4.048015</td>\n",
       "      <td>4.410013</td>\n",
       "      <td>-0.741955</td>\n",
       "      <td>-0.724952</td>\n",
       "      <td>1.782491</td>\n",
       "      <td>5.187237</td>\n",
       "      <td>-0.330979</td>\n",
       "      <td>-0.226760</td>\n",
       "      <td>1.715375</td>\n",
       "      <td>4.048015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.327242</td>\n",
       "      <td>-0.051567</td>\n",
       "      <td>-0.650795</td>\n",
       "      <td>0.684560</td>\n",
       "      <td>-0.409497</td>\n",
       "      <td>2.670880</td>\n",
       "      <td>0.696843</td>\n",
       "      <td>0.922512</td>\n",
       "      <td>0.019785</td>\n",
       "      <td>2.309167</td>\n",
       "      <td>...</td>\n",
       "      <td>2.329392</td>\n",
       "      <td>2.217152</td>\n",
       "      <td>2.662647</td>\n",
       "      <td>2.622486</td>\n",
       "      <td>2.903525</td>\n",
       "      <td>0.343727</td>\n",
       "      <td>0.390082</td>\n",
       "      <td>0.630955</td>\n",
       "      <td>2.896937</td>\n",
       "      <td>2.329392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-0.017532</td>\n",
       "      <td>-0.205998</td>\n",
       "      <td>0.650419</td>\n",
       "      <td>-0.367479</td>\n",
       "      <td>0.292689</td>\n",
       "      <td>1.132793</td>\n",
       "      <td>1.154954</td>\n",
       "      <td>0.220795</td>\n",
       "      <td>-0.938953</td>\n",
       "      <td>4.163881</td>\n",
       "      <td>...</td>\n",
       "      <td>3.908268</td>\n",
       "      <td>4.316245</td>\n",
       "      <td>1.029392</td>\n",
       "      <td>1.216825</td>\n",
       "      <td>2.904480</td>\n",
       "      <td>2.235917</td>\n",
       "      <td>-0.060883</td>\n",
       "      <td>-0.328746</td>\n",
       "      <td>2.794982</td>\n",
       "      <td>3.908268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>-0.335970</td>\n",
       "      <td>0.520732</td>\n",
       "      <td>-1.123963</td>\n",
       "      <td>-0.367479</td>\n",
       "      <td>-0.082899</td>\n",
       "      <td>-0.711704</td>\n",
       "      <td>0.009676</td>\n",
       "      <td>1.229513</td>\n",
       "      <td>0.760628</td>\n",
       "      <td>4.272981</td>\n",
       "      <td>...</td>\n",
       "      <td>4.012427</td>\n",
       "      <td>4.427652</td>\n",
       "      <td>-0.737617</td>\n",
       "      <td>-0.673893</td>\n",
       "      <td>1.794432</td>\n",
       "      <td>5.134111</td>\n",
       "      <td>-0.326156</td>\n",
       "      <td>-0.366866</td>\n",
       "      <td>1.713048</td>\n",
       "      <td>4.012427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>-0.589762</td>\n",
       "      <td>0.002937</td>\n",
       "      <td>-1.478839</td>\n",
       "      <td>-0.762588</td>\n",
       "      <td>-0.768755</td>\n",
       "      <td>-0.784910</td>\n",
       "      <td>0.369620</td>\n",
       "      <td>0.922512</td>\n",
       "      <td>0.346627</td>\n",
       "      <td>3.717476</td>\n",
       "      <td>...</td>\n",
       "      <td>3.410041</td>\n",
       "      <td>3.937462</td>\n",
       "      <td>-0.867757</td>\n",
       "      <td>-0.684723</td>\n",
       "      <td>1.456259</td>\n",
       "      <td>4.553116</td>\n",
       "      <td>-0.480497</td>\n",
       "      <td>-0.783488</td>\n",
       "      <td>1.344799</td>\n",
       "      <td>3.410041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>-0.608916</td>\n",
       "      <td>-0.959980</td>\n",
       "      <td>-1.123963</td>\n",
       "      <td>-0.881597</td>\n",
       "      <td>-0.801414</td>\n",
       "      <td>-0.726043</td>\n",
       "      <td>0.369620</td>\n",
       "      <td>0.264652</td>\n",
       "      <td>0.041574</td>\n",
       "      <td>3.695656</td>\n",
       "      <td>...</td>\n",
       "      <td>3.367510</td>\n",
       "      <td>3.938390</td>\n",
       "      <td>-0.829438</td>\n",
       "      <td>-0.605041</td>\n",
       "      <td>1.482052</td>\n",
       "      <td>4.437821</td>\n",
       "      <td>-0.494967</td>\n",
       "      <td>-0.899838</td>\n",
       "      <td>1.358300</td>\n",
       "      <td>3.367510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>-0.333575</td>\n",
       "      <td>0.929518</td>\n",
       "      <td>-1.833716</td>\n",
       "      <td>-0.510290</td>\n",
       "      <td>-0.850404</td>\n",
       "      <td>-0.835475</td>\n",
       "      <td>0.107843</td>\n",
       "      <td>0.878655</td>\n",
       "      <td>0.564522</td>\n",
       "      <td>4.168426</td>\n",
       "      <td>...</td>\n",
       "      <td>3.886569</td>\n",
       "      <td>4.348739</td>\n",
       "      <td>-0.877879</td>\n",
       "      <td>-0.777557</td>\n",
       "      <td>1.661169</td>\n",
       "      <td>5.189497</td>\n",
       "      <td>-0.210401</td>\n",
       "      <td>-0.538065</td>\n",
       "      <td>1.569193</td>\n",
       "      <td>3.886569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>-0.556242</td>\n",
       "      <td>-0.769213</td>\n",
       "      <td>-1.123963</td>\n",
       "      <td>-0.676902</td>\n",
       "      <td>-0.736095</td>\n",
       "      <td>-0.844531</td>\n",
       "      <td>0.500509</td>\n",
       "      <td>0.352367</td>\n",
       "      <td>-0.045584</td>\n",
       "      <td>3.723841</td>\n",
       "      <td>...</td>\n",
       "      <td>3.431741</td>\n",
       "      <td>3.927249</td>\n",
       "      <td>-0.911860</td>\n",
       "      <td>-0.759764</td>\n",
       "      <td>1.421869</td>\n",
       "      <td>4.650325</td>\n",
       "      <td>-0.451558</td>\n",
       "      <td>-0.699527</td>\n",
       "      <td>1.319659</td>\n",
       "      <td>3.431741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>-0.587368</td>\n",
       "      <td>-1.423270</td>\n",
       "      <td>-1.478839</td>\n",
       "      <td>-0.629299</td>\n",
       "      <td>-0.752425</td>\n",
       "      <td>-0.772080</td>\n",
       "      <td>0.042398</td>\n",
       "      <td>0.571653</td>\n",
       "      <td>0.455575</td>\n",
       "      <td>3.692020</td>\n",
       "      <td>...</td>\n",
       "      <td>3.410041</td>\n",
       "      <td>3.885472</td>\n",
       "      <td>-0.835222</td>\n",
       "      <td>-0.693233</td>\n",
       "      <td>1.451005</td>\n",
       "      <td>4.502250</td>\n",
       "      <td>-0.480497</td>\n",
       "      <td>-0.653491</td>\n",
       "      <td>1.352713</td>\n",
       "      <td>3.410041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>-0.589762</td>\n",
       "      <td>-1.078073</td>\n",
       "      <td>-1.360547</td>\n",
       "      <td>-0.676902</td>\n",
       "      <td>-0.670775</td>\n",
       "      <td>-0.734345</td>\n",
       "      <td>0.238732</td>\n",
       "      <td>0.396224</td>\n",
       "      <td>0.194101</td>\n",
       "      <td>3.676564</td>\n",
       "      <td>...</td>\n",
       "      <td>3.386605</td>\n",
       "      <td>3.878973</td>\n",
       "      <td>-0.806302</td>\n",
       "      <td>-0.646816</td>\n",
       "      <td>1.466768</td>\n",
       "      <td>4.426518</td>\n",
       "      <td>-0.494967</td>\n",
       "      <td>-0.697788</td>\n",
       "      <td>1.363421</td>\n",
       "      <td>3.386605</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 64 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Elevation    Aspect     Slope  Horizontal_Distance_To_Hydrology  \\\n",
       "0   -0.367095 -0.959980 -1.597132                          0.146639   \n",
       "1   -0.381461 -0.914559 -1.715424                         -0.072337   \n",
       "2    0.130912 -0.160577 -0.887379                          0.194243   \n",
       "3    0.085421 -0.015231  0.177250                          0.070474   \n",
       "4   -0.369489 -1.014485 -1.715424                         -0.353198   \n",
       "5   -0.407798 -0.224166 -1.242255                          0.346574   \n",
       "6   -0.343152 -1.014485 -1.123963                          0.203764   \n",
       "7   -0.345547 -0.978148 -1.478839                          0.032391   \n",
       "8   -0.316816 -1.014485 -0.887379                          0.060953   \n",
       "9   -0.328787 -0.887307 -0.769087                          0.094276   \n",
       "10  -0.328787  0.402639 -1.478839                         -0.224668   \n",
       "11   0.327242 -0.051567 -0.650795                          0.684560   \n",
       "12  -0.017532 -0.205998  0.650419                         -0.367479   \n",
       "13  -0.335970  0.520732 -1.123963                         -0.367479   \n",
       "14  -0.589762  0.002937 -1.478839                         -0.762588   \n",
       "15  -0.608916 -0.959980 -1.123963                         -0.881597   \n",
       "16  -0.333575  0.929518 -1.833716                         -0.510290   \n",
       "17  -0.556242 -0.769213 -1.123963                         -0.676902   \n",
       "18  -0.587368 -1.423270 -1.478839                         -0.629299   \n",
       "19  -0.589762 -1.078073 -1.360547                         -0.676902   \n",
       "\n",
       "    Vertical_Distance_To_Hydrology  Horizontal_Distance_To_Roadways  \\\n",
       "0                        -0.834074                        -0.908681   \n",
       "1                        -0.932054                        -0.999246   \n",
       "2                         0.227369                         1.106379   \n",
       "3                         1.092853                         1.038455   \n",
       "4                        -0.850404                        -0.998491   \n",
       "5                        -1.079023                        -1.243015   \n",
       "6                        -0.752425                        -0.815853   \n",
       "7                        -0.719765                        -0.861135   \n",
       "8                         0.080400                        -0.790947   \n",
       "9                        -0.654445                        -0.813589   \n",
       "10                       -0.001250                        -0.738873   \n",
       "11                       -0.409497                         2.670880   \n",
       "12                        0.292689                         1.132793   \n",
       "13                       -0.082899                        -0.711704   \n",
       "14                       -0.768755                        -0.784910   \n",
       "15                       -0.801414                        -0.726043   \n",
       "16                       -0.850404                        -0.835475   \n",
       "17                       -0.736095                        -0.844531   \n",
       "18                       -0.752425                        -0.772080   \n",
       "19                       -0.670775                        -0.734345   \n",
       "\n",
       "    Hillshade_9am  Hillshade_Noon  Hillshade_3pm  \\\n",
       "0        0.271454        0.571653       0.281259   \n",
       "1        0.238732        0.703225       0.346627   \n",
       "2        0.696843        0.834797      -0.002005   \n",
       "3        0.827731        0.834797      -0.285268   \n",
       "4        0.238732        0.659368       0.324838   \n",
       "5        0.565954        0.790940       0.106943   \n",
       "6        0.304176        0.264652       0.063364   \n",
       "7        0.304176        0.483939       0.194101   \n",
       "8        0.336898        0.089223      -0.045584   \n",
       "9        0.500509        0.001508      -0.241689   \n",
       "10       0.173287        1.054084       0.564522   \n",
       "11       0.696843        0.922512       0.019785   \n",
       "12       1.154954        0.220795      -0.938953   \n",
       "13       0.009676        1.229513       0.760628   \n",
       "14       0.369620        0.922512       0.346627   \n",
       "15       0.369620        0.264652       0.041574   \n",
       "16       0.107843        0.878655       0.564522   \n",
       "17       0.500509        0.352367      -0.045584   \n",
       "18       0.042398        0.571653       0.455575   \n",
       "19       0.238732        0.396224       0.194101   \n",
       "\n",
       "    Horizontal_Distance_To_Fire_Points      ...             HF1       HF2  \\\n",
       "0                             4.334805      ...        4.165194  4.390517   \n",
       "1                             4.285710      ...        4.078395  4.383090   \n",
       "2                             4.191156      ...        4.036731  4.234547   \n",
       "3                             4.272981      ...        4.092283  4.342240   \n",
       "4                             4.237524      ...        3.981180  4.388660   \n",
       "5                             4.109330      ...        3.986388  4.121283   \n",
       "6                             4.313894      ...        4.155646  4.358023   \n",
       "7                             4.288437      ...        4.100095  4.365450   \n",
       "8                             4.302984      ...        4.119190  4.374734   \n",
       "9                             4.290256      ...        4.113114  4.355238   \n",
       "10                            4.282982      ...        4.048015  4.410013   \n",
       "11                            2.309167      ...        2.329392  2.217152   \n",
       "12                            4.163881      ...        3.908268  4.316245   \n",
       "13                            4.272981      ...        4.012427  4.427652   \n",
       "14                            3.717476      ...        3.410041  3.937462   \n",
       "15                            3.695656      ...        3.367510  3.938390   \n",
       "16                            4.168426      ...        3.886569  4.348739   \n",
       "17                            3.723841      ...        3.431741  3.927249   \n",
       "18                            3.692020      ...        3.410041  3.885472   \n",
       "19                            3.676564      ...        3.386605  3.878973   \n",
       "\n",
       "         HR1       HR2       FR1       FR2  ele_vert  slope_hyd  \\\n",
       "0  -0.848236 -0.959357  1.702247  5.505993 -0.246574   0.102333   \n",
       "1  -0.968254 -1.016605  1.619137  5.580595 -0.246574  -0.110745   \n",
       "2   1.089401  1.098462  2.902092  2.309390  0.098281   0.184798   \n",
       "3   1.005533  1.048950  2.902092  2.512851 -0.075352   0.154476   \n",
       "4  -1.010188 -0.970188  1.594299  5.519557 -0.246574  -0.384926   \n",
       "5  -1.138159 -0.974056  1.372194  5.726409 -0.251397   0.298982   \n",
       "6  -0.750631 -0.873486  1.750011  5.340963 -0.234516   0.158236   \n",
       "7  -0.820039 -0.892053  1.707979  5.377134 -0.241751  -0.008558   \n",
       "8  -0.748462 -0.824748  1.760042  5.290098 -0.330979   0.048718   \n",
       "9  -0.765091 -0.853372  1.739026  5.308183 -0.234516   0.052422   \n",
       "10 -0.741955 -0.724952  1.782491  5.187237 -0.330979  -0.226760   \n",
       "11  2.662647  2.622486  2.903525  0.343727  0.390082   0.630955   \n",
       "12  1.029392  1.216825  2.904480  2.235917 -0.060883  -0.328746   \n",
       "13 -0.737617 -0.673893  1.794432  5.134111 -0.326156  -0.366866   \n",
       "14 -0.867757 -0.684723  1.456259  4.553116 -0.480497  -0.783488   \n",
       "15 -0.829438 -0.605041  1.482052  4.437821 -0.494967  -0.899838   \n",
       "16 -0.877879 -0.777557  1.661169  5.189497 -0.210401  -0.538065   \n",
       "17 -0.911860 -0.759764  1.421869  4.650325 -0.451558  -0.699527   \n",
       "18 -0.835222 -0.693233  1.451005  4.502250 -0.480497  -0.653491   \n",
       "19 -0.806302 -0.646816  1.466768  4.426518 -0.494967  -0.697788   \n",
       "\n",
       "    Mean_Amenities  Mean_Fire_Hyd  \n",
       "0         1.673476       4.165194  \n",
       "1         1.571055       4.078395  \n",
       "2         2.847589       4.036731  \n",
       "3         2.835485       4.092283  \n",
       "4         1.519379       3.981180  \n",
       "5         1.371335       3.986388  \n",
       "6         1.725617       4.155646  \n",
       "7         1.667889       4.100095  \n",
       "8         1.721427       4.119190  \n",
       "9         1.704202       4.113114  \n",
       "10        1.715375       4.048015  \n",
       "11        2.896937       2.329392  \n",
       "12        2.794982       3.908268  \n",
       "13        1.713048       4.012427  \n",
       "14        1.344799       3.410041  \n",
       "15        1.358300       3.367510  \n",
       "16        1.569193       3.886569  \n",
       "17        1.319659       3.431741  \n",
       "18        1.352713       3.410041  \n",
       "19        1.363421       3.386605  \n",
       "\n",
       "[20 rows x 64 columns]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_tf.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.76752646  0.75562169  0.78075397  0.78637566  0.85846561]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.78974867724867726"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "xgbc=xgb.XGBClassifier(objective = \"multi:softmax\",\n",
    "                       eval_metric = \"merror\",\n",
    "                       max_depth = 40,\n",
    "                       eta = 0.0399,\n",
    "                       gamma = 1.2393,\n",
    "                       subsample = 0.7052,\n",
    "                       colsample_bytree = 0.6296,\n",
    "                       min_child_weight = 1,\n",
    "                       colsample_bylevel = 1,\n",
    "                       reg_lambda = 1, \n",
    "                       reg_alpha = 0, \n",
    "                       num_class = 7,\n",
    "                       booster = \"gbtree\",\n",
    "                       silent = 0)\n",
    "results=cross_val_score(xgbc,X_train,Y_train,cv=5,n_jobs=8)\n",
    "print(results)\n",
    "results.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n",
      "End\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process ForkPoolWorker-491:\n",
      "Process ForkPoolWorker-490:\n",
      "Process ForkPoolWorker-489:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/site-packages/sklearn/externals/joblib/pool.py\", line 360, in get\n",
      "    racquire()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/site-packages/sklearn/externals/joblib/pool.py\", line 360, in get\n",
      "    racquire()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/site-packages/sklearn/externals/joblib/pool.py\", line 362, in get\n",
      "    return recv()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/connection.py\", line 250, in recv\n",
      "    buf = self._recv_bytes()\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-5f503cbe11c7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     23\u001b[0m                                                    \u001b[0mmax_depth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_depth\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m                                                    min_child_weight = min_child_weight),\n\u001b[0;32m---> 25\u001b[0;31m                                 X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n\u001b[0m\u001b[1;32m     26\u001b[0m             \u001b[0mresults_rf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'accuracy_min'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0mresults_rf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'accuracy_max'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_val_score\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch)\u001b[0m\n\u001b[1;32m    340\u001b[0m                                 \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    341\u001b[0m                                 \u001b[0mfit_params\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 342\u001b[0;31m                                 pre_dispatch=pre_dispatch)\n\u001b[0m\u001b[1;32m    343\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcv_results\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'test_score'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    344\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36mcross_validate\u001b[0;34m(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch, return_train_score)\u001b[0m\n\u001b[1;32m    204\u001b[0m             \u001b[0mfit_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_train_score\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_train_score\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m             return_times=True)\n\u001b[0;32m--> 206\u001b[0;31m         for train, test in cv.split(X, y, groups))\n\u001b[0m\u001b[1;32m    207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_train_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    787\u001b[0m                 \u001b[0;31m# consumption.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 789\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    790\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/site-packages/sklearn/externals/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    697\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    698\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 699\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    700\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    701\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 602\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    603\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mready\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    597\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 599\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    600\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    547\u001b[0m             \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 549\u001b[0;31m                 \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    550\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/z013lw7/anaconda/lib/python3.5/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    291\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    292\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 293\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    294\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "results_rf = pd.DataFrame(data=None,columns=['n_estimator','nin_child_weight','max_depth',\n",
    "                                            'accuracy_min','accuracy_max','accuracy_mean'])\n",
    "\n",
    "N_estimators = [40,60,80,100,120]\n",
    "Min_child_weight = [1,2,3,4,5,6,7]\n",
    "Max_depth = [20,30,40,50,60,70]\n",
    "i=0\n",
    "for n_estimator in N_estimators:\n",
    "    for min_child_weight in Min_child_weight:\n",
    "        for max_depth in Max_depth:\n",
    "            results_rf.loc[i,'n_estimator'] = n_estimator\n",
    "            results_rf.loc[i,'min_child_weight'] = min_child_weight\n",
    "            results_rf.loc[i,'max_depth'] = max_depth\n",
    "            r = cross_val_score(xgb.XGBClassifier( objective = \"multi:softmax\",\n",
    "                                                   booster = \"gbtree\",\n",
    "                                                   num_class = 7,\n",
    "                                                   random_state=100,\n",
    "                                                   n_estimators=n_estimator,\n",
    "                                                   max_depth=max_depth,\n",
    "                                                   min_child_weight = min_child_weight),\n",
    "                                X=X_train_tf,y=Y_train,scoring='accuracy',cv=5,n_jobs=4)\n",
    "            results_rf.loc[i,'accuracy_min'] = r.min()\n",
    "            results_rf.loc[i,'accuracy_max'] = r.max()\n",
    "            results_rf.loc[i,'accuracy_mean'] = r.mean()\n",
    "            print(\"End\")\n",
    "            i = i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "results_rf.to_csv(\"gridsearch.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    " max depth=40  \n",
    "n_estimators>80\n",
    "min_child weight=1\n",
    "    \n",
    "    eta = 0.0399,\n",
    "                                                   gamma = 1.2393,\n",
    "                                                   subsample = 0.7052,\n",
    "                                                   colsample_bytree = 0.6296,\n",
    "\n",
    "                                                   colsample_bylevel = 1,\n",
    "                                                   reg_lambda = 1, \n",
    "                                                   reg_alpha = 0, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.79828042  0.78075397  0.78968254  0.80059524  0.87334656]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.80853174603174605"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "xgbc=xgb.XGBClassifier(objective = \"multi:softmax\",\n",
    "                       n_estimator=80,\n",
    "                       max_depth = 40,\n",
    "                       min_child_weight = 1,\n",
    "                       num_class = 7,\n",
    "                       booster = \"gbtree\",\n",
    "                       n_jobs=4)\n",
    "results=cross_val_score(xgbc,X_train,Y_train,cv=5,n_jobs=8)\n",
    "print(results)\n",
    "results.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}