base_algos/python/7.2. Regression tree.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import sklearn\n",
    "from sklearn.tree import DecisionTreeRegressor, export_graphviz\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from mlxtend.plotting import plot_decision_regions, plot_confusion_matrix\n",
    "from IPython.display import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>price</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>sqft_living</th>\n",
       "      <th>sqft_lot</th>\n",
       "      <th>floors</th>\n",
       "      <th>waterfront</th>\n",
       "      <th>view</th>\n",
       "      <th>...</th>\n",
       "      <th>grade</th>\n",
       "      <th>sqft_above</th>\n",
       "      <th>sqft_basement</th>\n",
       "      <th>yr_built</th>\n",
       "      <th>yr_renovated</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "      <th>sqft_living15</th>\n",
       "      <th>sqft_lot15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7129300520</td>\n",
       "      <td>20141013T000000</td>\n",
       "      <td>221900.0</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1180</td>\n",
       "      <td>5650</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>1180</td>\n",
       "      <td>0</td>\n",
       "      <td>1955</td>\n",
       "      <td>0</td>\n",
       "      <td>98178</td>\n",
       "      <td>47.5112</td>\n",
       "      <td>-122.257</td>\n",
       "      <td>1340</td>\n",
       "      <td>5650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6414100192</td>\n",
       "      <td>20141209T000000</td>\n",
       "      <td>538000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.25</td>\n",
       "      <td>2570</td>\n",
       "      <td>7242</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>2170</td>\n",
       "      <td>400</td>\n",
       "      <td>1951</td>\n",
       "      <td>1991</td>\n",
       "      <td>98125</td>\n",
       "      <td>47.7210</td>\n",
       "      <td>-122.319</td>\n",
       "      <td>1690</td>\n",
       "      <td>7639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5631500400</td>\n",
       "      <td>20150225T000000</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1.00</td>\n",
       "      <td>770</td>\n",
       "      <td>10000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>770</td>\n",
       "      <td>0</td>\n",
       "      <td>1933</td>\n",
       "      <td>0</td>\n",
       "      <td>98028</td>\n",
       "      <td>47.7379</td>\n",
       "      <td>-122.233</td>\n",
       "      <td>2720</td>\n",
       "      <td>8062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2487200875</td>\n",
       "      <td>20141209T000000</td>\n",
       "      <td>604000.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1960</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>1050</td>\n",
       "      <td>910</td>\n",
       "      <td>1965</td>\n",
       "      <td>0</td>\n",
       "      <td>98136</td>\n",
       "      <td>47.5208</td>\n",
       "      <td>-122.393</td>\n",
       "      <td>1360</td>\n",
       "      <td>5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1954400510</td>\n",
       "      <td>20150218T000000</td>\n",
       "      <td>510000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.00</td>\n",
       "      <td>1680</td>\n",
       "      <td>8080</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>8</td>\n",
       "      <td>1680</td>\n",
       "      <td>0</td>\n",
       "      <td>1987</td>\n",
       "      <td>0</td>\n",
       "      <td>98074</td>\n",
       "      <td>47.6168</td>\n",
       "      <td>-122.045</td>\n",
       "      <td>1800</td>\n",
       "      <td>7503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           id             date     price  bedrooms  bathrooms  sqft_living  \\\n",
       "0  7129300520  20141013T000000  221900.0         3       1.00         1180   \n",
       "1  6414100192  20141209T000000  538000.0         3       2.25         2570   \n",
       "2  5631500400  20150225T000000  180000.0         2       1.00          770   \n",
       "3  2487200875  20141209T000000  604000.0         4       3.00         1960   \n",
       "4  1954400510  20150218T000000  510000.0         3       2.00         1680   \n",
       "\n",
       "   sqft_lot  floors  waterfront  view     ...      grade  sqft_above  \\\n",
       "0      5650     1.0           0     0     ...          7        1180   \n",
       "1      7242     2.0           0     0     ...          7        2170   \n",
       "2     10000     1.0           0     0     ...          6         770   \n",
       "3      5000     1.0           0     0     ...          7        1050   \n",
       "4      8080     1.0           0     0     ...          8        1680   \n",
       "\n",
       "   sqft_basement  yr_built  yr_renovated  zipcode      lat     long  \\\n",
       "0              0      1955             0    98178  47.5112 -122.257   \n",
       "1            400      1951          1991    98125  47.7210 -122.319   \n",
       "2              0      1933             0    98028  47.7379 -122.233   \n",
       "3            910      1965             0    98136  47.5208 -122.393   \n",
       "4              0      1987             0    98074  47.6168 -122.045   \n",
       "\n",
       "   sqft_living15  sqft_lot15  \n",
       "0           1340        5650  \n",
       "1           1690        7639  \n",
       "2           2720        8062  \n",
       "3           1360        5000  \n",
       "4           1800        7503  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "house = pd.read_csv('../../datasets/kc_house_data.csv')\n",
    "house.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = house.loc[:, ['bedrooms', 'sqft_living', 'grade', 'waterfront', 'yr_built', 'sqft_basement']]\n",
    "y = house.loc[:, 'price']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train - Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split( X, y, test_size = 0.3, \n",
    "                                                   random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=None, max_features=None,\n",
       "           max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           presort=False, random_state=None, splitter='best')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = DecisionTreeRegressor()\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualization\n",
    "If graphviz is not installed then paste the generated graph in http://webgraphviz.com/ to visualize it.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dotfile = open(\"D:/regressor.dot\", 'w')\n",
    "export_graphviz(model, out_file = dotfile, feature_names = X.columns)\n",
    "dotfile.close()\n",
    "# Image(filename='D:/regressor_image.PNG') # Tree takes time to build"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction and RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE:  28860.57996223972\n"
     ]
    }
   ],
   "source": [
    "# Train RMSE\n",
    "est = model.predict(X_train)\n",
    "rmse = np.sqrt(metrics.mean_squared_error(y_train, est))\n",
    "print('Train RMSE: ', rmse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test RMSE:  286675.2055023114\n"
     ]
    }
   ],
   "source": [
    "# Test RMSE\n",
    "est = model.predict(X_test)\n",
    "rmse = np.sqrt(metrics.mean_squared_error(y_test, est))\n",
    "print('Test RMSE: ', rmse)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>which feature plays more imp. role?</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bedrooms</th>\n",
       "      <td>2.857266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sqft_living</th>\n",
       "      <td>33.732786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grade</th>\n",
       "      <td>39.850454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>waterfront</th>\n",
       "      <td>2.882267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yr_built</th>\n",
       "      <td>15.228335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sqft_basement</th>\n",
       "      <td>5.448891</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               which feature plays more imp. role?\n",
       "bedrooms                                  2.857266\n",
       "sqft_living                              33.732786\n",
       "grade                                    39.850454\n",
       "waterfront                                2.882267\n",
       "yr_built                                 15.228335\n",
       "sqft_basement                             5.448891"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(model.feature_importances_*100, index=X.columns, columns=['which feature plays more imp. role?'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Changing the model parameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeRegressor(criterion='mse', max_depth=6, max_features='auto',\n",
       "           max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=3,\n",
       "           min_samples_split=3, min_weight_fraction_leaf=0.0,\n",
       "           presort=False, random_state=None, splitter='best')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_n = DecisionTreeRegressor(max_depth=6, # If None then internal nodes are splitted until \n",
    "                                             # leaf nodes have less than min_samples_split samples. \n",
    "                                min_samples_split=3, # The minimum number of samples required to split an internal node\n",
    "                                min_samples_leaf=3,  # The minimum number of samples required to be at a leaf node\n",
    "                                max_features='auto', # Number of features to be cosidered while splitting \n",
    "                                                     # auto = sqrt(n_features)\n",
    "                               )\n",
    "model_n.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE:  191606.86524988583\n",
      "Test RMSE:  229863.73440058585\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>which feature plays more imp. role?</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bedrooms</th>\n",
       "      <td>0.134542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sqft_living</th>\n",
       "      <td>33.189000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>grade</th>\n",
       "      <td>53.779681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>waterfront</th>\n",
       "      <td>2.873081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yr_built</th>\n",
       "      <td>9.461005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sqft_basement</th>\n",
       "      <td>0.562691</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               which feature plays more imp. role?\n",
       "bedrooms                                  0.134542\n",
       "sqft_living                              33.189000\n",
       "grade                                    53.779681\n",
       "waterfront                                2.873081\n",
       "yr_built                                  9.461005\n",
       "sqft_basement                             0.562691"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train RMSE\n",
    "est = model_n.predict(X_train)\n",
    "rmse = np.sqrt(metrics.mean_squared_error(y_train, est))\n",
    "print('Train RMSE: ', rmse)\n",
    "# Test RMSE\n",
    "est = model_n.predict(X_test)\n",
    "rmse = np.sqrt(metrics.mean_squared_error(y_test, est))\n",
    "print('Test RMSE: ', rmse)\n",
    "# Feature importance\n",
    "pd.DataFrame(model_n.feature_importances_*100, index=X.columns, columns=['which feature plays more imp. role?'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding the best paramters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 45 candidates, totalling 135 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 135 out of 135 | elapsed:    1.5s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=KFold(n_splits=3, random_state=42, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=DecisionTreeRegressor(criterion='mse', max_depth=None, max_features=None,\n",
       "           max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "           min_impurity_split=None, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           presort=False, random_state=None, splitter='best'),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'max_depth': range(1, 11, 2), 'min_samples_split': range(2, 5), 'min_samples_leaf': range(2, 5)},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "param_grid = dict(max_depth=range(1, 11, 2), \n",
    "                  min_samples_split=range(2,5), \n",
    "                  min_samples_leaf=range(2,5))\n",
    "Kcv = KFold(n_splits=3, shuffle=True, random_state=42)\n",
    "grid_search = GridSearchCV(model, param_grid, verbose=1, cv=Kcv)\n",
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE:  184199.15828133587\n",
      "Test RMSE:  231424.3994334205\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'max_depth': 7, 'min_samples_leaf': 3, 'min_samples_split': 2}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train RMSE\n",
    "est = grid_search.predict(X_train)\n",
    "rmse = np.sqrt(metrics.mean_squared_error(y_train, est))\n",
    "print('Train RMSE: ', rmse)\n",
    "# Test RMSE\n",
    "est = grid_search.predict(X_test)\n",
    "rmse = np.sqrt(metrics.mean_squared_error(y_test, est))\n",
    "print('Test RMSE: ', rmse)\n",
    "\n",
    "# Best parameters\n",
    "grid_search.best_params_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}