finetuning_forest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine-Tuning Random Forest Regressor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook assumes the data has been split into a training and a test set. If not, run get_data.ipynb first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "TRAINING_FILEPATH = 'data/training_set.csv'\n",
    "TEST_FILEPATH = 'data/test_set.csv'\n",
    "\n",
    "training_set = pd.read_csv(TRAINING_FILEPATH, index_col='index')\n",
    "test_set = pd.read_csv(TEST_FILEPATH, index_col='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocessing_utils import FeaturePreprocessor, separate_features_targets\n",
    "\n",
    "train_X, train_y = separate_features_targets(training_set)\n",
    "\n",
    "# preprocess training features (add combinations, power transform)\n",
    "preprocessor = FeaturePreprocessor(add_combinations=True, powertransform_num=True, onehot_type=True)\n",
    "train_X = preprocessor.fit_transform(train_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Starting 5-fold cross validation\nfit 0\ttrain RMSE: 0.299\t val RMSE: 0.815\t train time: 5.72 s\nfit 1\ttrain RMSE: 0.299\t val RMSE: 0.797\t train time: 5.69 s\nfit 2\ttrain RMSE: 0.298\t val RMSE: 0.811\t train time: 5.62 s\nfit 3\ttrain RMSE: 0.300\t val RMSE: 0.815\t train time: 5.64 s\nfit 4\ttrain RMSE: 0.300\t val RMSE: 0.786\t train time: 5.62 s\n"
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "         fold 0    fold 1    fold 2    fold 3    fold 4      mean      std\ntrain  0.299417  0.299162  0.298058  0.300439  0.300183  0.299452  0.00094\nval    0.814788  0.797459  0.811265  0.814552  0.785646  0.804742  0.01282",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>fold 0</th>\n      <th>fold 1</th>\n      <th>fold 2</th>\n      <th>fold 3</th>\n      <th>fold 4</th>\n      <th>mean</th>\n      <th>std</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>train</th>\n      <td>0.299417</td>\n      <td>0.299162</td>\n      <td>0.298058</td>\n      <td>0.300439</td>\n      <td>0.300183</td>\n      <td>0.299452</td>\n      <td>0.00094</td>\n    </tr>\n    <tr>\n      <th>val</th>\n      <td>0.814788</td>\n      <td>0.797459</td>\n      <td>0.811265</td>\n      <td>0.814552</td>\n      <td>0.785646</td>\n      <td>0.804742</td>\n      <td>0.01282</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {}
    }
   ],
   "source": [
    "from train_utils import cross_val_rmse\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# random forest regressor model with default hyperparameters\n",
    "base_forest = RandomForestRegressor(n_estimators=100)\n",
    "base_forest_errors = cross_val_rmse(base_forest, train_X, train_y, cv=5,\n",
    "                                    random_state=42, verbose=True)\n",
    "display(base_forest_errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The base model is overfitting the training folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from train_utils import save_model, load_model\n",
    "\n",
    "save_model(base_forest, \"base_forest.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest Regressor Hyperparameters:\n",
    "- n_estimators: number of trees in the foreset\n",
    "- max_features: max number of features considered for splitting a node\n",
    "- max_depth: max number of levels in each decision tree\n",
    "- min_samples_split: min number of data points placed in a node before the node is split\n",
    "- min_samples_leaf: min number of data points allowed in a leaf node\n",
    "- bootstrap: sample points without replacement (True) or use the whole dataset for each tree (False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "# run a randomized search on hyperparameters\n",
    "forest_random_params = {\n",
    "    'n_estimators': [int(x) for x in np.linspace(200, 2000, 10)],\n",
    "    'max_features': ['auto', 'sqrt'],\n",
    "    'max_depth': [int(x) for x in np.linspace(10, 110, 11)] + [None],\n",
    "    'min_samples_split': [2, 10, 100, 1000],\n",
    "    'min_samples_leaf': [1, 5, 50, 500],\n",
    "    'bootstrap': [False, True]\n",
    "}\n",
    "\n",
    "forest_random_search = RandomizedSearchCV(RandomForestRegressor(), forest_random_params, n_iter=100,\n",
    "                                          scoring='neg_mean_squared_error', cv=5, n_jobs=-1, verbose=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": [
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "[Parallel(n_jobs=-1)]: Done 168 tasks      | elapsed: 16.6min\n[Parallel(n_jobs=-1)]: Done 169 tasks      | elapsed: 16.6min\n[Parallel(n_jobs=-1)]: Done 170 tasks      | elapsed: 16.6min\n[Parallel(n_jobs=-1)]: Done 171 tasks      | elapsed: 16.6min\n[Parallel(n_jobs=-1)]: Done 172 tasks      | elapsed: 16.7min\n[Parallel(n_jobs=-1)]: Done 173 tasks      | elapsed: 16.7min\n[Parallel(n_jobs=-1)]: Done 174 tasks      | elapsed: 16.7min\n[Parallel(n_jobs=-1)]: Done 175 tasks      | elapsed: 16.8min\n[Parallel(n_jobs=-1)]: Done 176 tasks      | elapsed: 16.8min\n[Parallel(n_jobs=-1)]: Done 177 tasks      | elapsed: 16.8min\n[Parallel(n_jobs=-1)]: Done 178 tasks      | elapsed: 16.9min\n[Parallel(n_jobs=-1)]: Done 179 tasks      | elapsed: 17.0min\n[Parallel(n_jobs=-1)]: Done 180 tasks      | elapsed: 17.0min\n[Parallel(n_jobs=-1)]: Done 181 tasks      | elapsed: 17.1min\n[Parallel(n_jobs=-1)]: Done 182 tasks      | elapsed: 17.1min\n[Parallel(n_jobs=-1)]: Done 183 tasks      | elapsed: 17.1min\n[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed: 17.1min\n[Parallel(n_jobs=-1)]: Done 185 tasks      | elapsed: 17.1min\n[Parallel(n_jobs=-1)]: Done 186 tasks      | elapsed: 17.7min\n[Parallel(n_jobs=-1)]: Done 187 tasks      | elapsed: 17.7min\n[Parallel(n_jobs=-1)]: Done 188 tasks      | elapsed: 17.7min\n[Parallel(n_jobs=-1)]: Done 189 tasks      | elapsed: 17.8min\n[Parallel(n_jobs=-1)]: Done 190 tasks      | elapsed: 17.9min\n[Parallel(n_jobs=-1)]: Done 191 tasks      | elapsed: 17.9min\n[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed: 18.0min\n[Parallel(n_jobs=-1)]: Done 193 tasks      | elapsed: 18.0min\n[Parallel(n_jobs=-1)]: Done 194 tasks      | elapsed: 18.0min\n[Parallel(n_jobs=-1)]: Done 195 tasks      | elapsed: 18.0min\n[Parallel(n_jobs=-1)]: Done 196 tasks      | elapsed: 18.0min\n[Parallel(n_jobs=-1)]: Done 197 tasks      | elapsed: 18.1min\n[Parallel(n_jobs=-1)]: Done 198 tasks      | elapsed: 18.1min\n[Parallel(n_jobs=-1)]: Done 199 tasks      | elapsed: 18.2min\n[Parallel(n_jobs=-1)]: Done 200 tasks      | elapsed: 18.4min\n[Parallel(n_jobs=-1)]: Done 201 tasks      | elapsed: 18.5min\n[Parallel(n_jobs=-1)]: Done 202 tasks      | elapsed: 18.5min\n[Parallel(n_jobs=-1)]: Done 203 tasks      | elapsed: 18.5min\n[Parallel(n_jobs=-1)]: Done 204 tasks      | elapsed: 18.5min\n[Parallel(n_jobs=-1)]: Done 205 tasks      | elapsed: 18.5min\n[Parallel(n_jobs=-1)]: Done 206 tasks      | elapsed: 18.5min\n[Parallel(n_jobs=-1)]: Done 207 tasks      | elapsed: 18.6min\n[Parallel(n_jobs=-1)]: Done 208 tasks      | elapsed: 18.6min\n[Parallel(n_jobs=-1)]: Done 209 tasks      | elapsed: 18.6min\n[Parallel(n_jobs=-1)]: Done 210 tasks      | elapsed: 18.6min\n[Parallel(n_jobs=-1)]: Done 211 tasks      | elapsed: 19.8min\n[Parallel(n_jobs=-1)]: Done 212 tasks      | elapsed: 19.8min\n[Parallel(n_jobs=-1)]: Done 213 tasks      | elapsed: 19.8min\n[Parallel(n_jobs=-1)]: Done 214 tasks      | elapsed: 19.9min\n[Parallel(n_jobs=-1)]: Done 215 tasks      | elapsed: 19.9min\n[Parallel(n_jobs=-1)]: Done 216 tasks      | elapsed: 20.0min\n[Parallel(n_jobs=-1)]: Done 217 tasks      | elapsed: 20.0min\n[Parallel(n_jobs=-1)]: Done 218 tasks      | elapsed: 20.1min\n[Parallel(n_jobs=-1)]: Done 219 tasks      | elapsed: 20.2min\n[Parallel(n_jobs=-1)]: Done 220 tasks      | elapsed: 20.2min\n[Parallel(n_jobs=-1)]: Done 221 tasks      | elapsed: 20.4min\n[Parallel(n_jobs=-1)]: Done 222 tasks      | elapsed: 20.5min\n[Parallel(n_jobs=-1)]: Done 223 tasks      | elapsed: 20.6min\n[Parallel(n_jobs=-1)]: Done 224 tasks      | elapsed: 21.6min\n[Parallel(n_jobs=-1)]: Done 225 tasks      | elapsed: 21.7min\n[Parallel(n_jobs=-1)]: Done 226 tasks      | elapsed: 22.4min\n[Parallel(n_jobs=-1)]: Done 227 tasks      | elapsed: 23.4min\n[Parallel(n_jobs=-1)]: Done 228 tasks      | elapsed: 23.5min\n[Parallel(n_jobs=-1)]: Done 229 tasks      | elapsed: 24.1min\n[Parallel(n_jobs=-1)]: Done 230 tasks      | elapsed: 24.1min\n[Parallel(n_jobs=-1)]: Done 231 tasks      | elapsed: 24.1min\n[Parallel(n_jobs=-1)]: Done 232 tasks      | elapsed: 24.3min\n[Parallel(n_jobs=-1)]: Done 233 tasks      | elapsed: 24.3min\n[Parallel(n_jobs=-1)]: Done 234 tasks      | elapsed: 24.3min\n[Parallel(n_jobs=-1)]: Done 235 tasks      | elapsed: 24.5min\n[Parallel(n_jobs=-1)]: Done 236 tasks      | elapsed: 24.5min\n[Parallel(n_jobs=-1)]: Done 237 tasks      | elapsed: 24.5min\n[Parallel(n_jobs=-1)]: Done 238 tasks      | elapsed: 24.7min\n[Parallel(n_jobs=-1)]: Done 239 tasks      | elapsed: 24.7min\n[Parallel(n_jobs=-1)]: Done 240 tasks      | elapsed: 24.7min\n[Parallel(n_jobs=-1)]: Done 241 tasks      | elapsed: 24.9min\n[Parallel(n_jobs=-1)]: Done 242 tasks      | elapsed: 24.9min\n[Parallel(n_jobs=-1)]: Done 243 tasks      | elapsed: 24.9min\n[Parallel(n_jobs=-1)]: Done 244 tasks      | elapsed: 25.1min\n[Parallel(n_jobs=-1)]: Done 245 tasks      | elapsed: 25.2min\n[Parallel(n_jobs=-1)]: Done 246 tasks      | elapsed: 25.2min\n[Parallel(n_jobs=-1)]: Done 247 tasks      | elapsed: 25.2min\n[Parallel(n_jobs=-1)]: Done 248 tasks      | elapsed: 25.3min\n[Parallel(n_jobs=-1)]: Done 249 tasks      | elapsed: 25.4min\n[Parallel(n_jobs=-1)]: Done 250 tasks      | elapsed: 25.4min\n[Parallel(n_jobs=-1)]: Done 251 tasks      | elapsed: 25.4min\n[Parallel(n_jobs=-1)]: Done 252 tasks      | elapsed: 25.4min\n[Parallel(n_jobs=-1)]: Done 253 tasks      | elapsed: 25.5min\n[Parallel(n_jobs=-1)]: Done 254 tasks      | elapsed: 25.5min\n[Parallel(n_jobs=-1)]: Done 255 tasks      | elapsed: 25.5min\n[Parallel(n_jobs=-1)]: Done 256 tasks      | elapsed: 26.4min\n[Parallel(n_jobs=-1)]: Done 257 tasks      | elapsed: 26.5min\n[Parallel(n_jobs=-1)]: Done 258 tasks      | elapsed: 26.5min\n[Parallel(n_jobs=-1)]: Done 259 tasks      | elapsed: 27.3min\n[Parallel(n_jobs=-1)]: Done 260 tasks      | elapsed: 27.5min\n[Parallel(n_jobs=-1)]: Done 261 tasks      | elapsed: 28.2min\n[Parallel(n_jobs=-1)]: Done 262 tasks      | elapsed: 29.0min\n[Parallel(n_jobs=-1)]: Done 263 tasks      | elapsed: 29.2min\n[Parallel(n_jobs=-1)]: Done 264 tasks      | elapsed: 29.2min\n[Parallel(n_jobs=-1)]: Done 265 tasks      | elapsed: 29.2min\n[Parallel(n_jobs=-1)]: Done 266 tasks      | elapsed: 29.3min\n[Parallel(n_jobs=-1)]: Done 267 tasks      | elapsed: 29.3min\n[Parallel(n_jobs=-1)]: Done 268 tasks      | elapsed: 29.4min\n[Parallel(n_jobs=-1)]: Done 269 tasks      | elapsed: 29.4min\n[Parallel(n_jobs=-1)]: Done 270 tasks      | elapsed: 29.5min\n[Parallel(n_jobs=-1)]: Done 271 tasks      | elapsed: 29.5min\n[Parallel(n_jobs=-1)]: Done 272 tasks      | elapsed: 29.6min\n[Parallel(n_jobs=-1)]: Done 273 tasks      | elapsed: 29.6min\n[Parallel(n_jobs=-1)]: Done 274 tasks      | elapsed: 29.7min\n[Parallel(n_jobs=-1)]: Done 275 tasks      | elapsed: 29.8min\n[Parallel(n_jobs=-1)]: Done 276 tasks      | elapsed: 29.8min\n[Parallel(n_jobs=-1)]: Done 277 tasks      | elapsed: 29.8min\n[Parallel(n_jobs=-1)]: Done 278 tasks      | elapsed: 29.8min\n[Parallel(n_jobs=-1)]: Done 279 tasks      | elapsed: 29.9min\n[Parallel(n_jobs=-1)]: Done 280 tasks      | elapsed: 29.9min\n[Parallel(n_jobs=-1)]: Done 281 tasks      | elapsed: 30.0min\n[Parallel(n_jobs=-1)]: Done 282 tasks      | elapsed: 30.0min\n[Parallel(n_jobs=-1)]: Done 283 tasks      | elapsed: 30.1min\n[Parallel(n_jobs=-1)]: Done 284 tasks      | elapsed: 30.1min\n[Parallel(n_jobs=-1)]: Done 285 tasks      | elapsed: 30.8min\n[Parallel(n_jobs=-1)]: Done 286 tasks      | elapsed: 31.0min\n[Parallel(n_jobs=-1)]: Done 287 tasks      | elapsed: 31.1min\n[Parallel(n_jobs=-1)]: Done 288 tasks      | elapsed: 31.2min\n[Parallel(n_jobs=-1)]: Done 289 tasks      | elapsed: 31.3min\n[Parallel(n_jobs=-1)]: Done 290 tasks      | elapsed: 31.3min\n[Parallel(n_jobs=-1)]: Done 291 tasks      | elapsed: 31.4min\n[Parallel(n_jobs=-1)]: Done 292 tasks      | elapsed: 31.4min\n[Parallel(n_jobs=-1)]: Done 293 tasks      | elapsed: 31.4min\n[Parallel(n_jobs=-1)]: Done 294 tasks      | elapsed: 31.5min\n[Parallel(n_jobs=-1)]: Done 295 tasks      | elapsed: 31.6min\n[Parallel(n_jobs=-1)]: Done 296 tasks      | elapsed: 31.6min\n[Parallel(n_jobs=-1)]: Done 297 tasks      | elapsed: 31.7min\n[Parallel(n_jobs=-1)]: Done 298 tasks      | elapsed: 32.2min\n[Parallel(n_jobs=-1)]: Done 299 tasks      | elapsed: 32.4min\n[Parallel(n_jobs=-1)]: Done 300 tasks      | elapsed: 32.5min\n[Parallel(n_jobs=-1)]: Done 301 tasks      | elapsed: 32.5min\n[Parallel(n_jobs=-1)]: Done 302 tasks      | elapsed: 32.5min\n[Parallel(n_jobs=-1)]: Done 303 tasks      | elapsed: 32.5min\n[Parallel(n_jobs=-1)]: Done 304 tasks      | elapsed: 32.6min\n[Parallel(n_jobs=-1)]: Done 305 tasks      | elapsed: 32.7min\n[Parallel(n_jobs=-1)]: Done 306 tasks      | elapsed: 32.7min\n[Parallel(n_jobs=-1)]: Done 307 tasks      | elapsed: 32.7min\n[Parallel(n_jobs=-1)]: Done 308 tasks      | elapsed: 32.8min\n[Parallel(n_jobs=-1)]: Done 309 tasks      | elapsed: 32.8min\n[Parallel(n_jobs=-1)]: Done 310 tasks      | elapsed: 32.8min\n[Parallel(n_jobs=-1)]: Done 311 tasks      | elapsed: 32.8min\n[Parallel(n_jobs=-1)]: Done 312 tasks      | elapsed: 32.8min\n[Parallel(n_jobs=-1)]: Done 313 tasks      | elapsed: 32.9min\n[Parallel(n_jobs=-1)]: Done 314 tasks      | elapsed: 32.9min\n[Parallel(n_jobs=-1)]: Done 315 tasks      | elapsed: 32.9min\n[Parallel(n_jobs=-1)]: Done 316 tasks      | elapsed: 33.3min\n[Parallel(n_jobs=-1)]: Done 317 tasks      | elapsed: 33.3min\n[Parallel(n_jobs=-1)]: Done 318 tasks      | elapsed: 33.3min\n[Parallel(n_jobs=-1)]: Done 319 tasks      | elapsed: 33.6min\n[Parallel(n_jobs=-1)]: Done 320 tasks      | elapsed: 33.6min\n[Parallel(n_jobs=-1)]: Done 321 tasks      | elapsed: 34.0min\n[Parallel(n_jobs=-1)]: Done 322 tasks      | elapsed: 34.0min\n[Parallel(n_jobs=-1)]: Done 323 tasks      | elapsed: 34.0min\n[Parallel(n_jobs=-1)]: Done 324 tasks      | elapsed: 34.0min\n[Parallel(n_jobs=-1)]: Done 325 tasks      | elapsed: 34.1min\n[Parallel(n_jobs=-1)]: Done 326 tasks      | elapsed: 34.4min\n[Parallel(n_jobs=-1)]: Done 327 tasks      | elapsed: 34.4min\n[Parallel(n_jobs=-1)]: Done 328 tasks      | elapsed: 34.4min\n[Parallel(n_jobs=-1)]: Done 329 tasks      | elapsed: 34.5min\n[Parallel(n_jobs=-1)]: Done 330 tasks      | elapsed: 34.6min\n[Parallel(n_jobs=-1)]: Done 331 tasks      | elapsed: 34.7min\n[Parallel(n_jobs=-1)]: Done 332 tasks      | elapsed: 34.8min\n[Parallel(n_jobs=-1)]: Done 333 tasks      | elapsed: 34.8min\n[Parallel(n_jobs=-1)]: Done 334 tasks      | elapsed: 34.8min\n[Parallel(n_jobs=-1)]: Done 335 tasks      | elapsed: 34.9min\n[Parallel(n_jobs=-1)]: Done 336 tasks      | elapsed: 34.9min\n[Parallel(n_jobs=-1)]: Done 337 tasks      | elapsed: 35.0min\n[Parallel(n_jobs=-1)]: Done 338 tasks      | elapsed: 35.0min\n[Parallel(n_jobs=-1)]: Done 339 tasks      | elapsed: 35.1min\n[Parallel(n_jobs=-1)]: Done 340 tasks      | elapsed: 35.2min\n[Parallel(n_jobs=-1)]: Done 341 tasks      | elapsed: 35.2min\n[Parallel(n_jobs=-1)]: Done 342 tasks      | elapsed: 35.2min\n[Parallel(n_jobs=-1)]: Done 343 tasks      | elapsed: 35.3min\n[Parallel(n_jobs=-1)]: Done 344 tasks      | elapsed: 35.4min\n[Parallel(n_jobs=-1)]: Done 345 tasks      | elapsed: 35.4min\n[Parallel(n_jobs=-1)]: Done 346 tasks      | elapsed: 35.5min\n[Parallel(n_jobs=-1)]: Done 347 tasks      | elapsed: 35.5min\n[Parallel(n_jobs=-1)]: Done 348 tasks      | elapsed: 35.5min\n[Parallel(n_jobs=-1)]: Done 349 tasks      | elapsed: 35.5min\n[Parallel(n_jobs=-1)]: Done 350 tasks      | elapsed: 35.6min\n[Parallel(n_jobs=-1)]: Done 351 tasks      | elapsed: 35.8min\n[Parallel(n_jobs=-1)]: Done 352 tasks      | elapsed: 35.8min\n[Parallel(n_jobs=-1)]: Done 353 tasks      | elapsed: 35.9min\n[Parallel(n_jobs=-1)]: Done 354 tasks      | elapsed: 35.9min\n[Parallel(n_jobs=-1)]: Done 355 tasks      | elapsed: 36.0min\n[Parallel(n_jobs=-1)]: Done 356 tasks      | elapsed: 36.0min\n[Parallel(n_jobs=-1)]: Done 357 tasks      | elapsed: 36.1min\n[Parallel(n_jobs=-1)]: Done 358 tasks      | elapsed: 36.1min\n[Parallel(n_jobs=-1)]: Done 359 tasks      | elapsed: 36.2min\n[Parallel(n_jobs=-1)]: Done 360 tasks      | elapsed: 36.2min\n[Parallel(n_jobs=-1)]: Done 361 tasks      | elapsed: 36.7min\n[Parallel(n_jobs=-1)]: Done 362 tasks      | elapsed: 36.9min\n[Parallel(n_jobs=-1)]: Done 363 tasks      | elapsed: 36.9min\n[Parallel(n_jobs=-1)]: Done 364 tasks      | elapsed: 37.0min\n[Parallel(n_jobs=-1)]: Done 365 tasks      | elapsed: 37.0min\n[Parallel(n_jobs=-1)]: Done 366 tasks      | elapsed: 37.2min\n[Parallel(n_jobs=-1)]: Done 367 tasks      | elapsed: 37.3min\n[Parallel(n_jobs=-1)]: Done 368 tasks      | elapsed: 37.4min\n[Parallel(n_jobs=-1)]: Done 369 tasks      | elapsed: 37.5min\n[Parallel(n_jobs=-1)]: Done 370 tasks      | elapsed: 37.6min\n[Parallel(n_jobs=-1)]: Done 371 tasks      | elapsed: 37.9min\n[Parallel(n_jobs=-1)]: Done 372 tasks      | elapsed: 38.1min\n[Parallel(n_jobs=-1)]: Done 373 tasks      | elapsed: 38.1min\n[Parallel(n_jobs=-1)]: Done 374 tasks      | elapsed: 38.3min\n[Parallel(n_jobs=-1)]: Done 375 tasks      | elapsed: 38.6min\n[Parallel(n_jobs=-1)]: Done 376 tasks      | elapsed: 38.7min\n[Parallel(n_jobs=-1)]: Done 377 tasks      | elapsed: 39.0min\n[Parallel(n_jobs=-1)]: Done 378 tasks      | elapsed: 39.3min\n[Parallel(n_jobs=-1)]: Done 379 tasks      | elapsed: 39.4min\n[Parallel(n_jobs=-1)]: Done 380 tasks      | elapsed: 39.7min\n[Parallel(n_jobs=-1)]: Done 381 tasks      | elapsed: 40.0min\n[Parallel(n_jobs=-1)]: Done 382 tasks      | elapsed: 40.1min\n[Parallel(n_jobs=-1)]: Done 383 tasks      | elapsed: 40.3min\n[Parallel(n_jobs=-1)]: Done 384 tasks      | elapsed: 40.4min\n[Parallel(n_jobs=-1)]: Done 385 tasks      | elapsed: 40.4min\n[Parallel(n_jobs=-1)]: Done 386 tasks      | elapsed: 40.5min\n[Parallel(n_jobs=-1)]: Done 387 tasks      | elapsed: 40.6min\n[Parallel(n_jobs=-1)]: Done 388 tasks      | elapsed: 40.7min\n[Parallel(n_jobs=-1)]: Done 389 tasks      | elapsed: 40.8min\n[Parallel(n_jobs=-1)]: Done 390 tasks      | elapsed: 40.8min\n[Parallel(n_jobs=-1)]: Done 391 tasks      | elapsed: 40.8min\n[Parallel(n_jobs=-1)]: Done 392 tasks      | elapsed: 40.9min\n[Parallel(n_jobs=-1)]: Done 393 tasks      | elapsed: 40.9min\n[Parallel(n_jobs=-1)]: Done 394 tasks      | elapsed: 40.9min\n[Parallel(n_jobs=-1)]: Done 395 tasks      | elapsed: 40.9min\n[Parallel(n_jobs=-1)]: Done 396 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 397 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 398 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 399 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 400 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 401 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 402 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 403 tasks      | elapsed: 41.0min\n[Parallel(n_jobs=-1)]: Done 404 tasks      | elapsed: 41.1min\n[Parallel(n_jobs=-1)]: Done 405 tasks      | elapsed: 41.1min\n[Parallel(n_jobs=-1)]: Done 406 tasks      | elapsed: 41.2min\n[Parallel(n_jobs=-1)]: Done 407 tasks      | elapsed: 41.2min\n[Parallel(n_jobs=-1)]: Done 408 tasks      | elapsed: 41.2min\n[Parallel(n_jobs=-1)]: Done 409 tasks      | elapsed: 41.3min\n[Parallel(n_jobs=-1)]: Done 410 tasks      | elapsed: 41.3min\n[Parallel(n_jobs=-1)]: Done 411 tasks      | elapsed: 41.5min\n[Parallel(n_jobs=-1)]: Done 412 tasks      | elapsed: 41.6min\n[Parallel(n_jobs=-1)]: Done 413 tasks      | elapsed: 41.6min\n[Parallel(n_jobs=-1)]: Done 414 tasks      | elapsed: 41.7min\n[Parallel(n_jobs=-1)]: Done 415 tasks      | elapsed: 41.8min\n[Parallel(n_jobs=-1)]: Done 416 tasks      | elapsed: 41.9min\n[Parallel(n_jobs=-1)]: Done 417 tasks      | elapsed: 41.9min\n[Parallel(n_jobs=-1)]: Done 418 tasks      | elapsed: 41.9min\n[Parallel(n_jobs=-1)]: Done 419 tasks      | elapsed: 41.9min\n[Parallel(n_jobs=-1)]: Done 420 tasks      | elapsed: 41.9min\n[Parallel(n_jobs=-1)]: Done 421 tasks      | elapsed: 41.9min\n[Parallel(n_jobs=-1)]: Done 422 tasks      | elapsed: 41.9min\n[Parallel(n_jobs=-1)]: Done 423 tasks      | elapsed: 42.0min\n[Parallel(n_jobs=-1)]: Done 424 tasks      | elapsed: 42.0min\n[Parallel(n_jobs=-1)]: Done 425 tasks      | elapsed: 42.0min\n[Parallel(n_jobs=-1)]: Done 426 tasks      | elapsed: 42.4min\n[Parallel(n_jobs=-1)]: Done 427 tasks      | elapsed: 42.5min\n[Parallel(n_jobs=-1)]: Done 428 tasks      | elapsed: 42.5min\n[Parallel(n_jobs=-1)]: Done 429 tasks      | elapsed: 42.7min\n[Parallel(n_jobs=-1)]: Done 430 tasks      | elapsed: 42.7min\n[Parallel(n_jobs=-1)]: Done 431 tasks      | elapsed: 42.7min\n[Parallel(n_jobs=-1)]: Done 432 tasks      | elapsed: 42.7min\n[Parallel(n_jobs=-1)]: Done 433 tasks      | elapsed: 42.8min\n[Parallel(n_jobs=-1)]: Done 434 tasks      | elapsed: 42.9min\n[Parallel(n_jobs=-1)]: Done 435 tasks      | elapsed: 42.9min\n[Parallel(n_jobs=-1)]: Done 436 tasks      | elapsed: 43.9min\n[Parallel(n_jobs=-1)]: Done 437 tasks      | elapsed: 44.0min\n[Parallel(n_jobs=-1)]: Done 438 tasks      | elapsed: 44.1min\n[Parallel(n_jobs=-1)]: Done 439 tasks      | elapsed: 44.1min\n[Parallel(n_jobs=-1)]: Done 440 tasks      | elapsed: 44.1min\n[Parallel(n_jobs=-1)]: Done 441 tasks      | elapsed: 44.2min\n[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed: 44.2min\n[Parallel(n_jobs=-1)]: Done 443 tasks      | elapsed: 44.2min\n[Parallel(n_jobs=-1)]: Done 444 tasks      | elapsed: 44.4min\n[Parallel(n_jobs=-1)]: Done 445 tasks      | elapsed: 44.6min\n[Parallel(n_jobs=-1)]: Done 446 tasks      | elapsed: 44.6min\n[Parallel(n_jobs=-1)]: Done 447 tasks      | elapsed: 44.6min\n[Parallel(n_jobs=-1)]: Done 448 tasks      | elapsed: 44.7min\n[Parallel(n_jobs=-1)]: Done 449 tasks      | elapsed: 44.7min\n[Parallel(n_jobs=-1)]: Done 450 tasks      | elapsed: 44.7min\n[Parallel(n_jobs=-1)]: Done 451 tasks      | elapsed: 44.7min\n[Parallel(n_jobs=-1)]: Done 452 tasks      | elapsed: 44.8min\n[Parallel(n_jobs=-1)]: Done 453 tasks      | elapsed: 44.8min\n[Parallel(n_jobs=-1)]: Done 454 tasks      | elapsed: 45.1min\n[Parallel(n_jobs=-1)]: Done 455 tasks      | elapsed: 45.2min\n[Parallel(n_jobs=-1)]: Done 456 tasks      | elapsed: 45.8min\n[Parallel(n_jobs=-1)]: Done 457 tasks      | elapsed: 46.0min\n[Parallel(n_jobs=-1)]: Done 458 tasks      | elapsed: 46.1min\n[Parallel(n_jobs=-1)]: Done 459 tasks      | elapsed: 46.7min\n[Parallel(n_jobs=-1)]: Done 460 tasks      | elapsed: 46.8min\n[Parallel(n_jobs=-1)]: Done 461 tasks      | elapsed: 46.8min\n[Parallel(n_jobs=-1)]: Done 462 tasks      | elapsed: 47.0min\n[Parallel(n_jobs=-1)]: Done 463 tasks      | elapsed: 47.1min\n[Parallel(n_jobs=-1)]: Done 464 tasks      | elapsed: 47.1min\n[Parallel(n_jobs=-1)]: Done 465 tasks      | elapsed: 47.2min\n[Parallel(n_jobs=-1)]: Done 466 tasks      | elapsed: 47.4min\n[Parallel(n_jobs=-1)]: Done 467 tasks      | elapsed: 47.5min\n[Parallel(n_jobs=-1)]: Done 468 tasks      | elapsed: 47.5min\n[Parallel(n_jobs=-1)]: Done 469 tasks      | elapsed: 47.5min\n[Parallel(n_jobs=-1)]: Done 470 tasks      | elapsed: 47.5min\n[Parallel(n_jobs=-1)]: Done 471 tasks      | elapsed: 47.5min\n[Parallel(n_jobs=-1)]: Done 472 tasks      | elapsed: 47.5min\n[Parallel(n_jobs=-1)]: Done 473 tasks      | elapsed: 47.5min\n[Parallel(n_jobs=-1)]: Done 474 tasks      | elapsed: 47.6min\n[Parallel(n_jobs=-1)]: Done 475 tasks      | elapsed: 47.7min\n[Parallel(n_jobs=-1)]: Done 476 tasks      | elapsed: 47.9min\n[Parallel(n_jobs=-1)]: Done 477 tasks      | elapsed: 48.0min\n[Parallel(n_jobs=-1)]: Done 478 tasks      | elapsed: 48.0min\n[Parallel(n_jobs=-1)]: Done 479 tasks      | elapsed: 48.2min\n[Parallel(n_jobs=-1)]: Done 480 tasks      | elapsed: 48.2min\n[Parallel(n_jobs=-1)]: Done 481 tasks      | elapsed: 48.4min\n[Parallel(n_jobs=-1)]: Done 482 tasks      | elapsed: 48.5min\n[Parallel(n_jobs=-1)]: Done 483 tasks      | elapsed: 48.6min\n[Parallel(n_jobs=-1)]: Done 484 tasks      | elapsed: 48.6min\n[Parallel(n_jobs=-1)]: Done 485 tasks      | elapsed: 48.8min\n[Parallel(n_jobs=-1)]: Done 496 out of 500 | elapsed: 49.3min remaining:   23.9s\n[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed: 49.5min finished\n{'n_estimators': 600, 'min_samples_split': 10, 'min_samples_leaf': 1, 'max_features': 'auto', 'max_depth': None, 'bootstrap': True}\n"
    }
   ],
   "source": [
    "forest_random_search.fit(train_X, train_y)\n",
    "print(forest_random_search.best_params_)\n",
    "save_model(forest_random_search.best_estimator_, \"rndsearch_forest.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "from train_utils import save_cv_results\n",
    "\n",
    "save_cv_results(forest_random_search.cv_results_, 'forest_rndsearch_results.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "   bootstrap  max_depth max_features  min_samples_leaf  min_samples_split  \\\n0      False        NaN         auto               500                100   \n1       True       20.0         auto                 5                 10   \n2       True      110.0         sqrt                50               1000   \n3       True      100.0         auto                50                  2   \n4      False       60.0         sqrt                 1                 10   \n\n   n_estimators  train_error_mean  val_error_mean  error_diff_mean  \\\n0           200          0.957593        0.963620         0.006027   \n1           400          0.556216        0.801190         0.244974   \n2          1800          0.963081        0.968507         0.005427   \n3          2000          0.813110        0.851608         0.038498   \n4          1400          0.387070        0.803025         0.415955   \n\n   train_error_std  val_error_std  \n0         0.001671       0.007002  \n1         0.002654       0.006210  \n2         0.001884       0.009572  \n3         0.002626       0.004816  \n4         0.001367       0.008934  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>bootstrap</th>\n      <th>max_depth</th>\n      <th>max_features</th>\n      <th>min_samples_leaf</th>\n      <th>min_samples_split</th>\n      <th>n_estimators</th>\n      <th>train_error_mean</th>\n      <th>val_error_mean</th>\n      <th>error_diff_mean</th>\n      <th>train_error_std</th>\n      <th>val_error_std</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>False</td>\n      <td>NaN</td>\n      <td>auto</td>\n      <td>500</td>\n      <td>100</td>\n      <td>200</td>\n      <td>0.957593</td>\n      <td>0.963620</td>\n      <td>0.006027</td>\n      <td>0.001671</td>\n      <td>0.007002</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>True</td>\n      <td>20.0</td>\n      <td>auto</td>\n      <td>5</td>\n      <td>10</td>\n      <td>400</td>\n      <td>0.556216</td>\n      <td>0.801190</td>\n      <td>0.244974</td>\n      <td>0.002654</td>\n      <td>0.006210</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>True</td>\n      <td>110.0</td>\n      <td>sqrt</td>\n      <td>50</td>\n      <td>1000</td>\n      <td>1800</td>\n      <td>0.963081</td>\n      <td>0.968507</td>\n      <td>0.005427</td>\n      <td>0.001884</td>\n      <td>0.009572</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>True</td>\n      <td>100.0</td>\n      <td>auto</td>\n      <td>50</td>\n      <td>2</td>\n      <td>2000</td>\n      <td>0.813110</td>\n      <td>0.851608</td>\n      <td>0.038498</td>\n      <td>0.002626</td>\n      <td>0.004816</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>False</td>\n      <td>60.0</td>\n      <td>sqrt</td>\n      <td>1</td>\n      <td>10</td>\n      <td>1400</td>\n      <td>0.387070</td>\n      <td>0.803025</td>\n      <td>0.415955</td>\n      <td>0.001367</td>\n      <td>0.008934</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 212
    }
   ],
   "source": [
    "from train_utils import summarize_cv_results\n",
    "\n",
    "forest_rndsearch_summary = summarize_cv_results(forest_random_search.cv_results_)\n",
    "forest_rndsearch_summary.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    bootstrap  max_depth max_features  min_samples_leaf  min_samples_split  \\\n81       True        NaN         auto                 1                 10   \n\n    n_estimators  train_error_mean  val_error_mean  error_diff_mean  \\\n81           600          0.453795        0.799826         0.346031   \n\n    train_error_std  val_error_std  \n81         0.001457       0.007141  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>bootstrap</th>\n      <th>max_depth</th>\n      <th>max_features</th>\n      <th>min_samples_leaf</th>\n      <th>min_samples_split</th>\n      <th>n_estimators</th>\n      <th>train_error_mean</th>\n      <th>val_error_mean</th>\n      <th>error_diff_mean</th>\n      <th>train_error_std</th>\n      <th>val_error_std</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>81</th>\n      <td>True</td>\n      <td>NaN</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>10</td>\n      <td>600</td>\n      <td>0.453795</td>\n      <td>0.799826</td>\n      <td>0.346031</td>\n      <td>0.001457</td>\n      <td>0.007141</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 222
    }
   ],
   "source": [
    "forest_rndsearch_summary.loc[[forest_random_search.best_index_]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The best model of random search does less overfitting than the base model but is still too much"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "    bootstrap  max_depth max_features  min_samples_leaf  min_samples_split  \\\n47       True        NaN         auto                 1                100   \n29       True       60.0         auto                 5                100   \n19       True      100.0         auto                 1                100   \n17       True       60.0         auto                 5                100   \n58       True       40.0         auto                 5                100   \n33      False       40.0         sqrt                 5                100   \n66      False       80.0         sqrt                 5                100   \n22      False       30.0         sqrt                 5                100   \n65       True       20.0         auto                50                100   \n3        True      100.0         auto                50                  2   \n\n    n_estimators  train_error_mean  val_error_mean  error_diff_mean  \\\n47           800          0.746378        0.829199         0.082821   \n29           600          0.759906        0.829323         0.069418   \n19          1200          0.746162        0.829471         0.083309   \n17          2000          0.759973        0.829500         0.069527   \n58          1600          0.760010        0.829541         0.069532   \n33          2000          0.782186        0.849682         0.067496   \n66          1200          0.782246        0.849708         0.067462   \n22          1600          0.782232        0.850119         0.067888   \n65          2000          0.813129        0.851497         0.038368   \n3           2000          0.813110        0.851608         0.038498   \n\n    train_error_std  val_error_std  \n47         0.002351       0.004911  \n29         0.002310       0.004571  \n19         0.002391       0.004434  \n17         0.002270       0.004565  \n58         0.002218       0.004620  \n33         0.001351       0.007261  \n66         0.001288       0.007630  \n22         0.001468       0.007159  \n65         0.002635       0.004493  \n3          0.002626       0.004816  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>bootstrap</th>\n      <th>max_depth</th>\n      <th>max_features</th>\n      <th>min_samples_leaf</th>\n      <th>min_samples_split</th>\n      <th>n_estimators</th>\n      <th>train_error_mean</th>\n      <th>val_error_mean</th>\n      <th>error_diff_mean</th>\n      <th>train_error_std</th>\n      <th>val_error_std</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>47</th>\n      <td>True</td>\n      <td>NaN</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>100</td>\n      <td>800</td>\n      <td>0.746378</td>\n      <td>0.829199</td>\n      <td>0.082821</td>\n      <td>0.002351</td>\n      <td>0.004911</td>\n    </tr>\n    <tr>\n      <th>29</th>\n      <td>True</td>\n      <td>60.0</td>\n      <td>auto</td>\n      <td>5</td>\n      <td>100</td>\n      <td>600</td>\n      <td>0.759906</td>\n      <td>0.829323</td>\n      <td>0.069418</td>\n      <td>0.002310</td>\n      <td>0.004571</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>True</td>\n      <td>100.0</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>100</td>\n      <td>1200</td>\n      <td>0.746162</td>\n      <td>0.829471</td>\n      <td>0.083309</td>\n      <td>0.002391</td>\n      <td>0.004434</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>True</td>\n      <td>60.0</td>\n      <td>auto</td>\n      <td>5</td>\n      <td>100</td>\n      <td>2000</td>\n      <td>0.759973</td>\n      <td>0.829500</td>\n      <td>0.069527</td>\n      <td>0.002270</td>\n      <td>0.004565</td>\n    </tr>\n    <tr>\n      <th>58</th>\n      <td>True</td>\n      <td>40.0</td>\n      <td>auto</td>\n      <td>5</td>\n      <td>100</td>\n      <td>1600</td>\n      <td>0.760010</td>\n      <td>0.829541</td>\n      <td>0.069532</td>\n      <td>0.002218</td>\n      <td>0.004620</td>\n    </tr>\n    <tr>\n      <th>33</th>\n      <td>False</td>\n      <td>40.0</td>\n      <td>sqrt</td>\n      <td>5</td>\n      <td>100</td>\n      <td>2000</td>\n      <td>0.782186</td>\n      <td>0.849682</td>\n      <td>0.067496</td>\n      <td>0.001351</td>\n      <td>0.007261</td>\n    </tr>\n    <tr>\n      <th>66</th>\n      <td>False</td>\n      <td>80.0</td>\n      <td>sqrt</td>\n      <td>5</td>\n      <td>100</td>\n      <td>1200</td>\n      <td>0.782246</td>\n      <td>0.849708</td>\n      <td>0.067462</td>\n      <td>0.001288</td>\n      <td>0.007630</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>False</td>\n      <td>30.0</td>\n      <td>sqrt</td>\n      <td>5</td>\n      <td>100</td>\n      <td>1600</td>\n      <td>0.782232</td>\n      <td>0.850119</td>\n      <td>0.067888</td>\n      <td>0.001468</td>\n      <td>0.007159</td>\n    </tr>\n    <tr>\n      <th>65</th>\n      <td>True</td>\n      <td>20.0</td>\n      <td>auto</td>\n      <td>50</td>\n      <td>100</td>\n      <td>2000</td>\n      <td>0.813129</td>\n      <td>0.851497</td>\n      <td>0.038368</td>\n      <td>0.002635</td>\n      <td>0.004493</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>True</td>\n      <td>100.0</td>\n      <td>auto</td>\n      <td>50</td>\n      <td>2</td>\n      <td>2000</td>\n      <td>0.813110</td>\n      <td>0.851608</td>\n      <td>0.038498</td>\n      <td>0.002626</td>\n      <td>0.004816</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 224
    }
   ],
   "source": [
    "# find models with a low test error that do not overfit too much\n",
    "condition = (forest_rndsearch_summary['val_error_mean'] < 0.9) & (forest_rndsearch_summary['error_diff_mean'] < 0.1)\n",
    "forest_rndsearch_summary[condition].sort_values(by='val_error_mean').head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Best parameters so far:\n",
    "# - bootstrap = True\n",
    "# - max_depth = [None, 60, 100]\n",
    "# - max_features = 'auto'\n",
    "# - min_samples_leaf = [1, 5]\n",
    "# - min_samples_split = 100\n",
    "# - n_estimators = [600, 800, 1200, 2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# run a grid search on a narrow range of hyperparameters\n",
    "forest_grid_params = {\n",
    "    'n_estimators': [1200, 1600, 2000, 2400],\n",
    "    'max_features': ['auto'],\n",
    "    'max_depth': [60, 100, 150, None],\n",
    "    'min_samples_split': [80, 100, 140, 200],\n",
    "    'min_samples_leaf': [1, 5, 10],\n",
    "    'bootstrap': [True]\n",
    "}\n",
    "\n",
    "forest_grid_search = GridSearchCV(RandomForestRegressor(), forest_grid_params,\n",
    "                                  scoring='neg_mean_squared_error', cv=5, n_jobs=-1, verbose=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "tags": [
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "| elapsed: 147.4min\n[Parallel(n_jobs=-1)]: Done 630 tasks      | elapsed: 147.5min\n[Parallel(n_jobs=-1)]: Done 631 tasks      | elapsed: 148.0min\n[Parallel(n_jobs=-1)]: Done 632 tasks      | elapsed: 148.1min\n[Parallel(n_jobs=-1)]: Done 633 tasks      | elapsed: 148.1min\n[Parallel(n_jobs=-1)]: Done 634 tasks      | elapsed: 148.6min\n[Parallel(n_jobs=-1)]: Done 635 tasks      | elapsed: 148.8min\n[Parallel(n_jobs=-1)]: Done 636 tasks      | elapsed: 149.2min\n[Parallel(n_jobs=-1)]: Done 637 tasks      | elapsed: 149.3min\n[Parallel(n_jobs=-1)]: Done 638 tasks      | elapsed: 149.5min\n[Parallel(n_jobs=-1)]: Done 639 tasks      | elapsed: 149.5min\n[Parallel(n_jobs=-1)]: Done 640 tasks      | elapsed: 149.8min\n[Parallel(n_jobs=-1)]: Done 641 tasks      | elapsed: 149.9min\n[Parallel(n_jobs=-1)]: Done 642 tasks      | elapsed: 150.0min\n[Parallel(n_jobs=-1)]: Done 643 tasks      | elapsed: 150.0min\n[Parallel(n_jobs=-1)]: Done 644 tasks      | elapsed: 150.4min\n[Parallel(n_jobs=-1)]: Done 645 tasks      | elapsed: 150.5min\n[Parallel(n_jobs=-1)]: Done 646 tasks      | elapsed: 151.1min\n[Parallel(n_jobs=-1)]: Done 647 tasks      | elapsed: 151.1min\n[Parallel(n_jobs=-1)]: Done 648 tasks      | elapsed: 151.4min\n[Parallel(n_jobs=-1)]: Done 649 tasks      | elapsed: 151.5min\n[Parallel(n_jobs=-1)]: Done 650 tasks      | elapsed: 151.6min\n[Parallel(n_jobs=-1)]: Done 651 tasks      | elapsed: 152.0min\n[Parallel(n_jobs=-1)]: Done 652 tasks      | elapsed: 152.3min\n[Parallel(n_jobs=-1)]: Done 653 tasks      | elapsed: 152.4min\n[Parallel(n_jobs=-1)]: Done 654 tasks      | elapsed: 153.1min\n[Parallel(n_jobs=-1)]: Done 655 tasks      | elapsed: 153.1min\n[Parallel(n_jobs=-1)]: Done 656 tasks      | elapsed: 153.7min\n[Parallel(n_jobs=-1)]: Done 657 tasks      | elapsed: 153.8min\n[Parallel(n_jobs=-1)]: Done 658 tasks      | elapsed: 153.9min\n[Parallel(n_jobs=-1)]: Done 659 tasks      | elapsed: 154.2min\n[Parallel(n_jobs=-1)]: Done 660 tasks      | elapsed: 154.3min\n[Parallel(n_jobs=-1)]: Done 661 tasks      | elapsed: 154.4min\n[Parallel(n_jobs=-1)]: Done 662 tasks      | elapsed: 154.5min\n[Parallel(n_jobs=-1)]: Done 663 tasks      | elapsed: 154.8min\n[Parallel(n_jobs=-1)]: Done 664 tasks      | elapsed: 154.9min\n[Parallel(n_jobs=-1)]: Done 665 tasks      | elapsed: 155.0min\n[Parallel(n_jobs=-1)]: Done 666 tasks      | elapsed: 155.5min\n[Parallel(n_jobs=-1)]: Done 667 tasks      | elapsed: 155.8min\n[Parallel(n_jobs=-1)]: Done 668 tasks      | elapsed: 155.8min\n[Parallel(n_jobs=-1)]: Done 669 tasks      | elapsed: 155.9min\n[Parallel(n_jobs=-1)]: Done 670 tasks      | elapsed: 156.1min\n[Parallel(n_jobs=-1)]: Done 671 tasks      | elapsed: 156.7min\n[Parallel(n_jobs=-1)]: Done 672 tasks      | elapsed: 156.7min\n[Parallel(n_jobs=-1)]: Done 673 tasks      | elapsed: 156.9min\n[Parallel(n_jobs=-1)]: Done 674 tasks      | elapsed: 157.3min\n[Parallel(n_jobs=-1)]: Done 675 tasks      | elapsed: 157.6min\n[Parallel(n_jobs=-1)]: Done 676 tasks      | elapsed: 157.9min\n[Parallel(n_jobs=-1)]: Done 677 tasks      | elapsed: 158.0min\n[Parallel(n_jobs=-1)]: Done 678 tasks      | elapsed: 158.1min\n[Parallel(n_jobs=-1)]: Done 679 tasks      | elapsed: 158.4min\n[Parallel(n_jobs=-1)]: Done 680 tasks      | elapsed: 158.4min\n[Parallel(n_jobs=-1)]: Done 681 tasks      | elapsed: 158.7min\n[Parallel(n_jobs=-1)]: Done 682 tasks      | elapsed: 159.0min\n[Parallel(n_jobs=-1)]: Done 683 tasks      | elapsed: 159.0min\n[Parallel(n_jobs=-1)]: Done 684 tasks      | elapsed: 159.1min\n[Parallel(n_jobs=-1)]: Done 685 tasks      | elapsed: 159.2min\n[Parallel(n_jobs=-1)]: Done 686 tasks      | elapsed: 159.5min\n[Parallel(n_jobs=-1)]: Done 687 tasks      | elapsed: 159.7min\n[Parallel(n_jobs=-1)]: Done 688 tasks      | elapsed: 159.8min\n[Parallel(n_jobs=-1)]: Done 689 tasks      | elapsed: 160.0min\n[Parallel(n_jobs=-1)]: Done 690 tasks      | elapsed: 160.3min\n[Parallel(n_jobs=-1)]: Done 691 tasks      | elapsed: 160.7min\n[Parallel(n_jobs=-1)]: Done 692 tasks      | elapsed: 160.8min\n[Parallel(n_jobs=-1)]: Done 693 tasks      | elapsed: 161.1min\n[Parallel(n_jobs=-1)]: Done 694 tasks      | elapsed: 161.2min\n[Parallel(n_jobs=-1)]: Done 695 tasks      | elapsed: 161.5min\n[Parallel(n_jobs=-1)]: Done 696 tasks      | elapsed: 162.0min\n[Parallel(n_jobs=-1)]: Done 697 tasks      | elapsed: 162.1min\n[Parallel(n_jobs=-1)]: Done 698 tasks      | elapsed: 162.1min\n[Parallel(n_jobs=-1)]: Done 699 tasks      | elapsed: 162.1min\n[Parallel(n_jobs=-1)]: Done 700 tasks      | elapsed: 162.4min\n[Parallel(n_jobs=-1)]: Done 701 tasks      | elapsed: 162.4min\n[Parallel(n_jobs=-1)]: Done 702 tasks      | elapsed: 162.7min\n[Parallel(n_jobs=-1)]: Done 703 tasks      | elapsed: 162.9min\n[Parallel(n_jobs=-1)]: Done 704 tasks      | elapsed: 162.9min\n[Parallel(n_jobs=-1)]: Done 705 tasks      | elapsed: 163.1min\n[Parallel(n_jobs=-1)]: Done 706 tasks      | elapsed: 163.3min\n[Parallel(n_jobs=-1)]: Done 707 tasks      | elapsed: 163.5min\n[Parallel(n_jobs=-1)]: Done 708 tasks      | elapsed: 163.6min\n[Parallel(n_jobs=-1)]: Done 709 tasks      | elapsed: 163.7min\n[Parallel(n_jobs=-1)]: Done 710 tasks      | elapsed: 164.0min\n[Parallel(n_jobs=-1)]: Done 711 tasks      | elapsed: 164.5min\n[Parallel(n_jobs=-1)]: Done 712 tasks      | elapsed: 164.6min\n[Parallel(n_jobs=-1)]: Done 713 tasks      | elapsed: 164.7min\n[Parallel(n_jobs=-1)]: Done 714 tasks      | elapsed: 164.9min\n[Parallel(n_jobs=-1)]: Done 715 tasks      | elapsed: 165.1min\n[Parallel(n_jobs=-1)]: Done 716 tasks      | elapsed: 165.5min\n[Parallel(n_jobs=-1)]: Done 717 tasks      | elapsed: 165.6min\n[Parallel(n_jobs=-1)]: Done 718 tasks      | elapsed: 165.9min\n[Parallel(n_jobs=-1)]: Done 719 tasks      | elapsed: 166.0min\n[Parallel(n_jobs=-1)]: Done 720 tasks      | elapsed: 166.4min\n[Parallel(n_jobs=-1)]: Done 721 tasks      | elapsed: 166.4min\n[Parallel(n_jobs=-1)]: Done 722 tasks      | elapsed: 166.5min\n[Parallel(n_jobs=-1)]: Done 723 tasks      | elapsed: 166.6min\n[Parallel(n_jobs=-1)]: Done 724 tasks      | elapsed: 166.9min\n[Parallel(n_jobs=-1)]: Done 725 tasks      | elapsed: 166.9min\n[Parallel(n_jobs=-1)]: Done 726 tasks      | elapsed: 167.6min\n[Parallel(n_jobs=-1)]: Done 727 tasks      | elapsed: 167.9min\n[Parallel(n_jobs=-1)]: Done 728 tasks      | elapsed: 168.1min\n[Parallel(n_jobs=-1)]: Done 729 tasks      | elapsed: 168.1min\n[Parallel(n_jobs=-1)]: Done 730 tasks      | elapsed: 168.2min\n[Parallel(n_jobs=-1)]: Done 731 tasks      | elapsed: 168.7min\n[Parallel(n_jobs=-1)]: Done 732 tasks      | elapsed: 169.1min\n[Parallel(n_jobs=-1)]: Done 733 tasks      | elapsed: 169.1min\n[Parallel(n_jobs=-1)]: Done 734 tasks      | elapsed: 169.8min\n[Parallel(n_jobs=-1)]: Done 735 tasks      | elapsed: 170.2min\n[Parallel(n_jobs=-1)]: Done 736 tasks      | elapsed: 170.3min\n[Parallel(n_jobs=-1)]: Done 737 tasks      | elapsed: 170.7min\n[Parallel(n_jobs=-1)]: Done 738 tasks      | elapsed: 170.8min\n[Parallel(n_jobs=-1)]: Done 739 tasks      | elapsed: 170.9min\n[Parallel(n_jobs=-1)]: Done 740 tasks      | elapsed: 171.1min\n[Parallel(n_jobs=-1)]: Done 741 tasks      | elapsed: 171.3min\n[Parallel(n_jobs=-1)]: Done 742 tasks      | elapsed: 171.5min\n[Parallel(n_jobs=-1)]: Done 743 tasks      | elapsed: 171.7min\n[Parallel(n_jobs=-1)]: Done 744 tasks      | elapsed: 171.7min\n[Parallel(n_jobs=-1)]: Done 745 tasks      | elapsed: 171.9min\n[Parallel(n_jobs=-1)]: Done 746 tasks      | elapsed: 172.4min\n[Parallel(n_jobs=-1)]: Done 747 tasks      | elapsed: 172.5min\n[Parallel(n_jobs=-1)]: Done 748 tasks      | elapsed: 172.7min\n[Parallel(n_jobs=-1)]: Done 749 tasks      | elapsed: 172.9min\n[Parallel(n_jobs=-1)]: Done 750 tasks      | elapsed: 173.1min\n[Parallel(n_jobs=-1)]: Done 751 tasks      | elapsed: 173.7min\n[Parallel(n_jobs=-1)]: Done 752 tasks      | elapsed: 173.9min\n[Parallel(n_jobs=-1)]: Done 753 tasks      | elapsed: 173.9min\n[Parallel(n_jobs=-1)]: Done 754 tasks      | elapsed: 174.5min\n[Parallel(n_jobs=-1)]: Done 755 tasks      | elapsed: 174.6min\n[Parallel(n_jobs=-1)]: Done 756 tasks      | elapsed: 175.1min\n[Parallel(n_jobs=-1)]: Done 757 tasks      | elapsed: 175.1min\n[Parallel(n_jobs=-1)]: Done 758 tasks      | elapsed: 175.4min\n[Parallel(n_jobs=-1)]: Done 759 tasks      | elapsed: 175.6min\n[Parallel(n_jobs=-1)]: Done 760 tasks      | elapsed: 175.7min\n[Parallel(n_jobs=-1)]: Done 761 tasks      | elapsed: 175.7min\n[Parallel(n_jobs=-1)]: Done 762 tasks      | elapsed: 176.1min\n[Parallel(n_jobs=-1)]: Done 763 tasks      | elapsed: 176.1min\n[Parallel(n_jobs=-1)]: Done 764 tasks      | elapsed: 176.2min\n[Parallel(n_jobs=-1)]: Done 765 tasks      | elapsed: 176.3min\n[Parallel(n_jobs=-1)]: Done 766 tasks      | elapsed: 176.8min\n[Parallel(n_jobs=-1)]: Done 767 tasks      | elapsed: 177.2min\n[Parallel(n_jobs=-1)]: Done 768 tasks      | elapsed: 177.2min\n[Parallel(n_jobs=-1)]: Done 769 tasks      | elapsed: 177.3min\n[Parallel(n_jobs=-1)]: Done 770 tasks      | elapsed: 177.6min\n[Parallel(n_jobs=-1)]: Done 771 tasks      | elapsed: 178.0min\n[Parallel(n_jobs=-1)]: Done 772 tasks      | elapsed: 178.1min\n[Parallel(n_jobs=-1)]: Done 773 tasks      | elapsed: 178.2min\n[Parallel(n_jobs=-1)]: Done 774 tasks      | elapsed: 178.8min\n[Parallel(n_jobs=-1)]: Done 775 tasks      | elapsed: 179.2min\n[Parallel(n_jobs=-1)]: Done 776 tasks      | elapsed: 179.3min\n[Parallel(n_jobs=-1)]: Done 777 tasks      | elapsed: 179.6min\n[Parallel(n_jobs=-1)]: Done 778 tasks      | elapsed: 179.9min\n[Parallel(n_jobs=-1)]: Done 779 tasks      | elapsed: 180.0min\n[Parallel(n_jobs=-1)]: Done 780 tasks      | elapsed: 180.2min\n[Parallel(n_jobs=-1)]: Done 781 tasks      | elapsed: 180.3min\n[Parallel(n_jobs=-1)]: Done 782 tasks      | elapsed: 180.4min\n[Parallel(n_jobs=-1)]: Done 783 tasks      | elapsed: 180.5min\n[Parallel(n_jobs=-1)]: Done 784 tasks      | elapsed: 180.6min\n[Parallel(n_jobs=-1)]: Done 785 tasks      | elapsed: 180.7min\n[Parallel(n_jobs=-1)]: Done 786 tasks      | elapsed: 181.3min\n[Parallel(n_jobs=-1)]: Done 787 tasks      | elapsed: 181.5min\n[Parallel(n_jobs=-1)]: Done 788 tasks      | elapsed: 181.6min\n[Parallel(n_jobs=-1)]: Done 789 tasks      | elapsed: 181.8min\n[Parallel(n_jobs=-1)]: Done 790 tasks      | elapsed: 181.9min\n[Parallel(n_jobs=-1)]: Done 791 tasks      | elapsed: 182.3min\n[Parallel(n_jobs=-1)]: Done 792 tasks      | elapsed: 182.5min\n[Parallel(n_jobs=-1)]: Done 793 tasks      | elapsed: 182.6min\n[Parallel(n_jobs=-1)]: Done 794 tasks      | elapsed: 183.0min\n[Parallel(n_jobs=-1)]: Done 795 tasks      | elapsed: 183.3min\n[Parallel(n_jobs=-1)]: Done 796 tasks      | elapsed: 183.9min\n[Parallel(n_jobs=-1)]: Done 797 tasks      | elapsed: 184.0min\n[Parallel(n_jobs=-1)]: Done 798 tasks      | elapsed: 184.1min\n[Parallel(n_jobs=-1)]: Done 799 tasks      | elapsed: 184.2min\n[Parallel(n_jobs=-1)]: Done 800 tasks      | elapsed: 184.6min\n[Parallel(n_jobs=-1)]: Done 801 tasks      | elapsed: 184.6min\n[Parallel(n_jobs=-1)]: Done 802 tasks      | elapsed: 184.8min\n[Parallel(n_jobs=-1)]: Done 803 tasks      | elapsed: 184.9min\n[Parallel(n_jobs=-1)]: Done 804 tasks      | elapsed: 185.4min\n[Parallel(n_jobs=-1)]: Done 805 tasks      | elapsed: 185.5min\n[Parallel(n_jobs=-1)]: Done 806 tasks      | elapsed: 186.1min\n[Parallel(n_jobs=-1)]: Done 807 tasks      | elapsed: 186.3min\n[Parallel(n_jobs=-1)]: Done 808 tasks      | elapsed: 186.6min\n[Parallel(n_jobs=-1)]: Done 809 tasks      | elapsed: 186.7min\n[Parallel(n_jobs=-1)]: Done 810 tasks      | elapsed: 186.9min\n[Parallel(n_jobs=-1)]: Done 811 tasks      | elapsed: 187.5min\n[Parallel(n_jobs=-1)]: Done 812 tasks      | elapsed: 188.2min\n[Parallel(n_jobs=-1)]: Done 813 tasks      | elapsed: 188.3min\n[Parallel(n_jobs=-1)]: Done 814 tasks      | elapsed: 188.8min\n[Parallel(n_jobs=-1)]: Done 815 tasks      | elapsed: 188.9min\n[Parallel(n_jobs=-1)]: Done 816 tasks      | elapsed: 189.7min\n[Parallel(n_jobs=-1)]: Done 817 tasks      | elapsed: 189.8min\n[Parallel(n_jobs=-1)]: Done 818 tasks      | elapsed: 190.0min\n[Parallel(n_jobs=-1)]: Done 819 tasks      | elapsed: 190.1min\n[Parallel(n_jobs=-1)]: Done 820 tasks      | elapsed: 190.3min\n[Parallel(n_jobs=-1)]: Done 821 tasks      | elapsed: 190.4min\n[Parallel(n_jobs=-1)]: Done 822 tasks      | elapsed: 190.7min\n[Parallel(n_jobs=-1)]: Done 823 tasks      | elapsed: 191.3min\n[Parallel(n_jobs=-1)]: Done 824 tasks      | elapsed: 191.3min\n[Parallel(n_jobs=-1)]: Done 825 tasks      | elapsed: 191.3min\n[Parallel(n_jobs=-1)]: Done 826 tasks      | elapsed: 191.9min\n[Parallel(n_jobs=-1)]: Done 827 tasks      | elapsed: 192.0min\n[Parallel(n_jobs=-1)]: Done 828 tasks      | elapsed: 192.2min\n[Parallel(n_jobs=-1)]: Done 829 tasks      | elapsed: 192.3min\n[Parallel(n_jobs=-1)]: Done 830 tasks      | elapsed: 192.6min\n[Parallel(n_jobs=-1)]: Done 831 tasks      | elapsed: 193.8min\n[Parallel(n_jobs=-1)]: Done 832 tasks      | elapsed: 193.8min\n[Parallel(n_jobs=-1)]: Done 833 tasks      | elapsed: 193.8min\n[Parallel(n_jobs=-1)]: Done 834 tasks      | elapsed: 194.4min\n[Parallel(n_jobs=-1)]: Done 835 tasks      | elapsed: 194.5min\n[Parallel(n_jobs=-1)]: Done 836 tasks      | elapsed: 195.1min\n[Parallel(n_jobs=-1)]: Done 837 tasks      | elapsed: 195.1min\n[Parallel(n_jobs=-1)]: Done 838 tasks      | elapsed: 195.2min\n[Parallel(n_jobs=-1)]: Done 839 tasks      | elapsed: 195.4min\n[Parallel(n_jobs=-1)]: Done 840 tasks      | elapsed: 195.7min\n[Parallel(n_jobs=-1)]: Done 841 tasks      | elapsed: 195.8min\n[Parallel(n_jobs=-1)]: Done 842 tasks      | elapsed: 196.3min\n[Parallel(n_jobs=-1)]: Done 843 tasks      | elapsed: 196.4min\n[Parallel(n_jobs=-1)]: Done 844 tasks      | elapsed: 196.5min\n[Parallel(n_jobs=-1)]: Done 845 tasks      | elapsed: 196.5min\n[Parallel(n_jobs=-1)]: Done 846 tasks      | elapsed: 196.9min\n[Parallel(n_jobs=-1)]: Done 847 tasks      | elapsed: 197.1min\n[Parallel(n_jobs=-1)]: Done 848 tasks      | elapsed: 197.4min\n[Parallel(n_jobs=-1)]: Done 849 tasks      | elapsed: 197.4min\n[Parallel(n_jobs=-1)]: Done 850 tasks      | elapsed: 197.9min\n[Parallel(n_jobs=-1)]: Done 851 tasks      | elapsed: 198.4min\n[Parallel(n_jobs=-1)]: Done 852 tasks      | elapsed: 198.5min\n[Parallel(n_jobs=-1)]: Done 853 tasks      | elapsed: 198.6min\n[Parallel(n_jobs=-1)]: Done 854 tasks      | elapsed: 198.9min\n[Parallel(n_jobs=-1)]: Done 855 tasks      | elapsed: 199.2min\n[Parallel(n_jobs=-1)]: Done 856 tasks      | elapsed: 199.8min\n[Parallel(n_jobs=-1)]: Done 857 tasks      | elapsed: 199.9min\n[Parallel(n_jobs=-1)]: Done 858 tasks      | elapsed: 200.1min\n[Parallel(n_jobs=-1)]: Done 859 tasks      | elapsed: 200.1min\n[Parallel(n_jobs=-1)]: Done 860 tasks      | elapsed: 200.4min\n[Parallel(n_jobs=-1)]: Done 861 tasks      | elapsed: 200.6min\n[Parallel(n_jobs=-1)]: Done 862 tasks      | elapsed: 201.0min\n[Parallel(n_jobs=-1)]: Done 863 tasks      | elapsed: 201.1min\n[Parallel(n_jobs=-1)]: Done 864 tasks      | elapsed: 201.2min\n[Parallel(n_jobs=-1)]: Done 865 tasks      | elapsed: 201.2min\n[Parallel(n_jobs=-1)]: Done 866 tasks      | elapsed: 201.7min\n[Parallel(n_jobs=-1)]: Done 867 tasks      | elapsed: 201.7min\n[Parallel(n_jobs=-1)]: Done 868 tasks      | elapsed: 202.1min\n[Parallel(n_jobs=-1)]: Done 869 tasks      | elapsed: 202.2min\n[Parallel(n_jobs=-1)]: Done 870 tasks      | elapsed: 202.6min\n[Parallel(n_jobs=-1)]: Done 871 tasks      | elapsed: 203.1min\n[Parallel(n_jobs=-1)]: Done 872 tasks      | elapsed: 203.2min\n[Parallel(n_jobs=-1)]: Done 873 tasks      | elapsed: 203.2min\n[Parallel(n_jobs=-1)]: Done 874 tasks      | elapsed: 203.6min\n[Parallel(n_jobs=-1)]: Done 875 tasks      | elapsed: 203.7min\n[Parallel(n_jobs=-1)]: Done 876 tasks      | elapsed: 204.4min\n[Parallel(n_jobs=-1)]: Done 877 tasks      | elapsed: 204.6min\n[Parallel(n_jobs=-1)]: Done 878 tasks      | elapsed: 204.6min\n[Parallel(n_jobs=-1)]: Done 879 tasks      | elapsed: 204.9min\n[Parallel(n_jobs=-1)]: Done 880 tasks      | elapsed: 205.1min\n[Parallel(n_jobs=-1)]: Done 881 tasks      | elapsed: 205.2min\n[Parallel(n_jobs=-1)]: Done 882 tasks      | elapsed: 205.6min\n[Parallel(n_jobs=-1)]: Done 883 tasks      | elapsed: 205.6min\n[Parallel(n_jobs=-1)]: Done 884 tasks      | elapsed: 205.9min\n[Parallel(n_jobs=-1)]: Done 885 tasks      | elapsed: 206.1min\n[Parallel(n_jobs=-1)]: Done 886 tasks      | elapsed: 206.7min\n[Parallel(n_jobs=-1)]: Done 887 tasks      | elapsed: 206.9min\n[Parallel(n_jobs=-1)]: Done 888 tasks      | elapsed: 207.0min\n[Parallel(n_jobs=-1)]: Done 889 tasks      | elapsed: 207.1min\n[Parallel(n_jobs=-1)]: Done 890 tasks      | elapsed: 207.4min\n[Parallel(n_jobs=-1)]: Done 891 tasks      | elapsed: 208.0min\n[Parallel(n_jobs=-1)]: Done 892 tasks      | elapsed: 208.2min\n[Parallel(n_jobs=-1)]: Done 893 tasks      | elapsed: 208.5min\n[Parallel(n_jobs=-1)]: Done 894 tasks      | elapsed: 208.9min\n[Parallel(n_jobs=-1)]: Done 895 tasks      | elapsed: 209.1min\n[Parallel(n_jobs=-1)]: Done 896 tasks      | elapsed: 209.8min\n[Parallel(n_jobs=-1)]: Done 897 tasks      | elapsed: 209.9min\n[Parallel(n_jobs=-1)]: Done 898 tasks      | elapsed: 209.9min\n[Parallel(n_jobs=-1)]: Done 899 tasks      | elapsed: 210.3min\n[Parallel(n_jobs=-1)]: Done 900 tasks      | elapsed: 210.3min\n[Parallel(n_jobs=-1)]: Done 901 tasks      | elapsed: 210.6min\n[Parallel(n_jobs=-1)]: Done 902 tasks      | elapsed: 210.8min\n[Parallel(n_jobs=-1)]: Done 903 tasks      | elapsed: 211.2min\n[Parallel(n_jobs=-1)]: Done 904 tasks      | elapsed: 211.3min\n[Parallel(n_jobs=-1)]: Done 905 tasks      | elapsed: 211.4min\n[Parallel(n_jobs=-1)]: Done 906 tasks      | elapsed: 211.9min\n[Parallel(n_jobs=-1)]: Done 907 tasks      | elapsed: 212.2min\n[Parallel(n_jobs=-1)]: Done 908 tasks      | elapsed: 212.3min\n[Parallel(n_jobs=-1)]: Done 909 tasks      | elapsed: 212.6min\n[Parallel(n_jobs=-1)]: Done 910 tasks      | elapsed: 212.8min\n[Parallel(n_jobs=-1)]: Done 911 tasks      | elapsed: 213.7min\n[Parallel(n_jobs=-1)]: Done 912 tasks      | elapsed: 213.7min\n[Parallel(n_jobs=-1)]: Done 913 tasks      | elapsed: 213.8min\n[Parallel(n_jobs=-1)]: Done 914 tasks      | elapsed: 214.3min\n[Parallel(n_jobs=-1)]: Done 915 tasks      | elapsed: 214.6min\n[Parallel(n_jobs=-1)]: Done 916 tasks      | elapsed: 215.2min\n[Parallel(n_jobs=-1)]: Done 917 tasks      | elapsed: 215.2min\n[Parallel(n_jobs=-1)]: Done 918 tasks      | elapsed: 215.5min\n[Parallel(n_jobs=-1)]: Done 919 tasks      | elapsed: 215.7min\n[Parallel(n_jobs=-1)]: Done 920 tasks      | elapsed: 215.7min\n[Parallel(n_jobs=-1)]: Done 921 tasks      | elapsed: 215.9min\n[Parallel(n_jobs=-1)]: Done 922 tasks      | elapsed: 216.5min\n[Parallel(n_jobs=-1)]: Done 923 tasks      | elapsed: 216.5min\n[Parallel(n_jobs=-1)]: Done 924 tasks      | elapsed: 216.7min\n[Parallel(n_jobs=-1)]: Done 925 tasks      | elapsed: 216.7min\n[Parallel(n_jobs=-1)]: Done 926 tasks      | elapsed: 217.3min\n[Parallel(n_jobs=-1)]: Done 927 tasks      | elapsed: 217.4min\n[Parallel(n_jobs=-1)]: Done 928 tasks      | elapsed: 217.4min\n[Parallel(n_jobs=-1)]: Done 929 tasks      | elapsed: 217.7min\n[Parallel(n_jobs=-1)]: Done 930 tasks      | elapsed: 218.2min\n[Parallel(n_jobs=-1)]: Done 931 tasks      | elapsed: 218.8min\n[Parallel(n_jobs=-1)]: Done 932 tasks      | elapsed: 218.8min\n[Parallel(n_jobs=-1)]: Done 933 tasks      | elapsed: 218.9min\n[Parallel(n_jobs=-1)]: Done 934 tasks      | elapsed: 219.5min\n[Parallel(n_jobs=-1)]: Done 935 tasks      | elapsed: 219.7min\n[Parallel(n_jobs=-1)]: Done 936 tasks      | elapsed: 220.1min\n[Parallel(n_jobs=-1)]: Done 937 tasks      | elapsed: 220.2min\n[Parallel(n_jobs=-1)]: Done 938 tasks      | elapsed: 220.4min\n[Parallel(n_jobs=-1)]: Done 939 tasks      | elapsed: 220.8min\n[Parallel(n_jobs=-1)]: Done 940 tasks      | elapsed: 220.9min\n[Parallel(n_jobs=-1)]: Done 941 tasks      | elapsed: 220.9min\n[Parallel(n_jobs=-1)]: Done 942 tasks      | elapsed: 221.3min\n[Parallel(n_jobs=-1)]: Done 943 tasks      | elapsed: 221.4min\n[Parallel(n_jobs=-1)]: Done 944 tasks      | elapsed: 221.5min\n[Parallel(n_jobs=-1)]: Done 945 tasks      | elapsed: 221.6min\n[Parallel(n_jobs=-1)]: Done 960 out of 960 | elapsed: 225.2min finished\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "GridSearchCV(cv=5, error_score='raise-deprecating',\n       estimator=RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n           max_features='auto', max_leaf_nodes=None,\n           min_impurity_decrease=0.0, min_impurity_split=None,\n           min_samples_leaf=1, min_samples_split=2,\n           min_weight_fraction_leaf=0.0, n_estimators='warn', n_jobs=None,\n           oob_score=False, random_state=None, verbose=0, warm_start=False),\n       fit_params=None, iid='warn', n_jobs=-1,\n       param_grid={'n_estimators': [1200, 1600, 2000, 2400], 'max_features': ['auto'], 'max_depth': [60, 100, 150, None], 'min_samples_split': [80, 100, 140, 200], 'min_samples_leaf': [1, 5, 10], 'bootstrap': [True]},\n       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n       scoring='neg_mean_squared_error', verbose=50)"
     },
     "metadata": {},
     "execution_count": 142
    }
   ],
   "source": [
    "forest_grid_search.fit(train_X, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{'bootstrap': True, 'max_depth': None, 'max_features': 'auto', 'min_samples_leaf': 1, 'min_samples_split': 80, 'n_estimators': 1200}\n"
    }
   ],
   "source": [
    "print(forest_grid_search.best_params_)\n",
    "save_model(forest_grid_search.best_estimator_, \"gridsearch_forest.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_cv_results(forest_grid_search.cv_results_, \"forest_gridsearch_results.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "   bootstrap  max_depth max_features  min_samples_leaf  min_samples_split  \\\n0       True       60.0         auto                 1                 80   \n1       True       60.0         auto                 1                 80   \n2       True       60.0         auto                 1                 80   \n3       True       60.0         auto                 1                 80   \n4       True       60.0         auto                 1                100   \n\n   n_estimators  train_error_mean  val_error_mean  error_diff_mean  \\\n0          1200          0.725895        0.824152         0.098257   \n1          1600          0.725903        0.823978         0.098076   \n2          2000          0.725848        0.823915         0.098067   \n3          2400          0.725925        0.824183         0.098258   \n4          1200          0.746206        0.829122         0.082916   \n\n   train_error_std  val_error_std  \n0         0.002671       0.004940  \n1         0.002743       0.004960  \n2         0.002682       0.005157  \n3         0.002630       0.004870  \n4         0.002466       0.005022  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>bootstrap</th>\n      <th>max_depth</th>\n      <th>max_features</th>\n      <th>min_samples_leaf</th>\n      <th>min_samples_split</th>\n      <th>n_estimators</th>\n      <th>train_error_mean</th>\n      <th>val_error_mean</th>\n      <th>error_diff_mean</th>\n      <th>train_error_std</th>\n      <th>val_error_std</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>True</td>\n      <td>60.0</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>80</td>\n      <td>1200</td>\n      <td>0.725895</td>\n      <td>0.824152</td>\n      <td>0.098257</td>\n      <td>0.002671</td>\n      <td>0.004940</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>True</td>\n      <td>60.0</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>80</td>\n      <td>1600</td>\n      <td>0.725903</td>\n      <td>0.823978</td>\n      <td>0.098076</td>\n      <td>0.002743</td>\n      <td>0.004960</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>True</td>\n      <td>60.0</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>80</td>\n      <td>2000</td>\n      <td>0.725848</td>\n      <td>0.823915</td>\n      <td>0.098067</td>\n      <td>0.002682</td>\n      <td>0.005157</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>True</td>\n      <td>60.0</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>80</td>\n      <td>2400</td>\n      <td>0.725925</td>\n      <td>0.824183</td>\n      <td>0.098258</td>\n      <td>0.002630</td>\n      <td>0.004870</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>True</td>\n      <td>60.0</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>100</td>\n      <td>1200</td>\n      <td>0.746206</td>\n      <td>0.829122</td>\n      <td>0.082916</td>\n      <td>0.002466</td>\n      <td>0.005022</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 231
    }
   ],
   "source": [
    "forest_gridsearch_summary = summarize_cv_results(forest_grid_search.cv_results_)\n",
    "forest_gridsearch_summary.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "     bootstrap  max_depth max_features  min_samples_leaf  min_samples_split  \\\n144       True        NaN         auto                 1                 80   \n\n     n_estimators  train_error_mean  val_error_mean  error_diff_mean  \\\n144          1200          0.725771        0.823648         0.097876   \n\n     train_error_std  val_error_std  \n144         0.002576       0.004816  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>bootstrap</th>\n      <th>max_depth</th>\n      <th>max_features</th>\n      <th>min_samples_leaf</th>\n      <th>min_samples_split</th>\n      <th>n_estimators</th>\n      <th>train_error_mean</th>\n      <th>val_error_mean</th>\n      <th>error_diff_mean</th>\n      <th>train_error_std</th>\n      <th>val_error_std</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>144</th>\n      <td>True</td>\n      <td>NaN</td>\n      <td>auto</td>\n      <td>1</td>\n      <td>80</td>\n      <td>1200</td>\n      <td>0.725771</td>\n      <td>0.823648</td>\n      <td>0.097876</td>\n      <td>0.002576</td>\n      <td>0.004816</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 232
    }
   ],
   "source": [
    "forest_gridsearch_summary.loc[[forest_grid_search.best_index_]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The best model from grid search does not overfit too much and has an error that is slightly higher than the base model\n",
    "- Error doesn't seem to be decreasing sharply so more fine-tuning probably won't yield a much better model\n",
    "- This is the best random forest regressor so far"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(forest_grid_search.best_estimator_, 'best_forest.pkl')"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python37364bit5171d068c3054b8f80dd7b7dde0249a4",
   "display_name": "Python 3.7.3 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}