HyperoptTuningH2o.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321. connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>3 days 3 hours 17 mins</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.10.0.9</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>10 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_arvc_lgnmyd</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>35.31 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>16</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>16</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.5.2 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ---------------------------\n",
       "H2O cluster uptime:         3 days 3 hours 17 mins\n",
       "H2O cluster version:        3.10.0.9\n",
       "H2O cluster version age:    10 days\n",
       "H2O cluster name:           H2O_from_python_arvc_lgnmyd\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    35.31 Gb\n",
       "H2O cluster total cores:    16\n",
       "H2O cluster allowed cores:  16\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://localhost:54321\n",
       "H2O connection proxy:\n",
       "Python version:             3.5.2 final\n",
       "--------------------------  ---------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import explained_variance_score,r2_score,mean_absolute_error\n",
    "from sklearn import preprocessing\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from hyperopt import hp\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, rand\n",
    "import sys\n",
    "import os\n",
    "from utils import get_allstate_train_valid_test_testids\n",
    "import h2o\n",
    "import os\n",
    "import time\n",
    "h2o.init(max_mem_size = \"40G\")             #specify max number of bytes. uses all cores by default.\n",
    "h2o.remove_all()  \n",
    "from h2o.estimators.gbm import H2OGradientBoostingEstimator\n",
    "from h2o.estimators.deeplearning import H2ODeepLearningEstimator\n",
    "from h2o.estimators.glm import H2OGeneralizedLinearEstimator\n",
    "from h2o.estimators.random_forest import H2ORandomForestEstimator\n",
    "from h2o.estimators.glrm import H2OGeneralizedLowRankEstimator\n",
    "from h2o import H2OFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape is: (188318, 132)\n",
      "Test shape is: (125546, 131)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvc/t81_558_deep_learning/utils.py:140: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  train.drop(\"type\", axis=1, inplace=True)\n",
      "/home/arvc/t81_558_deep_learning/utils.py:141: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  test.drop(\"type\", axis=1, inplace=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Train shape is: (160070, 1191)\n",
      "Final Valid shape is: (28248, 1191)\n",
      "Final Test shape is: (125546, 1191)\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "#Read Input CSV file\n",
    "shift = 203\n",
    "\n",
    "train, valid, test, testids = get_allstate_train_valid_test_testids(0.15, shift, True)\n",
    "test[\"loss\"] = 0.0\n",
    "\n",
    "trainframe = H2OFrame(train)\n",
    "validframe = H2OFrame(valid)\n",
    "testframe = H2OFrame(test)\n",
    "del(train)\n",
    "del(valid)\n",
    "del(test)\n",
    "y = \"loss\"\n",
    "x = list(trainframe.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with params : \n",
      "{'gamma': 0.65, 'sample_rate': 0.04, 'learn_rate': 4.62, 'max_depth': 100, 'ntrees': 750, 'col_sample_rate': 0.33}\n"
     ]
    },
    {
     "ename": "H2OResponseError",
     "evalue": "Server error water.exceptions.H2OKeyNotFoundArgumentException:\n  Error: Object 'None' not found in function: predict for argument: model\n  Request: POST /4/Predictions/models/None/frames/Key_Frame__upload_8fe6e9bca7ea6ddf6fe559dff4ec4da1.hex\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mH2OResponseError\u001b[0m                          Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-5266f37f4ab6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0mtrials\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrials\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-22-5266f37f4ab6>\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(trials)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m     \u001b[0mbest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malgo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtpe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msuggest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_evals\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Best params:\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin)\u001b[0m\n\u001b[1;32m    304\u001b[0m             \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    305\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 306\u001b[0;31m             \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreturn_argmin\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    307\u001b[0m             )\n\u001b[1;32m    308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(self, fn, space, algo, max_evals, rstate, verbose, pass_expr_memo_ctrl, catch_eval_exceptions, return_argmin)\u001b[0m\n\u001b[1;32m    631\u001b[0m             \u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpass_expr_memo_ctrl\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m             \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 633\u001b[0;31m             return_argmin=return_argmin)\n\u001b[0m\u001b[1;32m    634\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mfmin\u001b[0;34m(fn, space, algo, max_evals, trials, rstate, allow_trials_fmin, pass_expr_memo_ctrl, catch_eval_exceptions, verbose, return_argmin)\u001b[0m\n\u001b[1;32m    317\u001b[0m                     verbose=verbose)\n\u001b[1;32m    318\u001b[0m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_eval_exceptions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcatch_eval_exceptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m     \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreturn_argmin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmin\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mexhaust\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    196\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mexhaust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    197\u001b[0m         \u001b[0mn_done\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 198\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_evals\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mn_done\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblock_until_done\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    199\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, N, block_until_done)\u001b[0m\n\u001b[1;32m    170\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m                 \u001b[0;31m# -- loop over trials and do the jobs directly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 172\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mserial_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    173\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstopped\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/fmin.py\u001b[0m in \u001b[0;36mserial_evaluate\u001b[0;34m(self, N)\u001b[0m\n\u001b[1;32m     87\u001b[0m                 \u001b[0mctrl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbase\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCtrl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrials\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_trial\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrial\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdomain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mctrl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m                     \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'job exception: %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/hyperopt/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, config, ctrl, attach_attachments)\u001b[0m\n\u001b[1;32m    836\u001b[0m                 \u001b[0mmemo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmemo\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    837\u001b[0m                 print_node_on_error=self.rec_eval_print_node_on_error)\n\u001b[0;32m--> 838\u001b[0;31m             \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpyll_rval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    839\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    840\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-22-5266f37f4ab6>\u001b[0m in \u001b[0;36mscore\u001b[0;34m(params)\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m     \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m     \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0mmean_absolute_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidframe\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_data_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mshift\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_data_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mshift\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mmin_mae\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/h2o/model/model_base.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, test_data)\u001b[0m\n\u001b[1;32m    146\u001b[0m         \"\"\"\n\u001b[1;32m    147\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh2o\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mH2OFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"test_data must be an instance of H2OFrame\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m         j = H2OJob(h2o.api(\"POST /4/Predictions/models/%s/frames/%s\" % (self.model_id, test_data.frame_id)),\n\u001b[0m\u001b[1;32m    149\u001b[0m                    self._model_json['algo'] + \" prediction\")\n\u001b[1;32m    150\u001b[0m         \u001b[0mj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/h2o/h2o.py\u001b[0m in \u001b[0;36mapi\u001b[0;34m(endpoint, data, json, filename, save_to)\u001b[0m\n\u001b[1;32m     82\u001b[0m     \u001b[0;31m# type checks are performed in H2OConnection class\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m     \u001b[0m_check_connection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 84\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mh2oconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mendpoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfilename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_to\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msave_to\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/h2o/backend/connection.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, endpoint, data, json, filename, save_to)\u001b[0m\n\u001b[1;32m    257\u001b[0m                                     auth=self._auth, verify=self._verify_ssl_cert, proxies=self._proxies)\n\u001b[1;32m    258\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_end_transaction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart_time\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_to\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    260\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mConnectionError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHTTPError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/h2o/backend/connection.py\u001b[0m in \u001b[0;36m_process_response\u001b[0;34m(response, save_to)\u001b[0m\n\u001b[1;32m    584\u001b[0m         \u001b[0;31m# Client errors (400 = \"Bad Request\", 404 = \"Not Found\", 412 = \"Precondition Failed\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    585\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstatus_code\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;36m400\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m404\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m412\u001b[0m\u001b[0;34m}\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mH2OErrorV3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mH2OModelBuilderErrorV3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 586\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mH2OResponseError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    587\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m         \u001b[0;31m# Server errors (notably 500 = \"Server Error\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mH2OResponseError\u001b[0m: Server error water.exceptions.H2OKeyNotFoundArgumentException:\n  Error: Object 'None' not found in function: predict for argument: model\n  Request: POST /4/Predictions/models/None/frames/Key_Frame__upload_8fe6e9bca7ea6ddf6fe559dff4ec4da1.hex\n"
     ]
    }
   ],
   "source": [
    "min_mae = 10000\n",
    "\n",
    "def score(params):\n",
    "    print(\"Training with params : \")\n",
    "    print(params)\n",
    "    \n",
    "    model = H2OGradientBoostingEstimator(\n",
    "                 ntrees=params[\"ntrees\"],\n",
    "                 learn_rate=params[\"learn_rate\"],\n",
    "                 max_depth=params[\"max_depth\"],\n",
    "                 sample_rate=params[\"sample_rate\"],\n",
    "                 col_sample_rate=0.7,\n",
    "                 stopping_rounds=200,\n",
    "                 stopping_tolerance=0.01, #10-fold increase in threshold as defined in rf_v1\n",
    "                 score_each_iteration=True,\n",
    "                 seed=200000,\n",
    "                 nfolds=6,                 \n",
    "                 keep_cross_validation_predictions=True\n",
    "                )\n",
    "    \n",
    "   \n",
    "    predictions = model.predict(validframe)\n",
    "    score =  mean_absolute_error(np.exp(validframe[\"loss\"].as_data_frame()) - shift, np.exp(predictions.as_data_frame()) - shift)\n",
    "    if score < min_mae:\n",
    "        min_mae = score\n",
    "        \n",
    "    print(\"MAE {}\".format(score))\n",
    "    return {'loss': score, 'status': STATUS_OK}\n",
    "\n",
    "\n",
    "def optimize(trials):\n",
    "    space = {\n",
    "             'ntrees' : hp.choice('ntrees', [100, 250, 500, 750,1000,1250, 1500, 1750, 2000]),\n",
    "             'learn_rate' : hp.quniform('learn_rate', 0.01, 5, 0.01),\n",
    "             'max_depth' : hp.choice('max_depth', [100, 250, 500, 750,1000,1250, 1500, 1750, 2000,2250]),            \n",
    "             'sample_rate' : hp.quniform('sample_rate', 0.01, 1, 0.01),\n",
    "             'gamma' : hp.quniform('gamma', 0.5, 1, 0.05),\n",
    "             'col_sample_rate' : hp.quniform('col_sample_rate', 0.01, 1, 0.01),\n",
    "             }\n",
    "    \n",
    "    \n",
    "    best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=1000)\n",
    "    print(\"Best params:\")\n",
    "    print(best)\n",
    "    print(\"Min MAE={}\".format(min_mae))\n",
    "\n",
    "\n",
    "#Trials object where the history of search will be stored\n",
    "trials = Trials()\n",
    "\n",
    "optimize(trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}