HyperoptTuning-AllState-XGBScikit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape is: (188318, 132)\n",
      "Test shape is: (125546, 131)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvc/t81_558_deep_learning/utils.py:139: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  train.drop(\"type\", axis=1, inplace=True)\n",
      "/home/arvc/t81_558_deep_learning/utils.py:140: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  test.drop(\"type\", axis=1, inplace=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Train shape is: (160070, 131)\n",
      "Final Valid shape is: (28248, 131)\n",
      "Final Test shape is: (125546, 131)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.ensemble import BaggingRegressor\n",
    "from xgboost import XGBRegressor\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import explained_variance_score,r2_score,mean_absolute_error\n",
    "from sklearn import preprocessing\n",
    "from random import randint\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from hyperopt import hp\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, rand\n",
    "import sys\n",
    "import xgboost as xgb\n",
    "import os\n",
    "from utils import get_allstate_train_valid_test_testids\n",
    "\n",
    "shift = 200\n",
    "train, valid, test, testids = get_allstate_train_valid_test_testids(0.15, shift, True)\n",
    "y_train = train[\"loss\"]    \n",
    "x_train = train.drop(\"loss\", axis=1)\n",
    "\n",
    "y_valid = valid[\"loss\"]    \n",
    "x_valid = valid.drop(\"loss\", axis=1)\n",
    "\n",
    "#xgtrain = xgb.DMatrix(x_train, label=y_train)\n",
    "#xgvalid = xgb.DMatrix(x_valid, label=y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with params : \n",
      "{'learning_rate': 0.35000000000000003, 'max_depth': 9, 'reg_alpha': 1, 'min_child_weight': 2.0, 'silent': True, 'colsample_bytree': 0.7000000000000001, 'nthread': -1, 'reg_lambda': 2.2, 'base_score': 1.2000000000000002, 'subsample': 0.9, 'objective': 'reg:linear', 'n_estimators': 2971, 'gamma': 1.2000000000000002, 'seed': 145932}\n"
     ]
    }
   ],
   "source": [
    "def score(params):           \n",
    "    print(\"Training with params : \")\n",
    "    print(params)\n",
    "   \n",
    "    #evallist  = [(x_valid, 'eval')]\n",
    "    base_regressor = XGBRegressor(**params)\n",
    "    base_regressor.fit(x_train, y_train)\n",
    "    predictions = base_regressor.predict(x_valid)\n",
    "                                  \n",
    "                                  \n",
    "                                  \n",
    "    #xgtrain, num_round, evals=evallist, early_stopping_rounds=20, verbose_eval=False,\n",
    "    #                 feval=custom_evalfn)\n",
    "    \n",
    "    \n",
    "    #print(predictions)\n",
    "    score =  mean_absolute_error(np.exp(y_valid) - shift, np.exp(predictions) - shift)\n",
    "    print(\"\\tMAE {0}\\n\\n\".format(score))\n",
    "    return {'loss': score, 'status': STATUS_OK}\n",
    "\n",
    "def custom_evalfn(preds, dtrain):\n",
    "    labels = dtrain.get_label()\n",
    "    return 'error', mean_absolute_error(np.exp(preds) - shift, np.exp(labels) - shift)\n",
    "\n",
    "\n",
    "def optimize(trials):\n",
    "    space = {\n",
    "        'max_depth' : hp.choice('max_depth', [9,10,11,12,13,14,15]),\n",
    "        'learning_rate' : hp.quniform('eta', 0.01, 0.5, 0.01),\n",
    "        'n_estimators' : hp.choice('n_estimators', np.arange(85, 3000, dtype=int)),\n",
    "        'silent' : True,\n",
    "        'objective': 'reg:linear',\n",
    "        'nthread' : -1,\n",
    "        'gamma' : hp.quniform('gamma', 0.0, 3, 0.2),\n",
    "        'min_child_weight' : hp.quniform('min_child_weight', 1, 3, 1),\n",
    "             'subsample' : hp.quniform('subsample', 0.6, 1.0, 0.1),\n",
    "             \n",
    "             'reg_lambda' : hp.quniform('reg_lambda', 0.0, 3, 0.2),\n",
    "             'base_score' : hp.quniform('base_score', 0.0, 3, 0.2),\n",
    "             'colsample_bytree' : hp.quniform('colsample_bytree', 0.4, 0.8, 0.1),\n",
    "             #'eval_metric': 'mae',\n",
    "             \n",
    "             \n",
    "            \n",
    "             'reg_alpha' : 1,\n",
    "             #'verbose_eval': False,\n",
    "             'seed': randint(1,429496)\n",
    "             }\n",
    "\n",
    "    best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=250)\n",
    "\n",
    "    print(\"Best params are:\")\n",
    "    print(best)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#Trials object where the history of search will be stored\n",
    "trials = Trials()\n",
    "\n",
    "optimize(trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}