HyperoptTuning-AllState-Keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvc/anaconda3/envs/tensorflow/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "Using Theano backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape is: (188318, 132)\n",
      "Test shape is: (125546, 131)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/arvc/t81_558_deep_learning/utils.py:139: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  train.drop(\"type\", axis=1, inplace=True)\n",
      "/home/arvc/t81_558_deep_learning/utils.py:140: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  test.drop(\"type\", axis=1, inplace=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Train shape is: (160070, 131)\n",
      "Final Valid shape is: (28248, 131)\n",
      "Final Test shape is: (125546, 131)\n",
      "float64\n",
      "float64\n",
      "float64\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import explained_variance_score,r2_score,mean_absolute_error\n",
    "from sklearn import preprocessing\n",
    "from random import randint\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from hyperopt import hp\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, rand\n",
    "import sys\n",
    "import xgboost as xgb\n",
    "import os\n",
    "from utils import get_allstate_train_valid_test_testids, to_xy\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers.advanced_activations import PReLU\n",
    "\n",
    "shift = 205\n",
    "train, valid, test, testids = get_allstate_train_valid_test_testids(0.15, shift, True)\n",
    "x_train,y_train = to_xy(train, \"loss\")\n",
    "x_valid,y_valid = to_xy(valid, \"loss\")\n",
    "x_test,y_test = to_xy(test, \"loss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with params : \n",
      "{'nb_epoch': 502, 'validation_split': 0.17, 'batch_size': 231}\n"
     ]
    }
   ],
   "source": [
    "def build_fn_reg(hidden_dims=50):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Dense(input_dim = x_train.shape[1], output_dim=400))\n",
    "    model.add(PReLU())\n",
    "    model.add(Dense(hidden_dims))\n",
    "    model.add(Dropout(0.4))\n",
    "    model.add(Dense(input_dim=400, output_dim=200, init=\"uniform\"))\n",
    "    model.add(PReLU())\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(input_dim=200,output_dim=1))\n",
    "    model.compile(loss = 'mean_absolute_error', optimizer = 'sgd', metrics=['accuracy']) #try optimizer adam, adadelta\n",
    "    return(model)\n",
    "\n",
    "def score(params):           \n",
    "    print(\"Training with params : \")\n",
    "    print(params)   \n",
    "   \n",
    "    model = KerasRegressor(build_fn=build_fn_reg, \n",
    "                           nb_epoch=params[\"nb_epoch\"], \n",
    "                           verbose=0, \n",
    "                           batch_size=params[\"batch_size\"],\n",
    "                           validation_split=params[\"validation_split\"], shuffle='batch', \n",
    "                           #hidden_dims=params[\"hidden_dims\"]\n",
    "                          )\n",
    "\n",
    "    model.fit(x_train, y_train)\n",
    "    \n",
    "    predictions = model.predict(x_valid)\n",
    "    \n",
    "    print(predictions)\n",
    "    score =  mean_absolute_error(np.exp(y_valid) - shift, np.exp(predictions) - shift)\n",
    "    print(\"\\tMAE {0}\\n\\n\".format(score))\n",
    "    return {'loss': score, 'status': STATUS_OK}\n",
    "\n",
    "def optimize(trials):\n",
    "    space = {\n",
    "             'nb_epoch' : hp.choice('nb_epoch', np.arange(3, 3000, dtype=int)),\n",
    "             'batch_size' : hp.choice('batch_size', np.arange(1, 300, dtype=int)),            \n",
    "             'validation_split' : hp.quniform('validation_split', 0.01, 0.95, 0.01),\n",
    "             #'hidden_dims' : hp.choice('hidden_dims', np.arange(1, 125, dtype=int))\n",
    "             }\n",
    "\n",
    "    best = fmin(score, space, algo=tpe.suggest, trials=trials, max_evals=250)\n",
    "\n",
    "    print(\"Best params are:\")\n",
    "    print(best)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#Trials object where the history of search will be stored\n",
    "trials = Trials()\n",
    "\n",
    "optimize(trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow]",
   "language": "python",
   "name": "conda-env-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}