8.2 Hyperopt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jannes/anaconda/lib/python3.5/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n",
      "Couldn't import dot_parser, loading of dot files will not be possible.\n"
     ]
    }
   ],
   "source": [
    "from hyperopt import Trials, STATUS_OK, tpe\n",
    "from hyperas import optim\n",
    "from hyperas.distributions import choice, uniform\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.optimizers import RMSprop\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data():\n",
    "    '''\n",
    "    Data providing function:\n",
    "\n",
    "    This function is separated from model() so that hyperopt\n",
    "    won't reload data for each evaluation run.\n",
    "    '''\n",
    "    (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "    X_train = X_train.reshape(60000, 784)\n",
    "    X_test = X_test.reshape(10000, 784)\n",
    "    X_train = X_train.astype('float32')\n",
    "    X_test = X_test.astype('float32')\n",
    "    X_train /= 255\n",
    "    X_test /= 255\n",
    "    nb_classes = 10\n",
    "    Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
    "    Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
    "    return X_train, Y_train, X_test, Y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(X_train, Y_train, X_test, Y_test):\n",
    "    '''\n",
    "    Model providing function:\n",
    "\n",
    "    Create Keras model with double curly brackets dropped-in as needed.\n",
    "    Return value has to be a valid python dictionary with two customary keys:\n",
    "        - loss: Specify a numeric evaluation metric to be minimized\n",
    "        - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n",
    "    The last one is optional, though recommended, namely:\n",
    "        - model: specify the model just created so that we can later use it again.\n",
    "    '''\n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_shape=(784,)))\n",
    "    \n",
    "    model.add(Activation('relu'))\n",
    "    \n",
    "    model.add(Dropout({{uniform(0, 0.5)}}))\n",
    "    \n",
    "    model.add(Dense({{choice([256, 512, 1024])}}))\n",
    "    \n",
    "    model.add(Activation({{choice(['relu','tanh'])}}))\n",
    "    \n",
    "    model.add(Dropout({{uniform(0, 0.5)}}))\n",
    "    \n",
    "    model.add(Dense(10))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    rms = RMSprop()\n",
    "    model.compile(loss='categorical_crossentropy', \n",
    "                  optimizer=rms, \n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    model.fit(X_train, y_train,\n",
    "              batch_size={{choice([64, 128])}},\n",
    "              epochs=1,\n",
    "              verbose=2,\n",
    "              validation_data=(X_test, Y_test))\n",
    "    score, acc = model.evaluate(X_test, Y_test, verbose=0)\n",
    "    print('Test accuracy:', acc)\n",
    "    return {'loss': -acc, 'status': STATUS_OK, 'model': model}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>> Imports:\n",
      "#coding=utf-8\n",
      "\n",
      "try:\n",
      "    from hyperopt import Trials, STATUS_OK, tpe\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas import optim\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from hyperas.distributions import choice, uniform\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.models import Sequential\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.layers import Dense, Dropout, Activation\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.optimizers import RMSprop\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.datasets import mnist\n",
      "except:\n",
      "    pass\n",
      "\n",
      "try:\n",
      "    from keras.utils import np_utils\n",
      "except:\n",
      "    pass\n",
      "\n",
      ">>> Hyperas search space:\n",
      "\n",
      "def get_space():\n",
      "    return {\n",
      "        'Dropout': hp.uniform('Dropout', 0, 0.5),\n",
      "        'Dense': hp.choice('Dense', [256, 512, 1024]),\n",
      "        'Activation': hp.choice('Activation', ['relu','tanh']),\n",
      "        'Dropout_1': hp.uniform('Dropout_1', 0, 0.5),\n",
      "        'batch_size': hp.choice('batch_size', [64, 128]),\n",
      "    }\n",
      "\n",
      ">>> Data\n",
      "  1: \n",
      "  2: '''\n",
      "  3: Data providing function:\n",
      "  4: \n",
      "  5: This function is separated from model() so that hyperopt\n",
      "  6: won't reload data for each evaluation run.\n",
      "  7: '''\n",
      "  8: (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
      "  9: X_train = X_train.reshape(60000, 784)\n",
      " 10: X_test = X_test.reshape(10000, 784)\n",
      " 11: X_train = X_train.astype('float32')\n",
      " 12: X_test = X_test.astype('float32')\n",
      " 13: X_train /= 255\n",
      " 14: X_test /= 255\n",
      " 15: nb_classes = 10\n",
      " 16: Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
      " 17: Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
      " 18: \n",
      " 19: \n",
      " 20: \n",
      ">>> Resulting replaced keras model:\n",
      "\n",
      "   1: def keras_fmin_fnct(space):\n",
      "   2: \n",
      "   3:     '''\n",
      "   4:     Model providing function:\n",
      "   5: \n",
      "   6:     Create Keras model with double curly brackets dropped-in as needed.\n",
      "   7:     Return value has to be a valid python dictionary with two customary keys:\n",
      "   8:         - loss: Specify a numeric evaluation metric to be minimized\n",
      "   9:         - status: Just use STATUS_OK and see hyperopt documentation if not feasible\n",
      "  10:     The last one is optional, though recommended, namely:\n",
      "  11:         - model: specify the model just created so that we can later use it again.\n",
      "  12:     '''\n",
      "  13:     model = Sequential()\n",
      "  14:     model.add(Dense(512, input_shape=(784,)))\n",
      "  15:     \n",
      "  16:     model.add(Activation('relu'))\n",
      "  17:     \n",
      "  18:     model.add(Dropout(space['Dropout']))\n",
      "  19:     \n",
      "  20:     model.add(Dense(space['Dense']))\n",
      "  21:     \n",
      "  22:     model.add(Activation(space['Activation']))\n",
      "  23:     \n",
      "  24:     model.add(Dropout(space['Dropout_1']))\n",
      "  25:     \n",
      "  26:     model.add(Dense(10))\n",
      "  27:     model.add(Activation('softmax'))\n",
      "  28: \n",
      "  29:     rms = RMSprop()\n",
      "  30:     model.compile(loss='categorical_crossentropy', \n",
      "  31:                   optimizer=rms, \n",
      "  32:                   metrics=['accuracy'])\n",
      "  33: \n",
      "  34:     model.fit(X_train, Y_train,\n",
      "  35:               batch_size=space['batch_size'],\n",
      "  36:               epochs=1,\n",
      "  37:               verbose=2,\n",
      "  38:               validation_data=(X_test, Y_test))\n",
      "  39:     score, acc = model.evaluate(X_test, Y_test, verbose=0)\n",
      "  40:     print('Test accuracy:', acc)\n",
      "  41:     return {'loss': -acc, 'status': STATUS_OK, 'model': model}\n",
      "  42: \n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 10s - loss: 0.2395 - acc: 0.9268 - val_loss: 0.1246 - val_acc: 0.9648\n",
      "Test accuracy: 0.9648\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 10s - loss: 0.2225 - acc: 0.9323 - val_loss: 0.1066 - val_acc: 0.9670\n",
      "Test accuracy: 0.967\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 7s - loss: 0.2672 - acc: 0.9174 - val_loss: 0.1273 - val_acc: 0.9589\n",
      "Test accuracy: 0.9589\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 10s - loss: 0.2476 - acc: 0.9235 - val_loss: 0.1337 - val_acc: 0.9615\n",
      "Test accuracy: 0.9615\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/1\n",
      " - 10s - loss: 0.2328 - acc: 0.9292 - val_loss: 0.0982 - val_acc: 0.9707\n",
      "Test accuracy: 0.9707\n"
     ]
    }
   ],
   "source": [
    "best_run, best_model = optim.minimize(model=model,\n",
    "                                      data=data,\n",
    "                                      algo=tpe.suggest,\n",
    "                                      max_evals=5,\n",
    "                                      trials=Trials(),\n",
    "                                      notebook_name='simple_notebook')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evalutation of best performing model:\n",
      "10000/10000 [==============================] - 1s 95us/step\n",
      "[0.12623950149156155, 0.9624]\n"
     ]
    }
   ],
   "source": [
    "print(\"Evalutation of best performing model:\")\n",
    "print(best_model.evaluate(X_test, Y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Activation': 1,\n",
       " 'Dense': 2,\n",
       " 'Dropout': 0.11654484411565585,\n",
       " 'Dropout_1': 0.4566135465496464,\n",
       " 'batch_size': 1}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_79 (Dense)             (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "activation_79 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_53 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_80 (Dense)             (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "activation_80 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_54 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_81 (Dense)             (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_81 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 669,706\n",
      "Trainable params: 669,706\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "best_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "??choice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}