notebooks/07_01_bidirectional_rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Users\\Yue\\Anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.keras.datasets import mnist\n",
    "from tensorflow.contrib.eager.python import tfe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# enable eager mode\n",
    "tf.enable_eager_execution()\n",
    "tf.set_random_seed(0)\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('weights/'):\n",
    "    os.makedirs('weights/')\n",
    "\n",
    "# constants\n",
    "units = 64\n",
    "batch_size = 256\n",
    "epochs = 2\n",
    "num_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x train (60000, 28, 28)\n",
      "y train (60000, 10)\n",
      "x test (10000, 28, 28)\n",
      "y test (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "# dataset loading\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "x_train = x_train.astype('float32') / 255.\n",
    "x_test = x_test.astype('float32') / 255.\n",
    "x_train = x_train.reshape((-1, 28, 28))  # 28 timesteps, 28 inputs / timestep\n",
    "x_test = x_test.reshape((-1, 28, 28))  # 28 timesteps, 28 inputs / timeste\n",
    "\n",
    "# one hot encode the labels. convert back to numpy as we cannot use a combination of numpy\n",
    "# and tensors as input to keras\n",
    "y_train_ohe = tf.one_hot(y_train, depth=num_classes).numpy()\n",
    "y_test_ohe = tf.one_hot(y_test, depth=num_classes).numpy()\n",
    "\n",
    "print('x train', x_train.shape)\n",
    "print('y train', y_train_ohe.shape)\n",
    "print('x test', x_test.shape)\n",
    "print('y test', y_test_ohe.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bi-Directional LSTM\n",
    "\n",
    "Writing a Bi-directional LSTM in keras is super simple with the Bidirectional wrapper. However the speed of such a model is slower than expected.\n",
    "\n",
    "Some fixes for it are to use the GPU implementation for all the cells, and to unroll the entire RNN before hand. In normal Keras and Tensorflow, unrolling the RNN yields significant speed improvements since the symbolic loop is replaced with the unrolled graph representation of the RNN. \n",
    "\n",
    "In Eager, I don't believe it is doing much to help with the speed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiRNN(tf.keras.Model):\n",
    "    def __init__(self, units, num_classes, merge_mode='concat', num_layers=1):\n",
    "        super(BiRNN, self).__init__()\n",
    "        self.impl = 1 if tfe.num_gpus() == 0 else 2\n",
    "        self.cells = [tf.keras.layers.LSTMCell(units, implementation=self.impl) for _ in range(num_layers)]\n",
    "        self.rnn = tf.keras.layers.RNN(self.cells, unroll=True)  # slower if not unrolled - probably because it is using K.rnn() internally.\n",
    "        self.bidirectional = tf.keras.layers.Bidirectional(self.rnn, merge_mode=merge_mode)\n",
    "        self.classifier = tf.keras.layers.Dense(num_classes)\n",
    "\n",
    "    def call(self, inputs, training=None, mask=None):\n",
    "        x = self.bidirectional(inputs)\n",
    "        output = self.classifier(x)\n",
    "\n",
    "        # softmax op does not exist on the gpu, so always use cpu\n",
    "        with tf.device('/cpu:0'):\n",
    "            output = tf.nn.softmax(output)\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "60000/60000 [==============================] - 275s 5ms/step - loss: 0.3881 - acc: 0.8740 - val_loss: 0.1089 - val_acc: 0.9657\n",
      "Epoch 2/2\n",
      "60000/60000 [==============================] - 260s 4ms/step - loss: 0.0927 - acc: 0.9721 - val_loss: 0.0747 - val_acc: 0.9748\n",
      "10000/10000 [==============================] - 13s 1ms/step\n",
      "Final test loss and accuracy : [0.07469581732600927, 0.9748]\n"
     ]
    }
   ],
   "source": [
    "device = '/cpu:0' if tfe.num_gpus() == 0 else '/gpu:0'\n",
    "\n",
    "with tf.device(device):\n",
    "    # build model and optimizer\n",
    "    model = BiRNN(units, num_classes, num_layers=2)\n",
    "    model.compile(optimizer=tf.train.AdamOptimizer(0.01), loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "     # TF Keras tries to use entire dataset to determine shape without this step when using .fit()\n",
    "    # Fix = Use exactly one sample from the provided input dataset to determine input/output shape/s for the model\n",
    "    dummy_x = tf.zeros((1, 28, 28))\n",
    "    model._set_inputs(dummy_x)\n",
    "\n",
    "    # train\n",
    "    model.fit(x_train, y_train_ohe, batch_size=batch_size, epochs=epochs,\n",
    "              validation_data=(x_test, y_test_ohe), verbose=1)\n",
    "\n",
    "    # evaluate on test set\n",
    "    scores = model.evaluate(x_test, y_test_ohe, batch_size, verbose=1)\n",
    "    print(\"Final test loss and accuracy :\", scores)\n",
    "\n",
    "    saver = tfe.Saver(model.variables)\n",
    "    saver.save('weights/07_01_bi_rnn/weights.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:Anaconda3]",
   "language": "python",
   "name": "conda-env-Anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}