RNN in Deep Learning with tf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Running...\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "print(\"TensorFlow Running...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-3-008c45951f32>:3: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<tf.Tensor 'zeros:0' shape=(1, 4) dtype=float32>,\n",
       " <tf.Tensor 'zeros:0' shape=(1, 4) dtype=float32>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LSTM_CELL_SIZE = 4  # output size (dimension), which is same as hidden size in the cell\n",
    "\n",
    "lstm_cell = tf.contrib.rnn.BasicLSTMCell(LSTM_CELL_SIZE, state_is_tuple=True)\n",
    "state = (tf.zeros([1,LSTM_CELL_SIZE]),)*2\n",
    "state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3. 2. 2. 2. 2. 2.]]\n"
     ]
    }
   ],
   "source": [
    "sample_input = tf.constant([[3,2,2,2,2,2]],dtype=tf.float32)\n",
    "print (sess.run(sample_input))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\users\\prasa\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\ops\\rnn_cell_impl.py:735: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "WARNING:tensorflow:From c:\\users\\prasa\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow_core\\python\\ops\\rnn_cell_impl.py:739: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "LSTMStateTuple(c=array([[-0.58901733,  0.41764838, -0.31162852, -0.03253254]],\n",
      "      dtype=float32), h=array([[-0.09600677,  0.05632699, -0.07111567, -0.00346462]],\n",
      "      dtype=float32))\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope(\"LSTM_sample1\"):\n",
    "    output, state_new = lstm_cell(sample_input, state)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "print (sess.run(state_new))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.09600677  0.05632699 -0.07111567 -0.00346462]]\n"
     ]
    }
   ],
   "source": [
    "print (sess.run(output))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacked LSTM\n",
    "What about if we want to have a RNN with stacked LSTM? For example, a 2-layer LSTM. In this case, the output of the first layer will become the input of the second.\n",
    "\n",
    "Lets start with a new session:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dim = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cells = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-10-0e975edc57d7>:2: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    }
   ],
   "source": [
    "LSTM_CELL_SIZE_2 = 5 #5 hidden nodes\n",
    "cell2 = tf.contrib.rnn.LSTMCell(LSTM_CELL_SIZE_2)\n",
    "cells.append(cell2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-11-4cb0e5b3e12a>:1: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n"
     ]
    }
   ],
   "source": [
    "stacked_lstm = tf.contrib.rnn.MultiRNNCell(cells)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-12-2941a7a9b793>:3: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n"
     ]
    }
   ],
   "source": [
    "# Batch size x time steps x features.\n",
    "data = tf.placeholder(tf.float32, [None, None, input_dim])\n",
    "output, state = tf.nn.dynamic_rnn(stacked_lstm, data, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[[1, 2, 3, 4, 3, 2], [1, 2, 1, 1, 1, 2], [1, 2, 2, 2, 2, 2]],\n",
       " [[1, 2, 3, 4, 3, 2], [3, 2, 2, 1, 1, 2], [0, 0, 0, 0, 3, 2]]]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Batch size x time steps x features.\n",
    "sample_input = [[[1,2,3,4,3,2], [1,2,1,1,1,2],[1,2,2,2,2,2]],[[1,2,3,4,3,2],[3,2,2,1,1,2],[0,0,0,0,3,2]]]\n",
    "sample_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'rnn/transpose_1:0' shape=(?, ?, 5) dtype=float32>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.538081  ,  0.02986366,  0.43410382,  0.13935864,\n",
       "         -0.42896175],\n",
       "        [ 0.47635466, -0.04281311,  0.29826024, -0.17105499,\n",
       "         -0.39975625],\n",
       "        [ 0.5618018 , -0.03158254,  0.44011858, -0.08130615,\n",
       "         -0.5479461 ]],\n",
       "\n",
       "       [[ 0.538081  ,  0.02986366,  0.43410382,  0.13935864,\n",
       "         -0.42896175],\n",
       "        [ 0.6272034 ,  0.00506311,  0.3022686 , -0.2458497 ,\n",
       "         -0.35341293],\n",
       "        [ 0.3985198 ,  0.1732146 ,  0.26848492, -0.02126782,\n",
       "         -0.10924675]]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sess.run(tf.global_variables_initializer())\n",
    "sess.run(output, feed_dict={data: sample_input})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}