5. Sequence Models/1. Building a Recurrent Neural Network - Step by Step.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building a Recurrent Neural Network - Step by Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from rnn_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Forward propagation for basic RNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 RNN cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_cell_forward(xt, a_prev,parameters):\n",
    "    Wax = parameters[\"Wax\"]\n",
    "    Waa = parameters[\"Waa\"]\n",
    "    Wya = parameters[\"Wya\"]\n",
    "    \n",
    "    ba = parameters[\"ba\"]\n",
    "    by = parameters[\"by\"]\n",
    "    \n",
    "    a_next = np.tanh(np.dot(Waa,a_prev) + np.dot(Wax,xt) + ba)\n",
    "    \n",
    "    yt_pred = softmax(np.dot(Wya,a_next)+by)\n",
    "    \n",
    "    cache = (a_next,a_prev,xt,parameters)\n",
    "    \n",
    "    return a_next,yt_pred,cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a_next.shape =  (5, 10)\n",
      "a_next[4] =  [ 0.59584544  0.18141802  0.61311866  0.99808218  0.85016201  0.99980978\n",
      " -0.18887155  0.99815551  0.6531151   0.82872037]\n",
      "yt_pred.shape =  (2, 10)\n",
      "yt_pred[1] =  [0.9888161  0.01682021 0.21140899 0.36817467 0.98988387 0.88945212\n",
      " 0.36920224 0.9966312  0.9982559  0.17746526]\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "xt = np.random.randn(3,10)\n",
    "a_prev = np.random.randn(5,10)\n",
    "Waa = np.random.randn(5,5)\n",
    "Wax = np.random.randn(5,3)\n",
    "Wya = np.random.randn(2,5)\n",
    "ba = np.random.randn(5,1)\n",
    "by = np.random.randn(2,1)\n",
    "parameters = {\"Waa\": Waa, \"Wax\": Wax, \"Wya\": Wya, \"ba\": ba, \"by\": by}\n",
    "\n",
    "a_next, yt_pred, cache = rnn_cell_forward(xt,a_prev,parameters)\n",
    "print(\"a_next.shape = \", str(a_next.shape))\n",
    "print(\"a_next[4] = \", str(a_next[4]))\n",
    "print(\"yt_pred.shape = \", str(yt_pred.shape))\n",
    "print(\"yt_pred[1] = \", str(yt_pred[1]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 RNN forward pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_forward(x, a0, parameters):\n",
    "    caches = []\n",
    "    n_x, m, T_x = x.shape\n",
    "    n_y, n_a = parameters[\"Wya\"].shape\n",
    "    \n",
    "    a = np.zeros((n_a,m,T_x))\n",
    "    y_pred = np.zeros((n_y, m, T_x))\n",
    "    \n",
    "    a_next = a0\n",
    "    \n",
    "    for t in range(T_x):\n",
    "        a_next,yt_pred,cache = rnn_cell_forward(x[:,:,t],a_next,parameters)\n",
    "        a[:,:,t] = a_next\n",
    "        y_pred[:,:,t] = yt_pred\n",
    "        caches.append(cache)\n",
    "        \n",
    "    caches = (caches,x)\n",
    "    \n",
    "    return a, y_pred, caches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a[4][1] =  [-0.99999375  0.77911235 -0.99861469 -0.99833267]\n",
      "a.shape =  (5, 10, 4)\n",
      "y_pred[1][3] = [0.79560373 0.86224861 0.11118257 0.81515947]\n",
      "y_pred.shape =  (2, 10, 4)\n",
      "caches[1][1][3] = [-1.1425182  -0.34934272 -0.20889423  0.58662319]\n",
      "len(caches) =  2\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "x = np.random.randn(3,10,4)\n",
    "a0 = np.random.randn(5,10)\n",
    "Waa = np.random.randn(5,5)\n",
    "Wax = np.random.randn(5,3)\n",
    "Wya = np.random.randn(2,5)\n",
    "ba = np.random.randn(5,1)\n",
    "by = np.random.randn(2,1)\n",
    "parameters = {\"Waa\": Waa, \"Wax\": Wax, \"Wya\": Wya, \"ba\": ba, \"by\": by}\n",
    "\n",
    "a, y_pred, caches = rnn_forward(x, a0, parameters)\n",
    "print(\"a[4][1] = \", a[4][1])\n",
    "print(\"a.shape = \", a.shape)\n",
    "print(\"y_pred[1][3] =\", y_pred[1][3])\n",
    "print(\"y_pred.shape = \", y_pred.shape)\n",
    "print(\"caches[1][1][3] =\", caches[1][1][3])\n",
    "print(\"len(caches) = \", len(caches))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. LSTM network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_cell_forward(xt, a_prev, c_prev, parameters):\n",
    "    Wf = parameters[\"Wf\"]\n",
    "    bf = parameters[\"bf\"]\n",
    "    Wi = parameters[\"Wi\"]\n",
    "    bi = parameters[\"bi\"]\n",
    "    Wc = parameters[\"Wc\"]\n",
    "    bc = parameters[\"bc\"]\n",
    "    Wo = parameters[\"Wo\"]\n",
    "    bo = parameters[\"bo\"]\n",
    "    Wy = parameters[\"Wy\"]\n",
    "    by = parameters[\"by\"]\n",
    "    \n",
    "    n_x, m = xt.shape\n",
    "    n_y, n_a = Wy.shape\n",
    "    \n",
    "    concat = np.zeros((n_a + n_x, m))\n",
    "    concat[: n_a, :] = a_prev\n",
    "    concat[n_a :, :] = xt\n",
    "    \n",
    "    ft = sigmoid(np.dot(Wf, concat) + bf)\n",
    "    it = sigmoid(np.dot(Wi, concat) + bi)\n",
    "    cct = np.tanh(np.dot(Wc, concat) + bc)\n",
    "    c_next = ft * c_prev + it * cct \n",
    "    ot = sigmoid(np.dot(Wo, concat) + bo)\n",
    "    a_next = ot * np.tanh(c_next)\n",
    "    \n",
    "    yt_pred = softmax(np.dot(Wy, a_next) + by)\n",
    "    \n",
    "    cache = (a_next, c_next, a_prev, c_prev, ft, it, cct, ot, xt, parameters)\n",
    "\n",
    "    return a_next, c_next, yt_pred, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a_next[4] =  [-0.66408471  0.0036921   0.02088357  0.22834167 -0.85575339  0.00138482\n",
      "  0.76566531  0.34631421 -0.00215674  0.43827275]\n",
      "a_next.shape =  (5, 10)\n",
      "c_next[2] =  [ 0.63267805  1.00570849  0.35504474  0.20690913 -1.64566718  0.11832942\n",
      "  0.76449811 -0.0981561  -0.74348425 -0.26810932]\n",
      "c_next.shape =  (5, 10)\n",
      "yt[1] = [0.79913913 0.15986619 0.22412122 0.15606108 0.97057211 0.31146381\n",
      " 0.00943007 0.12666353 0.39380172 0.07828381]\n",
      "yt.shape =  (2, 10)\n",
      "cache[1][3] = [-0.16263996  1.03729328  0.72938082 -0.54101719  0.02752074 -0.30821874\n",
      "  0.07651101 -1.03752894  1.41219977 -0.37647422]\n",
      "len(cache) =  10\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "xt = np.random.randn(3,10)\n",
    "a_prev = np.random.randn(5,10)\n",
    "c_prev = np.random.randn(5,10)\n",
    "Wf = np.random.randn(5, 5+3)\n",
    "bf = np.random.randn(5,1)\n",
    "Wi = np.random.randn(5, 5+3)\n",
    "bi = np.random.randn(5,1)\n",
    "Wo = np.random.randn(5, 5+3)\n",
    "bo = np.random.randn(5,1)\n",
    "Wc = np.random.randn(5, 5+3)\n",
    "bc = np.random.randn(5,1)\n",
    "Wy = np.random.randn(2,5)\n",
    "by = np.random.randn(2,1)\n",
    "\n",
    "parameters = {\"Wf\": Wf, \"Wi\": Wi, \"Wo\": Wo, \"Wc\": Wc, \"Wy\": Wy, \"bf\": bf, \"bi\": bi, \"bo\": bo, \"bc\": bc, \"by\": by}\n",
    "\n",
    "a_next, c_next, yt, cache = lstm_cell_forward(xt, a_prev, c_prev, parameters)\n",
    "print(\"a_next[4] = \", a_next[4])\n",
    "print(\"a_next.shape = \", c_next.shape)\n",
    "print(\"c_next[2] = \", c_next[2])\n",
    "print(\"c_next.shape = \", c_next.shape)\n",
    "print(\"yt[1] =\", yt[1])\n",
    "print(\"yt.shape = \", yt.shape)\n",
    "print(\"cache[1][3] =\", cache[1][3])\n",
    "print(\"len(cache) = \", len(cache))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_forward(x, a0, parameters):\n",
    "    \n",
    "    caches = []\n",
    "    n_x, m, T_x = x.shape\n",
    "    n_y, n_a = parameters[\"Wy\"].shape\n",
    "    a = np.zeros((n_a, m, T_x))\n",
    "    c = a\n",
    "    y = np.zeros((n_y, m, T_x))\n",
    "    a_next = a0\n",
    "    c_next = np.zeros(a_next.shape)\n",
    "    \n",
    "    for t in range(T_x):\n",
    "        a_next, c_next, yt, cache = lstm_cell_forward(x[:,:,t], a_next, c_next, parameters)\n",
    "        a[:,:,t] = a_next\n",
    "        y[:,:,t] = yt\n",
    "        c[:,:,t]  = c_next\n",
    "        caches.append(cache)\n",
    "        \n",
    "    caches = (caches, x)\n",
    "\n",
    "    return a, y, c, caches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a[4][3][6] =  0.7316245102699646\n",
      "a.shape =  (5, 10, 7)\n",
      "y[1][4][3] = 0.9508734618501101\n",
      "y.shape =  (2, 10, 7)\n",
      "caches[1][1[1]] = [ 0.82797464  0.23009474  0.76201118 -0.22232814 -0.20075807  0.18656139\n",
      "  0.41005165]\n",
      "c[1][2][1] -0.8555449167181981\n",
      "len(caches) =  2\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "x = np.random.randn(3,10,7)\n",
    "a0 = np.random.randn(5,10)\n",
    "Wf = np.random.randn(5, 5+3)\n",
    "bf = np.random.randn(5,1)\n",
    "Wi = np.random.randn(5, 5+3)\n",
    "bi = np.random.randn(5,1)\n",
    "Wo = np.random.randn(5, 5+3)\n",
    "bo = np.random.randn(5,1)\n",
    "Wc = np.random.randn(5, 5+3)\n",
    "bc = np.random.randn(5,1)\n",
    "Wy = np.random.randn(2,5)\n",
    "by = np.random.randn(2,1)\n",
    "\n",
    "parameters = {\"Wf\": Wf, \"Wi\": Wi, \"Wo\": Wo, \"Wc\": Wc, \"Wy\": Wy, \"bf\": bf, \"bi\": bi, \"bo\": bo, \"bc\": bc, \"by\": by}\n",
    "\n",
    "a, y, c, caches = lstm_forward(x, a0, parameters)\n",
    "print(\"a[4][3][6] = \", a[4][3][6])\n",
    "print(\"a.shape = \", a.shape)\n",
    "print(\"y[1][4][3] =\", y[1][4][3])\n",
    "print(\"y.shape = \", y.shape)\n",
    "print(\"caches[1][1[1]] =\", caches[1][1][1])\n",
    "print(\"c[1][2][1]\", c[1][2][1])\n",
    "print(\"len(caches) = \", len(caches))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Backpropagation in RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_cell_backward(da_next, cache):\n",
    "    (a_next, a_prev, xt, parameters) = cache\n",
    "    \n",
    "    Wax = parameters[\"Wax\"]\n",
    "    Waa = parameters[\"Waa\"]\n",
    "    Wya = parameters[\"Wya\"]\n",
    "    ba = parameters[\"ba\"]\n",
    "    by = parameters[\"by\"]\n",
    "\n",
    "    dtanh = (1- a_next**2) * da_next\n",
    "\n",
    "    dxt = np.dot(Wax.T, dtanh)\n",
    "    dWax = np.dot(dtanh, xt.T)\n",
    "\n",
    "    da_prev = np.dot(Waa.T, dtanh)\n",
    "    dWaa = np.dot(dtanh, a_prev.T)\n",
    "\n",
    "    dba = np.sum(dtanh, 1, keepdims=True)\n",
    "\n",
    "    gradients = {\"dxt\": dxt, \"da_prev\": da_prev, \"dWax\": dWax, \"dWaa\": dWaa, \"dba\": dba}\n",
    "    \n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gradients[\"dxt\"][1][2] = -0.4605641030588796\n",
      "gradients[\"dxt\"].shape = (3, 10)\n",
      "gradients[\"da_prev\"][2][3] = 0.08429686538067718\n",
      "gradients[\"da_prev\"].shape = (5, 10)\n",
      "gradients[\"dWax\"][3][1] = 0.3930818739219303\n",
      "gradients[\"dWax\"].shape = (5, 3)\n",
      "gradients[\"dWaa\"][1][2] = -0.2848395578696067\n",
      "gradients[\"dWaa\"].shape = (5, 5)\n",
      "gradients[\"dba\"][4] = [0.80517166]\n",
      "gradients[\"dba\"].shape = (5, 1)\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "xt = np.random.randn(3,10)\n",
    "a_prev = np.random.randn(5,10)\n",
    "Wax = np.random.randn(5,3)\n",
    "Waa = np.random.randn(5,5)\n",
    "Wya = np.random.randn(2,5)\n",
    "b = np.random.randn(5,1)\n",
    "by = np.random.randn(2,1)\n",
    "parameters = {\"Wax\": Wax, \"Waa\": Waa, \"Wya\": Wya, \"ba\": ba, \"by\": by}\n",
    "\n",
    "a_next, yt, cache = rnn_cell_forward(xt, a_prev, parameters)\n",
    "\n",
    "da_next = np.random.randn(5,10)\n",
    "gradients = rnn_cell_backward(da_next, cache)\n",
    "print(\"gradients[\\\"dxt\\\"][1][2] =\", gradients[\"dxt\"][1][2])\n",
    "print(\"gradients[\\\"dxt\\\"].shape =\", gradients[\"dxt\"].shape)\n",
    "print(\"gradients[\\\"da_prev\\\"][2][3] =\", gradients[\"da_prev\"][2][3])\n",
    "print(\"gradients[\\\"da_prev\\\"].shape =\", gradients[\"da_prev\"].shape)\n",
    "print(\"gradients[\\\"dWax\\\"][3][1] =\", gradients[\"dWax\"][3][1])\n",
    "print(\"gradients[\\\"dWax\\\"].shape =\", gradients[\"dWax\"].shape)\n",
    "print(\"gradients[\\\"dWaa\\\"][1][2] =\", gradients[\"dWaa\"][1][2])\n",
    "print(\"gradients[\\\"dWaa\\\"].shape =\", gradients[\"dWaa\"].shape)\n",
    "print(\"gradients[\\\"dba\\\"][4] =\", gradients[\"dba\"][4])\n",
    "print(\"gradients[\\\"dba\\\"].shape =\", gradients[\"dba\"].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn_backward(da, caches):\n",
    "    \n",
    "    (caches, x) = caches\n",
    "    (a1, a0, x1, parameters) = caches[0]\n",
    "    \n",
    "    n_a, m, T_x = da.shape\n",
    "    n_x, m = x1.shape\n",
    "    \n",
    "    dx = np.zeros((n_x, m, T_x))\n",
    "    dWax = np.zeros((n_a, n_x))\n",
    "    dWaa = np.zeros((n_a, n_a))\n",
    "    dba = np.zeros((n_a, 1))\n",
    "    da0 = np.zeros((n_a, m))\n",
    "    da_prevt = np.zeros((n_a, m))\n",
    "    \n",
    "    for t in reversed(range(T_x)):\n",
    "        gradients = rnn_cell_backward(da[:,:, t] + da_prevt, caches[t])\n",
    "        dxt, da_prevt, dWaxt, dWaat, dbat = gradients[\"dxt\"], gradients[\"da_prev\"], gradients[\"dWax\"], gradients[\"dWaa\"], gradients[\"dba\"]\n",
    "        dx[:, :, t] = dxt\n",
    "        dWax += dWaxt\n",
    "        dWaa += dWaat\n",
    "        dba += dbat\n",
    "        \n",
    "    da0 = da_prevt\n",
    "    gradients = {\"dx\": dx, \"da0\": da0, \"dWax\": dWax, \"dWaa\": dWaa,\"dba\": dba}\n",
    "    \n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gradients[\"dx\"][1][2] = [-2.07101689 -0.59255627  0.02466855  0.01483317]\n",
      "gradients[\"dx\"].shape = (3, 10, 4)\n",
      "gradients[\"da0\"][2][3] = -0.31494237512664996\n",
      "gradients[\"da0\"].shape = (5, 10)\n",
      "gradients[\"dWax\"][3][1] = 11.264104496527777\n",
      "gradients[\"dWax\"].shape = (5, 3)\n",
      "gradients[\"dWaa\"][1][2] = 2.303333126579893\n",
      "gradients[\"dWaa\"].shape = (5, 5)\n",
      "gradients[\"dba\"][4] = [-0.74747722]\n",
      "gradients[\"dba\"].shape = (5, 1)\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "x = np.random.randn(3,10,4)\n",
    "a0 = np.random.randn(5,10)\n",
    "Wax = np.random.randn(5,3)\n",
    "Waa = np.random.randn(5,5)\n",
    "Wya = np.random.randn(2,5)\n",
    "ba = np.random.randn(5,1)\n",
    "by = np.random.randn(2,1)\n",
    "parameters = {\"Wax\": Wax, \"Waa\": Waa, \"Wya\": Wya, \"ba\": ba, \"by\": by}\n",
    "a, y, caches = rnn_forward(x, a0, parameters)\n",
    "da = np.random.randn(5, 10, 4)\n",
    "gradients = rnn_backward(da, caches)\n",
    "\n",
    "print(\"gradients[\\\"dx\\\"][1][2] =\", gradients[\"dx\"][1][2])\n",
    "print(\"gradients[\\\"dx\\\"].shape =\", gradients[\"dx\"].shape)\n",
    "print(\"gradients[\\\"da0\\\"][2][3] =\", gradients[\"da0\"][2][3])\n",
    "print(\"gradients[\\\"da0\\\"].shape =\", gradients[\"da0\"].shape)\n",
    "print(\"gradients[\\\"dWax\\\"][3][1] =\", gradients[\"dWax\"][3][1])\n",
    "print(\"gradients[\\\"dWax\\\"].shape =\", gradients[\"dWax\"].shape)\n",
    "print(\"gradients[\\\"dWaa\\\"][1][2] =\", gradients[\"dWaa\"][1][2])\n",
    "print(\"gradients[\\\"dWaa\\\"].shape =\", gradients[\"dWaa\"].shape)\n",
    "print(\"gradients[\\\"dba\\\"][4] =\", gradients[\"dba\"][4])\n",
    "print(\"gradients[\\\"dba\\\"].shape =\", gradients[\"dba\"].shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 LSTM Bachpropagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_cell_backward(da_next, dc_next, cache):\n",
    "    \n",
    "    (a_next, c_next, a_prev, c_prev, ft, it, cct, ot, xt, parameters) = cache\n",
    "    \n",
    "    n_x, m = xt.shape\n",
    "    n_a, m = a_next.shape\n",
    "    \n",
    "    dot = da_next * np.tanh(c_next) * ot * (1 - ot)\n",
    "    dcct = (dc_next * it + ot * (1 - np.square(np.tanh(c_next))) * it * da_next) * (1 - np.square(cct))\n",
    "    dit = (dc_next * cct + ot * (1 - np.square(np.tanh(c_next))) * cct * da_next) * it * (1 - it)\n",
    "    dft = (dc_next * c_prev + ot *(1 - np.square(np.tanh(c_next))) * c_prev * da_next) * ft * (1 - ft)\n",
    "    \n",
    "    concat = np.concatenate((a_prev, xt), axis=0)\n",
    "\n",
    "    dWf = np.dot(dft, concat.T)\n",
    "    dWi = np.dot(dit, concat.T)\n",
    "    dWc = np.dot(dcct, concat.T)\n",
    "    dWo = np.dot(dot, concat.T)\n",
    "    dbf = np.sum(dft, axis=1 ,keepdims = True)\n",
    "    dbi = np.sum(dit, axis=1, keepdims = True)\n",
    "    dbc = np.sum(dcct, axis=1,  keepdims = True)\n",
    "    dbo = np.sum(dot, axis=1, keepdims = True)\n",
    "\n",
    "    da_prev = np.dot(parameters['Wf'][:, :n_a].T, dft) + np.dot(parameters['Wi'][:, :n_a].T, dit) + np.dot(parameters['Wc'][:, :n_a].T, dcct) + np.dot(parameters['Wo'][:, :n_a].T, dot)\n",
    "    dc_prev = dc_next * ft + ot * (1 - np.square(np.tanh(c_next))) * ft * da_next\n",
    "    dxt = np.dot(parameters['Wf'][:, n_a:].T, dft) + np.dot(parameters['Wi'][:, n_a:].T, dit) + np.dot(parameters['Wc'][:, n_a:].T, dcct) + np.dot(parameters['Wo'][:, n_a:].T, dot)\n",
    "    gradients = {\"dxt\": dxt, \"da_prev\": da_prev, \"dc_prev\": dc_prev, \"dWf\": dWf,\"dbf\": dbf, \"dWi\": dWi,\"dbi\": dbi,\n",
    "                \"dWc\": dWc,\"dbc\": dbc, \"dWo\": dWo,\"dbo\": dbo}\n",
    "\n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gradients[\"dxt\"][1][2] = 3.2305591151091884\n",
      "gradients[\"dxt\"].shape = (3, 10)\n",
      "gradients[\"da_prev\"][2][3] = -0.06396214197109241\n",
      "gradients[\"da_prev\"].shape = (5, 10)\n",
      "gradients[\"dc_prev\"][2][3] = 0.7975220387970015\n",
      "gradients[\"dc_prev\"].shape = (5, 10)\n",
      "gradients[\"dWf\"][3][1] = -0.1479548381644968\n",
      "gradients[\"dWf\"].shape = (5, 8)\n",
      "gradients[\"dWi\"][1][2] = 1.0574980552259903\n",
      "gradients[\"dWi\"].shape = (5, 8)\n",
      "gradients[\"dWc\"][3][1] = 2.3045621636876668\n",
      "gradients[\"dWc\"].shape = (5, 8)\n",
      "gradients[\"dWo\"][1][2] = 0.3313115952892109\n",
      "gradients[\"dWo\"].shape = (5, 8)\n",
      "gradients[\"dbf\"][4] = [0.18864637]\n",
      "gradients[\"dbf\"].shape = (5, 1)\n",
      "gradients[\"dbi\"][4] = [-0.40142491]\n",
      "gradients[\"dbi\"].shape = (5, 1)\n",
      "gradients[\"dbc\"][4] = [0.25587763]\n",
      "gradients[\"dbc\"].shape = (5, 1)\n",
      "gradients[\"dbo\"][4] = [0.13893342]\n",
      "gradients[\"dbo\"].shape = (5, 1)\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "xt = np.random.randn(3,10)\n",
    "a_prev = np.random.randn(5,10)\n",
    "c_prev = np.random.randn(5,10)\n",
    "Wf = np.random.randn(5, 5+3)\n",
    "bf = np.random.randn(5,1)\n",
    "Wi = np.random.randn(5, 5+3)\n",
    "bi = np.random.randn(5,1)\n",
    "Wo = np.random.randn(5, 5+3)\n",
    "bo = np.random.randn(5,1)\n",
    "Wc = np.random.randn(5, 5+3)\n",
    "bc = np.random.randn(5,1)\n",
    "Wy = np.random.randn(2,5)\n",
    "by = np.random.randn(2,1)\n",
    "\n",
    "parameters = {\"Wf\": Wf, \"Wi\": Wi, \"Wo\": Wo, \"Wc\": Wc, \"Wy\": Wy, \"bf\": bf, \"bi\": bi, \"bo\": bo, \"bc\": bc, \"by\": by}\n",
    "\n",
    "a_next, c_next, yt, cache = lstm_cell_forward(xt, a_prev, c_prev, parameters)\n",
    "\n",
    "da_next = np.random.randn(5,10)\n",
    "dc_next = np.random.randn(5,10)\n",
    "gradients = lstm_cell_backward(da_next, dc_next, cache)\n",
    "print(\"gradients[\\\"dxt\\\"][1][2] =\", gradients[\"dxt\"][1][2])\n",
    "print(\"gradients[\\\"dxt\\\"].shape =\", gradients[\"dxt\"].shape)\n",
    "print(\"gradients[\\\"da_prev\\\"][2][3] =\", gradients[\"da_prev\"][2][3])\n",
    "print(\"gradients[\\\"da_prev\\\"].shape =\", gradients[\"da_prev\"].shape)\n",
    "print(\"gradients[\\\"dc_prev\\\"][2][3] =\", gradients[\"dc_prev\"][2][3])\n",
    "print(\"gradients[\\\"dc_prev\\\"].shape =\", gradients[\"dc_prev\"].shape)\n",
    "print(\"gradients[\\\"dWf\\\"][3][1] =\", gradients[\"dWf\"][3][1])\n",
    "print(\"gradients[\\\"dWf\\\"].shape =\", gradients[\"dWf\"].shape)\n",
    "print(\"gradients[\\\"dWi\\\"][1][2] =\", gradients[\"dWi\"][1][2])\n",
    "print(\"gradients[\\\"dWi\\\"].shape =\", gradients[\"dWi\"].shape)\n",
    "print(\"gradients[\\\"dWc\\\"][3][1] =\", gradients[\"dWc\"][3][1])\n",
    "print(\"gradients[\\\"dWc\\\"].shape =\", gradients[\"dWc\"].shape)\n",
    "print(\"gradients[\\\"dWo\\\"][1][2] =\", gradients[\"dWo\"][1][2])\n",
    "print(\"gradients[\\\"dWo\\\"].shape =\", gradients[\"dWo\"].shape)\n",
    "print(\"gradients[\\\"dbf\\\"][4] =\", gradients[\"dbf\"][4])\n",
    "print(\"gradients[\\\"dbf\\\"].shape =\", gradients[\"dbf\"].shape)\n",
    "print(\"gradients[\\\"dbi\\\"][4] =\", gradients[\"dbi\"][4])\n",
    "print(\"gradients[\\\"dbi\\\"].shape =\", gradients[\"dbi\"].shape)\n",
    "print(\"gradients[\\\"dbc\\\"][4] =\", gradients[\"dbc\"][4])\n",
    "print(\"gradients[\\\"dbc\\\"].shape =\", gradients[\"dbc\"].shape)\n",
    "print(\"gradients[\\\"dbo\\\"][4] =\", gradients[\"dbo\"][4])\n",
    "print(\"gradients[\\\"dbo\\\"].shape =\", gradients[\"dbo\"].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lstm_backward(da, caches):\n",
    "    \n",
    "    \n",
    "    (caches, x) = caches\n",
    "    (a1, c1, a0, c0, f1, i1, cc1, o1, x1, parameters) = caches[0]\n",
    "    \n",
    "    n_a, m, T_x = da.shape\n",
    "    n_x, m = x1.shape\n",
    "    \n",
    "    dx = np.zeros((n_x, m, T_x))\n",
    "    da0 = np.zeros((n_a, m))\n",
    "    da_prevt = np.zeros(da0.shape)\n",
    "    dc_prevt = np.zeros(da0.shape)\n",
    "    dWf = np.zeros((n_a, n_a + n_x))\n",
    "    dWi = np.zeros(dWf.shape)\n",
    "    dWc = np.zeros(dWf.shape)\n",
    "    dWo = np.zeros(dWf.shape)\n",
    "    dbf = np.zeros((n_a, 1))\n",
    "    dbi = np.zeros(dbf.shape)\n",
    "    dbc = np.zeros(dbf.shape)\n",
    "    dbo = np.zeros(dbf.shape)\n",
    "    \n",
    "    for t in reversed(range(T_x)):\n",
    "        gradients = lstm_cell_backward(da[:, :, t], dc_prevt, caches[t])\n",
    "        dx[:,:,t] = gradients[\"dxt\"]\n",
    "        dWf += gradients[\"dWf\"]\n",
    "        dWi += gradients[\"dWi\"]\n",
    "        dWc += gradients[\"dWc\"]\n",
    "        dWo += gradients[\"dWo\"]\n",
    "        dbf += gradients[\"dbf\"]\n",
    "        dbi += gradients[\"dbi\"]\n",
    "        dbc += gradients[\"dbc\"]\n",
    "        dbo += gradients[\"dbo\"]\n",
    "    da0 = gradients[\"da_prev\"]\n",
    "    \n",
    "    gradients = {\"dx\": dx, \"da0\": da0, \"dWf\": dWf,\"dbf\": dbf, \"dWi\": dWi,\"dbi\": dbi,\n",
    "                \"dWc\": dWc,\"dbc\": dbc, \"dWo\": dWo,\"dbo\": dbo}\n",
    "    \n",
    "    return gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gradients[\"dx\"][1][2] = [-0.00173313  0.08287442 -0.30545663 -0.43281115]\n",
      "gradients[\"dx\"].shape = (3, 10, 4)\n",
      "gradients[\"da0\"][2][3] = -0.09591150195400465\n",
      "gradients[\"da0\"].shape = (5, 10)\n",
      "gradients[\"dWf\"][3][1] = -0.06981985612744009\n",
      "gradients[\"dWf\"].shape = (5, 8)\n",
      "gradients[\"dWi\"][1][2] = 0.10237182024854771\n",
      "gradients[\"dWi\"].shape = (5, 8)\n",
      "gradients[\"dWc\"][3][1] = -0.062498379492745226\n",
      "gradients[\"dWc\"].shape = (5, 8)\n",
      "gradients[\"dWo\"][1][2] = 0.04843891314443013\n",
      "gradients[\"dWo\"].shape = (5, 8)\n",
      "gradients[\"dbf\"][4] = [-0.0565788]\n",
      "gradients[\"dbf\"].shape = (5, 1)\n",
      "gradients[\"dbi\"][4] = [-0.15399065]\n",
      "gradients[\"dbi\"].shape = (5, 1)\n",
      "gradients[\"dbc\"][4] = [-0.29691142]\n",
      "gradients[\"dbc\"].shape = (5, 1)\n",
      "gradients[\"dbo\"][4] = [-0.29798344]\n",
      "gradients[\"dbo\"].shape = (5, 1)\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "x = np.random.randn(3,10,7)\n",
    "a0 = np.random.randn(5,10)\n",
    "Wf = np.random.randn(5, 5+3)\n",
    "bf = np.random.randn(5,1)\n",
    "Wi = np.random.randn(5, 5+3)\n",
    "bi = np.random.randn(5,1)\n",
    "Wo = np.random.randn(5, 5+3)\n",
    "bo = np.random.randn(5,1)\n",
    "Wc = np.random.randn(5, 5+3)\n",
    "bc = np.random.randn(5,1)\n",
    "\n",
    "parameters = {\"Wf\": Wf, \"Wi\": Wi, \"Wo\": Wo, \"Wc\": Wc, \"Wy\": Wy, \"bf\": bf, \"bi\": bi, \"bo\": bo, \"bc\": bc, \"by\": by}\n",
    "\n",
    "a, y, c, caches = lstm_forward(x, a0, parameters)\n",
    "\n",
    "da = np.random.randn(5, 10, 4)\n",
    "gradients = lstm_backward(da, caches)\n",
    "\n",
    "print(\"gradients[\\\"dx\\\"][1][2] =\", gradients[\"dx\"][1][2])\n",
    "print(\"gradients[\\\"dx\\\"].shape =\", gradients[\"dx\"].shape)\n",
    "print(\"gradients[\\\"da0\\\"][2][3] =\", gradients[\"da0\"][2][3])\n",
    "print(\"gradients[\\\"da0\\\"].shape =\", gradients[\"da0\"].shape)\n",
    "print(\"gradients[\\\"dWf\\\"][3][1] =\", gradients[\"dWf\"][3][1])\n",
    "print(\"gradients[\\\"dWf\\\"].shape =\", gradients[\"dWf\"].shape)\n",
    "print(\"gradients[\\\"dWi\\\"][1][2] =\", gradients[\"dWi\"][1][2])\n",
    "print(\"gradients[\\\"dWi\\\"].shape =\", gradients[\"dWi\"].shape)\n",
    "print(\"gradients[\\\"dWc\\\"][3][1] =\", gradients[\"dWc\"][3][1])\n",
    "print(\"gradients[\\\"dWc\\\"].shape =\", gradients[\"dWc\"].shape)\n",
    "print(\"gradients[\\\"dWo\\\"][1][2] =\", gradients[\"dWo\"][1][2])\n",
    "print(\"gradients[\\\"dWo\\\"].shape =\", gradients[\"dWo\"].shape)\n",
    "print(\"gradients[\\\"dbf\\\"][4] =\", gradients[\"dbf\"][4])\n",
    "print(\"gradients[\\\"dbf\\\"].shape =\", gradients[\"dbf\"].shape)\n",
    "print(\"gradients[\\\"dbi\\\"][4] =\", gradients[\"dbi\"][4])\n",
    "print(\"gradients[\\\"dbi\\\"].shape =\", gradients[\"dbi\"].shape)\n",
    "print(\"gradients[\\\"dbc\\\"][4] =\", gradients[\"dbc\"][4])\n",
    "print(\"gradients[\\\"dbc\\\"].shape =\", gradients[\"dbc\"].shape)\n",
    "print(\"gradients[\\\"dbo\\\"][4] =\", gradients[\"dbo\"][4])\n",
    "print(\"gradients[\\\"dbo\\\"].shape =\", gradients[\"dbo\"].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}