END_P2S3_10_10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "END P2S3_10_10.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jofyc9OC4Qcf"
      },
      "source": [
        "#Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ahBVnrNc3E0U"
      },
      "source": [
        "import numpy as np\n",
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from IPython import display\n",
        "plt.style.use('seaborn-white')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "crQSAaIz4SkA"
      },
      "source": [
        "# Read and process data. \n",
        "\n",
        "Download the file from this URL: https://drive.google.com/file/d/1UWWIi-sz9g0x3LFvkIZjvK1r2ZaCqgGS/view?usp=sharing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pjG_Gyd6JnU3",
        "outputId": "0b7c2383-4ace-4676-bb99-2ce1e01d7960"
      },
      "source": [
        "!gdown --id 1JBVfScTakEGsfZjLDFPtKx-uVa9n1TYb"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1JBVfScTakEGsfZjLDFPtKx-uVa9n1TYb\n",
            "To: /content/text.txt\n",
            "\r  0% 0.00/10.3k [00:00<?, ?B/s]\r100% 10.3k/10.3k [00:00<00:00, 22.0MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rgOGxPDP3Wpp"
      },
      "source": [
        "data = open('text.txt', 'r').read()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZeXXMLRb4kXb"
      },
      "source": [
        "Process data and calculate indices"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5TKeiOp4jtl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c11eec0d-f20f-4df4-9d4c-a012c51a0ed3"
      },
      "source": [
        "chars = list(set(data))\n",
        "data_size, X_size = len(data), len(chars)\n",
        "print(\"Corona Virus article has %d characters, %d unique characters\" %(data_size, X_size))\n",
        "char_to_idx = {ch:i for i,ch in enumerate(chars)}\n",
        "idx_to_char = {i:ch for i,ch in enumerate(chars)}"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Corona Virus article has 10223 characters, 75 unique characters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4C53MB135LRY"
      },
      "source": [
        "# Constants and Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dfj21ORa49Ps"
      },
      "source": [
        "Hidden_Layer_size = 10 #size of the hidden layer\n",
        "Time_steps = 10 # Number of time steps (length of the sequence) used for training\n",
        "learning_rate = 1e-1 # Learning Rate\n",
        "weight_sd = 0.1 #Standard deviation of weights for initialization\n",
        "z_size = Hidden_Layer_size + X_size #Size of concatenation(H, X) vector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdmJf4Du5uhb"
      },
      "source": [
        "# Activation Functions and Derivatives"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "seGHei_D5FGk"
      },
      "source": [
        "def sigmoid(x): # sigmoid function\n",
        "  return 1.0 / (1.0 + np.exp(-x)) # write your code here\n",
        "\n",
        "def dsigmoid(y): # derivative of sigmoid function\n",
        "  return y * (1.0 - y)     # write your code here\n",
        "\n",
        "def tanh(x): # tanh function\n",
        "  return np.tanh(x) # write your code here\n",
        "\n",
        "def dtanh(y): # derivative of tanh\n",
        "  return 1.0 - (y * y) # write your code here"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KeCvVH1v6Me-"
      },
      "source": [
        "# Quiz Question 1\n",
        "\n",
        "What is the value of sigmoid(0) calculated from  your code? (Answer up to 1 decimal point, e.g. 4.2 and NOT 4.29999999, no rounding off).\n",
        "\n",
        "# Quiz Question 2\n",
        "\n",
        "What is the value of dsigmoid(sigmoid(0)) calculated from your code?? (Answer up to 2 decimal point, e.g. 4.29 and NOT 4.29999999, no rounding off). \n",
        "\n",
        "# Quiz Question 3\n",
        "\n",
        "What is the value of tanh(dsigmoid(sigmoid(0))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off).\n",
        "\n",
        "# Quiz Question 4\n",
        "\n",
        "What is the value of dtanh(tanh(dsigmoid(sigmoid(0)))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Iuc7hFzqLeQq",
        "outputId": "5e0c9cb7-6240-4635-9a05-08737965ce07"
      },
      "source": [
        "# Quiz Question 1\n",
        "sigmoid(0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sGqN976PLeFg",
        "outputId": "01defabe-530e-44f8-9f05-29ec65d041e3"
      },
      "source": [
        "# Quiz Question 2\n",
        "dsigmoid(sigmoid(0))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.25"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O68gJ6VHLd98",
        "outputId": "8cfd569d-681b-4bb1-f5ee-ddc92b27006d"
      },
      "source": [
        "# Quiz Question 3\n",
        "tanh(dsigmoid(sigmoid(0))) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.24491866240370913"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ECRIOdNHLdyK",
        "outputId": "ad85958c-69a1-4f4d-81b3-83f1b2f18b3c"
      },
      "source": [
        "# Quiz Question 4\n",
        "dtanh(tanh(dsigmoid(sigmoid(0))))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.940014848806378"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EeSVipDu8iKE"
      },
      "source": [
        "# Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ICbWNemE6LGV"
      },
      "source": [
        "class Param:\n",
        "    def __init__(self, name, value):\n",
        "      self.name = name\n",
        "      self.v = value # parameter value\n",
        "      self.d = np.zeros_like(value) # derivative\n",
        "      self.m = np.zeros_like(value) # momentum for Adagrad"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j83pZNPE8212"
      },
      "source": [
        "We use random weights with normal distribution (0, weight_sd) for  tanh  activation function and (0.5, weight_sd) for  `sigmoid`  activation function.\n",
        "\n",
        "Biases are initialized to zeros."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swHwLXOI9E7V"
      },
      "source": [
        "# LSTM \n",
        "You are making this network, please note f, i, c and o (also \"v\") in the image below:\n",
        "![alt text](http://blog.varunajayasiri.com/ml/lstm.svg)\n",
        "\n",
        "Please note that we are concatenating the old_hidden_vector and new_input."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0DBzNY-90s5"
      },
      "source": [
        "# Quiz Question 4\n",
        "\n",
        "In the class definition below, what should be size_a, size_b, and size_c? ONLY use the variables defined above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFuHhqVq6Wge"
      },
      "source": [
        "size_a = Hidden_Layer_size # write your code here\n",
        "size_b = z_size # write your code here\n",
        "size_c = X_size # write your code here\n",
        "\n",
        "class Parameters:\n",
        "    def __init__(self):\n",
        "        self.W_f = Param('W_f', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_f = Param('b_f', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_i = Param('W_i', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_i = Param('b_i', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_C = Param('W_C', np.random.randn(size_a, size_b) * weight_sd)\n",
        "        self.b_C = Param('b_C', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_o = Param('W_o', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_o = Param('b_o', np.zeros((size_a, 1)))\n",
        "\n",
        "        #For final layer to predict the next character\n",
        "        self.W_v = Param('W_v', np.random.randn(X_size, size_a) * weight_sd)\n",
        "        self.b_v = Param('b_v', np.zeros((size_c, 1)))\n",
        "        \n",
        "    def all(self):\n",
        "        return [self.W_f, self.W_i, self.W_C, self.W_o, self.W_v,\n",
        "               self.b_f, self.b_i, self.b_C, self.b_o, self.b_v]\n",
        "        \n",
        "parameters = Parameters()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RzmfGLZt_xVs"
      },
      "source": [
        "Look at these operations which we'll be writing:\n",
        "\n",
        "**Concatenation of h and x:**\n",
        "\n",
        "$z\\:=\\:\\left[h_{t-1},\\:x\\right]$\n",
        "\n",
        "$f_t=\\sigma\\left(W_f\\cdot z\\:+\\:b_f\\:\\right)$\n",
        "\n",
        "$i_i=\\sigma\\left(W_i\\cdot z\\:+\\:b_i\\right)$\n",
        "\n",
        "$\\overline{C_t}=\\tanh\\left(W_C\\cdot z\\:+\\:b_C\\right)$\n",
        "\n",
        "$C_t=f_t\\ast C_{t-1}+i_t\\ast \\overline{C}_t$\n",
        "\n",
        "$o_t=\\sigma\\left(W_o\\cdot z\\:+\\:b_i\\right)$\n",
        "\n",
        "$h_t=o_t\\ast\\tanh\\left(C_t\\right)$\n",
        "\n",
        "**Logits:**\n",
        "\n",
        "$v_t=W_v\\cdot h_t+b_v$\n",
        "\n",
        "**Softmax:**\n",
        "\n",
        "$\\hat{y}=softmax\\left(v_t\\right)$\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-bUkseNnDott"
      },
      "source": [
        "def forward(x, h_prev, C_prev, p = parameters):\n",
        "    assert x.shape == (X_size, 1)\n",
        "    assert h_prev.shape == (Hidden_Layer_size, 1)\n",
        "    assert C_prev.shape == (Hidden_Layer_size, 1)\n",
        "    \n",
        "    z = np.row_stack((h_prev, x))\n",
        "    f = sigmoid(np.dot(p.W_f.v, z) + p.b_f.v) # write your code here\n",
        "    i = sigmoid(np.dot(p.W_i.v, z) + p.b_i.v) # write your code here\n",
        "    C_bar = tanh(np.dot(p.W_C.v, z) + p.b_C.v) # write your code here\n",
        "\n",
        "    C = f * C_prev + i * C_bar # write your code here\n",
        "    o = sigmoid(np.dot(p.W_o.v, z) + p.b_o.v) # write your code here\n",
        "    h = o * tanh(C) # write your code here\n",
        "\n",
        "    v = np.dot(p.W_v.v, h) + p.b_v.v # write your code here\n",
        "    y = np.exp(v) / np.sum(np.exp(v)) #softmax\n",
        "\n",
        "    return z, f, i, C_bar, C, o, h, v, y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jZrDhZIjFpdI"
      },
      "source": [
        "You must finish the function above before you can attempt the questions below. \n",
        "\n",
        "# Quiz Question 5\n",
        "\n",
        "What is the output of 'print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))'?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aUJxybI9TSYX",
        "outputId": "6210216b-d623-43cc-d818-486ee37faa77"
      },
      "source": [
        "# Quiz Question 5\n",
        "print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "9\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XV-YVl_GGiX8"
      },
      "source": [
        "# Quiz Question 6. \n",
        "\n",
        "Assuming you have fixed the forward function, run this command: \n",
        "z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))\n",
        "\n",
        "Now, find these values:\n",
        "\n",
        "\n",
        "1.   print(z.shape)\n",
        "2.   print(np.sum(z))\n",
        "3.   print(np.sum(f))\n",
        "\n",
        "Copy and paste exact values you get in the logs into the quiz.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1GvKVWmTDt3H"
      },
      "source": [
        "z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mmsoAG4xTnve",
        "outputId": "a1d13126-56d0-4ee7-d0e8-674b76f67257"
      },
      "source": [
        "# Quiz Question 6\n",
        "print(z.shape)\n",
        "print(np.sum(z))\n",
        "print(np.sum(f))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(85, 1)\n",
            "0.0\n",
            "5.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NeSvhkqwILsG"
      },
      "source": [
        "# Backpropagation\n",
        "\n",
        "Here we are defining the backpropagation. It's too complicated, here is the whole code. (Please note that this would work only if your earlier code is perfect)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zIa1jUZiGPmF"
      },
      "source": [
        "def backward(target, dh_next, dC_next, C_prev,\n",
        "             z, f, i, C_bar, C, o, h, v, y,\n",
        "             p = parameters):\n",
        "    \n",
        "    assert z.shape == (X_size + Hidden_Layer_size, 1)\n",
        "    assert v.shape == (X_size, 1)\n",
        "    assert y.shape == (X_size, 1)\n",
        "    \n",
        "    for param in [dh_next, dC_next, C_prev, f, i, C_bar, C, o, h]:\n",
        "        assert param.shape == (Hidden_Layer_size, 1)\n",
        "        \n",
        "    dv = np.copy(y)\n",
        "    dv[target] -= 1\n",
        "\n",
        "    p.W_v.d += np.dot(dv, h.T)\n",
        "    p.b_v.d += dv\n",
        "\n",
        "    dh = np.dot(p.W_v.v.T, dv)        \n",
        "    dh += dh_next\n",
        "    do = dh * tanh(C)\n",
        "    do = dsigmoid(o) * do\n",
        "    p.W_o.d += np.dot(do, z.T)\n",
        "    p.b_o.d += do\n",
        "\n",
        "    dC = np.copy(dC_next)\n",
        "    dC += dh * o * dtanh(tanh(C))\n",
        "    dC_bar = dC * i\n",
        "    dC_bar = dtanh(C_bar) * dC_bar\n",
        "    p.W_C.d += np.dot(dC_bar, z.T)\n",
        "    p.b_C.d += dC_bar\n",
        "\n",
        "    di = dC * C_bar\n",
        "    di = dsigmoid(i) * di\n",
        "    p.W_i.d += np.dot(di, z.T)\n",
        "    p.b_i.d += di\n",
        "\n",
        "    df = dC * C_prev\n",
        "    df = dsigmoid(f) * df\n",
        "    p.W_f.d += np.dot(df, z.T)\n",
        "    p.b_f.d += df\n",
        "\n",
        "    dz = (np.dot(p.W_f.v.T, df)\n",
        "         + np.dot(p.W_i.v.T, di)\n",
        "         + np.dot(p.W_C.v.T, dC_bar)\n",
        "         + np.dot(p.W_o.v.T, do))\n",
        "    dh_prev = dz[:Hidden_Layer_size, :]\n",
        "    dC_prev = f * dC\n",
        "    \n",
        "    return dh_prev, dC_prev"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tnc7WpRkIU5S"
      },
      "source": [
        "# Forward and Backward Combined Pass\n",
        "\n",
        "Let's first clear the gradients before each backward pass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJWoC3U1ITf8"
      },
      "source": [
        "def clear_gradients(params = parameters):\n",
        "    for p in params.all():\n",
        "        p.d.fill(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7XN93UnjIgmA"
      },
      "source": [
        "Clip gradients to mitigate exploding gradients"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0LTsublxIfFl"
      },
      "source": [
        "def clip_gradients(params = parameters):\n",
        "    for p in params.all():\n",
        "        np.clip(p.d, -1, 1, out=p.d)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T7XUpDTWIl_Y"
      },
      "source": [
        "Calculate and store the values in forward pass. Accumulate gradients in backward pass and clip gradients to avoid exploding gradients.\n",
        "\n",
        "input, target are list of integers, with character indexes.\n",
        "h_prev is the array of initial h at  h−1  (size H x 1)\n",
        "C_prev is the array of initial C at  C−1  (size H x 1)\n",
        "Returns loss, final  hT  and  CT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQNxjTuZIia_"
      },
      "source": [
        "def forward_backward(inputs, targets, h_prev, C_prev):\n",
        "    global paramters\n",
        "    \n",
        "    # To store the values for each time step\n",
        "    x_s, z_s, f_s, i_s,  = {}, {}, {}, {}\n",
        "    C_bar_s, C_s, o_s, h_s = {}, {}, {}, {}\n",
        "    v_s, y_s =  {}, {}\n",
        "    \n",
        "    # Values at t - 1\n",
        "    h_s[-1] = np.copy(h_prev)\n",
        "    C_s[-1] = np.copy(C_prev)\n",
        "    \n",
        "    loss = 0\n",
        "    # Loop through time steps\n",
        "    assert len(inputs) == Time_steps\n",
        "    for t in range(len(inputs)):\n",
        "        x_s[t] = np.zeros((X_size, 1))\n",
        "        x_s[t][inputs[t]] = 1 # Input character\n",
        "        \n",
        "        (z_s[t], f_s[t], i_s[t],\n",
        "        C_bar_s[t], C_s[t], o_s[t], h_s[t],\n",
        "        v_s[t], y_s[t]) = \\\n",
        "            forward(x_s[t], h_s[t - 1], C_s[t - 1]) # Forward pass\n",
        "            \n",
        "        loss += -np.log(y_s[t][targets[t], 0]) # Loss for at t\n",
        "        \n",
        "    clear_gradients()\n",
        "\n",
        "    dh_next = np.zeros_like(h_s[0]) #dh from the next character\n",
        "    dC_next = np.zeros_like(C_s[0]) #dh from the next character\n",
        "\n",
        "    for t in reversed(range(len(inputs))):\n",
        "        # Backward pass\n",
        "        dh_next, dC_next = \\\n",
        "            backward(target = targets[t], dh_next = dh_next,\n",
        "                     dC_next = dC_next, C_prev = C_s[t-1],\n",
        "                     z = z_s[t], f = f_s[t], i = i_s[t], C_bar = C_bar_s[t],\n",
        "                     C = C_s[t], o = o_s[t], h = h_s[t], v = v_s[t],\n",
        "                     y = y_s[t])\n",
        "\n",
        "    clip_gradients()\n",
        "        \n",
        "    return loss, h_s[len(inputs) - 1], C_s[len(inputs) - 1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tcy5u_vRItkV"
      },
      "source": [
        "# Sample the next character"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p8SrtJiwIsSm"
      },
      "source": [
        "def sample(h_prev, C_prev, first_char_idx, sentence_length):\n",
        "    x = np.zeros((X_size, 1))\n",
        "    x[first_char_idx] = 1\n",
        "\n",
        "    h = h_prev\n",
        "    C = C_prev\n",
        "\n",
        "    indexes = []\n",
        "    \n",
        "    for t in range(sentence_length):\n",
        "        _, _, _, _, C, _, h, _, p = forward(x, h, C)\n",
        "        idx = np.random.choice(range(X_size), p=p.ravel())\n",
        "        x = np.zeros((X_size, 1))\n",
        "        x[idx] = 1\n",
        "        indexes.append(idx)\n",
        "\n",
        "    return indexes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SiWFaWLNIx_L"
      },
      "source": [
        "# Training (Adagrad)\n",
        "\n",
        "Update the graph and display a sample output\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ENQYU-7AIw0t"
      },
      "source": [
        "def update_status(inputs, h_prev, C_prev):\n",
        "    #initialized later\n",
        "    global plot_iter, plot_loss\n",
        "    global smooth_loss\n",
        "    \n",
        "    # Get predictions for 200 letters with current model\n",
        "\n",
        "    sample_idx = sample(h_prev, C_prev, inputs[0], 200)\n",
        "    txt = ''.join(idx_to_char[idx] for idx in sample_idx)\n",
        "\n",
        "    # Clear and plot\n",
        "    plt.plot(plot_iter, plot_loss)\n",
        "    display.clear_output(wait=True)\n",
        "    plt.show()\n",
        "\n",
        "    #Print prediction and loss\n",
        "    print(\"----\\n %s \\n----\" % (txt, ))\n",
        "    print(\"iter %d, loss %f\" % (iteration, smooth_loss))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ACXcASJuI73a"
      },
      "source": [
        "# Update Parameters\n",
        "\n",
        "\\begin{align}\n",
        "\\theta_i &= \\theta_i - \\eta\\frac{d\\theta_i}{\\sum dw_{\\tau}^2} \\\\\n",
        "d\\theta_i &= \\frac{\\partial L}{\\partial \\theta_i}\n",
        "\\end{align}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bR08TvcjI4Pf"
      },
      "source": [
        "def update_paramters(params = parameters):\n",
        "    for p in params.all():\n",
        "        p.m += p.d * p.d # Calculate sum of gradients\n",
        "        #print(learning_rate * dparam)\n",
        "        p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "La9vyJ6RJLFK"
      },
      "source": [
        "To delay the keyboard interrupt to prevent the training from stopping in the middle of an iteration\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZVDHbMb7JNGT"
      },
      "source": [
        "# Exponential average of loss\n",
        "# Initialize to a error of a random model\n",
        "smooth_loss = -np.log(1.0 / X_size) * Time_steps\n",
        "\n",
        "iteration, pointer = 0, 0\n",
        "\n",
        "# For the graph\n",
        "plot_iter = np.zeros((0))\n",
        "plot_loss = np.zeros((0))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HF6vS0VWJqsS"
      },
      "source": [
        "# Training Loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQyNSL0iJOxH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 364
        },
        "outputId": "f6934ad9-6164-4d40-9dce-cc7d71db05dd"
      },
      "source": [
        "iter = 50000\n",
        "while iter > 0:\n",
        "  # Reset\n",
        "  if pointer + Time_steps >= len(data) or iteration == 0:\n",
        "      g_h_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      g_C_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      pointer = 0\n",
        "\n",
        "\n",
        "  inputs = ([char_to_idx[ch] \n",
        "              for ch in data[pointer: pointer + Time_steps]])\n",
        "  targets = ([char_to_idx[ch] \n",
        "              for ch in data[pointer + 1: pointer + Time_steps + 1]])\n",
        "\n",
        "  loss, g_h_prev, g_C_prev = \\\n",
        "      forward_backward(inputs, targets, g_h_prev, g_C_prev)\n",
        "  smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
        "\n",
        "  # Print every hundred steps\n",
        "  if iteration % 100 == 0:\n",
        "      update_status(inputs, g_h_prev, g_C_prev)\n",
        "\n",
        "  update_paramters()\n",
        "\n",
        "  plot_iter = np.append(plot_iter, [iteration])\n",
        "  plot_loss = np.append(plot_loss, [loss])\n",
        "\n",
        "  pointer += Time_steps\n",
        "  iteration += 1\n",
        "  iter = iter -1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAD1CAYAAACWXdT/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5hU1f3H8fcWWHpHFgEp4h5EikpHEBAUFNFEQKJYsceWaFTysyE2gpqYGGPsiiXGkkQRBQFRUYpgF/QqZZHe21J2YXd+f0xhZu697OzuzM7c2c/reXycPTNz59xl53vP/Z6W4fP5EBGR1JaZ7AqIiEjpFKxFRDxAwVpExAMUrEVEPEDBWkTEA7ITcVBjTA7QA1gPFCfiM0RE0lAW0BxYZFlWYfgTCQnW+AP13AQdW0Qk3fUHPg0vSFSwXg/wyiuvkJubm6CPEBFJLxs2bGDs2LEQiKHhEhWsiwFyc3Np2bJlgj5CRCRt2dLH6mAUEfEABWsREQ9QsBYR8QAFaxERD1CwFhHxAAVrEREPSPlg3faP03h4hpXsaoiIJFXKB2ufD/4+Z1myqyEiklQpH6xFRETBWkTEExSsRUQ8QMFaRMQDFKxFRDxAwVpExAMUrEVEPEDBWkTEAzwbrKd+s47XF69OdjVERCpFonaKSbjr//UVAOd2b5XkmoiIJJ5nW9YiIlWJgrWIiAcoWIuIeICCtYiIByhYi4h4gIK1iIgHlDp0zxgzEHgDWBIo+g6YDLwEZAHrgQstyypMUB3LZOe+AyzbVEC31g2TXRURkbiJtWX9sWVZAwP/XQ9MBB63LKs/sAwYl7AaltGlz3/OyCfmcaC4JNlVERGJm/KmQQYC7wQeTwWGxKU2cfD92l0AlPh8Sa6JiEj8xDqDsaMx5h2gEXAPUDss7bEJaJ6IyomIiF8swfpn/AH6daAdMCfqfRkJqJeIiIQpNQ1iWdZay7L+bVmWz7Ks5cAGoKExpmbgJS2AdfGs1KylG5VzFhEJU2qwNsaMNcb8IfA4F2gGPA+MDLxkJDA9XhX6+KfNXD5lMY/N/jlehwSg6GAJT368nKKDugiIiPfEkgZ5B3jVGHM2UB24BvgKmGKMuQpYBbwYrwpt2e1Pha/Zvi9ehwTghXkrefD9H8nMyOCKk9vF9dgiIolWarC2LGs3MMLhqVPjX53EKSgsBmBP0cEk10REpOxSbgajBtyJiNilXLAO0RgTEZGQ1A3WFRTrnJjiEh9txk/jb3Hu0BQRiae0DdaxCg4RfHzOsiTXRETEXcoFa1+cpolnKI0iImkk5YJ1UEaCktZluRbMXLqRT3/ekpB6iIiURcrubh6vFnZQeUL/FVMWA5A/aXhc6yIiUlYp17IOxuj/fLU2uRUREUkhKResvbC06Yad+5m3XOkREak8KResvWDoo59w/tMLk10NEalCFKzLYee+A8mugohUMSkXrJeu35XsKpTb/gPFvLRgVdw7R0VEUm40yJT5q+JynFjjZTzj6uTpFs99tpKmdXIY1ik3fgcWkSov5VrW4d7/bn3cj5nINu/2vUUA7NXKfiISZykdrK955ctyvzd6BqPbjMbKmOm4r6iYjndN54MlGxL/YSKSllI6WKeL1dv3sreomIdmWMmuioh4lIJ1kv2ydS9bCgpLf6GIVGkp18FY1Zz80BwyM2DFg5rSLiLu1LJOASUa6ScipVCwrkRliclXv/QFbcZPS1hdRMRbPBOs9x8oZk9hHIbEJWHCSnkGnEzXyBERCeOZYN130occd/eMcr8/UetjO0nk9eDdb9fRZvw0tu0pStyHiEjK8UywDganx2LcKzEZMxgrw/Of5QOwYnOB7blte4o01V0kTXkmWAc9MvOnw78gBbbzqoyJNtEhedmmAk68d2bcpuuLSGrxXLAuVRkblpW5V2M8Wr1u1c3fsgeAT37abHtuxpINLNu0u8KfLSLJk7bjrFNpw9xE1KUscf+ql74AtD2ZiJelX8s6zcUz8I96Yh6dKtBpKyKVp8oF63TpfotHSmXxqu0UOAyHXLG5gEc+sNRZKZJCqkywToW0SDxCX2UMQbzouc957MNlbNwVuWaJz+fjk582U6IplyKVrsoE6+SKf4BNZLg8UFziWD5jyUYueu5znvtspe25gsKDHHR5n4hUnIK115QS9xMZxDfs3AfA6m17bc91unsGv3/9mwR+ukjVpmCdAMlI9aZCmmfqN+tsZec/vYDfPDU/CbURSS9pG6zTdQZjkFfqPW/5Vhas2GYrn/btesY8qSAuEqv0C9blbGHGs2Xqeqw4BFjXQ3skeAdd++qXLFxpD+JbCgq5Yspidu0/kIRaiaSumCbFGGNqAt8D9wKzgZeALGA9cKFlWZ7f6iSRwS4hk2JcIn8KZEMq5B9zljNz6UZeX7Say/u3C5X7fD7un/YDo7q3pENuvSTWUCQ5Ym1Z3wEEm0ETgccty+oPLAPGJaJih7N2xz73J0sJutFBORVyvWXhtfrGy5aCIp75dCUXPLPQ9twrC1fx/nfrk1ArkcpTarA2xnQAOgLBlfAHAu8EHk8FhiSkZodx0qQPy/yetItxlZD2cGu9J8Ohutj/JW//7/dc88qXtvKPf9rMQzN+THDNRCpHLC3rR4Cbwn6uHZb22AQ0j3ut0pRXJsWki4uf+5zH5yy3lS/bVMCoJ+Y5zt4USVWHDdbGmIuA+ZZl2WdB+ClyxCARvyS3wB/PtnC6Xhj+NP1HFq/azmfLtkSUFxQe5Li7pjuuXCiSbKW1rIcDZxtjFgCXA3cCBYEOR4AWgH1wbSV7ffFqnvvU7XqSXtxy1lU1lx1P1oZd7Ckq5tFZ9jXT24yfxpVTFtvK9xQe1K49UikOOxrEsqwxwcfGmAlAPtAXGAm8HPj/9MRVLza3vvktAOP6tU1yTfwqYxidF4bqlWcNkRVb7DvgAK4BsbgS1yn5YOlGW9mQP3/M+p37bcvPFhQeZOGKrQw+tlllVU/SXHnGWd8NXGyMmQs0Al6Mb5UqVzyDXmU0biujBR29gFPQ03Od715u+NdXjuUPvv+DY/nHh0kzfGQ5Pzfs0bmO5be99a1j+bJNzkEfYKZD0AUoOlj2P4b1O/c7lt/yxjdc9uLi0KYQQQeKS5jwzhK2Fnh+tKtUspg3H7Asa0LYj6fGvyqVw3V8chyCYGlf9XguORp9Hm6HdluU6ZvVO8pUDu5DJt9xmGYO8N+v1jqW/7wxfrvWvPnFGsfylxc4b2+2ebd7kDzv6QWO5V/+st2x/HD/nisDQXrfgeKI8unfb+CFefns3HeAv4w5PuK57vfN4oLeR/G7IXkR5bv2H6CkxEeDWtVdP0/Sn+dnMO7cG9tMt8rM6UZ/VkYcP9yt029dYJGl6GdHPTHP8fUTpi5xLN+4y7mlmC7KkzY55x/Ov8NXP//FsbykxMePG5wvSMGZmU712FJQyKOz7BtCd5nwAcdPnGkr37Rrv+N6LJKePB+sr4ju9EnzjrZV2/wttuhG3V1vOwffb9bsTHSVEiKeF7hEmf3DJsfyrw5zd3L7f793LJ+xZINj+f6olnm4sc8s5Pp/fcXeosghiNv2FHHeUwvYtDu9L7xVjeeD9fLN7rnJsvBCh92GnftZve0wszeJ79C9VJoUE+SBGE5JOf6YgvtkRnPLya/YXMDPgbx8dCP9xHtnMn/FVl6clx9RPvfnzbQZP43J0yMnCm3eXciU+fm2uyqfz8dH1ibHdM8P63exx2Gc+sZd+x1TTXsKDzourevz+Vy/w6u37XVM4+3YW+T42YUHi9ntsKaMz+c77EXPKzwfrMO1GT+NooPOOVq3708qfvk/spxbbOEjIlIvjEoiLFm3y7H868O03t1c+OznAPzz4+VR5Qu56+0l9HpgdkT5W1+u5ZLnF3H+05FT/A8Ul3D6X+dy5Uv2oYy9HphNj/tn2crPf2Yh/SfPsZW/vGAVgx/5mEX5kYt6bSkopP/kOdz77lLbe46fOJMBD31kKz/3n/PpPOEDW/lLC1bR4c7prIvqc1m/cx9txk9z7HPp8+BsHnzP3kH+2Oyfudlh3fafN+5mwjtLEroVnueDdVmDbTInekT/My51+SK6dcxVtnSdFBNU0bNz+2JW9l1aRQPEqq32Fi8QCm7zV2yNKA/m2z9btpWd+2LrM3Lt0A6k6VZGjZoJHvfTn7fY3gP+YO52rGjvfuNfNya6Zf/jen+/wn++tHdSr9+5nyc/WWErf2TmT7zl8PpLnl/EC/PyWbP98He+FeH5YO0FbkHhjL85D0eLRUW/oG518kKuONH0O4jdey4LaL3kMhrHKU2RThL5p5MGwTr1vljRcXTgwx8lpR6SvioSFCqj4f/sXHurFNwXYVvksLa5RPJ8sHa6HQp6fdFqjrtresQwqUR2mkXfyiWSctbJ57o+Swx3PbEG20TmQINclzBIwGft2u+8eNYbLuPlV2zZU66ZsMni1mcWD54P1odz9ztL2FNUTOHBYtc/yHh9F/YfKC6108cLI04qQ7qkGWL590zUqYb3J6T7n9WClVsdy79Y5TxZKXoESbCBtixq1Emw3G3WLLgPnZz9g/Ms2HEvLHI9VkWldbAui4q2dA5W9tU/3b+hHpYq/zSxNg7c+y/iVpUKKXFprN70+teO5cfc/r5judsY98NZs92583WaS64+36WzNh5inm7uNeF/qHe/vcT1Nqus/vNl6SM1UuWP/HDcWrepUvVUqcfhpEpQLqvKuMOLZ9ByS126jWJJV2nbsv5l297QugzxCtQQv0k45REeXytjwkoqTorxgngGQ7dDxdIg8EKjwe13Vdkpww0uC3Jtj3E5i8qQtsF66KOfOJbvKSx2zHUF/zh27z/A2h37eGPx6ojn9xUVM/1751ufNdv3lmvWmlcl80zjGX88kTuP4Zft9qe3pzC2WXuud1ku5ZU/jjwx7w8vd1vw7KEZlmN5LHfY8Za2aRA3L8zL54V5+dx1ZkfymtWlW+uGgD/n/PAMi9cW/cKWgiKyMsM6cHw+7pm6hNcWreak9o0jjrdyyx4GPfwRVw1oR7TznlrAANP00HEC37xVW/ewa5/7llKxhJAqdG2QckrUFOvKvuOq6Ke5j9qphA+PoyoXrIMmBqax/jBxGOCflfX3OcscX9v2j++FHocH2be/XssnP/lnWC1YfqjHemtBEZ/+vIX5K7baZn9t21PkOFUW/Dve3Prmt5zaMXLBep/PxxxrE0fUrRHj2ZVfeGOq4jMYPdByrQDXGYwxfMODs+e8Lr3/hVNLlQ3WQf9yWeYyFje+5twbfb/DmgIAq7ft48R77UtdAnS/byZbCvxrf2wKWwjngyUbWLxqO099soIzOueGygsCC+PcM3UpvzrhSNvx3G7rgqK/ZMUlPscxonsKDzJl/iquPPnQnUMG/mVA75v2Axf0Psr2noUrttKodnVSqllSBpWx+0zhwdRZWCjVA25ljDX3giofrCc6LBQD5fgDrmD+Mxioo10Zthpb+Ip74ReKWWFjPpes20nfB2ezaXchmVF1mrd8C3sLi8nfap+8c92rX/L+9xt49uLuEeWTp//Ii/NX0bJhzYjyFVsKeO6zlXzys32M6pin/Iv4N6kTuVj+wzMs5lib+PUJLSLKfT4fB4p9VM9OjS6U8iyS5CZR66tkRPY2p7U0P72YVflg7cYLfU9OIrfkivwzD1897YSjGoQej3/rW97/3r+ecvjylnuKDvJLYPGbfUXOLcGytHqCaaboYP3ywl+483/fM/+Pp9jes2nXfprWzQn9XNGO3ErJt1bCaJCY3qsol1YUrF34W0T2v/bK+bJXwrC8sI94bdFqx8dD//JJaLLP3e8c2txg0+5C/vGRf5nN5ZvLPsU+epTB1K/9S1T+EjVudvnmAgY/8jG3n3FsqCx4B7J6217q16pW5s+OpxUu575tr/Nd0sFSUlNO3JYweOdr59EI4SvLxfq3utthbWjw96E4Weuyslx50hXLNjnn7iNSchX8OhS4THGP6cKfQo02BesU5JYyjeeFYv1O5y9c+AiC8FmZ4XsJBlvhTtqMnxZ6HJ7aWRy2XvF6l70ci6KCWbBV/+ky+zKZ/SfPoVWjmrbyNdv3sreomBeiFt4P+vBH57XCC1wC1uG47Ul5t8uuPd+7LIlbHrPCdqkJ/7sI3y+yoimY8EknPp8vdJF1W8K3PLl+t2VNH53106HPDju/8mw7Z7ns+fm0y2JTTpsklMbtbyGeFKxdRAeOoG0uueXCOA6T+m6t8x/w92vj92V328Hcbe/Aihr1z/mhx898emiX9MnTf+TzQCAPX1dh/8Filgd2Qvlxg/N5O+2a0+9P9gXuAbYWFPpH54SN2gkPZW5pnvIoTOBiPqVxCzTx7DQN3wQjUba73J3EczjionzntUXC72ZSqGGtYF1W61xmOiUqyKW7YDoF4EDxoYAyefqhyQjRF5Y/vGHfqQMObUbr5Iopi/nylx10aVk/ovxAcQnj3/qOy/q1LVO9D8ctHfDJT4c6YxM1ISd8HenwFmmsmwS48fkO9eOEp8rCxfM7EB5Iw3+d5bkDShcK1uI5b7osH9DFYUsngFlLN/LlL/4RHt+G3Xb7gH8vWs1bX65x3P0jPKUTbvYPG5n0/o+28v0Hirlv2lL2OLTSp327ns8c0jluLeEDxSWO+wymmvBrTsIm4YQF6w9dNimOp59c0ibJHkKoYC1p7/Ip9r0Cwd/aduMWqG987Sve/tq+Z9/jc5Y5Tk3+dNkWrn75C1tH4Y69RZzyyEeOnZS3vvkN//1qbcSdBsC7367j05+32Fq2K7fs4YtVa5i5dGPEXcjBEh+vLvyF2jlZtmGcX6zaTu2cLNtM2q0F/iGfO6Ja4iU+H8XFPqplpcbwykSaudR5+dNky0jE1cIY0wZYOXv2bFq2bFmm97p9SUQkdXXIrUurRrVsge6qAe1o2aAmd0Z1uj4yuit7iw7ayp+6sBslPh9Xv/xlRPndIzoyIK8pb3+9jr/O/jlUfknfNlw7qD11a2TT4c7pofLTOjbjTyO7UC07k053zwiVH3dkPf59VR+yMjI49q7pEZ/xxR1DaFwnhzFPzmdh2M41027ox3FH1sfn80XMZv77+SdwZhf/hLTwuDX+9A5cPeDow//CXKxZs4bBgwcDtLUsKz/8OQVrEZEYXNynNS/Ot+8teVL7xny2LHJZifxJw8v1GYcL1ul/TyMiEgdOgRqwBepEUbAWEfEABWsREQ9QsBYR8QAFaxERD1CwFhHxAAVrEREPULAWEfEABWsREQ8odW0QY0wt4AWgGVADuBf4BngJyALWAxdaluW85qaIiFRYLC3rEcBiy7IGAOcCfwYmAo9bltUfWAaMS1wVRUSk1Ja1ZVn/DvuxFbAGGAhcHSibCvwBeCLelRMREb+Yl0g1xswDWgJnArPC0h6bgOYJqJuIiATE3MFoWVZf4CzgZSJ3u0mlnW9ERJLu3W/ta55XVKnB2hjTzRjTCsCyrK/xt8Z3G2OCu5W2AOJfMxERj3LaVKKiYmlZnwzcDGCMaQbUAWYBIwPPjwSmO79VRETiIZac9T+BZ40xc4GawLXAYmCKMeYqYBXwYuKqKCIisYwG2Qec7/DUqfGvjoiI9xUejP/mwZrBKCISZ8s2FcT9mArWIiIeoGAtIhJnGQkY0axgLSISZxkJmH2iYC0i4gEK1iIiHqBgLSISZ0qDiIhUUQrWIiIeoGAtIuIBCtYiIh6QcsE6JzvlqiQiknSKjCIiHqBgLSISZ1ViunkixieKiHhdygVrERGxS7lgPWHEccmugohIhZT4fHE/ZsoF69/0PCrZVRARqZCFK7fF/ZgpF6xFRLzOVxVa1iIiXhf/UK1gLSISdwloWCtYi4jEm9IgIiIeoDSIiIgXVJU0yHWD2ie7CiIi5VZlWtYNalVLdhVERMqtoPBg3I+ZksE6K1MLhIiIhEvJYJ2p1ZxERCKkZLBWrBYRiZSSwVpERCKlZLA2zeomuwoiIiklJYN1r3aNk10FEZGUkpLBWkREIqVssG7XpHayqyAikjKyY3mRMWYy0D/w+geBRcBLQBawHrjQsqzCeFbsvRv70+HO6fE8pIiIZ5XasjbGDAI6WZbVBxgGPApMBB63LKs/sAwYF++K1aiWFe9Dioh4VixpkE+A0YHHO4DawEDgnUDZVGBI3GsmIiIhpaZBLMsqBvYEfrwMeA8YGpb22AQ0T0z1REQEytDBaIw5G3+wvi7qqYTNN3z1il6JOrSIiKfEFKyNMUOB24HTLcvaCRQYY2oGnm4BrEtE5foe3SQRhxUR8ZxYOhjrAw8BZ1qWFdxffRYwMvB4JKBhGyIiAUc1qhX3Y8YydG8M0AR43RgTLLsYeMYYcxWwCngx7jUTEfEoXwK2H4ilg/Ep4CmHp06Ne21ERNJAldzd/MSjGiS7CiIiSZfywbp2jr/xf8fwY5NcExGR2FTJlnVwJmOH3HpJromISPLEtDZIMk06pzPH5tal79FaNlVEqq6UD9aN6+Rw02mm9BeKiKSxlE+DhGvVqGbpLxIRSUOeCtZ/Gtkl2VUQEUkKTwXrbq0bJrsKIiKl8iVgOIingnVOdhYrHjgj2dUQETmsBIzc81awBsjMTNgifyIiKctzwVpEJNVVyUkxTqZe149XL9da1yKSmhKxkJMng3XnlvXp2/7QWtcjT2yZxNqIiCSeJ4N1tEfO7ZrsKoiIhCgNIiLiAfVrVov7MT0frAeZpgDMvXWQ7bncejUquzoiIozr1zbux0z5tUEOx7pvGNmZ/utNq0a1aFo3h827C0PPm9y6bNi1P1nVE5EqKisj/kOMPd2yzsnOIits3PXdIzrSsNah24+WDbWWiIhUPo0GKcWZXY7kq7tOC/1855kdmTCiI8OOy4143ehuGj0iIt6SVsE6Wo1qWVxyUlt6tm0UUd4/r2mSaiQiVYFGg8RJMHHSvH5kB2SNalXy1yEiHpCW0emWoYYLeh8V+nmAiWxJd2lZn2OOqMM/xp4YUX5+z9aVUj8RkbLy9GgQN9cOah/x89FN65A/aTjd75vJloIialXPZuZNA5JUOxFJd/2OaVL6i8ooLVvWFXVG59zSXyQi4iIRczyqVLBuFvgFZh1mmdX8ScP5x9huMR+zXZPaQGJmLImIN2VonHXFPH9pD/4ypiuNalcPlX1z92mMP71Dqe8d5zCqJFyXlvUjfu7cor7LK0VEyq5KBesj6tbg1ydEjrGuX7Ma2TFsaHDXiI68flWfmD/rwj7+zsr+CchdiUjVU6WCtZvjjvS3gsP3eHzvhv5Mu6Ffqe994oJunNu9JW0D6ZBo0bdDE88+rgI1FREvOFyqtbwUrIE+Rzdm4f8NZniX5qGyjkfWCwXxcBPPPi4ijWJy6zJ5VFcyo4JyMD/epnGtuNWzdzv3NIyIpDcF64Bmh+m9verkdqHHF/Vpw5d3nmp7zdheR1Grelbo5wF5TXnpsp5cd0p722vBfuWNnqDjpFqW8z+XUi0i6U/BuhT5k4bzxzOOLfV1xzSry9KJwyLK+h/TlCPq1uDBczpzasdmgH8a6uPnn8ismwZQu3oWzerlhMrdVM92/mca3OGIw9apZrWswz7vJMfls0QkufTNLKe5tw5yXEO7e1jeO+i8nkdFtJyHd2lO2ya1WTJxGP+79qRQ+SuX9+K5S7rz4c0DePbi7qHyvGZ1ALhh8DE0CFtVsFNgxEn0XcHtMVxc3LR2Sdv0ade43McUkYpTsC6nVo1q0aqRPbC9MK4nsxxmR44KrPQ3IGoRqRrZ/tZvXm5dTmrfhFM6NKNd0zoMPrZZ6DVTxvXi5ct60aNNI74OW1Xw2Ob1ABhybDM+vW1Q6Ni5YReGFy7twf2/7gREXkgmj+oSkd4JuvQk50XTTW5dx/JjjqjjWC4i8aVgHWd1crJp7xDAurRsQP6k4bSJGjXSsHZ1XruyN4+ff4LtPQ+P7srzl/SgUe3qEdNX7znrOPq1b8KwTrl8ePMAhnXKpWXDWlwUGC7YrXVD6uRkM+Gsjgw0RzAkEPhHdD0ydIxzu7cKpXfC63tez6PInzQ8tKhVsNO1S8v6zL11EGcGfu7Z5lBn51vX9OXBczrb6v/m1X2YMq6nrXz67/pz3SB7Lv+x806gg8NFoV2T2lTPyqRp3Rzbc03r5lAnJy1XTRCJoL/yFNDbJcUwymXd7Yv7tuHivm0AaNf0UKAdfGwz8icNB+D7e4aGypvVq8FP951OtawMcuvXiMhlf3PXaeRUy+Ti5z5n4cptofL54wez90AxD8+wQmWtGtVi8qgujO3VmsZ1qnPaXz4B/BeHE49qQHZmBiO6HkmHO6cD0D0Q0B8e3ZWC/QeYMHUpAB1y69Ehtx6dWtTnwfd/YNXWvaF6Tv/dySxcsZUxTy0IfW7bJrX58A8DWbtjHydN+jDid7Ho9iGUlPjoNGEGe4uKQ+XL7j+d/QdLGPPkfJas2xUqf+PqPjSuXZ3nPlvJywt+iTjW1Ov68c2aHdzxv+8jyt++9iSmL9nAEx8tjyhvUKsaOdmZbNxViEjQnxO0gXdMLWtjTCdjzHJjzHWBn1sZYz4yxsw1xrxujLE3eSSlVM/OJCMjg6HH5XJyWCqmfq1q1KiWxYvjerLo9iGh8oa1q9OiQU1uHWY4tWMzhnXyr5dSq3o2fY5uTMNa/uGLwWNlZGQwunsrajh0ao7q1pJLHNIrwzrl8vEtg2x5/l7tGrPs/tN54NeRrfUWDWqy/IEzuPPMjhHlmZkZLJ04LGKcfHZWJnVyspl2Q/+IDtoWDWrSrmkd7vtV54j8/IW9W9O5ZX0u6N2a3/RoFXH8rq0acNuwDsy66eSI8s//bwgL/28IS8IujAB/GtmZ/EnD+fHeyA5ngB/vHcZ7N/S3ld8y1HB8qwYRfRLhjm7qPI5fUs85JyZmc5NSg7UxpjbwGDA7rHgi8LhlWf2BZcC4hNROKk2NalmOaYbm9Wvy9EXdqVU98iasad0cPht/Cn90mKr/198cz7+v7G0rH92tJfVq2G/mOjT3pz4a1T4UqLKzMkMjZcInFmVlZnCZy2akb13T17H827tPixgbH/56p0W7/njGsRFL7AZF91EELwK1o9IwY3r43xt94eJQvCgAAAlZSURBVLpm4NHUqJZFxyPr0aNN5AXq2kHt+d+1J/HomOMjyl8c15P8ScOZ+Xt7P8jcWwcx/Xf2wP/omOO1G1IaiqVlXQicAawLKxsIvBN4PBUYglQ5LRrUJNth7PfZx7egl0Nq56HRXfl2wlBb+Z1nduSNq/vQ/ojIfPWAvKZc2Ls1DwQ6SKOd19MeUJ3UqJbFEYELUfjSAk3q5NCrrb+e4XOa6tesxn2/sufgc7KzQmmmaN9OOM2x/LPxp9A1sG5Mo1qHLhj/GNuNXx3v70MIv5B0a90wIk0V7DTOjBqXb903jFaNatEht15E+Vldj+RXJ7TgodFdadEgcg/SxXcMYcUDZ3DH8MjRQjnZmbx1TV8+G3+Krf5/P/8E2wUE4JwTWzieryROqcHasqyDlmXtiyqubVlWMFG3CWiOSDnlZGfRo419dmZ2Vib3/qoTRzhMWMqfNNyxU3PxHUMcg87zl/ZwPNaZXZrTsXk9ruhvHxkD/gW8YlGvRjVbcAT/Ba330Y0D53Mo4Datm8MdgXROeBiuW6MaPzikT4CItEpO9qGAvuCPg0PLHYTPwv3Pb/tyeiB9dcJRDWhSJ4fMTH+6qlOLQ0Heuu90urVuaKv/+zf258wu/uAf7ZHRXcmfNJwmdSLvxl69vBcf3zKQK/rbf29PXtiNOX8YaCv/08jOPBG1EQjAtYOO5iiHEVeJmMrtBfEYDVI1f3OSkprUyXEMms3r1+TC3vadgBrXyeG9G/s7DsPMnzScu0Z0tJV/cccQFt9hv5n8z2/78vwlPWzl159yDJf1a2u7E2hYqzrdWjfkzw4t1zaNazEwaoejGtWyHMe759avwYhAkA4fc9+sXg1uGHyM/zVh5fVrVuPd6+3pEyDiziE4NBTgqztPDaXJ3ruhfyg19f6N/TkrMMpoRNcj6du+Ca0b1+b3p+ZxcZ9Dv+8l9wxl6HG5tG5UKyL//tCoLozpcRSnd25uy9fffKrhk1sH2Va0fGlcT5ZOHMqVDkNP372+Hx/8/mRb+SuX93Isf/XyXqGhreF+06MVHZvXs5UHOY34AiL6g+KtvMG6wBgT/Ea0IDJFIpLWGtfJsbUowR8cBznMKq2Tk82dZ3a05bCzMjN465q+trH3AB/dMogXLrUPe3xxXE++c0i53Dgkj//+ti/Ht2oQUX5s83r89TfHM3lUF9t7Hh7dlTevtq8k+cjorlw1IDIQNqxdnQd+3ZlqWRkRHbNN6+Zw06l5AKFhneDviL7nbH8Q7Ne+SSivn5mZweybB4ZeN7r7oc7cr+86LVT/HyYOC6V+Xruydygw3zD4GPq2b0Kt6tncMtTwt/MODXn9bsJpdGrh37LvtmGH+lIeO+8ETmrfhLxmdW3n1bd9E8b2as3lUf0gl/Vry3s39ufeqIXXTuvoH3H1+Pn2O4GlE4fyXNhktrjz+Xwx/ZeXlzchLy/vusDjp/Ly8i4IPP5bXl7e5VGvbZOXl+dbvXq1T0Sqrh17inz7Dxy0lW/cuc+3ZvteW/m2gkLf3J8228oPHCz2PTN3ha/wQLHtuSc+WuazNuyylT//6QrfKwtW2crfXLzaN/qJebbyxfnbfK1ve9e3fNPuUFlJSYlva0Ghr/Vt7/rO+vunEeeybNNuX+vb3vW1vu1d33drdtiOVx6rV6/25eXl+fLy8tr4omJwhq+UPdONMd2AR4A2wAFgLTAWeAGoAawCLrUs60DYe9oAK2fPnk3LluqVFpH0tP9AMZkZGa7r95TVmjVrGDx4MEBby7Lyw58rdVKMZVlf4B/9Ec2+9JyISBXiNK8gUTTdXETEAxSsRUQ8QMFaRMQDFKxFRDxAwVpExAMUrEVEPCBR61lnAWzYsCFBhxcRST9hMdM2JjBRwbo5wNixYxN0eBGRtNYciNjtIlHBehHQH1gPFJfyWhER8cvCH6gXRT9R6nRzERFJPnUwioh4QMptmGuM+QvQG/ABN1qWZbsd8AJjTCfgbeAvlmX93RjTCngJ/23OeuBCy7IKjTFjgd8BJcBTlmU9a4yphn+hrNb400iXWpa1whjTFXgC/+/mW8uyrqn0EzsMY8xk/OmvbOBB/LdyaXnOxpha+OvbDP+CZvcC35Cm5xsusDzy9/jPeTZpfs7GmIHAG8CSQNF3wGQq+bxTqmVtjBkAHGNZVh/gMuBvSa5SucS6b2XgdXfh3xZtIPB7Y0wj4Hxgh2VZ/YD78Qc+gEfxX8BOAuobY06vjPOJhTFmENAp8G83DH9d0/mcRwCLLcsaAJwL/Jn0Pt9wdwDbAo+ryjl/bFnWwMB/15OE806pYA0MBv4HYFnWD0BDY4z7dg2pK9Z9K3sBiyzL2hnYOu0z4CT8v4f/Bl47CzjJGFMd/7KJi6KOkSo+AUYHHu8AapPG52xZ1r8ty5oc+LEVsIY0Pt8gY0wHoCMwLVA0kDQ/ZxcDqeTzTrVgnQtsDvt5c6DMU8qwb2X0+drKLcsqwX+blAtsd3htSrAsq9iyrD2BHy8D3iPNzxnAGDMPeBX/rW/any/+te1vCvu5KpwzQEdjzDvGmE+NMaeShPNOtWAdLV33d3Q7r7KUp+TvxhhzNv5gfV3UU2l5zpZl9QXOAl4msn5pd77GmIuA+ZZlrXR5Sdqdc8DPwD3A2cDFwLNE9vdVynmnWrBeR2RL+kj8yft04LRvZfT52soDnRMZ+H8PjR1emzKMMUOB24HTLcvaSRqfszGmW6DTGMuyvsb/5d2drucbMBw42xizALgcuJM0/jcOsixrbSDt5bMsazmwAX+KtlLPO9WC9QfAKABjzInAOsuydie3SnEzCxgZeDwSmA4sBHoYYxoYY+rgz2/Nxf97COZ/RwBzAtum/WiM6RcoPydwjJRgjKkPPAScaVlWsPMpnc/5ZOBmAGNMM6AO6X2+WJY1xrKsHpZl9QaewT8aJK3PGcAYM9YY84fA41z8I4Cep5LPO+UmxRhjJuH/IpQA11qW9U2Sq1RmZdm30hgzCrgFfx7rMcuyXjHGZOH/MhyDv7PyEsuyVhtjOgJP4r/ILrQs6yZShDHmSmAC8FNY8cX4zyPtzjnQqnoWf+diTfy3yYuBKaTh+UYzxkwA8oEZpPk5G2Pq4u+XaABUx/9v/RWVfN4pF6xFRMQu1dIgIiLiQMFaRMQDFKxFRDxAwVpExAMUrEVEPEDBWkTEAxSsRUQ8QMFaRMQD/h+VtbSNOi23hAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "----\n",
            " or CCSChinl� fibnet bestizas vicriost mad ronten ares youd, toptiond Mecspiot cere ninst puad and Healses ans.\n",
            "\n",
            "Whoatif countes Cibsion C1Thrings can cadore treape neptiruthcent ifiver in U\"rinfen ocn \n",
            "----\n",
            "iter 49900, loss 20.332579\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2AKpa1BGOItQ"
      },
      "source": [
        "# Quiz Question 7. \n",
        "\n",
        "Run the above code for 50000 iterations making sure that you have 100 hidden layers and time_steps is 40. What is the loss value you're seeing?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 466
        },
        "id": "Glf5nwwMVf7w",
        "outputId": "d48d76b2-0127-4735-9211-c115e1adaf7f"
      },
      "source": [
        "update_status(inputs, g_h_prev, g_C_prev)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAD1CAYAAACWXdT/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wU9f3H8ddx1KP3DiflvqggiDRBBAUURUWDShSJxujPWJIYKxqToIgSe2KJXSOKJbGLFRAEpCpFVAYph1TpHQ7ubn9/7N7d7s7s3d6xczuz934+Hj7c/e7uzHdul8985zPfkhYIBBAREW+rlOwKiIhIyRSsRUR8QMFaRMQHFKxFRHxAwVpExAcqu7FRY0w1oCewCchzYx8iIikoHWgOLLAsKyf8BVeCNcFAPdOlbYuIpLr+wKzwAreC9SaA1157jWbNmrm0CxGR1LJ582ZGjRoFoRgazq1gnQfQrFkzWrVq5dIuRERSli19rBuMIiI+oGAtIuIDCtYiIj6gYC0i4gMK1iIiPqBgLSLiA54P1pljJvPI51ayqyEiklSeD9YA/5q2MtlVEBFJKl8EaxGRik7BWkTEBxSsRUR8QMFaRMQHFKxFRHxAwVpExAcUrEVEfEDBWkTEB3wbrN9fvIF3vl2f7GqIiJQLt1aKcd2f3lgMwK+6ayUaEUl9vm1Zi4hUJArWIiI+oGAtIuIDCtYiIj6gYC0i4gMK1iIiPlBi1z1jzEDgv8D3oaLvgAeAiUA6sAkYbVlWjkt1LJVdBw6zZtt+TmxTP9lVERFJmHhb1jMsyxoY+u8PwD3Ak5Zl9QdWAle6VsNSGvnMXC546utkV0NEJKHKmgYZCHwQevwhMDghtUkA65e9ya6CiEjCxTuC8ThjzAdAA+BuoGZY2mML0NyNyomISFA8wfonggH6LaAd8GXU59JcqJeIiIQpMQ1iWdYGy7LetCwrYFnWKmAzUN8YUyP0lpbAxkRWasoPv3AkLz+RmxQR8bUSg7UxZpQx5pbQ42ZAU+AlYEToLSOATxNVoRkrtnLVKwt5fOpPidokAIeO5PHsV6vIyw8kdLsiIuUhnjTIB8AkY8xwoCpwLbAIeMUYcw2wFvhPoiq0bW8wFb5+58FEbRKAR79YwTNfraZRrWqaqU9EfKfEYG1Z1l7gXIeXhiS+Ou7ZcygXgINH8pJcExGR0vPcCEYlKURE7DwXrAupj4mISCHvButysi8nl8wxk3llTnayqyIiElOFD9a/7DkEwMuzs5NbERGRYnguWAcCylqLiETzXLAukOZS0ro054IPl2xkYfYOV+ohIlIanl3dPNEt7LQyxP4/vL4IgOwJwxJaFxGR0vJcy7ogRr+zaENyKyIi4iGeC9b5PshZr92+n2/W7kx2NUSkAvFsGsTLBjw4HVB6RETKj+da1iIiYue5YP3Dpj1J2W8iki879x/mvwvXJWBLIiKRPJcGeWXOWle3Hx2UE9lB8I9vLGLmT9s4qW192jWulcAti0hF57mWdbhPl21K2LZiBeVE3s7csic4vethLZwgIgnm6WD9+1e/Lbd9uTlv1C97DpE5ZjJL1+9ycS8ikso8HaxTxYwVWwH3UzwikroUrF1Qmq7iK7fsZe+hI+5VRkRSgoJ1ApVlSPvgR77i0ufmJb4yIpJSFKw94LsNu5NdBRHxuIoXrJM4nL00uz79oekMePBL9yojIr7iuX7WsRw6kkdefoCa1cpW5ZJSFG6G8LL0NFm9bX/C6yEi/uWblnWv8VM4/u+fJXy7bnTZc7Px/sgXK8gcM9m9HYiIJ/kmWO85lAvAU9NXJrkmyfWvqT/FfG37vpxyrImIlCffBOsCD3xqJXR73p+QNT7vLlrPSfdOYfE6DbwRSUW+C9ZucXMEY4GAi6eGOau2A2Bttk+E9cb8n9XqFvE5BetykFaWDtgJsmbbfsa88x3XTyq/ofsikngVLliXR9rDzRZ0aR0JTSq1fd9h22uZYybz25fml3eVRKQMKkywdmu19Ih9JLEFXVZfWlttZbN+2sbr839OQm1EJJYKE6w9wTsN7mJd9sI87njnO1v5gcO5zFu9PQk1EhEF6xCvDYrxotv+t5SRz85l466Dttd2H9RkVCJuqvDB2m+DYpK57x9DS64dOJwbUb5k3S663v05Hy3d6N7ORSq4Ch+sEymZLejyyMnHsmxjcCKq2SvtKZLMMZN5YlrsgTwiEh8F6woimenyhz5fYSsbP/kH7v/kxyTURsSfKlywTmqKwtVtO2/dq/ny52au4ZkZq23ls1duY+wH3yehRiLeFtcUdsaYGsAyYBwwFZgIpAObgNGWZXl+eFysXnXlEbvLs0df9L580gGl0KjngwsxjD3v+IjybftyeODT5dwzvDPVq6Qno2oiSRVvy/ouYEfo8T3Ak5Zl9QdWAle6UbHibHDojXC0vNoCTRS/H999H//IWwvX8+GSyJuY+fkB/vb+MtZoSllJcSUGa2NMJ+A4oGBezoHAB6HHHwKDXalZMfpNmFbeu4xLebagA8nM5yRR9MCj5Zv38sqctVz76je29z4x7Sc++35zeVVNxFXxtKwfBm4Ke14zLO2xBWie8FolQbkMQ6+gATZZHvp8BddMtAfx1+atVV5cfKfYYG2M+Q0wx7KsNTHe4ver63I5ADda3LHivs4HJfvLu8t4+etsW/nn32+m692fc+hIXvlXSqQEJbWshwHDjTFzgauAvwL7QjccAVoCSR8JMWnez0yck53saiRVMvtZp4rxH//I7oNH2Lz7UES5tXkvmWMmM3vlNttntuw5RE6ugru4r9jeIJZljSx4bIwZC2QDfYERwKuh/3/qXvXic+e7wXksRp+cWeJ7yyMVUR6t29LuIhkN7sO5+Y7lZQlu63YccCwvmFXQTfPWBAf7fLpsM/06NIp4rdd9UxnUqQkvXNEzovzHTXtIr5RGVtPartdPKoay9LP+O3C5MWYm0AD4T2Kr5A6/pjuOVmmrlBsj+OXlB1i11bnHxV/eXeZYfvrD0x3Lb/vfUsfyT77b5Fh+ODefBdk7HV8b/uRsx/KXZjtn7vYeij2HSVnTH1OXb7GVnfXPmZzx6Fe28iXrdvHMjFVl2o9UbHEvFW5Z1tiwp0MSX5XUl4xBMTkxWrfvLlrvWP7YFOeh4fPX7HAsL876nc5dLGessE/LCrBq6z7H8tz80ree7/7wB8fyv73vfGNx/c4D/LLHebjAa3Odp4styxVCwcnlmgHtI8oXZO/goqfnMOeO02let0bEaxt2HaRZneqkV/Jga0DKje9HMO7Yb59UH7zV88KNfHL08a0OtXqjW/fnPD7L8fN/fnOJY3nBZE22/flueI2zWL+X4vppW7/sdSwf8MB0x/LVMU46xXl17loA5q2OPClu2n2QfhOm8cBny22fmbFiKz/FqJukHt8H68tfPLqVTtwIQeUd2HLz8lm41jlNIO7ZvOeQY/nIZ+c6ln8V44oiEAjw/mLn+/QFVzRONzcvf3E+QxxSLW8tWMc/PrUHd/E33wfrn7YkpmWRiLav2z0ypi3/pfBx+OkgN7/oWawLilS5gI51HF5apefgYef0yDcxTqh7c3IdywH+9MZix/Ib31jkWL588x5ue3sp/54emRfPzw9wwtjPyBwzmf1R+5v501Y+WrrRdrW2bscBVjr8+9qfk+t4BRYIBFi2YbdjvbK37Xe8J7B1bw67D9jvI+zPyWWLw8kwPz/Apt3O6bUd+w87XlEfzs0nL9//V4a+D9bhMsdMjvlaSV+Vl77K52faJzgCsDaX/vJaUtN7MVri4z5yztV/vWo7ew4Fg3R0Kmj0C/O5YdIiJkUt5db/gS8Z/MhX/Lw9sifOta99y1n/nGnL2X+wZCPnPD7LdqP4wOFcBj40nZv/a0+99Rw/hRPHfW4rH/7kbHrdN9VW/tT0lZx8/zSyo9JWK7fso/u4L5gYSieFy7rrE37vMML1vCdm0XP8FFv5K3OyyRwzmfx8+8mr34RptpNFIBDgng9/iJlCTBTfB+ujbc2WZ3ss+qQ/Z5XzEllPx9FboLxT8urH7W9H4rhJ+9Mvzo2B4U9G3veYH+rKGD1r4opQ/jz6RnHOkeC+nVI5AE6N3pVbnOsyK7SNjVEBs+CeQ6xU0xc//GIrW7p+N1v32m8qF5zwcqMq9uq8tWzYdZD3FkWeKLftO8yLs9dwWWgSMrf4PljHy4uh5pLnnHObbog3th9dNsFL1yelE34y8lBGxRN2OqQpAB75wj5POTjPXw6wK8Z2UoXbvxvfB2sv/sOKbvXe+GYw75jIuqZK7ww/iff7i9UTSd8YLF2/y7F8XzF5ewnyfbA+EONmDsDLs9fQfdwX5VaXRN3s9C8PnjnjlMyTX/hf7Wjrkcweq/GkyrK3O49EHfO280CpsvTvT5ZY3UITxffBujhjP/zB9T9ggTXb9nPoSPF5wUT+Q/JQN/JSK21IV7786CXyL+jGb29djAFUFz8zx7H8D5Mie8MUXM3EGohVnOjeKAXH9863zgPHYnWLdLvDSUoHazfE6gLkdKMilXgx3eRVMf/NxhHlYp2Y/HDCivUbCS+PmSIq5Rlge4xG2PLNezlw2Dml4tQVEOCfUyNH7RbUZMw730WUx/xuyumrSdlgHf7d3/6/pfxnjr1LT1k87sGVuhPb0kleUIinr7Rb/zASGQxjTl8b6/0J23N8Yk+vm7zLtaXrnftnl0WsG5lvLljnWP7LXucg7jUpG6zXbC/qh/nmwqIv6Wh/jt9vLLkvpRczFN5vlx2dVD++eB1NvC3u/k/EPsq+C9fEU6flm53vKU1eGtkv3EtTVYSLeyInvxn08AzH8j0Hc1m8bhfdWtdzfH3XgcPk5Obb1vrbffAI36x1vtkRa/rORIu8nCzdZxP58/Pob9m3EppPjuObjnV1Eiu1EK94juNofzvJ/O3F+tuWV51SNljH8uiUFTw6ZQXjL+hMxya1aVSrKhAcIvzgZ8t5avoq2x8/EAhw/WvfMmvlNluQn71yG6Oen8dlfdrY9jX4kRlcN7BodrWCzS7fvIfKxc6g5td24tH2ZPDGWcCL+eHvYgzjjldCh+MfxdfkVq+bhN68P4rPWpv3Ypq5M4d5hQvWBQrmYJ528wAgOCnPk186jxw85o6PCx+HTzn66ty1hS3wJeuK/jFt2HmQ3Lx8Vm7Zx01vFQ2xDQQC7Nx/mKGPzXTcz4RPlvPsV6toULNaRPmRvHxm/RQ5+sutH33Mm0Su7E285KhPljF+PF48+ZVFPDcYV23dp2DtlpdmZ5f5s3e95zzp/vWTvnUs/2jpJj5a6jzBftZdnziurPLWgnV8smwTX1pbGZDVuLB8+77DbOcwN7y+iDFDO9k+dzg3v1TzH8eamzl7234WZO+gTYOMwrK0tGC6aMInyxl73vG2z3y0dCP92jeylfvdnoPxjcBL9uWyl7l0PqgQKnywdpr4pSyO9kcUawms28IGC4T3Ge//wJeFj+/+sGhC/Y+WbmLi3LV8v3EPfdo1KCzPDwSYbm1h0+5D1M+oattPj3FTOHAkj9M7NYkoP/fxWezNyeXN/+sTUf7IFyt4Y8E6jmtRJ6w0jQ27DnLDpEX0bd/Qto+Lnv6aE1rVs4Wy/PwA+cX8K/bKaM0dKT5cujyk+gnLzeOr8ME61cwKmyxnbthE9qu37ueKlxY4fmbkM3MKp+kMn2ls5/7DheXRv8FYP8qc0DSYm3bbu0MtyN7Jguyd1MuoElH+6+fmMn/NDm45IyuiPC8/wPb9OdSqVvLPNNbEP9HKI1bE7rrn/EIi88nJHcGYGo7mb+hmwyJlu+65JZk3wdy6vJ4XNqR3676iwT0nhg3Vnxa2zuCSdbt5b/EGADbuKn0f1eh/1LGGFD/yhUWv8VMdBxz9uGlPxHdxOM6Fc/NizD6Xl8DvNdZSauH3NcJt3OU86i7WepjFmRfjb/npsqL0W/ihxhooUpxYf+tYs0juCxukcrR/ZadGAEQunBzz3Ofzs4la1h4UKyVwJDfG6Ml9iRs9GesfwwyraOrJ8HmJw6dz/WDxBl4PzYkcvUzWZ99vLnwcaxa31VEL8k79MXiCCG81p6UFg/vFz8zhrmHH2raxeN2uiPx6tNkrnQPK41OdBzvFmnioLGbFmCI01mCNRE5u9Pp8530cbS+TcPOznU8UT4cthBB+gt1Zhq6Ca2PMLTJ+8o+O5d9vLDq+o43VsaYuXvRz0W9EaRAPibWU084DiZuDJNZ8JrHWAkzk/Cexhs3H2ne4/cUMqrhmon3yd4BfP1s098M7izZE1GNLqC5Pfrky4jM/h/q1O7Xkzn9yNm0b2oP1knW7im3Vffuz8yousU4siRS9ckuB6PmUEyW85Rnr95zIK8hYN6+Lu09RWrGOoyxzhZTWp8uKGiJuXncrWJdSrKGs63a4/6NIRXNXO7fGwlfw+Das5bJk3W4+CV3STw1LzUBRS82p9VWwqni0lVv28sqctcG8cSh4xJNCTmQweyNGy3ruauergER6Y77z6N5EturfWlg0IVL4PopbpDhRyjtr6WaaVMFafOXlr7NjvnZijOlwo9cjLJCXH2DwI/YFZwG27cuhx71TaN+4pu212/+3lGpV7Ld7FmbvYPE6e9pkf05uYY4/2oIYqYPdB48UrrCSUsJiWayURiK9seBnx3Kn9SDLKpFX1cVRsJaUF2tKy/Z3fuxY/vr8n3kqFOBXheXRd+zP4bSHpju2CIc/OZslDoH6jneWOuaL3/5mPY98sYINUTcXs7fvZ9Tzcx1z6397fxlzV2/nYFSgeefb9Wzafcg2cOrHTXvYsjeHr1dFlu/LyWVh9g72H86zpdBWb91H3RpVbPs4nJtPlfQ0Ty1MHI9YaZBYixqXxbSoKzy3pLnRbDfGZAJrpk6dSqtWrUr12eIWvRUR72lSuxrpldJsN6eHHt+MujWqREykBtC/YyPOPaEF63Ye4PFpRfcjsprWYvwFXaifUSXiiqd53er858pe1K5emZPvn1ZYnl4pjak3DSCjarptcd1P/tSfJrWrMfLZuRE3qF+4vAf9OjRiX04uPe4tSrXdfd7xjO7TlkqV0iJi0DUD2nHHWcEb2eHlw7u14LGR3UhLS7PFrKVjz6BO9cjuqfFav349gwYNAjjGsqzs8NcUrEWkwmlapxq/7HFvDvp5dw6iaZ3qpf5cccFa/axFpMJxM1BDZJfBRFGwFhHxAQVrEREfULAWEfEBBWsRER9QsBYR8QEFaxGRBHNjdRwFaxERH1CwFhFJMDcWIShxbhBjTAbwMtAUqA6MA5YAE4F0YBMw2rIsd3uZi4hUYPG0rM8FFlqWNQC4GHgEuAd40rKs/sBK4Er3qigiIiW2rC3LejPsaWtgPTAQ+H2o7EPgFuDfia6ciIgfuXGDMe4pUo0xXwOtgHOAKWFpjy1A84TXTERECsV9g9GyrL7AecCrRC5n5q8JbkVEfKjEYG2MOckY0xrAsqzFBFvje40xNUJvaQlsdK+KIiIST8v6VOBmAGNMU6AWMAUYEXp9BPCpK7UTEREgvpz108ALxpiZQA3gemAh8Iox5hpgLfAf96ooIiLx9AY5CFzq8NKQxFdHRMT/Nu52XvvxaGgEo4hIguUnfgCjgrWIiB8oWIuI+ICCtYhIgrkx+ETBWkQkwdJciNYK1iIiPqBgLSLiAwrWIiI+oGAtIpJgn3//S8K3qWAtIpJgM1ZsTfg2FaxFRHzAc8G6WmXPVUlEJOkUGUVEfEDBWkTEBzwXrN0Y+SMi4neeC9YiImLnuWA99tzjk10FERHP8Vyw/nWvNsmugoiI53guWIuIiJ2CtYiIDyhYi4j4gIK1iIgPKFiLiPiAgrWIiA94MlhfN7B9sqsgIuIpngzW9TOqJrsKIiKe4slgnV5JE4SIiITzZLDWZE4iIpG8GayTXQEREY/xZrBW01pEJIIng7VpVjvZVRAR8RRPBus+7RomuwoiIp7iyWAtIiKRPBusMxtmJLsKIiKeUTmeNxljHgD6h95/P7AAmAikA5uA0ZZl5SSyYp/eeCqd/vppIjcpIuJbJbasjTGnAZ0tyzoZGAo8BtwDPGlZVn9gJXBloitWvUp6ojcpIuJb8aRBvgIuCj3eBdQEBgIfhMo+BAYnvGYiIlKoxDSIZVl5wP7Q098BHwNnhqU9tgDN3ameiIhAKW4wGmOGEwzWN0S95NoIlteu6u3WpkVEfCWuYG2MORP4C3CWZVm7gX3GmBqhl1sCG92oXL8OjdzYrIiI78Rzg7Eu8CBwjmVZO0LFU4ARoccjAHXbEBFxUTxd90YCjYC3jDEFZZcDzxtjrgHWAv9xp3oiIgLx3WB8FnjW4aUhia+OiIg48ewIxgKdW9ZJdhVERJLO88G6VrVg4//u845Pck1ERJLH88G6RmgkYydNmyoiFVhcc4Mk0wMXdmXi3LX0zGyQ7KqIiCSN54N149rVuGlIVrKrISKSVJ5Pg4SrWVWTO4lIxeSrYP3oyG7JroKISFL4Klj379g42VUQEUkKXwXrGlXTWXXf2cmuhohIsdzovearYA2QXsm1Sf5ERBKiQ5NaCd+m74K1iIjXVUpLfKPSl8H6/ev78frVfZJdDRGRcuPLYN21dT1Obt+w8PmQ45omsTYiIpFcaFj7M1hHe+43PZJdBRGRQm7cWUuJYC0i4iVpylnbndW5GQAzbzvN9lrBJFAiIuXJjZa15+cGKY5171AqVwqeb1o3yKBa5Urk5OYXvt6xaS2Wrt+drOqJiCSMr1vW1SqnR/S7vu+CLjSqVa3weZsGGcmolohUdLrBWLwRJ7Vi4V2DC5//Y8QJjD33OPq0i5xe9eIercq7aiJSgaS5EK1TKlhHq1mtMlf0O4aT2taPKO/bvlGSaiQiUjYpHaxjKbhRe0yjmhHlbRsqbSIiR0/9rOP0x9M7cGW/Ywqfn9W5ecTrA7Iac0yjmtz/qy4R5aP7tC2X+olIanNjCiNf9waJ5aYzTMTzzi3rkj1hGJljJgNQL6MqX94ykJ37D8e1vRZ1q7Nx96GE11NEUpNy1uUkuoUdSFI9REQKVKhg3bxu9YjnNRyWCcueMIxx53cu9bYb1apa5nqJiJSkQgXrt6/tyxOXnlj4vHqVdJb87Qyu6JtZ4mcvOLElQ49vFvP19o0j56/VepEiFZduMB6lFvVqcM4JLSLK6mZUiWvu2UdHduPp0SfFva9fdQ/25T6lg7oJilQ0CtYu6ZEZ7Id9XIs6hWXvXd+PV67sVeJn//f7k/ltv8yYX050+eOXBFv2Gl0pksp0g9EVZ3dpzrw7B0UMlunWup7j0jx/HNQxorxHZgP+fu7xtvcV9NluWicyTx7rZmV0n28nWtFMpOJSsA6JDqoA9TOCNw3D+2zfNCSLKTcNsL33xsFZ1K1RpfD5lf2O4ZUre3Fu18i0S5VQxG0YdUPyhFZ1S6xjrHRNvw4NHcszlDcXSQqlQcpZjarpZE8YxtWntivxvX3aNWTJ388ofF6pUhqnZjXm1I6NuGvYsbQLtZyb16vBP0Z04dnRwQUTKoeCd6AM/QMLpoCN1aczepi9iJQPLT7gIVNuGsDcOwbZyhvUjGwxp6WlcVX/dtSuXjT+aGTPNjSuXY3sCcN4+OKuheVPjerO29eezPvX9+PN/7OvMfnwxV0j0iXtGgcfd4lqld9xVicgsSvBn9imXsK2JZLq3JhLX8G6jDo0qUWzuvbUybSbB/DVrfaFEK4d2AGw56Ybh6Z0bdMgg7O7NOektg3o2roevdsVpTZm3nYa717Xl+HdWvLlLQMLy/uG1qEc3adtxOILjWsHt1k/oyqPjexWuEDD3ecV5dZvOK0D1w5sb6vnr05s6Xi8/Ts2diwf3q2FY3nrBjUcy0Uqgpb1E//7V7BOsHoZVWnjMCHU0M7NyJ4wLCKvDdC3QyNe/m1P/jS4o+0zfzn7WN67vh+tG2RwYpuilMbvTjmGS3q14fahnfjq1tNoUa8GrRtkcPOQLAD6hboLjj65Leef2JKLQlPChreObznT8PtTg8H6t/0yC8sfGdmN7AnDgMgbmjcO6sis24tOCNWrBH86vY5pwItX9OA0ExnM09PS+NclJ/LnwVm24/r3qO48fFFXW/nzv+lB/472ro5/Pec4W5mIl0VfYSdCWqAsydISGGMygTVTp06lVSvNHe0FObl5VKuczsS5a+mZWZ9OzYLdFLfuzaFBzaq0v/NjgMJAvXHXQWpUSefEcV9ElG/efYgd+w8zce5aXp//M+Mv6Myo3m3Zc+gIXy7fQpeWdTn94RlkNsxg+q2nkZcf4OkZq6ifUZU73/0uYlv/nr6Kp2esYvfBIxHlL8xaw7iPfiis+6c39qdTszq8t2gDN765uLD8wpNa8dBFXflu/W7OfWJWxPH+cM+ZHM7Np9s9X0SUL/n7GeQcyePkCdPIyy/67T8z+iTqZ1Tlvo9/ZPG6XRGfefDCE3jn2w3MWb09onzc+Z3563vL4vsCpEIp+C2X1vr16xk0aBDAMZZlZYe/FlfL2hjT2RizyhhzQ+h5a2PMdGPMTGPMW8aYaiVtQ5KrWuVgDm10n7aFgRqCKZP0SmksHXtGxA3SFvVqUL9mVV6/ug+X9WlTWN6sbnWOa1GnsJXesUltAOpUr8Lwbi0Le6xkVA3m6NMrpXH9aR24tHfRNgpcO7B9xD4L/O6UY1h295m26QHOP7Els8ecXpj+KdClVV2+uvU0WtYruvTMqFqZehlV+f7uMyPeW7dGFZrUqc6q+86OKD/z+Gb0OqYB71zbN6L8rmHHclGP1ky6urftHsDoPm3JnjCMUVHHtuivQ8ieMCwiZQXB7pxfjzmdaTfbexM9fVl3RnS3N2zqZ1SxlUnFVGKwNsbUBB4HpoYV3wM8aVlWf2AlcKU71ZPyUqd6FVuKBuDk9g259/wutvKLTmrFzNtOo9cxkavwtG2Ywa1nGp67vIftM03rVLN1ZYylVrVgsI3Wsl4NznfIq7dpmMHsMafbymtWq8yyqIBd4KM/nGIrq1QpjUlX9474PARvFD99WdEI1o5hfe0vi5r4q37oEjj6/sS0mwfSol4N2kVNTfDQRV0Z2rl5xM3mAgvvCgb+JrUj20P3nt+ZSdQPmpQAAAkPSURBVFf15oELT7B95tYzDU9e2t1WfnI75y6e4g/xtKxzgLOBjWFlA4EPQo8/BAYjFUpaWhqtHUZhpqUFW9LhrdwC8+4cXDiCM9z8OwfxxZ9PtZU/fVl3rjrlGLJCrfcCvTKDJ4jzuznfDO0a1TumVjXnmYA7t3Tu2963fSNG9mgNRHbBGnJcUz53qOexzesw7057zyCAN8J69YS3zMMDf3gf+3+MKDoxXtKrTeFnoudev6xPW/p2aMSgTk0iyr+5azDXn9aBYSdEzuFerXIlJl3dmzX3R15RAHzx51N59Xe9beXHt6jDqN5tbCfkgmO5pJf9akncU2Kwtiwr17Ksg1HFNS3Lygk93gI0R6SMmtSpTsemtW3lbRvW5K5zjqNSVPohs1FNsicM4xSHm5FTbx7Aa1fbuz1Ouqo3r11lD0hPX9bdFvAAzuoS7EET3Ve9Xujqo3fUup5Og6og2P/eydDOzQpHwoYf3ciebRh/gX3Wx0HHNnW8QmhYqxpvh6VuGoYtGP30ZUWta+ves0hLSyMtLS0i8N93QRc6Nq3NKR0bRZwoAN65ri/jL+jCnWcfG1H+3G96sOq+s20nEIDFfxvC4r8NsZVP/uMpvHiF/Wpr8LFN+HXP1hFdWwtUSU+LeSXWxeFEWzU9tftLJOLoNAhaPKN941qOLem+HRoV9pIJN7Rzc164oqetfKBpQvaEYbaTSJM61Zl28wDHKQbeuuZkpjrkoydd7ZyuGH9+Z7q2qmvrPVSQrojuFllwXGd3iZz9Mdbgp6Gdm1PPIed9Sa82ha3i8FGuI3u24eM/9gegU7Pahfc5urWuFzFqd8hxTQsfh8+fkz1hGPUyqlIvo2rEieJfl5zI8S3qcnqnpjwSlep54tLuTBhxgi1t8+CFJ/DT+LMjupsWWD5uKO9e19dW/vUdpzPj1oG28vEXdOaxkd1s5QBX9M2MGKFcoL/DyQuCPap6OPy9u7V2fxxCWVeK2WeMqRFqcbckMkUiktKic84FnNIFEHuB5t7tGvL+Dfa8ebvGtWL2Jlg69gzHARdfO+TrASb/sT8rNu+1lY85qxP1MqpwTlS6pGPTWgw+tik3RnUl7dCkFl1b1aVP1M3dU7Mac0qHRsxauS2ifGjn5lxzajue+Wo1/cNOkr/q3ooGNatyxUsLGDf8eKqHjuXUrMaFqzllNszgolAaqkHNqky/ZSADH5oOwOr7zi680nr3ur5c8NTXALx2VW8a1apGo9CVxoh/f124z1/3DKaTOjSpxRUvLWDbvmBS4O1r+3JS2/ps3HWQF2evKXx/5UppTAylhW5/+7uI43r3un4AhatOFXhqVHfq1qjC6Q9P55c9ObgiEAjE9V9WVtbYrKysG0KPn83Kyros9PhfWVlZV0W9NzMrKyuwbt26gIhUTLl5+YFtew85vrb4552B/Px8W/nslVsDWx0+M2fVtsCCNdtt5d+t3xV4cdZqW/nGXQcCv5+4MHAgJzei/ODh3MBpD30ZmLNqm+0zAx6YFrj1v4sjyg7n5gUueXZOoO3tHwVWb91XWL59X05gzNtLA21v/ygw5u2lEeXzHeoZr3Xr1gWysrICWVlZmYGoGFxiP2tjzEnAw0AmcATYAIwCXgaqA2uB31qWdSTsM5mon7WIpLD8/AAHjuTFvIFdFsX1sy5xL5ZlfUOw90c0+10EEZEKolKltIQG6hL3V257EhGRMlOwFhHxAQVrEREfULAWEfEBBWsRER9QsBYR8QG3+p2kA2zevNmlzYuIpJ6wmGkbpupWsG4OMGrUKJc2LyKS0poDq8IL3ArWC4D+wCYgz6V9iIikmnSCgXpB9AuuLOslIiKJpRuMIiI+UH4D2+NkjHkU6AMEgD9ZlmW7HPADY0xn4H3gUcuynjDGtAYmErzM2QSMtiwrxxgzCrgRyAeetSzrBWNMFYITZbUlmEb6rWVZq40xXYF/E/zbLLUs69pyP7AYjDEPEEx9VQbuJ3gZl8rHm0Gwzk0JTmg2DlhCCh9zAWNMDWAZwWOeSgofszFmIPBf4PtQ0XfAAyThmD3VsjbGDAA6WpZ1MvA74F9JrlKZxLtuZeh9fyO4LNpA4M/GmAbApcAuy7JOAcYTDH4AjxE8gfUD6hpjziqP4ymJMeY0oHPoextKsJ4pe7wh5wILLcsaAFwMPELqH3OBu4AdoccV4ZhnWJY1MPTfH0jSMXsqWAODgPcALMv6EahvjKlT/Ec8Kd51K3sDCyzL2h1ayGE20I/g3+Hd0HunAP2MMVUJTpu4IGobXvAVcFHo8S6gJql9vFiW9aZlWQ+EnrYG1pPixwxgjOkEHAcUzL4/kBQ/ZgcDScIxey1YNwO2hj3fGirzlVKsWxl9vLZyy7LyCV4qNQN2Orw36SzLyrMsa3/o6e+Aj0nh4w1njPkamETw8rciHPPDwE1hzyvCMR9njPnAGDPLGDOEJB2z14J1tFRd3zHWcZWm3HN/G2PMcILB+oaol1LyeAEsy+oLnAe8SmQdU+6YjTG/AeZYlrUmxltS7piBn4C7geHA5cALRN7rK7dj9lqw3khkS7oFwQR+KtgXujEDRetWRh+vrTx0gyKN4N+hocN7PcEYcybwF+Asy7J2k/rHe1LopjGWZS0m+A94byofMzAMGG6MmQtcBfyVFP+eLcvaEEp5BSzLWgVsJpieLfdj9lqw/hy4EMAY0x3YaFmWfbVPf5oCjAg9HgF8CswDehpj6hljahHMcc0k+HcoyAGfC3wZWjZtuTGmYIXVX4W2kXTGmLrAg8A5lmUV3HhK2eMNORW4GcAY0xSoRYofs2VZIy3L6mlZVh/geYK9QVL6mI0xo4wxt4QeNyPY++clknDMnhsUY4yZQPAfQj5wvWVZS5JcpVIrzbqVxpgLgVsJ5rIetyzrNWNMOsF/DB0J3qy8wrKsdcaY44BnCJ5k51mWdRMeYIz5P2AssCKs+HKCx5ByxwuF3ddeIHhzsQbBS+WFwCuk6DGHM8aMBbKBz0jhYzbG1CZ4T6IeUJXg97yIJByz54K1iIjYeS0NIiIiDhSsRUR8QMFaRMQHFKxFRHxAwVpExAcUrEVEfEDBWkTEBxSsRUR84P8BHpOE3frEOGwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "----\n",
            " gna, (Wushomon't gugirun oonkifr soulg.\n",
            "\n",
            "A5 theies nerer altien nfd oxthac firy hithion no, tout al ames Gedin\n",
            "\n",
            "Sheth was.\n",
            "\n",
            "\n",
            "AHe ane s9nmetre xhepire sirhtiru soiat aver honayd an sinr ond\n",
            "Onigon wins \n",
            "----\n",
            "iter 50000, loss 20.327018\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fr8Xm4dVVgZT"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}