END_P2S3_100_40.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "END P2S3_100_40.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jofyc9OC4Qcf"
      },
      "source": [
        "#Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ahBVnrNc3E0U"
      },
      "source": [
        "import numpy as np\n",
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from IPython import display\n",
        "plt.style.use('seaborn-white')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "crQSAaIz4SkA"
      },
      "source": [
        "# Read and process data. \n",
        "\n",
        "Download the file from this URL: https://drive.google.com/file/d/1UWWIi-sz9g0x3LFvkIZjvK1r2ZaCqgGS/view?usp=sharing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pjG_Gyd6JnU3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d2235464-15ba-448b-8bf9-5527652700f0"
      },
      "source": [
        "!gdown --id 1JBVfScTakEGsfZjLDFPtKx-uVa9n1TYb"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1JBVfScTakEGsfZjLDFPtKx-uVa9n1TYb\n",
            "To: /content/text.txt\n",
            "\r  0% 0.00/10.3k [00:00<?, ?B/s]\r100% 10.3k/10.3k [00:00<00:00, 19.9MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rgOGxPDP3Wpp"
      },
      "source": [
        "data = open('text.txt', 'r').read()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZeXXMLRb4kXb"
      },
      "source": [
        "Process data and calculate indices"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5TKeiOp4jtl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "90128556-2538-448b-f1a8-84bfd02b1f4a"
      },
      "source": [
        "chars = list(set(data))\n",
        "data_size, X_size = len(data), len(chars)\n",
        "print(\"Corona Virus article has %d characters, %d unique characters\" %(data_size, X_size))\n",
        "char_to_idx = {ch:i for i,ch in enumerate(chars)}\n",
        "idx_to_char = {i:ch for i,ch in enumerate(chars)}"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Corona Virus article has 10223 characters, 75 unique characters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4C53MB135LRY"
      },
      "source": [
        "# Constants and Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dfj21ORa49Ps"
      },
      "source": [
        "Hidden_Layer_size = 100 #size of the hidden layer\n",
        "Time_steps = 40 # Number of time steps (length of the sequence) used for training\n",
        "learning_rate = 1e-1 # Learning Rate\n",
        "weight_sd = 0.1 #Standard deviation of weights for initialization\n",
        "z_size = Hidden_Layer_size + X_size #Size of concatenation(H, X) vector"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdmJf4Du5uhb"
      },
      "source": [
        "# Activation Functions and Derivatives"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "seGHei_D5FGk"
      },
      "source": [
        "def sigmoid(x): # sigmoid function\n",
        "  return 1.0 / (1.0 + np.exp(-x)) # write your code here\n",
        "\n",
        "def dsigmoid(y): # derivative of sigmoid function\n",
        "  return y * (1.0 - y)     # write your code here\n",
        "\n",
        "def tanh(x): # tanh function\n",
        "  return np.tanh(x) # write your code here\n",
        "\n",
        "def dtanh(y): # derivative of tanh\n",
        "  return 1.0 - (y * y) # write your code here"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KeCvVH1v6Me-"
      },
      "source": [
        "# Quiz Question 1\n",
        "\n",
        "What is the value of sigmoid(0) calculated from  your code? (Answer up to 1 decimal point, e.g. 4.2 and NOT 4.29999999, no rounding off).\n",
        "\n",
        "# Quiz Question 2\n",
        "\n",
        "What is the value of dsigmoid(sigmoid(0)) calculated from your code?? (Answer up to 2 decimal point, e.g. 4.29 and NOT 4.29999999, no rounding off). \n",
        "\n",
        "# Quiz Question 3\n",
        "\n",
        "What is the value of tanh(dsigmoid(sigmoid(0))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off).\n",
        "\n",
        "# Quiz Question 4\n",
        "\n",
        "What is the value of dtanh(tanh(dsigmoid(sigmoid(0)))) calculated from your code?? (Answer up to 5 decimal point, e.g. 4.29999 and NOT 4.29999999, no rounding off)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iuc7hFzqLeQq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fb906363-d66c-40ff-92ee-65084fe8bcd8"
      },
      "source": [
        "# Quiz Question 1\n",
        "sigmoid(0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sGqN976PLeFg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "344da3f3-9ad9-4718-84ee-ec26fcff5117"
      },
      "source": [
        "# Quiz Question 2\n",
        "dsigmoid(sigmoid(0))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.25"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O68gJ6VHLd98",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e5addd32-e39d-4700-978d-313dd1419eb3"
      },
      "source": [
        "# Quiz Question 3\n",
        "tanh(dsigmoid(sigmoid(0))) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.24491866240370913"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ECRIOdNHLdyK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4f6a09c-a154-4b3a-8101-25a2430c6789"
      },
      "source": [
        "# Quiz Question 4\n",
        "dtanh(tanh(dsigmoid(sigmoid(0))))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.940014848806378"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EeSVipDu8iKE"
      },
      "source": [
        "# Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ICbWNemE6LGV"
      },
      "source": [
        "class Param:\n",
        "    def __init__(self, name, value):\n",
        "      self.name = name\n",
        "      self.v = value # parameter value\n",
        "      self.d = np.zeros_like(value) # derivative\n",
        "      self.m = np.zeros_like(value) # momentum for Adagrad"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j83pZNPE8212"
      },
      "source": [
        "We use random weights with normal distribution (0, weight_sd) for  tanh  activation function and (0.5, weight_sd) for  `sigmoid`  activation function.\n",
        "\n",
        "Biases are initialized to zeros."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swHwLXOI9E7V"
      },
      "source": [
        "# LSTM \n",
        "You are making this network, please note f, i, c and o (also \"v\") in the image below:\n",
        "![alt text](http://blog.varunajayasiri.com/ml/lstm.svg)\n",
        "\n",
        "Please note that we are concatenating the old_hidden_vector and new_input."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A0DBzNY-90s5"
      },
      "source": [
        "# Quiz Question 4\n",
        "\n",
        "In the class definition below, what should be size_a, size_b, and size_c? ONLY use the variables defined above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFuHhqVq6Wge"
      },
      "source": [
        "size_a = Hidden_Layer_size # write your code here\n",
        "size_b = z_size # write your code here\n",
        "size_c = X_size # write your code here\n",
        "\n",
        "class Parameters:\n",
        "    def __init__(self):\n",
        "        self.W_f = Param('W_f', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_f = Param('b_f', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_i = Param('W_i', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_i = Param('b_i', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_C = Param('W_C', np.random.randn(size_a, size_b) * weight_sd)\n",
        "        self.b_C = Param('b_C', np.zeros((size_a, 1)))\n",
        "\n",
        "        self.W_o = Param('W_o', np.random.randn(size_a, size_b) * weight_sd + 0.5)\n",
        "        self.b_o = Param('b_o', np.zeros((size_a, 1)))\n",
        "\n",
        "        #For final layer to predict the next character\n",
        "        self.W_v = Param('W_v', np.random.randn(X_size, size_a) * weight_sd)\n",
        "        self.b_v = Param('b_v', np.zeros((size_c, 1)))\n",
        "        \n",
        "    def all(self):\n",
        "        return [self.W_f, self.W_i, self.W_C, self.W_o, self.W_v,\n",
        "               self.b_f, self.b_i, self.b_C, self.b_o, self.b_v]\n",
        "        \n",
        "parameters = Parameters()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RzmfGLZt_xVs"
      },
      "source": [
        "Look at these operations which we'll be writing:\n",
        "\n",
        "**Concatenation of h and x:**\n",
        "\n",
        "$z\\:=\\:\\left[h_{t-1},\\:x\\right]$\n",
        "\n",
        "$f_t=\\sigma\\left(W_f\\cdot z\\:+\\:b_f\\:\\right)$\n",
        "\n",
        "$i_i=\\sigma\\left(W_i\\cdot z\\:+\\:b_i\\right)$\n",
        "\n",
        "$\\overline{C_t}=\\tanh\\left(W_C\\cdot z\\:+\\:b_C\\right)$\n",
        "\n",
        "$C_t=f_t\\ast C_{t-1}+i_t\\ast \\overline{C}_t$\n",
        "\n",
        "$o_t=\\sigma\\left(W_o\\cdot z\\:+\\:b_i\\right)$\n",
        "\n",
        "$h_t=o_t\\ast\\tanh\\left(C_t\\right)$\n",
        "\n",
        "**Logits:**\n",
        "\n",
        "$v_t=W_v\\cdot h_t+b_v$\n",
        "\n",
        "**Softmax:**\n",
        "\n",
        "$\\hat{y}=softmax\\left(v_t\\right)$\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-bUkseNnDott"
      },
      "source": [
        "def forward(x, h_prev, C_prev, p = parameters):\n",
        "    assert x.shape == (X_size, 1)\n",
        "    assert h_prev.shape == (Hidden_Layer_size, 1)\n",
        "    assert C_prev.shape == (Hidden_Layer_size, 1)\n",
        "    \n",
        "    z = np.row_stack((h_prev, x))\n",
        "    f = sigmoid(np.dot(p.W_f.v, z) + p.b_f.v) # write your code here\n",
        "    i = sigmoid(np.dot(p.W_i.v, z) + p.b_i.v) # write your code here\n",
        "    C_bar = tanh(np.dot(p.W_C.v, z) + p.b_C.v) # write your code here\n",
        "\n",
        "    C = f * C_prev + i * C_bar # write your code here\n",
        "    o = sigmoid(np.dot(p.W_o.v, z) + p.b_o.v) # write your code here\n",
        "    h = o * tanh(C) # write your code here\n",
        "\n",
        "    v = np.dot(p.W_v.v, h) + p.b_v.v # write your code here\n",
        "    y = np.exp(v) / np.sum(np.exp(v)) #softmax\n",
        "\n",
        "    return z, f, i, C_bar, C, o, h, v, y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jZrDhZIjFpdI"
      },
      "source": [
        "You must finish the function above before you can attempt the questions below. \n",
        "\n",
        "# Quiz Question 5\n",
        "\n",
        "What is the output of 'print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))'?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aUJxybI9TSYX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0906a1f6-620c-4d36-c2a6-eec076731741"
      },
      "source": [
        "# Quiz Question 5\n",
        "print(len(forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)), parameters)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "9\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XV-YVl_GGiX8"
      },
      "source": [
        "# Quiz Question 6. \n",
        "\n",
        "Assuming you have fixed the forward function, run this command: \n",
        "z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))\n",
        "\n",
        "Now, find these values:\n",
        "\n",
        "\n",
        "1.   print(z.shape)\n",
        "2.   print(np.sum(z))\n",
        "3.   print(np.sum(f))\n",
        "\n",
        "Copy and paste exact values you get in the logs into the quiz.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1GvKVWmTDt3H"
      },
      "source": [
        "z, f, i, C_bar, C, o, h, v, y = forward(np.zeros((X_size, 1)), np.zeros((Hidden_Layer_size, 1)), np.zeros((Hidden_Layer_size, 1)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mmsoAG4xTnve",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a29fc561-bde1-4dfb-b280-b8db489fd0bd"
      },
      "source": [
        "# Quiz Question 6\n",
        "print(z.shape)\n",
        "print(np.sum(z))\n",
        "print(np.sum(f))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(175, 1)\n",
            "0.0\n",
            "50.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NeSvhkqwILsG"
      },
      "source": [
        "# Backpropagation\n",
        "\n",
        "Here we are defining the backpropagation. It's too complicated, here is the whole code. (Please note that this would work only if your earlier code is perfect)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zIa1jUZiGPmF"
      },
      "source": [
        "def backward(target, dh_next, dC_next, C_prev,\n",
        "             z, f, i, C_bar, C, o, h, v, y,\n",
        "             p = parameters):\n",
        "    \n",
        "    assert z.shape == (X_size + Hidden_Layer_size, 1)\n",
        "    assert v.shape == (X_size, 1)\n",
        "    assert y.shape == (X_size, 1)\n",
        "    \n",
        "    for param in [dh_next, dC_next, C_prev, f, i, C_bar, C, o, h]:\n",
        "        assert param.shape == (Hidden_Layer_size, 1)\n",
        "        \n",
        "    dv = np.copy(y)\n",
        "    dv[target] -= 1\n",
        "\n",
        "    p.W_v.d += np.dot(dv, h.T)\n",
        "    p.b_v.d += dv\n",
        "\n",
        "    dh = np.dot(p.W_v.v.T, dv)        \n",
        "    dh += dh_next\n",
        "    do = dh * tanh(C)\n",
        "    do = dsigmoid(o) * do\n",
        "    p.W_o.d += np.dot(do, z.T)\n",
        "    p.b_o.d += do\n",
        "\n",
        "    dC = np.copy(dC_next)\n",
        "    dC += dh * o * dtanh(tanh(C))\n",
        "    dC_bar = dC * i\n",
        "    dC_bar = dtanh(C_bar) * dC_bar\n",
        "    p.W_C.d += np.dot(dC_bar, z.T)\n",
        "    p.b_C.d += dC_bar\n",
        "\n",
        "    di = dC * C_bar\n",
        "    di = dsigmoid(i) * di\n",
        "    p.W_i.d += np.dot(di, z.T)\n",
        "    p.b_i.d += di\n",
        "\n",
        "    df = dC * C_prev\n",
        "    df = dsigmoid(f) * df\n",
        "    p.W_f.d += np.dot(df, z.T)\n",
        "    p.b_f.d += df\n",
        "\n",
        "    dz = (np.dot(p.W_f.v.T, df)\n",
        "         + np.dot(p.W_i.v.T, di)\n",
        "         + np.dot(p.W_C.v.T, dC_bar)\n",
        "         + np.dot(p.W_o.v.T, do))\n",
        "    dh_prev = dz[:Hidden_Layer_size, :]\n",
        "    dC_prev = f * dC\n",
        "    \n",
        "    return dh_prev, dC_prev"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tnc7WpRkIU5S"
      },
      "source": [
        "# Forward and Backward Combined Pass\n",
        "\n",
        "Let's first clear the gradients before each backward pass"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJWoC3U1ITf8"
      },
      "source": [
        "def clear_gradients(params = parameters):\n",
        "    for p in params.all():\n",
        "        p.d.fill(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7XN93UnjIgmA"
      },
      "source": [
        "Clip gradients to mitigate exploding gradients"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0LTsublxIfFl"
      },
      "source": [
        "def clip_gradients(params = parameters):\n",
        "    for p in params.all():\n",
        "        np.clip(p.d, -1, 1, out=p.d)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T7XUpDTWIl_Y"
      },
      "source": [
        "Calculate and store the values in forward pass. Accumulate gradients in backward pass and clip gradients to avoid exploding gradients.\n",
        "\n",
        "input, target are list of integers, with character indexes.\n",
        "h_prev is the array of initial h at  h−1  (size H x 1)\n",
        "C_prev is the array of initial C at  C−1  (size H x 1)\n",
        "Returns loss, final  hT  and  CT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQNxjTuZIia_"
      },
      "source": [
        "def forward_backward(inputs, targets, h_prev, C_prev):\n",
        "    global paramters\n",
        "    \n",
        "    # To store the values for each time step\n",
        "    x_s, z_s, f_s, i_s,  = {}, {}, {}, {}\n",
        "    C_bar_s, C_s, o_s, h_s = {}, {}, {}, {}\n",
        "    v_s, y_s =  {}, {}\n",
        "    \n",
        "    # Values at t - 1\n",
        "    h_s[-1] = np.copy(h_prev)\n",
        "    C_s[-1] = np.copy(C_prev)\n",
        "    \n",
        "    loss = 0\n",
        "    # Loop through time steps\n",
        "    assert len(inputs) == Time_steps\n",
        "    for t in range(len(inputs)):\n",
        "        x_s[t] = np.zeros((X_size, 1))\n",
        "        x_s[t][inputs[t]] = 1 # Input character\n",
        "        \n",
        "        (z_s[t], f_s[t], i_s[t],\n",
        "        C_bar_s[t], C_s[t], o_s[t], h_s[t],\n",
        "        v_s[t], y_s[t]) = \\\n",
        "            forward(x_s[t], h_s[t - 1], C_s[t - 1]) # Forward pass\n",
        "            \n",
        "        loss += -np.log(y_s[t][targets[t], 0]) # Loss for at t\n",
        "        \n",
        "    clear_gradients()\n",
        "\n",
        "    dh_next = np.zeros_like(h_s[0]) #dh from the next character\n",
        "    dC_next = np.zeros_like(C_s[0]) #dh from the next character\n",
        "\n",
        "    for t in reversed(range(len(inputs))):\n",
        "        # Backward pass\n",
        "        dh_next, dC_next = \\\n",
        "            backward(target = targets[t], dh_next = dh_next,\n",
        "                     dC_next = dC_next, C_prev = C_s[t-1],\n",
        "                     z = z_s[t], f = f_s[t], i = i_s[t], C_bar = C_bar_s[t],\n",
        "                     C = C_s[t], o = o_s[t], h = h_s[t], v = v_s[t],\n",
        "                     y = y_s[t])\n",
        "\n",
        "    clip_gradients()\n",
        "        \n",
        "    return loss, h_s[len(inputs) - 1], C_s[len(inputs) - 1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tcy5u_vRItkV"
      },
      "source": [
        "# Sample the next character"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p8SrtJiwIsSm"
      },
      "source": [
        "def sample(h_prev, C_prev, first_char_idx, sentence_length):\n",
        "    x = np.zeros((X_size, 1))\n",
        "    x[first_char_idx] = 1\n",
        "\n",
        "    h = h_prev\n",
        "    C = C_prev\n",
        "\n",
        "    indexes = []\n",
        "    \n",
        "    for t in range(sentence_length):\n",
        "        _, _, _, _, C, _, h, _, p = forward(x, h, C)\n",
        "        idx = np.random.choice(range(X_size), p=p.ravel())\n",
        "        x = np.zeros((X_size, 1))\n",
        "        x[idx] = 1\n",
        "        indexes.append(idx)\n",
        "\n",
        "    return indexes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SiWFaWLNIx_L"
      },
      "source": [
        "# Training (Adagrad)\n",
        "\n",
        "Update the graph and display a sample output\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ENQYU-7AIw0t"
      },
      "source": [
        "def update_status(inputs, h_prev, C_prev):\n",
        "    #initialized later\n",
        "    global plot_iter, plot_loss\n",
        "    global smooth_loss\n",
        "    \n",
        "    # Get predictions for 200 letters with current model\n",
        "\n",
        "    sample_idx = sample(h_prev, C_prev, inputs[0], 200)\n",
        "    txt = ''.join(idx_to_char[idx] for idx in sample_idx)\n",
        "\n",
        "    # Clear and plot\n",
        "    plt.plot(plot_iter, plot_loss)\n",
        "    display.clear_output(wait=True)\n",
        "    plt.show()\n",
        "\n",
        "    #Print prediction and loss\n",
        "    print(\"----\\n %s \\n----\" % (txt, ))\n",
        "    print(\"iter %d, loss %f\" % (iteration, smooth_loss))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ACXcASJuI73a"
      },
      "source": [
        "# Update Parameters\n",
        "\n",
        "\\begin{align}\n",
        "\\theta_i &= \\theta_i - \\eta\\frac{d\\theta_i}{\\sum dw_{\\tau}^2} \\\\\n",
        "d\\theta_i &= \\frac{\\partial L}{\\partial \\theta_i}\n",
        "\\end{align}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bR08TvcjI4Pf"
      },
      "source": [
        "def update_paramters(params = parameters):\n",
        "    for p in params.all():\n",
        "        p.m += p.d * p.d # Calculate sum of gradients\n",
        "        #print(learning_rate * dparam)\n",
        "        p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "La9vyJ6RJLFK"
      },
      "source": [
        "To delay the keyboard interrupt to prevent the training from stopping in the middle of an iteration\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZVDHbMb7JNGT"
      },
      "source": [
        "# Exponential average of loss\n",
        "# Initialize to a error of a random model\n",
        "smooth_loss = -np.log(1.0 / X_size) * Time_steps\n",
        "\n",
        "iteration, pointer = 0, 0\n",
        "\n",
        "# For the graph\n",
        "plot_iter = np.zeros((0))\n",
        "plot_loss = np.zeros((0))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HF6vS0VWJqsS"
      },
      "source": [
        "# Training Loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQyNSL0iJOxH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 415
        },
        "outputId": "c1fbba47-f4d2-4b14-85a8-58512ac29465"
      },
      "source": [
        "iter = 50000\n",
        "while iter > 0:\n",
        "  # Reset\n",
        "  if pointer + Time_steps >= len(data) or iteration == 0:\n",
        "      g_h_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      g_C_prev = np.zeros((Hidden_Layer_size, 1))\n",
        "      pointer = 0\n",
        "\n",
        "\n",
        "  inputs = ([char_to_idx[ch] \n",
        "              for ch in data[pointer: pointer + Time_steps]])\n",
        "  targets = ([char_to_idx[ch] \n",
        "              for ch in data[pointer + 1: pointer + Time_steps + 1]])\n",
        "\n",
        "  loss, g_h_prev, g_C_prev = \\\n",
        "      forward_backward(inputs, targets, g_h_prev, g_C_prev)\n",
        "  smooth_loss = smooth_loss * 0.999 + loss * 0.001\n",
        "\n",
        "  # Print every hundred steps\n",
        "  if iteration % 100 == 0:\n",
        "      update_status(inputs, g_h_prev, g_C_prev)\n",
        "\n",
        "  update_paramters()\n",
        "\n",
        "  plot_iter = np.append(plot_iter, [iteration])\n",
        "  plot_loss = np.append(plot_loss, [loss])\n",
        "\n",
        "  pointer += Time_steps\n",
        "  iteration += 1\n",
        "  iter = iter -1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAD1CAYAAACm0cXeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUVcLH8W8SCIQiiIUg8IpYDtbdFbEhim1BsewurruvrKuru66+q69l3X1R7GXtytpFsZdVsYE06VUg0oscIEgJISShhPQyM+8fdyaZJJNkksxkcoff53l4mLn3zr3nJpnfnDnn3HMTfD4fIiLiTomxLoCIiDSdQlxExMUU4iIiLqYQFxFxMYW4iIiLtWnJgxlj2gEDgJ2ApyWPLSLiUklADyDNWltac2WLhjhOgM9r4WOKiMSDQcD8mgtbOsR3Anz00Uekpqa28KFFRNwnKyuLESNGgD8/a2rpEPcApKam0qtXrxY+tIiIq4VsglbHpoiIiynERURcTCEuIuJiCnERERdTiIuIuJhCXETExVwT4sfcO4mnpqyPdTFERFqVBseJG2M6AO8C3YH2wKPASuADnMtBdwLXWmtLjTEjgDsALzDGWjs2UgWt8Pp4bXY6/ze0X6R2KSLieuHUxC8HfrDWngdcDTwPPAK8Yq0dBGwCbjDGdAQeAC4CBgN3GmO6RaXUIiIChFETt9Z+GvS0N5CBE9I3+5dNAO4GLM4ELXkAxpgFwED/ehERiYKwL7s3xiwEegGXAdODZtPKxplhKxXICXpJYLmIiERJ2B2b1tqzgSuAD4GEoFUJoV9R53IREYmQBkPcGNPfGNMbwFq7Aqf2nm+MSfFv0hPI9P8LnpowsFxERKIknJr4ucDfAYwx3YFOwHRguH/9cGAKsBgYYIzpaozphNMerrnDRUSiKJwQfx043BgzD5gI/A14ELjOv6wb8J61thgYCUzFCfmHA52cIiISHeGMTikGrgmx6uIQ244DxkWgXCIiEgbXXLEpIiK1KcRFRFxMIS4i4mIKcRERF1OIi4i4mEJcRMTFFOIiIi6mEBcRcTGFuIiIiynERURcTCEuIuJiCnERERdTiIuIuJhCXETExRTiIiIuphAXEXExhbiIiIspxEVEXEwhLiLiYgpxEREXU4iLiLiYQlxExMUU4iIiLqYQFxFxsTbhbGSMeRoY5N/+CeAKoD+w27/JM9baicaYEcAdgBcYY60dG/kii4hIQIMhbow5HzjJWnuWMeYQYDkwE7jHWvtt0HYdgQeA04EyIM0Y85W1dk90ii4iIuE0p8wFfut/vA/oCCSF2O4MIM1am2etLQYWAAMjUkoREQmpwZq4tdYDFPqf3ghMAjzArcaYu4Bs4FYgFcgJemk20COipRURkWrC7tg0xlyJE+K3Ah8AI621FwArgIdCvCQhEgUUEZG6hduxOQQYBQy11uYBM4JWjwdeA8bh1MYDegKLIlROEREJocGauDGmC/AMcFmgk9IY84Uxpq9/k8HAGmAxMMAY09UY0wmnPXxeVEotIiJAeDXx3wGHAp8ZYwLL3gE+NcYUAQXAn6y1xcaYkcBUwAc87K+1i4hIlITTsTkGGBNi1Xshth2H06wiIiItQFdsioi4mEJcRMTFFOIiIi6mEBcRcTGFuIiIiynERURcTCEuIuJiCnERERdTiIuIuJhCXETExRTiIiIuphAXEXExhbiIiIspxEVEXEwhLiLiYgpxEREXU4iLiLiYQlxExMUU4iIiLqYQFxFxMYW4iIiLKcRFRFxMIS4i4mJtwtnIGPM0MMi//RNAGvABkATsBK611pYaY0YAdwBeYIy1dmxUSi0iIkAYNXFjzPnASdbas4ChwGjgEeAVa+0gYBNwgzGmI/AAcBEwGLjTGNMtWgUXEZHwmlPmAr/1P94HdMQJ6fH+ZRNwgvsMIM1am2etLQYWAAMjWloREammweYUa60HKPQ/vRGYBAyx1pb6l2UDPYBUICfopYHlIiISJWG1iQMYY67ECfFfAhuDViXU8ZK6louISISENTrFGDMEGAVcYq3NAwqMMSn+1T2BTP+/1KCXBZaLiEiUhNOx2QV4BrjMWrvHv3g6MNz/eDgwBVgMDDDGdDXGdMJpD58X+SKLiEhAOM0pvwMOBT4zxgSWXQe8ZYz5K7AVeM9aW26MGQlMBXzAw/5au4iIREk4HZtjgDEhVl0cYttxwLgIlEtERMKgKzZFRFxMIS4i4mIKcRERF1OIi4i4mEJcRMTFFOIiIi6mEBcRcTGFuIiIiynERURcTCEuIuJiCnERERdTiIuIuJhCXETExRTiIiIuphAXEXExhbiIiIspxEVEXEwhLiLiYgpxEREXU4iLiLiYQlxExMUU4iIiLqYQFxFxsTbhbGSMOQn4BnjBWvuyMeZdoD+w27/JM9baicaYEcAdgBcYY60dG4Uyi4iIX4MhbozpCLwEzKix6h5r7bc1tnsAOB0oA9KMMV9Za/dEsLwiIhIknOaUUuBSILOB7c4A0qy1edbaYmABMLCZ5RMRkXo0WBO31lYAFcaYmqtuNcbcBWQDtwKpQE7Q+mygR4TKKSIiITS1Y/MDYKS19gJgBfBQiG0SmlooEREJT1gdmzVZa4Pbx8cDrwHjcGrjAT2BRU0vmoiINKRJNXFjzBfGmL7+p4OBNcBiYIAxpqsxphNOe/i8iJRSRERCCmd0Sn/gOaAPUG6MuQpntMqnxpgioAD4k7W22BgzEpgK+ICHrbV5USu5iIiE1bG5FKe2XdMXIbYdh9OsIiIiLUBXbIqIuJjrQnxzTkGsiyAi0mq4LsQveG4O09ftinUxRERaBdeFOIDdlR/rIoiItAquDHEREXG4MsQrPL5YF0FEpFVwZYiLiIjDlSHuQzVxERFwa4grw0VEALeGeKwLICLSSrgyxFUVFxFxuDLEvcpwERHApSGujk0REYcrQ/yVWemxLoKISKvgyhAXERGHQlxExMVcG+I+jVAREXFziMe6BCIisefaEJ+xPhuvxhqKyAHOtSH+l/d/4N2FW2JdDBGRmHJtiAN8mrY91kUQEYkpV4e43ZXPje+mxboYIiIx4+oQB6dt/KPFWyn3eGNdFBGRFuf6EAcY9dUa3pr3U6yLISLS4tqEs5Ex5iTgG+AFa+3LxpjewAdAErATuNZaW2qMGQHcAXiBMdbasVEqdy1T1mbRu1sKl51yREsdUkQk5hqsiRtjOgIvATOCFj8CvGKtHQRsAm7wb/cAcBEwGLjTGNMt4iWuw8rt+7j14+UtdTgRkVYhnOaUUuBSIDNo2WBgvP/xBJzgPgNIs9bmWWuLgQXAwMgVNTynPTadRZt3t/RhRURiosEQt9ZW+EM5WEdrban/cTbQA0gFcoK2CSxvUbkFpTz4zVo25xS09KFFRFpcJDo2Exq5POrsrnwueG5OrA4vItJimhriBcaYFP/jnjhNLZk4tXFqLBcRkShpaohPB4b7Hw8HpgCLgQHGmK7GmE447eHzml9EERGpS4NDDI0x/YHngD5AuTHmKmAE8K4x5q/AVuA9a225MWYkMBXnhvQPW2vzolbyMMzfmMsJRxxEt47JsSyGiEjUNBji1tqlOKNRaro4xLbjgHHNL1Ztl//sCCasbFzrzB/GLqZfamem3HFuNIokIhJzrrli808D+zTpdZuyNUpFROKXa0I8MaFpg10qvD5GT99Aucer+cdFJO64JsRN985Nfu3o6Rs5dtRkrteMhyISZ1wT4inJSc3ex9wNObwwbQOLNu9mVca+CJRKRCS2wpoAK578e8ZG/j1jIwBbnhwW49KIiDSPa2riIiJSm0JcRMTFFOIiIi7mqhD/8zlHRXR/T05eT1FZBQDrMvfrFm8i4jqu6tg8rU833pofuduwvT4nndfnpJN6UHuy9pdw07l9uffS4yO2fxGRaHNVTfywztGZAyVrfwkA45Zm8NXyjKgcQ0QkGlwV4qf+18FR3f+ewjLu/HRlVI8hIhJJrgrxhCZeet8ceUXllJR7Wvy4IiLhcFWIt5Qb3k1jts0G4GePfMdvX/8+xiUSEQlNIR7CzPXZXP9OGpe/NB+A1Tvy6DNyItn5JTEumUh8KiytYF3m/lgXw5UU4vVYvaP6PS1WbNN8KyLRcPOHS7n0xXlqumwChXgTZOeX8PXyHVE/zv98tJS/fbws6scRaaq7PlvBzx/5rtn7+WHLXgC8Pk0X3ViuGicOzqRVfUZOjMmxN2YXMOrr6eTklwJw7nGHRfXWb5NWZwHwyjVRO4RIs3y5LLKVGWV446km3gjPTLWVAQ6wYFMuqzL2cc5TM8krKq/3tYWlFfqqKFKHGAw8ixsK8Wa47ZPlXPHyAjL2FvP95t2Ue7y8MmtTyLA+8cGpDHp6VgxKKeIeqog3nuuaU1qrmz9cSofkJIrKPFR4fNx+0bEAPPbtOt5e4EwVEFyLD5a5r5hO7dtwUPu29R6joLQCj9dHl5T6txNxG1XEm04hHkFFZU4NfOueQjbnFPDq7HTGLW34Mv6zn5xJz64pLBh5Qb3b9X90GqUVXt3MQuKWT43ijaYQj4Ivl+2os8MnK6+EdxduoXP7Njz3nWXkJf0A2LGvuMH9llZolkWJT7G4GjteNCnEjTGDgc+Btf5Fq4GngQ+AJGAncK21NnT7QTNdenJq5cgNtznziRnVnv9r0voYlUSk9VE9vPGa07E5x1o72P/vNuAR4BVr7SBgE3BDREoYwqUn94jWrg8YXq9Po2Wk1VA9vOkiOTplMDDe/3gCcFEE913NZaccwfpHh0Zr9zEz4PHp3PPlaqauzaK4LLoB+8x3ln73T2n0cT5N20afkRMpLK1o1Ou8Xl/lDThE6hJOk/jCTbn0GTmRjL1F0S+QCzSnTfwEY8x4oBvwMNAxqPkkG4hqdbl926Ro7j4mcvJL+WTJNj5Zso0z+3artm5PYVmdr/P5fNhd+fRLPSjsY33+w3YA8kvLSUkO/2f52ux0ALLzSzmqXfh/Pv+a9CNvzf+JHx8Z2qjjyQGiEVXxT9Kcv92lW/fS6+AOUSqQezS1Jr4RJ7ivBK4DxlL9A0Hfjppp0eY9lY9LKzxc/UbVTIo2K59vVlR1nH65bAdDR89j8uqdPDVlfa3A/3JZBu8t3FLjCE37FTW1zfJL/zQFhaqNt7ituwvZX1L/xWitRhh/YBrBUl2TQtxau8Na+6m11metTQeygIONMSn+TXoCmZEq5IHO3DeFTdkFlc+HjJ7L7f9ZUfl8fZYz+9stHy3jtdnpnProtGrNJHd9tpIHx68lpEa+HwLvn8Z+BAS21/uv6fKKy7nrsxXsLihl2IvzWLp1b1ivO++Z2fzq5QVRLl3zNObvKfAnpBEtjiaFuDFmhDHmbv/jVKA78A4w3L/JcGBKREpYj78MiuyNk90or6icN+fVvu/oSQ9NxeutOzHD/fvPKyrn+/TdYb9+1vrsyqaaphwv2nw+H1PWZNX7s2mtXp29iS+X7eDer1azNnM/j0yo44M5hM25hVEsmfNz3VLPMbL3hzdpnC+sqrjzX2Ir+ZuKtaY2p4wHzjPGzAO+AW4BRgHX+Zd1A96LTBHrNmrYCaTEYdt4uPqMnMjP6phBzuP10ffeSWzfU73z56fcQuZsyKl8HnjLjJ3/E09NqT3c8c/vp/Hfby6q7JRs6E32p3fT+Me4VXWuD379jB938Ye3Ftfb3h9pXy7bwc0fLuW977e02DFDWbp1DzvzGr42IJQEf721NX0Mfb40g8HPzg75gQ9w3Ttp3PHpCvYVNf93HZjpMKGZrbZPTPqRgU/ObHZ5Yq1JHZvW2nzg8hCrLm5ecRqvtdTwWqua87Wc/+zsas/L/BcQPfrtOgAuO6UHJx7RhXkbc8jeX0qaf4rQco/zxgk0h+QWlHHeM7O555J+/PW8o8Moif8XFZQ8N773AwAjv1jFmD+e1oizatiElZmc3+9wOtXofM32T32Qldf4G3ysythHjy4pHNa5XVjbf7E0g1+e2J3OIaZTGP7a9yS3SWTDY5eEffyaoRXNpimblU/3g9rRtUN4s3Su2O7Mtb8pp4Czjj6k1vpd/puReyLwDWjyGucakQ278hlWY/xESbmHe79czchL+3F45/b17ueNuZubXZbWwPUTYJ3Wp1vDGwkA366q3U0x6OlZ3PLh0srnw1507mZ07dgl/P3zum8avSk7H4AnJlfV3iesrLsbJKF2hlcqasZwyrkbcmp9TV+bmcdtnyznni9X11kOgDfmpLNtd/jD1K54eQG/fGFOWNuu2ZHH3z9fycgvapchoKyRV+AGyl71s4xeig8ZPZcrGtGO3tS+koCmtG//uLP2nYAmrMzky+U7eGqybWJJ3Mf1If7S739B30M7xroYrnDrx8tDLg/UbAJCjb8trfCwt7Cs8s06d2Nu5brr3l5CSbmnWogPe3FetXHh9XVsNub9W/MCpT++vYQ7Pl0RcpsJKzP5ePG2kPvZXVjGE5PXM2LsovAPDuxtYMrhmmXI2t+4Gv+Hi7byyqxNIdfV/DF5mzkLw7R1u8grrvt8tu1p/Djs5n4zbsy3i1DHak1NTC3F9SHepUNbZt49ONbFiCvnPFV7ytzTH5/BLx6dVjnHS/Ct6uZsyGH5tn0kBfU0rc3cz7R1u/D5fNz8wdLKZoyZ67N5dqoNq2OxsLSCC5+bTZ+REykp9/BZ2nb63T+l3g40qF6ru/er0DXhRZudttuiUg8L03P5NC102DdVoAiNvVPNfV+v4ZmpoWuRtWviTbdjXzF/ef8Hbv9P6A/2xqu/NA0NC2xK+CeF6tk8ADs9NQGWNEnNCbv++83aNdrb/7OCdTv3M2VtVU0/EKpHdE2pXJa5r5jR0zfw61/05KD2bfH6fLRJSuSRCetIz3ECu6jMw1T/fjZmF9Cnnm9fiWEkQsbeqvJf8+ZiAH434L8afF04nMDydz5GsWrYnPHSgSGoDTUn9X90Gkvvv5i8onIen7SOBy8/kY4hLvKqak5pXno25oxCNcEEmpgOpL6yuAnx1/9wKl8u28F363bFuigS5I05oTuP9gaNUkjPKWT09I2Mnr6xzv0kEFQD9fnwen2VFxCF2jbY1a9/T8+DU/B4fZx4RPWrWiOdsY99u4635lcN+QwnaD9Zso3DO7fjwuO717tdcwOyuvAasXf7Rw69PGsjn/2QwTGHd+Kmc2t3ZFeGeB37a6jNuylnNnHVzlq3LozUh4mbxE2IDz2pB5CgEHeJupoM6uJkQNXQur73Tqq2funWvZzQ4yBSkpNq1cSXbNkDW5zHfQ+rXoNvTm12YXouqzLyuDlodE5wgAfK2pBAB2xD88RXNqf4fw7b9xSRX1IecvTLpuwCSso9nNSzS8h9NbUjsqEfV3Ojs7lXY3ob+DBpqk3ZBVz0/Bzu/uVx3HrBsZHdeTO5vk082MBjDuH4HuHPHyLuMezF+ZVvzOApBwKGv7aQ4x+YwsL0XKatq3ua4pq1/eCOyrIKL7kFpUxavbPB8vh8Pq55czFPTq5/KmGvz5ktMpJj4Sf6y1dY5uHKV5wRJAWlFYwP6li+6Pk5XPbS/Ab3Fe6okMB2dUVsc0fKROrqy6rmlMimeKDP5NnvNlDh8Tb6w+b1OemMnr4homUKiKsQ79y+LZNvHxTrYkgU7NhXzDT/t6z65pK/5s3FvDgz9OiOhlz/zhKuf2cJ//PRsnpHbeSXlPP+91srn3/+w3aOvndSyAtZiso8nPPUTE59dFqTyhRQ7vFWa4IK2OzvMzjv6Vn87yfLWbgpt9Y2oQQiKHg6h/o0NG1CQ80pgdDLLag6h5oXogWXK1x7C8tYtq1q+oGGyhEJx4yaHGIuovo9OXl9vc2FzRFXIR4w++7B/Pv3P491McRlFqbvZs0OZ+zxvI3OVa3jlmawbXdRteGSJz/0XbW5aP4xbhUer4/BNS6kAidkA8H18syNbM4pYHoTmvz+OW4VHy6qewRNoO16yZY9dW4TrLGtFoFRNg3VuOtqiw6MiR/xltOJPGXNTgY9PYuZ63f5X9c0w19fyG9eXcjOvGJ27S+pmlelifurS82afaipLmIlbtrEg/U5tCNHHtKBNTvyWtUPW9zj1o+XM+zkHtxdzwVPNe1rYAz5s99t4KPF29gZxtWif/9sJV8sy2DczWdxWp9u9V5IteSnquAu99Q9eLy4zMNsm83pR3VrVPNHucdb+T6qK/xnrs+udx8V3kBN3BlquiojD4Afd+ZzQb/6O3TrE/gmctYTzuXz9w07HohsTdzj9dVqPgnndootJS5r4uB8co4adgJvXNuf0448ONbFERc66p5JDW/USDUD/LO0qsnCFgQ1hXyxzLnB9lWvf8/C9NzKEAwleJriV2als9ofkDU9OH4Nt3y0jP6PTSctKPjv/nwlz9bT0fz4xB9rLcsrLq/27STwTWDq2tBNXTXHywc/m78xl3z/TUaaOyTzMX9ZIzk65eh7JzW5MrinsIwNu/IjVpZQ4rImHmzIiakMOTEVgA8WbeX+r9fEuEQiVf75RdVkYYGmhpoC49jDdfnLVR2az0+r6kzbvqeq9hhoNgKnyQjg7iGG7P0l/L7GmP/l26su7PL5fCzevJvfjQl9pesMf43c5/Ph8fpITEggMTGhWjgvD2rDzi0o5Q9ja5/fzrxiunduT2LQVTtrduTVGiJal0jUxF+asbHauTfFkNFzycmPyq2GK8VtTTyUa888ki1PDmPEGZG5qEOktXtxRlVn2vebq2YYLAhxc44nJv3I6f+aUdlEEVAaNNXBs99tqHbxVl3Gzv+JY0ZNrhwKGlwT//WrCytDveY0Cj6cKW3PemImr81Jr1y+YFMul700n79+sJRwBGf4puwCKuppZgrF5/Px3LQN9TYTbd1d2OAolWgHOBxgIR7w+K9PZux1zqx53TqGN0ubSDyZuKr2MMq6ZvVbn1W9OeCdBVvq3fdpj02vbNYAZx6dmq1BH3zv7KPm/C9eb1V7c/C1BIF5XMK9DiTQEbklt5CLnp/Ds99VfSPx+Xy8OrtqBNND49dydI1pm0vDmJzsvGdmM6aemRB3F0Q/wOEADXGAC4/vzpYnh7Hs/otJ/9eldO1Q+4IJEWm83BrhtTaz9myDhf7L/r+uMeb/zCdmVKtFn/zg1IiUJS1o1M6cDTk8PaXqA+LdhVvweH1MXrOT9Vn7Ka0If1bNwN2V7v96DW8HXei1JbeQ/o9Nb1bZwxX3beLhSEpMYMUDv2RTdgFT1uykTVJigxdxiEjzharxXhPUN5BfWsFHi7cy6qvG9WWV+ZtPgiciKyn30CYxgVs+XBbyNZn7Shg6eh4Avzm1Z1jHKSrzUFLu4YNFznUDPQ9OIXNfMUcf1qlR5W2OhJa86agxpg/w04wZM+jVq1eLHbcpiss8ZOYV8/sxi1qkXUtEImvByAtiduee9244neveXlJreUNTK4SSkZHBhRdeCHCUtXZLzfWqidchJTmJow/rRNqoi6jwePnshwyu6t+LCq+XWz9eTnJSYlgdPCISG7G89VqoAI8WhXgY2iQlco1/REsyibx9/QDAqa17fT5WZuzj7flbuPxnPbBZ+bw6O72+3YmIRIxCvBlSkp2bNJ999KGcffShlcv/ObQfm3MKaNc2iZ5dU0jbsoefcgtJSkhgzNzN2CgP/heRA4dCPEr6BnVsDOjTjQH+e4EO79+L4jIPuQWl/HPcKo7r3onz+x3Of5ZsZ33WfrYETdLfITmpWfefFJH4pxCPgZTkJHp368AnN51ZuWywObzycUFpBe3aJNI2KZE/vr2EYw/vxH3DjmfymixWZeRx2wXH8ItHp1FW4WXOPwYzdPQ8issV9iIHIoV4K9Qp6PZX799weuXjS0/uwaUn9wBg7cNDqPD4SElOYvytA5ltc/jLuX3ZXVDKV8t3kNwmkbyicp6btoHHfnUSHq+PB8ev5dErT+TIQzryx7eX0C+1M22TElm9w5lrI6VtUuWHwV8GHVU5X8SdFx3HC1GaC1lEmkch7lJtkxJp6zTJc2z3zhzbvTMAh3Rqx58H9a3c7rYLq+5Cct3ZfSofBw91Wp2RxxFd23NIp3bkFZeTnJRISnISo4adULlNjy7t+WHrHrbkFnHHxcdy9tGHsn1PEd06JtOxXRu+T99Nx3ZJpG3ZS35JOXnF5WTsLWbYyT14bXY6dlc+px15MB6fj+XbmjcfhYhUiXiIG2NeAM7EmajsdmttWqSPIZF1cq+qW3h1SQl95erVA3pz9YDe1Zb17tah8vFZRx8CwCm9utZ67a9+0ZOc/FIO7tCWNkmJLNyUS+9uHSgq8zBj/S56HdwB070zJrVzrddm7itmX1E5/56xgXOPO4zEhAQmr8nikpNSKfd4eeCbtdx+4bFk55cy5MTuXP9O1Z/b/15wDH8+ty+vzkrnsM7tePTbdRx1aEcOSmnLymZObCTSWkT0Yh9jzHnAP6y1lxljjgfettaeFbS+Dy652EcOHIFJmErLvczekM2ZfQ8hMSGBwzq3q9wmv6Sc/JIKnpqyntsvPJbEhAQW/7SbHl1SmLhqJ16fD5PamQmrdnL0YR0Z0Kcbx3XvxMMT1nFElxQGm8N4YfoGdu0v5cy+3fxTlBbwf0P70a5NIo98u47kNomVN0+Q+BSNi30iHeKPANustW/5n68HTrfW7vc/74NCXKRBJeUekhITaJuUSEm5B5/P6RDfX1JOUamH1C7tK7fdvqeIg9q3ZfveInp2Tan8MEhuk4gPp4+lpNxDfklFtQ+mrbsLOaRTO5KTEiksrSC/pIL/OqQD09btwuP1+m8+7libmUdyUiJrM/ezYvs+zjnmUC7odzgZe4vZXVjKz3t35d2FWygq87BtdxE3ndeXdm2c7dskJrD4pz10SWnLKb26kPbTHk7p1ZWDOyZzbPdOTFq1k/u+XsPXfxvIhFWZjJm7uda84sf3OIjDOrcjK6+YDbucW8od170TmftKKCitPSNja3Rm327856azGt6whpYO8THARGvtN/7n84AbrbUb/M/7oBAXEZfyep17IiUlJlDh8VLuH1wQ4PP5KC730CE5ci3Vsb7sPoq3KxURaVnBN6lok5RIm6Tq6xMSEiIa4GGVKcL7ywRSg54fAdSeuBz2B5kAAARJSURBVFhERCIi0iH+HXAVgDHmVCDTWqtrzEVEoiSiIW6tXQgsNcYsBF4E/hbJ/YuISHURb7yx1o6M9D5FRCS0A/b2bCIi8UAhLiLiYi09d0oSQFaW7ogjIhKOoLxMCrW+pUO8B8CIESNa+LAiIq7XA6h127CWDvE0YBDO2HFNgC0i0rAknAAPOZlgi97tXkREIksdmyIiLuaKm0LEyxzlxpiTgG+AF6y1LxtjegMf4Hxd2glca60tNcaMAO4AvMAYa+1YY0xb4F3gSJymqD9ZazcbY34GvIbzs1llrb2lxU+sHsaYp3Ga0NoAT+B8JYzbczbGdMApc3egPfAosJI4PucAY0wKsAbnnGcQx+dsjBkMfA6s9S9aDTxNDM651dfE/XOUH+ufl/xGnCtBXccY0xF4CeePO+AR4BVr7SBgE3CDf7sHgIuAwcCdxphuwDXAPmvtOcDjOIEIMBrng20g0MUYc0lLnE84jDHnAyf5f3dDccoa1+cMXA78YK09D7gaeJ74P+eA+4A9/scHwjnPsdYO9v+7jRidc6sPceBC4GsAa+2PwMHGmINiW6QmKQUuxZkkLGAwMN7/eALOL/oMIM1am2etLQYWAANxfg5f+bedDgw0xiTjTE+ZVmMfrcVc4Lf+x/uAjsT5OVtrP7XWPu1/2hvIIM7PGcAY0w84AZjoXzSYOD/nEAYTg3N2Q4inAjlBz3OoPlOiK1hrK/y/xGAdrbWl/sfZOD3QNc+31nJrrRfn61YqsDfEtq2CtdZjrS30P70RmEScn3OAf/6gj3G+Rh8I5/wccFfQ8wPhnE8wxow3xsw3xlxMjM7ZDSFeU7zOUV7XeTVmeav82RhjrsQJ8VtrrIrbc7bWng1cAXxI9TLG3TkbY/4IfG+t/amOTeLunIGNwMPAlcB1wFiq9zG22Dm7IcTjeY7yAn9nEEBPnHOteb61lvs7RRJwfg6HhNi21TDGDAFGAZdYa/OI83M2xvT3d1hjrV2B88bOj+dzBoYBVxpjFgF/Bu4nzn/P1tod/qYzn7U2HcjCaept8XN2Q4jH8xzl04Hh/sfDgSnAYmCAMaarMaYTTvvZPJyfQ6B9+XJglrW2HFhvjDnHv/w3/n20CsaYLsAzwGXW2kCHV1yfM3Au8HcAY0x3oBNxfs7W2t9ZawdYa88E3sIZnRLX52yMGWGMudv/OBVnNNI7xOCcXXGxjzHmSZw3hxf4m7V2ZYyL1GjGmP447YZ9gHJgBzACZ5hRe2ArzjCjcmPMVcA/cNrJXrLWfmSMScJ5gxyL00l6vbV2uzHmBOANnA/kxdbau2gljDE3AQ8BG4IWX4dzHvF6zik4X617Ayk4X7l/AN4nTs85mDHmIWALMJU4PmdjTGecPo+uQDLO73k5MThnV4S4iIiE5obmFBERqYNCXETExRTiIiIuphAXEXExhbiIiIspxEVEXEwhLiLiYgpxEREX+3+Z9KftaL7IPgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "----\n",
            " sauths.\n",
            "\n",
            "Even infection symptoms, China. In the United States, cat nasew abous winters mou. In paricke, China. Their plane lake dinfeys wort wome at her dealth uppaat.\n",
            "\n",
            "Even including the virus.\n",
            "Wash  \n",
            "----\n",
            "iter 49900, loss 6.294803\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2AKpa1BGOItQ"
      },
      "source": [
        "# Quiz Question 7. \n",
        "\n",
        "Run the above code for 50000 iterations making sure that you have 100 hidden layers and time_steps is 40. What is the loss value you're seeing?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Glf5nwwMVf7w",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        },
        "outputId": "49abd03e-e672-4eb8-a004-26dfba823e2b"
      },
      "source": [
        "update_status(inputs, g_h_prev, g_C_prev)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAD1CAYAAACm0cXeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1cH/8U8SthAUEBAQVBR9DrXuFMUFxe2Ruv6sW1tqfdT+tFVb9z60tj6K1gUVrVuViuJaUXysIrLIKpsxyA5ySJCwhZBASMi+zTx/3JlkJplJJslMJnf8vl8vXq87d+5ybkK+c+bcc89J8nq9iIiIOyXHuwAiItJ6CnERERdTiIuIuJhCXETExRTiIiIu1qk9T2aM6QqMAHYDte15bhERl0oBBgIZ1trKhm+2a4jjBPjidj6niEgiGAUsabiyvUN8N8B7773HgAED2vnUIiLuk5uby9ixY8GXnw21d4jXAgwYMIDBgwe386lFRFwtZBO0bmyKiLiYQlxExMUU4iIiLqYQFxFxMYW4iIiLKcRFRFzMNSF+zJ+/4KlZm+JdDBGRDqXZfuLGmO7AFKA/0A14FFgDvIPzOOhu4AZrbaUxZixwN+ABJllrJ0eroDUeL/9YuIX/HjMsWocUEXG9SGrilwMrrLXnAtcBE4HxwMvW2lFAFnCzMSYNeAi4EBgN3GOMOSQmpRYRESCCmri1dmrAy8OBnTgh/VvfuunA/YDFGaClCMAYsxQ4y/e+iIjEQMSP3RtjlgGDgcuAuQGjaeXhjLA1AMgP2MW/XkREYiTiG5vW2jOBK4B3gaSAt5JC7xF2vYiIREmzIW6MGW6MORzAWrsap/ZebIxJ9W0yCMjx/QscmtC/XkREYiSSmvg5wH0Axpj+QA9gLnC17/2rgVlAOjDCGNPLGNMDpz1cY4eLiMRQJCH+KnCoMWYxMAO4A/gf4EbfukOAt6y15cA4YDZOyD/iv8kpIiKxEUnvlHLglyHeuijEttOAaVEol4iIRMA1T2yKiEhjCnERERdTiIuIuJhCXETExRTiIiIuphAXEXExhbiIiIspxEVEXEwhLiLiYgpxEREXU4iLiLiYQlxExMUU4iIiLqYQFxFxMYW4iIiLKcRFRFxMIS4i4mIKcRERF1OIi4i4mEJcRMTFFOIiIi6mEBcRcTGFuIiIiynERURcrFMkGxljJgCjfNs/AVwBDAf2+TZ52lo7wxgzFrgb8ACTrLWTo19kERHxazbEjTHnAcdba88wxvQBVgHzgT9Zaz8P2C4NeAg4DagCMowxn1hrC2JTdBERiaQ55SvgWt9yIZAGpITY7nQgw1pbZK0tB5YCZ0WllCIiElKzNXFrbS1Q6nt5C/AFUAvcaYy5F8gD7gQGAPkBu+YBA6NaWhERCRLxjU1jzJU4IX4n8A4wzlp7PrAaeDjELknRKKCIiIQX6Y3Ni4EHgTHW2iJgXsDbnwH/AKbh1Mb9BgFfR6mcIiISQrM1cWNMT+Bp4DL/TUpjzMfGmKN9m4wG1gPpwAhjTC9jTA+c9vDFMSm1iIgAkdXErwf6Ah8aY/zr3gSmGmPKgBLgJmttuTFmHDAb8AKP+GrtIiISI5Hc2JwETArx1lshtp2G06wiIiLtQE9sioi4mEJcRMTFFOIiIi6mEBcRcTGFuIiIiynERURcTCEuIuJiCnERERdTiIuIuJhCXETExRTiIiIuphAXEXExhbiIiIspxEVEXEwhLiLiYgpxEREXU4iLiLiYQlxExMUU4iIiLqYQFxFxMYW4iIiLKcRFRFxMIS4i4mKdItnIGDMBGOXb/gkgA3gHSAF2AzdYayuNMWOBuwEPMMlaOzkmpRYRESCCmrgx5jzgeGvtGcAY4HlgPPCytXYUkAXcbIxJAx4CLgRGA/cYYw6JVcFFRCSy5pSvgGt9y4VAGk5If+ZbNx0nuE8HMqy1RdbacmApcFZUSysiIkGabU6x1tYCpb6XtwBfABdbayt96/KAgcAAID9gV/96ERGJkYjaxAGMMVfihPh/ApkBbyWF2SXcehERiZKIeqcYYy4GHgR+aq0tAkqMMam+twcBOb5/AwJ2868XEZEYieTGZk/gaeAya22Bb/Vc4Grf8tXALCAdGGGM6WWM6YHTHr44+kUWERG/SJpTrgf6Ah8aY/zrbgReN8bcBmwD3rLWVhtjxgGzAS/wiK/WLiIiMRLJjc1JwKQQb10UYttpwLQolEtERCKgJzZFRFxMIS4i4mIKcRERF1OIi4i4mEJcRMTFFOIiIi6mEBcRcTGFuIiIiynERURcTCEuIuJiCnERERdTiIuIuJhCXETExRTiIiIuphAXEXExhbiIiIspxEVEXEwhLiLiYgpxEREXU4iLiLiYQlxExMUU4iIiLqYQFxFxsU6RbGSMOR74FHjOWvuSMWYKMBzY59vkaWvtDGPMWOBuwANMstZOjkGZRUTEp9kQN8akAS8C8xq89Sdr7ecNtnsIOA2oAjKMMZ9YawuiWF4REQkQSXNKJXAJkNPMdqcDGdbaImttObAUOKuN5RMRkSY0WxO31tYANcaYhm/daYy5F8gD7gQGAPkB7+cBA6NUThERCaG1NzbfAcZZa88HVgMPh9gmqbWFEhGRyER0Y7Mha21g+/hnwD+AaTi1cb9BwNetL5qIiDSnVTVxY8zHxpijfS9HA+uBdGCEMaaXMaYHTnv44qiUUkREQoqkd8pw4FlgCFBtjLkGp7fKVGNMGVAC3GStLTfGjANmA17gEWttUcxKLiIiEd3Y/Bantt3QxyG2nYbTrCIiIu1AT2yKiLiY60L8+/ySeBdBRKTDcF2In//sIuZv2hPvYoiIdAiuC3EAm6vauIgIuDTEvXjjXQQRkQ7BnSGuDBcRAVwa4iIi4nBliHs8qoqLiIBLQ1xERByuDHHVw0VEHK4McY/ubIqIAC4NcWW4iIjDnSEe7wKIiHQQrgzxF+ZlxrsIIiIdgitDXEREHApxEREXU4iLiLiYQlxExMVcG+LzN+3Bq76GIvID59oQv3nKCt5evi3exRARiSvXhjjA/67cGe8iiIjElatDfM3OIn7z1op4F0NEJG5cHeIAc7/bw3vp26ip9cS7KCIi7c71IQ7w4CfrmbIsO97FEBFpd50i2cgYczzwKfCctfYlY8zhwDtACrAbuMFaW2mMGQvcDXiASdbayTEqdyNfrNvNYb1SueSEge11ShGRuGu2Jm6MSQNeBOYFrB4PvGytHQVkATf7tnsIuBAYDdxjjDkk6iUOY+X2Qm5/b2V7nU5EpEOIpDmlErgEyAlYNxr4zLc8HSe4TwcyrLVF1tpyYClwVvSKGpnhj37JiuyC9j6tiEhcNBvi1toaXygHSrPWVvqW84CBwAAgP2Ab//p2ta+0ikc/38jWvaXtfWoRkXYXjRubSS1cH3NrdhZx3jML43V6EZF209oQLzHGpPqWB+E0teTg1MZpsF5ERGKktSE+F7jat3w1MAtIB0YYY3oZY3rgtIcvbnsRRUQknGa7GBpjhgPPAkOAamPMNcBYYIox5jZgG/CWtbbaGDMOmI0zg9oj1tqimJU8Al9tzufEwT3p1b1LPIshIhIzzYa4tfZbnN4oDV0UYttpwLS2F6uxy04cyOdrd7don1+/8Q3Dj+zNx787MxZFEhGJO9c8sXnTWUe1ar8t+SVRLomISMfhmhBPamVfl8Kyal5ekEV1rQePR+OPi0hicU2IDxtwUKv3fXq25dgHZ+qJThFJOK4J8e5dIhrmpUmzNuTywrxMlmXtZUNOXO+5iohERduT0WUmfrm5bjn7yUvjWBIRkbZzTU1cREQaU4iLiLiYQlxExMVcFeL/deaQqB7vyZmbqKypBWD9riJ1QRQR13HVjc3TjjokqtOwvbpoC68u2kLfHl3YW1LFH8cYbh99TNSOLyISa66qifc7qGtMjru3pAqAqRk7+HytBl4UEfdwVYj/5MjeMT3+tn1l3Pn+qpieQ0QkmlwV4kmtffa+DQrLqqiq8bT7eUVEIuGqEG8vY1//mm+2OvN0njz+S259Z0WcSyQiEppCPISlWfu47rXlXPfqcgAW2nyGjJtBSWVNnEsmkphsbjEHKqrjXQxXUog34ZvsgqDX2/eVxakkIont4ue/Yuw/0+NdDFdSiLfCjoIyZq5r2QQVrXH5i0t4YuZ3MT+PSGsUV1QzZNwMPlqxIyrHW7dLg9K1hutCPJ6DVm3KPcAJD89m1IQF/K4dhrVdt6uI1xZ9H/PziLTGrsJyAF5fvDXOJflhc12Ix9O9H66huKK+XXxFdgGLM/O54NmFdU9+hnOgolq9XCShePWAc4fgqic2O5prfDc+wWliGdgzlbeWZ3PbOUNJSQ7uDnniw3MYMaQ3H/1W831KYvCHeBx6/koAhXiU/OKf6eQXVwJweO/uXH7SYQDc8f5KVm8vBCAje3/IfbftK6X/wd3o1jmlyXMUlVXTKSWJtK76tYmIQ80pUeIPcICte0vZkl/CH/61ihlrd9e1HYbi9Xo59+mF/Pbdb5s9x0nj53Du0wuiUl6RtvKi9pSOQFW6GJj45eagGYQC5RdX8uwcy8CeqTw3dzMTrzsJgEWb8yM6tn+cF5F4q29OUXtKPLUqxI0xo4GPgA2+VeuACcA7QAqwG7jBWlsZ8gA/YCP+Njfo9b0frolTSUSiQxEeX21pTllkrR3t+/d7YDzwsrV2FJAF3ByVEobw3PUnxerQcdPefwi1Hq96y4gkgGi2iY8GPvMtTwcujOKxg1x1ymA2PTomVoePC48XRk2Yz8QvNzN9TQ61MZ6g4ueTlvMff5nZ4v3u/2gNJ4+f0+L9qmo8VFQ33Q1T3Km9W1POfXoBd32g0Ub92tImfpwx5jPgEOARIC2g+SQPGNjWwjWluZ4cbrSjoJwX5mUCsPz7fQ3eC//If1WNhx37yxjar0fE5wrXU6Y5077d2ar9Lpy4iO0FZXF9WEuiK179xLftK2PbvjL+/vNT4lOADqa1NfFMnOC+ErgRmEzwB4Kaydro/fTtdcsej5dRE+p7pXyztYAFNq/u9SPTN3DBs4vYkFPExDm2UY33lYVZzFqfG/tCN2F7Ex9CEjsej5cNObF5nN3fO0X3NeOrVSFurd1lrZ1qrfVaa7cAuUBvY0yqb5NBgKbIiZKzn5of9Pq615Zz05sZda+/ynR6tlz6whJemJ/FsL/OCmrvnjDLRtSFUTq2Lzfu4Y0lW5mzIZfrX1uON4Kq8MsLsrj0hSWs2VEYs3Ilqc4WV60KcWPMWGPM/b7lAUB/4E3gat8mVwOzolLCJtx81lGxPkWHkFNUEfa97L2l7Cho3A/9F//8OqI/8ubsKChrUU3u7eXZfN2gKagj2VtSSXoHLl9T/v/bKxj/+UZue/db0rcWRNSc4R9UancT/4daK/D8+0urKCwL3/11zoZcsveWRr0M0vrmlM+Ac40xi4FPgd8BDwI3+tYdArwVnSKG99Dlx/2gv8oNGTeD0c8sDPnet9v2M/yxuRSUBv9hrdq+P2St7KFP1zM1Y3uj9aMmLODSF5ZEXKaHPt3Azyd9HdG2f5+bybiP10blwyZSV72ylOsjLF+s1Hq8zFq/u83X3bK9o/8z9h8xKQlOefRLTh7/Zdhtb33n27D/V+Mhv7iSIeNmsCRzb7yL0maturFprS0GLg/x1kVtK07LaRCe8ApKqzj10eA/rKteWRb0uqbWQ6eUZN5evg2As4/tx6BeqXy0Ygc9mni8f4HN46Y3M5h+59mcMLhni8tWWlnDc3OdB6KuPHkQZwzt0+JjhFNcUc1Xm/dy6YmN762H+tYSCa/Xy0Kbz2jTL6KHW/KLK/lma0HIMry5dCuPzfiO564/iatOGdzisiThBKjzIdB0WfxFbcnfSVWNhw05RZxyRGRz2naketQnq3ZSXuXhl6cf0eR2q7Y7N/anLNvK2cf2bY+ixYzrH7sfMSS2kycnklBjoB/z4Ezunbq67vWlLywG4IFpa5scbnfaCqeXyuUv1dfS72vBg0uBmVJd2/r+6m8u3cq6ncHNPfd/tIY73l9JVl5x2P0Ky6p4do6NuCvnhyt2cNOUDD5aEVnvnBvf+IY73l8ZcraaXF/Txt7i1j19m+xL5khK3pr26se/+I6rXlnW5M8PaNdvUJG6Z+oa/vzJuma3S6SnTF0f4pNu+AlH902LdzFcIVwo/++qXXXLhWXVFJU1Dp4DFdVB09PNCPhAeND3R/PxyvqAu6MF463XRhgGtR5voyF/H5m+MeiDBOrHub5h8jdh2/Mf+nQDL87PYsGmvJDvN7SrsCLo2M3JKXK2q62NPOhqaj3c/t63rG9mcgR//nhaEKINt8wvrgw71IP/Z1ZQ2vR0aXXHdHEgdsDPoRZzfYj3TuvC/PtHx7sYCeWkEA/znPjwHI7/n9kht38vvXFb+ox1uykqq2ZfSSXXvlrfhPOPhVv4dPWuoG09YWrDW/eWMmTcDE582Dnvr15Px/yl+fvl/j/M3UUVPPDR2pDbfLbG6TxV4/HwxpKtzfbe8MdUpH/z/u1DBW24Y2TvK+WLdbn8oZkHWfy160gCKFy+Xj9pOTe+8U3zB4iAGyO8pb/Pjsz1IS4dw5BxMxqtO2n8HMa+nh70YNFTszZx1werg2qom/eU8NjnG6mq8ZBTWE5hWRXlVbWc57sRdsA3EUfDB6CiZfznG7ny5aVNbtOSyqbH4637ut5USDQ+pr8Bu5kTtCI1Gwb+9/nN9xS57rXlvPO1c6/kszU5fNhgGjY312Lr7xW4+CJ8EmYUw1fGnsrUjB0RjwYo7WNTbph21YAgemrWJgC+/G4P23yTUXfr3HT9Iu9ARURzMm7cfYD7PlzD2p2F3Hbu0EbvR/NvuKC0ij/8axVLsup7PDTX5FFRXcv4zzfywH+a+mBp5jx1tcg21MQj9cxsyw0jj+QP/3K+HVz3k8PbdsAoyi+upN9BXVu1b6Q/azdImJr4JScM5KpTBsW7GBKh37yV0WidP8ABKqqDb3YG1ph2FJRx2uPzuOWtFXXrlm0J31Xs45U7ycwr4f6PGt94jbQ9PqAgdYv/WLiF5Vvqvx28vTw7KMAbbB5y3aerd/F++nYmzN4UEM5Nl8kfQN/lHgi7zdyNeygNuIfR2rG/kztwW0lbxk9p6Q3fIeNmhPy22REkTIgDnDfsUIYNOCjexZAItHTsllcWbqlbDhyCwO+X/0zn/fTtvJe+jc17mu5VEejO94ODYNX2/Y16u0BAO7TvdVFZNU/N2sQv/tl0n3Ov13nAKNzgX/7bAR4PETXBQP0H3M9eWUZGdgEAq3cUsjHHCfWsvBJ+8/YK/vjx2mbbz0N9YAQGXNO9OOL72H3gjfbWaulnuMfjbfHgdFvyS7j21WVRKW8oCRXiPVM7M+vuc+JdDImBp2fbZrd5Y+lWHvxkPdUt6BESaM6GXK56ZVmj3i5QH1SZe0qoqvEEjQv/39PWcsuUjJB/3BXVtfzksblBwyQE1ooD86+pZpJQPYagfmC0//fyUi7xdQ/118C37ytrtv38QEXTwRJJTbw1GV4UphdUSyThjCNUHKIbZ0Q70/LmlDF//4qhf/6iRfs8NXMTGdn7WZIZm6behApxv/n3ncvjV50Q72JIO8vKK2nT/re+Uz++zN6SSsqqapj01RY8Hm/dk6+zNuTyH3+ZSVVA3/apK3Ywb1MeL87PanTMCyYuApybss/MtlTW1PKO78GqwFpuTlF5QDtt42gJ1WPIr7wquJYf6jjhwqr5IQjCR3Rr7if4a/4njZ/T5DVF4kBFDde9tpw73l/V4rFhIm26amjznpb/Hwv9kR09CXNjM9DR/XpwVN80svJKeGPp1ngXR1zov6et5fBDujNlWTardxTyxbrWjQIZWDt/aUEW63YVUROwzh+4izP31j3Es6OgvK79tbmhe6trPTwx87ugdXUPA3mDY2PdziK8eIOGLG4uwvaWVDb6kGi478rtkQeoxwspUcoyfxPVV5vz+WpzPlNuGhHxvpE+7OP1emnr0P7109i17TjhJGSIg/NLeujy4zj1yF68+/U2vv6+IN5FEheZF/AQUGsDPJTA3lPzN+0JupmbGeIJyfziSt5L3xb2eM99mUnugfrBrV6an8moY/sBBIWP1+sN2UyUU1jOyMfn8eFtZ3BEn+7OygZhU1wZ3FxRWVNLRbUn5JO2pZU1pDUxXEOtx0tKQBtNXnGF0+zTCvsajAsUtidUCJH28Hlg2tpWj6EPkJFdUNdDKTlGKZ6wIe532YmHcdmJhwHOuNoTZjXftirSHpZmBTdl3DxlRaNtGs7J2lBggAM8M2czb/maa77bfYBM303efy7+PuT+j0zfCMAHGdv545hhvLZoC99sbVDhaRB05z29MOzImlOWZXPHecc4NwC9XjqnBLfY3jA5nam3nVH3+pK/L240+XdZVQ2V1R56p3WpW5dfXEmNx8PAnql16xpOL1gW5htDKM3l6Z4DFYx5/iv2t6HdfknmXn41Ob3+nK0+UtMSsk08nNtHH0P2k5fyM3VFlASWX1w/P7m/6Wb9rvDdEcEZq/yP09bwxMxNjd7bkFO/71//vb7JoZF3+4YbOPrPX3DsgzODJi8BSN9awPxNe+peNwxwgDHPL+aUBgO3jfjbXM54Yn6TU/yFat8uKqtmz4Hw5Q3X9fLBT9Y3GeCFZVXsL2167JtdhcHfMGLVnPKDCnG/idefzDPXOpMtD+qV2szWIokvM6+ED8MM7nXTlPqeNf4nOMN59+vtnB8w5OxNb2bUjRjo11TzlNfrrZsFquF+4Dx8FE6o3kEjn5jH6Y/PC1qX/v0+Zqx1xv5ZmrWPs5+az6OfbwzaZu53e2jKyeO/bPRBE6i8qpaSyuAPHHUxjLJrhg8m+8lLWTrufOxjY+jeJfHm7BSJh+8bTP7QcPjjptqYJy+p74hw1SvL+GRV8LavLwnfUSHUDcjyEDX36yd9zQcZ9UMI7NxfzuQlWykorWpyLttw0r/fx3WvLg+6R3D8w7MbfTDc9cHqhrtGRcK3iUeia6cUNo4fw4acIpZl7aO0qobn52bGu1giPziPzQjuaTPxy83cMzWyIY6bmjnI6/VSXl3b6EngQP6x98f9dFhE5wNnqIV7P1zDrsJyPlyxg5XbCnn4iuNa/EBQWyjEA/z4sJ78+DBngoPfjDqavcWVXDhxUVCXMBFpPy2ZxGPWhvpmmulrcoLGrTnqT84DOrePbjx+TkNPhrgvEE7gpCsPfrIegAE9WzeeS2spxMPo0bUTPbp2IuvxS6ioruXfq3ZxzfDBlFTW8Lt3V9LvoK51w5mKSMfy+3+FHlclcPiGWHl5QezPEUghHoFunVP4+WnOdE+9unfhX7eOBOCJn51AclISizbnMWfDHkYe3YeNuw8wZVl2HEsrIj8kCvE28D/UMOb4gYw5vn4uxYev+DGbcg/QJ60r/Q7qygKbx4HyavaVVPHv1btYG2KAJRGR1lCIx8iwAQfXLZ9nDq1bvvnsoyitrCGnsJy7PljNFScfxmG9Uvn3ql2kf7+P0oAHFnqmdqaovG2DBIlIYlOIx0Fa104c2/8gvrhrVN26K046rG75QEU1Pbp0Ijk5iQueXcgNI4/kxjOH8ObSbJKT4IqTB9XdUFn514sazWgvIj8cCvEO6OBuneuW5903um755rOPqlve9OgYwGmv/+DWkezaX87VwweTlVfM6h1FZO8tparWw9vLs3n48h+z50Alz83dzKu/Gk5VrYc//GsVZxzdh+LK6rqn+Q4/JJUdBeX0TO3Mr0YeUXeD5k8/HRbyST4RiT+FuEt161z/cNLIo/vULR9z6EEcc2j9xBh/vuRHdct3XXhs3XJgzX/5ln2cemQvunZKYV9JJQendqZzSjIPXFzfX7agrAqPx8vizL28dfNp9D+4G5l7ihnarwfJyUl8vjaHIw7pzpb8EuZuzGPooT3YklfCSYf35OUFWygqr+aXpx/B+l1FuicgEkVRD3FjzHPASJxhc+6y1jaeh0s6lDOG1n8I9OkRuo/rn376o0brju1f/2HhH2TsxMG9uOqUwUHb3XrOUPYcqKD/wd3wer18vnY3I4/uQ0V1Lc/PzWTsyCPom9a1fhQ9H6/Xy46Ccr7JLiCnsJzSyhr6HdSVPQcq6NIpmZTkZKat2MGIow7hwh/1Z/Oe4qAxvWfffQ690zozfvpGjuqbxtSMHeQVV3LBsENZuDm/XR/IEImVpGjO9myMORd4wFp7mTHmR8Ab1tozAt4fAmydN28egwcPDncYkXbj8XipqvVQ7JvhZoHN4+IfD6BTclLQkKp7SypZu9OZAu38Yf3pnJLElvxStheUsmt/ObVeL8lJSXRKTmb2hlwm/Xo4Hg9c/tISXrthODmF5TwyfSNnDu1Dr+6d+WJdLqOO7ctxhx1McUUN+0urmLk+eEyRXt07U9jG2W+kY2lufPhQdu7cyQUXXABwlLU2u+H70a6JXwD8G8Ba+50xprcx5mBrbdNDqInESXJyEt2SU+qap8LN5t63R1fOH9af84f1r1sX+E0k0EOXH1e3HPhHe9NZR4XaPKTAcbmLyqo5OLUTSUlJbN9XxqEHd60rr8fj5fu9JXTrnEKXTslU1Xjo26MrHq+Xkgrnm0tSUhL5xZUcnNqJrp2c/WpqPWzJL2VovzSSk5LIzCvhyD7dSUlO4unZlttHD6VXd2co2JLKGnYXlrNx9wFSO6fw8cqdvPiLU0lOgsVZexl+ZG8Afv/+Ki45YQCllbWcdUxfeqd1Zua6XFK7pLC/tIqqGg+DeqfSO60LWXtK+NXIIymuqOaCiYt44GJDWpdOPDFzE0P7pZHeYDjcy04cyJA+aby0wPmmdfYxfVmStZcLhh0aNPZ7R3bDyCNjctxo18QnATOstZ/6Xi8GbrHWbva9HoJq4iLiUtW1nrox0osrqunRtVPQLEHlVbV07ZRMciSTk0aovWviDcVpHmwRkegLnOTioIBeZH6pcRgNNdpD0eYAAwJeHwbsjvI5RETEJ9ohPge4BsAYcyqQY62NfOI7ERFpkaiGuLV2GfCtMWYZ8AJwRzSPLyIiwaLeJm6tHRftY4qISGg/2OnZREQSgUJcRMTF2nvslBSA3L665VsAAAP5SURBVNzws12LiEi9gLwM2X+xvUN8IMDYsWPb+bQiIq43EGg091t7h3gGMAqn73htM9uKiIhTAx+Ik5+NRPWxexERaV+6sSki4mKumBQiUcYoN8YcD3wKPGetfckYczjwDs7Xpd3ADdbaSmPMWOBuwANMstZONsZ0BqYAR+I0Rd1krf3eGHMS8A+cn81aa+3v2v3CmmCMmYDThNYJeALnK2HCXrMxpjtOmfsD3YBHgTUk8DUDGGNSgfU41zuPxL/e0cBHwAbfqnXABOJw3R2+Ju4bo/xY37jkt+A8Ceo6xpg04EWc/+B+44GXrbWjgCzgZt92DwEXAqOBe4wxhwC/BAqttWcDf8MJRIDncT7YzgJ6GmN+2h7XEwljzHnA8b7f3Ricsib0NQOXAyustecC1wETSfxrBvgL4B8/9odwvQCLrLWjff9+T5yuu8OHOA3GKAd6G2MObnqXDqkSuARnkDC/0cBnvuXpOL/o04EMa22RtbYcWAqchfNz+MS37VzgLGNMF5zhKTMaHKOj+Aq41rdcCKSR4NdsrZ1qrZ3ge3k4sJMEv2ZjzDDgOGCGb9VoEvh6mzCaOFy3G0J8AJAf8Dqf4JESXcFaW+P7JQZKs9ZW+pbzcO5AN7zeRuuttR6cr1sDgP0htu0QrLW11tpS38tbgC9I8Gv2840f9D7O1+hEv+ZngXsDXif69fodZ4z5zBizxBhzEXG6bjeEeEOJOkZ5uOtqyfoO+bMxxlyJE+J3NngrYa/ZWnsmcAXwLsFlTKhrNsb8Glhurd0aZpOEut4AmcAjwJXAjcBkgu8xttt1uyHEE3mM8hLfDSGAQTjX2vB6G6333RRJwvk59AmxbYdhjLkYeBD4qbW2iAS/ZmPMcN8Na6y1q3H+sIsT+JovBa40xnwN/Ab4Kwn+Owaw1u7yNZ15rbVbgFycpt52v243hHgij1E+F7jat3w1MAtIB0YYY3oZY3rgtJ8txvk5+NuXLwcWWGurgU3GmLN963/mO0aHYIzpCTwNXGat9d/0SuhrBs4B7gMwxvQHepDA12ytvd5aO8JaOxJ4Had3SsJer58xZqwx5n7f8gCc3khvEofrdsXDPsaYJ3H+ODzAHdbaNXEuUosZY4bjtB0OAaqBXcBYnG5G3YBtON2Mqo0x1wAP4LSTvWitfc8Yk4LzR3Iszk3S/7LW7jDGHAe8hvOBnG6tvZcOwhhzK/AwsDlg9Y0415Go15yK89X6cCAV5yv3CuBtEvSa/YwxDwPZwGwS/HqNMQfh3PPoBXTB+T2vIg7X7YoQFxGR0NzQnCIiImEoxEVEXEwhLiLiYgpxEREXU4iLiLiYQlxExMUU4iIiLqYQFxFxsf8DpruJo/SxHicAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "----\n",
            " \n",
            "Chlers. They had ald patient imficines from Chicatored for and cases mas has sapperic7nos infold out the knew me8 tame. They are scwermmon s mwacine Soroun, The fird from theilach aA Oroplron sind ca \n",
            "----\n",
            "iter 50000, loss 6.301059\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fr8Xm4dVVgZT"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}