Python/MINST_with_one_hidden_layer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MINST-with-one-hidden-layer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyOuOC0kTs5Q+JGYsmKKUCEE",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/happyrabbit/IntroDataScience/blob/master/Python/MINST_with_one_hidden_layer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wIstG9VsPyb2"
      },
      "source": [
        "# 01-Packages\n",
        "\n",
        "Import packages for the notebook."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2e5WqQGj2tlo"
      },
      "source": [
        "import numpy as np\n",
        "import math\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Flatten, Dropout\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9-m2iScR3RvE"
      },
      "source": [
        "# 02-Load Data\n",
        "\n",
        "Let's load the MINST data set. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgbepWh429f2",
        "outputId": "4d8be34f-e073-4c46-a19b-0684a3b41ad2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
        "\n",
        "img_width = X_train.shape[1]\n",
        "img_height = X_train.shape[2]\n",
        "\n",
        "# one hot encode outputs\n",
        "y_train = to_categorical(y_train)\n",
        "y_test = to_categorical(y_test)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dTRwGbNJ4Faj",
        "outputId": "ef43e6b4-bb24-4d8d-d2d3-3b3deec11556",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "print('The shape of X_train is: ' + str(X_train.shape)) \n",
        "print('The shape of y_train is: ' + str(y_train.shape)) \n",
        "print('The shape of X_test is: ' + str(X_test.shape)) \n",
        "print('The shape of y_test is: ' + str(y_test.shape))\n",
        "print('I have %d traning and %d testing examples!' %(X_train.shape[0], X_test.shape[0])) "
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The shape of X_train is: (60000, 28, 28)\n",
            "The shape of y_train is: (60000, 10)\n",
            "The shape of X_test is: (10000, 28, 28)\n",
            "The shape of y_test is: (10000, 10)\n",
            "I have 60000 traning and 10000 testing examples!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qq1P18bhWT3A"
      },
      "source": [
        "Check some of the input images "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7hsXDMtEWc-C",
        "outputId": "09ca19b6-d3f2-49fc-e416-af6495b51937",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "# assign the number of the image to show\n",
        "img_id = 10\n",
        "plt.imshow(X_train[img_id], cmap='gray')\n",
        "print(\"Label: \"+ str(np.where(y_train[img_id] ==1)[0]))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Label: [3]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANcElEQVR4nO3df6gd9ZnH8c9ntVE0kSSK8WL9kUZFg2KyRlFWF9eSkhUlFqQ2yOKyws0fVaoI2VDBCJuC7hpXglhIUZtduimFGCql0rghrOs/JVGzGhPbZENic40J7kVr/Scan/3jTuSq98y5OTNz5uQ+7xdczjnznJl5OOSTmTM/ztcRIQBT31+03QCA/iDsQBKEHUiCsANJEHYgiVP7uTLbHPoHGhYRnmh6pS277SW2f297r+2VVZYFoFnu9Ty77VMk/UHSYkkHJW2TtCwidpXMw5YdaFgTW/brJO2NiH0RcVTSLyQtrbA8AA2qEvbzJf1x3OuDxbQvsT1se7vt7RXWBaCixg/QRcQ6SeskduOBNlXZso9IumDc628W0wAMoCph3ybpUttzbU+T9H1JL9bTFoC69bwbHxGf2b5P0m8lnSLpuYh4u7bOANSq51NvPa2M7+xA4xq5qAbAyYOwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgST6OmQzmjF//vyOtdtuu6103uHh4dL6tm3bSutvvPFGab3MU089VVo/evRoz8vG17FlB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkGMX1JLB8+fLS+hNPPNGxNn369Lrbqc0tt9xSWt+6dWufOplaOo3iWumiGtv7JX0s6ZikzyJiUZXlAWhOHVfQ/U1EfFDDcgA0iO/sQBJVwx6SNtt+zfaEF1nbHra93fb2iusCUEHV3fgbI2LE9rmSXrb9TkS8Mv4NEbFO0jqJA3RAmypt2SNipHg8ImmTpOvqaApA/XoOu+0zbc84/lzSdyTtrKsxAPXq+Ty77W9pbGsujX0d+I+I+HGXediN78Hs2bNL67t37+5YO/fcc+tupzYffvhhaf2uu+4qrW/evLnOdqaM2s+zR8Q+SVf33BGAvuLUG5AEYQeSIOxAEoQdSIKwA0nwU9IngdHR0dL6qlWrOtbWrFlTOu8ZZ5xRWn/33XdL6xdeeGFpvczMmTNL60uWLCmtc+rtxLBlB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk+CnpKW7Hjh2l9auvLr9xcefO8p8ouPLKK0+4p8maN29eaX3fvn2Nrftk1ukWV7bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE97NPcatXry6tP/zww6X1BQsW1NnOCZk2bVpr656K2LIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBLcz57ceeedV1rv9tvsV111VZ3tfMnGjRtL63feeWdj6z6Z9Xw/u+3nbB+xvXPctNm2X7a9p3icVWezAOo3md34n0n66tAcKyVtiYhLJW0pXgMYYF3DHhGvSPrq+ENLJa0vnq+XdEfNfQGoWa/Xxs+JiEPF8/clzen0RtvDkoZ7XA+AmlS+ESYiouzAW0Ssk7RO4gAd0KZeT70dtj0kScXjkfpaAtCEXsP+oqR7iuf3SPpVPe0AaErX3XjbGyTdLOkc2wclrZL0mKRf2r5X0gFJ32uySfTu7rvvLq13+934Jn8XvptXX321tXVPRV3DHhHLOpS+XXMvABrE5bJAEoQdSIKwA0kQdiAJwg4kwS2uJ4HLL7+8tL5p06aOtUsuuaR03lNPHdxfE2fI5t4wZDOQHGEHkiDsQBKEHUiCsANJEHYgCcIOJDG4J1nxhSuuuKK0Pnfu3I61QT6P3s2DDz5YWr///vv71MnUwJYdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5I4eU/CJlJ2v7okrVixomPt8ccfL5339NNP76mnfhgaGmq7hSmFLTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJMF59ilg7dq1HWt79uwpnXfmzJmV1t3tfvmnn366Y+2ss86qtG6cmK5bdtvP2T5ie+e4aY/aHrG9o/i7tdk2AVQ1md34n0laMsH0f42IBcXfb+ptC0DduoY9Il6RNNqHXgA0qMoBuvtsv1ns5s/q9Cbbw7a3295eYV0AKuo17D+RNE/SAkmHJK3p9MaIWBcRiyJiUY/rAlCDnsIeEYcj4lhEfC7pp5Kuq7ctAHXrKey2x997+F1JOzu9F8Bg6Hqe3fYGSTdLOsf2QUmrJN1se4GkkLRf0vIGe0QFL730UqPLtyccCvwLZePDP/LII6XzLliwoLR+0UUXldYPHDhQWs+ma9gjYtkEk59toBcADeJyWSAJwg4kQdiBJAg7kARhB5LgFldUMm3atNJ6t9NrZT799NPS+rFjx3pedkZs2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCc6zo5LVq1c3tuxnny2/ufLgwYONrXsqYssOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0k4Ivq3Mrt/K6vZ2Wef3bH2/PPPl867YcOGSvU2DQ0Nldbfeeed0nqVYZnnzZtXWt+3b1/Py57KImLC3/dmyw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSXA/+yStXbu2Y+32228vnfeyyy4rrb/33nul9ZGRkdL63r17O9auueaa0nm79bZixYrSepXz6GvWrCmtd/tccGK6btltX2B7q+1dtt+2/cNi+mzbL9veUzzOar5dAL2azG78Z5Ieioj5kq6X9APb8yWtlLQlIi6VtKV4DWBAdQ17RByKiNeL5x9L2i3pfElLJa0v3rZe0h1NNQmguhP6zm77YkkLJf1O0pyIOFSU3pc0p8M8w5KGe28RQB0mfTTe9nRJGyU9EBF/Gl+LsbtpJrzJJSLWRcSiiFhUqVMAlUwq7La/obGg/zwiXigmH7Y9VNSHJB1ppkUAdeh6i6tta+w7+WhEPDBu+r9I+r+IeMz2SkmzI6L0PM3JfIvr9ddf37H25JNPls57ww03VFr3/v37S+u7du3qWLvppptK550xY0YvLX2h27+fsltgr7322tJ5P/nkk556yq7TLa6T+c7+V5L+TtJbtncU034k6TFJv7R9r6QDkr5XR6MAmtE17BHxqqQJ/6eQ9O162wHQFC6XBZIg7EAShB1IgrADSRB2IAl+SroG3W7VLLsFVZKeeeaZOtvpq9HR0dJ62U9woxn8lDSQHGEHkiDsQBKEHUiCsANJEHYgCcIOJMFPSdfgoYceKq2fdtpppfXp06dXWv/ChQs71pYtW1Zp2R999FFpffHixZWWj/5hyw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSXA/OzDFcD87kBxhB5Ig7EAShB1IgrADSRB2IAnCDiTRNey2L7C91fYu22/b/mEx/VHbI7Z3FH+3Nt8ugF51vajG9pCkoYh43fYMSa9JukNj47H/OSKemPTKuKgGaFyni2omMz77IUmHiucf294t6fx62wPQtBP6zm77YkkLJf2umHSf7TdtP2d7Vod5hm1vt729UqcAKpn0tfG2p0v6L0k/jogXbM+R9IGkkPRPGtvV/4cuy2A3HmhYp934SYXd9jck/VrSbyPiyQnqF0v6dURc2WU5hB1oWM83wti2pGcl7R4f9OLA3XHflbSzapMAmjOZo/E3SvpvSW9J+ryY/CNJyyQt0Nhu/H5Jy4uDeWXLYssONKzSbnxdCDvQPO5nB5Ij7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJNH1Bydr9oGkA+Nen1NMG0SD2tug9iXRW6/q7O2iToW+3s/+tZXb2yNiUWsNlBjU3ga1L4neetWv3tiNB5Ig7EASbYd9XcvrLzOovQ1qXxK99aovvbX6nR1A/7S9ZQfQJ4QdSKKVsNteYvv3tvfaXtlGD53Y3m/7rWIY6lbHpyvG0Dtie+e4abNtv2x7T/E44Rh7LfU2EMN4lwwz3upn1/bw533/zm77FEl/kLRY0kFJ2yQti4hdfW2kA9v7JS2KiNYvwLD915L+LOnfjg+tZfufJY1GxGPFf5SzIuIfB6S3R3WCw3g31FunYcb/Xi1+dnUOf96LNrbs10naGxH7IuKopF9IWtpCHwMvIl6RNPqVyUslrS+er9fYP5a+69DbQIiIQxHxevH8Y0nHhxlv9bMr6asv2gj7+ZL+OO71QQ3WeO8habPt12wPt93MBOaMG2brfUlz2mxmAl2H8e6nrwwzPjCfXS/Dn1fFAbqvuzEi/lLS30r6QbG7OpBi7DvYIJ07/YmkeRobA/CQpDVtNlMMM75R0gMR8afxtTY/uwn66svn1kbYRyRdMO71N4tpAyEiRorHI5I2aexrxyA5fHwE3eLxSMv9fCEiDkfEsYj4XNJP1eJnVwwzvlHSzyPihWJy65/dRH3163NrI+zbJF1qe67taZK+L+nFFvr4GttnFgdOZPtMSd/R4A1F/aKke4rn90j6VYu9fMmgDOPdaZhxtfzZtT78eUT0/U/SrRo7Iv+/kh5uo4cOfX1L0v8Uf2+33ZukDRrbrftUY8c27pV0tqQtkvZI+k9Jsweot3/X2NDeb2osWEMt9XajxnbR35S0o/i7te3PrqSvvnxuXC4LJMEBOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4v8BbAEsnwu8EY8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZH2gGO_yRfW4"
      },
      "source": [
        "Scale and reshape the inputs and outputs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "og6T69tsP-OD",
        "outputId": "5d9eaa18-3995-4f31-c14b-4ce3d571b22a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "# Scale the input\n",
        "X_train = X_train / 255.\n",
        "X_test = X_test / 255.\n",
        "\n",
        "# Flatten the input\n",
        "X_train = np.array([X_train[i].flatten() for i in range(0,X_train.shape[0])])\n",
        "X_test = np.array([X_test[i].flatten() for i in range(0,X_test.shape[0])])\n",
        "\n",
        "# Reshape the input and output\n",
        "X_train = X_train.T\n",
        "X_test = X_test.T\n",
        "y_train = y_train.T\n",
        "y_test = y_test.T\n",
        "\n",
        "print('The shape of X_train is: ' + str(X_train.shape)) \n",
        "print('The shape of y_train is: ' + str(y_train.shape)) \n",
        "print('The shape of X_test is: ' + str(X_test.shape)) \n",
        "print('The shape of y_test is: ' + str(y_test.shape))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The shape of X_train is: (784, 60000)\n",
            "The shape of y_train is: (10, 60000)\n",
            "The shape of X_test is: (784, 10000)\n",
            "The shape of y_test is: (10, 10000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dfufnet5VBPQ"
      },
      "source": [
        "# 03-One Hidden Layer Neural Network\n",
        "<center><img src=\"https://raw.githubusercontent.com/happyrabbit/course2020/master/slides/02DeepLearning/DNN/images/1hiddenexp.png\" width=\"700\"/></center>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VPgV2WF7Do4I"
      },
      "source": [
        "## 03.1-Defining the neural network structure\n",
        "\n",
        "- `n_x` -- the size of the input layer\n",
        "- `n_h` -- the size of the hidden layer\n",
        "- `n_y` -- the size of the output layer\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sa4lvy8dCx8B"
      },
      "source": [
        "## 03.2- Initialize the model's parameters\n",
        "\n",
        "- Initialize the weights matrices, $W_1$ and $W_2$, with random values.\n",
        "- Initialize the bias vectors, $b_1$ and $b_2$, as zeros."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DGbe96XiQhW2"
      },
      "source": [
        "def initialize_parameters(n_x, n_h, n_y):\n",
        "    \"\"\"\n",
        "    Argument:\n",
        "    n_x -- size of the input layer\n",
        "    n_h -- size of the hidden layer\n",
        "    n_y -- size of the output layer\n",
        "    \n",
        "    Returns:\n",
        "    params -- python dictionary containing your parameters:\n",
        "                    W1 -- weight matrix of shape (n_h, n_x)\n",
        "                    b1 -- bias vector of shape (n_h, 1)\n",
        "                    W2 -- weight matrix of shape (n_y, n_h)\n",
        "                    b2 -- bias vector of shape (n_y, 1)\n",
        "    \"\"\"\n",
        "        \n",
        "    W1 = np.random.randn(n_h, n_x) * 0.001\n",
        "    b1 = np.zeros(shape=(n_h, 1))\n",
        "    W2 = np.random.randn(n_y, n_h) * 0.001\n",
        "    b2 = np.zeros(shape=(n_y, 1))\n",
        "    \n",
        "    assert (W1.shape == (n_h, n_x))\n",
        "    assert (b1.shape == (n_h, 1))\n",
        "    assert (W2.shape == (n_y, n_h))\n",
        "    assert (b2.shape == (n_y, 1))\n",
        "    \n",
        "    parameters = {\"W1\": W1,\n",
        "                  \"b1\": b1,\n",
        "                  \"W2\": W2,\n",
        "                  \"b2\": b2}\n",
        "    \n",
        "    return parameters"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Duea8jLXHMEm"
      },
      "source": [
        "## 03.3-Defining Activation Functions and the Gradients\n",
        "\n",
        "![](https://raw.githubusercontent.com/happyrabbit/course2020/master/slides/02DeepLearning/DNN/images/activation.png)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hbupxz65Hm6S"
      },
      "source": [
        "def ReLU(Z1):\n",
        "    \"\"\"\n",
        "    Argument:\n",
        "    Z1 -- the input of hidden layer activation function\n",
        "    \n",
        "    Returns:\n",
        "    A1 -- activations from the hidden layer\n",
        "    \"\"\"\n",
        "    A1 = np.maximum(Z1, 0)\n",
        "    return A1\n",
        "\n",
        "def ReLU_dev(Z1):\n",
        "    \"\"\"\n",
        "    Argument:\n",
        "    Z1 -- the input of hidden layer activation function\n",
        "    \n",
        "    Returns:\n",
        "    dZ1 -- gradient of ReLU given input Z1\n",
        "    \"\"\"\n",
        "    positive = Z1 > 0\n",
        "    dZ1 = np.zeros(Z1.shape)\n",
        "    dZ1[positive] = 1\n",
        "    return dZ1\n",
        "\n",
        "def softmax(Z2):\n",
        "    \"\"\"\n",
        "    Argument:\n",
        "    Z2 -- the input of output layer activation function\n",
        "    \n",
        "    Returns:\n",
        "    A2 -- activations from the hidden layer\n",
        "    \"\"\"\n",
        "    exp_scores = np.exp(Z2 - Z2.max(axis = 0, keepdims=True))\n",
        "    A2 = exp_scores / exp_scores.sum(axis = 0, keepdims=True)\n",
        "    return A2"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YkS9OaatW8NF"
      },
      "source": [
        "## 03.4-Loss and Accuracy\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f8Cskf4wWdcH"
      },
      "source": [
        "def compute_cost(A2, Y, parameters):\n",
        "    \"\"\"\n",
        "    Computes the cross-entropy cost given in equation (13)\n",
        "    \n",
        "    Arguments:\n",
        "    A2 -- The sigmoid output of the second activation, of shape (10, number of examples)\n",
        "    Y -- \"true\" labels vector of shape (10, number of examples)\n",
        "    parameters -- python dictionary containing your parameters W1, b1, W2 and b2\n",
        "    \n",
        "    Returns:\n",
        "    cost -- cross-entropy cost given equation (13)\n",
        "    \"\"\"\n",
        "    m = Y.shape[1] # number of example\n",
        "    # Retrieve W1 and W2 from parameters\n",
        "    W1 = parameters['W1']\n",
        "    W2 = parameters['W2']\n",
        "    \n",
        "    # Compute the cross-entropy cost\n",
        "    logprobs = np.multiply(np.log(A2), Y)\n",
        "    cost = - np.sum(logprobs) / m\n",
        "    \n",
        "    cost = np.squeeze(cost)     # makes sure cost is the dimension we expect. \n",
        "                                # E.g., turns [[17]] into 17 \n",
        "    assert(isinstance(cost, float))\n",
        "    return cost"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V4hoXxx9FLw4"
      },
      "source": [
        "## 03.5-Forward Propagation\n",
        "\n",
        "Forward and backward propagation.\n",
        "\n",
        "![](https://raw.githubusercontent.com/happyrabbit/course2020/master/slides/02DeepLearning/DNN/images/1hiddenfb_math.png)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ujmmyE0CEysi"
      },
      "source": [
        "def forward_propagation(X, parameters):\n",
        "    \"\"\"\n",
        "    Argument:\n",
        "    X -- input data of size (n_x, m)\n",
        "    parameters -- python dictionary containing your parameters (output of initialization function)\n",
        "    \n",
        "    Returns:\n",
        "    A2 -- The sigmoid output of the second activation\n",
        "    cache -- a dictionary containing \"Z1\", \"A1\", \"Z2\" and \"A2\"\n",
        "    \"\"\"\n",
        "    # Retrieve each parameter from the dictionary \"parameters\"\n",
        "    W1 = parameters['W1']\n",
        "    b1 = parameters['b1']\n",
        "    W2 = parameters['W2']\n",
        "    b2 = parameters['b2']\n",
        "    \n",
        "    # Implement Forward Propagation to calculate A2 (probabilities)\n",
        "    Z1 = np.dot(W1, X) + b1\n",
        "    A1 = ReLU(Z1)\n",
        "    Z2 = np.dot(W2, A1) + b2\n",
        "    A2 = softmax(Z2)\n",
        "    \n",
        "    assert(A2.shape == (10, X.shape[1]))\n",
        "    \n",
        "    cache = {\"Z1\": Z1,\n",
        "             \"A1\": A1,\n",
        "             \"Z2\": Z2,\n",
        "             \"A2\": A2}\n",
        "    \n",
        "    return A2, cache"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OdQpBDdnaQIO"
      },
      "source": [
        "## 03.6-Backward Propagation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y8KONXGME3Y_"
      },
      "source": [
        "def backward_propagation(parameters, cache, X, Y):\n",
        "    \"\"\"\n",
        "    Implement the backward propagation using the instructions above.\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing our parameters \n",
        "    cache -- a dictionary containing \"Z1\", \"A1\", \"Z2\" and \"A2\".\n",
        "    X -- input data of shape (784, number of examples)\n",
        "    Y -- \"true\" labels vector of shape (10, number of examples)\n",
        "    \n",
        "    Returns:\n",
        "    grads -- python dictionary containing your gradients with respect to different parameters\n",
        "    \"\"\"\n",
        "    m = X.shape[1]\n",
        "    \n",
        "    # First, retrieve W1 and W2 from the dictionary \"parameters\".\n",
        "    W1 = parameters['W1']\n",
        "    W2 = parameters['W2']\n",
        "        \n",
        "    # Retrieve also A1 and A2 from dictionary \"cache\".\n",
        "    A1 = cache['A1']\n",
        "    A2 = cache['A2']\n",
        "    Z1 = cache['Z1']\n",
        "    Z2 = cache['Z2']\n",
        "    \n",
        "    # Backward propagation: calculate dW1, db1, dW2, db2. \n",
        "    dZ2= A2 - Y\n",
        "    dW2 = (1 / m) * np.dot(dZ2, A1.T)\n",
        "    db2 = (1 / m) * np.sum(dZ2, axis=1, keepdims=True)\n",
        "    dZ1 = np.multiply(np.dot(W2.T, dZ2), ReLU_dev(Z1))\n",
        "    dW1 = (1 / m) * np.dot(dZ1, X.T)\n",
        "    db1 = (1 / m) * np.sum(dZ1, axis=1, keepdims=True)\n",
        "    \n",
        "    grads = {\"dW1\": dW1,\n",
        "             \"db1\": db1,\n",
        "             \"dW2\": dW2,\n",
        "             \"db2\": db2}\n",
        "    \n",
        "    return grads"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WyVadBizc4Kc"
      },
      "source": [
        "def update_parameters(parameters, grads, learning_rate= 0.1, l2 = 0):\n",
        "    \"\"\"\n",
        "    Updates parameters using the gradient descent update rule given above\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters \n",
        "    grads -- python dictionary containing your gradients \n",
        "    learning_rate -- a number between 0 to 1\n",
        "    l2 -- parameter for l2 penality, a number >= 0\n",
        "    \n",
        "    Returns:\n",
        "    parameters -- python dictionary containing your updated parameters \n",
        "    \"\"\"\n",
        "    # Retrieve each parameter from the dictionary \"parameters\"\n",
        "    W1 = parameters['W1']\n",
        "    b1 = parameters['b1']\n",
        "    W2 = parameters['W2']\n",
        "    b2 = parameters['b2']\n",
        "    \n",
        "    # Retrieve each gradient from the dictionary \"grads\"\n",
        "    dW1 = grads['dW1'] + np.sum(l2*W1)\n",
        "    db1 = grads['db1'] \n",
        "    dW2 = grads['dW2'] + np.sum(l2*W2)\n",
        "    db2 = grads['db2'] \n",
        "    \n",
        "    # Update rule for each parameter\n",
        "    W1 = W1 - learning_rate * dW1\n",
        "    b1 = b1 - learning_rate * db1\n",
        "    W2 = W2 - learning_rate * dW2\n",
        "    b2 = b2 - learning_rate * db2\n",
        "    \n",
        "    parameters = {\"W1\": W1,\n",
        "                  \"b1\": b1,\n",
        "                  \"W2\": W2,\n",
        "                  \"b2\": b2}\n",
        "    \n",
        "    return parameters"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NON-Sf9QdFnb"
      },
      "source": [
        "# 04-Optimization Methods\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LmZPoEA2MMHS"
      },
      "source": [
        "## 04.1-Stochastic Gradient Descent (SGD) "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QtghT07DdOCA"
      },
      "source": [
        "def nn_model(X, Y, n_h, num_iterations=10000, print_cost=False, learning_rate= 0.1, l2 = 0):\n",
        "    \"\"\"\n",
        "    Arguments:\n",
        "    X -- dataset of shape (784, number of examples)\n",
        "    Y -- labels of shape (10, number of examples)\n",
        "    n_h -- size of the hidden layer\n",
        "    num_iterations -- Number of iterations in gradient descent loop\n",
        "    print_cost -- if True, print the cost every 1000 iterations\n",
        "    learning_rate -- a number between 0 to 1\n",
        "    l2 -- parameter for l2 penality, a number >= 0\n",
        "    \n",
        "    Returns:\n",
        "    parameters -- parameters learnt by the model. They can then be used to predict.\n",
        "    costs -- a list of cost per 100 interations\n",
        "    \"\"\"\n",
        "    \n",
        "    np.random.seed(1)\n",
        "    n_x = X.shape[0]\n",
        "    n_y = Y.shape[0]\n",
        "    costs = []\n",
        "    \n",
        "    # Initialize parameters, then retrieve W1, b1, W2, b2. Inputs: \"n_x, n_h, n_y\". Outputs = \"W1, b1, W2, b2, parameters\".\n",
        "    parameters = initialize_parameters(n_x, n_h, n_y)\n",
        "    W1 = parameters['W1']\n",
        "    b1 = parameters['b1']\n",
        "    W2 = parameters['W2']\n",
        "    b2 = parameters['b2']\n",
        "    \n",
        "    # Loop (gradient descent)\n",
        "\n",
        "    for i in range(0, num_iterations):\n",
        "         \n",
        "        # Forward propagation. Inputs: \"X, parameters\". Outputs: \"A2, cache\".\n",
        "        A2, cache = forward_propagation(X, parameters)\n",
        "        \n",
        "        # Cost function. Inputs: \"A2, Y, parameters\". Outputs: \"cost\".\n",
        "        cost = compute_cost(A2, Y, parameters)\n",
        " \n",
        "        # Backpropagation. Inputs: \"parameters, cache, X, Y\". Outputs: \"grads\".\n",
        "        grads = backward_propagation(parameters, cache, X, Y)\n",
        "        \n",
        "        # Gradient descent parameter update. Inputs: \"parameters, grads\". Outputs: \"parameters\".\n",
        "        parameters = update_parameters(parameters, grads, learning_rate= learning_rate, l2 = l2)\n",
        "        \n",
        "        # Print the cost every 1000 iterations\n",
        "        if print_cost and i % 1000 == 0:\n",
        "          print (\"Cost after iteration %i: %f\" % (i, cost))\n",
        "        if i % 100 == 0:\n",
        "          costs.append(cost)\n",
        "\n",
        "    return parameters, costs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jb5eD9qOduZK",
        "outputId": "0600bc01-b4d2-4d67-c54c-4431d9fb191c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "# Build a model with a n_h-dimensional hidden layer\n",
        "parameters, costs = nn_model(X_train, y_train, n_h = 4, num_iterations=10000, print_cost=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cost after iteration 0: 2.302587\n",
            "Cost after iteration 1000: 0.565407\n",
            "Cost after iteration 2000: 0.499799\n",
            "Cost after iteration 3000: 0.479786\n",
            "Cost after iteration 4000: 0.466273\n",
            "Cost after iteration 5000: 0.454446\n",
            "Cost after iteration 6000: 0.444719\n",
            "Cost after iteration 7000: 0.437433\n",
            "Cost after iteration 8000: 0.431851\n",
            "Cost after iteration 9000: 0.427439\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vgv_l_WfDoce",
        "outputId": "0738ad84-32b7-47d6-db9f-a4fa53dc7b9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "# plot the cost\n",
        "plt.plot(costs)\n",
        "plt.ylabel('cost')\n",
        "plt.xlabel('epochs (per 100)')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5hkdX3n8fenqqtvM9M9V4a5wSBCgInc0oAGNyLscjGJGKMGvKGrS5IHN+q6STSbR3bJ5cnmopF4QYIIZBWCCIouEQghQpYgNIjADALDRZhhYK7Oracv1fXdP86p7pqe6p6emT5d3X0+r+epp+r8zqW+NQX16XN+5/yOIgIzM7ORCo0uwMzMpiYHhJmZ1eWAMDOzuhwQZmZWlwPCzMzqamp0ARNp4cKFsXLlykaXYWY2bTzyyCObI2JRvXkzKiBWrlxJd3d3o8swM5s2JP1stHk+xGRmZnU5IMzMrC4HhJmZ1eWAMDOzuhwQZmZWlwPCzMzqckCYmVldDgjgynueZfUr2xtdhpnZlJL7gNi2u5+bHnqJ91z17/zr0xsbXY6Z2ZSR+4CYN6uZ2y47kyMXzOIj13fzjw+/1OiSzMymhNwHBMDijlZu/p03cebrF/KH336Cmx9+udElmZk1XGYBIWmFpHslrZG0WtLH6yzzPkmPS3pC0gOSTqqZ92La/pikzAdYmt3SxNcu6eLwjlYeeG5z1m9nZjblZTlYXxn4VEQ8KmkO8IikuyNiTc0yLwBviYhtki4ArgbOqJn/1oiYtF/rUrHA3PYSewYGJ+stzcymrMwCIiI2ABvS1zslPQUsA9bULPNAzSoPAsuzqme8WktFevodEGZmk9IHIWklcArwozEW+wjwTzXTAdwl6RFJl46x7UsldUvq3rRp0yHX2t5cpNd7EGZm2d8PQtJs4NvAJyJixyjLvJUkIN5c0/zmiFgv6TDgbkk/jYj7Rq4bEVeTHJqiq6srDrXetlKR7XsGDnUzZmbTXqZ7EJJKJOHwjYi4dZRlTgSuAS6MiC3V9ohYnz5vBG4DTs+y1qq25qL7IMzMyPYsJgFfA56KiM+NsswRwK3AByLimZr2WWnHNpJmAecCT2ZVa622UpE97oMwM8v0ENOZwAeAJyQ9lrb9EXAEQERcBXwWWAB8OckTyhHRBSwGbkvbmoBvRsQPMqx1SLv3IMzMgGzPYvo3QPtZ5qPAR+u0Pw+ctO8a2Wtt9h6EmRn4Sup9tJWK9JUrDFYOub/bzGxac0CM0N5cBPCprmaWew6IEdpKSUD4YjkzyzsHxAhtzUm3jPcgzCzvHBAjeA/CzCzhgBih2gfhU13NLO8cECO0Du1BlBtciZlZYzkgRmjzWUxmZoADYh9Dh5j6Kw2uxMyssRwQI7T5EJOZGeCA2IcPMZmZJRwQI/g0VzOzhANihGpA+DRXM8s7B8QIhYJoaSp4RFczyz0HRB2+q5yZmQOirvZS0X0QZpZ7Dog6Wr0HYWaW6T2pV0i6V9IaSaslfbzOMpJ0paS1kh6XdGrNvEskPZs+Lsmqznram4v0eg/CzHIuy3tSl4FPRcSjkuYAj0i6OyLW1CxzAXBM+jgD+ApwhqT5wOVAFxDpurdHxLYM6x3S5kNMZmbZ7UFExIaIeDR9vRN4Clg2YrELgRsi8SAwV9IS4Dzg7ojYmobC3cD5WdU6Ultzkw8xmVnuTUofhKSVwCnAj0bMWga8XDO9Lm0brb3eti+V1C2pe9OmTRNSb1vJp7mamWUeEJJmA98GPhEROyZ6+xFxdUR0RUTXokWLJmSbbSV3UpuZZRoQkkok4fCNiLi1ziLrgRU108vTttHaJ0Vbc5P7IMws97I8i0nA14CnIuJzoyx2O/DB9GymNwLbI2IDcCdwrqR5kuYB56Ztk6KtVPRgfWaWe1mexXQm8AHgCUmPpW1/BBwBEBFXAXcAbwPWAj3Ah9N5WyX9CfBwut4VEbE1w1r30t5cpKe/TESQ5JyZWf5kFhAR8W/AmL+uERHAZaPMuxa4NoPS9qutuUgloH+wQktTsRElmJk1nK+krqM6omuv7ypnZjnmgKijetOgngHfVc7M8ssBUcfQPSF8JpOZ5ZgDoo6hPQgHhJnlmAOijqE+CJ/qamY55oCoo917EGZmDoh6Wn1fajMzB0Q91T0IH2IyszxzQNThTmozMwdEXT7N1czMAVFXdQ/CfRBmlmcOiDqaiwUK8h6EmeWbA6IOSbT7nhBmlnMOiFG0+q5yZpZzDohRtDcX2dPvwfrMLL8cEKPwfanNLO8cEKNobS6yZ8D3gzCz/MrsjnKSrgV+DdgYEb9YZ/7vA++rqeN4YFF6u9EXgZ3AIFCOiK6s6hxNe8mHmMws37Lcg7gOOH+0mRHxVxFxckScDHwG+OGI+06/NZ0/6eEAybUQPsRkZnmWWUBExH3A1v0umLgYuDGrWg5GW3PRp7maWa41vA9CUjvJnsa3a5oDuEvSI5Iu3c/6l0rqltS9adOmCaurrVSk1wFhZjnW8IAAfh34fyMOL705Ik4FLgAuk/Qro60cEVdHRFdEdC1atGjCimpvLtLjQ0xmlmNTISAuYsThpYhYnz5vBG4DTp/sotpKRQ+1YWa51tCAkNQJvAX4bk3bLElzqq+Bc4EnJ7u21lKRvnKFwUpM9lubmU0JWZ7meiNwFrBQ0jrgcqAEEBFXpYv9BnBXROyuWXUxcJukan3fjIgfZFXnaGpvGjSrJbN/JjOzKSuzX76IuHgcy1xHcjpsbdvzwEnZVDV+tUN+OyDMLI+mQh/ElOSbBplZ3jkgRuGbBplZ3jkgRtHu+1KbWc45IEbR6kNMZpZzDohRDPVBDHjAPjPLJwfEKNqbkzOX9vR7yG8zyycHxCiqexA9HvLbzHLKATGKtpoL5czM8sgBMQqf5mpmeeeAGMXwISYHhJnlkwNiFMWCaG4qeA/CzHLLATGG9mYP+W1m+eWAGIPvCWFmeeaAGENbyXeVM7P8ckCMob2lSE+fr4Mws3xyQIyho7XEjl4HhJnlkwNiDJ1tJXbsGWh0GWZmDeGAGENHa4ntDggzy6nMAkLStZI2SnpylPlnSdou6bH08dmaeedLelrSWkmfzqrG/eloa2JHrwPCzPIpyz2I64Dz97PM/RFxcvq4AkBSEfgScAFwAnCxpBMyrHNUnW0legcq9JV9JpOZ5U9mARER9wFbD2LV04G1EfF8RPQDNwEXTmhx49TRVgJgxx53VJtZ/jS6D+JNkn4i6Z8krUrblgEv1yyzLm2rS9KlkroldW/atGlCi+usBoQPM5lZDjUyIB4FjoyIk4C/A75zMBuJiKsjoisiuhYtWjShBXa0JgHhjmozy6OGBURE7IiIXenrO4CSpIXAemBFzaLL07ZJN3yIyQFhZvnTsICQdLgkpa9PT2vZAjwMHCPpKEnNwEXA7Y2osbMtue2oL5YzszxqymrDkm4EzgIWSloHXA6UACLiKuBdwO9KKgN7gIsiIoCypI8BdwJF4NqIWJ1VnWPxISYzy7PMAiIiLt7P/C8CXxxl3h3AHVnUdSB8iMnM8mxch5gkvXs8bTNNa6lIc1PBAWFmuTTePojPjLNtxulsK/k0VzPLpTEPMUm6AHgbsEzSlTWzOoBc9Nx2tDa5D8LMcml/fRCvAN3A24FHatp3Ap/MqqipJBnRNRdZaGa2lzEDIiJ+AvxE0jcjYgBA0jxgRURsm4wCG62jrcTW3f2NLsPMbNKNtw/ibkkdkuaTXAH995I+n2FdU4aH/DazvBpvQHRGxA7gncANEXEGcE52ZU0dvmmQmeXVeAOiSdIS4D3A9zOsZ8pJ7glRJrmGz8wsP8YbEFeQXNn8XEQ8LOl1wLPZlTV1dLaVGKwEu/t9Twgzy5dxXUkdEd8CvlUz/Tzwm1kVNZVUh9vYsWeA2S2ZXXhuZjbljPdK6uWSbktvIbpR0rclLc+6uKmgek8Id1SbWd6M9xDT10lGVF2aPr6Xts14Ho/JzPJqvAGxKCK+HhHl9HEdMLF355miPKKrmeXVeANii6T3Syqmj/eT3Lthxhu+7aivpjazfBlvQPxnklNcXwU2kNzL4UMZ1TSldFRvGuQ9CDPLmfGelnMFcEl1eI30iuq/JgmOGW2ODzGZWU6Ndw/ixNqxlyJiK3BKNiVNLcWCmNPS5CG/zSx3xhsQhXSQPmBoD2J/Q4Vfm54S++Qo898n6XFJT0h6QNJJNfNeTNsfk9Q9zhoz09Hm8ZjMLH/Ge4jpb4B/l1S9WO7dwJ/tZ53rSG4pesMo818A3hIR29L7TlwNnFEz/60RsXmc9WWqw0N+m1kOjfdK6hvSv+TPTpveGRFr9rPOfZJWjjH/gZrJB4Epe+FdZ1uTO6nNLHfGPXZEGghjhsIh+AjwT7VvB9wlKYCvRsTVo60o6VLgUoAjjjgik+I6Wku8tLUnk22bmU1VDR9cSNJbSQLizTXNb46I9ZIOI7kXxU8j4r5666fhcTVAV1dXJkOudnjIbzPLofF2UmdC0onANcCFETF04V1ErE+fNwK3Aac3psJEpzupzSyHGhYQko4AbgU+EBHP1LTPkjSn+ho4F6h7JtRk6Wgtsbt/kPJgpZFlmJlNqswOMUm6ETgLWChpHXA5UAKIiKuAzwILgC9LAihHRBewGLgtbWsCvhkRP8iqzvHorF5N3Vtm/qzmRpZiZjZpMguIiLh4P/M/Cny0TvvzwEn7rtE4tSO6OiDMLC8a2gcxXQwP2Od+CDPLDwfEOHT4pkFmlkMOiHEYvu2or6Y2s/xwQIyDbztqZnnkgBiHoXtCuA/CzHLEATEObaUipaK8B2FmueKAGAdJdLR6uA0zyxcHxDh1tpV8X2ozyxUHxDjN8XhMZpYzDohxWjS7mVe372l0GWZmk8YBMU7HL+nguU276R0YbHQpZmaTwgExTics6WCwEjzz2s5Gl2JmNikcEOO0amknAKtf2dHgSszMJocDYpxWzG9jTksTq1/Z3uhSzMwmhQNinCRx/NIO70GYWW44IA7AqqUd/HTDTgYrmdz62sxsSnFAHIBVSzvZMzDIC5t3N7oUM7PMZRoQkq6VtFFS3XtKK3GlpLWSHpd0as28SyQ9mz4uybLO8Vq1tAPA/RBmlgtZ70FcB5w/xvwLgGPSx6XAVwAkzSe5h/UZwOnA5ZLmZVrpOLz+sNk0Fwus2eB+CDOb+TINiIi4D9g6xiIXAjdE4kFgrqQlwHnA3RGxNSK2AXczdtBMilKxwLGHz2aNO6rNLAca3QexDHi5Znpd2jZa+z4kXSqpW1L3pk2bMiu0atWSTla/soMId1Sb2czW6IA4ZBFxdUR0RUTXokWLMn+/E5Z2sHV3P6/u6M38vczMGqnRAbEeWFEzvTxtG6294aod1T7MZGYzXaMD4nbgg+nZTG8EtkfEBuBO4FxJ89LO6XPTtoY7bkkHkofcMLOZrynLjUu6ETgLWChpHcmZSSWAiLgKuAN4G7AW6AE+nM7bKulPgIfTTV0REWN1dk+a2S1NrFwwy6e6mtmMl2lARMTF+5kfwGWjzLsWuDaLug7VG5Z18sBzmykPVmgqNnonzMwsG/51Owhve8MSNu/q5/61mxtdiplZZhwQB+Hs4w5jbnuJWx+dEv3mZmaZcEAchOamAm8/aSl3rX6VHb2+T7WZzUwOiIP0zlOX01eucMfjGxpdiplZJhwQB+mk5Z0cvWiWDzOZ2YzlgDhIknjnqct56MWtvLSlp9HlmJlNOAfEIfiNU5Yhwa0/XtfoUszMJpwD4hAsndvGLx+9gNt+vN6D95nZjOOAOES/fuJSfralh5++urPRpZiZTSgHxCE65/jFSHDn6lcbXYqZ2YRyQByiRXNa6DpyHnetfq3RpZiZTSgHxAQ494TDWbNhBy9v9dlMZjZzOCAmwHmrDgd8mMnMZhYHxAQ4YkE7xx0+h7vW+DCTmc0cDogJcu6qw+l+cStbdvU1uhQzswnhgJgg561aTCXgnqc2NroUM7MJ4YCYICcs6WDZ3Db3Q5jZjJFpQEg6X9LTktZK+nSd+Z+X9Fj6eEbSz2vmDdbMuz3LOieCJM5dtZj7126md2Cw0eWYmR2yzAJCUhH4EnABcAJwsaQTapeJiE9GxMkRcTLwd8CtNbP3VOdFxNuzqnMi/fLRC+kvV3h8ne9XbWbTX5Z7EKcDayPi+YjoB24CLhxj+YuBGzOsJ3NdR84D4OEXtza4EjOzQ5dlQCwDXq6ZXpe27UPSkcBRwL/UNLdK6pb0oKR3ZFfmxJk3q5ljF8/moRccEGY2/U2VTuqLgFsiovbg/ZER0QW8F/hbSUfXW1HSpWmQdG/atGkyah3TaSvn8+jPtjFY8eiuZja9ZRkQ64EVNdPL07Z6LmLE4aWIWJ8+Pw/8K3BKvRUj4uqI6IqIrkWLFh1qzYfs9KPms7OvzFMbdjS6FDOzQ5JlQDwMHCPpKEnNJCGwz9lIko4D5gH/XtM2T1JL+nohcCawJsNaJ8xpK+cD7ocws+kvs4CIiDLwMeBO4Cng5ohYLekKSbVnJV0E3BR733HneKBb0k+Ae4G/iIhpERBL57axbG6bA8LMpr2mLDceEXcAd4xo++yI6f9ZZ70HgDdkWVuWTj9qPvc/u5mIQFKjyzEzOyhTpZN6Rjlt5Xw27+rjxS0e/tvMpi8HRAZOPyq9HsKnu5rZNOaAyMDRi2Yzf1YzD7kfwsymMQdEBiTRdeQ8d1Sb2bTmgMjIm45ewM+29PiqajObthwQGfmt01awbG4b/+O2J+gvVxpdjpnZAXNAZKS9uYkrLlzFsxt38ff3P9/ocszMDpgDIkPnHL+YC37xcK6851l+tmV3o8sxMzsgDoiMXf7rqygVC/zxd55k74vFzcymNgdExg7vbOX3z/sF7n92M5/61k/cH2Fm00amQ21Y4oNvOpLtewb43N3PsGlnH19+36nMaS01uiwzszF5D2ISSOL3zjmGv3zXiTzw3BZ+66sP8tymXY0uy8xsTA6ISfSerhVc+6HTWLethwv+9n6uvOdZH3IysynLATHJ3nLsIu751Fmcu2oxn7v7GX71yvu5c/WrVHwHOjObYhwQDbBoTgtffO+pfP1Dp9E/WOG3/+ERLvjC/Xz3sfUMDHqPwsymBs2kUy+7urqiu7u70WUckPJghe8/voEv3buWZzfuYuHsZi48eRm/eepyTlja0ejyzGyGk/RIRHTVneeAmBoqleDepzdyyyPr+OenXmNgMFi5oJ2zj1vM2ccdRtfKebSWio0u08xmGAfENLNtdz/ff2ID9zz1Gg88t4X+coXmYoGTVnRy2sr5nHLEPN6wrJPFHS2+Y52ZHZKGBYSk84EvAEXgmoj4ixHzPwT8FbA+bfpiRFyTzrsE+OO0/U8j4vr9vd9MCYhaPf1lHli7hR+9sIWHXtzGk+u3M5h2aC+c3cIJSzs49rDZHLt4DkcfNpujFs5iXnvJwWFm49KQgJBUBJ4B/hOwDngYuDgi1tQs8yGgKyI+NmLd+UA30AUE8AjwSxGxbaz3nIkBMVJPf5k1r+zgyfXbeWL9Dp5+bQfPvraLvprTZTvbShy5oJ0V89pZPq+NZfPaWNLZxpLOVpbObXOAmNmQsQIiyyupTwfWRsTzaRE3ARcCa8ZcK3EecHdEbE3XvRs4H7gxo1qnjfbmJrpWzqdr5fyhtsFK8PLWHp7fvIvnN+3mhc27eWlrD2s27ODuNa/RP+LMqOZigcM6Wljc0crC2c0smtPCwtktLJjVzILZLcyf1cyCWc3MbW9mXnuJpqJPdjPLoywDYhnwcs30OuCMOsv9pqRfIdnb+GREvDzKusvqvYmkS4FLAY444ogJKHv6KRbEyoWzWLlwFmcft/e8SiXYvKuPV7b3suHne9iwvZfXdvaycUcfr+3o5YXNu3n4xW1s3d0/6vbntDYxt73EvPZmOttKdLSVmJs+d6aPOa1NdLQmz3NaS3Skz62lgvdWzKapRo/F9D3gxojok/TbwPXA2QeygYi4GrgakkNME1/i9FYoiMM6Wjmso5WTV8wddbmBwQrbevrZurufLbv62dbTz7bd/WzZ3c/Pewb4eU8/23oG2L5ngPXb9rB9T/K6vJ8L/JoKYnZrE7Nbah6tTcxqaWJ2c/rcUqS9peZ1c7Jce3ORWdXn5ibaW4o0Fx04ZpMly4BYD6yomV7OcGc0ABGxpWbyGuAva9Y9a8S6/zrhFdqQUrHAYXNaOWxO67jXiQj2DAyyfc8AO3vL7Kg+9ybPO3vL7OobYFf6emdfmd19Zbbs6uelrT3s7iuzu2+Q3f1lxtsVVixor8Bob04CJXku0lYafj3U3rLvvNbS8DJtaVvJh9LM9pJlQDwMHCPpKJIf/IuA99YuIGlJRGxIJ98OPJW+vhP4c0nz0ulzgc9kWKsdBEnpj3ATSzoPfjvVoNlVDYy+Mj39Nc/95aHXPf3JMj391enk9bbd/azbNsie6jL9gwc8zlWpKNpKw6HRlobI3q+b9mqvBk1bKXnd1lyktakwtE7r0KNAW6no/hybVjILiIgoS/oYyY99Ebg2IlZLugLojojbgd+T9HagDGwFPpSuu1XSn5CEDMAV1Q5rm3lqg4Y5E7fd8mCFPQPV0EiCZs9QqAyyZ6DMnv4KPdX2geGA6ekfpHcgWW5nb5lNO/vYk05XlzmY4bOaChoKjNrwaG1KwqWlKZmuPtfOb0mnW5qGn1ua0udSzet02eZiYejZwWQHwxfKmR2EiKCvXKFvoJIGR5negeFA6h0YpLecvi5X6Ku2lwfZ058s11e7zECF3nLy3DdQXT99Xa4MXftysIoF0Vws0NyUPooFWtLXpeJwW3W6palAqah95peqj6bh7TUV9l42eWgomEpFDbU3FUWpkKxfXa+pWKApra9QcP/SZGvUaa5mM5akoT2ATrK/+dPAYIW+ciUJjoHk8FlfebitryZM+tNH78Ag/YPD0/2DyTL96bZq2wfS5Xp6yvQPBv3lQQYGY6/51WWyHHi4IJJQKWgoXIqFfcOkqZi07fW6qGS6UKBY87qpoKHpYiEJq2SbqnkuDE8X920v1CxfVLK9orTvvJpHU0EUlNRQKDA8T8lzoWaZavtUC0gHhNk0UP0LfHZL4/+XHazEUGCVBysMDMZQgFRDZaBSYSANl/LQ/KBcGV6+XLctGKgkz+XBCgOV5LlciaStOq+SPtJ1esplypVgYDCoVNLl0nUGa6YH0+1XKlCuZBt2B2s4LEjCRcPhItUGDEOhsnBWCzf/zpsmvJbG/9dmZtNK8mNVnBGDR1YqwWBUQyQJnMFKMj1QScKmGjCDaahU51fXqW6jGkbV7dVdpmabldh3/nA9UEm3WV1uMIIY2jZ7tc/J6A8HB4SZ5VahIAqIGZB1mfCpDWZmVpcDwszM6nJAmJlZXQ4IMzOrywFhZmZ1OSDMzKwuB4SZmdXlgDAzs7pm1GB9kjYBPzvI1RcCmyewnOkgj58Z8vm58/iZIZ+f+0A/85ERsajejBkVEIdCUvdoIxrOVHn8zJDPz53Hzwz5/NwT+Zl9iMnMzOpyQJiZWV0OiGFXN7qABsjjZ4Z8fu48fmbI5+eesM/sPggzM6vLexBmZlaXA8LMzOrKfUBIOl/S05LWSvp0o+vJiqQVku6VtEbSakkfT9vnS7pb0rPp87xG1zrRJBUl/VjS99PpoyT9KP3O/1FSc6NrnGiS5kq6RdJPJT0l6U0z/buW9Mn0v+0nJd0oqXUmfteSrpW0UdKTNW11v1slrkw//+OSTj2Q98p1QEgqAl8CLgBOAC6WdEJjq8pMGfhURJwAvBG4LP2snwbuiYhjgHvS6Znm48BTNdP/G/h8RLwe2AZ8pCFVZesLwA8i4jjgJJLPP2O/a0nLgN8DuiLiF4EicBEz87u+Djh/RNto3+0FwDHp41LgKwfyRrkOCOB0YG1EPB8R/cBNwIUNrikTEbEhIh5NX+8k+cFYRvJ5r08Xux54R2MqzIak5cCvAtek0wLOBm5JF5mJn7kT+BXgawAR0R8RP2eGf9ckt1Buk9QEtAMbmIHfdUTcB2wd0Tzad3shcEMkHgTmSloy3vfKe0AsA16umV6Xts1oklYCpwA/AhZHxIZ01qvA4gaVlZW/Bf4AqKTTC4CfR0Q5nZ6J3/lRwCbg6+mhtWskzWIGf9cRsR74a+AlkmDYDjzCzP+uq0b7bg/pNy7vAZE7kmYD3wY+ERE7audFcs7zjDnvWdKvARsj4pFG1zLJmoBTga9ExCnAbkYcTpqB3/U8kr+WjwKWArPY9zBMLkzkd5v3gFgPrKiZXp62zUiSSiTh8I2IuDVtfq26y5k+b2xUfRk4E3i7pBdJDh+eTXJsfm56GAJm5ne+DlgXET9Kp28hCYyZ/F3/R+CFiNgUEQPArSTf/0z/rqtG+24P6Tcu7wHxMHBMeqZDM0mn1u0NrikT6bH3rwFPRcTnambdDlySvr4E+O5k15aViPhMRCyPiJUk3+2/RMT7gHuBd6WLzajPDBARrwIvS/qFtOkcYA0z+LsmObT0Rknt6X/r1c88o7/rGqN9t7cDH0zPZnojsL3mUNR+5f5KaklvIzlOXQSujYg/a3BJmZD0ZuB+4AmGj8f/EUk/xM3AESRDpb8nIkZ2gE17ks4C/ntE/Jqk15HsUcwHfgy8PyL6GlnfRJN0MknHfDPwPPBhkj8IZ+x3Lel/Ab9Fcsbej4GPkhxvn1HftaQbgbNIhvV+Dbgc+A51vts0LL9IcritB/hwRHSP+73yHhBmZlZf3g8xmZnZKBwQZmZWlwPCzMzqckCYmVldDggzM6vLAWE2BklnVUeBPcj13yHpsxNZU822/0zSy5J2jWhvSUcuXZuOZLqyZt5n0vanJZ2XtjVLuq/mgjIzwAFhlrU/AL58qBsZ5cf7eyQDTo70EWBbOoLp50lGNCUdvfciYBXJefFfllRMB6q8h+QaArMhDgib9iS9X9JDkh6T9NV0GHck7ZL0+fQeAfdIWpS2nyzpwXR8/Ntqxs5/vaR/lvQTSY9KOjp9i9k191b4RnrxEZL+Qsn9NR6X9Nd16joW6IuIzen0dZKuktQt6Zl0rKjq/Sr+StLD6bZ+O20/S9L9km4nuSp4LxHx4ChXxdaO7AGvlL8AAALoSURBVHkLcE5a84XATRHRFxEvAGsZDpjvAO87wH96m+EcEDatSTqe5C/fMyPiZGCQ4R+6WUB3RKwCfkhyxSnADcAfRsSJJFeWV9u/AXwpIk4CfplkVFBIRr79BMk9Q14HnClpAfAbwKp0O39ap7wzgUdHtK0k+VH+VeAqSa0kf/Fvj4jTgNOA/yLpqHT5U4GPR8SxB/DPMjSCZzqS6XaSUWzHGtnzyfS9zYb4mKNNd+cAvwQ8nP5h38bwQGUV4B/T1/8HuDW9V8LciPhh2n498C1Jc4BlEXEbQET0AqTbfCgi1qXTj5H8yD8I9AJfS/so6vVTLCEZdrvWzRFRAZ6V9DxwHHAucKKk6phBnSQ3eOlP3/uFA/1HOVARMSipX9Kc9H4hZg4Im/YEXB8RnxnHsgc7rkzt2D2DQFNElCWdThJQ7wI+RjJabK09JD/2Y9UQJJ/hv0bEnbUz0vGjdh9EvdURPNelfRedwBb2P7JnC0nomQE+xGTT3z3AuyQdBkP35j0ynVdgeCTP9wL/FhHbgW2S/kPa/gHgh+lfzeskvSPdTouk9tHeNL2vRmdE3AF8kuS2niM9Bbx+RNu7JRXS/o3XAU8DdwK/mw7HjqRjldzg52DVjuz5LpJRbCNtvyj9bEeR7KU8lL7nAmBzOlS2GeA9CJvmImKNpD8G7pJUAAaAy0hGtNwNnJ7O38jwWTqXkBz/b2d4pFNIwuKrkq5It/PuMd56DvDdtA9BwH+rs8x9wN9IUgyPivkSyY9yB/A7EdEr6RqSw1aPpp3JmxjHrTEl/SVJ8LVLWgdcExH/k2RY93+QtJbk1pQXpf9WqyXdTNLhXQYui4jBdHNvBf7v/t7T8sWjudqMJWlXRMxucA1fAL4XEf8s6Trg+xFxy35Wm3SSbgU+HRHPNLoWmzp8iMksW38OjHqoaipQcrOs7zgcbCTvQZiZWV3egzAzs7ocEGZmVpcDwszM6nJAmJlZXQ4IMzOr6/8DIXyAY7u4HYoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jmVWTypFYLI1"
      },
      "source": [
        "def predict(parameters, X):\n",
        "    \"\"\"\n",
        "    Using the learned parameters, predicts a class for each example in X\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters \n",
        "    X -- input data of size (n_x, m)\n",
        "    \n",
        "    Returns\n",
        "    predictions -- vector of predictions of our model\n",
        "    \"\"\"\n",
        "    \n",
        "    # Computes probabilities using forward propagation, and classifies to 0/1 using 0.5 as the threshold.\n",
        "    A2, cache = forward_propagation(X, parameters)\n",
        "\n",
        "    predictions = np.argmax(A2, axis = 0)\n",
        "    return predictions"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rdGHLTyTI6Gs",
        "outputId": "a840d99b-41a0-4d5d-e837-c52371b8e6d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "yhat_test = predict(parameters, X_test)\n",
        "y = np.argmax(y_test, axis = 0)\n",
        "np.mean(yhat_test == y)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.873"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9Xx-OKoSExSy"
      },
      "source": [
        "## 04.2-Mini-Batch Gradient Descent\n",
        "\n",
        "![](https://raw.githubusercontent.com/happyrabbit/course2020/master/slides/02DeepLearning/DNN/images/batchexp.png)\n",
        "\n",
        "Let's update the parameters using mini-batches gradient descent."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ENpD3RL8ZWMq"
      },
      "source": [
        "def random_mini_batches(X, Y, mini_batch_size = 64):\n",
        "    \"\"\"\n",
        "    Creates a list of random minibatches from (X, Y)\n",
        "    \n",
        "    Arguments:\n",
        "    X -- input data, of shape (input size, number of examples)\n",
        "    Y -- true \"label\" of shape (10, number of examples)\n",
        "    mini_batch_size -- size of the mini-batches, integer\n",
        "    \n",
        "    Returns:\n",
        "    mini_batches -- list of synchronous (mini_batch_X, mini_batch_Y)\n",
        "    \"\"\"\n",
        "    \n",
        "    m = X.shape[1]                  # number of training examples\n",
        "    mini_batches = []\n",
        "        \n",
        "    # Step 1: Shuffle (X, Y)\n",
        "    permutation = list(np.random.permutation(m))\n",
        "    shuffled_X = X[:, permutation]\n",
        "    shuffled_Y = Y[:, permutation]\n",
        "\n",
        "    # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case.\n",
        "    num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning\n",
        "    for k in range(0, num_complete_minibatches):\n",
        "        mini_batch_X = shuffled_X[:,k * mini_batch_size:(k + 1) * mini_batch_size]\n",
        "        mini_batch_Y = shuffled_Y[:,k * mini_batch_size:(k + 1) * mini_batch_size]\n",
        "        mini_batch = (mini_batch_X, mini_batch_Y)\n",
        "        mini_batches.append(mini_batch)\n",
        "    \n",
        "    # Handling the end case (last mini-batch < mini_batch_size)\n",
        "    if m % mini_batch_size != 0:\n",
        "        end = m - mini_batch_size * math.floor(m / mini_batch_size)\n",
        "        mini_batch_X = shuffled_X[:,num_complete_minibatches * mini_batch_size:]\n",
        "        mini_batch_Y = shuffled_Y[:,num_complete_minibatches * mini_batch_size:]\n",
        "        mini_batch = (mini_batch_X, mini_batch_Y)\n",
        "        mini_batches.append(mini_batch)\n",
        "    \n",
        "    return mini_batches"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "blDQeacfGmA0"
      },
      "source": [
        "def model(X, Y, n_h = 4,  learning_rate= 0.1, l2 = 0, mini_batch_size=64, num_epochs=20, print_cost=True):\n",
        "    \"\"\"\n",
        "    3-layer neural network model which can be run in different optimizer modes.\n",
        "    \n",
        "    Arguments:\n",
        "    X -- input data, of shape (784, number of examples)\n",
        "    Y -- true \"label\" of shape (10, number of examples)\n",
        "    layers_dims -- python list, containing the size of each layer\n",
        "    learning_rate -- the learning rate, scalar.\n",
        "    l2 -- l2 panalty parameter\n",
        "    mini_batch_size -- the size of a mini batch\n",
        "    num_epochs -- number of epochs\n",
        "    print_cost -- True to print the cost every 1000 epochs\n",
        "\n",
        "    Returns:\n",
        "    parameters -- python dictionary containing your updated parameters\n",
        "    costs -- a list of cost per 100 epoch \n",
        "    \"\"\"\n",
        "\n",
        "    costs = []  # to keep track of the cost\n",
        "    n_x = X.shape[0]\n",
        "    n_y = Y.shape[0]\n",
        "    \n",
        "    # Initialize parameters\n",
        "    parameters = initialize_parameters(n_x, n_h, n_y)\n",
        "    \n",
        "    # Optimization loop\n",
        "    for i in range(num_epochs):\n",
        "        \n",
        "        # Define the random minibatches. We increment the seed to reshuffle differently the dataset after each epoch\n",
        "        minibatches = random_mini_batches(X, Y, mini_batch_size)\n",
        "\n",
        "        for minibatch in minibatches:\n",
        "\n",
        "            # Select a minibatch\n",
        "            (minibatch_X, minibatch_Y) = minibatch\n",
        "            \n",
        "            # Forward propagation. Inputs: \"X, parameters\". Outputs: \"A2, cache\".\n",
        "            A2, cache = forward_propagation(X, parameters)\n",
        "            \n",
        "            # Cost function. Inputs: \"A2, Y, parameters\". Outputs: \"cost\".\n",
        "            cost = compute_cost(A2, Y, parameters)\n",
        "            \n",
        "            # Backpropagation. Inputs: \"parameters, cache, X, Y\". Outputs: \"grads\".\n",
        "            grads = backward_propagation(parameters, cache, X, Y)\n",
        " \n",
        "            # Gradient descent parameter update. Inputs: \"parameters, grads\". Outputs: \"parameters\".\n",
        "            parameters = update_parameters(parameters, grads, learning_rate= learning_rate, l2 = l2)\n",
        "        \n",
        "        if print_cost:\n",
        "            print(\"Cost after epoch %i: %f\" % (i, cost))\n",
        "        costs.append(cost)\n",
        "                \n",
        "    # plot the cost\n",
        "    plt.plot(costs)\n",
        "    plt.ylabel('cost')\n",
        "    plt.xlabel('epochs')\n",
        "    plt.title(\"Learning rate = \" + str(learning_rate))\n",
        "    plt.show()\n",
        "\n",
        "    return parameters, costs, dw1s"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bgtJmQIJJms3",
        "outputId": "812b9dd6-42fa-43b8-8249-e5d6b3ff890e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 782
        }
      },
      "source": [
        "parameters, costs = model(X_train, y_train, num_epochs=20)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cost after epoch 0: 0.613234\n",
            "Cost after epoch 1: 0.517477\n",
            "Cost after epoch 2: 0.479951\n",
            "Cost after epoch 3: 0.456474\n",
            "Cost after epoch 4: 0.442755\n",
            "Cost after epoch 5: 0.434352\n",
            "Cost after epoch 6: 0.428594\n",
            "Cost after epoch 7: 0.424224\n",
            "Cost after epoch 8: 0.420781\n",
            "Cost after epoch 9: 0.417941\n",
            "Cost after epoch 10: 0.415551\n",
            "Cost after epoch 11: 0.416622\n",
            "Cost after epoch 12: 0.416572\n",
            "Cost after epoch 13: 0.415844\n",
            "Cost after epoch 14: 0.414793\n",
            "Cost after epoch 15: 0.413713\n",
            "Cost after epoch 16: 0.412917\n",
            "Cost after epoch 17: 0.412736\n",
            "Cost after epoch 18: 0.411680\n",
            "Cost after epoch 19: 0.409923\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xdZZ3v8c83SZP0stNb0qQ3aKFNkVsrVEBAZWTE4miLoIgigqiMF0YdZxzxzDmOg+M5ema8nuGoiAiMKCjqUAQGQQUPINiAbaVAaXqB3q/0mra5/c4fayXshlyb7Oxcvu/Xa7322s961tq/tZLsX571rPUsRQRmZmbdVZDvAMzMbHBx4jAzsx5x4jAzsx5x4jAzsx5x4jAzsx5x4jAzsx5x4jDrhKQ3SFqZ7zjMBhInDhuwJK2T9Jf5jCEi/l9EzMlnDC0knSdpQ54++32SXpR0QNJ/SprQSd0bJa2U1Czpqn4M0/qJE4cNa5IK8x0DgBID8u9R0knA94ArgEqgDvi/nayyDPg48HTuo7N8GJC/qGadkVQg6TpJqyXtlPTT7P+AJf1M0hZJeyT9Pv3ia1l2i6TvSLpP0gHgL9KWzd9LWp6uc6ek0rT+Ef/ld1Y3Xf4PkjZL2iTpw5JC0qwO9uNhSV+W9BjJl/Fxkj4o6TlJ+yStkfTXad3RwP3AFEn702lKV8eij1wO3BMRv4+I/cD/AC6WlGmvckTcEBG/AQ71cRw2QDhx2GD0N8BFwJuAKcDLwA1Zy+8HZgOTSP7rvb3N+u8DvgxkgEfTskuBBcBM4FTgqk4+v926khYAnwH+EpgFnNeNfbkCuCaN5UVgG/B2oAz4IPANSadFxAHgQmBTRIxJp03dOBatJB0jaXcn0/s6iPEkklYEABGxGqgHqruxfzYEFeU7ALOj8FHg2ojYACDpi8BLkq6IiMaIuLmlYrrsZUljI2JPWnx3RDyWzh+SBPDt9IsYSfcA8zr5/I7qXgr8MCJWZH325V3syy0t9VP3Zs0/IunXwBvo+LRPp8ciu2JEvASM6yKe9owB9rQp20OS7GwYcovDBqNjgV+2/KcMPAc0AZWSCiV9JT11sxdYl65TnrX++na2uSVrvo7ky7IjHdWd0mbb7X1OW0fUkXShpCck7Ur37W0cGXtbHR6Lbnx2d+0naQFlKwP29eFn2CDixGGD0XrgwogYlzWVRsRGktNQi0hOF40FZqTrKGv9XA0JvRmYlvV+ejfWaY1FUgnwc+DfgMqIGAfcxyuxtxd3Z8fiCOmpqv2dTB21jlYAc7O2cxxQArzQjf2zIciJwwa6EZJKs6Yi4LvAlyUdCyCpQtKitH4GOAzsBEYB/7MfY/0p8EFJr5E0iqQTuSeKSb6QtwONki4ELshavhWYKGlsVllnx+IIEfFSVv9Ie1PbvqAWtwPvUHJPy2jgeuAXEdFui0NScXrBgHjl5+fvmiHEP0wb6O4DDmZNXwS+BSwGfi1pH/AEcGZa/zaSTuaNwLPpsn4REfcD3wZ+B9Rmffbhbq6/D/gkSQJ6maT1tDhr+fPAT4A16ampKXR+LPpE2gfzUZIEso0kOX+8Zbmk+yX9t6xVfk3yszobuDGdf2NfxmT5JT/IySw3JL0GeAYoadtRbTaYucVh1ockvVNSiaTxwFdJ7n9w0rAhxYnDrG/9NcnpnNUkVzd9LL/hmPU9n6oyM7MecYvDzMx6ZFjcOV5eXh4zZszIdxhmZoPKU089tSMiKtqWD4vEMWPGDGpqavIdhpnZoCLpxfbKfarKzMx6xInDzMx6xInDzMx6xInDzMx6xInDzMx6xInDzMx6xInDzMx6xImjE3cv3ciPnmj3MmYzs2HLiaMT9/95Czc/ujbfYZiZDShOHJ2orsqwbucBDjU05TsUM7MBw4mjE3MqMzQH1G7bn+9QzMwGDCeOTsypGgPAC1vbfbSymdmw5MTRiWMnjqa4sICVThxmZq2cODoxorCA4ypG88IWJw4zsxZOHF2YU5Xhha3u4zAza+HE0YXqygwbdx9k36GGfIdiZjYgOHF0YU5lBsCtDjOzlBNHF+ZUtSQO93OYmUGOE4ekBZJWSqqVdF0HdS6V9KykFZJ+nFV+paRV6XRlVvnpkv6cbvPbkpTLfZg6biSjigtZ6Q5yMzMgh88cl1QI3AC8BdgALJG0OCKezaozG/g8cE5EvCxpUlo+AfgnYD4QwFPpui8D3wE+AjwJ3AcsAO7P1X4UFIjZlRm3OMzMUrlscZwB1EbEmoioB+4AFrWp8xHghjQhEBHb0vK3Ag9GxK502YPAAkmTgbKIeCIiArgNuCiH+wDAnMoxThxmZqlcJo6pwPqs9xvSsmzVQLWkxyQ9IWlBF+tOTec72yYAkq6RVCOpZvv27b3YjeTKqh3769mx/3CvtmNmNhTku3O8CJgNnAe8F/i+pHF9seGIuDEi5kfE/IqKil5tyx3kZmavyGXi2AhMz3o/LS3LtgFYHBENEbEWeIEkkXS07sZ0vrNt9rmWS3LdQW5mltvEsQSYLWmmpGLgMmBxmzr/SdLaQFI5yamrNcADwAWSxksaD1wAPBARm4G9ks5Kr6b6AHB3DvcBgIpMCeNHjXCLw8yMHF5VFRGNkq4lSQKFwM0RsULS9UBNRCzmlQTxLNAEfDYidgJI+hJJ8gG4PiJ2pfMfB24BRpJcTZWzK6paSKK6MuMWh5kZOUwcABFxH8kls9llX8iaD+Az6dR23ZuBm9sprwFO7vNguzCnKsMvnt5IRJDjW0fMzAa0fHeODxrVlRn2H25k055D+Q7FzCyvnDi6qfXKKp+uMrNhzomjm6onpVdWuYPczIY5J45uGjtqBFVlpW5xmNmw58TRA9VVGbc4zGzYc+LogTmVY1i1bT9NzZHvUMzM8saJoweqKzPUNzbz4s4D+Q7FzCxvnDh6wGNWmZk5cfTIrEljkGDlFj9G1syGLyeOHhhVXMQxE0a5xWFmw5oTRw9VV/rKKjMb3pw4emhOZYa1Ow5wuLEp36GYmeWFE0cPVVdlaGoO1mz3lVVmNjw5cfRQy0Od3M9hZsOVE0cPzSwfTVGB/GwOMxu2cpo4JC2QtFJSraTr2ll+laTtkpam04fT8r/IKlsq6ZCki9Jlt0ham7VsXi73oa3iogKOrxjjFoeZDVs5e5CTpELgBuAtJM8WXyJpcUQ826bqnRFxbXZBRPwOmJduZwJQC/w6q8pnI+KuXMXeleqqDEvXv5yvjzczy6tctjjOAGojYk1E1AN3AIuOYjvvAu6PiLo+ja4X5lSOYf2ugxw43JjvUMzM+l0uE8dUYH3W+w1pWVuXSFou6S5J09tZfhnwkzZlX07X+YakkvY+XNI1kmok1Wzfvv2odqAj1WkH+aptvoPczIaffHeO3wPMiIhTgQeBW7MXSpoMnAI8kFX8eeAE4HXABOBz7W04Im6MiPkRMb+ioqJPg24Zs2rllr19ul0zs8Egl4ljI5DdgpiWlrWKiJ0RcTh9exNwepttXAr8MiIastbZHInDwA9JTon1q+njR1E6osBjVpnZsJTLxLEEmC1ppqRiklNOi7MrpC2KFguB59ps4720OU3Vso4kARcBz/Rx3F0qKBDVlRlfWWVmw1LOrqqKiEZJ15KcZioEbo6IFZKuB2oiYjHwSUkLgUZgF3BVy/qSZpC0WB5ps+nbJVUAApYCH83VPnSmujLDIy/0bd+JmdlgkLPEARAR9wH3tSn7Qtb850n6LNpbdx3tdKZHxJv7NsqjM6cyw11PbWDXgXomjC7OdzhmZv0m353jg1a1H+pkZsOUE8dR8phVZjZcOXEcpcqyEspKizxmlZkNO04cR0kSc6p8ZZWZDT9OHL1QXZlh5ZZ9RES+QzEz6zdOHL0wpyrD3kONbN17uOvKZmZDhBNHL7SMWeVnkJvZcOLE0QstieMFd5Cb2TDixNELE0YXU5EpcYvDzIYVJ45emuMxq8xsmHHi6KWWS3Kbm31llZkND04cvTSnMsOhhmbWvzxgHlBoZpZTThy9VN36UCefrjKz4cGJo5dmTxoDeMwqMxs+nDh6aXRJEdMnjGTlVj8N0MyGByeOPjCnMuN7Ocxs2Mhp4pC0QNJKSbWSrmtn+VWStktamk4fzlrWlFW+OKt8pqQn023emT6WNq+qKzOs3r6f+sbmfIdiZpZzOUsckgqBG4ALgROB90o6sZ2qd0bEvHS6Kav8YFb5wqzyrwLfiIhZwMvAh3K1D901pypDY3OwbueBfIdiZpZzuWxxnAHURsSaiKgH7gAW9WaDkgS8GbgrLboVuKhXUfaB1jGrfLrKzIaBXCaOqcD6rPcbaOcZ4sAlkpZLukvS9KzyUkk1kp6Q1JIcJgK7I6Kxi20i6Zp0/Zrt27f3clc6d1zFaAoL5MRhZsNCvjvH7wFmRMSpwIMkLYgWx0bEfOB9wDclHd+TDUfEjRExPyLmV1RU9F3E7SgpKmRm+WiPWWVmw0IuE8dGILsFMS0taxUROyOi5WEWNwGnZy3bmL6uAR4GXgvsBMZJKupom/niMavMbLjIZeJYAsxOr4IqBi4DFmdXkDQ56+1C4Lm0fLykknS+HDgHeDaSR+39DnhXus6VwN053Iduq67M8NKuOurqG7uubGY2iOUscaT9ENcCD5AkhJ9GxApJ10tquUrqk5JWSFoGfBK4Ki1/DVCTlv8O+EpEPJsu+xzwGUm1JH0eP8jVPvTEnKoxREDtNt8IaGZDW1HXVY5eRNwH3Nem7AtZ858HPt/Oeo8Dp3SwzTUkV2wNKNlXVp06bVyeozEzy518d44PGcdOHE1xUYH7OcxsyHPi6COFBWL2pDEes8rMhjwnjj7kMavMbDhw4uhD1VUZtuw9xJ66hnyHYmaWM04cfWhO+lCnF7a51WFmQ5cTRx+a4zGrzGwYcOLoQ5PHlpIpKfKVVWY2pDlx9CFJVFdl3OIwsyHNiaOPVadjViWjo5iZDT1OHH1sTuUYXq5rYPv+w11XNjMbhJw4+lh1y5VVW3wjoJkNTU4cfaz1yip3kJvZEOXE0ccmjimhfEyx7yA3syHLiSMHqiszbnGY2ZDlxJED1ZUZVm3dR3Ozr6wys6Enp4lD0gJJKyXVSrquneVXSdouaWk6fTgtnyfpD+lDnpZLek/WOrdIWpu1zrxc7sPRmFOV4UB9Ext3H8x3KGZmfS5nD3KSVAjcALwF2AAskbQ460l+Le6MiGvblNUBH4iIVZKmAE9JeiAidqfLPxsRd+Uq9t5qeajTC1v3MX3CqDxHY2bWt3LZ4jgDqI2INRFRD9wBLOrOihHxQkSsSuc3AduAipxF2seqK8cA8Lw7yM1sCMpl4pgKrM96vyEta+uS9HTUXZKmt10o6QygGFidVfzldJ1vSCpp78MlXSOpRlLN9u3be7EbPZcpHcHUcSM9ZpWZDUn57hy/B5gREacCDwK3Zi+UNBn4D+CDEdGcFn8eOAF4HTAB+Fx7G46IGyNifkTMr6jo/8ZKdeUYj1llZkNSLhPHRiC7BTEtLWsVETsjomVsjpuA01uWSSoD7gX+MSKeyFpncyQOAz8kOSU24FRXZViz/QANTc1dVzYzG0RymTiWALMlzZRUDFwGLM6ukLYoWiwEnkvLi4FfAre17QRvWUeSgIuAZ3K2B70wpzJDfVMzL+48kO9QzMz6VM6uqoqIRknXAg8AhcDNEbFC0vVATUQsBj4paSHQCOwCrkpXvxR4IzBRUkvZVRGxFLhdUgUgYCnw0VztQ2+0PA1w5Zb9zJqUyXM0ZmZ9p1uJQ9K7I+JnXZW1FRH3Afe1KftC1vznSfos2q73I+BHHWzzzd2JOd+OrxhDgZIxq/6KyV2vYGY2SHT3VNWrvtw7KLNU6YhCZpSP9phVZjbkdNrikHQh8DZgqqRvZy0qIzm9ZJ04ecpYHq3dwaGGJkpHFOY7HDOzPtFVi2MTUAMcAp7KmhYDb81taIPfZWdMZ9eBeu5eurHrymZmg0SnLY6IWAYsk/TjiGgAkDQemB4RL/dHgIPZ64+byAlVGW5+dB2Xzp9OciGYmdng1t0+jgcllUmaADwNfF/SN3IY15AgiQ+dO5OVW/fxWO3OfIdjZtYnups4xkbEXuBiknsrzgTOz11YQ8c75k6hfEwxNz+2Nt+hmJn1ie4mjqL0xrtLgV/lMJ4hp3REIe8/61h++/w21mz3c8jNbPDrbuK4nuRGvtURsUTSccCq3IU1tFx+5rEUFxbww8fW5TsUM7Ne61biiIifRcSpEfGx9P2aiLgkt6ENHRWZEhbOm8JdT21gT11DvsMxM+uVbiUOSdMk/VLStnT6uaRpuQ5uKLn6nJkcbGjiJ0teyncoZma90t1TVT8kuXdjSjrdk5ZZN504pYzXHzeRWx9f5xFzzWxQ627iqIiIH0ZEYzrdwiB6It9A8aFzZ7J5zyEeWLEl36GYmR217iaOnZLeL6kwnd4P+MaEHnrzCZM4duIofvCoL801s8Gru4njapJLcbcAm4F38coQ6NZNBQXig2fP4E8v7ebpl3zjvZkNTj25HPfKiKiIiEkkieSfcxfW0PXu+dPJlBZxs1sdZjZIdTdxnJo9NlVE7AJe29VKkhZIWimpVtJ17Sy/StJ2SUvT6cNZy66UtCqdrswqP13Sn9NtfluDbACo0SVFXPa66dz/zBY27T6Y73DMzHqsu4mjIB3cEIB0zKquhmQvBG4ALgROBN4r6cR2qt4ZEfPS6aas7f8TcCbJM8X/KevzvwN8BJidTgu6uQ8DxpVnzyAiuPUP6/IdiplZj3U3cXwN+IOkL0n6EvA48L+7WOcMoDa9WbAeuANY1M3PeyvwYETsSls6DwIL0mFPyiLiiYgI4DaS544PKtPGj2LByVX85MmXqKv3Y03MbHDp7p3jt5EMcLg1nS6OiP/oYrWpwPqs9xvSsrYukbRc0l2Spnex7tR0vqttIukaSTWSarZv395FqP3v6nNmsvdQIz9/2s/qMLPBpbstDiLi2Yj493R6to8+/x5gRkScStKquLWPtktE3BgR8yNifkXFwLvl5PRjxzN32lh++Ohampsj3+GYmXVbtxPHUdgITM96Py0taxUROyPicPr2JuD0LtbdmM53uM3BQhJXnzuTNTsO8MgLA69FZGbWkVwmjiXAbEkzJRUDl5EMW9Iq7bNosRB4Lp1/ALhA0vi0U/wC4IGI2AzslXRWejXVB4C7c7gPOfW2UyZTWVbiGwLNbFDJWeKIiEbgWpIk8Bzw04hYIel6SQvTap+UtELSMuCTpDcVppf7fokk+SwBrk/LAD5O0jqpBVYD9+dqH3JtRGEBH3j9DB6t3cHKLfvyHY6ZWbcouThpaJs/f37U1NTkO4x2vXygntd/5TdcNG8qX7nk1HyHY2bWStJTETG/bXkuT1VZN4wfXczFp03jF3/ayM79h7tewcwsz5w4BoCrz5lBfWMzP37Sz+ows4HPiWMAmDUpw5uqK7jtiRc53NiU73DMzDrlxDFAXH3uTLbvO8y9yzfnOxQzs045cQwQb5xdzqxJY/jBo2sZDhcsmNng5cQxQEji6nNmsmLTXv64dlfXK5iZ5YkTxwDyztdOZdyoEdz8mG8INLOBy4ljABlZXMjlZx7Dr5/dyks76/IdjplZu5w4BpgrzppBocQtj6/LdyhmZu1y4hhgqsaW8vZTJ/PTmvXsO9SQ73DMzF7FiWMAuvrcmew/3MhPazZ0XdnMrJ85cQxAp04bx/xjx3PL42tp8rM6zGyAceIYoD507kzW7zrIg89uzXcoZmZHcOIYoN5yYiVTx43kO4+sdqvDzAYUJ44BqqiwgL+7oJpl63dz4+/X5DscM7NWOU0ckhZIWimpVtJ1ndS7RFJImp++v1zS0qypWdK8dNnD6TZblk3K5T7k0ztfO5W/OmUyX39wJc9s3JPvcMzMgBwmDkmFwA3AhcCJwHslndhOvQzwKeDJlrKIuD0i5kXEPOAKYG1ELM1a7fKW5RGxLVf7kG+S+PI7T2bC6GI+fedSDjV45Fwzy79ctjjOAGojYk1E1AN3AIvaqfcl4KvAoQ6289503WFp3KhivvbuedRu289X7n8+3+GYmeU0cUwF1me935CWtZJ0GjA9Iu7tZDvvAX7SpuyH6Wmq/yFJfRLtAHbu7HKuPmcmtzy+jodXDtkGlpkNEnnrHJdUAHwd+LtO6pwJ1EXEM1nFl0fEKcAb0umKDta9RlKNpJrt27f3YeT58Q8L5lBdOYbP3rWcXQfq8x2OmQ1juUwcG4HpWe+npWUtMsDJwMOS1gFnAYtbOshTl9GmtRERG9PXfcCPSU6JvUpE3BgR8yNifkVFRS93Jf9KRxTyzfe8lj11Dfy3X/zZz+wws7zJZeJYAsyWNFNSMUkSWNyyMCL2RER5RMyIiBnAE8DCiKiB1hbJpWT1b0gqklSezo8A3g5kt0aGtBOnlPH3b63mv1Zs4WdPeTgSM8uPnCWOiGgErgUeAJ4DfhoRKyRdL2lhNzbxRmB9RGTfxFACPCBpObCUpAXz/T4OfUD78LnH8frjJvLPi1d46HUzywsNh1Me8+fPj5qamnyH0Wc27T7IW7/5e6orM9x5zVkUFfo+TjPre5Keioj5bcv9jTMITRk3kn+56GSeevFlvvPw6nyHY2bDjBPHILVo3lQWzZvCt36zimXrd+c7HDMbRpw4BrHrF53MpEwJf3vnUurqG/MdjpkNE04cg9jYkSP4t0vnsnbnAb5873P5DsfMhgknjkHu7OPLueYNx3H7ky/x2+f97A4zyz0njiHgMxdU85rJZfzDXcvZsf9wvsMxsyHOiWMIKCkq5JvvmcfeQ41c9/PlvqvczHLKiWOImFOV4boFJ/DQc9u4Y8n6rlcwMztKThxDyFVnz+DcWeVcf8+zrN1xIN/hmNkQ5cQxhBQUiH9791yKiwr49J1LaWhqzndIZjYEOXEMMVVjS/lfF5/CsvW7+fff1uY7HDMbgpw4hqC3nTKZi0+byr//rpanXnw53+GY2RDjxDFE/fPCk5g8tpSP3/4Uz2zck+9wzGwIceIYojKlI7jpyvkUFRTwru8+zn1/3pzvkMxsiHDiGMJOqCrjPz9xDidOLuPjtz/Ntx5a5Xs8zKzXnDiGuIpMCT/+yFlc/NqpfOOhF/jkHUs51NCU77DMbBDLaeKQtEDSSkm1kq7rpN4lkqLleeOSZkg6KGlpOn03q+7pkv6cbvPbkpTLfRgKSkcU8rVL5/K5BSfwq+WbeM/3/sDWvYfyHZaZDVI5SxySCoEbgAuBE4H3SjqxnXoZ4FPAk20WrY6Ieen00azy7wAfAWan04JcxD/USOJj5x3P995/Oqu27Wfhvz/Knze409zMei6XLY4zgNqIWBMR9cAdwKJ26n0J+CrQ5b/AkiYDZRHxRCQn628DLurDmIe8C06q4ucfO5uiggLe/b3HuXe5O83NrGdymTimAtmDJm1Iy1pJOg2YHhH3trP+TEl/kvSIpDdkbXNDZ9vM2vY1kmok1Wzfvv2od2Ioes3kMu6+9hxOmjKWT/z4ab750AvuNDezbstb57ikAuDrwN+1s3gzcExEvBb4DPBjSWU92X5E3BgR8yNifkVFRe8DHmLKx5Tw44+cycWnTeWbD63i2p/8iYP17jQ3s64V5XDbG4HpWe+npWUtMsDJwMNp/3YVsFjSwoioAQ4DRMRTklYD1en60zrZpvVASVEhX3v3XOZUZvjKfz3P+l113HjFfKrGluY7NDMbwHLZ4lgCzJY0U1IxcBmwuGVhROyJiPKImBERM4AngIURUSOpIu1cR9JxJJ3gayJiM7BX0lnp1VQfAO7O4T4MeZL46zcdz41XzGf1tv0suuFRlm/Yne+wzGwAy1niiIhG4FrgAeA54KcRsULS9ZIWdrH6G4HlkpYCdwEfjYhd6bKPAzcBtcBq4P6c7MAw85YTK/n5x9NO8+/+gV8t35TvkMxsgNJw6BSdP39+1NTU5DuMQWHH/sN89D+eoubFl/nU+bP51PmzKSjwrTJmw5GkpyJiftty3zluRygfU8LtHzmTd50+jW/9ZhUfua2G2m378h2WmQ0gThz2KiVFhfzru07lv//Va/jDmp1c8I3f87d3LmWdnypoZvhUlXVh5/7DfO/3a7jtD+toaAoufu1UPnn+bKZPGJXv0Mwsxzo6VeXEYd2ybd8hvvPwam5/8iWam4N3z5/O37x5FlPGjcx3aGaWI04cThx9YvOeg/zf363mjiUvIcRlZ0znE38xi8oy3/thNtQ4cThx9KkNL9dxw+9q+VnNBgoLxOVnHsvHzjueikxJvkMzsz7ixOHEkRMv7azj279dxS+e3kBJUSEfOPtY/vqNxzNhdHG+QzOzXnLicOLIqTXb9/Pt36zi7mWbGDWikA+eM5OPvOE4xo4ake/QzOwoOXE4cfSLVVv38c3frOLe5ZvJlBRxxeuP5eLTpjJrUibfoZlZDzlxOHH0q+c27+WbD73Ag89upTngxMllLJo3hXfMneIrscwGCScOJ4682Lb3EL9avpnFyzaxdH0yeOIZMybwjnlT+KtTJrsvxGwAc+Jw4si7F3ce4J5lm7h76SZWbdtPUYE4d3Y5i+ZN4S0nVjGmJJej/JtZTzlxOHEMGBHB81v2cffSTdyzbBMbdx+kdEQB57+mkoVzp3DenApKigrzHabZsOfE4cQxIDU3B0+/9DKLl23i3uWb2XmgnkxpEReeXMWieVM567iJFHp0XrO8cOJw4hjwGpuaeWz1Tu5eupFfr9jK/sONTBhdzNnHT+ScWeWcO6vcY2SZ9aOOEkdOTypLWgB8CygEboqIr3RQ7xKSBza9Ln0C4FuArwDFQD3w2Yj4bVr3YWAycDBd/YKI2JbL/bD+UVRYwJuqK3hTdQWHGpr47fPbeOi5rTxWu4NfLd8MwDETRrUmkdcfP9Gd62Z5kLPEkT769QbgLcAGYImkxRHxbJt6GeBTwJNZxTuAd0TEJkknkzxFcGrW8svT55LbEFU6opC3nTKZt50ymYhg9fb9PLpqB4+t3smvlm3iJ398CQlOmlLGObPKOef4cl43YwIji903YpZruWxxnAHURsQaAEl3AIuAZ9vU+xLwVeCzLQUR8aes5UWh0ioAAA7JSURBVCuAkZJKIuJwDuO1AUoSsyZlmDUpw1XnzKSxqZnlG/fw2KodPFq7g5sfXcv3HllDcWEBpx87nnNnl3POrHJOmTrW/SNmOZDLxDEVWJ/1fgNwZnYFSacB0yPiXkmfpX2XAE+3SRo/lNQE/Bz4l2ino0bSNcA1AMccc8zR74UNOEWFBZx2zHhOO2Y8f3P+bOrqG/nj2l08VruDR2t38q8PrORfH1hJWWkRZ8ycwLzp4zh12jjmThvnIVDM+kDeLpyXVAB8HbiqkzonkbRGLsgqvjwiNqanuH4OXAHc1nbdiLgRuBGSzvG+i9wGmlHFRZw3ZxLnzZkEJM9Nf3z1Th6v3cEf1+7ioede6QKbMXEUc9NEMm/6WE6aMpbSET69ZdYTuUwcG4HpWe+npWUtMsDJwMOSAKqAxZIWph3k04BfAh+IiNUtK0XExvR1n6Qfk5wSe1XisOGrfEwJC+dOYeHcKQDsOdjAMxv3sHT9bpZv2M2Ta3Zx99JNABQWiDmVGeZOH8vcaUlCqa4cQ1Ghn6ps1pFcJo4lwGxJM0kSxmXA+1oWRsQeoLzlfXq11N+nSWMccC9wXUQ8llWnCBgXETskjQDeDjyUw32wIWDsyBFJB/qs1l83tu49xLL1u1m2YTfLN+zh3uWb+ckfkzOrpSMKOHnKWOZOH8cpU8cypyrDcRWjfVOiWSpniSMiGiVdS3JFVCFwc0SskHQ9UBMRiztZ/VpgFvAFSV9Iyy4ADgAPpEmjkCRpfD9X+2BDV2VZKRecVMUFJ1UByd3s63bWtSaTZet386MnXuRwYzOQtExmlo9mTmWG6soMc6qS6ZgJo9wBb8OObwA060BDUzOrt+/nha37eWHLPlZu3cfKLft4aVdda52SogJmV45JkkllhuqqDCdUZagqKyU9BWs2aOXlBkCzwWxEYQEnVJVxQlUZzH2lvK6+kVVb97Ny677WhPJY7Q5+8fQrXXiZ0iLmVGaYXTmGmeWjmTFxNDPLRzN9wih3xtug58Rh1kOjiouYO30cc6ePO6J8d109L7RJKA+s2MquA/WtdSSYMnZkkkzKRzGzfAwzy0cxY2KSVEYcZad8RHCwoYk9BxvYe7CRPQcb2HOwgbr6xvRzhdLPBxBC4ogyjihLCosKxMjiQka1TkWMKi5kZHEhxYUFblUNU04cZn1k3Khizpg5gTNmTjiifM/BBtbtOMDadFq38wDrdhxg8dJN7D3U2FqvsEBMGz+ytXUys3w0E8cUs//QK4lg76EG9qSJYW86tZQ3NPXvaecjk0pRa3IZWVzEqBGFjCopZHRxEWUjixg7cgRlpSMoGzmidX7syBGUjSwiUzrC/USDjBOHWY6NHTmi3RZKRPByXQNrd+xn7Y66JLmkSWXJul3U1TcdUb+oQK988Y4cQVlpEdPGj2wty/5CbvlSHlXc8icetHRnBhABkV3W5n1LWWNzMwfrm6irb6KuoYm6w43U1TdxsKGJuvpGDhxuSpY3NHEwfb/nYANb9hykrr6J/Ycb2XeokabmzpNapqSIspEjyJQWZe3fK/txxP6NOnJ/S0e45dPfnDjM8kQSE0YXM2H0BE4/9shWSkSwfd9hdtXVt36BjiouHJRfkBHBgfqmpIV0qIE9dQ3sPdR4RGup5fRaMt/A+l117EtbWvsPN3a6/eLCgjR5Fh2RRMemiWh0SRFjSpIkOrq4kNElRYwuSV/TltLokiJKipyAusuJw2wAksSkslImlZXmO5Rek8SY9Mt7Cj1/3nxjU3NrEsmektN2Da2n7Vr6d3bur2fN9gPsOdjAvkMNdNHYaVVYIEYVF6ZJJnktHVFIYYEoLBAFannllfkCUZjOS7TOt5QXFYpMS8spqzWV/Tp6EP5D4MRhZgNaUWEB40cXM/4ohtCPCA41NHOgvpG6w8mps7r6Rg7UN3HgcCMH0lNvreWHm1rLDtQnr/WNzTRF0Nwc6Ss0R9DU+j5oDmhqjtbylteGpuiyxVQg2u37SVpNxVSWlVBVVkrl2FImjy2lYkxJ3kc2cOIwsyFLSjrwRxYXwpj8xNDUHK0XOLS0kjo6TdeybPOeg+w91MjuuvpXXfRQoGRYncljS6ksK6UqfZ08trQ1wVSVlTK6JHdf704cZmY5VFigpEP/KEZmjgh2Hahny95DbNlziC17D7E1fd285xDrdh7giTU7j7g6r0WmtIiqslK+d8XpHFfRt1nTicPMbICSxMQxJUwcU8JJU8Z2WK+uvvGVxJImlZYEUzay7x8l4MRhZjbIjSou4riKMX3esuiIx442M7MeceIwM7MeceIwM7MeceIwM7MeyWnikLRA0kpJtZKu66TeJZJC0vysss+n662U9NaebtPMzHIjZ1dVSSoEbgDeAmwAlkhaHBHPtqmXAT4FPJlVdiLJo2ZPAqYAD0mqThd3uU0zM8udXLY4zgBqI2JNRNQDdwCL2qn3JeCrwKGsskXAHRFxOCLWArXp9rq7TTMzy5FcJo6pwPqs9xvSslaSTgOmR8S93Vy3y22amVlu5e0GQEkFwNeBq3K0/WuAa9K3+yWtPMpNlQM7+iaqnHB8veP4esfx9c5Aj+/Y9gpzmTg2AtOz3k9Ly1pkgJOBh9MhhauAxZIWdrFuZ9tsFRE3Ajf2In4AJNW097D2gcLx9Y7j6x3H1zsDPb6O5PJU1RJgtqSZkopJOrsXtyyMiD0RUR4RMyJiBvAEsDAiatJ6l0kqkTQTmA38sattmplZ7uWsxRERjZKuBR4ACoGbI2KFpOuBmojo8As/rfdT4FmgEfhERDQBtLfNXO2DmZm9Wk77OCLiPuC+NmVf6KDueW3efxn4cne2mWO9Pt2VY46vdxxf7zi+3hno8bVLEd18rqKZmRkecsTMzHrIicPMzHrEiSPV1RhY6RVed6bLn5Q0ox9jmy7pd5KelbRC0qfaqXOepD2SlqZTu31JOYxxnaQ/p59d085ySfp2evyWpzd/9ldsc7KOy1JJeyV9uk2dfj1+km6WtE3SM1llEyQ9KGlV+jq+g3WvTOusknRlP8b3r5KeT39+v5Q0roN1O/1dyGF8X5S0Metn+LYO1s35eHcdxHdnVmzrJC3tYN2cH79ei4hhP5FcobUaOA4oBpYBJ7ap83Hgu+n8ZcCd/RjfZOC0dD4DvNBOfOcBv8rjMVwHlHey/G3A/YCAs4An8/iz3gIcm8/jB7wROA14JqvsfwPXpfPXAV9tZ70JwJr0dXw6P76f4rsAKErnv9pefN35XchhfF8E/r4bP/9O/9ZzFV+b5V8DvpCv49fbyS2ORHfGwFoE3JrO3wWcr/TOxVyLiM0R8XQ6vw94jsE31Moi4LZIPAGMkzQ5D3GcD6yOiBfz8NmtIuL3wK42xdm/Y7cCF7Wz6luBByNiV0S8DDwILOiP+CLi1xHRmL59guQG3Lzo4Ph1R7+Md9dZfOn3xqXAT/r6c/uLE0eiO2NgtdZJ/3j2ABP7Jbos6Smy15I1mnCW10taJul+SSf1a2AQwK8lPZUO99LWQBln7DI6/oPN5/EDqIyIzen8FqCynToD5TheTdKCbE9Xvwu5dG16Ku3mDk71DYTj9wZga0Ss6mB5Po9ftzhxDCKSxgA/Bz4dEXvbLH6a5PTLXOD/AP/Zz+GdGxGnARcCn5D0xn7+/C6low0sBH7WzuJ8H78jRHLOYkBeKy/pH0luzL29gyr5+l34DnA8MA/YTHI6aCB6L523Ngb835ITR6KrcbWOqCOpCBgL7OyX6JLPHEGSNG6PiF+0XR4ReyNifzp/HzBCUnl/xRcRG9PXbcAvSU4JZOvOMc61C4GnI2Jr2wX5Pn6prS2n79LXbe3UyetxlHQV8Hbg8jS5vUo3fhdyIiK2RkRTRDQD3+/gc/N9/IqAi4E7O6qTr+PXE04cie6MgbUYaLmC5V3Abzv6w+lr6TnRHwDPRcTXO6hT1dLnIukMkp9tvyQ2SaOVPJALSaNJOlGfaVNtMfCB9Oqqs4A9Wadl+kuH/+nl8/hlyf4duxK4u506DwAXSBqfnoq5IC3LOUkLgH8gGVOuroM63fldyFV82X1m7+zgc/M93t1fAs9HxIb2Fubz+PVIvnvnB8pEctXPCyRXXPxjWnY9yR8JQCnJKY5akgEXj+vH2M4lOW2xHFiaTm8DPgp8NK1zLbCC5CqRJ4Cz+zG+49LPXZbG0HL8suMTydMbVwN/Bub38893NEkiGJtVlrfjR5LANgMNJOfZP0TSZ/YbYBXwEDAhrTsfuClr3avT38Na4IP9GF8tSf9Ay+9gy1WGU4D7Ovtd6Kf4/iP93VpOkgwmt40vff+qv/X+iC8tv6Xldy6rbr8fv95OHnLEzMx6xKeqzMysR5w4zMysR5w4zMysR5w4zMysR5w4zMysR5w4zAagdLTeX+U7DrP2OHGYmVmPOHGY9YKk90v6Y/rshO9JKpS0X9I3lDw75TeSKtK68yQ9kfU8i/Fp+SxJD6UDLD4t6fh082Mk3ZU+A+P2rDvbv6Lk2SzLJf1bnnbdhjEnDrOjJOk1wHuAcyJiHtAEXE5yl3pNRJwEPAL8U7rKbcDnIuJUkjucW8pvB26IZIDFs0nuOIZkFORPAyeS3FF8jqSJJMNpnJRu519yu5dmr+bEYXb0zgdOB5akT3M7n+QLvplXBrH7EXCupLHAuIh4JC2/FXhjOi7R1Ij4JUBEHIpXxoH6Y0RsiGTQvqXADJLh/A8BP5B0MdDumFFmueTEYXb0BNwaEfPSaU5EfLGdekc7rs/hrPkmkqfvNZKMlnoXySi1/3WU2zY7ak4cZkfvN8C7JE2C1meGH0vyd/WutM77gEcjYg/wsqQ3pOVXAI9E8kTHDZIuSrdRImlURx+YPpNlbCRDv/8tMDcXO2bWmaJ8B2A2WEXEs5L+O8nT2gpIRkL9BHAAOCNdto2kHwSSodK/myaGNcAH0/IrgO9Juj7dxrs7+dgMcLekUpIWz2f6eLfMuuTRcc36mKT9ETEm33GY5YpPVZmZWY+4xWFmZj3iFoeZmfWIE4eZmfWIE4eZmfWIE4eZmfWIE4eZmfXI/wdLt6Njv3/RoQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-d1b6b4804538>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcosts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m: too many values to unpack (expected 2)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncTf3SUSJ6qX",
        "outputId": "34d6e411-a33f-4198-dd9c-c502ae6e20ca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "yhat_test = predict(parameters, X_test)\n",
        "y = np.argmax(y_test, axis = 0)\n",
        "np.mean(yhat_test == y)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8774"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E5OmZx6lAdmn"
      },
      "source": [
        "## 04.3-Momentum\n",
        "\n",
        "Initialize the velocity for momentum optimization."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E_EPrQenGFoA"
      },
      "source": [
        "def initialize_velocity(parameters):\n",
        "    \"\"\"\n",
        "    Initializes the velocity as a python dictionary with:\n",
        "                - keys: \"dW1\", \"db1\", \"dW2\", \"db2\" \n",
        "                - values: numpy arrays of zeros of the same shape as the corresponding gradients/parameters.\n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters.\n",
        "    \n",
        "    Returns:\n",
        "    v -- python dictionary containing the current velocity.\n",
        "                    v['dW1'] = velocity of dWl\n",
        "                    v['db1'] = velocity of dbl\n",
        "    \"\"\"\n",
        "    \n",
        "    v = {}\n",
        "    \n",
        "    # Initialize velocity\n",
        "    v[\"dW1\"] = np.zeros_like(parameters[\"W1\"])\n",
        "    v[\"db1\"] = np.zeros_like(parameters[\"b1\"])\n",
        "    v[\"dW2\"] = np.zeros_like(parameters[\"W2\"])\n",
        "    v[\"db2\"] = np.zeros_like(parameters[\"b2\"])\n",
        "    \n",
        "    return v"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xin0AoAQqqj1"
      },
      "source": [
        "Implement the parameters update with momentum. The momentum update rule is:\n",
        "\n",
        "$$V_{dw} = \\beta V_{dw}+(1-\\beta)dw$$\n",
        "\n",
        "$$V_{db} = \\beta V_{db}+(1-\\beta)db$$\n",
        "\n",
        "$$w=w-\\alpha V_{dw};\\ \\ b=b-\\alpha V_{db}$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_j7RL7ya3mv"
      },
      "source": [
        "def update_parameters_with_momentum(parameters, grads, v, beta, learning_rate):\n",
        "    \"\"\"\n",
        "    Update parameters using Momentum\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters\n",
        "    grads -- python dictionary containing your gradients for each parameters\n",
        "    v -- python dictionary containing the current velocity\n",
        "    beta -- the momentum hyperparameter, scalar\n",
        "    learning_rate -- the learning rate, scalar\n",
        "    \n",
        "    Returns:\n",
        "    parameters -- python dictionary containing your updated parameters \n",
        "    v -- python dictionary containing your updated velocities\n",
        "    \"\"\"\n",
        "\n",
        "    # Momentum update for each parameter\n",
        "    # compute velocities\n",
        "    v[\"dW1\"] = beta * v[\"dW1\"] + (1 - beta) * grads['dW1']\n",
        "    v[\"db1\"] = beta * v[\"db1\"] + (1 - beta) * grads['db1']\n",
        "    v[\"dW2\"] = beta * v[\"dW2\"] + (1 - beta) * grads['dW2']\n",
        "    v[\"db2\"] = beta * v[\"db2\"] + (1 - beta) * grads['db2']\n",
        "    # update parameters\n",
        "    parameters[\"W1\"] = parameters[\"W1\"] - learning_rate * v[\"dW1\"]\n",
        "    parameters[\"b1\"] = parameters[\"b1\"] - learning_rate * v[\"db1\"]\n",
        "    parameters[\"W2\"] = parameters[\"W2\"] - learning_rate * v[\"dW2\"]\n",
        "    parameters[\"b2\"] = parameters[\"b2\"] - learning_rate * v[\"db2\"]\n",
        "        \n",
        "    return parameters, v"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZGjNYslKJqEt"
      },
      "source": [
        "## 04.4-Adam\n",
        "\n",
        "Adam is one of the most effective optimization algorithms for training neural networks. It combines ideas from RMSProp and Momentum.\n",
        "\n",
        "On iteration t, compute $dw$, $db$ using the current mini-batch.\n",
        "\n",
        "$$\\begin{cases} \\begin{array}{c} V_{dw}=\\beta_{1}V_{dw}+(1-\\beta_{1})dw\\\\ V_{db}=\\beta_{1}V_{db}+(1-\\beta_{1})db \\end{array} & momantum\\ update\\ \\beta_{1}\\end{cases}$$\n",
        "\n",
        "$$\\begin{cases} \\begin{array}{c} S_{dw}=\\beta_{2}S_{dw}+(1-\\beta_{2})dw^{2}\\\\ S_{db}=\\beta_{2}S_{db}+(1-\\beta_{2})db^{2} \\end{array} & RMSprop\\ update\\ \\beta_{2}\\end{cases}$$\n",
        "\n",
        "$$\\begin{cases} \\begin{array}{c} V_{dw}^{corrected}=\\frac{V_{dw}}{1-\\beta_{1}^{t}}\\\\ V_{db}^{corrected}=\\frac{V_{db}}{1-\\beta_{1}^{t}} \\end{array}\\end{cases};\\ \\ \\begin{cases} \\begin{array}{c} S_{dw}^{corrected}=\\frac{S_{dw}}{1-\\beta_{2}^{t}}\\\\ S_{db}^{corrected}=\\frac{S_{db}}{1-\\beta_{2}^{t}} \\end{array}\\end{cases}$$\n",
        "\n",
        "$$w=w-\\alpha \\frac{V_{dw}^{corrected}}{\\sqrt{S_{dw}^{corrected}} +\\epsilon};\\ b=b-\\alpha\\frac{V_{db}^{corrected}}{\\sqrt{S_{db}^{corrected}}+\\epsilon}$$\n",
        "\n",
        "where:\n",
        "\n",
        "- $t$ counts the number of steps taken of Adam\n",
        "- $\\beta_1$ and $\\beta_2$ are hyperparameters that control the two exponentially weighted averages.\n",
        "- $\\alpha$ is the learning rate\n",
        "- $\\epsilon$ is a very small number to avoid dividing by zero\n",
        "\n",
        "As before, let's initialize the Adam variables $v, s$ which keep track of the past information."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V01-TLpqWnCG"
      },
      "source": [
        "def initialize_adam(parameters) :\n",
        "    \"\"\"\n",
        "    Initializes v and s as two python dictionaries with:\n",
        "                - keys: \"dW1\", \"db1\",\"dW2\", \"db2\" \n",
        "                - values: numpy arrays of zeros of the same shape as the corresponding gradients/parameters.\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters.\n",
        "    \n",
        "    Returns: \n",
        "    v -- python dictionary that will contain the exponentially weighted average of the gradient.\n",
        "    s -- python dictionary that will contain the exponentially weighted average of the squared gradient.\n",
        "    \"\"\"\n",
        "    \n",
        "    v = {}\n",
        "    s = {}\n",
        "    \n",
        "    # Initialize v, s. Input: \"parameters\". Outputs: \"v, s\".\n",
        "    v[\"dW1\"] = np.zeros_like(parameters[\"W1\"])\n",
        "    v[\"db1\"] = np.zeros_like(parameters[\"b1\"])\n",
        "    v[\"dW2\"] = np.zeros_like(parameters[\"W2\"])\n",
        "    v[\"db2\"] = np.zeros_like(parameters[\"b2\"])\n",
        "\n",
        "    s[\"dW1\"] = np.zeros_like(parameters[\"W1\"])\n",
        "    s[\"db1\"] = np.zeros_like(parameters[\"b1\"])\n",
        "    s[\"dW2\"] = np.zeros_like(parameters[\"W2\"])\n",
        "    s[\"db2\"] = np.zeros_like(parameters[\"b2\"])\n",
        "\n",
        "    return v, s"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7KFa7474Xoyo"
      },
      "source": [
        "Now, implement the parameters update with Adam."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rTXiWjzPXpnk"
      },
      "source": [
        "def update_parameters_with_adam(parameters, grads, v, s, t, learning_rate=0.01,\n",
        "                                beta1=0.9, beta2=0.999, epsilon=1e-8):\n",
        "    \"\"\"\n",
        "    Update parameters using Adam\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters\n",
        "    grads -- python dictionary containing gradients for your parameters\n",
        "    v -- Adam variable, moving average of the first gradient, python dictionary\n",
        "    s -- Adam variable, moving average of the squared gradient, python dictionary\n",
        "    learning_rate -- the learning rate, scalar.\n",
        "    beta1 -- Exponential decay hyperparameter for the first moment estimates \n",
        "    beta2 -- Exponential decay hyperparameter for the second moment estimates \n",
        "    epsilon -- hyperparameter preventing division by zero in Adam updates\n",
        "\n",
        "    Returns:\n",
        "    parameters -- python dictionary containing your updated parameters \n",
        "    v -- Adam variable, moving average of the first gradient, python dictionary\n",
        "    s -- Adam variable, moving average of the squared gradient, python dictionary\n",
        "    \"\"\"\n",
        "    \n",
        "    v_corrected = {}                         # Initializing first moment estimate, python dictionary\n",
        "    s_corrected = {}                         # Initializing second moment estimate, python dictionary\n",
        "    \n",
        "    # Perform Adam update on all parameters\n",
        "    # Moving average of the gradients. Inputs: \"v, grads, beta1\". Output: \"v\".\n",
        "    v[\"dW1\"] = beta1 * v[\"dW1\"] + (1 - beta1) * grads['dW1']\n",
        "    v[\"db1\"] = beta1 * v[\"db1\"] + (1 - beta1) * grads['db1']\n",
        "    v[\"dW2\"] = beta1 * v[\"dW2\"] + (1 - beta1) * grads['dW2']\n",
        "    v[\"db2\"] = beta1 * v[\"db2\"] + (1 - beta1) * grads['db2']\n",
        "        \n",
        "    # Compute bias-corrected first moment estimate. Inputs: \"v, beta1, t\". Output: \"v_corrected\".\n",
        "    v_corrected[\"dW1\"] = v[\"dW1\"] / (1 - np.power(beta1, t))\n",
        "    v_corrected[\"db1\"] = v[\"db1\"] / (1 - np.power(beta1, t))\n",
        "    v_corrected[\"dW2\"] = v[\"dW2\"] / (1 - np.power(beta1, t))\n",
        "    v_corrected[\"db2\"] = v[\"db2\"] / (1 - np.power(beta1, t))\n",
        "       \n",
        "    # Moving average of the squared gradients. Inputs: \"s, grads, beta2\". Output: \"s\".\n",
        "    s[\"dW1\"] = beta2 * s[\"dW1\"] + (1 - beta2) * np.power(grads['dW1'], 2)\n",
        "    s[\"db1\"] = beta2 * s[\"db1\"] + (1 - beta2) * np.power(grads['db1'], 2)\n",
        "    s[\"dW2\"] = beta2 * s[\"dW2\"] + (1 - beta2) * np.power(grads['dW2'], 2)\n",
        "    s[\"db2\"] = beta2 * s[\"db2\"] + (1 - beta2) * np.power(grads['db2'], 2)\n",
        "\n",
        "    # Compute bias-corrected second raw moment estimate. Inputs: \"s, beta2, t\". Output: \"s_corrected\".\n",
        "\n",
        "    s_corrected[\"dW1\"] = s[\"dW1\"] / (1 - np.power(beta2, t))\n",
        "    s_corrected[\"db1\"] = s[\"db1\"] / (1 - np.power(beta2, t))\n",
        "    s_corrected[\"dW2\"] = s[\"dW2\"] / (1 - np.power(beta2, t))\n",
        "    s_corrected[\"db2\"] = s[\"db2\"] / (1 - np.power(beta2, t))\n",
        "\n",
        "    # Update parameters. Inputs: \"parameters, learning_rate, v_corrected, s_corrected, epsilon\". Output: \"parameters\".\n",
        "    parameters[\"W1\"] = parameters[\"W1\"] - learning_rate * v_corrected[\"dW1\"] / np.sqrt(s_corrected[\"dW1\"] + epsilon)\n",
        "    parameters[\"b1\"] = parameters[\"b1\"] - learning_rate * v_corrected[\"db1\"] / np.sqrt(s_corrected[\"db1\"] + epsilon)\n",
        "    parameters[\"W2\"] = parameters[\"W2\"] - learning_rate * v_corrected[\"dW2\"] / np.sqrt(s_corrected[\"dW2\"] + epsilon)\n",
        "    parameters[\"b2\"] = parameters[\"b2\"] - learning_rate * v_corrected[\"db2\"] / np.sqrt(s_corrected[\"db2\"] + epsilon)\n",
        "\n",
        "    return parameters, v, s"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KjVm7e07Gfus"
      },
      "source": [
        "# 05-Model with different optimization algorithms"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SSH1UcHxrdtp"
      },
      "source": [
        "def model(X, Y, n_h, optimizer, learning_rate= 0.1, mini_batch_size=64, beta=0.9,\n",
        "          beta1=0.9, beta2=0.999, epsilon=1e-8, num_epochs= 20, print_cost=True):\n",
        "    \"\"\"\n",
        "    3-layer neural network model which can be run in different optimizer modes.\n",
        "    \n",
        "    Arguments:\n",
        "    X -- input data\n",
        "    Y -- true \"label\" vector\n",
        "    layers_dims -- python list, containing the size of each layer\n",
        "    learning_rate -- the learning rate, scalar.\n",
        "    mini_batch_size -- the size of a mini batch\n",
        "    beta -- Momentum hyperparameter\n",
        "    beta1 -- Exponential decay hyperparameter for the past gradients estimates \n",
        "    beta2 -- Exponential decay hyperparameter for the past squared gradients estimates \n",
        "    epsilon -- hyperparameter preventing division by zero in Adam updates\n",
        "    num_epochs -- number of epochs\n",
        "    print_cost -- True to print the cost \n",
        "\n",
        "    Returns:\n",
        "    parameters -- python dictionary containing your updated parameters \n",
        "    \"\"\"\n",
        "\n",
        "    costs = []                       # to keep track of the cost\n",
        "    t = 0                            # initializing the counter required for Adam update\n",
        "    gradients = []                   # gradients per epoch\n",
        "    n_x = X.shape[0]\n",
        "    n_y = Y.shape[0]\n",
        "    # Initialize parameters\n",
        "    parameters = initialize_parameters(n_x, n_h, n_y)\n",
        "\n",
        "    # Initialize the optimizer\n",
        "    if optimizer == \"gd\":\n",
        "        pass # no initialization required for gradient descent\n",
        "    elif optimizer == \"momentum\":\n",
        "        v = initialize_velocity(parameters)\n",
        "    elif optimizer == \"adam\":\n",
        "        v, s = initialize_adam(parameters)\n",
        "    \n",
        "    # Optimization loop\n",
        "    for i in range(num_epochs):\n",
        "        \n",
        "        minibatches = random_mini_batches(X, Y, mini_batch_size)\n",
        "\n",
        "        for minibatch in minibatches:\n",
        "\n",
        "            # Select a minibatch\n",
        "            (minibatch_X, minibatch_Y) = minibatch\n",
        "\n",
        "            # Forward propagation. Inputs: \"X, parameters\". Outputs: \"A2, cache\".\n",
        "            A2, cache = forward_propagation(X, parameters)\n",
        "            \n",
        "            # Cost function. Inputs: \"A2, Y, parameters\". Outputs: \"cost\".\n",
        "            cost = compute_cost(A2, Y, parameters)\n",
        "           \n",
        "           # Backpropagation. Inputs: \"parameters, cache, X, Y\". Outputs: \"grads\".\n",
        "            grads = backward_propagation(parameters, cache, X, Y)\n",
        "            \n",
        "            # Update parameters\n",
        "            if optimizer == \"gd\":\n",
        "                parameters = update_parameters(parameters, grads, learning_rate)\n",
        "            elif optimizer == \"momentum\":\n",
        "                parameters, v = update_parameters_with_momentum(parameters, grads, v, beta, learning_rate)\n",
        "            elif optimizer == \"adam\":\n",
        "                t = t + 1 # Adam counter\n",
        "                parameters, v, s = update_parameters_with_adam(parameters, grads, v, s,\n",
        "                                                               t, learning_rate, beta1, beta2,  epsilon)\n",
        "        \n",
        "        gradients.append([grads['dW1'], grads['dW2'], grads['db1'], grads['db2']])\n",
        "        if print_cost:\n",
        "            print(\"Cost after epoch %i: %f\" % (i, cost))\n",
        "            costs.append(cost)\n",
        "\n",
        "                \n",
        "    # plot the cost\n",
        "    plt.plot(costs)\n",
        "    plt.ylabel('cost')\n",
        "    plt.xlabel('epochs')\n",
        "    plt.title(\"Learning rate = \" + str(learning_rate))\n",
        "    plt.show()\n",
        "\n",
        "    return parameters, costs, gradients"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RRtP79Yb8SK_",
        "outputId": "e3746eb4-3068-4b4b-96c7-dd5632020d9a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 635
        }
      },
      "source": [
        "parameters = model(X_train, y_train, n_h = 4, beta=0.9, optimizer=\"momentum\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cost after epoch 0: 0.584021\n",
            "Cost after epoch 1: 0.517832\n",
            "Cost after epoch 2: 0.497935\n",
            "Cost after epoch 3: 0.486989\n",
            "Cost after epoch 4: 0.479612\n",
            "Cost after epoch 5: 0.474131\n",
            "Cost after epoch 6: 0.469694\n",
            "Cost after epoch 7: 0.466067\n",
            "Cost after epoch 8: 0.462972\n",
            "Cost after epoch 9: 0.460300\n",
            "Cost after epoch 10: 0.457927\n",
            "Cost after epoch 11: 0.455797\n",
            "Cost after epoch 12: 0.453861\n",
            "Cost after epoch 13: 0.452080\n",
            "Cost after epoch 14: 0.450376\n",
            "Cost after epoch 15: 0.448701\n",
            "Cost after epoch 16: 0.447044\n",
            "Cost after epoch 17: 0.445284\n",
            "Cost after epoch 18: 0.443355\n",
            "Cost after epoch 19: 0.441043\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXzddZ3v8dc7S9M2OU2XpEn3tLRNaaFWGuoCKG4VvAN1xAUXFEcHHYfLzKgzF2auyuD1jjiOzjiXcQREcJkBQXGKFKE6IqACDUgLbeneQkqXdG9SuqT53D/OL+E0nKTpcnKyvJ+Px+/Rc36/7++cT349Oe98f8v3p4jAzMyso4J8F2BmZr2TA8LMzLJyQJiZWVYOCDMzy8oBYWZmWTkgzMwsKweEDXiSLpC0Kt91mPU2DgjLK0kbJb09nzVExKMRUZvPGtpIulBSQ57e+0OSNklqlvQzSSO7aHuzpFWSWiVd2YNlWg9yQFi/J6kw3zUAKK1X/s5JmgV8B7gCqAIOAP/WxSpLgc8AT+e+OsuXXvlhNZNUIOlaSesk7ZT048y/aCXdLWmrpL2SHkm+4NqW3S7p25IWSWoG3pL0VD4vaVmyzl2SBiftj/mrvau2yfK/kbRF0kuSPikpJE3t5Od4WNJXJP2W9JfuFEkfl7RS0n5J6yV9KmlbCjwAjJXUlExjj7ctTpMPA/dFxCMR0QR8AXiPpFS2xhFxU0T8Cjh4muuwXsQBYb3V/wTeDbwZGAvsBm7KWP4AMA0YTfqv2B91WP9DwFeAFPBYMu/9wEXAZGA2cGUX75+1raSLgM8CbwemAhd242e5ArgqqWUTsB34I2AY8HHgm5LOiYhm4GLgpYgoS6aXurEt2kmaKGlPF9OHOqlxFuleAQARsQ44DEzvxs9n/VRRvgsw68SngasjogFA0vXAC5KuiIiWiLitrWGybLek8ojYm8z+r4j4bfL4oCSAbyVfuEi6D5jTxft31vb9wPciYnnGe3/4OD/L7W3tE/dnPP6NpIeAC+h8d02X2yKzYUS8AAw/Tj3ZlAF7O8zbSzrUbIByD8J6q0nAvW1/+QIrgaNAlaRCSV9NdrnsAzYm61RkrP9iltfcmvH4AOkvxc501nZsh9fO9j4dHdNG0sWSHpe0K/nZ3sWxtXfU6bboxnt3VxPpHk2mYcD+0/ge1sc4IKy3ehG4OCKGZ0yDI2Iz6d1HC0jv5ikHapJ1lLF+roYp3gKMz3g+oRvrtNciqQT4CfB1oCoihgOLeKX2bHV3tS2Okexiaupi6qy3sxx4TcbrTAFKgNXd+Pmsn3JAWG9QLGlwxlQE/DvwFUmTACRVSlqQtE8Bh4CdwFDg//ZgrT8GPi7pTElDSR/MPRGDSH/xNgItki4G5mcs3waMklSeMa+rbXGMiHgh4/hFtqnjsZo2PwIuUfqakFLgBuCnEZG1ByFpUHLgXrzy/+fvk37G/6HWGywCXs6Yrgf+BVgIPCRpP/A48Lqk/fdJH+zdDKxIlvWIiHgA+Bbwa2Btxnsf6ub6+4FrSAfNbtK9oYUZy58H/hNYn+xSGkvX2+K0SI6RfJp0UGwnHcKfaVsu6QFJf5uxykOk/6/eCNycPH7T6azJ8k++YZDZyZN0JvAcUNLxgLFZX+cehNkJkvTHkkokjQBuJH39gMPB+h0HhNmJ+xTp3TDrSJ9N9Gf5LccsN7yLyczMsnIPwszMsuo3V1JXVFRETU1NvsswM+tTnnrqqR0RUZltWb8JiJqaGurr6/NdhplZnyJpU2fLvIvJzMyyckCYmVlWDggzM8vKAWFmZlk5IMzMLCsHhJmZZeWAMDOzrAZ8QOw9cIR/+eUaljXsyXcpZma9Sr+5UO5kFRTAN3+5mqJCMXv8ydzK18ysfxrwPYjU4GLGDR/Cqq2+9a6ZWaYBHxAAM6pTDggzsw5yGhCSLpK0StJaSddmWX6lpEZJzyTTJzOWfU3SckkrJX1Lkjquf7pMr06xrrGJwy2tuXoLM7M+J2cBIakQuAm4GJgJfFDSzCxN74qIOcl0a7LuG4HzgNnAWcC5wJtzVeuM6hQtrcH6HU25egszsz4nlz2IecDaiFgfEYeBO4EF3Vw3gMHAIKAEKAa25aRKoLY6BeDdTGZmGXIZEOOAFzOeNyTzOrpM0jJJ90iaABARvwd+DWxJpgcjYmXHFSVdJaleUn1jY+NJFzqlooyiAjkgzMwy5Psg9X1ATUTMBhYDdwBImgqcCYwnHSpvlXRBx5Uj4uaIqIuIusrKrPe76JZBRQWcUVnmgDAzy5DLgNgMTMh4Pj6Z1y4idkbEoeTprcDc5PEfA49HRFNENAEPAG/IYa1Mr07xvAPCzKxdLgNiCTBN0mRJg4DLgYWZDSSNyXh6KdC2G+kF4M2SiiQVkz5A/apdTKfTjOoUm/e8zP6DR3L5NmZmfUbOAiIiWoCrgQdJf7n/OCKWS7pB0qVJs2uSU1mXAtcAVybz7wHWAc8CS4GlEXFfrmoFqK1KH6hevc1nMpmZQY6H2oiIRcCiDvO+mPH4OuC6LOsdBT6Vy9o6yjyTae6kET351mZmvVK+D1L3GuNHDKGspIhVW/fluxQzs17BAZGQxPSqMh+oNjNLOCAy1FanWLVtPxGR71LMzPLOAZGhtirFngNH2L7/0PEbm5n1cw6IDLXVwwAPuWFmBg6IY8zwmExmZu0cEBlGlA5idKrEB6rNzHBAvEr6QLVPdTUzc0B0UFuVYs22Jo62+kwmMxvYHBAd1FanONTSyqadzfkuxcwsrxwQHczwmUxmZoAD4lWmVZUh4QPVZjbgOSA6GFxcSM2oUvcgzGzAc0BkUVuVHnLDzGwgc0BkUVudYuPOZg4eOZrvUszM8sYBkcWM6hQRsMY3DzKzAcwBkUXbzYOe970hzGwAc0BkMWlUKSVFBT5QbWYDWk4DQtJFklZJWivp2izLr5TUKOmZZPpkxrKJkh6StFLSCkk1uaw1U2GBmFZV5gPVZjag5eye1JIKgZuAdwANwBJJCyNiRYemd0XE1Vle4vvAVyJisaQyoDVXtWZTWzWMR9Y09uRbmpn1KrnsQcwD1kbE+og4DNwJLOjOipJmAkURsRggIpoi4kDuSn21GdUpGvcfYlfz4Z58WzOzXiOXATEOeDHjeUMyr6PLJC2TdI+kCcm86cAeST+V9AdJ/5j0SI4h6SpJ9ZLqGxtP71/7tb43hJkNcPk+SH0fUBMRs4HFwB3J/CLgAuDzwLnAFODKjitHxM0RURcRdZWVlae1sFduHuQzmcxsYMplQGwGJmQ8H5/MaxcROyOi7QbQtwJzk8cNwDPJ7qkW4GfAOTms9VUqUyUMH1rsA9VmNmDlMiCWANMkTZY0CLgcWJjZQNKYjKeXAisz1h0uqa1b8Fag48HtnJJEbVXKg/aZ2YCVs4BI/vK/GniQ9Bf/jyNiuaQbJF2aNLtG0nJJS4FrSHYjRcRR0ruXfiXpWUDALbmqtTMzqlOs3rqfCN88yMwGnpyd5goQEYuARR3mfTHj8XXAdZ2suxiYncv6jqe2ehjNh4/SsPtlJowcms9SzMx6XL4PUvdqPpPJzAYyB0QXpleVAfhAtZkNSA6ILqQGFzNu+BAfqDazAckBcRwzqlO+FsLMBiQHxHHUVqdY39jM4ZYeHQrKzCzvHBDHUVudoqU1WL/DNw8ys4HFAXEcM6qHAT6TycwGHgfEcUyuKKWoQD5QbWYDjgPiOAYVFXBGZZl7EGY24DgguqG2OuWAMLMBxwHRDbXVKTbveZn9B4/kuxQzsx7jgOiGtntDrPYV1WY2gDggumF6VTogfKDazAYSB0Q3jB8xhLKSIh+HMLMBxQHRDZKYXuUzmcxsYHFAdFNt9TBWbfPNg8xs4HBAdNOM6hR7Dhxh+/5Dx29sZtYPOCC6yQeqzWygcUB004z2u8t56G8zGxhyGhCSLpK0StJaSddmWX6lpEZJzyTTJzssHyapQdL/y2Wd3TGidBCjUyXuQZjZgFGUqxeWVAjcBLwDaACWSFoYESs6NL0rIq7u5GW+DDySqxpPVG11yhfLmdmAkcsexDxgbUSsj4jDwJ3Agu6uLGkuUAU8lKP6TtiM6hRrtjVxtNVnMplZ/5fLgBgHvJjxvCGZ19FlkpZJukfSBABJBcA/AZ/v6g0kXSWpXlJ9Y2Pj6aq7U7XVwzjU0srGnc05fy8zs3zL90Hq+4CaiJgNLAbuSOZ/BlgUEQ1drRwRN0dEXUTUVVZW5rhUqK1qO1Dt3Uxm1v/lMiA2AxMyno9P5rWLiJ0R0XZhwa3A3OTxG4CrJW0Evg58VNJXc1hrt0yrKqNAPtXVzAaGnB2kBpYA0yRNJh0MlwMfymwgaUxEbEmeXgqsBIiID2e0uRKoi4hXnQXV0wYXF1IzqpTVDggzGwByFhAR0SLpauBBoBC4LSKWS7oBqI+IhcA1ki4FWoBdwJW5qud0qa1OuQdhZgNCLnsQRMQiYFGHeV/MeHwdcN1xXuN24PYclHdSaqtT/GL5Vl4+fJQhgwrzXY6ZWc7k+yB1n1NblSIC1mx3L8LM+jcHxAmqrfaYTGY2MDggTtCkUaUMLi7wqa5m1u85IE5QYYGYNtpDbphZ/+eAOAk+k8nMBgIHxEmorUrRuP8Qu5oP57sUM7OccUCchFcOVPveEGbWfzkgTsIrNw/ybiYz678cECehMlXCiKHFPlBtZv2aA+IkSPKBajPr9xwQJ2lG9TBWb91Pq28eZGb9lAPiJE2vStF8+Cib97yc71LMzHLCAXGSPOSGmfV3DoiT1BYQPlBtZv2VA+IklZUUMX7EEPcgzKzfckCcghnVKVb5Yjkz66ccEKdgelWK9Y3NHG5pzXcpZmannQPiFNRWp2hpDdY1NuW7FDOz084BcQpmVA8DPOSGmfVPOQ0ISRdJWiVpraRrsyy/UlKjpGeS6ZPJ/DmSfi9puaRlkj6QyzpP1pTKUooLxSqfyWRm/VBRrl5YUiFwE/AOoAFYImlhRKzo0PSuiLi6w7wDwEcjYo2kscBTkh6MiD25qvdkFBcWcEZlmXsQZtYv5bIHMQ9YGxHrI+IwcCewoDsrRsTqiFiTPH4J2A5U5qzSUzC9KuWAMLN+KZcBMQ54MeN5QzKvo8uS3Uj3SJrQcaGkecAgYF2WZVdJqpdU39jYeLrqPiG11Sk273mZfQeP5OX9zcxyJd8Hqe8DaiJiNrAYuCNzoaQxwA+Aj0fEq84ljYibI6IuIuoqK/PTwWi7N8Rq9yLMrJ/pVkBIel935nWwGcjsEYxP5rWLiJ0RcSh5eiswN+P1hwH3A38XEY93p858aBtyo37T7jxXYmZ2enW3B3FdN+dlWgJMkzRZ0iDgcmBhZoOkh9DmUmBlMn8QcC/w/Yi4p5s15sW44UN4/ZSR/Ptv1rHngO9RbWb9R5cBIeliSf8KjJP0rYzpdqClq3UjogW4GniQ9Bf/jyNiuaQbJF2aNLsmOZV1KXANcGUy//3Am4ArM06BnXOyP2QuSeJLl8xi38tH+Obi1fkux8zstFFE5ze8kfQaYA5wA/DFjEX7gV9HRK/Zr1JXVxf19fV5e/8v/Ow5/uPJF1h0zQXtu53MzHo7SU9FRF22ZV32ICJiaUTcAUyNiDuSxwtJn77aa8KhN/jsO6ZTVlLE39+3nK5C18ysr+juMYjFkoZJGgk8Ddwi6Zs5rKvPGVE6iM/Nn87v1u3kweVb812Omdkp625AlEfEPuA9pA8cvw54W+7K6ps+NG8iM6pT/J/7V3LwyNF8l2Nmdkq6GxBFyRlH7wd+nsN6+rSiwgK+eMlMGna/zC2PrM93OWZmp6S7AXED6bOR1kXEEklTgDW5K6vveuMZFVx8VjX/9vA6tux9Od/lmJmdtG4FRETcHRGzI+LPkufrI+Ky3JbWd/3tu86kNYJ/WPR8vksxMztp3b2SerykeyVtT6afSBqf6+L6qgkjh/KpN01h4dKXeHLDrnyXY2Z2Urq7i+l7pE9vHZtM9yXzrBOfvvAMxpQP5vqFyzna6tNezazv6W5AVEbE9yKiJZlup5cOv91bDB1UxHXvOpMVW/Zx15IXj7+CmVkv092A2CnpI5IKk+kjwM5cFtYfXDJ7DPNqRvL1h1ax94CHAzezvqW7AfEnpE9x3QpsAd7LK+MmWSck8cVLZrL7wGH++Vcep8nM+pYTOc31YxFRGRGjSQfG3+eurP7jrHHlfHDeRL7/+02s8b2rzawP6W5AzM4ceykidgGvzU1J/c/n59dSOqiQG36+wuM0mVmf0d2AKJA0ou1JMiZTUW5K6n9Glg7ir94xnUfX7GDxim35LsfMrFu6GxD/BPxe0pclfRn4HfC13JXV/3zk9ZOYNrrM4zSZWZ/R3Supv096oL5tyfSeiPhBLgvrb4oLC/jSJbN4YdcBvvvYhnyXY2Z2XN3eTRQRK4AVOayl3zt/WgXzZ1Zx06/Xctk546kuH5zvkszMOtXdXUx2mvzv/zGTltbgxl94nCYz691yGhCSLpK0StJaSddmWX6lpMaM+05/MmPZxyStSaaP5bLOnjRx1FD+9ILJ3PuHzTy1yeM0mVnvlbOAkFQI3ARcDMwEPihpZpamd0XEnGS6NVl3JPAl4HXAPOBLmWdR9XWfuXAqVcNKuH7hClo9TpOZ9VK57EHMI33v6vURcRi4E1jQzXXfCSyOiF3J9ReLgYtyVGePKy0p4rqLz+TZzXu5+ymP02RmvVMuA2IckPnt15DM6+gyScsk3SNpwomsK+kqSfWS6hsbG09X3T1iwZyxzJ00gn98cBX7DnqcJjPrffJ9kPo+oCYiZpPuJdxxIitHxM0RURcRdZWVfWtwWUlcf8ksdjYf5lu/9M35zKz3yWVAbAYmZDwfn8xrFxE7I+JQ8vRWYG531+0Pzh5fzgfqJnD77zay2uM0mVkvk8uAWAJMkzRZ0iDgctI3HWonaUzG00uBlcnjB4H5kkYkB6fnJ/P6nc+/s5bU4CKu+O4THszPzHqVnAVERLQAV5P+Yl8J/Dgilku6QdKlSbNrJC2XtBS4hmQI8WQwwC+TDpklwA3JvH6noqyE/7zq9RxthQ/c/DjPbd6b75LMzABQfxldtK6uLurr6/NdxknbsKOZj9z6BPtePsL3Pn4udTUj812SmQ0Akp6KiLpsy/J9kNoSkytKufvTb6AyVcIV332SR9f0rbOyzKz/cUD0ImOHD+GuT72BmopSPnF7Pb94bmu+SzKzAcwB0ctUpkq4809fz6xxw/jz/3iae//QkO+SzGyAckD0QuVDi/nhJ17H6yaP5K/uWsoPHt+U75LMbAByQPRSpSVF3Hblubz9zNF84WfP8e2H1+W7JDMbYBwQvdjg4kK+/ZG5XPqasdz4i+f5xwef9z2tzazH+L7SvVxxYQHf/MAcSksKuenX62g62MKXLplFQYHyXZqZ9XMOiD6gsED83z8+m7KSIm55dANNh45y42VnU1ToDqCZ5Y4Doo+QxN++60xSg4v5xuLVHDjcwj9fPoeSosJ8l2Zm/ZT/BO1DJHHN26bxhT+ayQPPbeVPv/8ULx8+mu+yzKyfckD0QZ84fzI3XnY2j65p5GO3Pen7SZhZTjgg+qgPnDuRb13+Wp5+YTcfvuUJdjQdOv5KZmYnwAHRh13ymrHc/NG5rN62n3d84zf85KkGnwZrZqeNA6KPe+uMKu77n+czpbKMz929lI/e9iQv7DyQ77LMrB9wQPQD06tS3P2pN/DlBbP4wwt7mP/Pv+E7v1lHy9HWfJdmZn2YA6KfKCgQV7yhhsWffRMXTKvkHx54ngU3/ZZnG3wDIjM7OQ6IfmZM+RBuvmIu3/7wOWzff4gFNz3GV+5fwYHDLfkuzcz6GAdEPySJi88ewy8/+2Y+cO5Ebnl0A/O/+QiPrPZNiMys+xwQ/Vj5kGL+4T1nc9dVr2dQUQEfve1J/uquZ9jpU2LNrBtyGhCSLpK0StJaSdd20e4ySSGpLnleLOkOSc9KWinpulzW2d+9bsooFl1zAde8dSo/X/YSb/cpsWbWDTkLCEmFwE3AxcBM4IOSZmZplwL+AngiY/b7gJKIOBuYC3xKUk2uah0IBhcX8tn5tdx/zQVMrij1KbFmdly57EHMA9ZGxPqIOAzcCSzI0u7LwI3AwYx5AZRKKgKGAIeBfTmsdcCYXpXink+/kRs6nBJ7xKfEmlkHuQyIccCLGc8bknntJJ0DTIiI+zusew/QDGwBXgC+HhG7Or6BpKsk1Uuqb2z0AdjuKigQH01OiT1/avqU2Lf+08P84PFNHDziwf/MLC1vB6klFQDfAD6XZfE84CgwFpgMfE7SlI6NIuLmiKiLiLrKysqc1tsfjSkfwi0fncttV9ZRUVbCF372HOff+Gu+/fA69nsAQLMBL5f3g9gMTMh4Pj6Z1yYFnAU8LAmgGlgo6VLgQ8AvIuIIsF3Sb4E6YH0O6x2QJPHWGVW8pXY0j6/fxb89vJYbf/E8//bwWj72hho+fl4No8pK8l2mmeVBLnsQS4BpkiZLGgRcDixsWxgReyOiIiJqIqIGeBy4NCLqSe9WeiuApFLg9cDzOax1wJPEG84YxQ8+8Truu/p8zp9awU0Pr+W8G/+b6xcup2G3D2abDTQ560FERIukq4EHgULgtohYLukGoD4iFnax+k3A9yQtBwR8LyKW5apWO9bZ48v59kfmsnZ7E9/5zTp++Pgmfvj4JhbMGcefXTiFqaNT+S7RzHqA+su58HV1dVFfX5/vMvqll/a8zC2Pruc/n3yBQy2tzJ9ZxWcunMprJgzPd2lmdookPRURdVmXOSCsu3Y2HeL2323kjt9tZN/BFs6bOorPXDiVN54xiuQ4kpn1MQ4IO632HzzCfzzxArc+toHG/Yd4zYTh/Ml5NbxzVjWDiwvzXZ6ZnQAHhOXEwSNH+cnTDdz8yHo27TxAanARC+aM5f11Ezh7XLl7FWZ9gAPCcqq1NXh8w07urm9g0bNbONTSyozqFO+rm8C754z1abJmvZgDwnrMvoNHuG/pS/y4voGlL+6huFC8/cwq3l83gQumVVBU6AGEzXoTB4Tlxaqt+7m7/kXu/cNmdjYfpmpYCZedM5731U1gckVpvsszMxwQlmeHW1r57+e3c3f9i/x61XZaA+bVjOR9deP5H7PHMHRQLi/oN7OuOCCs19i27yA/fXozd9e/yPodzZQOKuSPZo9lwZyxnDt5JMXeBWXWoxwQ1utEBE9t2s2P61/k58u2cODwUcqHFPO2GaOZP6uKN02vdM/CrAc4IKxXO3C4hUfX7OCh5dv41fPb2HPgCCVFBVwwrZL5s6p424zRPhPKLEe6Cgj/iWZ5N3RQEe+cVc07Z1XTcrSVJRt38+DyrSxesY1frtxGgaCuZiTzZ1bxzlnVTBg5NN8lmw0I7kFYrxURLH9pHw+t2MZDy7fy/Nb9AMyoTjF/VjXvnFXFzDHDfEGe2SnwLibrF17YeYCHVmzloeXbqN+0i9aAccOHMH9W+n4W59aMZMggD/VhdiIcENbv7Gg6xH+v3M5DK7byyJodHG5pZVBhAXU1IzhvagXnT63grHHlFBa4d2HWFQeE9WsHDrfw5IZd/HbtDh5bu5OVW/YBUD6kmDeeMYrzp6UDY9IoX5xn1pEPUlu/NnRQERfWjubC2tEANO4/xO/W7eCxNTt4bO0OHnhuKwATRg7h/KkVnDe1gvPOqGBE6aB8lm3W67kHYf1aRLBhRzOPrU0Hxu/X7WT/oRYkmDV2GOdNreCCqZXMnTTCxy9sQPIuJrNEy9FWlm3ey2+T3sXTL+zmyNGgqEDMGjuMuZNGUlczgrpJIxg9bHC+yzXLOQeEWSeaD7Xw5MZdLNmwi/pNu1n64h4OtbQC6V1SdZNGMnfSCOpqRjB9dIoCH/S2fiZvxyAkXQT8C1AI3BoRX+2k3WXAPcC5EVGfzJsNfAcYBrQmyw7msl4beEpLinhL7Wjekhy/ONzSyvKX9vLUpt3Ub9zNo2t2cO8fNgOQGlyUDotJI5g7aSRzJgz3binr13LWg5BUCKwG3gE0AEuAD0bEig7tUsD9wCDg6oiol1QEPA1cERFLJY0C9kTE0c7ezz0Iy4WI4IVdB6jfuJv6Tbt5atMuVm9rAjhmt9RrJw5n9vhyJo4c6gv3rE/JVw9iHrA2ItYnRdwJLABWdGj3ZeBG4K8z5s0HlkXEUoCI2JnDOs06JYlJo0qZNKqUy+aOB2DvgSM8/cJu6jfton7jbn70xCZu++0GIH1q7ezx5Zw9rpzZ49OhMaZ8sEPD+qRcBsQ44MWM5w3A6zIbSDoHmBAR90vKDIjpQEh6EKgE7oyIr3V8A0lXAVcBTJw48TSXb5Zd+dBi3jJjNG+Zkd4tdeRoK6u27ufZzXtZ1rCHZQ17ufmR9bS0pnvnFWUlzB5f3j6dPW44lSkPPmi9X96ug5BUAHwDuDLL4iLgfOBc4ADwq6Qb9KvMRhFxM3AzpHcx5bRgs04UFxZw1rhyzhpXzgfnpf9QOXjkKCu37GNZw16WNezl2c17+PWq7bTt0R1bPpizx7/SyzhrbLmvy7BeJ5cBsRmYkPF8fDKvTQo4C3g46X5XAwslXUq6t/FIROwAkLQIOAc4JiDMeqvBxYW8duIIXjtxRPu85kMtLH9pX3svY1nDHh5cvq19efWwwZw5JsWZY4a1T5MrSj1ciOVNLgNiCTBN0mTSwXA58KG2hRGxF6hoey7pYeDzyUHqdcDfSBoKHAbeDHwzh7Wa5VxpSRHzJo9k3uSR7fP2HjjCs5v3smLLXlZu2c/KLft4dM2O9t1Tg4sLqK06NjRmjEkxbHBxvn4MG0ByFhAR0SLpauBB0qe53hYRyyXdANRHxMIu1t0t6RukQyaARRFxf65qNcuX8qHF6bGiprX/rcShlqOs3d7UHhgrXtrHL5Zv5c4lrxzSGz9iSHtgzByTorZ6GBNHDnVvw04rXyhn1gdEBFv3HWTlln2s3LKfFVv2sXLLPjbsaG4/rlFSVMAZlWVMrypjWlWKaaPLmF6VYoKDw7rgwfrM+jhJjCkfwpjyIbx1RlX7/JcPH2XVtv2s3rqf1dv2s2Z7E09u2MXPnnmpvU1JUQFTR5cxbXQ6OKZXpZheVcb4ETd2fAEAAAydSURBVA4O65oDwqwPGzKokDkThjNnwvBj5u8/eIQ125tYu62J1dv2s3p7E090CI7BxW09jhRTR5dxRmUpUyrLmDRqKCVFvkLcHBBm/VJqcDHnTBzBORlnUcErwbFm235Wb2tizfYmHl+/s304EYACwYSRQ5lSkQ6MMyrLmFJZypTKUirLSnzR3wDigDAbQDoLjqZDLWxobGb9jibWNTazrrGJ9Y3N/G7dzvbBC9PrF6VDo6KUM0aXtYfIpFFDGVzsXkd/44AwM8pKijh7fDlnjy8/Zn5ra/DS3pdZnxEa63c08bt1O/lpRq9DSt8ffHJFaftUU1HKlIpSxg0fQlFhQU//SHYaOCDMrFMFBWL8iKGMHzGUN02vPGZZZq9jfWMzG3Y0s3FnM/c+vZn9h1ra2xUXqn2XVVtwTK4oZUpFGVXDvMuqN3NAmNlJ6azXERHsbD7Mhh3NbGhsZsPO5N8dzTy6Zscxu6yGFBcmgTGUmlGvhEfNqFIqygY5PPLMAWFmp5UkKspKqCgr4dyakccsa20Ntuw7yMYdzaxvC5Ad6YsCH1q+rf0KckgHUE0SHG2h0RYgI4YWOzx6gAPCzHpMQYEYN3wI44YP4bypFccsO3K0lc27X2bDzmY27khPG3YeYFnDXhY9u4WM7GDY4KL23VXtAVJRyuRRpZQP9TAkp4sDwsx6heLCgvQXfkUp1B677HBLKy/uPpAOjeRYx8Yd6Rs5LVz6EpkDQgwfWpzubYwa2t7jmDTK4XEyHBBm1usNSoYROaOy7FXLDh45you7DrBxZxIgO5vZtLOZJRt3818dwmPE0OJ0WLTvsnrl2Ef5EIdHRw4IM+vTBhcXpseeqkq9allbeLT3OpIQeaLDxYEAFWWDmFLxykWBbY8njBxK8QA9TdcBYWb91vHC44UkPDbsaGZ9YxMbdjSzeMU2djYfbm9XVCAmjhyaBMcrFwdOqSxlVGn/PtPKAWFmA9Lg4sJk4MJXh8feA0dYl1zf0RYc6xubeWTNDg5nnKY7bHARkyvLmFpZxrSqZCTd0SnGDR9CQT8YCNEBYWbWQfnQ7EOSHG0NXtrz8jFXla9vbObRNY385OmG9nZDBxUmI+im+nRwOCDMzLqpsCB9VfiEkUO5sMOZVnsPHGHN9vSQ66u37WfNtiYeW9u3g8MBYWZ2GpQPLaauZiR1HS4O3HvgCGsb06Pnrt62n7XbXx0cqZIiZo4dxtnj0lemnzWunMmjSvMeGg4IM7McKh9azNxJI5k7qUNwvHyEtdvTwbHipX08u3kvP3h8U/tQJGUZoTE7T6GR04CQdBHwL6TvSX1rRHy1k3aXAfcA50ZEfcb8icAK4PqI+HouazUz60nlQ14dHEeOtrJ2exPPbt7Lc5v38uzmvfywk9A4e1w6NKZU5C40chYQkgqBm4B3AA3AEkkLI2JFh3Yp4C+AJ7K8zDeAB3JVo5lZb1JcWMCZY4Zx5phhvL9uAgAtR1tZ29jEsw2vhMaPntjEwSPp0CgdVMhbz6ziXz/42tNeTy57EPOAtRGxHkDSncAC0j2CTF8GbgT+OnOmpHcDG4DmHNZoZtarFRUWMKN6GDOqh/G+TkKjbHBuvspzGRDjgBcznjcAr8tsIOkcYEJE3C/przPmlwH/i3Tv4/OdvYGkq4CrACZOnHj6Kjcz68WyhUYu5O36cUkFpHchfS7L4uuBb0ZEU1evERE3R0RdRNRVVlZ21dTMzE5QLnsQm4HMaBufzGuTAs4CHk4uVa8GFkq6lHRP472SvgYMB1olHYyI/5fDes3MLEMuA2IJME3SZNLBcDnwobaFEbEXaB8QXtLDwOeTs5guyJh/PdDkcDAz61k528UUES3A1cCDwErgxxGxXNINSS/BzMx6MUXmYOl9WF1dXdTX1x+/oZmZtZP0VETUZVs2MAc5NzOz43JAmJlZVg4IMzPLqt8cg5DUCGw6hZeoAHacpnJywfWdGtd3alzfqenN9U2KiKwXkvWbgDhVkuo7O1DTG7i+U+P6To3rOzW9vb7OeBeTmZll5YAwM7OsHBCvuDnfBRyH6zs1ru/UuL5T09vry8rHIMzMLCv3IMzMLCsHhJmZZTWgAkLSRZJWSVor6dosy0sk3ZUsf0JSTQ/WNkHSryWtkLRc0l9kaXOhpL2SnkmmL/ZUfRk1bJT0bPL+rxr8SmnfSrbhsuSmUD1VW23GtnlG0j5Jf9mhTY9uQ0m3Sdou6bmMeSMlLZa0Jvl3RCfrfixps0bSx3qwvn+U9Hzy/3evpOGdrNvlZyGH9V0vaXPG/+G7Olm3y9/3HNZ3V0ZtGyU908m6Od9+pywiBsQEFALrgCnAIGApMLNDm88A/548vhy4qwfrGwOckzxOAauz1Hch8PM8b8eNQEUXy99F+j7iAl4PPJHH/++tpC8Cyts2BN4EnAM8lzHva8C1yeNrgRuzrDcSWJ/8OyJ5PKKH6psPFCWPb8xWX3c+Czms73rStwY43v9/l7/vuaqvw/J/Ar6Yr+13qtNA6kG03yM7Ig4DbffIzrQAuCN5fA/wNiV3M8q1iNgSEU8nj/eTHiJ9XE+892m2APh+pD0ODJc0Jg91vA1YFxGncnX9KYuIR4BdHWZnfs7uAN6dZdV3AosjYldE7AYWAxf1RH0R8VCkh+sHeJz0zb7yopPt1x3d+X0/ZV3Vl3x3vB/4z9P9vj1lIAVEtntkd/wCbm+T/ILsBUb1SHUZkl1brwWeyLL4DZKWSnpA0qweLSwtgIckPZXcE7yj7mznnnA5nf9i5nsbVkXEluTxVqAqS5vesh3/hHSPMJvjfRZy6epkF9htneyi6w3b7wJgW0Ss6WR5PrdftwykgOgTJJUBPwH+MiL2dVj8NOldJq8B/hX4WU/XB5wfEecAFwN/LulNeaihS5IGAZcCd2dZ3Bu2YbtI72voleeaS/o7oAX4USdN8vVZ+DZwBjAH2EJ6N05v9EG67j30+t+lgRQQx7tH9jFtJBUB5cDOHqku/Z7FpMPhRxHx047LI2JfRDQljxcBxZIqOrbLpYjYnPy7HbiXdFc+U3e2c65dDDwdEds6LugN2xDY1rbbLfl3e5Y2ed2Okq4E/gj4cBJir9KNz0JORMS2iDgaEa3ALZ28b763XxHwHuCuztrka/udiIEUEO33yE7+wrwcWNihzUKg7WyR9wL/3dkvx+mW7K/8LrAyIr7RSZvqtmMikuaR/v/ryQArlZRqe0z6YOZzHZotBD6anM30emBvxu6UntLpX2753oaJzM/Zx4D/ytLmQWC+pBHJLpT5ybyck3QR8DfApRFxoJM23fks5Kq+zGNaf9zJ+3bn9z2X3g48HxEN2Rbmc/udkHwfJe/JifQZNqtJn93wd8m8G0j/IgAMJr1bYi3wJDClB2s7n/SuhmXAM8n0LuDTwKeTNlcDy0mfkfE48MYe3n5TkvdemtTRtg0zaxRwU7KNnwXqerjGUtJf+OUZ8/K2DUkH1RbgCOn94J8gfVzrV8Aa4JfAyKRtHXBrxrp/knwW1wIf78H61pLef9/2OWw7s28ssKirz0IP1feD5LO1jPSX/piO9SXPX/X73hP1JfNvb/vMZbTt8e13qpOH2jAzs6wG0i4mMzM7AQ4IMzPLygFhZmZZOSDMzCwrB4SZmWXlgDDLo2R02Z/nuw6zbBwQZmaWlQPCrBskfUTSk8nY/d+RVCipSdI3lb5/x68kVSZt50h6PON+CiOS+VMl/TIZKPBpSWckL18m6Z7kHgw/yrjS+6tK3x9kmaSv5+lHtwHMAWF2HJLOBD4AnBcRc4CjwIdJX7VdHxGzgN8AX0pW+T7wvyJiNukrftvm/wi4KdIDBb6R9BW4kB659y+BmaSvsD1P0ijSw0jMSl7n/+T2pzR7NQeE2fG9DZgLLEnuDvY20l/krbwyGNsPgfMllQPDI+I3yfw7gDcl4+6Mi4h7ASLiYLwyztGTEdEQ6cHnngFqSA81fxD4rqT3AFnHRDLLJQeE2fEJuCMi5iRTbURcn6XdyY5bcyjj8VHSd3NrIT265z2kR1X9xUm+ttlJc0CYHd+vgPdKGg3t95SeRPr3571Jmw8Bj0XEXmC3pAuS+VcAv4n0XQIbJL07eY0SSUM7e8PkviDlkR6S/K+A1+TiBzPrSlG+CzDr7SJihaT/TfruXwWkR+78c6AZmJcs2076OAWkh/D+9yQA1gMfT+ZfAXxH0g3Ja7yvi7dNAf8laTDpHsxnT/OPZXZcHs3V7CRJaoqIsnzXYZYr3sVkZmZZuQdhZmZZuQdhZmZZOSDMzCwrB4SZmWXlgDAzs6wcEGZmltX/B7KKantoZmLrAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dXlKfwzSLBgB",
        "outputId": "1fc6ab68-1ba4-4f30-9aba-e12a08e76cf2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 635
        }
      },
      "source": [
        "parameters = model(X_train, y_train, learning_rate = 0.1, n_h = 4, optimizer=\"adam\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cost after epoch 0: 1.453199\n",
            "Cost after epoch 1: 1.449830\n",
            "Cost after epoch 2: 1.451291\n",
            "Cost after epoch 3: 1.451958\n",
            "Cost after epoch 4: 1.449140\n",
            "Cost after epoch 5: 1.449319\n",
            "Cost after epoch 6: 1.458471\n",
            "Cost after epoch 7: 1.448479\n",
            "Cost after epoch 8: 1.457921\n",
            "Cost after epoch 9: 1.449281\n",
            "Cost after epoch 10: 1.474954\n",
            "Cost after epoch 11: 1.448607\n",
            "Cost after epoch 12: 1.448502\n",
            "Cost after epoch 13: 1.454827\n",
            "Cost after epoch 14: 1.451984\n",
            "Cost after epoch 15: 1.487578\n",
            "Cost after epoch 16: 1.450886\n",
            "Cost after epoch 17: 1.448374\n",
            "Cost after epoch 18: 1.456553\n",
            "Cost after epoch 19: 1.455893\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hc5bXo/++aGY26JVmSm+RgjMGAARuwHTqmBBySEHpCEgKkEELCOTnn5oTklJvc/JJ7Q8oJIYWSE1oKEDAQ000LhADBBXcDNsZgybIlWcXq0mjW74+9tzwWKtNnJK3P88xjae89M6/G0qx5y1qvqCrGGGNMtHyZboAxxpixxQKHMcaYmFjgMMYYExMLHMYYY2JigcMYY0xMLHAYY4yJiQUOY0YgIqeKyFuZbocx2cQCh8laIrJDRM7OZBtU9W+qOjeTbfCIyBIRqcnQc39GRN4TkQ4ReUREJo9w7e0i8paIhEXkqjQ206SJBQ4zoYmIP9NtABBHVv49isg84DbgCmAq0An8ZoS7rAOuA9akvnUmE7LyF9WYkYiIT0S+LSLviMheEflz5CdgEXlARHaLSKuIvOS+8Xnn7hKRW0TkCRHpAM5wezbfFJH17n3uF5E89/oDPuWPdK17/lsiUiciu0TkSyKiIjJnmJ/jryLyQxH5O86b8WwRuVpEtohIm4hsF5GvuNcWAk8CM0Sk3b3NGO21SJLPAo+q6kuq2g78F3CRiBQPdbGq/lpVnwO6k9wOkyUscJix6HrgAuB0YAbQDPw64vyTwKHAFJxPvX8cdP/PAD8EioGX3WOXAUuBg4FjgKtGeP4hrxWRpcC/AmcDc4AlUfwsVwDXuG15D6gHPg5MAq4Gfi4ix6lqB/BRYJeqFrm3XVG8FgNE5EMi0jLC7TPDtHEeTi8CAFV9B+gFDovi5zPjUCDTDTAmDtcCX1fVGgAR+R7wvohcoaohVb3Du9A91ywiJara6h7+i6r+3f26W0QAbnbfiBGRR4EFIzz/cNdeBtypqpsinvuzo/wsd3nXux6P+PpFEVkBnMrwwz4jvhaRF6rq+0DpKO0ZShHQOuhYK06wMxOQ9TjMWHQQ8LD3SRnYAvQDU0XELyI/codu9gE73PtURNx/5xCPuTvi606cN8vhDHftjEGPPdTzDHbANSLyURF5TUSa3J/tPA5s+2DDvhZRPHe02nF6QJEmAW1JfA4zhljgMGPRTuCjqloacctT1VqcYahP4gwXlQCz3PtIxP1TVRK6DqiO+H5mFPcZaIuI5ALLgJ8CU1W1FHiC/W0fqt0jvRYHcIeq2ke4Ddc72gTMj3ic2UAu8HYUP58ZhyxwmGyXIyJ5EbcAcCvwQxE5CEBEKkXkk+71xUAPsBcoAP5vGtv6Z+BqETlCRApwJpFjEcR5Q24AQiLyUeCciPN7gHIRKYk4NtJrcQBVfT9ifmSo2+C5IM8fgU+Ik9NSCHwfeEhVh+xxiEjQXTAg7P//s/eaccT+M022ewLoirh9D/gFsBxYISJtwGvAh93r78GZZK4FNrvn0kJVnwRuBl4AtkU8d0+U928D/gknADXj9J6WR5x/E7gX2O4OTc1g5NciKdw5mGtxAkg9TnC+zjsvIk+KyL9H3GUFzv/VScDt7tenJbNNJrPENnIyJjVE5AhgI5A7eKLamLHMehzGJJGIXCgiuSJSBtyIk/9gQcOMKxY4jEmur+AM57yDs7rpq5ltjjHJZ0NVxhhjYmI9DmOMMTGZEJnjFRUVOmvWrEw3wxhjxpTVq1c3qmrl4OMTInDMmjWLVatWZboZxhgzpojIe0Mdt6EqY4wxMbHAYYwxJiYWOIwxxsTEAocxxpiYWOAwxhgTEwscxhhjYmKBwxhjTEwscBhjxqWWzl6Wr9uV6WaMSxY4jDHj0oOra/ine9+gfl93ppsy7ljgMMaMSw1tzv5Z9W1R7aNlYmCBwxgzLjW0OwFjb0dvhlsy/ljgMMaMS43tTsDY2249jmSzwGGMGZca3SGqve3W40g2CxzGmHGp0e1pNFqPI+kscBhjxp1wWGly5zYarceRdBY4jDHjTmtXH6Gwsy323g7rcSSbBQ5jzLgTOTxlcxzJZ4HDGDPueEtxq0rzbVVVCqQscIjIHSJSLyIbR7lukYiEROSSiGM/FpFNIrJFRG4WEXGP/1VE3hKRte5tSqrab4wZu7x5jbnTimns6EVVM9yi8SWVPY67gKUjXSAifuBGYEXEsZOAk4FjgKOARcDpEXf7rKoucG/1yW60MWbs85bizp1WTG8oTFtPKMMtGl9SFjhU9SWgaZTLrgeWAZEBQIE8IAjkAjnAnlS00RgzPjW29+D3CYdUFgE2z5FsGZvjEJEq4ELglsjjqvoq8AJQ596eVtUtEZfc6Q5T/Zc3hDXM418jIqtEZFVDQ0MKfgJjTLba297L5MIglcW57vc2z5FMmZwcvwm4QVXDkQdFZA5wBFANVAFnisip7unPqurRwKnu7YrhHlxVb1fVhaq6sLKyMiU/gDEmOzW291BRlEtFUdD93nocyRTI4HMvBO5zOw0VwHkiEgIOBV5T1XYAEXkSOBH4m6rWAqhqm4j8CVgM3JOJxhtjspcTOIJUFLk9DsvlSKqM9ThU9WBVnaWqs4AHgetU9RHgfeB0EQmISA7OxPgW9/sKAPf4x4ERV2wZYyamxvZeKotyKStwehw2x5FcKetxiMi9wBKgQkRqgO/iTHSjqreOcNcHgTOBDTgT5U+p6qMiUgg87QYNP/As8NtUtd8YMzapKg3tPVQU5xIM+CjJz7F6VUmWssChqpfHcO1VEV/3A18Z4poO4PikNM4YM2619YToDYUH5jfKi4LW40gyyxw3xowrXpAoL3TmNyoKc63HkWQWOIwx44oXJCrcpbgVxUHbBTDJLHAYY8YVL2t8YKiqMNfyOJLMAocxZlzxehyV7lLc8qIgzZ19hPrDI93NxMAChzFmXGlo70UEJhd6k+NOAGmy4aqkscBhjBlXGtt7KCsIEvA7b28VhZY9nmwWOIwx40pjWw/lbrCA/T0Oyx5PHgscxphxZW9H70CpEdg/SW65HMljgcMYM640ulnjHq/HYbkcyWOBwxgzrjS29Qz0MgAm5QXI8YvlciSRBQ5jzLjR1dtPR2//AUNVIkJ5Ye5AfodJnAUOY8y4MTiHw1NeZNnjyWSBwxgzbjQMlBsJHnC8vMiyx5PJAocxZtzwhqO8AoeeisKg5XEkkQUOY8y44Q1HRa6q8r7f29GDqmaiWeOOBQ5jzLixv8cxaKiqMEh3X5jO3v5MNGvcSWngEJE7RKReREbc4lVEFolISEQuiTj2YxHZJCJbRORmcTcnF5HjRWSDiGyLPG6MMY3tPRTnBcjL8R9w3HI5kivVPY67gKUjXSAifuBGYEXEsZOAk4FjgKOARTh7jwPcAnwZONS9jfj4xpiJw9trfLDyIqtXlUwpDRyq+hLQNMpl1wPLgPrIuwJ5QBDIxdmrfI+ITAcmqepr6gxW3gNckPSGG2PGpIb2ngNyODwV7mS5raxKjozOcYhIFXAhTi9igKq+CrwA1Lm3p1V1C1AF1ERcWuMeG+qxrxGRVSKyqqGhIRXNN8ZkGafcSPADx70eh+VyJEemJ8dvAm5Q1QN2WBGROcARQDVOYDhTRE6N5YFV9XZVXaiqCysrK5PWYGNM9trb3vuBpbgQETisx5EUgQw//0LgPnd+uwI4T0RCOHMXr6lqO4CIPAmcCPweJ5h4qoHatLbYGJOVekNhWrv6hhyqyg34Kc4L2BxHkmS0x6GqB6vqLFWdBTwIXKeqjwDvA6eLSEBEcnAmxreoah2wT0ROcFdTfR74S6bab4zJHt5+G0MNVQFUFOXaqqokSWmPQ0TuBZYAFSJSA3wXZ6IbVb11hLs+CJwJbMCZKH9KVR91z12Hs1orH3jSvRljJrjGNjf5b4geBzi5HLYnR3KkNHCo6uUxXHtVxNf9wFeGuW4VzhJdY4wZ4PUmhg0cRUHebexIZ5PGrUxPjhtjTFI0DFMZ1+MUOrQeRzJY4DDGjAuNw1TG9VQU5dLU2Ut/2OpVJcoChzFmXNjb3kt+jp+C4NAj8BVFQVShudN6HYmywGGMGReGS/7zePkdtrIqcRY4jDHjQuMw5UY8+5MArceRKAscxphxobGtd8TAUTFQ6NB6HImywGGMGRdG7XEMFDq0HkeiLHAYY8a8UH+Yps5eKouGn+Moyc8h4JOBDHMTPwscxpgxr6mzF9X9GzYNxecTJlv2eFJY4DDGjHleMBhpqAqcwGJzHImzwGGMGfP2lxsZfqjKO28VchNngcMYM+btzxofpcdRGLQ5jiSwwGGMGfNGq4zrsXpVyWGBwxgz5jW29xD0+5iUN3LB74qiXDp7++nsDaWpZeOTBQ5jzJjX0N5DRVEQdzfRYVn2eHJY4DDGjHmN7b0jLsX1WPZ4cljgMMaMeXvdHsdoLHs8OVIWOETkDhGpF5GNo1y3SERCInKJ+/0ZIrI24tYtIhe45+4SkXcjzi1IVfuNMWPHaOVGPANDVbayKiGp3Dr2LuBXwD3DXSAifuBGYIV3TFVfABa45ycD2yLPA/+mqg+moL3GmDEoHFb2tveOuhQX9q+6slyOxKSsx6GqLwFNo1x2PbAMqB/m/CXAk6ramcy2GTMW/PWter5x3xuZbkbWa+3qIxTWqHoceTl+inIDNlSVoIzNcYhIFXAhcMsIl30auHfQsR+KyHoR+bmIDPubIiLXiMgqEVnV0NCQhBYbk15Pb9rDI2t30drVl+mmZLVos8Y95UWWBJioTE6O3wTcoKrhoU6KyHTgaODpiMPfAQ4HFgGTgRuGe3BVvV1VF6rqwsrKyuS12pg0qW3pcv5t7spwS7Jbgxs4KqPocYCTPW6rqhKTycCxELhPRHbgDEn9xpsEd10GPKyqAx+3VLVOHT3AncDidDbYmHSqaXZGaL0AYobmDTtFsxzXu86GqhKTscChqger6ixVnQU8CFynqo9EXHI5g4ap3F4I4mT5XACMuGLLmLFKVQd6GrXNNsU3kliHqqzQYeJStqpKRO4FlgAVIlIDfBfIAVDVW0e57yxgJvDioFN/FJFKQIC1wLVJbbQxWaKxvZeekDOKaz2OkTW29+D3CWUFUc5xFObS1NFDOKz4fCNnmpuhpSxwqOrlMVx71aDvdwBVQ1x3ZsINM2YMqInoZVjgGFljWy+TC4NRB4GKoiBhhZauPiYXRhdszIEsc9yYLFTjDlNNKc6ltqU7w63JbtEm/3m8uZC9NkEeNwscxmQhr5ex+ODJtqpqFI1RlhvxeNnjDRY44maBw5gsVNPcSWlBDnOnFtPY3kN3X3+mm5S1Gtt7Y+pxVBRZvapEWeAwJgvVNHdRXZZPVVk+ALtsnmNIqhp7j6PQK61uPY54WeAwJgvVNHdRVZrPjFIncNgE+dDae0L0hMIx9ThKC4L4BPZ2WI8jXhY4jMkyXg5HdVkBVV7gsHmOIXn5GLEEDr9PmFyYa7kcCbDAYUyWaeropauvn+qyfKaV5OET63EMZyD5L4rKuJEqioI2VJUACxzGZBlvKW5VaT45fh/TJuVZj2MYjW2xZY17yousXlUiLHAYk2W83kV1WQEAVWX51uMYRmOMBQ495YW5NseRAAscxmQZL2vcW1FVVWqBYzgN7jxFWYwZ4OVFQVuOmwALHMZkmZrmLorzApTk5wBOANnd2k1/WDPcsuyzt72HsoIccvyxvZVVFOXS3hOy/Jg4WeAwJst4K6o8M0rzCYWVPfus9MhgsZYb8VQM7D1uvY54WOAwJst4yX+eKsvlGFasWeOe8kKrV5UICxzGZBFVpaa5cyBYAANBxFZWfVBje0/MS3Fhf70qW1kVHwscxmSR1q4+Onr7D+hxWPb48BrbYis34vF6KZYEGB8LHMZkES+HI3KOoyAYYHJhcOCccXT19tPR2x/fUJU3x2GBIy4pCxwicoeI1IvIiNu7isgiEQmJyCXu92eIyNqIW7e3F7mIHCwi/xCRbSJyv4jYLixmXPGW4kb2OMCZ57BChweKdcvYSAXBAPk5fpvjiFMqexx3AUtHukBE/MCNwArvmKq+oKoLVHUBcCbQGXH+RuDnqjoHaAa+mIJ2G5Mx+3scHwwcNlR1oP2BI/YeB7i5HLaqKi4pCxyq+hLQNMpl1wPLgPphzl8CPKmqnSIiOIHkQffc3cAFyWirMdmiprmLotz9ORyeqrJ8apu7ULVcDk88BQ4jVRTl2uR4nDI2xyEiVcCFwC0jXPZp4F7363KgRVVD7vc1DLEvecTjXyMiq0RkVUNDQzKabEzKeUtxnc9J+80ozaerr5/mzr4MtSz7xFvg0FNRFLTJ8ThlcnL8JuAGVQ0PdVJEpgNHA0/H8+CqeruqLlTVhZWVlQk005j0qW3pOmAprsfKq3+QV+CwPMZyI57ywlyb44hTVIFDRC6N5liMFgL3icgOnCGp33iT4K7LgIdV1fuItRcoFZGA+301UJtgG4zJKjXNnR+Y34CIXI6WznQ3KWs1tvdQnBcgL8cf1/3Li4I0dfQStlIuMYu2x/GdKI9FTVUPVtVZqjoLZ97iOlV9JOKSy9k/TIU6g7sv4AQZgCuBvyTSBmOySWtXH23doQOW4nq8Hoctyd2vsb035qq4kcqLcgmFlX3dNvwXq8BIJ0Xko8B5QJWI3BxxahIQGvpeA/e9F1gCVIhIDfBdIAdAVW8d5b6zgJnAi4NO3YDTS/kB8Abwu5Eex5ixZHBV3EilBTkUBP3sarF6VZ7G9p6BfIx4VAxkj/dSWmAr+2MxYuAAdgGrgPOB1RHH24B/GemOqnp5tI1Q1asGfb+DISa+VXU7sDjaxzVmLKkdZikugIi4S3JtqMrT2N7DYVOL476/txprb3sPc6YUJatZE8KIgUNV1wHrRORP3lyDiJQBM1W1OR0NNGaiGCprPJJt6HSgxvZeTjokkaGq/T0OE5to5zieEZFJIjIZWAP8VkR+nsJ2GTPh1DR3kZ/jp6wgZ8jzM0rzbVWVqzcUprWrL+4cDoiokNthK6tiFW3gKFHVfcBFwD2q+mHgrNQ1y5iJp7alc8gcDk9VaT7NnX109o44vTgheG/2FcXxz02UFeQgYj2OeEQbOAJuXsVlwGMpbI8xE9bgfTgGs/Lq+zW2JZY1DhDw+ygrCFouRxyiDRzfx0nEe0dVV4rIbGBr6pplzMRT09w15Ioqz8CSXJvnSKjAYaTyQtt7PB6jraoCQFUfAB6I+H47cHGqGmXMRNPW3UdrV9+wE+Owf5mu9TgSL3DoqSjKtTmOOESbOV4tIg+7ZdLrRWSZiFSnunHGTBTeaqmRhqqmFOcR8ImVVyfxAoeecqtXFZdoh6ruBJYDM9zbo+4xY0wS1DQ5wWCoOlUev0+YXppnS3Jxehz5OX4Kc6MaNBmWVciNT7SBo1JV71TVkHu7C7DKgcYkyf4ex/BDVQAzSmxJLnh7jSee7V1eGKStO0RPqD8JrZo4og0ce0XkcyLid2+fwyk6aIxJgprmTnIDvlEney0J0NHY3pPwMBU49aoAmmxDp5hEGzi+gLMUdzdQh1No8KoUtcmYCcdbUTVcDoenujSfPfu66esfcjeCCaOxrTdJgcP2Ho9HLMtxr1TVSlWdghNI/k/qmmXMxFLb0jXqMBU4PY6wwu7WiV3scG9HT8JLcWH/5LrNc8Qm2sBxTGRtKlVtAo5NTZOMmXhGS/7zVJUWDFw/UfWHlaaO5PQ4KqxeVVyiDRw+t7ghAG7NqsSWMxhjAOjsDdHU0TviiiqPl8sxkZfkNnX0EtbEl+LC/jkOyx6PTbRv/j8DXhURLwnwUuCHqWmSMRPLSOXUB5tekufcZwIHjmQl/wEUBv3kBnzstcnxmESbOX6PiKwCznQPXaSqm1PXLGMmjtHKqUfKy/FTUZQ7oZfkJqvcCDj7nFguR+yiHapCVTer6q/c26hBQ0TucLPMN45y3SIRCYnIJRHHPiQiK0Rki4hsdncERETuEpF3RWSte1sQbfuNyVbezn8zo+hxgC3JHQgcxYn3OMBZWWWrqmITdeCIw13A0pEuEBE/cCOwYtCpe4CfqOoRODv+1Uec+zdVXeDe1iaxvcZkRE1LF0G/L+qhl+rSCR44klAZN5LVq4pdygKHqr4ENI1y2fXAMiICg4gcCQRU9Rn3cdpV1fbLNOOWl8Ph842cw+HxehzhsKa4ZdmpsaOHoN/HpLzkrM8pLwwOBCMTnVT2OEYkIlXAhcAtg04dBrSIyEMi8oaI/MTtmXh+KCLrReTnIjLsRw4RuUZEVonIqoaGhhT8BMYkR7RLcT1Vpfn0hsI0TtBPyY1tvZQXBUdNloxWudvjUJ2YgTgeGQscwE3ADao6OAU2AJwKfBNYBMxmf5b6d4DD3eOTgRuGe3BVvV1VF6rqwspKK6tlsldtc2dUS3E93rUTdYI8WeVGPBVFQfr6lX3dtrNitDIZOBYC94nIDpwSJr8RkQuAGmCtqm5X1RDwCHAcgKrWqaMHpzrv4sw03Zjk6O7rp7G9N7Yex0Aux8TMHncCR+Irqjz7y45MzB5cPDIWOFT1YFWdpaqzgAeB61T1EWAlUCoiXjfhTGAzgLt9LeL0US8ARlyxZUy2i2UprmdgQ6eWiTn1l+weR3mhmwRouRxRS1n2t4jcCywBKkSkBvgukAOgqrcOdz9V7ReRbwLPuQFiNfBb9/Qf3YAiwFrg2lS135h08JbijrRl7GCT8nIozg1MyKGqcFjZ296btKW4sH91lvU4opeywKGql8dw7VWDvn8GOGaI684cfMyYsSyanf+GMlFzOVq7+giFlfLC5A1VecNeDZbLEbVMznEYM+HVNHeR4xemFOfFdL+q0vwJWejQy7eoTGKPo6zQ5jhiZYHDmAyqae5iekk+/ihzODwTtcfRkOTkP4Acv4/SghzLHo+BBQ5jMqi2uTPmYSpwehxt3SH2dfeloFXZK5kFDiOVFwYtezwGFjiMyaBYk/88AyurJthwVTILHEYqL8q1PTliYIHDmAzp7uunvq1nYHOmWHhJgBNtX47G9h78PqGsILmBo7Io1+Y4YmCBw5gMqXO3f413qAom3r4cjW29TC4MRl3XK1rlRUHrccTAAocxGeLlcMQTOCqKcgn6fRNuqGpvR09Sl+J6ygtzae3qozc0uAKSGYoFDmMyxFtOG0vyn8fnE2aU5lEzwXocDe29SV2K6/HKjjR3Wq8jGhY4jMmQ2uYu/D5h2qTYcjg8VWX5E67H0diW3HIjHm+y3XYCjI4FDmMypKa5k+kleQT88f0ZVk2wDZ1UNekFDj3lA2VHrMcRDQscxmRITXNXTOXUB6sqLaChrYfuvv4ktip7tfeE6AmFU9Tj8AodWo8jGhY4jMmQ2paumKriDubNjexunRjl1b1VT6kIHN4ch+0EGB0LHMZkQG8ozO593XGtqPLMKHXmRibKcNVA8l8KJseLcwME/b4Ju6tirCxwGJMBda1dqMa3FNdT7SYOTpQJci9BLxXLcUWE8qKgzXFEyQKHMRmQyFJcz7SSPESYMEtyvbLnqViOC7iBw3oc0bDAYUwGeL2EmQnMcQQDPqYW502YHkdjm/OmPjkFPQ5wkgBtF8DopCxwiMgdIlIvIiNu7yoii0QkJCKXRBz7kIisEJEtIrJZRGa5xw8WkX+IyDYRuV9EUvMbZEyK1TR34hOn15AIp7z6xNhCtrG9h7KCHHLiXL48Ghuqil4qexx3AUtHukBE/MCNwIpBp+4BfqKqRwCLgXr3+I3Az1V1DtAMfDGZDTYmXWqau5g2KS/hN8GJlMuR7L3GB6ssyqWhvQdVTdlzjBcpCxyq+hLQNMpl1wPL2B8YEJEjgYC7fSyq2q6qne7+42cCD7qX3g1ckPSGG5MGNQkuxfVUleVT19JNf3j8v9k1tvemNHCUFwXpDYVp7wml7DnGi4zNcYhIFXAhcMugU4cBLSLykIi8ISI/cXsm5UCLqnr/qzVA1QiPf42IrBKRVQ0NDan4EYyJW22c+3AMVlWaTyisNLSN/0ndxvaelCzF9ZQXWvZ4tDI5OX4TcIOqDi5HGQBOBb4JLAJmA1fF+uCqeruqLlTVhZWVlYm21Zik6esPU9faldCKKs/+8urjf55jb3tvSpbierwkQMseH10gg8+9ELjPGYGiAjhPREI4PYm1qrodQEQeAU4A7gBKRSTg9jqqgdqMtNyYBOxu7SacYA6Hxws+Nc1dHH9Qwg+Xtbr7+mnvCaVsKS7sz0i3fTlGl7HAoaoHe1+LyF3AY6r6iDssVSoilaragDOvsUpVVUReAC4B7gOuBP6SgaYbkxAvhyMpcxwTZEMnbyguFQUOPQM9Dgsco0rlctx7gVeBuSJSIyJfFJFrReTake6nqv04w1TPicgGQIDfuqdvAP5VRLbhzHn8LlXtNyZVvA2cEilw6CnMDVBakDPuczn27zWe+jkOK60+upT1OFT18hiuvWrQ988Axwxx3Xac5bnGjFm1LV2IwPTSxHI4PBNhSW4qCxx6ggEfk/IClj0eBcscNybNapq7mFqcR27An5THqyod/xs6pbLAYaSKolwaLXt8VBY4TNZ6bsse/nvFW5luRtLVNHcmZUWVp6osn10tXeM6cS2VBQ4jWb2q6FjgMFnrpyve5ubnt427/SacfTiSGDhK8+no7ae1qy9pj5ltGtt7Kc4NkJeTnF7acMoLc21yPAoWOExW2lbfxpa6fQCs2Lw7w61JnlB/mLqWxPbhGMybZK8Zx8NVDSlO/vOUFwWt0GEULHCYrLR8XR0+gekleTy9KTOB4/ev7uDSW18hnMRyHnvaegiFlarSxJfierxhr/E8Qd7Ylpq9xgerKMqlubOXUP/gvGQTyQKHyTqqyqPrdnHC7HIuPLaK17Y30ZzmT4Gqyl2v7GDljmZe2743aY9bO5DDkfwex3ieIE91gUNPRVEQVWjqtF7HSCxwmKyzsXYf7zZ2cP78GSw9ahr9YeW5N+tHv2MSvbWnjXcaOgBYtiZ5BQq8HI5kBo7JhUHycnzju8eR4gKHnvIiq1cVDQscJussX1dLjl/46FHTObqqhBkleTy1Mb3DVY+vd4bKzj5iKk9urLcKItEAACAASURBVKOzNzkVU715iBlJSP7ziMi4XpLbGwrT2tWXnsBRaNnj0bDAYbJKOKw8tr6O0w+rpKQgBxHhnHnT+NvWBjrSVO5aVXl8fR0nHlLONafNprO3P2nzLDXNnVQW5yZ9dVBVWcG47XE0ucOU5WmY4xjocVihwxFZ4DBZZeWOJupau/nE/BkDx86dN42eUJgX305PefwtdW1sb+zgY0fPYOFBZcycnM9DSRquSvZSXE9VaR67xmngSEe5EY83AW+FDkdmgcNkleXrdpGf4+cjR04dOLZoVhllBTlpW1312Ppd+H3C0qOm4fMJFx5bzcvbGpOST1LTnJwNnAarKs1nb0cvXb39SX/sTGtwA0dlcep7HCX5OQR8kjX1qvqydHWXBQ6TNfr6wzy5cTdnHTGFguD+MmoBv4+zj5jK81vq6Q2l9g9JVXl8Qx0nHVLOZHe8+6Jjq1CFR9Ym1usIh5VdLV1JKW442HhektvYlr4eh4hkTfb405t2c+h/PMnpP3mB6+99g//523ZW7mhK2nxbIjK5H4cxB/j7tkaaOno5P2KYyrP0qGk8sLqGV95pZMncKSlrw6Zd+3hvbyfXLTlk4NisikIWHlTGstU1fOW02bh7yMSsvq2Hvn5N0VCV04upbelizpSipD9+JqWjwGGkbMkev+vvO5hSnMsR0yaxekcTj67bBYBP4LCpxRxTXcL8maXMry5l7rTihPevj4UFDpM1lq/bxaS8AKfP/eCOjSfPqaAw6OfpTXtSGjgeW19HwCecc+S0A45fdFw1//7wBjbW7uPo6pK4HjsVS3E9Az2OcbiyqrG9h/wcP4W56Xm7Ki8KZrzQ4Xt7O3h1+17+10cO4/qzDgWgvq2b9TtbWV/TwrqaVp7ZvIc/r6oBnMq+R06fxPzqEo6pLmX+zFJmVxTi88X3IWc0FjhMVuju62fFpj2cd/S0IavG5uX4WXL4FJ7ZvJsfXHAU/hT8QTjDVLs4eU4FZYOK6X3s6Ol879FNLFtTk0DgSH7yn2dqcS5+n4zLLWSdvcZTP7/hqSjK5d3GjrQ931AeWFWDT+CShdUDx6YU53H2kXmc7c7/qSo7m7pYV9MyEEweWF3D3a++B0BxboCjqkr4vxcdzcEVhUltnwUOkxVeeLOe9p4Q58+vGvaac+dN4/H1dax5v5lFsyYnvQ0balvZ2dTF9Wce+oFzJQU5fOSIqSxft4v/+NgRcQ0LePMPySw34gn4fUyblDcuexzOXuPpGaYCJ5cjk0NVof4wD6zeyemHVTK9ZPgPGSLCh8oL+FB5wcAqxP6wsq2+fSCYrK9ppTQ/J+ltTOUOgHeISL2IbBzlukUiEhKRSyKO9YvIWve2POL4XSLybsS5Balqv0mv5et2UVGUy4mHlA97zRlzKwn6fTydomTAx9fXkeMXzh00TOW56Lgqmjp6efGt+JYF1zR3UlEUJD+YmgqvVaX57GoZX5WEIX3lRjwVxbl09fWnLW9osJe2NrBnXw+fWvShmO/r9wlzpxVz2cKZ/OCCo1n+9VM+0HtOhlTOptwFLB3pAnd/8RuBFYNOdanqAvd2/qBz/xZxbm3ymmsypa27j+ferOdjR08bcQiqOC+Hk+eU89Sm3Unfe0LVSTw89VAn8XAopx1WSXlhkIfeqInrOWqaU7OiylNVNj53Amxs70nLUlxPprPH73t9JxVFQc46InVzeYlKWeBQ1ZeAplEuux5YBqS3EJHJKis27aE3FOb8BR9cTTXYufOmUdPcxWa35HqyrN3ZQm1LFx87evqw1+T4fZy/YAbPbq6nJY4ieLUpyuHwVJXms3tf97iq7NofVpo60lOnyuM9V2MGssfr27p5/s16Lj6uOq2rpGKVsZaJSBVwIXDLEKfzRGSViLwmIhcMOvdDEVkvIj8XkfT9NpmUeXT9LqpK8znuQ2WjXnv2kVPxCUkfrnp8fR1Bv29g4nE4Fx9XTW9/mMfW18X0+OGwUpOirHFPVVk+/WFl977sGK7q7A3x+rujfXYcWVNHL2FN31Jc2F/aJBM9jofW1BIKK5cunJn2545FJkPaTcANqjrUx6ODVHUh8BngJhHxFtV/BzgcWARMBm4Y7sFF5Bo3+KxqaEhPqQoTu6aOXl7e2sgn5s+IKj+ioiiXhbMm8/SmPUlrQzisPLGhjtMOq6BklInEeTMmMXdqMQ+tiW24qrG9h95QOKlbxg6WbeXVv7d8E5fd9io/eGxz3HuapLPciGd/hdz09jhUlT+v3MmiWWVZn4uTycCxELhPRHYAlwC/8XoXqlrr/rsd+CtwrPt9nTp6gDuBxcM9uKrerqoLVXVhZeUH8wJMdnhiQx2hsA6Z9Decc+dN4609bUlbMvnGzhZ2tXbzsWOGH6byiAgXHVfFmvdbYnr+mpbULcX1ZFP2+Pt7O1m2ppaZk/P5n5ff5Rv3r40r69/71J+OAoeegTmONOdyrNzRzPbGDi7L8t4GZDBwqOrBqjpLVWcBDwLXqeojIlLmDUGJSAVwMrDZ/X66+68AFwAjrtgy2W/5ul3MmVLEEdOLo77PufOc4aRk1a56fH0dwYBT1iQaFxxbhU/g4Rh6HftzOFI7xwHZ0eP45fNb8fuEB689iW8tncvydbu4+q7XaeuObV/0TPQ48nL8FOcGaGhLb4/j/pU7KcoNRPUBJtNSuRz3XuBVYK6I1IjIF0XkWhG5dpS7HgGsEpF1wAvAj1R1s3vujyKyAdgAVAA/SFX7TerVtXaxckcT50c5TOWpLivgqKpJSQkc3jDV6YdVUpwX3Xr3qZPyOHlOBQ+9URv1EIyXNZ7KVVV5OX4qioIZ73G8t7eDh96o5TOLP8TUSXlct2QOP7t0Pv/Y3sSnbnuN+hjmYLzAUZnGwAHp33t8X3cfT2yo4xPzZxxQpy1bpayFqnp5DNdeFfH1K8DRw1x3ZuItM9nisXV1qHJACfVoLZ03jZ+ueJvdrd1MK8mLuw1r3m9m975uvnPM4THd7+LjqvnG/WtZuaOJD88ePvfEU9vcRVlBTsrLZswozfyS3F8+v42ATw6o93Xx8dWUFwW57o9ruOiWV7j7C4s5pHL0cfyG9h6Cfh+T8tP7ZlpelJvWOY5H1+2iq6+fTy3K/mEqsOq4JoOWr9vF0VUlcZVDOHeek6T3zObEeh2Pra8jN+DjrCiHqTznzJtKYdDPsiiHq1JVTn2wqgwHjh2NHTz8Ri2f/fBBTJl0YEBfMncK911zAl29/Vxyyyuseb951MdrbOulvCgYd2HJeKU7e/zPK3dy+LRi5sdZzibdLHCYjHi3sYMNta0xTYpHmjOliNkVhTyVwHBVvztMdcbcKRTF2BMoCAY47+jpPLFhd1R7YNQ0d6Z0mMrjZI93JT1BMlpeb+Pa02cPef6Y6lKWffUkJuXn8JnfvsZzW0ZeHZfurHFPeVFu2nYB3FK3j3U1rVy2cGbaA2S8LHCMoLG9h/f2ZrbY2Xj16LpdiMDH58c3ESginHvUNF7b3hRXMh7Aqh1N1Lf1xD0ZedFx1bT3hFgxSq9HVVO2899gVWX5dPeF074iCJwPAw+/UcPnTvhgbyPSrIpCln31JA6bWsyX71nFfa+/P+y1TuBI34oqT0VRkKaOXvrjXEYci/tX7iTo93HhscPXacs2FjiGoap89Q+r+cxv/zFut+TMFFVl+bpdLJo1ecQibqM5d940+sPKc1viKzzw+IY68nJ8nHl4fKUdPnzwZKpKR99Wdm9HL9194fQEjgyurPrl81sJBnx8ZZjeRqSKolzu/fIJnHJoJd9+aAO/eHbrkL2kve29A3kV6VRRlEtYoTnODyXR6u7r5+E3ajln3tSU1JRKFQscwxARvvuJeezr6uNz//OPrNlKcjzYUtfGtvr2uIepPMdUlTC9JC+u4SpnmGo3Zx4+Je4Ja2db2Sr+trVhxJVC3lLcqnTMcWQol2N7QzuPvFHL5z58EFOKo1usUJgb4HdXLuSi46r4+bNv8+8PbzygXIqqsrcjU0NV6ckeX7F5D61dfXw6joKGmWSBYwRHVZVw59WLqGvt5orfvU5rZ2xr0M3Qlq9z9vQ+b4S6UNHw+YRzjpzKS283xLyd5uvvNtHY3sPHjk4seF14XBVhhb+s3TXsNancwGmwam8nwDT3OH71/Da3t3HI6BdHyPH7+Nml87luySHc+/r7XPuHNQNzRq1dffT1a0aGqrwy7qleWXX/yvepLsvnpBGqQmcjCxyjWDhrMrddcTzv1Ldz9V2vZ6zU8nihqjy6bhenzKkY2NM7EefOm0ZPKBxzqfPHN+wiP8fPGYcnVlXgkMoiFswsZdmammEnpGsHehypDxyT8gMUBv1p7XG809DOI2trueKEg6gsjr13ICJ8a+nh/J/z5/Hcm3v47P+8RnNH7/4cjjgeM1FesErlToA7mzr5+7a9XHr8zJTt1JcqFjiicNphldx8+QLW7mzhmt+vortv9FU0Zmhr3m+mtqUr4WEqz+KDJ1NakBNTMmCoP8xTG3dz1hFTkpJsdfHx1by5u23Yir01zV2U5OcwKcoEw0SISNrLq//yua1x9TYGu/KkWfz6M8excdc+Lrn1FdbubAXSmzXuSUe9qgdW7UQELo3Y5W+ssMARpaVHTecnl8zn79v2cv29b9A3jkpXp9PytbvIDfg4Z15seRPDCfidUiHPvVkfdS2kf7zbRGN7Lx9PUmmHTxwznRy/DDtJnq6luJ6q0vy0DVVtq29n+bpdfP7EWUl5gz/v6On8/guLqW/r4VsPrgMyEzhK83Pw+yRlcxz9YeWB1TWcdmglM9L4u5EsFjhicPHx1Xz/k/N4ZvMe/u2BdXFX/EyV/rDy9p42Hli1k/98ZAOf+OXLfOnuVTRnYGnmUEL9YR7fUMeZh0+JurxHNJbOm0Zbd4hXt++N6vrH1tdREPSzZG5yNsopLQhy1uFT+cva2iH3wkjXUlxPOnscv3x+K7kBP9ecNvpKqmh9eHY5D1570sAkeyaGqnw+YXJhMGWLYl7a2kBdazefHiOZ4oNlf1GULPP5E2fR1h3iJ0+/RWFugB9ccFRGknZUlV2t3azb2cK6mhbW7WxhQ00rHe7EYlFugHkzJvHS1gYuvvUV7r56MTMnp35Vz0he2+580k/WMJXnlEMrKAj6eXrTbk4/bOQ5C2eYqo6zj5hKXk7ytnC96Lgqntq0m79tbeSMiOW9qkpNcxenzElfheaq0gJau/po7wnFnNgYC6+3cc1ps5PeK5g7rZhHvnYyq95rSspcWDzKC4M0pqjHcf/rOykvDMZcsSBbWOCIw3VLDqGtO8StL75DcV4O3/5obHWO4tHS2cu6mlbW7XQ2oV+7s3Xg01DQ7+OI6cVcfHw186tLmT+zhNkVRfh8wuvvNvGlu1dy0S2vcOdViziqKnMlDZavq6UoN3DAG2sy5OX4WTK3khWb9vD/ffKoEbeffXX7Xpo7+5JegXTJ3CmUFeSwbE3NAT9fc2cfnb39aZkY9wwsyW3uYu606KsOx+rm57aSn+PnmlOT19uINK0kj48fk9wPGbGoSFH2eGN7D89u2cPVJ88iGBibgz4WOOIgItywdC7tPX1u8AjwtTPmJPU5+vrDPLGhjuffrGfdzhZ27O10n9tZyXPaYRUsmFnK/OpSDp9eTG5g6E/Piw+ezLKvnsRVd67kU7e9ym8+d/yon8pToSfUz5Mbd3POkcn9pO85d940ntiwmzfeb2bhrMnDXvf4+joKg/6kvwbBgI/z58/g3pU7ae3qG9gQqrY59ftwDDaQBNjSmbLAsXVPG4+u38VXTjskIwl66VBeFOT99zuT/rgPrakhFNYxU9BwKBY44iQifP/8o2h3h62KcgNcedKshB+3oyfE/St38ruX36W2pYupk3I5dmYZn1r0IeZXl3BUdUnMq3MOnVrMQ9edxJV3vM4X71rJjy4+hkuOT+9KjhffaqCtO8QnothXPB5nHD6FHL/w9KbdwwaOvv4wT23azUdSFLwuOq6au199jyc21HH5YiehK505HJ50ZI/f/Pw2p7eRxLmNbFNemPwKuarK/St3cvxBZcyZkrreYKpZ4EiAzyf85NL5dPT2893lmyjMDcT9htzY3sPdr+zgnlffo7Wrj8WzJvP9T87jjLlTkrLGe+qkPB649kSu/cNqvvnAOna3dvG1M+akbX5m+bpdlBXkcMqcipQ8/qS8HE6eU8FTm3bz7+cdMeTP9co7e2np7EvZ8Mcx1SXMmVLEQ2tqIgKH2+MoTd/80pTiXHL8Qm1LavYef3tPG4+t38W1px+SsfmHdCgvCtLR25/UuaLV7zXzTkMHP744saXLmTY2B9iySI7fxy8vP5aT55TzrQfX8dTGupju/25jB//+8AZO+tHz/OqFbZww2xla+vO1J3LWEVOTmhhUnJfDnVct5sJjq/jpirf5j0c2DrkKKNk6ekI8u2UP5x09nRx/6n7lzp03jZ1NXWypaxvy/OPrd1GcG+DUw1ITvLxtZVfuaB4ojlnb0kVxbiCt+0n4fML0ktStrLr5ua0U5Pj5cormNrLF0e584Od/9w/2xLD51EjuX7mTwqB/TOzyNxILHEmQl+Pn9isWsmBmKdff+wYvvT16FvMb7zfz1T+s5syf/ZUHV9dw8XHVPPevp3PbFQs5/qCylLU1GPDx35fN56tLDuFP/3ifa/+wOqqy4Il4dsseuvvCSV9NNdjZR0xFhCFrV/WGnKS/j8ybOux8UDJcsKAKEXj4DSeno6a5k6qy/LSvvHNyOZI/Pv/2njYe31DHlSfNGte9DXASf3/9meN4c3cbH//ly6zc0ZTQ47V19/HYemeXv1Rv6JVqqdw69g4RqReREfcFF5FFIhISkUsijvWLyFr3tjzi+MEi8g8R2SYi94tI1vzmFuYGuPOqxcyZUsw1v1815C9ZOKw8/+YeLrvtVS78zSv8fVsj1y05hJdvOIP/d9HRzI5iR7RkcCb3D+f7n5zHc2/Wc/lvX0tphuyj63YxvSSPRSNMWidDZXEuiw6azIohAsfftzWyrzuUtKS/4cwodeoOPbSmdmApbjrnNzypyuX4xXNbKQwGxn1vw/OxY6bzyNdOpig3wOW3v8bdr+yIe6+Tx9bX0dXXz2VjeFLck8oex13A0pEuEBE/cCOwYtCpLlVd4N7Ojzh+I/BzVZ0DNANfTGJ7E1ZSkMPvv7iYGSX5fOHOlWysdUom9IbCPLi6hqW/eIkv3LWKmqZO/uvjR/LKd87i3849POpqosn2+RNncctnj2dL3T4uvuWVlOw90tLZy4tvN/DxY6anpR7POfOm8ubuNnY0HvizPLa+juK8QFryKS46tpr3mzpZ9V4ztWna+W+wqtJ86tt6os6mj8Zbu9t4YkMdV500a0yVAE/UYVOdnJIlcyv57vJN/K8/r4url37/yp0cNrWIY2eWpqCV6ZWywKGqLwGj9e2uB5YBo26oIE5f/0zgQffQ3cAFibQxFSqKcvnDlz7MpPwcPn/H69z07Nuc9uMX+OYD6/CJ8PNPzefFb53BF085OKXJWdFaetQ0/vTlD9PS1cdFv3mFdTtbkvr4T23cTV+/xrWveDy8LWUja1f1hPpZsXk3586blpZ180uPmkZ+jp87//4ubT2hjPU4VKGuNXm9jl889zaFwQBfOvXgpD3mWFGSn8PtVyzkXz9yGA+vreXiW15hZ1P0Q4Fv7W5j7c6WMbXL30gyNschIlXAhcAtQ5zOE5FVIvKaiHjBoRxoUVWvPG0NMOyWWSJyjfsYqxoaYqucmqgZpfn84UsfxifCTc9uZXZlIXd/YTFP/vOpXHhsdUoniONx/EHOhHx+0M+nb3+NF96Mb2OkoSxft4tZ5QUDE42pNnNyAfNmTDogcLy8tZG27lDaJiQLcwN89CgnrwRIa50qT3WSl+S+uXsfT2zYzdUnz6K0YOL0NiL5fMI/nXUod1y5iJrmTj7xq5f529bo3lvuX7mTHL9w0XFjr6DhUDL5DnYTcIOqDtWXPkhVFwKfAW4SkZjXrqnq7aq6UFUXVlamP+Ht4IpCln/9ZJ7851P505dP4PTDKrP6k8YhlUU8dN1JzK4s5Ev3rOL+lcNv5xmt+n3dvLp9L+fPn5HWn/3cedNY837LwEqYx9fXUZKfuqXAQ7k4Yll2JoaqvMJ5NUma5/jFs1spzg3wxVMmXm9jsDMOn8Lyr5/CtEl5XHnH6/zmr9tGnPfoCfXz0Bs1nHPktHGzoCCTYyULgfvcN5QK4DwRCanqI6paC6Cq20Xkr8CxOENapSIScHsd1cDIe3Zm2IzS/DFV+XJKcR73f+VErvvjGm5YtoFdLd184+xDP/CmHw4r7b0h2rtDtHWHaO/po23g6xBt3X20d4fYuGsfqnB+ipL+hrP0qGn89zNvs2LzHi49vppnNqd+KfBgJ8wuZ3pJHnWt3RkZqppe6sybJWPb4y11+3hy427+6cw5E7a3MdisikIeuu4kbli2gR8/9Rbrd7by08vmDzn8/MzmPbR09o3pTPHBMhY4VHXgo4uI3AU8pqqPiEgZ0KmqPSJSAZwM/FhVVUReAC4B7gOuBP6SgaaPa0Xudp7fXraBXzy3lb++VY/PJxFBwrmNRgSKggHOnTc17Rmyh04p4uCKQp7euJtpk/Jo60nfMJXH7xM+d8JB3Pv6+5QWpH4fjsFyA36mFOcmZahqf29jYqykilZBMMDNn17A/OoS/t+Tb/LJX73MbVcsZM6UA1dH3r9yJ1Wl+Wnt8aZaygKHiNwLLAEqRKQG+C6QA6Cqt45w1yOA20QkjDOU9iNV3eyeuwGnl/ID4A3gdylq/oSW4/fx00uPYXZlIc9s3kNRboBpk/Ioyg1QnJdDUV6ASXmBA74vynWP5TnHCnL8GdvVTEQ4Z95Ufve3dwn4hbKCHE7MwNac1y05hGtPPyRjQ5TJWJK7aVcrT23azT+ddSglGQiA2U5E+NKpszlyxiSu/9MbXPDrv/Ozy+YPLNLY2dTJy9sa+eezDh1zu/yNJGWBQ1Uvj+HaqyK+fgU4epjrtgOLE26cGZWI8LUz5iS9eGO6LJ03jdte3M5f32rg8sUzM7IgQUTwZ/C9oqo0nw3ukvB43fzcVorzbG5jNCcdUsGj15/CV/+wmq/8fjVfP2MO//KRw3hgdQ0Aly4cP8NUYLWqzDg1v7qUqZNy2bOvh48dnbnS3JlUXVbAY+vrOOw/nyQv4CM3x09ejo/cgJ/cgI+8nJH/BXh60x6+cfahA9V+zfBmlOZz/1dO5HvLN/GrF7axvraVbXvaOGVORUZW1qWSBQ4zLvl8wgULqnhsfR0nzE5txnq2+vyJB5Gf46ezL0RPX5ieUL/7b5juvv6Bf9u6Qwd8H/nv1Em5XH2y9TailZfj50cXH8Mx1aV8d/lG+vqV//jYkZluVtJJvOnzY8nChQt11apVmW6GSbP+sNIbCpMfTF1tqvFMVVFlXI3Np9Mb7zfzzOY9fOPsw8bshk0istpNjTiA9TjMuOX3iQWNBIgIWZx6lPWO/VAZx34odQVLM2lshkFjjDEZY4HDGGNMTCxwGGOMiYkFDmOMMTGxwGGMMSYmFjiMMcbExAKHMcaYmFjgMMYYE5MJkTkuIg3Ae3HevQJoTGJzks3alxhrX2KsfYnJ9vYdpKof2AlvQgSORIjIqqFS7rOFtS8x1r7EWPsSk+3tG44NVRljjImJBQ5jjDExscAxutsz3YBRWPsSY+1LjLUvMdneviHZHIcxxpiYWI/DGGNMTCxwGGOMiYkFDpeILBWRt0Rkm4h8e4jzuSJyv3v+HyIyK41tmykiL4jIZhHZJCL/PMQ1S0SkVUTWurf/na72uc+/Q0Q2uM/9ge0WxXGz+/qtF5Hj0ti2uRGvy1oR2Sci3xh0TVpfPxG5Q0TqRWRjxLHJIvKMiGx1/x1yFyARudK9ZquIXJnG9v1ERN50//8eFpHSYe474u9CCtv3PRGpjfg/PG+Y+474t57C9t0f0bYdIrJ2mPum/PVLmLM95MS+AX7gHWA2EATWAUcOuuY64Fb3608D96exfdOB49yvi4G3h2jfEuCxDL6GO4CKEc6fBzwJCHAC8I8M/l/vxklsytjrB5wGHAdsjDj2Y+Db7tffBm4c4n6Tge3uv2Xu12Vpat85QMD9+sah2hfN70IK2/c94JtR/P+P+LeeqvYNOv8z4H9n6vVL9GY9DsdiYJuqblfVXuA+4JODrvkkcLf79YPAWSLp2VhTVetUdY37dRuwBahKx3Mn0SeBe9TxGlAqItMz0I6zgHdUNd5KAkmhqi8BTYMOR/6O3Q1cMMRdzwWeUdUmVW0GngGWpqN9qrpCVUPut68B1cl+3mgN8/pFI5q/9YSN1D73feMy4N5kP2+6WOBwVAE7I76v4YNvzAPXuH88rUB5WloXwR0iOxb4xxCnTxSRdSLypIjMS2vDQIEVIrJaRK4Z4nw0r3E6fJrh/2Az+foBTFXVOvfr3cDUIa7JltfxCzg9yKGM9ruQSl93h9LuGGaoLxtev1OBPaq6dZjzmXz9omKBYwwRkSJgGfANVd036PQanOGX+cAvgUfS3LxTVPU44KPA10TktDQ//6hEJAicDzwwxOlMv34HUGfMIivXyovIfwAh4I/DXJKp34VbgEOABUAdznBQNrqckXsbWf+3ZIHDUQvMjPi+2j025DUiEgBKgL1paZ3znDk4QeOPqvrQ4POquk9V292vnwByRKQiXe1T1Vr333rgYZwhgUjRvMap9lFgjaruGXwi06+fa483fOf+Wz/ENRl9HUXkKuDjwGfd4PYBUfwupISq7lHVflUNA78d5nkz/foFgIuA+4e7JlOvXywscDhWAoeKyMHup9JPA8sHXbMc8FawXAI8P9wfTrK5Y6K/A7ao6n8Pc800b85FRBbj/N+mJbCJSKGIFHtf40yibhx02XLg8+7qqhOA1ohhmXQZ9pNeJl+/S1A2hwAAAyJJREFUCJG/Y1cCfxnimqeBc0SkzB2KOcc9lnIishT4FnC+qnYOc000vwupal/knNmFwzxvNH/rqXQ28Kaq1gx1MpOvX0wyPTufLTecVT9v46y4+A/32Pdx/kgA8nCGOLYBrwOz09i2U3CGLdYDa93becC1wLXuNV8HNuGsEnkNOCmN7ZvtPu86tw3e6xfZPgF+7b6+G4CFaf7/LcQJBCURxzL2+uEEsDqgD2ec/Ys4c2bPAVuBZ4HJ7rULgf+JuO8X3N/DbcDVaWzfNpz5Ae930FtlOAN4YqTfhTS17/fu79Z6nGAwfXD73O8/8Leejva5x+/yfucirk3765fozUqOGGOMiYkNVRljjImJBQ5jjDExscBhjDEmJhY4jDHGxMQChzHGmJhY4DAmC7nVeh/LdDuMGYoFDmOMMTGxwGFMAkTkcyLyurt3wm0i4heRdhH5uTh7pzwnIpXutQtE5LWI/SzK3ONzRORZt8DiGhE5xH34IhF50N0D448Rme0/EmdvlvUi8tMM/ehmArPAYUycROQI4FPAyaq6AOgHPouTpb5KVecBLwLfde9yD3CDqh6Dk+HsHf8j8Gt1CiyehJNxDE4V5G8AR+JkFJ8sIuU45TTmuY/zg9T+lMZ8kAUOY+J3FnA8sNLdze0snDf4MPuL2P0BOEVESoBSVX3RPX43cJpbl6hKVR8GUNVu3V8H6nVVrVGnaN9aYBZOOf9u4HcichEwZM0oY1LJAocx8RPgblVd4N7mqur3hrgu3ro+PRFf9+PsvhfCqZb6IE6V2qfifGxj4maBw5j4PQdcIiJTYGDP8INw/q4uca/5DPCyqrYCzSJyqnv8CuBFdXZ0rBGRC9zHyBWRguGe0N2TpUSd0u//AsxPxQ9mzEgCmW6AMWOVqm4Wkf/E2a3Nh1MJ9WtAB7DYPVePMw8CTqn0W93AsB242j1+BXCbiHzffYxLR3jaYuAvIpKH0+P51yT/WMaMyqrjGpNkItKuqkWZbocxqWJDVcYYY2JiPQ5jjDExsR6HMcaYmFjgMMYYExMLHMYYY2JigcMYY0xMLHAYY4yJyf8PmpOWYrvIeRQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z05qkjuMLDOw"
      },
      "source": [
        "When $\\beta_1 = 0$ , adam optimizer becomes RMSprop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cddq9ugpJpUc",
        "outputId": "85782614-32d6-4b25-b65f-c0a54dab4c0c",
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "parameters = model(X_train, y_train, n_h = 4, learning_rate= 0.1, mini_batch_size=64, beta=0.9,\n",
        "          beta1=0, beta2=0.9, optimizer=\"adam\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cost after epoch 0: 2.163012\n",
            "Cost after epoch 1: 1.811483\n",
            "Cost after epoch 2: 1.807051\n",
            "Cost after epoch 3: 1.820496\n",
            "Cost after epoch 4: 1.822814\n",
            "Cost after epoch 5: 1.816322\n",
            "Cost after epoch 6: 1.851430\n",
            "Cost after epoch 7: 2.076641\n",
            "Cost after epoch 8: 1.779753\n",
            "Cost after epoch 9: 1.810285\n",
            "Cost after epoch 10: 1.772520\n",
            "Cost after epoch 11: 1.797026\n",
            "Cost after epoch 12: 1.921602\n",
            "Cost after epoch 13: 2.113433\n",
            "Cost after epoch 14: 1.994257\n",
            "Cost after epoch 15: 1.796781\n",
            "Cost after epoch 16: 1.840291\n",
            "Cost after epoch 17: 1.847565\n",
            "Cost after epoch 18: 1.849388\n",
            "Cost after epoch 19: 1.825749\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXijd3Xo8e+RbUm25V22Z7dnxtnJPpksE3JZekOS0gbShAZCNpaQsrdwW0pp4aalpS3Qy1aSlIQkNIQ9ECBAUiAhy2SZSWaSzGTxMvbsXmSP913n/qFXtsYj27KtV69sn8/z6BlZ7yvpjCzr6Ledn6gqxhhjzFQ+rwMwxhiTnSxBGGOMScoShDHGmKQsQRhjjEnKEoQxxpikLEEYY4xJyhKEWfZE5PUi8qrXcRiTbSxBGE+JSLOI/JGXMajqY6p6gpcxxInIG0Rkv0fP/S4RaRGRfhH5qYiUz3Du7SLyqohEReSGDIZpMsgShFnyRCTH6xgAJCYr/+ZE5BTgNuBaoBoYAP5zhrvsBD4IPOd+dMYrWflmNUZEfCLyKRFpFJGIiPwg8RutiPxQRA6LSLeI/MH5gIsfu0tEvikiD4pIP/BGp6XySRF5wbnP90Uk6Jx/1Lf2mc51jv+1iBwSkYMi8j4RURGpm+b/8YiIfF5EniD2obtBRG4UkZdFpFdEmkTkA865hcCvgFUi0udcVs32WqTJNcDPVfUPqtoH/D1whYgUJTtZVb+hqr8FhtIch8kiliBMtvoI8DbgfwGrgC7gGwnHfwUcB1QR+xZ775T7vwv4PFAEPO7c9g7gEmA9cBpwwwzPn/RcEbkE+Cvgj4A64A0p/F+uBW5yYmkB2oC3AsXAjcB/iMhZqtoPXAocVNWQczmYwmsxQUTWiciRGS7vmibGU4i1CgBQ1UZgBDg+hf+fWaJyvQ7AmGncDHxYVfcDiMjngL0icq2qjqnqnfETnWNdIlKiqt3OzT9T1Sec60MiAvBV5wMXEfk5cMYMzz/due8Avq2quxKe+5pZ/i93xc93/DLh+qMi8hDweqbvrpnxtUg8UVX3AqWzxJNMCOiecls3saRmlilrQZhsVQPcH//mC7wMjAPVIpIjIl9wulx6gGbnPuGE++9L8piHE64PEPtQnM50566a8tjJnmeqo84RkUtF5CkR6XT+b5dxdOxTTftapPDcqeoj1qJJVAz0pvE5zCJjCcJkq33ApapamnAJquoBYt1HlxPr5ikBap37SML93SpTfAhYk/Dz2hTuMxGLiASAHwNfBKpVtRR4kMnYk8U902txFKeLqW+Gy3StnV3A6QmPswEIAK+l8P8zS5QlCJMN8kQkmHDJBW4FPi8iNQAiUikilzvnFwHDQAQoAP45g7H+ALhRRE4SkQJig7lz4Sf2wdsOjInIpcDFCcdbgQoRKUm4babX4iiqujdh/CLZZepYTdy9wJ9IbE1IIXAL8BNVTdqCEBG/M3AvTP7+7PNkibFfqMkGDwKDCZfPAV8BHgAeEpFe4CngXOf8e4gN9h4AdjvHMkJVfwV8Ffg90JDw3MMp3r8X+CixRNNFrDX0QMLxV4D7gCanS2kVM78WaeGMkdxMLFG0EUvCH4wfF5FficinE+7yELHf1QXA7c71i9IZk/Ge2IZBxsyfiJwEvAQEpg4YG7PYWQvCmDkSkbeLSEBEyoB/JbZ+wJKDWXIsQRgzdx8g1g3TSGw20V94G44x7rAuJmOMMUlZC8IYY0xSS2oldTgc1traWq/DMMaYRWP79u0dqlqZ7NiSShC1tbVs27bN6zCMMWbREJGW6Y5ZF5MxxpikLEEYY4xJyhKEMcaYpCxBGGOMScoShDHGmKQsQRhjjEnKEoQxxpikln2CUFW+9tt6Hn2t3etQjDEmqyz7BCEi3P5YE79/pc3rUIwxJqss+wQBUBkK0N6X0n4vxhizbFiCAMKhAB29liCMMSaRJQggXOSnw1oQxhhzFNcShIisFZHfi8huEdklIh9Lcs6JIrJVRIZF5JNTjjWLyIsiskNEXK3AFw4F6OgbcfMpjDFm0XGzmusY8AlVfU5EioDtIvKwqu5OOKeT2Abub5vmMd6oqh0uxgjEEkT34CgjY1H8udaoMsYYcLEFoaqHVPU553ov8DKweso5bar6LDDqVhypCIcCAET6rZvJGGPiMvJ1WURqgTOBp+dwNwUeEpHtInLTDI99k4hsE5Ft7e3zW8sQDvkB6Oi1biZjjIlzPUGISAj4MfBxVe2Zw10vVNWzgEuBD4nIRclOUtXbVXWTqm6qrEy6KdKswkWxFoQNVBtjzCRXE4SI5BFLDveq6k/mcl9VPeD82wbcD2xOf4QxlU4Xk62FMMaYSW7OYhLgDuBlVf3yHO9b6AxsIyKFwMXAS+mPMiY+BmEtCGOMmeTmLKYtwLXAiyKyw7nt08A6AFW9VURWANuAYiAqIh8HTgbCwP2xHEMu8F1V/bVbgeb7cyj059gYhDHGJHAtQajq44DMcs5hYE2SQz3A6W7ENZ1wUcBaEMYYk8Am/Ttii+UsQRhjTJwlCEc4ZOU2jDEmkSUIh5XbMMaYo1mCcIRDAboGRhgbj3odijHGZAVLEI5wUQBV6Oy3VoQxxoAliAmVTrmNNtsXwhhjAEsQE2yxnDHGHM0ShKNyoh6TdTEZYwxYgphgLQhjjDmaJQhHYSCX/Lwc25vaGGMcliAS2N7UxhgzyRJEAlssZ8zcNXf081prr9dhGBdYgkhg9ZiMmbu/++mLfOA7270Ow7jAEkQCSxDGzN0rh3rZ09HPvs4Br0MxaWYJIkFlyE9n/wjjUfU6FGMWhUjfMBGn+sCTjR0eR2PSzRJEgnBRgKiV2zAmZQ1tfRPXH2+IeBiJcYObW46uFZHfi8huEdklIh9Lcs6JIrJVRIZF5JNTjl0iIq+KSIOIfMqtOBPZWghj5qbeSRDnbSjnyYYOotb6XlLcbEGMAZ9Q1ZOB84APicjJU87pBD4KfDHxRhHJAb4BXEpsC9J3Jrlv2lmCMGZu6lt7KfTncNXZa4n0j/DKYZvNtJS4liBU9ZCqPudc7wVeBlZPOadNVZ8FRqfcfTPQoKpNqjoCfA+43K1Y48JOwT5LEMakpr6tj7rqIi48LgzAEw02DrGUZGQMQkRqgTOBp1O8y2pgX8LP+5mSXBIe+yYR2SYi29rb2xcSJuF4PaZeG4MwJhX1bX0cVxWiujjIcVUhHrcEsaS4niBEJAT8GPi4qvak+/FV9XZV3aSqmyorKxf0WEWBXPy5PmtBGJOCIwMjtPcOc1xVCIAtdWGe2dPJ8Ni4x5GZdHE1QYhIHrHkcK+q/mQOdz0ArE34eY1zm6tEhMpQgHZLEMbMKj5AfVz1ZIIYHB3n+b1HvAzLpJGbs5gEuAN4WVW/PMe7PwscJyLrRcQPXA08kO4YkwmH/FZuw5gU1Lc6CaKqCIBzN5ST4xMbh1hC3GxBbAGuBd4kIjucy2UicrOI3AwgIitEZD/wV8BnRGS/iBSr6hjwYeA3xAa3f6Cqu1yMdUI4FLCKrsakoL6tl/y8HFaX5gNQHMzj9DUlNg6xhOS69cCq+jggs5xzmFj3UbJjDwIPuhDajMKhAC8e6M700xqz6DS09VFXFcLnm/wzv7AuzNd/30DP0CjFwTwPozPpYCuppwgX+Yn0j9iCH2NmUd/aNzFAHbelLkxU4alGW1W9FFiCmCIcCjAeVboGbBzCmOl0D45yuGeI46qLjrr9zHVl5Ofl2DjEEmEJYorJ1dSWIIyZTrwG09QWhD/Xx7kbym0cYomwBDGFldswZnYNbbGSGvEprokurAvT2N7Poe7BTIdl0swSxBSVRVZuw5jZ1Lf2Ecj1saas4JhjW+riZTdsHGKxswQxRWUoCEC7TXVdNG59tJGfPu/6OkqT4DVnBlOO79iJiidUFxEO+W0cYgmwBDFFcX4u/hyfjUEsIt96bA8/2LZv9hNN2jS09h4z/hDn8wkXbAzzeEMHqjYbcDGzBDGFiFAR8lsX0yLRNzxGR98wh3uGvA5l2egdGuVg97EzmBJdWBemvXd4ohyHWZwsQSRhe1MvHi2RfgBauy1BZEpje+w1r5umBQGwxSn//Xi9dTMtZpYgkghbC2LRaIkMANA/Mk7v0NRtRYwb6ltjM5iOn6EFsbo0n/XhQhuHWOQsQSQRq8dkYxCLQbPTggA4bK2IjKhv68Of62NtWf6M522pq+Cppgij49EMRWbSzRJEEuGiAJH+YRtgWwRaOgYmrts4RGbUt/ayIVxIbs7MHx8X1oXpHxln5z4r/71YWYJIIhwKMDqudA9al0W2a470U10cW9xoLYjMqG/rm3GAOu78DWFEsFXVi5gliCRsb+rFoyUywOb1FYAliEwYGBljf9fgtFNcE5UU5HHa6hIbh1jELEEkUemU22i3cYisNjgyzuGeIY6vClFWkGddTBkQr8F0fJISG8lsqQvz/N4j9A2PuRmWcYkliCTCRVaPaTHY2xkbf6gJF1JdHKTVEoTr4rvI1VXN3sUEsXGIsajyzB4ru7EYubnl6FoR+b2I7BaRXSLysSTniIh8VUQaROQFETkr4dh4wk50GdluNM4K9i0O8RlMtRUFrCwJcsi6mFxX39ZHXo5QU3FsDaZkzqopI5Dr4/F6SxCLkWs7ygFjwCdU9TkRKQK2i8jDqro74ZxLgeOcy7nAN51/AQZV9QwX45tWaX4eOT6xBJHl4ovkasoLWVEStJ0AM6ChrZf14ULyZpnBFBfMy2Hz+nIbh1ikXGtBqOohVX3Oud5LbG/p1VNOuxy4R2OeAkpFZKVbMaXK5xMqCv22FiLLNUcGKCvIo6Qgj+riIB19I4yM2Zx7N6U6gynRlrowr7b20tZrLbzFJiNjECJSC5wJPD3l0GogscrafiaTSFBEtonIUyLythke+ybnvG3t7e1pi9nKbWS/lkg/NRWFAKwsiVXhtQ8h9wyOjLO3cyClGUyJLnTKfz9p5b8XHdcThIiEgB8DH1fVnjnctUZVNwHvAv6fiGxMdpKq3q6qm1R1U2VlZRoijgkXWYLIds0dA9Q6feHVxbEEYVNd3dPY3ocqHJfiAHXcySuLKS3Is/UQi5CrCUJE8oglh3tV9SdJTjkArE34eY1zG6oa/7cJeIRYCyRjwiG/7QmRxYbHxjnYPTjRgljhtCBsqqt7JrYZTXGKa5zPJ2zZGOYJK/+96Lg5i0mAO4CXVfXL05z2AHCdM5vpPKBbVQ+JSJmIBJzHCQNbgN3TPIYrKkMBOvpG7A2dpfZ1DqIKteFYC2KFtSBcV9/WS65PqHWS8lxsqQtzqHuIpo7+2U82WcPNWUxbgGuBF0Vkh3Pbp4F1AKp6K/AgcBnQAAwANzrnnQTcJiJRYknsC1NmP7kuHAowMh6lZ2iMkvy8TD61ScHEDCbnw6okP49gns8ShItea+2jNlyIP3fu3ysvnNiGtIONlXNrgRjvuJYgVPVx4Nj9CI8+R4EPJbn9SeBUl0JLSThhb2pLENmn2SnzHf82KyKsKA5aF5OLGtr6OHHF3MYf4tZVFLC2PJ/H6zu47vza9AZmXGMrqacxsVjOxiGyUkukn6JgLmUFk8nbVlO7Z2h0nJZI/5xnMCW6sC7M1qYIY1b+e9GwBDGNydXUthYiGzVHBqitKCQ21BWzssRaEG7Z09FPVKFujmsgEm2pC9M7NGYLGhcRSxDTqLR6TFkttgbi6HIP1SVBWrttHw831M+xSF8yF2ycHIcwi4MliGmUFfjxiSWIbDQ6HmV/1+Axs2lWFAcZGY/S2W+tvnSrb+3FJ7A+PPcZTHHlhX5OWVVs6yEWEUsQ08jxCeWFtlguGx3oGmQ8qse0ICamulo3U9rVt/ZRW1FIIDdnQY9zYV2Y51qOMDgynqbIjJssQcwgtljOvo1mm4kqrlO+zU4slrOprmlX39ZL3QIGqOO21IUZGY/ybHNnGqIybrMEMYNKK7eRlVqcKa7HtCBsNbUrRsaiNEcGOH4BA9Rx59SW48/x2TjEImEJYgZWsC87NUf6KfDnTOz8F1cZCuATaLUWRFrt6ehnPKpzLrGRTL4/h7NrymwcYpGwBDGDcMhPR5/Nisk2LZEBaqZMcQXIzfFRWRSwFkSa1bf1AqSliwngwuPC7DrYY5MJFgFLEDMIhwIMjUbptwG1rNIc6Z+o4jrVimLbWS7d6lv78AlpK5GxJV7+u9FaEdnOEsQMbDV19hmPKvs6ByZqME1lq6nTr6Gtj3XlBQTzFjaDKe7U1SUUBXNtHGIRsAQxg7Atlss6B48MMjqu07cgSoI2iynNYjOYFj5AHZfjEy7YWGHjEIuAJYgZhEOTBftMdpicwZS8BbGiJEjP0BgDI2OZDGvJGh2PsqejPy0D1IkurAuzr3OQvc7v02QnSxAziM+Sabd6TFljcg3E9GMQYGsh0qUl0s/ouC6oSF8y8XEIa0VkN0sQMygv9CNiYxDZpCXSTyDXR3VRMOlxW02dXvWt8RpM6etigljJjlUlQRuHyHKWIGaQm+OjrMBvXUxZpDkyQE1FAT5f8q1GbDV1etW39SFpnMEUJyJsqQvzRGMH0ahNI89WliBmEV8LYbJDrIrr9AXjbDV1er3W2suasnzy/emZwZTowuPCHBkYZfehnrQ/tkkPN/ekXisivxeR3SKyS0Q+luQcEZGvikiDiLwgImclHLteROqdy/VuxTmbcChAu3UxZYVoVNnbOUBNefLxB4ACfy5FwVxbTZ0mDW19HJfGGUyJ4uW/bRwie7nZghgDPqGqJwPnAR8SkZOnnHMpcJxzuQn4JoCIlAOfBc4FNgOfFZEyF2OdVqzchg1SZ4O23mGGRqPUzFJy2rYeTY+x8ShN7QvbRW4mlUUBTlxRZOMQWcy1BKGqh1T1Oed6L/AysHrKaZcD92jMU0CpiKwE3gI8rKqdqtoFPAxc4lasM7F6TNljYgbTNGsg4mwtRHrs7RxgZDzKcWkeoE60pS7MM3s6GRq1agXZKCNjECJSC5wJPD3l0GpgX8LP+53bprs92WPfJCLbRGRbe3t7ukKeEC7yMzAybvPqs0DLRIKwFkQmxHeRc6sFAbClroLhsSjPtXS59hxm/lxPECISAn4MfFxV0z4apaq3q+omVd1UWVmZ7odPKLdh3Uxea44MkJcjrCxJPsU1bkVJkPbeYcbGoxmKbGmqb40V6dvoYoLYvL6CXJ/YOESWcjVBiEgeseRwr6r+JMkpB4C1CT+vcW6b7vaMm1wsZ91MXmuJ9LO2rIDcnJnftitKgkTVfmcLVd/Wx+rSfEKBXNeeIxTI5cx1pTzZGHHtOcz8uTmLSYA7gJdV9cvTnPYAcJ0zm+k8oFtVDwG/AS4WkTJncPpi57aMm2hB2IeN55o7Bo7ZJCgZW02dHvWtfWkvsZHMKatKqG/ttbL6Wci9rwawBbgWeFFEdji3fRpYB6CqtwIPApcBDcAAcKNzrFNE/hF41rnfLarqyR6F4SKrx5QNVJWWSD+b15fPem61kyCsquv8jUeVxvY+ttRVuP5cdVUh+kfGOdQ9xKrSfNefz6TOtQShqo8DyZe7Tp6jwIemOXYncKcLoc1JRaGNQWSDjr4R+kfGZ53BBLaaOh32dQ4wPBZ1bQ1EovhGRA1tfZYgsoytpJ6FP9dHaUGetSA8Fp/BNNsaCIDyAj/+HB+HrAUxb/EZTHUZ6GJKTBAmu1iCSIGthfBes1MWerYprgA+n1BVHLDV1AsQ32bUzSmucRWFfkoL8mhotwSRbVJKECJyVSq3LVVWj8l7LZF+cnzC6hS7IGwtxMI0tPaxsiRIUTDP9ecSEeoqQ9aCyEKptiD+NsXbliQrt+G95sgAq0vz8eem9pa11dQLU9/WN9H1kwl1VSEaLUFknRn/2kTkUhH5GrDaKaoXv9xFrNbSshAOBWxPCI/FqrjOPkAdF29B2NTJuYtG1dUifcnUVYWI9I/Q1W9fxLLJbF/HDgLbgCFge8LlAWL1kpaFyqIAvcNjVi/GI6rKno7+lMYf4laUBBkajdIzuGy+x6TNgSODDI6Oc3wGBqjj4qu1bRwiu8w4zVVVdwI7ReS7qjoK4CxcW+sU0VsWEvemXlOW+rdYkx5HBkbpHRqbUwuiOmFnuZIC9/vRl5KJAeoMJoi6ysmZTOfUzr7WxWRGqmMQD4tIsVOG+zngv0TkP1yMK6tMrqa25q8XmlMs0pcoXq/pUPegKzEtZfFtRusqM9fFtLo0n/y8HBuozjKpJogSp9DeFcTKc58LvNm9sLLLZME+G4fwQkt8imt47i0IW009d6+19lFVFMhoy8vnEzZUFlqCyDKpJohcZ5+GdwC/cDGerBQusnpMXmqO9CPCnLr3JrqYuu13NlcNbb0Z7V6Kq6uyqa7ZJtUEcQuxYnmNqvqsiGwA6t0LK7tUFFo9Ji+1RAZYVZJPMC/1fZH9uT7CIT+He6yLaS5UlfoMz2CKq6sMceDIoO29kkVSShCq+kNVPU1V/8L5uUlV/8zd0LJHMC+HomCu7U3tkeY5TnGNqy62tRBzdbB7iIGRcc9aEABN7f0Zf26TXKorqdeIyP0i0uZcfiwia9wOLptU2mI5z7REBqiZwwB1XGwthCX1uYhvEuRJC8JqMmWdVLuYvk1s7cMq5/Jz57ZlIxwK2AY0HugeHKWzfySlKq5TVZcEOWyzmOYkPoMpEzWYpqqpKCTHJ5YgskiqCaJSVb+tqmPO5S4g/ft7ZrFwkdVj8sJeZwbTfFoQK4uDdA2M2gLHOahv6yUcClDmjLtlkj/XR01FgSWILJJqgoiIyLtFJMe5vBtYVnsEWrkNb0ysgZjDFNe4amctRJt1M6UsNkCd+dZDXF1lyFZTZ5FUE8R7iE1xPQwcAq4EbpjpDiJypzNe8dI0x8uccY0XROQZEXldwrFmEXlRRHaIyLYUY3RVOBSgZ2iM4TH7NppJ8X0g1pXPPUGsSFhNbWanqjRkaJvR6dRVhWju6Gd0POpZDGbSXKa5Xq+qlapaRSxh/N9Z7nMXcMkMxz8N7FDV04DrgK9MOf5GVT1DVTelGKOr4ovlIjZQnVHNkQGqiwMU+Oe++aGtpp6bwz1D9A6PeduCqAoxFtWJxZHGW6kmiNMSay85+0OfOdMdVPUPwEz7SJ8M/M459xWgVkSqU4wn4xLrMZnMiVVxnfv4A0x2Mdlq6tRMDFBXZ34GU5zNZMouqSYIn1OkDwCnJtNC97PeSax0ByKyGagB4lNnFXhIRLaLyE0zPYiI3CQi20RkW3t7+wJDmp6tpvZGc2RgXjOYAIoCuRT4c2w1dYri24x62YLY6BTta7RxiKyQ6of8l4CtIvJD5+ergM8v8Lm/AHxFRHYALwLPA/EO/gtV9YCIVBErFPiK0yI5hqreDtwOsGnTJteK/1dO1GOyLqZM6R8eo713eN4tCBFx1kJYF1MqGtp6KS/0U+G8171QGMhlVUlwYj2G8VZKCUJV73EGi9/k3HSFqu5eyBM7xf9uBBARAfYATc6xA86/bSJyP7AZSJogMiU+BmFrITKnZWKK6/xLrNvOcql7rTWzu8hNZ2OVzWTKFql2MaGqu1X1685lQckBQERKRSQ+2fp9wB9UtUdECkWkyDmnELgYSDoTKpPy/TmEArnWxZRBLfMo8z3ViuIgrTbNdVaqSn1rb0Y3CZpObPvRfqJR2w3QawsdR5iWiNwHvAEIi8h+4LNAHoCq3gqcBNwtIgrsAt7r3LUauD/WqCAX+K6q/tqtOOciHPJbuY0ManZaEOsW0IKoLgnS2jNENKr4fJKu0Jac9t5heobGPCmxMVVdVYjB0XEOdg/aBl0ecy1BqOo7Zzm+FTg+ye1NwOluxbUQtlgus1oi/VQU+ikOzn9fgpUlQcaiSkf/MFVFwTRGt7RkwwB1XOLucpYgvJVyF5NxEoR1MWXMfKu4JprYOMhmMs0oPihclyVdTGBTXbOBJYg5sHpMmdUSGVjQ+APYaupUvdbWR0l+3sRsPS9VhAKUFeTZVNcsYAliDsKhAF0Do1YGIAOGRsc51D007ymucStK4jvL2VTXmTS09nF8dQhn7M9ztrtcdrAEMQfxqa6d/TZQ7ba9nXPfhzqZcChAjk+sBTEDVeW1tl7qsmCAOs4SRHawBDEHE2shbKDadS0LKPOdKMcnVBUFbDX1DCL9IxwZGM2KAeq4jZUhugZGiViXrqcsQcxBZZHVY8qUyTUQC5/FUl0ctHpMM3gtvotcFgxQx9lAdXawBDEH8RaErYVwX3Okn5L8PEoLFr5xzcqSoFV0nUH8Q/h4D4v0TTWRIGyg2lOWIObAupgyp2UBRfqmqrbV1DOqb+2jKJhLVZH3M5jiVpXkk5+XYy0Ij1mCmIPCQC75eTnWxZQBzQso8z3VipIgfcNj9A6NpuXxlprdh3o4vrooa2YwAfh8wsaqQksQHrMEMUe2FsJ9I2NRDnQNpq0FEV8LYeMQxxoYGeOF/Uc4p7bc61COUVcZotEShKcsQcyRraZ23/6uAaK68BlMcZNrIez3NtW25i5Gx5XzN1Z4Hcox6qpCHOweon94zOtQli1LEHMUq8dkg9Ruik9xXegaiDhbTT29rU0Rcn3CObVls5+cYfGBaltR7R1LEHNkLQj3NTtTXNPfgrCZTFNtbYxwxtrSee357Tab6uo9SxBzVBny0zkwwpiV23BNS2SAUCCXisKFT3EFCOblUFqQZy2IKfqGx3jxQHdWdi9B7AtCrk8sQXjIEsQchYsCqELngHUzuSVexTWds2pWFAdtDGKKZ/d0Mh5Vzt+QnQkiL8dHbdhmMnnJEsQchW1vatelo4rrVLaa+lhbmyL4c3ycVZN94w9xdZW2/aiXXEsQInKniLSJSNLtQkWkTETuF5EXROQZEXldwrFLRORVEWkQkU+5FeN8TK6mtm+jbhgbj7Kvc2DB+0BMtaI4yCHbm/ooTzZ2cOa6UoJ5OV6HMq26qhAtkQFGxqxL1wtutiDuAi6Z4fingR2qehpwHfAVABHJAb4BXAqcDLxTRE52Mc45CYesHpObDh4ZYiyqaW9BrCgJEukftlLtju6BUXYd7Mna8Ye4uqoQ41GdqM1lMsu1BKGqfwA6ZzjlZOB3zrmvALUiUg1sBhpUtRTJbi8AAB7jSURBVElVR4DvAZe7FedchYusBeGmyRlMaW5BlARRhTYrkwLA03siqJK14w9xNpPJW16OQewErgAQkc1ADbAGWA3sSzhvv3NbUiJyk4hsE5Ft7e3tLoYbUxTIJZDrs4J9Lpmo4hpOcwui2Ka6JtraFCGQ6+OMdaVehzKjDZWx94ElCG94mSC+AJSKyA7gI8DzwPhcH0RVb1fVTaq6qbKyMt0xHkNEnMVy9k3UDc2RAYJ5vrQXjrPV1Efb2hhhU20ZgdzsHX8AKPDnsro03waqPeLZ6hhV7QFuBJDYfMY9QBOQD6xNOHUNcCDjAc4gXBSg3bqYXNES6ae2ojDtheNsNfWkzv4RXjncyycvPt7rUFJiu8t5x7MWhIiUikh8JdT7gD84SeNZ4DgRWe8cvxp4wKs4k6kM+a2LySXNkfTPYAIoLcjDn+uzqa7A000RgKwfoI6rqwrR2N5HNKpeh7LsuDnN9T5gK3CCiOwXkfeKyM0icrNzyknASyLyKrEZSx8DUNUx4MPAb4CXgR+o6i634pwPK7fhjvGosteFNRAQ6xq0qa4xW5siFPhzOG1Ndo8/xNVVhRgajXLgiI0fZZprXUyq+s5Zjm8FkrZxVfVB4EE34kqHcChAZ/8I0aji82VPDf3F7nDPECPjUda50IKA2DhEqyUIZ/yhnLycxbFONnF3ubXl7rw3THKL4x2SZcIhP+NRpcvKbaRVS0d8H+r0tyDAKbexzLuY2nuHqW/ry/rprYnqKp2qrjYOkXGWIOZhci2EJYh0anbKfLsxBgGxFsThniFUl29f9lOLbPwBoKzQT0Wh3waqPWAJYh5sb2p3tET68ef4WFmS78rjrygOMjIWpWtg+W49urUpQiiQy+tWFXsdypxstJlMnrAEMQ9Wj8kdzZF+1pbnk+PSuM7kWojl2830VGOEzevLyV0k4w9xdVWxon3LufXnhcX1LskSlZYgXOFGFddE1ct8b+rWniGaOvoX1fhDXF1liCMDo0T6rVs3kyxBzENxfi7+HJ8tlksjVXX2gXAvQcRbEMt1quvWxsU3/hBnNZm8YQliHkSEipDf9oRIo7beYYZGo2nbhzqZqqIAIst3NfXWxgjFwVxOWrm4xh/AEoRXLEHMky2WS6/mjvTuQ51MXo6PcCiwbNdCbG2KcO6GCtfGeNy0siRIoT/HEkSGWYKYp3DIbwkijVqcKa61Lk1xjVtRHOTQMmxBHDgyyN7OgUU5/gCxVvtGp+SGyRxLEPNkLYj0ao70k+sTVpe6M8U1rrp4ea6mXszjD3F1lTbVNdMsQcxTuChApG/ECoilSUtkgDVl+a5Pv1xZsjxXU29tjFBWkMcJ1UVehzJvG6tCHOoeondo+a5jyTRLEPMUDgUYiyrdg/ZmTQe3ZzDFrSgJ0j04yuDInLceWbRUlaeaIpy3oWJR1w6LD1Q3ttv2o5liCWKebG/q9FFVZw2E+4XYqpfhvhD7Ogc5cGRwUXcvgc1k8oIliHmKL5aztRALF+kfoW94LCMtiJXLcDX11qYOIPv3n55NTXkBeTliCSKDLEHMkxXsS5/Jfagz14JYTquptzZGCIcCE9/AF6vcHB+1FYWWIDLIEsQ8TdRjsoJ9C9bcEa/impkxCFg+q6lVla1NEc7bUJ72bVy9UGdTXTPKEsQ8lebnkeMTG4NIg5ZIPz6BNWXuTnEFCAVyCQVyl00LYk9HP609w4t+/CGuripES6Sf4bHlM8nAS25uOXqniLSJyEvTHC8RkZ+LyE4R2SUiNyYcGxeRHc4lq/ajjvP5xBbLpUlL5wCrSvMJ5OZk5PlWlASXzRjEk/H1D4t8/CGuripEVCdbncZdbrYg7gIumeH4h4Ddqno68AbgSyLid44NquoZzuVPXYxxQWKL5WwMYiHGo8rLh3pcreI61XJaTb21KUJ1cYD14cy9vm7aWGkzmTLJzT2p/yAitTOdAhRJrGM0BHQCY27F4wZbTb0wqsrnHtjFa619vGfL+ow9b3VxkIaGjow9n1dUlaebIlxYF14S4w8QSxAi2Z0gRsej9A6N0Ts06vw7eb1vePJ6T8LPa8ry+etLTqQ4mOd1+EdxLUGk4OvAA8BBoAj4c1WNOseCIrKNWML4gqr+dLoHEZGbgJsA1q1b527EU4RDAepbezP6nEvJ13/XwHeeauEDF23g6s2Z+92tLAnS3jfMeFQXZeG6VNW39dHRN8IFG8Neh5I2+f4cVpfm0+DxQPXgyDg79x9he0sX21u6aO7op8dJBMNj0VnvH8j1URTMoygYGxN7rL6Dx+o7+Po7z+LUNSUZ+B+kxssE8RZgB/AmYCPwsIg8pqo9QI2qHhCRDcDvRORFVW1M9iCqejtwO8CmTZsyWvciXOSno28EVV0y39Ay5XvP7OVLD7/GFWeu5m8uOTGjz11dEmQ8qnT0DU9Me12KlkL9pWTqPNh+tK1niO0tXWxzLrsOdDPmlNnZWFnISauKKQ7mUex84BcFcycTQDCXYud6UTCPUCAXf+7RvfvPNnfy0fue58+++SR/98cncd35NVnxmeJlgriRWOtAgQYR2QOcCDyjqgcAVLVJRB4BzgSSJggvVYYCjIxH6RkaoyQ/u5qG2ezh3a18+v4Xuej4Sv71ytMyXv5hRfHkVNelniBWl+azttz99SWZVFcZYmtjxLUWYDSqvNbWy7bmLicpdLKvcxCIffM/fU0p779oA5tqyjhrXRllhf5ZHnF259SW88uPvp5P/nAnn31gF1sbI/zrlad5/rniZYLYC7wZeExEqoETgCYRKQMGVHVYRMLAFuDfPIxzWvG1EO29w57/IheL7S2dfPi7z3Hq6hK+ec1Z5HmwN3I8QRzuHoK1GX/6jIhGlaf2RPijk6q9DiXt6qpCDI9FOdA1yLo0lGcZGYuyrblzooXw3N4ueodiw6HhUIBNNWVcf34tZ9WU8bpVJcd8+0+X8kI/37puE996vIl/+/WrvPVrj/H1d57F6WtLXXm+VLiWIETkPmKzk8Iish/4LJAHoKq3Av8I3CUiLwIC/I2qdojIBcBtIhIlNsvqC6q62604FyKcsDf1Yl+lmgn1rb28565trCrN584bzqEw4M33k/hiuaW8FuKVw70cGRhdMtNbE03UZGrvXXCCGI8q77nrWR5v6EAEjq8q4q2nrWJTTRmbastYV16Q0a4en0+46aKNnF1Tzkfve54rb32Sv730JG7cUutJl5Obs5jeOcvxg8DFSW5/EjjVrbjSKVxkBftSdah7kOvvfAZ/ro973rOZCie5eqGi0E9ejizp1dRbm5bm+AMcXbTvTScurIX0zUcaeLyhg09fdiJ/fs66rOkJOLumjF9+9EI++cOd3PKL3TzVFOHfrzydkoLMxmcrqRfAym2kpntglOvvfIaeoTHuuvEcz/vEfT6hqii4pFsQWxsj1FQUsMrlDZi8UFrgJxzyL3igentLJ//xP/X86emreP/rN2RNcogrLfDzX9dt4jN/fBK/e6WNy776GM/v7cpoDJYgFqCswI9PrGDfTIZGx3nfPc/S3DHA7deezSmrsmMK31JeTT0eVZ7eE1mS3UtxGxe4u1z34CgfvW8Hq0qD/NPbX5cVM4aSERHe9/oN/PDm8wG46tatfOuxJmJze9xnCWIBcnxCeaEtlpvO2HiUj973PNtauvjyn5/OBXXZMx9/RfHS3Vlu98EeeofGlmT3Ulx8qut8PihVlU//5EVae4b46tVnZt3itGTOXFfGgx99PW86sYp/+uXLvP+ebRwZcP+LqSWIBbJ6TMmpKn//s108tLuVz771ZN562iqvQzpKdXGsBZGpb2KZtFT2f5hJXVWInqGxee3H8oNt+/jli4f4xMUncOa6Mheic0dJQR63XXs2n/2Tk3n0tXYu+8pjbG9xt8vJEsQCVRYFaLcupmN85bf13PfMXj74ho3ckMEyGqlaURJgcHScnqFFVd0lJVsbI2yoLKRqCa/xmO/ucg1tvXzugd1cWBfmAxdtcCM0V4kIN25Zz49uvoCcHOHPb9vKbY82Eo2680XHEsQChUMBG6Se4t6nW/h//1PPlWev4f+85QSvw0lqRUls8HapDVSPjUd5trlrSbceIGF/6jkkiKHRcT783efJ9+fw5Xecvqj35z59bSm/+Mjr+d8nV/Mvv3qF992zjYGR9H/Z8XKh3JIQ72JaTOU2VJW+4TGODIzGLoMjVBQGOL46RO4CF679Ztdh/v6nL/HGEyr5lytOzdrXJHE19fHVRR5Hkz4vHuimb3hpjz9A7PcXCuTOqQXxhV+9wiuHe/n2DecsidZVSX4e/3nNWXznqRaebuokPy/95fItQSxQOBRgeCxK3/AYRR4NdvUNj3Gga5CugRGODIxwZGCUroFRjgyM0DUwQtfAKN0Do5PXB0cYHT+2SRrM83HKqhJOXV3C6WtLOHV1KRvChSl/04rXkzltTSnf8GiVdKriCaJ1ic1kiq9/OG+JtyBEhI2VhSkX7fuf3a3c9WQz79mynjeeWOVydJkjIlx3fi3XnudO7SZLEAs0uZp6JOMJYtfBbu5+spmf7TiYtIKkP9dHWUEeZQV+SvLz2FgZoqwwj9ICP2UFeZTm+yktiP18qHuQF/Z388L+I3z/2X3c9WQzENuB7XWrizl9TSmnrinhtNWlrC3PP+bN+OrhXt5717OsLoutki7wZ/dbq6o49ntbajOZtjZGOL46NPG+XMo2VoV4IoWy7Ye7h/g/P9rJKauK+ZtLs7PLc6Hcaqln91/xIhAumiy3kYlNWUbHo/z6pcPc/WQz21q6yM/L4c/OXsMFGysoK5j8wC8ryCM/L2dOb5zLz1gNxObRN7T18cL+I7GkcaCbbz/RzMh4LAmVFuTFWhlO0lhVks/779lGMC+He96zmfI0FC9zWzAvh/JC/5JaTR2rKdTFOzat8TqUjKirCvGT5w7QMzQ67VTV8ajyl9/fwfBYlK+988yM7Vq4VFiCWKBwyCm34fJAdXvvMPc9s5d7n26htWeYdeUFfOaPT+Kqs9emffl9jk84YUURJ6wo4qpNsWp2I2NRXmvtZef+I7y4v5ud+7v55qONjDuzJ4qCufzgA+ezpmzxVA6tLl5aq6lf2H+EwdHxJT/+EFdXOTlQPd101W8+0sDWpgj/fuVpbKi0emlzZQligSoTCva54fm9Xdz9ZDO/fPEQo+PKRcdX8i9X1PCG46syOgvDn+vjdatLeN3qEjg3dtvQ6Di7Dvaw+1APm2vLOWHF4hrsXVEcWFKrqbc2RhCBc9cvkwSRMNU1WYJILKVx5dnLo1WVbpYgFqi80I8I/PaVNsoLA9RUFFBTUbCg8YjhsXF+sfMQ92xtZuf+bkKBXK45t4Zrz6+Z2JM3GwTzcji7poyzaxbPYqNEK0ryeWF/t9dhpM3WpggnrihOy/4Ei8G68gL8Ob6kA9WLpZRGtrMEsUC5OT7OXV/OI6+288ir7RO3h0N+aioKqSkvoKaikNqw829FAaUFyf+AD3UPcu9Te7nvmb1E+kfYWFnILZefwhVnrSHkUWnspWxFcZBI/wjDY+Pz7pvuHRrlo/c9T1mhny9e6d3c+uGxcba3dHHNuTWePL8XcnN81IYLjlkLkVhK44c3n78oSmlkK/vUSYPv3XQ+fcNjtET6aYkM0BzpZ6/z79amCD95/sBR55fk51FbUcA6J2GsLs3nD/Xt/GZXK1FV3nxiNTdcUMuWugr75uOiFSWx7sG2nuF5VZjtHhjl+m8/w879R1CFqqIgn7o0s9unxj2/9wjDY9FlM/4QV1cVYvfBnqNui5fS+OtLFlcpjWxkCSJNQoFcTllVkrRa6dDoOHs7B2ju6I/96ySSHfu6+OULB4lqLGm878L1vPu8Gs/LYS8X8dXUh3uG5vyad/aP8O5vPU1DWx+3vftsHnmtnVsfbaSuKuRJf/fWxgg+gc3ryzP+3F6qqyri1y8dZmh0nGBezlGlNG6+aKPX4S16riYIEbkTeCvQpqqvS3K8BPhvYJ0TyxdV9dvOseuBzzin/pOq3u1mrG4K5uVwfHVR0hW7I2NRDnUPUl0cJOjCSkgzvcTV1HPR1jvEu7/1NC2RAW6/7mzecEIVbzyxiuaOfv72Jy9QU1HAObWZ/aDe2hjhlFUlWbengdvqqkJEFZoj/dRWFC6ZUhrZwu2lrncBl8xw/EPAblU9ndj2pF8SEb+IlBPbovRcYDPwWWev6iXHn+ujpqLQkoMH5rOa+lD3IFff9hT7Ogf59g3n8IYTYqty83J8/Oc1Z7GmrIAPfGc7+zoHXIk5mcGRcZ7f17XsupdgcqprQ1vfRCmNL111+pIopZENXE0QqvoHoHOmU4AiiXW0h5xzx4C3AA+raqeqdgEPM3OiMWbOivNzCeb5Ul5Nva9zgHfctpW23mG+897Nx+xvUVrg547rNzE2HuW9dz9L79CoG2EfY3tLF6PjuuQL9CWzobIQEbjrieYlWUrDa14Xy/k6cBJwEHgR+JiqRoHVwL6E8/Y7tx1DRG4SkW0isq29vT3ZKcYkJSKsLMlPKUHs6ejnHbdtpWdwjHvfdy6bpulC2lAZ4pvvPpvG9n4+ct/zEwsJ3bS1qYMcn3DOMht/gFj37dqyAra1dC3pUhpe8TpBvAXYAawCzgC+LiLFc3kAVb1dVTep6qbKyko3YjRLWHUKi+XqW3t5x21bGR6Lct/7z+P0taUznr+lLswtl5/CI6+28/lfvpzOcI/x+1fb+M7WFs5aV7psp0IfX11EgT/HSmm4wOt31I3AFzS2rVeDiOwBTgQOEBuTiFsDPJLx6MySt6I4yLPN0+/KtetgN9fe8Qw5PuH7N53HcSmWBr/m3Boa2vq484k91FWFeNe569IVMhCrMfSV39bztd/Vc0J1EV+86vS0Pv5i8rk/PZmewTErpeECrxPEXuDNwGMiUg2cADQBDcA/JwxMXwz8rTchmqVsRUk+bb2HiEb1mFkvO/cd4do7niYUyOXe958352KMf3fZSTS19/MPP3uJ2nABF2xMz57cnf0jfOx7z/NYfQdXnLWaz7/tVPL9y/eb85qyAliSU1i852oXk4jcB2wFThCR/SLyXhG5WURudk75R+ACEXkR+C3wN6raoaqdzrFnncstzm3GpNWK4gCj40qk/+htY59t7uSabz1NSUEe3//A+fOq1Jub4+Nr7zqT9eFC/uK/n2NPR/+C49257wh/8rXHebqpk39++6l86arTl3VyMO5ytQWhqu+c5fhBYq2DZMfuBO50Iy5j4laUOFNde4aodEq3P9nQwXvv3sbKkiD3vv9cVjoL6uajOJjHHdefw9v+8wnee9ez3P/BLfOqvquq3Pv0Xm75+W4qiwL86C/O57Q1M4+FGLNQXg9SG+Opame+fHyg+pFX27jxrmdZW57P9z5w3oKSQ9y6igJufffZ7Osa4IPf3c7o+LGbO81kcGScT/xwJ5/56Uucv7GCX3zkQksOJiMsQZhlLZ4ADvUM8dCuw7z/nm3UVYX43k3nU1WUvsVWm9eX889vP5UnGiL835/vIjYvY3Z7Ovp5+38+wf3PH+Av/+h4vn3DOcumWqvxnteD1MZ4Khzy4xP48fb9vHSgm9etLuHuGzenfRMmgKs2raWhvY/bHm2irjLEDVvWz3j+Q7sO84kf7CQnR45atW1MpliCMMtabo6PyqIAO/YdYXNtOXfeeI6r6wn++i0n0tjWzy2/2E1tuDDph/7YeJQvPvQatz7ayGlrSiZKeBiTadbFZJa9c2rLedOJVdz1HneTA8S2c/3K1WdwwopiPvLd56lv7T3qeHvvMNfe8Qy3PtrINeeu44c3L65tXM3SIqn2hS4GmzZt0m3btnkdhllkVDXj+24cODLI5V9/ggJ/Dj/90BbKC/1sa+7kQ999jiMDo/zz20/lz2ybTJMBIrJdVTclO2YtCLPsebEp0+rSfG6/7mwO9wxx839v51uPNXH17U8RzMvh/g9useRgsoKNQRjjkbPWlfHvV57Gx763g2f2dPK/T67mi1edvuz2dDDZyxKEMR66/IzVDI6MMzIe5drzamyLWZNVLEEY47GrN6e3kJ8x6WJjEMYYY5KyBGGMMSYpSxDGGGOSsgRhjDEmKUsQxhhjkrIEYYwxJilLEMYYY5KyBGGMMSapJVWsT0TagZZ53j0MdKQxnHSz+BbG4lsYi29hsjm+GlWtTHZgSSWIhRCRbdNVNMwGFt/CWHwLY/EtTLbHNx3rYjLGGJOUJQhjjDFJWYKYdLvXAczC4lsYi29hLL6Fyfb4krIxCGOMMUlZC8IYY0xSliCMMcYktewShIhcIiKvikiDiHwqyfGAiHzfOf60iNRmMLa1IvJ7EdktIrtE5GNJznmDiHSLyA7n8g+Zis95/mYRedF57m1JjouIfNV5/V4QkbMyGNsJCa/LDhHpEZGPTzkno6+fiNwpIm0i8lLCbeUi8rCI1Dv/lk1z3+udc+pF5PoMxvfvIvKK8/u7X0RKp7nvjO8FF+P7nIgcSPgdXjbNfWf8W3cxvu8nxNYsIjumua/rr9+CqeqyuQA5QCOwAfADO4GTp5zzQeBW5/rVwPczGN9K4CznehHwWpL43gD8wsPXsBkIz3D8MuBXgADnAU97+Ls+TGwRkGevH3ARcBbwUsJt/wZ8yrn+KeBfk9yvHGhy/i1zrpdlKL6LgVzn+r8miy+V94KL8X0O+GQKv/8Z/9bdim/K8S8B/+DV67fQy3JrQWwGGlS1SVVHgO8Bl08553Lgbuf6j4A3S4Y2ClbVQ6r6nHO9F3gZWJ2J506jy4F7NOYpoFREVnoQx5uBRlWd78r6tFDVPwCdU25OfI/dDbwtyV3fAjysqp2q2gU8DFySifhU9SFVHXN+fApYk+7nTdU0r18qUvlbX7CZ4nM+N94B3Jfu582U5ZYgVgP7En7ez7EfwBPnOH8k3UBFRqJL4HRtnQk8neTw+SKyU0R+JSKnZDQwUOAhEdkuIjclOZ7Ka5wJVzP9H6aXrx9Ataoecq4fBqqTnJMtr+N7iLUIk5ntveCmDztdYHdO00WXDa/f64FWVa2f5riXr19KlluCWBREJAT8GPi4qvZMOfwcsW6T04GvAT/NcHgXqupZwKXAh0Tkogw//6xExA/8KfDDJIe9fv2OorG+hqycay4ifweMAfdOc4pX74VvAhuBM4BDxLpxstE7mbn1kPV/S8stQRwA1ib8vMa5Lek5IpILlACRjEQXe848YsnhXlX9ydTjqtqjqn3O9QeBPBEJZyo+VT3g/NsG3E+sKZ8oldfYbZcCz6lq69QDXr9+jtZ4t5vzb1uSczx9HUXkBuCtwDVOEjtGCu8FV6hqq6qOq2oU+K9pntfr1y8XuAL4/nTnePX6zcVySxDPAseJyHrnW+bVwANTznkAiM8YuRL43XR/IOnm9FneAbysql+e5pwV8TEREdlM7HeYkQQmIoUiUhS/Tmww86Uppz0AXOfMZjoP6E7oTsmUab+5efn6JUh8j10P/CzJOb8BLhaRMqcL5WLnNteJyCXAXwN/qqoD05yTynvBrfgSx7TePs3zpvK37qY/Al5R1f3JDnr5+s2J16Pkmb4Qm2XzGrEZDn/n3HYLsT8GgCCxrokG4BlgQwZju5BYd8MLwA7nchlwM3Czc86HgV3EZmU8BVyQwfg2OM+704kh/volxifAN5zX90VgU4Z/v4XEPvBLEm7z7PUjlqgOAaPE+sHfS2xM67dAPfA/QLlz7ibgWwn3fY/zPmwAbsxgfA3E+u/j78H4rL5VwIMzvRcyFN93nPfWC8Q+9FdOjc/5+Zi/9UzE59x+V/w9l3Buxl+/hV6s1IYxxpikllsXkzHGmBRZgjDGGJOUJQhjjDFJWYIwxhiTlCUIY4wxSVmCMMZDTnXZX3gdhzHJWIIwxhiTlCUIY1IgIu8WkWec2v23iUiOiPSJyH9IbO+O34pIpXPuGSLyVMJ+CmXO7XUi8j9OocDnRGSj8/AhEfmRswfDvQkrvb8gsb1BXhCRL3r0XzfLmCUIY2YhIicBfw5sUdUzgHHgGmKrtrep6inAo8BnnbvcA/yNqp5GbMVv/PZ7gW9orFDgBcRW4EKsau/HgZOJrbDdIiIVxMpInOI8zj+5+7805liWIIyZ3ZuBs4Fnnd3B3kzsgzzKZDG2/wYuFJESoFRVH3Vuvxu4yKm7s1pV7wdQ1SGdrHP0jKru11jxuR1ALbEy80PAHSJyBZC0JpIxbrIEYczsBLhbVc9wLieo6ueSnDffujXDCdfHie3mNkasuuePiFVV/fU8H9uYebMEYczsfgtcKSJVMLGndA2xv58rnXPeBTyuqt1Al4i83rn9WuBRje0QuF9E3uY8RkBECqZ7QmdPkBKNlST/S+B0N/5jxswk1+sAjMl2qrpbRD5DbPcvH7HKnR8C+oHNzrE2YuMUECvhfauTAJqAG53brwVuE5FbnMe4aoanLQJ+JiJBYi2Yv0rzf8uYWVk1V2PmSUT6VDXkdRzGuMW6mIwxxiRlLQhjjDFJWQvCGGNMUpYgjDHGJGUJwhhjTFKWIIwxxiRlCcIYY0xS/x+8GuKWOzqhZwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n_CPW_ae858R",
        "outputId": "3761e1e1-7557-4e43-a892-fca1fa36a974",
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "parameters = model(X_train, y_train, n_h = 4,  mini_batch_size=128, learning_rate= 0.001,  beta=0.9,\n",
        "          beta1=0, beta2=0.9, optimizer=\"adam\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cost after epoch 0: 0.605447\n",
            "Cost after epoch 1: 0.485888\n",
            "Cost after epoch 2: 0.455039\n",
            "Cost after epoch 3: 0.437966\n",
            "Cost after epoch 4: 0.426471\n",
            "Cost after epoch 5: 0.419440\n",
            "Cost after epoch 6: 0.414431\n",
            "Cost after epoch 7: 0.411123\n",
            "Cost after epoch 8: 0.409106\n",
            "Cost after epoch 9: 0.406916\n",
            "Cost after epoch 10: 0.405556\n",
            "Cost after epoch 11: 0.404031\n",
            "Cost after epoch 12: 0.402654\n",
            "Cost after epoch 13: 0.401189\n",
            "Cost after epoch 14: 0.399858\n",
            "Cost after epoch 15: 0.398767\n",
            "Cost after epoch 16: 0.397490\n",
            "Cost after epoch 17: 0.396462\n",
            "Cost after epoch 18: 0.395467\n",
            "Cost after epoch 19: 0.394695\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhcdZ3v8fen9zVJp9MJZCEhElYHESPLgAyKYOA64KjjRVwAHZFRxBkdHbzeqz54nSt33Ma5zCjjoKAoIC4EJ4gggjMqS4OAhAQIEchG0ul0lk6n13zvH+d0p9JUdypJV1V31+f1POepqt85p+rbp6vr079zfueUIgIzM7PhyopdgJmZjU8OCDMzy8oBYWZmWTkgzMwsKweEmZll5YAwM7OsHBBWEiS9TtLTxa7DbCJxQFjeSXpe0huLWUNE/GdEHFXMGgZJOlPS2iK99kWSXpC0U9JPJU0fZdkTJD0iqSu9PSFjniRdI6k9na6RpIz510l6WtJuSZfk+ceyPHFA2KQgqbzYNcDQB+e4/LuSdBzwTeA9wCygC/iXEZatAm4Hvgc0ATcAt6ftAJcBbwFeBRwP/DnwwYyneBz4EPDomP8gVjDj8o1spUFSmaSrJD2X/hd6a+Z/tJJ+KOklSdsk/Tr9gBuc9x1J/yppmaSdwOvTnsrfSXoiXecWSTXp8nv91z7asun8T0raIGm9pL+SFJKOGOHnuE/SFyT9huRDd6GkSyWtkLRD0mpJH0yXrQfuBGZL6kyn2fvaFmPkXcAdEfHriOgE/hfwVkmNWZY9E6gAvhYRPRHxdUDAG9L5FwNfjoi1EbEO+DJwyeDKEXFtRPwS6B7jn8EKyAFhxfQRkv9C/wyYDXQA12bMvxNYBMwk+U/0pmHrXwR8AWgE/ittewewBDic5D/bS0Z5/azLSloCfAx4I3AEyYflvryH5L/qRuAFYBPwZmAKcCnwVUknRsRO4FxgfUQ0pNP6HLbFEEmHSdo6ynTRCDUeR/KfPQAR8RzQCxw5wrJPxN7X4nkibX/Zc6X3j8MmlYpiF2Al7XLgiohYCyDpc8CLkt4TEf0Rcf3ggum8DklTI2Jb2nx7RPwmvd+d7gL/evqBi6Q7gKH95lmMtOw7gG9HxPKM137XPn6W7wwun/qPjPv3S/oF8DpG3uUy6rbIXDAiXgSm7aOebBqAbcPatpGE2v4uO3z+NqBBksIXeJs03IOwYpoP/GTwP19gBTAAzJJULumL6S6X7cDz6TozMtZfk+U5X8q430XyQTaSkZadPey5s73OcHstI+lcSQ9I2pL+bOexd+3DjbgtcnjtXHWS9GgyTQF2HMCyw+dPATodDpOLA8KKaQ1wbkRMy5hq0n3aFwEXkOzmmQosSNdRxvr5+jDaAMzNeDwvh3WGapFUDfwI+BIwKyKmAcvYU3u2ukfbFntJdzF1jjKN1NtZTnJQefB5FgLVwDMjLHt85sgkkt1wyzPmvypj3qsy5tkk4YCwQqmUVJMxVQDfAL4gaT6ApBZJF6TLNwI9QDtQB/xDAWu9FbhU0jGS6kgO5u6PKpIP3jagX9K5wDkZ8zcCzZKmZrSNti32EhEvZhy/yDYNP1Yz6Cbgz5WcE1IPXA38OCKy9SDuI+nBXCmpWtIVafu96e2NwMckzZE0G/g48J3BlSVVpQf9xZ7fvT9vJhj/wqxQlgG7MqbPAf8ELAV+IWkH8ABwcrr8jSQHe9cBT6XzCiIi7gS+DvwKWJXx2j05rr8DuJIkaDpIekNLM+avBH4ArE53Kc1m9G0xJtJjJJeTBMUmkhD+0OB8SXdK+h/psr0kB83fC2wF3ge8JW2HZLjsHcAfgCdJjrl8M+PlfkHye/5T4Lr0/hlj+fNY/sm7DM1GJ+kYkg/B6uEHjM0mM/cgzLKQ9BfprpUm4BqS8wccDlZSHBBm2X2QZDfMcyT74v+6uOWYFZ53MZmZWVbuQZiZWVaT5kzqGTNmxIIFC4pdhpnZhPLII49sjoiWbPMmTUAsWLCA1tbWYpdhZjahSHphpHnexWRmZlk5IMzMLCsHhJmZZeWAMDOzrBwQZmaWlQPCzMyyckCYmVlWJR8Q27r6+Kd7nuXxNVuLXYqZ2bgyaU6UO1Aqg6/e8wzVlWW8at6BfM2vmdnkVPI9iCk1lTTXV/H85p3FLsXMbFzJa0BIWiLpaUmrJF01wjLvkPSUpOWSvp/RfrGkZ9Pp4nzWuWBGPc+3OyDMzDLlbReTpHLgWuBsYC3wsKSlEfFUxjKLgE8Bp0VEh6SZaft04LPAYpIveH8kXbcjH7UuaK7nN6s25+OpzcwmrHz2IE4CVkXE6vR7bG8Ghn8J+weAawc/+CNiU9r+JuDuiNiSzrsbWJKvQhc01/HS9m529Q7k6yXMzCacfAbEHGBNxuO1aVumI4EjJf1G0gOSluzHuki6TFKrpNa2trYDLnTBjHoAXtji3UxmZoOKfZC6AlgEnAm8E/g3STkPJYqI6yJicUQsbmnJejnznCxoTgLCB6rNzPbIZ0CsA+ZlPJ6btmVaCyyNiL6I+CPwDElg5LLumJk/ow6A59u78vUSZmYTTj4D4mFgkaTDJVUBFwJLhy3zU5LeA5JmkOxyWg3cBZwjqUlSE3BO2pYXHupqZvZyeRvFFBH9kq4g+WAvB66PiOWSrgZaI2Ipe4LgKWAA+EREtANI+jxJyABcHRFb8lUreKirmdlweT2TOiKWAcuGtX0m434AH0un4eteD1yfz/oyeairmdnein2QetzwUFczs705IFIe6mpmtjcHRMpDXc3M9uaASHmoq5nZ3hwQKQ91NTPbmwMig4e6mpnt4YDIsKC5nuc3exeTmRk4IPbioa5mZns4IDJ4qKuZ2R4OiAwe6mpmtocDIoOHupqZ7eGAyOChrmZmezgghvFQVzOzhANiGA91NTNLOCCG8VBXM7OEA2IYD3U1M0s4IIbxUFczs4QDYhgPdTUzSzgghvFQVzOzhAMiCw91NTNzQGTloa5mZg6IrDzU1czMAZGVh7qamTkgsvJQVzMzB0RWHupqZuaAyMpDXc3MHBAj8lBXMyt1DogReKirmZU6B8QIPNTVzEqdA2IEHupqZqXOATECD3U1s1LngBiBh7qaWalzQIzAQ13NrNQ5IEbhoa5mVsocEKPwUFczK2UOiFF4qKuZlTIHxCg81NXMSpkDYhQe6mpmpSyvASFpiaSnJa2SdFWW+ZdIapP0WDr9Vca8gYz2pfmscyQe6mpmpawiX08sqRy4FjgbWAs8LGlpRDw1bNFbIuKKLE+xKyJOyFd9ufBQVzMrZfnsQZwErIqI1RHRC9wMXJDH18sLD3U1s1KVz4CYA6zJeLw2bRvubZKekHSbpHkZ7TWSWiU9IOkt2V5A0mXpMq1tbW1jWPoeHupqZqWq2Aep7wAWRMTxwN3ADRnz5kfEYuAi4GuSXjF85Yi4LiIWR8TilpaWvBTooa5mVqryGRDrgMwewdy0bUhEtEdET/rwW8BrMuatS29XA/cBr85jrSPyUFczK1X5DIiHgUWSDpdUBVwI7DUaSdKhGQ/PB1ak7U2SqtP7M4DTgOEHtwtiz1BX72Yys9KSt1FMEdEv6QrgLqAcuD4ilku6GmiNiKXAlZLOB/qBLcAl6erHAN+UtJskxL6YZfRTQewZ6uoehJmVlrwFBEBELAOWDWv7TMb9TwGfyrLeb4E/yWdtufJQVzMrVcU+SD0heKirmZUiB0QO5jfX+RiEmZUcB0QODm+u91BXMys5DogceKirmZUiB0QOPNTVzEqRAyIHHupqZqXIAZEDD3U1s1LkgMiRh7qaWalxQOTIQ13NrNQ4IHLkoa5mVmocEDnyUFczKzUOiBx5qKuZlRoHRI481NXMSo0DIkce6mpmpcYBsR881NXMSokDYj94qKuZlRIHxH7wUFczKyUOiP3goa5mVkocEPvBQ13NrJQ4IPaDh7qaWSlxQOwHD3U1s1LigNhPHupqZqXCAbGfPNTVzEqFA2I/eairmZUKB8R+8lBXMysVDoj95KGuZlYqHBD7yUNdzaxUOCD2k4e6mlmpcEAcAA91NbNS4IA4AB7qamalwAFxADzU1cxKgQPiAHioq5mVAgfEAfBQVzMrBTkFhKS/zKWtVHioq5mVglx7EJ/Ksa0keKirmZWCitFmSjoXOA+YI+nrGbOmAP35LGy881BXM5vsRg0IYD3QCpwPPJLRvgP423wVNRHMb67jt6vai12GmVnejBoQEfE48Lik70dEH4CkJmBeRHQUosDx6vDmen786Dp29Q5QW1Ve7HLMzMZcrscg7pY0RdJ04FHg3yR9dV8rSVoi6WlJqyRdlWX+JZLaJD2WTn+VMe9iSc+m08U5/0QF4qGuZjbZ5RoQUyNiO/BW4MaIOBk4a7QVJJUD1wLnAscC75R0bJZFb4mIE9LpW+m604HPAicDJwGfTXsu44aHuprZZJdrQFRIOhR4B/CzHNc5CVgVEasjohe4Gbggx3XfBNwdEVvSXVl3A0tyXLcgPNTVzCa7XAPiauAu4LmIeFjSQuDZfawzB1iT8Xht2jbc2yQ9Iek2SfP2Z11Jl0lqldTa1taW448yNjzU1cwmu5wCIiJ+GBHHR8Rfp49XR8TbxuD17wAWRMTxJL2EG/Zn5Yi4LiIWR8TilpaWMShn/3ioq5lNZrmeST1X0k8kbUqnH0mau4/V1gHzMh7PTduGRER7RPSkD78FvCbXdccDX9XVzCazXHcxfRtYCsxOpzvSttE8DCySdLikKuDC9DmGpMc1Bp0PrEjv3wWcI6kpPTh9Tto2rviqrmY2meUaEC0R8e2I6E+n7wCj7tOJiH7gCpIP9hXArRGxXNLVks5PF7tS0nJJjwNXApek624BPk8SMg8DV6dt44qHuprZZLavM6kHtUt6N/CD9PE7gX2eRhwRy4Blw9o+k3H/U4xwTaeIuB64Psf6iiJzqOvRh0wpcjVmZmMr1x7E+0iGuL4EbADeTvrffinzUFczm8xy7UFcDVw8eHmN9ES2L5EER8nyUFczm8xy7UEcn3ntpfR4wKvzU9LE4qGuZjZZ5RoQZZmXukh7ELn2PiY1D3U1s8kq1w/5LwO/k/TD9PFfAl/IT0kTi6/qamaTVa5nUt9IcqG+jen01oj4bj4Lmyg81NXMJqucdxNFxFPAU3msZULyUFczm6xyPQZhIzi8pZ7KcnHvyo3FLsXMbEw5IA5SQ3UF7z11Abc9spaVL20vdjlmZmPGATEGPvKGI2ioruD/LFtZ7FLMzMaMA2IMTKur4sqzFnH/M238+pnCfi+FmVm+OCDGyHtOnc+86bX8w7IVDOyOYpdjZnbQHBBjpLqinE++6WhWvrSDHz+6ttjlmJkdNAfEGHrz8YdywrxpfOkXT/s7IsxswnNAjCFJfPq/HcPG7T186z9XF7scM7OD4oAYY69dMJ03HTeLb9z/HG07eva9gpnZOOWAyIO/X3I0Pf27+do9zxS7FDOzA+aAyIOFLQ28+5T53PzwGp7duKPY5ZiZHRAHRJ5cedYi6irL+eKdPnnOzCYmB0SeTK+v4kOvP4JfrtzEb5/bXOxyzMz2mwMijy49bQFzpiUnz+32yXNmNsE4IPKoprKcT7zpKJ5ct53bH19X7HLMzPaLAyLPzn/VbP5kzlT+8edP093nk+fMbOJwQORZWZn4H+cdw/pt3Xz7N88Xuxwzs5w5IArg1Fc088ZjZvIvv1pFe6dPnjOzicEBUSBXnXs0XX0DfP2Xzxa7FDOznDggCuSImY2886R53PTgi6xu6yx2OWZm++SAKKCPnnUk1RVlXPNznzxnZuOfA6KAWhqr+eszX8Fdyzfy0B+3FLscM7NROSAK7P2nL+SQKTV8YdkKInzynJmNXw6IAqutKufj5xzJ42u28rMnNhS7HDOzETkgiuCtJ87lmEOncM3PV9LT75PnzGx8ckAUQXmZ+PR5x7C2Yxc3/vaFYpdjZpaVA6JITl80gzOPauGf732WrV29xS7HzOxlHBBF9Klzj6Gzp59/vndVsUsxM3sZB0QRHXVII+9YPI8bf/c8z2/eWexyzMz24oAoso+dfSQ1FeVc+p2H2bi9u9jlmJkNcUAU2cwpNXznfa9l0/ZuLrzuAYeEmY0beQ0ISUskPS1plaSrRlnubZJC0uL08QJJuyQ9lk7fyGedxfaa+dO58f0nOSTMbFzJW0BIKgeuBc4FjgXeKenYLMs1Ah8FHhw267mIOCGdLs9XneOFQ8LMxpt89iBOAlZFxOqI6AVuBi7IstzngWuAkv9EdEiY2XiSz4CYA6zJeLw2bRsi6URgXkT8R5b1D5f0e0n3S3pdtheQdJmkVkmtbW1tY1Z4MTkkzGy8KNpBakllwFeAj2eZvQE4LCJeDXwM+L6kKcMXiojrImJxRCxuaWnJb8EF5JAws/EgnwGxDpiX8Xhu2jaoEXglcJ+k54FTgKWSFkdET0S0A0TEI8BzwJF5rHXccUiYWbHlMyAeBhZJOlxSFXAhsHRwZkRsi4gZEbEgIhYADwDnR0SrpJb0IDeSFgKLgNV5rHVcckiYWTHlLSAioh+4ArgLWAHcGhHLJV0t6fx9rH4G8ISkx4DbgMsjoiS/YcchYWbFosnypTWLFy+O1tbWYpeRN4+8sIX3/vtDzJxSw82XncKsKTXFLsnMJgFJj0TE4mzzfCb1BOGehJkVmgNiAnFImFkhOSAmGIeEmRWKA2ICckiYWSE4ICao4SGxtqOr2CWZ2STjgJjABkOibUcP537tP7m1dQ2TZVSamRWfA2KCe8386Sy78nUcO3sKn7ztCd5/Q6t3OZnZmHBATAKHNdfxgw+cwmf//Fh++9xmzv7K/fzk92vdmzCzg+KAmCTKysSlpx3OnR89g0WzGvnbWx7nsu8+QtuOnmKXZmYTlANikjl8Rj23fvBUPn3eMdz/TBvnfPV+7nh8vXsTZrbfHBCTUHmZ+MAZC1l25es4rLmej/zg93z4+4/S3unehJnlzgExiR0xs4EfXX4qn1xyFPc8tYlzvvprfv7khmKXZWYThANikqsoL+NDZx7BHR85nUOn1XD59x7lyh/8no6dvcUuzczGOQdEiTjqkEZ+8qHT+PjZR3Lnkxs4+6u/5u6nNha7LDMbxxwQJaSyvIyPnLWI2z98Oi2N1XzgxlY+dutjbOvqK3ZpZjYOOSBK0LGzp3D7h0/jyjccwe2Preecr93PzQ+9SFdvf7FLM7NxxF8YVOL+sHYbV/34CZav305jTQVvO3Eu7z5lPkfMbCh2aWZWAKN9YZADwogIWl/o4Lu/e4E7n9xA30Bw6sJm3nPqfM4+dhaV5e5omk1WDgjLWduOHm5tXcP3H3yRdVt3MbOxmgtfO493nnwYh06tLXZ5ZjbGHBC23wZ2B/c9vYnvPfAC9z3TRpnEWUfP5N2nzOf0I2ZQVqZil2hmY2C0gKgodDE2MZSXibOOmcVZx8xizZYubnrwRW5tXcMvntrIguY63nXyfN7+mrk01VcVu1QzyxP3ICxnPf0D/PzJl/ju716g9YUOqivKePPxs3n3KYdxwrxpSO5VmE003sVkY27Fhu1874EX+Onv17Gzd4D5zXW8/qiZvOHomZy8cDrVFeXFLtHMcuCAsLzZ0d3HHY9v4J4VG/nNqs309O+mrqqc046YwRuOnsnrj5rJIVNril2mmY3AAWEF0d03wO+ea+eXKzfyq5VtrNu6C4BjD52ShMXRMzlh3jTKfYDbbNxwQFjBRQTPbOzk3pWb+NXKTTzyYgcDu4Pp9VX82ZEtvOHomZxxZAtTayuLXapZSXNAWNFt6+rj/mfb+NXKTdz39CY6uvooLxOvmd/E64+ayUmHT+eYQxupq/LAOrNCckDYuDKwO3hszVbuXbmRe1e2sWLDdgDKBAtbGnjl7CkcN3sqx81Jbt3LMMsfB4SNaxu3d/PE2m08uW4by9dvY/n67WzY1j00f970Wl45eyrHzZ7CcXOS25mNPvBtNhZ8opyNa7Om1HD2sTWcfeysobb2zh6Wr9/Ok+u3sXzddpav38adT740NH9mYzXHzZ7CK9PAWNjSwOxptTRU+y1tNlb812TjUnNDNWcc2cIZR7YMtW3v7mPF+u08uX47y9clPY37n2ljd0YneGptJbOn1TJnWk16W8vsdJozrZaZjdW+TIhZjhwQNmFMqank5IXNnLyweaitu2+AlS/t4MUtXazfuot1HbtYv3UXazt28dAft7C9e+/vuKgsF4dMrWH21CQw5jTtCY950+uYM62WqgpfvdYMHBA2wdVUlnPCvGmcMG9a1vk7uvtYv7U7CY90Wp9OD/5xCy893s1ARhdEgkOm1DCvqY65TbXMnV7HvKYkPOZNr+OQKTU+j8NKhgPCJrXGmkqOOqSSow5pzDq/f2A3G3f0sHZLF2s6drFmSxdrOrpY27GLB1a3s+GxdWSO46goE7On1TJvei3zmpLQmNtUy8zGGpobqmiqq6KprpIKf4eGTQIOCCtpFeVlya6mabWcnGV+b/9u1m/dxZqOLtZs2cXajj1Bcs+KjWzu7M36vFNrK5leX8X0+iQ0muuraKrfczu9vpLp9dVMr6tiekMV9VXlvtihjTsOCLNRVFWUsWBGPQtm1Ged39Xbz7qOXbTt6GFLVy9bdiZTx85e2nf20tHVy9qOLv6wbitbdvbSN5B9WHl1RRkzGqqZ0VCV3lYzo7GK5vpqZjQm7S1p+9TaSh9ot4JwQJgdhLqqChbNamTRrOy7sDJFBJ09/XTs7EvDpIctO/vYsrOH9s5e2jp72NzZy4Zt3Ty5fhvtnb307355oFSUien1aZA07gmVwZ7K9GG9lSk1Fe6d2AFxQJgViCQaaypprKnksOa6fS6/e3ewbVcfmzt7hsKjvbOHzZ09bN7Rm9x29vDcpk7ad/bQ3bc76/NUloumuqo9u7zq9wRJ5jSjoZrm+iqm1VX5QLwBeQ4ISUuAfwLKgW9FxBdHWO5twG3AayOiNW37FPB+YAC4MiLuymetZuNNWZloSj/Qc+mhdPX2D+3iak93c2Xeb08fr1i/nfadvWzb1Zf9dQXT65PdW80NVTSnwTEj435zujusuaHax08msbwFhKRy4FrgbGAt8LCkpRHx1LDlGoGPAg9mtB0LXAgcB8wG7pF0ZEQM5Ktes4murqqCuqoK5jbtu3cC0Dewm61dfWmIJLu52jt7aN/ZO9Rbad/Zyx/WbqW9s5cdPf1Zn6e6oizj4HvVUG+lqa6KpvrKvR4nPZhKf6HUBJHPHsRJwKqIWA0g6WbgAuCpYct9HrgG+ERG2wXAzRHRA/xR0qr0+X6Xx3rNSkpleRktjdW0NFYD++6hdPcNJGHS2cvmjEDZ3NlDR1df0mPp6mXNli627Ox92UmKmeqryvcKlGl1lUypqWRKbQVTawfvVw7dn1qbzGusqfTurwLKZ0DMAdZkPF4Le48klHQiMC8i/kPSJ4at+8CwdecMfwFJlwGXARx22GFjVLaZZVNTWT502ZJcDPZQOrr2jOzqGPZ4cOTX8+072b6rj+3d/XuduJhNY3UFU2oraaxJw6S2kmm1lTTVJ0EzeC7KtLTHMq2ukmm1VT5D/gAU7SC1pDLgK8AlB/ocEXEdcB0kV3Mdm8rMbCzs3UPJTUSws3eAbbv6ksDY1Zfc7+7f09adtu3qZ/uuPl5s7+IPu5KRYb392Q/UAzRUV+wJkPokRAZ7L9NqK9MBBHvCZ0pN0ntpqKko2V5LPgNiHTAv4/HctG1QI/BK4L70ANchwFJJ5+ewrplNQpJoqK6gobqCOTn2VAZFBLv6BoZ2dw32XrZ27em5DLZ17Ozl+c076ejqZccou8IGNVRXDIVGZojsaRv+uGKoraGmgoaqigl57ko+A+JhYJGkw0k+3C8ELhqcGRHbgBmDjyXdB/xdRLRK2gV8X9JXSA5SLwIeymOtZjbBSRo6UL8/4dI3sJsd3f3s6O5jR3f/0K6u7RmPB+cPtm3a0c1zbf1D87Odr7J3bdBQVbFXcOwVItUVae3l1FWXJ7dVFdRXVVBbVU59dTl1lRVD82orCzNyLG8BERH9kq4A7iIZ5np9RCyXdDXQGhFLR1l3uaRbSQ5o9wMf9ggmM8uHyvKyoXNBDkRE0N23Ow2QPUGTGTo7ejLbk9vNnb38cfNOdnT309nTT88ou8eGk6C2snwoVI6fO5X/d9GJB1T/qK/jb5QzMyu+gd1BV28/Xb0DdPUOsLOnn1196W3vADt7B9jV28/O3gG6epLlBtvmNNXyiTcdfUCv62+UMzMb58rL9pxpP1543JeZmWXlgDAzs6wcEGZmlpUDwszMsnJAmJlZVg4IMzPLygFhZmZZOSDMzCyrSXMmtaQ24IWDeIoZwOYxKicfXN/BcX0Hx/UdnPFc3/yIaMk2Y9IExMGS1DrS6ebjges7OK7v4Li+gzPe6xuJdzGZmVlWDggzM8vKAbHHdcUuYB9c38FxfQfH9R2c8V5fVj4GYWZmWbkHYWZmWTkgzMwsq5IKCElLJD0taZWkq7LMr5Z0Szr/QUkLCljbPEm/kvSUpOWSPpplmTMlbZP0WDp9plD1ZdTwvKQ/pK//sq/wU+Lr6TZ8QtLYfw/iyLUdlbFtHpO0XdLfDFumoNtQ0vWSNkl6MqNtuqS7JT2b3jaNsO7F6TLPSrq4gPX9o6SV6e/vJ5KmjbDuqO+FPNb3OUnrMn6H542w7qh/73ms75aM2p6X9NgI6+Z9+x20iCiJieR7sZ8DFgJVwOPAscOW+RDwjfT+hcAtBazvUODE9H4j8EyW+s4Eflbk7fg8MGOU+ecBdwICTgEeLOLv+yWSk4CKtg2BM4ATgScz2v4vcFV6/yrgmizrTQdWp7dN6f2mAtV3DlCR3r8mW325vBfyWN/ngL/L4fc/6t97vuobNv/LwGeKtf0OdiqlHsRJwKqIWB0RvcDNwAXDlrkAuCG9fxtwliQVoriI2BARj6b3dwArgDmFeO0xdgFwYyQeAKZJOrQIdZwFPBcRB3N2/UGLiF8DW4Y1Z77PbgDekmXVNwF3R8SWiOgA7gaWFKK+iPhFRPSnDx8A5o716+ZqhO2Xi1z+3g/aaPWlnx3vAH4w1q9bKDZFtfgAAATGSURBVKUUEHOANRmP1/LyD+ChZdI/kG1Ac0Gqy5Du2no18GCW2adKelzSnZKOK2hhiQB+IekRSZdlmZ/Ldi6ECxn5D7PY23BWRGxI778EzMqyzHjZju8j6RFms6/3Qj5dke4Cu36EXXTjYfu9DtgYEc+OML+Y2y8npRQQE4KkBuBHwN9ExPZhsx8l2WXyKuCfgZ8Wuj7g9Ig4ETgX+LCkM4pQw6gkVQHnAz/MMns8bMMhkexrGJdjzSV9GugHbhphkWK9F/4VeAVwArCBZDfOePRORu89jPu/pVIKiHXAvIzHc9O2rMtIqgCmAu0FqS55zUqScLgpIn48fH5EbI+IzvT+MqBS0oxC1Ze+7rr0dhPwE5KufKZctnO+nQs8GhEbh88YD9sQ2Di42y293ZRlmaJuR0mXAG8G3pWG2Mvk8F7Ii4jYGBEDEbEb+LcRXrfY268CeCtwy0jLFGv77Y9SCoiHgUWSDk//w7wQWDpsmaXA4GiRtwP3jvTHMdbS/ZX/DqyIiK+MsMwhg8dEJJ1E8vsrZIDVS2ocvE9yMPPJYYstBd6bjmY6BdiWsTulUEb8z63Y2zCV+T67GLg9yzJ3AedIakp3oZyTtuWdpCXAJ4HzI6JrhGVyeS/kq77MY1p/McLr5vL3nk9vBFZGxNpsM4u5/fZLsY+SF3IiGWHzDMnohk+nbVeT/CEA1JDsllgFPAQsLGBtp5PsangCeCydzgMuBy5Pl7kCWE4yIuMB4E8LvP0Wpq/9eFrH4DbMrFHAtek2/gOwuMA11pN84E/NaCvaNiQJqg1AH8l+8PeTHNf6JfAscA8wPV12MfCtjHXfl74XVwGXFrC+VST77wffh4Mj+2YDy0Z7LxSovu+m760nSD70Dx1eX/r4ZX/vhagvbf/O4HsuY9mCb7+DnXypDTMzy6qUdjGZmdl+cECYmVlWDggzM8vKAWFmZlk5IMzMLCsHhFkRpVeX/Vmx6zDLxgFhZmZZOSDMciDp3ZIeSq/d/01J5ZI6JX1Vyfd3/FJSS7rsCZIeyPg+haa0/QhJ96QXCnxU0ivSp2+QdFv6HQw3ZZzp/UUl3w/yhKQvFelHtxLmgDDbB0nHAP8dOC0iTgAGgHeRnLXdGhHHAfcDn01XuRH4+4g4nuSM38H2m4BrI7lQ4J+SnIELyZV7/wY4luQM29MkNZNcRuK49Hn+d35/SrOXc0CY7dtZwGuAh9NvBzuL5IN8N3suxvY94HRJU4FpEXF/2n4DcEZ63Z05EfETgIjojj3XOXooItZGcvG5x4AFJJea7wb+XdJbgazXRDLLJweE2b4JuCEiTkinoyLic1mWO9Dr1vRk3B8g+Ta3fpKre95GclXVnx/gc5sdMAeE2b79Eni7pJkw9J3S80n+ft6eLnMR8F8RsQ3okPS6tP09wP2RfEvgWklvSZ+jWlLdSC+Yfi/I1EguSf63wKvy8YOZjaai2AWYjXcR8ZSk/0ny7V9lJFfu/DCwEzgpnbeJ5DgFJJfw/kYaAKuBS9P29wDflHR1+hx/OcrLNgK3S6oh6cF8bIx/LLN98tVczQ6QpM6IaCh2HWb54l1MZmaWlXsQZmaWlXsQZmaWlQPCzMyyckCYmVlWDggzM8vKAWFmZln9f3hVkCwC16e5AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3bCGFZZQeauD",
        "outputId": "e38fc8b8-a851-4954-8567-10501d95f7f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 635
        }
      },
      "source": [
        "parameters = model(X_train, y_train, learning_rate = 0.001, mini_batch_size=128, n_h = 4, optimizer=\"adam\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cost after epoch 0: 0.652625\n",
            "Cost after epoch 1: 0.538039\n",
            "Cost after epoch 2: 0.494766\n",
            "Cost after epoch 3: 0.466445\n",
            "Cost after epoch 4: 0.446931\n",
            "Cost after epoch 5: 0.434147\n",
            "Cost after epoch 6: 0.424947\n",
            "Cost after epoch 7: 0.418039\n",
            "Cost after epoch 8: 0.412807\n",
            "Cost after epoch 9: 0.408511\n",
            "Cost after epoch 10: 0.404447\n",
            "Cost after epoch 11: 0.401307\n",
            "Cost after epoch 12: 0.399088\n",
            "Cost after epoch 13: 0.397500\n",
            "Cost after epoch 14: 0.396359\n",
            "Cost after epoch 15: 0.395433\n",
            "Cost after epoch 16: 0.394425\n",
            "Cost after epoch 17: 0.392661\n",
            "Cost after epoch 18: 0.389971\n",
            "Cost after epoch 19: 0.387413\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3hddZ3v8fc39+aeJuklaUta2gJpgVIiyFUQxOI4oIAIKBYvg6gIHh1nYM6oHDzOox4FRauCDgOMICAqFCwgKBdHpG2AUnoBeqfpNU3apE2apEm+54+1ku6GnXS3zc5O9v68nmc92Xut39r7m5Vkf/Jbt5+5OyIiIn2lJboAEREZnhQQIiISlQJCRESiUkCIiEhUCggREYlKASEiIlEpICQlmNlZZvZWousQGUkUEBJ3ZrbezM5PZA3u/ld3PyaRNfQws3PMrC5B732VmW0wsxYze9TMRg/QdpaZvWJmreHXWRHLzMy+Z2YN4fQ9M7OI5XeZ2Vtm1m1m18T525I4UUBIUjCz9ETXAL0fnMPy78rMZgB3AlcDY4FW4Gf9tM0CHgN+DZQA9wKPhfMBrgU+ApwInAD8I/D5iJd4Hfgi8OqgfyMyZIblL7KkBjNLM7ObzGxN+F/ow5H/0ZrZb81sq5k1mdmL4Qdcz7J7zOznZrbAzFqAc8Oeyj+b2dJwnYfMLCdsf8B/7QO1DZf/i5ltMbPNZvY5M3Mzm9rP9/G8mX3HzP5G8KE7xcw+bWYrzWy3ma01s8+HbfOAJ4EKM9sTThUH2xaD5BPA4+7+orvvAb4BXGJmBVHangNkAD9y93Z3vwMw4P3h8rnAD929zt03AT8ErulZ2d3nufufgbZB/h5kCCkgJJG+TPBf6PuACmAnMC9i+ZPANGAMwX+i9/dZ/yrgO0AB8D/hvMuBOcBkgv9srxng/aO2NbM5wFeB84GpBB+WB3M1wX/VBcAGYDvwYaAQ+DRwu5nNdvcW4EJgs7vnh9PmGLZFLzObZGa7Bpiu6qfGGQT/2QPg7muADmB6P22X+oH34lkazn/Xa4WPZyBJJSPRBUhKuw643t3rAMzsFuAdM7va3Tvd/e6ehuGynWZW5O5N4ezH3P1v4eO2cBf4HeEHLmb2ONC73zyK/tpeDvyXuy+PeO9PHOR7uaenfeiPEY9fMLM/AWfR/y6XAbdFZEN3fwcoPkg90eQDTX3mNRGE2qG27bu8Ccg3M3Pd4C1pqAchiXQU8Iee/3yBlUAXMNbM0s3su+Eul2ZgfbhOWcT6G6O85taIx60EH2T96a9tRZ/XjvY+fR3QxswuNLOXzawx/N4+xIG199XvtojhvWO1h6BHE6kQ2H0YbfsuLwT2KBySiwJCEmkjcKG7F0dMOeE+7auAiwl28xQBVeE6FrF+vD6MtgATIp5PjGGd3lrMLBv4HfADYKy7FwML2F97tLoH2hYHCHcx7Rlg6q+3s5zgoHLP60wBsoG3+2l7QuSZSQS74ZZHLD8xYtmJEcskSSggZKhkmllOxJQB/AL4jpkdBWBm5WZ2cdi+AGgHGoBc4D+GsNaHgU+b2XFmlktwMPdQZBF88NYDnWZ2IXBBxPJtQKmZFUXMG2hbHMDd34k4fhFt6nuspsf9wD9acE1IHnAr8Ht3j9aDeJ6gB3ODmWWb2fXh/L+EX+8DvmpmlWZWAXwNuKdnZTPLCg/6G/t/9vq8GWH0A5OhsgDYGzHdAvwYmA/8ycx2Ay8Dp4bt7yM42LsJWBEuGxLu/iRwB/AcsDrivdtjXH83cANB0Owk6A3Nj1j+JvAbYG24S6mCgbfFoAiPkVxHEBTbCUL4iz3LzexJM/u3sG0HwUHzTwG7gM8AHwnnQ3C67OPAG8AygmMud0a83Z8Ifs6nA3eFj88ezO9H4s+0y1BkYGZ2HMGHYHbfA8YiyUw9CJEozOyj4a6VEuB7BNcPKBwkpSggRKL7PMFumDUE++K/kNhyRIaedjGJiEhU6kGIiEhUSXMldVlZmVdVVSW6DBGREeWVV17Z4e7l0ZYlTUBUVVVRW1ub6DJEREYUM9vQ3zLtYhIRkagUECIiEpUCQkREolJAiIhIVAoIERGJSgEhIiJRKSBERCSquAaEmc0xs7fMbLWZ3dRPm8vNbIWZLTezByLmd5nZknCaH23dwdDUuo8fP7uKN+r6jq4oIpLa4nahnJmlEwy6/gGgDlhsZvPdfUVEm2nAzcAZ7r7TzMZEvMRedx9oPOFBkZYGtz/7NhnpxvETig6+gohIiohnD+IUYLW7rw0HGXmQYAjJSP8EzHP3nQDuvj2O9URVkJPJUaW5rNjcPNRvLSIyrMUzICo5cCD3unBepOnAdDP7WzjA+5yIZTlmVhvO/0i0NzCza8M2tfX19YddaPX4QlZsUUCIiERK9EHqDGAacA5wJfBLMysOlx3l7jUEwzX+yMyO7ruyu9/l7jXuXlNeHvVeUzGpHl/I+oYW9rRrPBgRkR7xDIhNwMSI5xPCeZHqgPnuvs/d1wFvEwQG7r4p/LqWYAD1k+JVaHVFIe7w1lb1IkREesQzIBYD08xsspllAVcQMXB76FGC3gNmVkawy2mtmZWYWXbE/DMIBq6Pi+qKQgAdhxARiRC3s5jcvdPMrgeeBtKBu919uZndCtS6+/xw2QVmtoJgWMevu3uDmZ0O3Glm3QQh9t3Is58G27jCHEpyM3UcQkQkQlzHg3D3BcCCPvO+GfHYga+GU2Sbl4Dj41lbJDOjuqJQPQgRkQiJPkg9bFSPL+TNrbvp7OpOdCkiIsOCAiJUXVFIe2c363a0JLoUEZFhQQERqh4fXEWt4xAiIgEFRGhKeR5ZGWk6DiEiElJAhDLT0zhmbIF6ECIiIQVEhOrxwZlMwclVIiKpTQERobqikIaWDrbvbk90KSIiCaeAiKArqkVE9lNARDh2XAGgM5lEREABcQCNDSEisp8Cog+NDSEiElBA9KGxIUREAgqIPjQ2hIhIQAHRh85kEhEJKCD60NgQIiIBBUQfGhtCRCSggIhCY0OIiCggouoZG2KtxoYQkRSmgIiid2wI7WYSkRSmgIiid2wIHagWkRSmgIiid2wI9SBEJIUpIPrRc8sNjQ0hIqlKAdGP6opCGls62NassSFEJDUpIPrRe0X1lqYEVyIikhgKiH70jg2h4xAikqIUEP3oHRtCZzKJSIpSQAygerxuuSEiqUsBMYBgbIhWjQ0hIikprgFhZnPM7C0zW21mN/XT5nIzW2Fmy83sgYj5c81sVTjNjWed/ek5UP2mdjOJSArKiNcLm1k6MA/4AFAHLDaz+e6+IqLNNOBm4Ax332lmY8L5o4FvATWAA6+E6+6MV73R7D+TqZmaqtFD+dYiIgkXzx7EKcBqd1/r7h3Ag8DFfdr8EzCv54Pf3beH8z8IPOPujeGyZ4A5caw1qt6xIXQcQkRSUDwDohLYGPG8LpwXaTow3cz+ZmYvm9mcQ1gXM7vWzGrNrLa+vn4QS+99/WBsCO1iEpEUlOiD1BnANOAc4Ergl2ZWHOvK7n6Xu9e4e015eXlcCtTYECKSquIZEJuAiRHPJ4TzItUB8919n7uvA94mCIxY1h0S1RWFdGhsCBFJQfEMiMXANDObbGZZwBXA/D5tHiXoPWBmZQS7nNYCTwMXmFmJmZUAF4TzhpzGhhCRVBW3gHD3TuB6gg/2lcDD7r7czG41s4vCZk8DDWa2AngO+Lq7N7h7I/BtgpBZDNwazhtyGhtCRFJV3E5zBXD3BcCCPvO+GfHYga+GU9917wbujmd9sdDYECKSqhJ9kHpE0NgQIpKKFBAx0NgQIpKKFBAx0NgQIpKKFBAx0NgQIpKKFBAx0NgQIpKKFBAx0tgQIpJqFBAx0tgQIpJqFBAx0tgQIpJqFBAxihwbQkQkFSggYqSxIUQk1SggYqSxIUQk1SggDoHGhhCRVKKAOAQaG0JEUokC4hBobAgRSSUKiEOgsSFEJJUoIA6BxoYQkVSigDhEGhtCRFKFAuIQaWwIEUkVCohDpLEhRCRVKCAOkcaGEJFUoYA4RAU5mVSV5rJcASEiSU4BcRh0yw0RSQUKiMNQPb6QDQ2t7G7bl+hSRETiRgFxGHrHhti6O8GViIjEjwLiMOiWGyKSChQQh2FsYTaj87IUECKS1BQQh8HMeq+oFhFJVgqIw1RdUchb23azT2NDiEiSUkAcpurx4dgQ9RobQkSSU1wDwszmmNlbZrbazG6KsvwaM6s3syXh9LmIZV0R8+fHs87DoVtuiEiyy4jXC5tZOjAP+ABQByw2s/nuvqJP04fc/fooL7HX3WfFq74jNaUsHBticzMfPSnR1YiIDL549iBOAVa7+1p37wAeBC6O4/sNqYz0NI4dV6AD1SKStOIZEJXAxojndeG8vi41s6Vm9oiZTYyYn2NmtWb2spl9JNobmNm1YZva+vr6QSw9NtXjC1mxWWNDiEhySvRB6seBKnc/AXgGuDdi2VHuXgNcBfzIzI7uu7K73+XuNe5eU15ePjQVR6iuKGRn6z62NrcN+XuLiMRbPANiExDZI5gQzuvl7g3u3jPyzq+AkyOWbQq/rgWeB4bdnv7q8eGBal0wJyJJKJ4BsRiYZmaTzSwLuAI44GwkMxsf8fQiYGU4v8TMssPHZcAZQN+D2wl3rAJCRJJY3M5icvdOM7seeBpIB+529+VmditQ6+7zgRvM7CKgE2gErglXPw6408y6CULsu1HOfkq4/OwMqkpzdaBaRJJS3AICwN0XAAv6zPtmxOObgZujrPcScHw8axss1RWFGjxIRJJSog9Sj3gaG0JEkpUC4ghpbAgRSVYKiCPUMzbEwrUNCa5ERGRwKSCO0LiiHM6cWsY9L61nb0dXossRERk0CohBcOP509ixp4P7F25IdCkiIoNGATEI3lM1mtOPLuUXL6xVL0JEkoYCYpDceN40duxp54FF7yS6FBGRQaGAGCSnTinltCml/OKFNbTtUy9CREa+mALCzD4Wy7xUd+P506jf3c4DC9WLEJGRL9YexLuudu5nXkp775RSTp08Wr0IEUkKAwaEmV1oZj8BKs3sjojpHoL7J0kfN54/je2723lQxyJEZIQ7WA9iM1ALtAGvREzzgQ/Gt7SR6bQppZxSNZqfqxchIiPcgAHh7q+7+73AVHe/N3w8n2Ao0Z1DUuEIY2bceP40tjW383DtxoOvICIyTMV6DOIZMys0s9HAq8Avzez2ONY1op1+dCnvqSrhZ8+tob1TvQgRGZliDYgid28GLgHuc/dTgfPiV9bIZmbceN50tja38fBi9SJEZGSKNSAywtHfLgeeiGM9SeOMqaWcfFQJP3tevQgRGZliDYhbCUaGW+Pui81sCrAqfmWNfEEvYhpbmtr4bW1dossRETlkMQWEu//W3U9w9y+Ez9e6+6XxLW3kO2taGSdNKubnz6+ho7M70eWIiBySWK+knmBmfzCz7eH0OzObEO/iRrqeXsSmXXt55BX1IkRkZIl1F9N/EZzeWhFOj4fz5CDeN72cEycWM++51epFiMiIEmtAlLv7f7l7ZzjdA5THsa6kYWZ85fygF/H7V9WLEJGRI9aAaDCzT5pZejh9EtAYmzE6Z3o5J04o4qfPrWZfl3oRIjIyxBoQnyE4xXUrsAW4DLgmTjUlnZ6rq+t2qhchIiPHoZzmOtfdy919DEFg/J/4lZV8zj1mDCeoFyEiI0isAXFC5L2X3L0ROCk+JSUnM+OG909jY+Ne/vDapkSXIyJyULEGRJqZlfQ8Ce/JlBGfkpLXeceNYWZlIfOeW02nehEiMszFGhA/BP5uZt82s28DLwHfj19ZyannHk0bGlp5dMnmRJcjIjKgWK+kvo/gRn3bwukSd//veBaWrM4/bgwzKgr56V9WqRchIsNarD0I3H2Fu/80nFbEso6ZzTGzt8xstZndFGX5NWZWb2ZLwulzEcvmmtmqcJoba53DnZlxw3nTWN/QymPqRYjIMBZzQBwqM0sH5gEXAtXAlWZWHaXpQ+4+K5x+Fa47GvgWcCpwCvCtyGMgI90F1WM5bnwhP9WxCBEZxuIWEAQf7KvDG/t1AA8CF8e47geBZ9y9MTx76hlgTpzqHHLBsYiprNvRwuNL1YsQkeEpngFRCUSOllMXzuvrUjNbamaPmNnEQ1nXzK41s1ozq62vrx+suofEBdXjOHZcAT/5y2q6uj3R5YiIvEs8AyIWjwNV7n4CQS/h3kNZ2d3vcvcad68pLx9Zt4ZKSwuORaytb+EJ9SJEZBiKZ0BsAiZGPJ8Qzuvl7g3u3h4+/RVwcqzrJoM5M8ZxzNgC7vjzKvUiRGTYiWdALAammdlkM8sCriC4ZXivcBjTHhcBK8PHTwMXmFlJeHD6gnBeUunpRaypb+F3ukeTiAwzcbsa2t07zex6gg/2dOBud19uZrcCte4+H7jBzC4COoFGwhsAuntjeEHe4vDlbg1v75F0Lpw5jpqjSrhl/nJmTyph6pj8RJckIgKAuSfHro2amhqvra1NdBmHZUvTXv7hjv+hPD+bR790BqOy0hNdkoikCDN7xd1roi1L9EFqAcYXjeJHH5/F29t3843HliW6HBERQAExbJw9vZwvnzuVR16p4+HajQdfQUQkzhQQw8iN50/n9KNL+cajy1i5pTnR5YhIilNADCPpacaPrziJwlGZfOn+V9ndti/RJYlIClNADDPlBdn85MqTWN/Qws2/f4NkOYlAREYeBcQw9N4ppXztgmN4YukWfv3yhkSXIyIpSgExTH3hfUdz7jHlfPuJlSyt25XockQkBSkghqm0NOO2y2dRlp/FF+9/laZWHY8QkaGlgBjGSvKy+OknZrO1qY2v/fZ1HY8QkSGlgBjmZk8q4eYPHcezK7fxq7+uS3Q5IpJCFBAjwGfOqGLOjHF896k3qV2flLekEpFhSAExApgZ3//YCVQWj+L6B16jYU/7wVcSETlCCogRojAnk599YjaNrR185aEldGv8CBGJMwXECDKzsohb/nEGf121g3nPrU50OSKS5BQQI8yVp0zkI7MquP3Zt3lp9Y5ElyMiSUwBMcKYGd/56PFMLsvjhgdfY3tzW6JLEpEkpYAYgfKyM/j5J0+mpb2LL//mNTq7uhNdkogkIQXECDV9bAHf+ehMFq5r5PZn3050OSKShBQQI9glsydwxXsmMu+5Nfxx6ZZElyMiSUYBMcLdctEMZk8q5su/eZXfLHon0eWISBJRQIxwOZnp/Ppzp3L29HJu/v0b/OTPq3TPJhEZFAqIJJCblcEvP1XDJSdV8sNn3uZb85fTpQvpROQIZSS6ABkcmelp/OBjJ1JWkM1dL66loaWD2y4/keyM9ESXJiIjlAIiiaSlGf/2oeMoy8/iPxa8yc6WDu68+mQKcjITXZqIjEDaxZSErj37aG67/EQWrWvkyl++TP1u3dxPRA6dAiJJXTJ7Ar+cW8Oa7S1c9ouXeKehNdElicgIo4BIYuceM4b7/+lUmvbu45Kfv8TyzU2JLklERhAFRJKbPamER647jax04+N3vszf1zQkuiQRGSHiGhBmNsfM3jKz1WZ20wDtLjUzN7Oa8HmVme01syXh9It41pnspo4p4HdfPJ3xRTnMvXsRT76hq65F5ODiFhBmlg7MAy4EqoErzaw6SrsC4EZgYZ9Fa9x9VjhdF686U8X4olH89rrTOH5CEV984FV+/fKGRJckIsNcPHsQpwCr3X2tu3cADwIXR2n3beB7gO5bHWfFuVn8+rOn8v5jxvDvjy7jR8++rauuRaRf8QyISmBjxPO6cF4vM5sNTHT3P0ZZf7KZvWZmL5jZWdHewMyuNbNaM6utr68ftMKT2aisdO68+mQ+dvIEfvTsKr7x2DJddS0iUSXsQjkzSwNuA66JsngLMMndG8zsZOBRM5vh7s2Rjdz9LuAugJqaGn3KxSgjPY3vX3YCZQXZ/Pz5NTTs6eD2j88iJ1NXXYvIfvHsQWwCJkY8nxDO61EAzASeN7P1wHuB+WZW4+7t7t4A4O6vAGuA6XGsNeWYGf8651i+8eFqnly2lQ/9+K+8+LZ6YSKyXzwDYjEwzcwmm1kWcAUwv2ehuze5e5m7V7l7FfAycJG715pZeXiQGzObAkwD1sax1pT12TMnc99nTsGBT929iM//dy0bG3VRnYjEMSDcvRO4HngaWAk87O7LzexWM7voIKufDSw1syXAI8B17t4Yr1pT3dnTy3nqK2fx9Q8ew4tv7+D8217gjj+vom1fV6JLE5EEsmQ5i6WmpsZra2sTXcaIt3nXXr6zYCV/XLqFSaNz+eaHqzm/emyiyxKRODGzV9y9JtoyXUktB6goHsW8q2bzwOdOJTsjjc/dV8tn7lnM+h0tiS5NRIaYAkKiOn1qGQtuPIt//4fjWLSukQtuf5EfPP0WrR2diS5NRIaIAkL6lZmexufOmsJfvvY+PnzCeH763GrO/+ELPPnGFl1gJ5ICFBByUGMKc7jt47P47XWnUTgqky/c/ypX/+ciVm/fnejSRCSOFBASs/dUjeaJL5/JrRfPYGndLub86K/8x4KV7GnXbieRZKSAkEOSkZ7Gp06r4i//fA6Xzp7AXS+u5f0/eJ57X1qvoBBJMjrNVY7Iko27uPXx5bz6zi4KsjO4rGYCc0+roqosL9GliUgMBjrNVQEhg+K1d3Zyz0vrWfDGFvZ1OeceU841Z0zmrKllpKVZossTkX4oIGTIbG9u4/6F73D/wnfYsaedKWV5zD29iktPnkB+dsLuDSki/VBAyJDr6OxmwRtbuOel9SzZuIv87AwuO3kCc0+vYrJ2P4kMGwoISaglG3dx70vreWLpZvZ1OeccU841p1dx9rRy7X4SSTAFhAwL23e38ZuFG/n1wg3U725nclkec087iktPnkBBTmaiyxNJSQoIGVY6Ort5clmw++m1d4LdTxfOHMecmeM4Y2qZBi4SGUIKCBm2Xt+4i/v+voE/rdjK7rZO8rMzOPfYMcyZMY5zjiknTwe2ReJqoIDQX58k1IkTi/nhxGI6Oo/npTU7eHr5Vv60fBuPv76ZrIw0zp5WzoUzx3H+cWMpytVuKJGhpB6EDDtd3c7i9Y08tWwrTy/fypamNjLSjNOOLmXOzHF8oHosYwpyEl2mSFLQLiYZsdyd1+uaeGrZVp5atoX1Da2YQc1RJXxwRnDcYkJJbqLLFBmxFBCSFNydt7ft4cllW3hq2Vbe3BrcTfb4yiLeN72cU6eMZvakEh23EDkECghJSut3tPD08q08tXwrS+ua6Op20tOMmZVFvHfyaE6ZPJqaqtEUjdKxC5H+KCAk6e1p7+TVDTtZuK6BResaeX1jEx1d3ZjBceMKOWXyaE4NQ6M0PzvR5YoMGwoISTlt+7p47Z1dLFrXyKL1DbyyYSdt+7oBmDomvzcwTp1cyrgiHfCW1KWAkJTX0dnNG5uagsBY10Dt+p3sDsevmDQ6l5MmFXN8ZRHHVxYxo7JINxaUlKGAEOmjq9tZuaWZhesaWbi2gaV1TWxtbgPADCaX5fUGxszKImZUFOp2IJKUFBAiMajf3c6yTU28samJpXVNLNuk0JDkpyupRWJQXpDNuceO4dxjx/TOiwyNnl1Ujy3Z3Lt8SlkeMyuLmFlZyIyKIqrHF1KSl5WI8kUGnQJCZACxhMbi9Y3Mf31/aFQU5VBdUUh1GBgzKgqZUDIKM93aXEYWBYTIIYoWGg172lm5ZTfLNzexYkszyzc385c3t9Md7sEtyMkIw6KI6oogNKaOySczPS1B34XIwSkgRAZBaX42Z07L5sxpZb3z9nZ08da2MDQ2B6HxwKINvafbZqWnMX1cPtXjC5k+toCq0jyqyvKYNDqXrAwFhyReXAPCzOYAPwbSgV+5+3f7aXcp8AjwHnevDefdDHwW6AJucPen41mryGAblZXOrInFzJpY3Duvq9tZt2MPyzc3s2JLMys2N/Psyu08XFvX2ybNYEJJLpPL8phclkdVaS6Ty/OZXJpHZcko0jUKnwyRuAWEmaUD84APAHXAYjOb7+4r+rQrAG4EFkbMqwauAGYAFcCzZjbd3bviVa/IUEhPM6aOKWDqmAIunlXZO39nSwfrGlpYv6OFdeG0vqGF2vWNtHTs/7XPSk9j4uhR+8Mj/DqhOJcxhdkabEkGVTx7EKcAq919LYCZPQhcDKzo0+7bwPeAr0fMuxh40N3bgXVmtjp8vb/HsV6RhCnJy6IkL4vZk0oOmO/u1O9pZ119EBjrdrSybsce1u9o5a+rdtDe2X1A++LcTMYV5jCmMIdxhdmM7X2cw9jCbMYV5lCan61eiMQkngFRCWyMeF4HnBrZwMxmAxPd/Y9m9vU+677cZ91KRFKMmTGmIIcxBTmcOqX0gGXd3c6W5jbW72hh0669bG9uY1tzO1ub29je3MZbW5up393ee6C8R3qaUZ6fzdgwQMYW5lCan0VJbhbFuZmU5O5/XJybSX52hs7ASlEJO0htZmnAbcA1R/Aa1wLXAkyaNGlwChMZIdLSjMriUVQWj+q3TVe3s2NPO9v6hMfWpja27W5nQ0MrC9c10rR3X7+vkZluFI3KoiQMj54QKc7LpDicXzQqk7zsDPJzMsjPDqecDPKyMtRbGcHiGRCbgIkRzyeE83oUADOB58P/TsYB883sohjWBcDd7wLuguBK6sEsXiQZpKdZby9hIJ1d3ezau49drR3sbN3HrtZ97GztiHjewc6WYN6GhlaWbNzFrtZ9dHR1D/i6ALlZ6e8KjfycDAqyM3pDJS8rnVFZGeRmpZOblc6ozHRyszIYFT7PzUoPH2eQm5lOmkJnSMQzIBYD08xsMsGH+xXAVT0L3b0J6D0n0MyeB/7Z3WvNbC/wgJndRnCQehqwKI61iqS0jPQ0yvKzKTuEW6G7O60dXexs7WB3Wyd72sOprZOW9gOf74l43tLeycbGVlo69i/b13Vo/9/lZKYFAZIZBkh2EDK5WRnkZYdBkpUezIuyrPdrVga52ekU5mTq1OIo4hYQ7t5pZtcDTxOc5nq3uy83s1uBWnefP8C6y83sYYID2p3Al3QGk8jwYmbkhb2AI9XR2c3eji5a93XS2tEVPO7oorWjc//jfV3s7ei7vIu94Tqt7SE+a2UAAAjfSURBVF1sadpLa0cXLe3BvJaOTmK93VxBdgaj87MYnZdFaV7wdXRe9v7H+fvnl+ZlMyor+c8Y0836RCRpuTtt+7pp6eiktT0IjNaOTlrau3q/tnR00tS6j4aWDhrDKXjcTmNLR7+9m1GZ6UFY5GcxpiCHCSWjwimXCSWjmDg6d0SMZqib9YlISjIzRoXHL8g/9PXdnd3tnTTu6YgIkPbg8Z79YVK3s5W/r9lxwDUrENxipScwIsOj5/FwDxAFhIhIP8yMwpxMCnMyqSrLG7Ctu9O0dx91O/dSt7OVjY3B17qde3mnoZWXVvcfIJXFQWhUFOdQWZwbfC0ZRVledkIPyCsgREQGgZlRnJtFcW4WMyuL3rXc3dnVuj9AeoMk/LpwbUPvKIc9sjLSqCjKoSI8nbmieBSVJaN6T28eX5xDdkb8joUoIEREhoCZ9V4xf/yEdwcIQHPbPjbt3MvmXXvZ1DOFz19cVc/23e3vOuheXpDNe6eU8pMrTxr0mhUQIiLDRGFOJoXjMzlufGHU5R2d3WxtausNj81hgJTmx2eQKgWEiMgIkZWRxqTSXCaV5g7J++nKEBERiUoBISIiUSkgREQkKgWEiIhEpYAQEZGoFBAiIhKVAkJERKJSQIiISFRJc7tvM6sHNhzBS5QBOwapnHhQfUdG9R0Z1XdkhnN9R7l7ebQFSRMQR8rMavu7J/pwoPqOjOo7MqrvyAz3+vqjXUwiIhKVAkJERKJSQOx3V6ILOAjVd2RU35FRfUdmuNcXlY5BiIhIVOpBiIhIVAoIERGJKqUCwszmmNlbZrbazG6KsjzbzB4Kly80s6ohrG2imT1nZivMbLmZ3RilzTlm1mRmS8Lpm0NVX0QN683sjfD9a6MsNzO7I9yGS81s9hDWdkzEtlliZs1m9pU+bYZ0G5rZ3Wa23cyWRcwbbWbPmNmq8GtJP+vODdusMrO5Q1jf/zOzN8Of3x/MrLifdQf8XYhjfbeY2aaIn+GH+ll3wL/3ONb3UERt681sST/rxn37HTF3T4kJSAfWAFOALOB1oLpPmy8CvwgfXwE8NIT1jQdmh48LgLej1HcO8ESCt+N6oGyA5R8CngQMeC+wMIE/760EFwElbBsCZwOzgWUR874P3BQ+vgn4XpT1RgNrw68l4eOSIarvAiAjfPy9aPXF8rsQx/puAf45hp//gH/v8aqvz/IfAt9M1PY70imVehCnAKvdfa27dwAPAhf3aXMxcG/4+BHgPDOzoSjO3be4+6vh493ASqByKN57kF0M3OeBl4FiMxufgDrOA9a4+5FcXX/E3P1FoLHP7Mjfs3uBj0RZ9YPAM+7e6O47gWeAOUNRn7v/yd07w6cvAxMG+31j1c/2i0Usf+9HbKD6ws+Oy4HfDPb7DpVUCohKYGPE8zre/QHc2yb8A2kCSoekugjhrq2TgIVRFp9mZq+b2ZNmNmNICws48Ccze8XMro2yPJbtPBSuoP8/zERvw7HuviV8vBUYG6XNcNmOnyHoEUZzsN+FeLo+3AV2dz+76IbD9jsL2Obuq/pZnsjtF5NUCogRwczygd8BX3H35j6LXyXYZXIi8BPg0aGuDzjT3WcDFwJfMrOzE1DDgMwsC7gI+G2UxcNhG/byYF/DsDzX3Mz+N9AJ3N9Pk0T9LvwcOBqYBWwh2I0zHF3JwL2HYf+3lEoBsQmYGPF8QjgvahszywCKgIYhqS54z0yCcLjf3X/fd7m7N7v7nvDxAiDTzMqGqr7wfTeFX7cDfyDoykeKZTvH24XAq+6+re+C4bANgW09u93Cr9ujtEnodjSza4APA58IQ+xdYvhdiAt33+buXe7eDfyyn/dN9PbLAC4BHuqvTaK236FIpYBYDEwzs8nhf5hXAPP7tJkP9Jwtchnwl/7+OAZbuL/yP4GV7n5bP23G9RwTMbNTCH5+QxlgeWZW0POY4GDmsj7N5gOfCs9mei/QFLE7Zaj0+59bordhKPL3bC7wWJQ2TwMXmFlJuAvlgnBe3JnZHOBfgIvcvbWfNrH8LsSrvshjWh/t531j+XuPp/OBN929LtrCRG6/Q5Loo+RDORGcYfM2wdkN/zucdyvBHwJADsFuidXAImDKENZ2JsGuhqXAknD6EHAdcF3Y5npgOcEZGS8Dpw/x9psSvvfrYR092zCyRgPmhdv4DaBmiGvMI/jAL4qYl7BtSBBUW4B9BPvBP0twXOvPwCrgWWB02LYG+FXEup8JfxdXA58ewvpWE+y/7/k97DmzrwJYMNDvwhDV99/h79ZSgg/98X3rC5+/6+99KOoL59/T8zsX0XbIt9+RTrrVhoiIRJVKu5hEROQQKCBERCQqBYSIiESlgBARkagUECIiEpUCQiSBwrvLPpHoOkSiUUCIiEhUCgiRGJjZJ81sUXjv/jvNLN3M9pjZ7RaM3/FnMysP284ys5cjxlMoCedPNbNnwxsFvmpmR4cvn29mj4RjMNwfcaX3dy0YH2Spmf0gQd+6pDAFhMhBmNlxwMeBM9x9FtAFfILgqu1ad58BvAB8K1zlPuBf3f0Egit+e+bfD8zz4EaBpxNcgQvBnXu/AlQTXGF7hpmVEtxGYkb4Ov83vt+lyLspIEQO7jzgZGBxODrYeQQf5N3svxnbr4EzzawIKHb3F8L59wJnh/fdqXT3PwC4e5vvv8/RInev8+Dmc0uAKoJbzbcB/2lmlwBR74kkEk8KCJGDM+Bed58VTse4+y1R2h3ufWvaIx53EYzm1klwd89HCO6q+tRhvrbIYVNAiBzcn4HLzGwM9I4pfRTB389lYZurgP9x9yZgp5mdFc6/GnjBg1EC68zsI+FrZJtZbn9vGI4LUuTBLcn/F3BiPL4xkYFkJLoAkeHO3VeY2b8TjP6VRnDnzi8BLcAp4bLtBMcpILiF9y/CAFgLfDqcfzVwp5ndGr7GxwZ42wLgMTPLIejBfHWQvy2Rg9LdXEUOk5ntcff8RNchEi/axSQiIlGpByEiIlGpByEiIlEpIEREJCoFhIiIRKWAEBGRqBQQIiIS1f8H9gWy9NFW2Z0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QtXTwgVI1ZAO"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}