Chapter_7.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOguwsmHsJdmAivufI2m+6J"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "RmiQnLG_y7qt"
      },
      "outputs": [],
      "source": [
        "# import necessary packages\n",
        "import math, argparse\n",
        "import numpy as np\n",
        "from numpy import random\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from sklearn import datasets\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# set seed to ensure reproducibility\n",
        "SEED = 8\n",
        "random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "np.random.seed(SEED)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data"
      ],
      "metadata": {
        "id": "AEaUSzBGyyaU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# prepare dummy data\n",
        "X, y = datasets.make_regression(n_samples=100, n_features=20, noise=80, random_state=SEED)\n",
        "# apply the bias trick\n",
        "X = np.c_[np.ones(X.shape[0]), X] \n",
        "# train test split\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=SEED)\n",
        "n_samples, n_features = X_train.shape\n"
      ],
      "metadata": {
        "id": "uhqT1jRyyUTf"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model "
      ],
      "metadata": {
        "id": "KhqxUFY1O7xg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Linear regression using gradient descent from scratch"
      ],
      "metadata": {
        "id": "XwgsXwAVgV1W"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Without ridge penalty"
      ],
      "metadata": {
        "id": "3fq_hlN4iKIq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class GD_regression:\n",
        "    def __init__(self, lr=1e-3, num_epochs=1000, l2_penalty=0):\n",
        "        self.lr = lr\n",
        "        self.num_epochs = num_epochs\n",
        "        self.l2_penalty = l2_penalty\n",
        "        self.train_loss = []\n",
        "        self.test_loss = []\n",
        "\n",
        "    # function to estimate model coefficients\n",
        "    def fit(self, X_train, y_train, X_test, y_test):\n",
        "        # initialize weights to be an array of zeros\n",
        "        weights = np.zeros(X_train.shape[1])\n",
        "        # iteration counter\n",
        "        epoch = 0 \n",
        "        # while not reaching maximum number of iterations:\n",
        "        while epoch < self.num_epochs:\n",
        "            # increment iteration counter\n",
        "            epoch += 1  \n",
        "            # compute the predictions based on current weights\n",
        "            predictions = np.dot(X_train, weights)\n",
        "            # compute the errors as predictions - output\n",
        "            errors = predictions.reshape(-1,1) - y_train.reshape(-1,1)\n",
        "            # coordinate descent over each weight\n",
        "            for i in range(len(weights)): \n",
        "                # compute the derivative for the ith weight\n",
        "                if i == 0:\n",
        "                    # compute the derivative of the constant weight when i=0\n",
        "                    derivative = self.feature_derivative_ridge(errors, X_train[:,i].reshape(1,-1), \n",
        "                                            weights[i], self.l2_penalty, True)\n",
        "                else:\n",
        "                    # compute the derivative of all other weights\n",
        "                    derivative = self.feature_derivative_ridge(errors, X_train[:,i].reshape(1,-1), \n",
        "                                            weights[i], self.l2_penalty, False)\n",
        "                # gradient descent update for each coordinate/weight\n",
        "                weights[i] = weights[i] - self.lr * derivative\n",
        "\n",
        "            # score using training and test sets\n",
        "            self.train_loss.append(self.calculate_mse(X_train, y_train, weights))\n",
        "            self.test_loss.append(self.calculate_mse(X_test, y_test, weights))\n",
        "        # store the final weights\n",
        "        self.weights = weights\n",
        "        return self\n",
        "\n",
        "    # function to calculate the mean squared error\n",
        "    def calculate_mse(self, X, y, weights):\n",
        "        # generate model prediction\n",
        "        f_star = X@weights\n",
        "        # calculate the residuals\n",
        "        resid = y - f_star\n",
        "        # MSE as the final cost \n",
        "        Q = np.dot(resid.T, resid).mean()\n",
        "        return Q\n",
        "\n",
        "    # function to calculate feature-wise closed-form derivative\n",
        "    def feature_derivative_ridge(self, errors, feature, weight, l2_penalty, feature_is_constant):\n",
        "        # compute the derivative of the weight corresponding to the bias term\n",
        "        if feature_is_constant == True:\n",
        "            derivative = np.dot(feature, errors) * 2\n",
        "        # compute the derivative of all other weights\n",
        "        elif feature_is_constant == False:\n",
        "            derivative = np.dot(feature, errors) * 2 + 2*l2_penalty*weight\n",
        "        else:\n",
        "            derivative = 'derivative not exist!!!'\n",
        "        return derivative"
      ],
      "metadata": {
        "id": "EcWzuZURyUQx"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = GD_regression(num_epochs=100)\n",
        "model.fit(X_train, y_train, X_test, y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eXyI-gLNyUOK",
        "outputId": "a09be108-caf1-427d-98e6-c9236356f4f0"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<__main__.GD_regression at 0x7f0d55243890>"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_axis = list(range(model.num_epochs))\n",
        "plt.plot(x_axis, model.train_loss, label = \"train\")\n",
        "plt.plot(x_axis, model.test_loss, ':', label = \"test\")\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "id": "vE0t1wB2yUL0",
        "outputId": "5e118330-83c1-4610-89cc-0317c5d0b66c"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxcdZnv8c/T1fuSpNOdQBayTIwghks2AwijoMOQ4FzBlw5XGHF0cIIzinFGuYLrMM44OONlEFEYFFRUEASBCGELJARIAnRCAtk3ErqzdneS3vd+7h+nKlUJCd1Juvt0nfq+X69+dT11Tlc9p071t0//6izm7oiISPrLCrsBERHpGwp0EZGIUKCLiESEAl1EJCIU6CIiEaFAFxGJiFAD3czuMbN9Zraml/NfYWbrzGytmd3X3/2JiKQTC3M/dDP7ENAI3OvuU3qYdzLwIPARdz9gZiPdfd9A9Ckikg5C3UJ39yXA/tT7zGySmT1lZivM7EUzOyM+6e+Bn7r7gfjPKsxFRFIMxjH0u4Dr3H0G8HXgZ/H73wu818xeNrPlZjY7tA5FRAah7LAbSGVmxcAHgT+YWeLuvPj3bGAycCEwFlhiZme5+8GB7lNEZDAaVIFO8B/DQXefepRpVcAr7t4BvGVmmwgC/rWBbFBEZLAaVEMu7l5PENZ/DWCBs+OTHyXYOsfMygmGYLaF0aeIyGAU9m6L9wPLgNPNrMrMrgH+BrjGzFYDa4HL4rM/DdSa2TpgEXC9u9eG0beIyGAU6m6LIiLSdwbVkIuIiJy40D4ULS8v9wkTJoT19CIiaWnFihU17j7iaNNCC/QJEyZQUVER1tOLiKQlM9txrGkachERiQgFuohIRCjQRUQiYrAdKSoi8q46OjqoqqqitbU17Fb6VX5+PmPHjiUnJ6fXP6NAF5G0UlVVRUlJCRMmTCDlnE+R4u7U1tZSVVXFxIkTe/1zGnIRkbTS2tpKWVlZZMMcwMwoKys77v9CFOgiknaiHOYJJ7KMaRfoG/c08KOnN7K/qT3sVkREBpW0C/Rt1Y3cvmgLe+uj/YGIiAxOBw8e5Gc/+1nPMx7h0ksv5eDB/r18Q9oFelFe8DluU1tnyJ2ISCY6VqB3dr57Ji1YsIBhw4b1V1tAGu7lkgj0RgW6iITghhtuYOvWrUydOpWcnBzy8/MpLS1lw4YNbNq0icsvv5zKykpaW1uZN28ec+fOBZKnO2lsbGTOnDlccMEFLF26lDFjxvDYY49RUFBw0r2lXaAXH9pC7wq5ExEJ201/Wsu6XfV9+phnjh7C9/73+485/eabb2bNmjWsWrWKxYsX87GPfYw1a9Yc2r3wnnvuYfjw4bS0tPCBD3yAT37yk5SVlR32GJs3b+b+++/n5z//OVdccQUPP/wwn/nMZ06697QL9KK8GABN7dpCF5HwzZo167B9xW+77TYeeeQRACorK9m8efM7An3ixIlMnRpcaXPGjBls3769T3pJv0DP1Ri6iATebUt6oBQVFR26vXjxYhYuXMiyZcsoLCzkwgsvPOq+5Hl5eYdux2IxWlpa+qQXfSgqInIcSkpKaGhoOOq0uro6SktLKSwsZMOGDSxfvnxAe0u7LfTc7CxyY1k0agxdREJQVlbG+eefz5QpUygoKOCUU045NG327NnceeedvO997+P000/n3HPPHdDe0i7QIRhHb9YYuoiE5L777jvq/Xl5eTz55JNHnZYYJy8vL2fNmjWH7v/617/eZ32l3ZALQGFutnZbFBE5QloGenFetsbQRUSOkJaBXpQX037oIiJHSNNAz9Z+6CIiR0jPQM/VkIuIyJHSM9DzsjXkIiJyhLQM9OK8mPZyEZFQnOjpcwFuvfVWmpub+7ijpLQM9KL4Xi7uHnYrIpJhBnOgp+mBRdl0djvtXd3kZcfCbkdEMkjq6XMvvvhiRo4cyYMPPkhbWxuf+MQnuOmmm2hqauKKK66gqqqKrq4uvvOd77B371527drFRRddRHl5OYsWLerz3tJzCz03fsZFjaOLyC8/Bq//Lrjd1RHUqx8I6vbmoF7zcFC31gX1uvlB3VQb1BvjR3c27O3x6W6++WYmTZrEqlWruPjii9m8eTOvvvoqq1atYsWKFSxZsoSnnnqK0aNHs3r1atasWcPs2bP5yle+wujRo1m0aFG/hDmka6DrBF0iMgg888wzPPPMM0ybNo3p06ezYcMGNm/ezFlnncWzzz7LN77xDV588UWGDh06IP2k5ZBLsa5aJCIJn38ieTuWc3idW3h4nT/08Lqo7PC6JHmird5wd2688Uauvfbad0xbuXIlCxYs4Nvf/jYf/ehH+e53v3tcj30i0noLXSfoEpGBlnr63EsuuYR77rmHxsZGAHbu3Mm+ffvYtWsXhYWFfOYzn+H6669n5cqV7/jZ/tDjFrqZnQbcC5wCOHCXu//4iHkM+DFwKdAMfM7dV/Z9u4HEVYt0Cl0RGWipp8+dM2cOV111Feeddx4AxcXF/Pa3v2XLli1cf/31ZGVlkZOTwx133AHA3LlzmT179qGx9L7WmyGXTuBr7r7SzEqAFWb2rLuvS5lnDjA5/nUOcEf8e7/QGLqIhOnI0+fOmzfvsHrSpElccskl7/i56667juuuu67f+upxyMXddye2tt29AVgPjDlitsuAez2wHBhmZqP6vNu4xGXoNIYuIpJ0XGPoZjYBmAa8csSkMUBlSl3FO0O/zyQ+FG1WoIuIHNLrQDezYuBh4KvuXn8iT2Zmc82swswqqqurT+QhACiMj6E3tWsMXSQTZcJR4ieyjL0KdDPLIQjz37n7H48yy07gtJR6bPy+Ixu8y91nuvvMESNGHHezCXnZMXJipiEXkQyUn59PbW1tpEPd3amtrSU/P/+4fq43e7kYcDew3t1vOcZs84Evm9nvCT4MrXP33cfVyXEq0lWLRDLS2LFjqaqq4mT+y08H+fn5jB079rh+pjd7uZwPXA28aWar4vd9ExgH4O53AgsIdlncQrDb4uePq4sTUKTriopkpJycHCZOnBh2G4NSj4Hu7i8B1sM8Dnypr5rqjaK8GM3aD11E5JC0PFIUdBk6EZEjpW2gF+dpyEVEJFXaBrquKyoicrj0DXRdV1RE5DBpHOgxjaGLiKRI40DXkIuISKq0DfTivGw6upy2Tg27iIhAGge6risqInK4tA30Qp0TXUTkMGkb6IlT6OqDURGRQNoGuq5aJCJyuLQN9GJdV1RE5DBpG+iFubpqkYhIqrQN9MQYus7nIiISSNtA1xi6iMjh0jjQdV1REZFUaRvoubEssrNMW+giInFpG+hmpvO5iIikSNtAh8RFLjTkIiICaR7oRXkxbaGLiMSleaDruqIiIgnpHei6DJ2IyCHpHeh5MZ0+V0QkLs0DPVtHioqIxKV1oBdrDF1E5JC0DvTC3GyaNeQiIgKkeaAX58Vo7+qmvbM77FZEREKX1oGuE3SJiCRFItD1waiISLoHeuIiFzrjoohImgf6ocvQdYTciYhI+NI60IcV5gJwsFmBLiKS1oFeVhQEem1je8idiIiEL60Dvbw4D4CapraQOxERCV9aB3pBboyi3Ji20EVESPNABygrzqO2UVvoIiIRCPRcapu0hS4ikv6BXpRHjYZcRETSP9DLi3Op0ZCLiEjPgW5m95jZPjNbc4zpF5pZnZmtin99t+/bPLay4lz2N7XT3e0D+bQiIoNOb7bQfwXM7mGeF919avzrX0++rd4rK8qjq9upa9HBRSKS2XoMdHdfAuwfgF5OSFlx/OAi7YsuIhmur8bQzzOz1Wb2pJm9/1gzmdlcM6sws4rq6uo+eeIRiYOL9MGoiGS4vgj0lcB4dz8b+Anw6LFmdPe73H2mu88cMWJEHzx1sB866PB/EZGTDnR3r3f3xvjtBUCOmZWfdGe9pCEXEZHASQe6mZ1qZha/PSv+mLUn+7i9VVqYixnUNCjQRSSzZfc0g5ndD1wIlJtZFfA9IAfA3e8EPgX8g5l1Ai3Ap919wPYhjGUZwwtzqdHRoiKS4XoMdHe/sofptwO391lHJ6CsOFfncxGRjJf2R4pCsC+6PhQVkUwXjUDXCbpERKIR6OXFeTqfi4hkvIgEei4NrZ20dXaF3YqISGgiEeg6uEhEJCqBrotFi4hEJNB1sWgRkWgEenmxttBFRCIR6MkxdG2hi0jmikSgF+XGyMvO0r7oIpLRIhHoZqZ90UUk40Ui0CEYR9cYuohkssgEepm20EUkw0Un0Iu0hS4imS06gV6cR21TGwN4KnYRkUElMoFeXpxLR5dT39oZdisiIqGITKAfuraoxtFFJENFJ9CL4gcXaV90EclQkQn0ESVBoO+tbw25ExGRcEQm0E8bXgjAjtrmkDsREQlHZAK9OC+bESV57KhtCrsVEZFQRCbQASaUFbK9RlvoIpKZIhXo48uK2K4tdBHJUJEK9InlRexraKO5Xfuii0jmiVSgjy8LPhjVsIuIZKJIBfqEsiIAfTAqIhkpUoF+aAtduy6KSAaKVKCX5OdQXpzL9hptoYtI5olUoEMw7KI9XUQkE0Uu0LXroohkqsgF+sTyQvbWa9dFEck8kQv08fE9Xd7erw9GRSSzRC7QE7su6oNREck0kQv08eXadVFEMlPkAn1Ifg5lRbk6uEhEMk7kAh1gQnkRb2nIRUQyTCQDfXxZoS50ISIZJ5KBPqGsiN11rbS0d4XdiojIgIlmoJdr10URyTw9BrqZ3WNm+8xszTGmm5ndZmZbzOwNM5ve920enwnxk3RpHF1EMklvttB/Bcx+l+lzgMnxr7nAHSff1smZNKKYLIP1u+vDbkVEZMD0GOjuvgTY/y6zXAbc64HlwDAzG9VXDZ6Iorxs3ntKCSvfPhBmGyIiA6ovxtDHAJUpdVX8vncws7lmVmFmFdXV1X3w1Mc2fXwpqyoP0t3t/fo8IiKDxYB+KOrud7n7THefOWLEiH59rmmnDaOhtZOt1Y39+jwiIoNFXwT6TuC0lHps/L5QTR9fCsDrbx8MuRMRkYHRF4E+H/hsfG+Xc4E6d9/dB497UiaWFTG0IEfj6CKSMbJ7msHM7gcuBMrNrAr4HpAD4O53AguAS4EtQDPw+f5q9nhkZRnTxg1ToItIxugx0N39yh6mO/ClPuuoD007rZQXNlVT39rBkPycsNsREelXkTxSNGH6+GG4w+pKjaOLSPRFOtDPPm0YZrByhwJdRKIv0oE+JD+HySOLeb1S4+giEn2RDnSA6eNKef1tHWAkItEX+UCfNm4YdS0dvKUrGIlIxEU+0KePCw4wWrFDwy4iEm2RD/RJI4oZUZLH4o37wm5FRKRfRT7Qs7KMS95/Cos2VOsKRiISaZEPdIBLp4yipaOLFzZpK11EoisjAn3WxOGUFuaw4M09YbciItJvMiLQs2NZXPL+U3l+wz7aOjXsIiLRlBGBDjB7yqk0tnXy0uaasFsREekXGRPoH5xUzpD8bA27iEhkZUyg52Zn8RdnnsLC9Xvp6OoOux0RkT6XMYEOMGfKKOpaOli2tTbsVkRE+lxGBfqfTy6nJC+bh1ZUhd2KiEify6hAz8+JceU543jizd1U7m8Oux0RkT6VUYEO8PnzJ5BlcPdLb4XdiohIn8q4QB81tICPnz2GB16r5EBTe9jtiIj0mYwLdIC5H/ozWjq6+O3yHWG3IiLSZzIy0E8/tYSLTh/Br5Zup7VDR46KSDRkZKADXPvhSdQ2tfOHisqwWxER6RMZG+jnTBzOzPGl/PfCzRpLF5FIyNhANzO+f/kU6lo6+I8n14fdjojIScvYQAd436ghfOHPJ/JgRRWvbNPRoyKS3jI60AHmfXQyY0sL+OYjb+rUuiKS1jI+0Atzs/n+5VPYWt3ETxdtDbsdEZETlvGBDnDR6SP5xLQx/OT5zSxctzfsdkRETogCPe4HnziLKaOHMu/3r7Npb0PY7YiIHDcFelxBboy7PjuDwrxsvvDrCu3KKCJpR4GeYtTQAu66egZ76lu55tevUd/aEXZLIiK9pkA/wrRxpdz26am8ubOOK+9aTk1jW9gtiYj0igL9KGZPGcXPPzuTrdWNXPE/y9h1sCXslkREeqRAP4YLTx/JvX93DtX1bVz+05dZrgOPRGSQU6C/i1kTh/PgF8+jOC+bq36+nFsXbqKr28NuS0TkqBToPXjfqCHMv+4CLps6hlsXbubKu5azcY92axSRwUeB3gvFednccsXZ/Oivz2bTvgYuve1F/mX+WupatBeMiAweCvReMjM+NWMsi752IVfOOo17l23nw/+1iNue20xds4JdRMLXq0A3s9lmttHMtpjZDUeZ/jkzqzazVfGvL/R9q4NDaVEu/3b5Wcz/8gXMHF/KLc9u4oM3P8cPFqznrZqmsNsTkQxm7u/+IZ+ZxYBNwMVAFfAacKW7r0uZ53PATHf/cm+feObMmV5RUXEiPYN78JUV/j8Y63fXc8firTzx5m66up1ZE4bz1zPH8pdnnsrQwpyw2xORiDGzFe4+82jTepOIs4At7r7N3duB3wOX9WWDx6VhD/zPn8Oah0JrIdX7Rg3htiunseyGj/CN2WdQ3djG9Q+9wYx/e5ar736F3yzbzvaaJnr6wykicrKyezHPGCD1wptVwDlHme+TZvYhgq35f3L3/rlYZ9FIGDYecov75eFP1Mgh+fzDhZP44of/jFWVB3l67V6eWbuH7zy2FoAxwwo4b1IZM8aXMm3cMCaPLCGWZSF3LSJR0pshl08Bs939C/H6auCc1OEVMysDGt29zcyuBf6Pu3/kKI81F5gLMG7cuBk7duzouyUZhNydbTVNLN1Sw9KttSzbVsvB+AeoRbkxzhg1hDNOLeGMU0uYNLKYSSOKGVmSh5mCXkSO7t2GXHoT6OcB/+Lul8TrGwHc/T+OMX8M2O/uQ9/tcU9qDB2gqwOqXoPxHzzxxxhg7s722mZWVR5gdWUd63bXs2F3PfWtnYfmKc7L5rThhYwtLeC00kJGD8vn1KH5nDokn5El+ZSX5FKY25t/rEQkit4t0HuTDK8Bk81sIrAT+DRw1RFPMMrdd8fLjwP9f9XlF2+BF26Geath2Lh+f7q+YGZMLC9iYnkRn5g2FghCfnddK9uqm9hW08i26iYq9zfzdm0zL2+pobn9nZfFK8qNUVacR2lhDqVFuZQW5jK0IIchBTkMyc+mJD+bkvwcivKyKc6LUZibTVFuNgW5MQpzYxTkxMjScI9I5PQY6O7eaWZfBp4GYsA97r7WzP4VqHD3+cBXzOzjQCewH/hcP/YcmH41jJ4KQ8b2+1P1JzNj9LACRg8r4ILJ5YdNc3fqWzrZU9/K7roWqhvaqG5so6ahnf1Nbexv7qC2sZ2t1Y3UNXfQ0NZJbz97zY1lkZeTRX5OjPycrKDOjpGbnRV8xYLvOTEjJ5YV/zKyY1nkZBmxrCyyY0Ysy8jOCr7HzIjFgjrLgvuyzMhKTMsKljdmRlYWZJlhZmRZcDvLgulGYhrxr+C+d0wDMDCStVnq7fgM8duJP2GJx0mug/j3lHmPvq56N8+h2ynPcsz5j373Med/dyf+RzrKo3yDcdGGFuRQVpzX54/b45BLfznpIRd5h65up7Gtk4bWDhrbOmls7aSpvYumtk6a2jpp7eiipaOL5vYuWju6ae3oorWji/bObto6u2nr7KKts5v2zm7au7rp7HLaO7vp6OqmozuoO7qcrsTt7m66u6Gzuxud4kak97744UncMOeME/rZkx1yGdxe+wU07oOLvhl2J6GLZRlDC3IYWjDw+793dztd7nR1O53dwffubqfbg/u7uwludzvuHLo/OKQgebv70H3x2wTTgz8Y8Wnx5wumgRPcmVon5iP+84f+3iTmPzTtiO8py5S6seMp8xw+V+r8KbePcf9h8/ficXrrZP6eapfagfeekf2zl176B/qeNVBXBd1dkBULu5uMlZVlZGHkaBWIhCb9A33ODyGWG+1BQBGRXgj/2PmTlZ0XhHnzftj4VNjdiIiEJv0DPeG5m+CPc6FdJ8gSkcyU/kMuCRd9C86+CnKLwu5ERCQU0dlCLx4J4+KnmNn2ArTWhduPiMgAi06gJzTshfuugOf/PexOREQGVHSGXBJKToFP3wenzQq7ExGRARW9LXSA93wU8kqgqxOe+ibU7wq7IxGRfhfNQE+o2Qgr74Vti8PuRESk30VvyCXVKe+H61YEwzAA+zZA+WQdUSoikRTtLXRIhnnzfvjlbFhwfbj9iIj0k2hvoacqKIU5/wmjpwd1a11wFqSCYeH2JSLSR6K/hZ5gBv/rCih/T1Av+gHc/gFoawi3LxGRPpI5W+hHmnoVlE4M9oYB2PAEjDsPCoeH25eIyAnKnC30I406G879YnC7sRoe/Cy8fGtyus4RLSJpJnMDPVXxCJj7Asy6Nqh3vQ4/mQ67VoXbl4jIcVCgJ5w6BYaOCW53tgfDMaXjg3r94/DE13QmRxEZ1BToRzPuHLj6j8GeMQD7t8LWRZBTGNSv3AULbwqvPxGRo1Cg98b58+DLFcmrIlVvCIZlEh7+e3ji68m6elOw37uIyABSoPdWVspL9Ve3wNWPJOuSU6GwLFnfd0UwRJPw+D/DqvuTdVUFNOzpv15FJCMp0E9U6jVM//L7cNGNyfrS/4Jz4nvQuMPOCjiwPVn/cg4s+2myvvUseOV/grqrEx79EmxZGNSd7VBxD+xbn6x3LIXGfcn563ZCR0vy8bo6M2cvHffgAuEJXR3J1wKC26nnxm9rgKbaZN1yIDjlckJTTfB6JtTvhgM7kvXBt2H/tmS9fxvUbEnW1RuD/9AS9qwJTjmRsGsV7F2XrCtfg71rk/WOpcHPJGx74fB6y0LY82ay3vjU4dPX/+nweu0jycd3hzcfSj5/VyesfuDw99aq+5P9drTC678LlgmCz5BW/gZqNgd1az2s+DXUbg3qlgNQ8cvk69NUC6/dnXzvN1bDa78IXkMINmpe/XlwkXcIXvdX7kqeTO/ADlh+Z3L97N8Gy+8IHgeC133Zz5L/DVdvDH6vWg4G9d51sPT2oE8IXrelP0l+FrZrVVB3tAZ11Qp4+bbgPQRQ+Sq8/GPo7o6vm2XwUsqecNtfCn4+Ydvi5O81wJbngv4TNj0TLG8/UqD3h8kXJy+2YQbXLkkGvjtc9SBM/Zug7mqHCR+CoWODuqMZti1K/hK01sHj/xS8eQCaa4I/CBueCOr6nfDfZ8KaPwZ17Rb4fhmseTio962HH4yFjU8G9Z434f+dEXwmAMHQ0Y/PDt6sEATMT2bCzhVBvWMp3D4rGSLbXgjqxC/55oXB/Ilf6g1PwG3TkiG49pHgD1bil3T174PnT/xSrvgV/Oek5C/h8jvhP8ZBe3NQv3Qr/NupyV+yRT+Af035b2jh9+DfRyXrZ78LP3pvsn76W0E/CU/dCHeen6yf/AbcfXGyXvB1uPeyZP3E1+D3VyXrx/8ZHvq7ZP2nr8Jj/5hSz4Mn/jml/go8/c1kPf/L8Pz3k/VjX4IXfpisH/3HIEQO1f8QhFjCI18M/sAfqufC679J1g//PbzxQLJ+6Jrke8MdHr4mCH2A7o7g5zfFr8Xb2QKPfhG2PhfU7Y3Bsr21JKhbDgb971ga1M21wfJVvhrUjfvg8a8mhyMbdgWvReK9U/d28Hom/oDsfyt4vRN/IGq3wJPXB/cD1GyCp74BdZVBvW89PHVD8LgAe9+Ep2+Exnjg71oVvNbN8T/YO1fAM99K/kGvfBWe+Ta0NQb128uCujO+AbDjJXj2O8HvJATv9We/Cx4P9K3Pw8J/Sb62m5+F51LW5aanYfHNyXrjAljyn8l6/Xx48Rb6lbuH8jVjxgyXXujqdK/b5d5SF9TtLe5bnnM/WBnULQfdK37pXr05qBur3Rf/0H3vuqCu2+n+5I3uu98M6v1vuT/6pWRdvcn9oS+471kb1HvWuj/4Ofd9G4J612r3B65OPn7VCvcHPhs8jrv7268G8yf62b7U/aFr3Ot3B/VbL7k/PNe9sSaoty0Jnr/lYLx+wf1PX3VvawzqrYvcF/xf947WZP30t4PXIVE/9/3k67N1kfsL/3V4vfT2w+tXf5FSL3Zf+dtkve0F9zf+kFIvcV83P1m/9ZL7xqeT9faX3bc8n6x3LAvmSXj7leA1Sah8zX3nypS6IvnauwevZ+K1dnff+bp7zZZkvWu1e+22ZL37DfcDbyfrPWuCdZywd517/Z5kvW+De8O+4HZ3d7C+E+uiqyt4rub9ybp2m3vzgXjd6b5/e/K919kRPHdrQ7I+WJVcd53twXu1vTlZ1+8J3rOJumFvct12tgfv1462oO5oC3rrbE/WTbXB8yTq5v1H1AeS742OtuB9dVhdFyxXom6tf2fd3Z1SNxxetzUeUTclX9uj1YllP1rd2Z58LU4CUOHHyFXzkP41nzlzpldUVITy3CIi6crMVrj7zKNN05CLiEhEKNBFRCJCgS4iEhEKdBGRiFCgi4hEhAJdRCQiFOgiIhGhQBcRiYjQDiwys2pgR48zHl05UNOH7aSLTFzuTFxmyMzlzsRlhuNf7vHuPuJoE0IL9JNhZhXHOlIqyjJxuTNxmSEzlzsTlxn6drk15CIiEhEKdBGRiEjXQL8r7AZCkonLnYnLDJm53Jm4zNCHy52WY+giIvJO6bqFLiIiR1Cgi4hERNoFupnNNrONZrbFzG4Iu5/+YGanmdkiM1tnZmvNbF78/uFm9qyZbY5/Lw271/5gZjEze93MHo/XE83slfg6f8DMcsPusS+Z2TAze8jMNpjZejM7LxPWtZn9U/z9vcbM7jez/CiuazO7x8z2mdmalPuOun4tcFt8+d8ws+nH81xpFehmFgN+CswBzgSuNLMzw+2qX3QCX3P3M4FzgS/Fl/MG4Dl3nww8F6+jaB6wPqX+IfDf7v4e4ABwTShd9Z8fA0+5+xnA2QTLHul1bWZjgK8AM919ChADPk001/WvgNlH3Hes9TsHmBz/mgvcwXFIq0AHZgFb3H2bu7cDvwcu6+Fn0o6773b3lfHbDQS/4GMIlvXX8dl+DVweTof9x8zGAh8DfhGvDfgI8FB8lkgtt5kNBT4E3A3g7u3ufpAMWNdANlBgZl1bstgAAAI0SURBVNlAIbCbCK5rd18C7D/i7mOt38uAe+OXD10ODDOzUfRSugX6GKAypa6K3xdZZjYBmAa8Apzi7rvjk/YAp4TUVn+6Ffi/QPxS65QBB929M15HbZ1PBKqBX8aHmX5hZkVEfF27+07gR8DbBEFeB6wg2us61bHW70llXLoFekYxs2LgYeCr7l6fOi1+9e9I7XNqZn8F7HP3FWH3MoCygenAHe4+DWjiiOGViK7rUoKt0YnAaKCIdw5LZIS+XL/pFug7gdNS6rHx+yLHzHIIwvx37v7H+N17E/9+xb/vC6u/fnI+8HEz204wnPYRgvHlYfF/yyF667wKqHL3V+L1QwQBH/V1/RfAW+5e7e4dwB8J1n+U13WqY63fk8q4dAv014DJ8U/Ccwk+RJkfck99Lj5ufDew3t1vSZk0H/jb+O2/BR4b6N76k7vf6O5j3X0Cwbp93t3/BlgEfCo+W6SW2933AJVmdnr8ro8C64j4uiYYajnXzArj7/fEckd2XR/hWOt3PvDZ+N4u5wJ1KUMzPXP3tPoCLgU2AVuBb4XdTz8t4wUE/4K9AayKf11KMJ78HLAZWAgMD7vXfnwNLgQej9/+M+BVYAvwByAv7P76eFmnAhXx9f0oUJoJ6xq4CdgArAF+A+RFcV0D9xN8TtBB8B/ZNcdav4AR7Mm3FXiTYC+gXj+XDv0XEYmIdBtyERGRY1Cgi4hEhAJdRCQiFOgiIhGhQBcRiQgFuohIRCjQRUQi4v8DdRV1pklf1/4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### With ridge penalty"
      ],
      "metadata": {
        "id": "gq1CrW29iQyf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_ridge = GD_regression(num_epochs=100, l2_penalty=10)\n",
        "model_ridge.fit(X_train, y_train, X_test, y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gsuVR0MTiTWg",
        "outputId": "e4c74293-d6ee-444d-8005-17f65d2441c8"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<__main__.GD_regression at 0x7f0d54d00cd0>"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_axis = list(range(model.num_epochs))\n",
        "plt.plot(x_axis, model.train_loss, label = \"train-without-ridge\")\n",
        "plt.plot(x_axis, model.test_loss, ':', label = \"test-without-ridge\")\n",
        "plt.plot(x_axis, model_ridge.train_loss, '--', label = \"train-with-ridge\")\n",
        "plt.plot(x_axis, model_ridge.test_loss, '-.', label = \"test-with-ridge\")\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "id": "R2UoG-SkiTHa",
        "outputId": "f9c016f3-740f-422a-86d6-2d1e33bc14fd"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwUVbrw8d/pLensISFAWAMiOwTCKkbBFUVRRlwQt3FlRkd0HO7oexUVnXudcUNmVC4oOjouDIKOCwqyCSKLCQSFCBIgQCBsgexbL+f9ozqdhCSsCUV3nq+fmD51TlU91dU8VTlddUpprRFCCBH4LGYHIIQQonFIQhdCiCAhCV0IIYKEJHQhhAgSktCFECJISEIXQoggYWpCV0rNVkodVEptOsn2NymlMpVSm5VSHzZ1fEIIEUiUmdehK6UuAoqB97TWvU/Qtivwb+ASrfVRpVSC1vrg2YhTCCECgaln6FrrFcCRmtOUUl2UUt8opdKVUiuVUt19VfcBr2utj/rmlWQuhBA1nIt96DOBP2itU4A/AW/4pp8PnK+UWqWUWqOUGmVahEIIcQ6ymR1ATUqpCOACYK5SqmpyiO+3DegKjADaASuUUn201vlnO04hhDgXnVMJHeMvhnytdXI9dTnAWq21C9iplPoVI8H/eDYDFEKIc9U51eWitS7ESNY3AihDP1/1Zxhn5yil4jG6YHaYEacQQpyLzL5s8SNgNdBNKZWjlLoHmADco5TaCGwGrvM1XwjkKaUygWXAZK11nhlxCyHEucjUyxaFEEI0nnOqy0UIIcTpM+1L0fj4eN2pUyezVi+EEAEpPT39sNa6ZX11piX0Tp06kZaWZtbqhRAiICmldjVUJ10uQggRJCShCyFEkJCELoQQQeJcu1NUCAG4XC5ycnIoLy83OxRhktDQUNq1a4fdbj/peSShC3EOysnJITIykk6dOlFjXCPRTGitycvLIycnh6SkpJOeT7pchDgHlZeXExcXJ8m8mVJKERcXd8p/oUlCF+IcJcm8eTud/R9wCX3r/iJeWriVIyWVZocihBDnlIBL6Et3ruXtHX9iQ+6vZociRNDKz8/njTfeOHHDY1x99dXk5zfuIwqqlnlsTMuXL+eaa65plHUsX76cH3744aTb33vvvWRmZtaZ/u677/LQQw81SkynI+ASusVaiS0smwMlh80ORYig1VBCd7vdx51vwYIFxMTENGosVcs83YPMyTiVhO7xeHjrrbfo2bNnk8RyJgIuoceERgJwtKzI5EiECF6PP/4427dvJzk5mUGDBpGamsqYMWP8Sez6668nJSWFXr16MXPmTP98nTp14vDhw2RnZ9OjRw/uu+8+evXqxRVXXEFZWVmd9bz44otMnz4dgEcffZRLLrkEgKVLlzJhwoRay6wZ0+TJkwEoLi5m3LhxdO/enQkTJlA1euySJUvo378/ffr04e6776aioqLWsgDS0tIYMWIE2dnZzJgxg1dffZXk5GRWrlxZJ86IiAgee+wx+vXrx+rVqxkxYoR/6JJ33nmH888/n8GDB7Nq1Sr/PNu3b2fo0KH06dOHJ598koiIiFrbPWjQIPr27cvTTz99OruoXgF32WILp5HQCytKTI5EiLPj2S82k7mvsFGX2TMxiqev7dVg/QsvvMCmTZvIyMhg+fLljB49mk2bNvkvoZs9ezYtWrSgrKyMQYMGccMNNxAXF1drGdu2beOjjz5i1qxZ3HTTTcybN4/bbrutVpvU1FRefvllHn74YdLS0qioqMDlcrFy5UouuuiiBmMC46x6w4YNbN68mcTERIYPH86qVasYOHAgd911F0uWLOH888/njjvu4M033+SRRx6pd1s7derExIkTiYiI4E9/+lO9bUpKShgyZAgvv/xyrem5ubk8/fTTpKenEx0dzciRI+nfvz8AkyZNYtKkSYwfP54ZM2b451m0aBHbtm1j3bp1aK0ZM2YMK1asqLO9pyPgztDjwqIAKKwsNjkSIZqPwYMH17oeevr06fTr14+hQ4eyZ88etm3bVmeepKQkkpONp0mmpKSQnZ1dp01KSgrp6ekUFhYSEhLCsGHDSEtLY+XKlaSmpp5UXO3atcNisZCcnEx2djZbt24lKSmJ888/H4A777yTFStWnOaWG6xWKzfccEOd6WvXrmXEiBG0bNkSh8PBzTff7K9bvXo1N954IwC33nqrf/qiRYtYtGgR/fv3Z8CAAWzZsqXe9+90BNwZesvwaDxlbcEbanYoQpwVxzuTPlvCw8P9r5cvX87ixYtZvXo1YWFhjBgxot7rpUNCQvyvrVYrZWVl7Nmzh2uvvRaAiRMnMnHiRJKSknj33Xe54IIL6Nu3L8uWLSMrK4sePXqcMK5j13GiPn6bzYbX6wVo8Bpvj8dDSkoKAGPGjGHq1KmEhoZitVpPGM/J0FrzxBNP8MADDzTK8moKuDP0VhGxlGb/gbb2IWaHIkTQioyMpKio/u+pCgoKiI2NJSwsjC1btrBmzZqTXm779u3JyMggIyODiRMnAka3y0svvcRFF11EamoqM2bMoH///nWuwz5eTDV169aN7OxssrKyAHj//fe5+OKLAaN7JT09HYB58+bVu2yr1eqPcerUqcdd15AhQ/juu+/Iy8vD5XIxd+5cf93QoUP96/j444/906+88kpmz55NcbHRy7B3714OHjx4wu06GQGX0B02Cw6rheIKj9mhCBG04uLiGD58OL179/Z/AVll1KhRuN1uevToweOPP87QoUPPaF2pqank5uYybNgwWrVqRWhoaL3dLceLqabQ0FDeeecdbrzxRvr06YPFYvEfPJ5++mkmTZrEwIEDa51xX3vttXz66acNfinakDZt2vDMM88wbNgwhg8fXuuvimnTpvHKK6/Qt29fsrKyiI6OBuCKK67g1ltvZdiwYfTp04dx48ad1IHqZJj2TNGBAwfq033ARd8Zv6FbTF/m3vJM4wYlxDnil19+OakuB3HuKi0txel0opTi448/5qOPPuI///nPKS2jvs+BUipdaz2wvvYB14cOgP0I+e5cs6MQQogGpaen89BDD6G1JiYmhtmzZzf5OgMyoVsJpcJTanYYQgjRoNTUVDZu3HhW1xlwfegANuXE5ZVxooUQoqaATOh2FYpL173rTAghmrOATOgxli5YXe3MDkMIIc4pAdmH3sN5I2ty88wOQwghzikBeYYeEWKluOL4d4QJIU7fmYxsOG3aNEpLT++ihX379jFu3DgAMjIyWLBggb/umWee4aWXXjqt5R7r3XffZd++fSfd/oILLqh3+l133cUnn3zSKDE1hoBM6Ls9i3C3nYrHKzcXCdEUzEroiYmJ/gR5bEJvTCeb0KuGEjiVsdLNFJAJ3Wb1YLEXUFghly4K0RSOHaq2vuFeS0pKGD16NP369aN3797MmTOH6dOns2/fPkaOHMnIkSPrLHf06NH89NNPAPTv399/a/2UKVOYNWsW2dnZ9O7dm8rKSqZMmcKcOXNITk5mzpw5AGRmZjJixAg6d+7sH3YX4JVXXqF379707t2badOmAfiXVeWll17imWee4ZNPPiEtLY0JEyaQnJxcZ1jf5cuX1xkuuGroW601Dz30EN26deOyyy6rdcv+ggUL6N69OykpKTz88MP+h2+UlJRw9913M3jwYPr373/KNxedioBM6BF2Y6CgwyWNO6SoEOesd0bDhg+M1x6XUd5oJDkqS43yJt/YJOUFRjnzc6NckmeUt35tlIsOnHB1L7zwAl26dCEjI4PLL7/cP9xrRkYG6enprFixgm+++YbExEQ2btzIpk2bGDVqFA8//DCJiYksW7aMZcuW1VluamoqK1eupKCgAJvN5h8//Njhch0OB1OnTuXmm28mIyPDP4rhli1bWLhwIevWrePZZ5/F5XKRnp7OO++8w9q1a1mzZg2zZs1iw4YNDW7buHHjGDhwIB988AEZGRk4nc46bdavX89rr73Gr7/WfjLap59+ytatW8nMzOS9997zn7mXl5fzwAMP8PXXX5Oens6hQ4f88/zlL3/hkksuYd26dSxbtozJkydTUtI0w38HZEKPdBhHy8Ol8pALIZpaQ8O99unTh2+//ZY///nPrFy50j9WyfGkpqayYsUKVq1axejRoykuLqa0tJSdO3fSrVu3E84/evRoQkJCiI+PJyEhgQMHDvD9998zduxYwsPDiYiI4De/+c0pjcdSn2OHC66yYsUKxo8fj9VqJTEx0f9Aji1bttC5c2f/POPHj/fPs2jRIl544QWSk5P9I1Pu3r37jOJrSEBe5RIVYiT0I2Vyhi6aid9+Vf3aaq9ddoTVLodG1y6Hx9UuR7Y6pVUfb7jX9evXs2DBAp588kkuvfRSpkyZUqv+008/5dlnnwXgrbfeYtCgQaSlpdG5c2cuv/xyDh8+zKxZs/zD1Z7IqQyXW3OoXGh4uNy1a9f6t23q1KlERUXVGi74TGmtmTdv3kkdsM5UQJ6ht41IxFXYB6/HbnYoQgSlmsPJNjTc6759+wgLC+O2225j8uTJrF+/vs68Y8eO9Q9FO3DgQBwOB+3bt2fu3LkMGzas1tC5x4vheFJTU/nss88oLS2lpKSETz/9lNTUVFq1asXBgwfJy8ujoqKCL7/8st5lDxkyxB/jmDFjjruuiy66iDlz5uDxeMjNzfV3K3Xr1o0dO3b4H+JR1edf9f79/e9/9z8e73jdQWfqhGfoSqn2wHtAK0ADM7XWrx3TRgGvAVcDpcBdWuv1jR+u4fwW51O+dwJRNrm5SIimUHOo2quuuso/3CsYXxD+61//Iisri8mTJ2OxWLDb7bz55psA3H///YwaNcrfl36s1NRUlixZgtPpJDU1lZycnHqHyx05cqS/q+KJJ55oMNYBAwZw1113MXjwYADuvfde/2PgpkyZwuDBg2nbti3du3f3z3PXXXcxceJEnE4nq1evrrcfvT5jx45l6dKl9OzZkw4dOvjfE6fTyRtvvMGoUaMIDw9n0KBB/nmeeuopHnnkEfr27YvX6yUpKanWwaUxnXD4XKVUG6CN1nq9UioSSAeu11pn1mhzNfAHjIQ+BHhNa33cJ1CcyfC5W/YXMmraSt6YMICr+7Q5rWUIcS6T4XMDT3FxMREREWitefDBB+natSuPPvroGS3zVIfPPWGXi9Y6t+psW2tdBPwCtD2m2XXAe9qwBojxHQiaRKUuIOL8Z1i5v2mOckIIcapmzZpFcnIyvXr1oqCgoEkeMXcip/SlqFKqE9AfWHtMVVtgT41yjm9akwxaHueMQFnLKayQq1yEEOeGRx999IzPyM/USX8pqpSKAOYBj2itT+vyEqXU/UqpNKVUWs3rNE9Vi3DjKpcSV/FpL0MIIYLNSSV0pZQdI5l/oLWeX0+TvUD7GuV2vmm1aK1naq0Haq0HtmzZ8nTiBcBpt6O9DkrccqeoEEJUOWFC913B8jbwi9b6lQaafQ7coQxDgQKtdZM+I055QymThC6EEH4n04c+HLgd+FkpleGb9v+ADgBa6xnAAowrXLIwLlv8beOHWpujfBDhzs5NvRohhAgYJ3OVy/daa6W17qu1Tvb9LNBaz/Alc3xXtzyote6ite6jtT696xFPQVzlWGI9w5t6NUI0S6c72uLVV19Nfn5+o8ZStcxjY1q+fLl/AKxTFSjD4Z6qgLxTFCA8xEZxZYXZYQgRlBpK6Me71R6MEQdjYmIaNZaqZZ7JkL5VAm043FMVsAn9kHMmv6jnzA5DiKBUc/jcQYMG1RlO9vrrryclJYVevXoxc+ZM/3ydOnXi8OHDZGdn06NHD+677z569erFFVdcUWeYWoAXX3zRPwzuo48+6h/saunSpUyYMKHWMo8d0heMm3nGjRtH9+7dmTBhgv/2+poCeTjcUxWQg3MBOCwhFHvqH2xHiGDz22/qfi11ZacruaX7LZS5y/j94t/Xqb/uvOu4/rzrOVp+lD8u/2OtundGvXPc9b3wwgts2rSJjIwMli9fzujRo9m0aZN/NMHZs2fTokULysrKGDRoEDfccANxcXG1lrFt2zY++ugjZs2axU033cS8efO47bbbarVJTU3l5Zdf5uGHHyYtLY2KigpcLled4XSPjQmMRL1hwwY2b95MYmIiw4cPZ9WqVVx44YV1tmf9+vW14q9SczjcAwcO0LNnT+6++27/cLgrVqwgKSmp1uiJVcPhzp49m/z8fAYPHsxll13WqAN6na6APUMPsYbhRRK6EGfDscPJTp8+nX79+jF06FD27NnDtm3b6syTlJREcnIyACkpKf6Bq2pKSUkhPT2dwsJCQkJCGDZsGGlpaaxcubLe8V3qi6tdu3ZYLBaSk5PrXUd98Vc514fDPVUBe4butDnRLknoonk43hm10+Y8bn1saOwJz8hPpObZ5/Lly1m8eDGrV68mLCzMn9SOdexQt2VlZezZs4drr70WgIkTJzJx4kSSkpJ49913ueCCC+jbty/Lli0jKyvrpMayqW843WAaDvdUBewZutMWDhY3bq88LFqIxna8oWsLCgqIjY0lLCyMLVu2sGbNmpNebvv27f1D1U6cOBGg1hC6qampzJgxg/79+2PcAnNyMdUUTMPhnqqATegdw3tScfhiylwus0MRIujUHD636gvIKqNGjcLtdtOjRw8ef/xxhg4dekbrSk1NJTc3l2HDhtGqVStCQ0Pr7W45Xkyna+zYsXTt2pWePXtyxx131DscbkpKCpGRkf4nMj311FO4XC769u1Lr169eOqppxollsZwwuFzm8qZDJ8L8O6qnTzzRSbrn7qcFuGORoxMCPPJ8Lnma4rhcE9Vow+fe64KdSiwlFJQJv3oQojGdy4Mh3uqAvZL0X0VG4nsNpWfDrclKb7+u76EEOJ0nQvD4Z6qgD1DjwmNBCC/TMZEF0IICOCEHus07vQ6Wi5jogshBARyQvedoRdVSEIXQggI4IQeHxYFQKEkdCGEAAI4obeMiKHi0GW0sJ1ndihCBJ0zGdlw2rRplJae3sNn9u3bx7hx4wDIyMhgwYIF/rpnnnmGl1566ZSXmZaWxsMPP1xvXdXAX8EiYBN6jDOUysOXEWXpYnYoQgQdsxJ6YmKifzzyYxP66XC73QwcONA/omOwC9iE7rBasNmLOVyWZ3YoQgSdY4eqffHFFxk0aBB9+/bl6aefBoxhZEePHk2/fv3o3bs3c+bMYfr06ezbt4+RI0cycuTIOssdPXo0P/30EwD9+/dn6tSpAEyZMoVZs2aRnZ1N7969qaysZMqUKcyZM4fk5GT/rfeZmZmMGDGCzp07N5ikn3nmGW6//XaGDx/O7bffXutBGHl5eVxxxRX06tWLe++9t9Zwu8899xzdunXjwgsvZPz48f6/BrZv3+6/YzQ1NZUtW7Y00rvc+AL2OnSlFM5Or7MuvzcwxOxwhGhSu26/44RtIkaMIO6eu/3to8eOJeY3Y3EfPcrehyfVatvx/feOu6yaQ9UuWrSITz75hHXr1qG1ZsyYMaxYsYJDhw6RmJjIV199BRhjvERHR/PKK6+wbNky4uPj6yw3NTWVlStX0rFjR2w2G6tWrQJg5cqVzJgxw9/O4XAwdepU0tLS+Mc//gEYiXrLli0sW7aMoqIiunXrxu9+9zvsdnud9WRmZvL999/jdDpZvny5f/qzzz7LhRdeyJQpU/jqq694++23Afjxxx+ZN28eGzduxOVyMWDAAFJSUgC4//77mTFjBl27dmXt2rX8/ve/Z+nSpcd9/8wSsAkdwKJDKffUHTRfCNF4Fi1axKJFi+jfvz9g3BK/bds2UlNTeeyxx/jzn//MNddcc1LD3aampjJ9+nSSkpIYPXo03377LaWlpezcuZNu3bo1OPxtldGjRxMSEkJISAgJCQkcOHCAdu3a1Wk3ZswYnE5nnekrVqxg/vz5/mXFxsYCsGrVKq677jpCQ0MJDQ31jwhZXFzMDz/8wI033uhfRkXFufuktIBO6FZCqfRKQhfB70Rn1Mdrb4uNPeX5a9Ja88QTT9R76/v69etZsGABTz75JJdeeilTpkypVf/pp5/y7LPPAvDWW28xaNAg0tLS6Ny5M5dffjmHDx9m1qxZ/rPhE6lvuNzXX3+dWbNmAfj73BtruFyv10tMTIz/oRrnuoDtQwewKSeVWhK6EI2t5lC1V155JbNnz6a42LhEeO/evRw8eJB9+/YRFhbGbbfdxuTJk1m/fn2deceOHesfynbgwIE4HA7at2/P3LlzGTZsWK2hc48Xw/E8+OCD/nUkJiYet+1FF13Ehx9+CMDXX3/N0aNHARg+fDhffPEF5eXlFBcX8+WXXwIQFRVFUlISc+fOBYyD28aNG08Yk1kCOqE7LE7cWgbnEqKx1Ryq9ttvv+XWW29l2LBh9OnTh3HjxlFUVMTPP//M4MGDSU5O5tlnn+XJJ58EjD7nUaNG1fulKBjdLgkJCTidTlJTU8nJyam3u2bkyJFkZmbW+lL0TD399NOsWLGCXr16MX/+fDp06ADAoEGDGDNmDH379uWqq66iT58+/uFyP/jgA95++2369etHr169zqlniB4rYIfPBbjpvdnkFh1h5YN/aqSohDg3yPC5Z1/VcLmlpaVcdNFFzJw5kwEDBpga06kOnxvQfejtQgewd98Rs8MQQgSB+++/n8zMTMrLy7nzzjtNT+anI6ATut1eSrHORmtd53FVQghxKqr61gNZQPeh7/N8hzfxVSo85+5lREKcLrO6Q8W54XT2f0An9HC7cWlSfrmMiS6CS2hoKHl5eZLUmymtNXl5eYSGhp7SfAHd5RLpMBL64ZJCWke0NDkaIRpPu3btyMnJ4dChQ2aHIkwSGhpa701TxxPYCT3EeMjF4VI5QxfBxW63k5SUZHYYIsAEdJdLdIjxkIsjZYUmRyKEEOYL6ITeOboLZftuJNrWxuxQhBDCdAGd0NtHt8JdkAKeKLNDEUII0wV0Qo8Ld2AJ3c2WvCyzQxFCCNMFdEKPjwjB2f49VhyYb3YoQghhuoBO6E6HFeWJIL9Cbv8XQoiATugAdqIodh81OwwhhDBdwCf0UGs05d4Cs8MQQgjTBXxCj7DF4kKuQxdCiIC+UxTg/LBLydvVRUZcFEI0eyc8Q1dKzVZKHVRKbWqgfoRSqkApleH7mVJfu6bSKaozBUc6IWMYCSGau5PpcnkXGHWCNiu11sm+n6lnHtbJCw+tRIVvZPuRfWdztUIIcc45YULXWq8AztnrApX9CM52H7J23wazQxFCCFM11peiw5RSG5VSXyulejXUSCl1v1IqTSmV1ljDgnaIbgVATtHBRlmeEEIEqsZI6OuBjlrrfsDfgc8aaqi1nqm1Hqi1HtiyZeOMX94x1kjoB4rzGmV5QggRqM44oWutC7XWxb7XCwC7Uir+jCM7Sa2jwtEeJ4fLDp+tVQohxDnpjBO6Uqq18l0vqJQa7FvmWTtdjg1zoN0RHJXb/4UQzdwJr0NXSn0EjADilVI5wNOAHUBrPQMYB/xOKeUGyoBb9Fl8EKLVonAcuYPz4jqcrVUKIcQ56YQJXWs9/gT1/wD+0WgRnYaWIUmUlYabGYIQQpgu4G/9BwiL2E9W+UKzwxBCCFMFRUL3hP7CQcfHVHgqzA5FCCFMExQJvUVoHAB5ZXLpohCi+QqKhN4yzEjoucVyc5EQovkKioTeJsK4SWlXviR0IUTzFRQJvW1kAgA5hY0znIAQQgSioEjoXVq0pThrMt0jLzY7FCGEME1QJPSESCfaFUdhqdmRCCGEeYIiocdFhGCP/pFV+78xOxQhhDBNUCT0cIcVR2w6PxUsNjsUIYQwTVAkdKUUDqIpcR81OxQhhDBNUCR0gDBrNOXeArPDEEII0wRNQo+wt8CjSnB5XWaHIoQQpgiahB7jiAXgaLl0uwghmqcTDp8bKPrGXMb6H7rQ0tk4j7YTQohAEzRn6K0iI3C5bRSWu80ORQghTBE0CT0qDEISvmDhjqVmhyKEEKYImoTeKiICe+yPrM5dbXYoQghhiqBJ6AlRoXgr49lduMvsUIQQwhRBk9DbtwjDWxHP/rI9ZocihBCmCJqEHhFiI1S1ptB9kEpPpdnhCCHEWRc0CR0gIbQdVh3J4bLDZocihBBnXdBchw7QJ3okK7f1ITEi0exQhBDirAuqM/TOLSM4WFRBaaVciy6EaH6CKqF3jAsjpPV8Xl73utmhCCHEWRdUCb1TXDiWkIP8eGCt2aEIIcRZF1QJvWNcGN7KeA6U5ZgdihBCnHVBldAjQ+04aUWp5yjFlcVmhyOEEGdVUCV0gFbO9gDsLtptciRCCHF2BV1CT4rqjKWyvdxcJIRodoIuofdOOI+C7Q9yfkxvs0MRQoizKugSese4cAB2Hyk1ORIhhDi7gi6hd4oLJ6TV5/y/Hx4yOxQhhDirgi6hd4wPAzS7ireitTY7HCGEOGuCLqFH+S5drNSl5JXnmR2OEEKcNUGX0AFahxmXLu6Sh10IIZqRoEzoSdGdANiev93cQIQQ4iwKyoTePb4TrsK+xDjizQ5FCCHOmqBM6EktIyjfeysdnYPMDkUIIc6aEyZ0pdRspdRBpdSmBuqVUmq6UipLKfWTUmpA44d5ajrFhQGwef9+yt3lJkcjhBBnx8mcob8LjDpO/VVAV9/P/cCbZx7WmenSMgJb6F6e3vgbVu1dZXY4QghxVpwwoWutVwBHjtPkOuA9bVgDxCil2jRWgKcjPMRGl5guKG1jw8ENZoYihBBnTWP0obcF9tQo5/im1aGUul8plaaUSjt06FAjrLphKR0T0BXtJKELIZqNs/qlqNZ6ptZ6oNZ6YMuWLZt0Xf3bx1BR3JHMvEzpRxdCNAuNkdD3Au1rlNv5pplqQMdYPGUdcWs3mw7X+32uEEIElcZI6J8Dd/iudhkKFGitcxthuWckKS6ccN2V3iG/pZPvRiMhhAhmthM1UEp9BIwA4pVSOcDTgB1Aaz0DWABcDWQBpcBvmyrYU2GxKAa0S2RfbizxTrnBSAgR/E6Y0LXW409Qr4EHGy2iRtS/fSwrtm9n7pb/cEO3a7GooLyPSgghgCC9U7TKgI4xWMJ/ZeraJ9mRv8PscIQQokkFdULv1z4Gb1knANYfXEG+924AACAASURBVG9uMEII0cSCOqFHhdrpEtsRh45n6e6lZocjhBBNKqgTOkBKhxZUFvZhTe4ajpYfNTscIYRoMkGf0Pt3iKEkrzce7SHtQJrZ4QghRJM54VUugW5Ah1i8FYk83PVdLu+YYnY4QgjRZIL+DL1LywhaRoayYafZkQghRNMK+oRusSiu7NWKZVv38uDiP/DJr5+YHZIQQjSJoE/oAFf3bkNZpZWteTv5YvsXZocjhBBNolkk9MFJLYgNsxPmGsj6g+vZX7Lf7JCEEKLRNYuEbrNauLJXa3bsPA+Ab3Z+Y3JEQgjR+JpFQgcY1bs1xSWxdI7ozQdbPsDlcZkdkhBCNKqgv2yxygVd4okKtRFbeQ1X9SzCrd3YjUEjhRAiKDSbM3SHzcJlPVux4deW3NP7fpw2p9khCSFEo2o2CR3gqt5tKChzsSrrEAuzF/LD3h/MDkkIIRpNs+lyAUjtGk9kiI156TnsCnsdu8XOsMRhKKXMDk0IIc5YszpDD7VbGT+kAws2HeA3SXfw69FfWZGzwuywhBCiUTSrhA7w2+GdsCjYkd2VthFteTn9ZSo8FWaHJYQQZ6zZJfQ20U7G9GvL3LRcHkl+nJ0FO5n500yzwxJCiDPW7BI6wP0XdabM5WFbdlvu7HknPVr0MDskIYQ4Y83qS9Eq3VpHMrJbS979IZtVjz9KqN1qdkhCCHHGmuUZOsADF3chr6SSuWl78Govb/38Fv/c/E+zwxJCiNPWbBP6kKQWDOwYy6uLt5Ff4uKXvF94Jf0VuepFCBGwmm1CV0rx3PW9KShz8cI3W3hu+HN0i+3G5O8ms/XIVrPDE0KIU9ZsEzpAjzZR3JuaxL/Tcvh5Txn/uPQfRDgieHDJgxwqPWR2eEIIcUqadUIHmHRpV9rFOvl/n/5MtCOO1y99nWJXMT8f/tns0IQQ4pQ0+4Qe5rDx3PW92X6ohNeXbad7i+58fv3nXNLhEgAqPZUmRyiEECen2Sd0gJHdEhjbvy1/X7qNxZkHSAhLAGBN7hpGfzqazLxMkyMUQogTk4Tu8z9j+9A7MZpJH2/g1wNFACSEJaBQ3Pn1ncz9dS5aa5OjFEKIhklC93E6rMy8I4WwEBv3/jONoyWVdI7uzIejP6R/Qn+mrp7KY989RkFFgdmhCiFEvSSh19Am2snM21PYX1jOPf/8kcJyF/HOeGZcPoM/pvyRZbuXsXjXYrPDFEKIeklCP0b/DrFMvyWZn/cWMH7mGg4XV2BRFn7b+7d8fM3HXHfedQB8t+c7NudtNjlaIYSoJgm9HqN6t2HWHQPZfqiYm/5vNfvyywDo1qIbNosNr/byavqr3PLlLTy05CEyDmaYHLEQQkhCb9CIbgm8d/cQDhVWcP3rq1izI89fZ1EW3r/6fR5KfoiNhzZy+9e3c9c3d7H+wHoTIxZCNHeS0I9jcFIL/j1xGBEhNm6dtYZpi3/F4zWudIl0RPJAvwdYeMNC/mvQf7G/ZD/l7nIAcopyyDiYgVd7zQxfCNHMKLMuxRs4cKBOS0szZd2nqrjCzVOfbeLTDXsZ3KkFz13fm26tI2u1qUreFmVhWvo03t70NgnOBC7pcAmp7VIZ2GogYfYwM8IXQgQRpVS61npgvXWS0E+O1pp56/fy/FeZFJW7uX1oRx69/HyinfY6bYsqi1iRs4Jvd33Lqr2rKPeUExsSy/Kbl2NRFnYU7CDBmUCEI8KELRFCBDJJ6I3oaEklL3+7lQ/X7ibKaefu4UncOawT0WF1EztAhaeC9QfWs79kP2O7jgXgN5//hqyjWXSK7kTPuJ70aNGD/gn96duy79ncFCFEADrjhK6UGgW8BliBt7TWLxxTfxfwIrDXN+kfWuu3jrfMQE3oVTbtLWDa4l9Z/MtBwh1WJgztyPjBHUiKDz/hvGty17DhwAYy8zLJPJLJwdKDjO48mhdSX0BrzX3f3kdLZ0s6RHWgfWR7EsMT6RTdiRahLc7ClgkhzmVnlNCVUlbgV+ByIAf4ERivtc6s0eYuYKDW+qGTDepMErr2esHrRdnMf4LeL7mFvLl8O1/9nIvHqxncqQU3DmzHFT1bN3jWfqy8sjwqPZW0iWhDmbuMPyz5A7uKdrG/ZL+/zb197mXSgEkUVhbyu8W/I8GZQJwzjnhnPHHOOFJapdA5ujMur4uiyiKiHFHYLOa/P0KIxnW8hH4y/+IHA1la6x2+hX0MXAeYMmKVe2cmu++4ibibryP6ob+YEUItPdpEMX18f54c3YN56/fy77Q9TP7kJ56w/MywLnFc0bMVqV1b0jEuDKVUvcuIc8b5XzttTt660vjjptxdTm5JLvuK99E6vDUAZa4ynDYnOwt2sm7/OgorCwF4csiTdI7uzPb87dz4xY0ARNgjiHJEEemI5JGUR7iw7YXsKtzF+5nvE24P9/84bU4uSLyA1uGtyS/PJ7swG6fNSYg1hFBbKCHWECIdkXKAEOIcdzL/QtsCe2qUc4Ah9bS7QSl1EcbZ/KNa6z31tDlj1g7d0NjJ+2oNUQ/qBpPk2ZYQFcrvRnRh4sWdydiTz8LNB1i0eT9P/ce4m7RtjJNhXeJI6RhL/w4xdE2IxGo5fuyhtlCSopNIik7yT2sV3oq3rqjuzar0VHKk/Ij/Cpp4ZzyPD36cgooCCioKKHYVU1hRSKg1FICDpQdZmL2QElcJLq/Lv5w3L3uT1uGtST+QziPLH6kTyz9H/ZMBrQbw5Y4vmbp6Kg6rA4fFgcPqwG6x89olr9E5ujOLdy3mX7/8C5vFhs1iw67s2Cw2/nvofxPvjGdFzgqW7l7qr7coCzZlY2K/iYTZw1ibu5aMgxlYlAWLsmBVVizKwq09bsVmsbH+wHq2F2z3T69qM7rzaAA2Hd5EbkkuCoVSCgsWHFYHw9sOB2DLkS0cKT9Sqz7EFkK/lv0AyDqaRZGrCIWxbyzKgtPmpGtsVwB2FuykzF3mnx8gzBZGh6gOAOwu3O0fclkphUIRZg/zH5BzinLwaI9R7/vPaXcS74wHYF/xPrzaW+tzHWYLIzY0FqDWX23+ensYUY4otNYcKqv7YJYwWxgRjgi82suR8iP11ofZw/B4PeRX5NeqU0oRZgsj1BaK2+v2n0BUvT9V6w+xhuDyuiipLKmz/HB7OHarHZfXRamrtN747RY7Lo+LUnfd+nB7ODaLjUpPJWXusjrxhdvCsVqsVHoqKfeU15k/wh6BRVmo8FRQ4amoPT+KcHu4v76+4bIj7BEopSh3l9f6N1Ml0mFc8VbmLsPtdddZftXFD6WuUjzag1VZm+yKt8Y65foC+EhrXaGUegD4J3DJsY2UUvcD9wN06NDhtFakrFZaPPz/2P/UFErXriN8aH3HFvMopejfIZb+HWL586hu7Dhcwg9Zh/lhex6LfznAJ+k5AIQ7rHRvE0X31pF0bx1Jl4QIurSMICEy5JQOUg6rw58swEjoE3pMaLD9oNaDWHnLSgD/P6AydxnRIdEA9Evox5uXvUmZu8z4B+CuoNxTTvvI9gB0iurEjeffSKWnEpfXhcvrotJTSZit+gOqUFS4KyjxluDWbtxet/+yzj1Fe1iRswKP9uDyuvBqLx6vh3v63APAD/t+YPam2XXiHt9jPAALdi5gztY5tersFrs/oX+05SM+3/55rfqYkBj/Ns/YOIMlu5fUqm8b0ZZvbvgGgL/++FfW5K6pVd81tivzx8wH4Mnvn+Snwz/Vqk9umcz7V78PwKRlk8jKz6pVPzxxODMunwHA3QvvJrckt1b95R0v55URrwBw4xc3+pNmleu6XMfzFz4PwFXzrsKtayeNW7vfyhNDnsDldXHp3Es5VlV3XUFFASP/PbJO/aQBk7i3z73kluRy1fyr6tQ/MfgJbu1xK9vztzPui3F16p8f/jzXnXcdmw9v5vavb69T/+qIV7ms42Wsy13HxMUT69T/32X/xwVtL2DZnmU89t1jderfv+p9khOS+WrHV0z5YUqd+vlj5tM1titzf53LC+teqFO/8IaFJEYk8t7m95i+YXqd+pU3ryQmNIY3M97k7U1v16lff9t67FY7L6e9zMdbP65VZ7fYWX+7cUPh82ueP+5n77+//28W715M3/i+fDD6gzrraQwn04c+DHhGa32lr/wEgNb6fxtobwWOaK2jj7fcM+lD91ZUkDVyJM7z2tP+vTknnuEcobUmO6+UjD1H2bingMzcQrbkFlJYXv0PNCLERvsWYbSLddI+NozEmFBaR4fSOiqUhMhQ4iMdhDmCt+tDa41bu9Fa49EeI+FrD5H2SJRSFFUWUeoqxau9ePHi1V601v4z5P0l+ymqLDLqtReNxqIsdG/RHYAdBTvIL89Ho9Fao9E4rA7/GfrmvM0UlBcY9b42TpuTga2NLsv0A+kUVBj1RsAQFRLFoNaDAFi1dxXFrmKjXoNGE++M99cv2b2EMndZraGYW4e39tcv2LGASm/1WWLVtqW0SgHg022f+tddtYwuMV1ITkjG4/UwP2t+nfezR4se9GnZh3J3uT/h1Fx/n5Z96BnXk+LKYr7c8WX1vL71DEgYQLcW3ThafpRvsr+pM4z00DZD6RzTmUOlh1i0a1GdfZraNpUOUR3YV7yPZXuW1akf2X4kiRGJ7CrcxcqclXXqr+x0JS3DWpJ1NKvWwbYqvms6X0NsaCy/5P1C2oG6OWXseWOJcETw86GfyThUPUxH1Xbc1O0mQm2hrD+wnk2HN9WZf0KPCVgtVtbmrq3zvGGrxeo/gfp+7/fsyN9Rqz7EGsLN3W8GYPme5ewu3E28M56rO19dZz0n60y/FLVhdKNcinEVy4/ArVrrzTXatNFa5/pejwX+rLUeerzlnulVLof+NJ7DX2bQ+aNZhPS/8LSXYzatNbkF5ew4VMKOw8XsOFTCniOl5BwtY8/RUkorPXXmCXdYiYsIITbMTmy4g9gwB9FOO1FOO1GhNiJDbUSG2gkPsRERYiXMYSPcYcPpsBLmsOK0W7GcoLtHCHFuOqMvRbXWbqXUQ8BCjMsWZ2utNyulpgJpWuvPgYeVUmMAN3AEuKvRom9A7B+eIm/hzRyZv4g2AZzQlVIkxjhJjHFyYdf4WnVaawrL3OwvLCe3oIxDRRUcKq7gcFElR0oqOFLqIq+4ku2HiikodVFU4eZkbytwWC2E2C2E2q2E2i1G2WbFYbMYP1bjt92qsFstvh+FzWrBblFYLRZsVoXVorBZjN9WpbBajbJFGdMsSmGpqrMY22tVCosFLMrXj62M1xZV1e9cVYfvx5hWpw5A+fqifWWlar72NfC9rjqEVS2neh/4ftdoW/++Ork2/tc11tJg+/onN9j++E7/IH2OfBXVJM7FTYt22omLCGn05Qb0jUW5T02h4PPPOW/JYmzx8SeeIch5vJriCjdF5S6KK9wUl7spqfRQUuGmpMJNuctDmctDaaWHcpeXcpeHcpeHSreXCreXCreHCreXSreXSo8Xt0dT6fbi8nhxeY2yy6PxVL32evF6we314pWHOQlx0iZe3IXHr+p+WvOe6WWL56wWd/+W/Pnz2D/pNtp98I3Z4ZjOalFEO+31DkfQ1LxejUdrPF6N22v89no1Xm1M93oxXns1WuOfrjW+/vLq6cY032uMeuOA4avzrc+oo0Z/dXW5qh2++f3HG13d94qvfa3fNbap5smOrtGmdqua7Wu8bmB6rfYnsZyTdSbHU3m04tl3XkLTDPsR0Ak9JCmJlpd3Jn/1Ltx5h7HFyVm6WSwWhQWF3Wp2JEI0XwGd0AHiXviYWLfGGiEDXQkhmreAHw9dhYZjjYjAezSXgv973uxwhBDCNAGf0Kvk/+337Hv1A8o3yePghBDNU8B3uVSJ/fPrOAZ/TWjvZLNDEUIIUwTNGbqKSSRirHH7eMlnMyleutDkiIQQ4uwKmjP0KrpwP4deepHywhDaz4gk/IILzA5JCCHOiqA5Q6+iolrT7tW/4Wjfkd333c+Rf30g19kKIZqFoEvoALZBY+k4599EXJTKgeefZ/8Tk9GVdYfFFEKIYBKUCR3AGhFOuym/J65PBfmffcXOcTdS9vPPZoclhBBNJmgTOoBq04eE2atp9+YbeAoKyL75Fg789a94y8pOPLMQQgSYoE7oAES2InLkSDr/+z1iulZy5L1/4j540OyohBCi0QXdVS4NsSa0p81zfyGORBwdO0J5Afv/52+EDbuQqKvqPqVFCCECTbNJ6CgFfW/C4St6v5lK2dKvsLVqBVyFdrlwHTiAo107M6MUQojT1nwS+jEsQ+6g04vnwcB7ASh896/se/kDnAMGEH3tNUReeSW2Fi1MjlIIIU5eQD/gotEUH8L1fE8KPCMo+KWMyqztoBTO5GQiRowgIvVCQrp3R1mC/ysHIcS57YyeKdpUzqmEDrB/Ezhj0VGJVKz8jKJZUyguaE/5rzsBsERHEz54EG2nTUNZrWivVxK8EOKsC9onFjWq1r0B4/mDoZ0SCb2iKy3HvY2r0EXp/DcpWf09Ho8bZTWe4LDn/gewRkfT9uWXAChZtw5bfDyOdu1QDkdDaxFCiCYjCb0+HYbA7fMBsDshum8M0a48+MMio37tTMLji1C9LgZAe73smfg7dGkpWCzY27bF3q4t9sRE46d1a2wJCTiSkuRLVyFEk5Eul5Pl9UJVF8uXf4QjO+COzwDQc++l/JCmsu0YKnftonLrJlyHjuLKPYD70CH/ImJuuok2U59FezxkjRhJ3P330+L223AfPcrBv72INSoSS0QklsgIrBERWMLDsYSFoZxOLM4w7G0TsbVogfZ48JaVYXE6/X8xCCGaB+lyaQw1+8uveaXWk3xVdBuc8WE4R15nTHgtGUb0hxvn4q2sxP3xw7hDkrAO+A0AOnstERcOwdGhPQDe4mJK1qzBW1iIt6SkwRASJv+JuHvuwZWTw/YrR5H44t+IvvZaStevZ88DE1EOB8puN35sNpTNBnYbymZHWa3EP/QgEcOHU751KwdfepmEPz1GaLdulKalceS998FiMb4XsFjAolDKYlzuabGAgrh77iGkc2fKfvqJ/E/m0fLhP2CLj6fkhx8oWrwYo8Oq6k1RdX7HT3wAW1wcJWvWULRkKQl/egxLSAhFixdTsnbdCXdBwuQ/YXE4KPz2W8o2ZNDqvyYDkD9vPuWbN/ta1X+CouwOWj3xuK/9PFz7cmn5h4cAyHvnXSqzs32z13zSc/Vra3wcCZMmGe3ffhsw3g+Ag69Ow513uOGnRAOOpE7E33ef0f7ll7G3bUvsLbcAkDvlaXRFRQNbbSwotF8/Wtx6q9H+qacIGzKU6GtG4y0vJ3fKlPpmqSXioouIvvYao/2TTxF1zWgiR4zAdeAgB198sd5trinq2muq2//tb8ROmEDYgP5UZGVx+I03G4i9WuxtEwgbMMDfPv73vyPkvPMoTU/n6AcfnHD++N//vlb7hMcfx56QQNHSZRR++cUJB+Br9fgT2FslULRsGQWff07i//4vltBQ8j/7jOLvvjvh+mu2L129hsS/vgDAkffeo3TDhuPOa3GE1GrvzjtCwqOPnHCdp0MS+ulSNZLXFc/Vrrv6RQiJAsBit+Moy8TRviN07QpaY/nwOtqMnAgXXwxa45h3NV1feAiGPICuKMc7/2G8HS/D27I/3qICvD9/gTeyMyEDLgZ3JZbCrSQ88ntCe/YEjxub3UX0mGvQbi/aVYl2udAuN9rtArcH7fGg3S7/2bx2ufEUFKBdbgA8RUVU7tyB9njB60VrL3i8oLXx2qtBa2LGjQPAtX8/RUuXEnevkdAqdu6kcMHX1dvv+8eljym3uG0CxMVRsS2Lgv/8h4RHJkFICGU/b6Lgs89qv6f1SHhkEjgclG/eTOHXXxsJXWvKMjZQ9O3iGvvG/z9/FBZnmJHQK4ooS19HxY7dRkIvO0rp2lWUbdria+41fvsO4EprQGPv0MmYnr+bsh9XgyPcKB/ZQenqlbgO5Pnmd/vmtxvb46kEFN7SUmP6vgzKflyNt6y/Ud7zI6Vrf0C7vEZ7d4Xx2+r7HsZdDhYrluhoo5y1mLIf12Jv6+u627KQsrR11e0rS8BqB1uIr1wMVgch551n7Ief5lG2IY3woUPA40ZvnEfZhnSw2Y23qrIYZQvxLU9DRTHYQgm/YBhUlqDXf0z5xg14rhkN5YV40+dS/vNGY53aCxVF4Agztl97jPkdYXgKroHiQ3h//IjyTT8ZJy5F+/GkfUL55k1gsYHXbcTriDDKHpexPSERxvt3ZAeeH+dQvnmzMdje4Sw8P86jPHMbKCt4fe1Do4yyuwJcpRAajXa5YP/PeNbNpWLLTuMv7n0ZuNP+Q8WWA773vtxo74w1Pj+uMnCXgbOF0X7Xatw/fkbFtiLjvc3+HlfaAiqyfGVXqbFOZ2z1vvBUYoltY5R/XYRrwyLcKuG4n/MzorU25SclJUU3Sx6P1llLtT7wi1F2lWv96e+1/uVLo1xWoPXLPbReN8soFx3U+ukordfONMoFe43yj7ON8pFso7z+X0b50K9G+ae5RvlAptZ/aav1lgVGOfcnrV/qZsSgtdZ712s9ra/W2T8Y5d3rtJ6eonVOmlHOXqX13wcZ82mt9fblRvngFqP867dG+8NZRvmXL7V+LdmIS2utN83X+tXeRtxaa53xkbH+ooNGOe0drf/aWevSo0Z59Zta/097rStKjPLKV7V+rpXW7kqjvPQvWj/bovr9XPSU1lNbVpe/flzr/2lXXf7iUa3/mlRd/uxBY/1V5t2n9at9qsv/vtPYniofjtf6jQuqy+/foPX/XVxdfvdard+6vLr89pVavzO6ujxzpNbvja0uvzlc6w9vqS7/fZDWc26vLk/rp/Un91aXX+5hfD6q/K2L1l88Ul3+3/ZaL/iv6vJzrbRe+GR1+ZlYrRdPNV57PMZnY9kLRrmy1CivfMUol+Ub5R/+YZSLD9X+7OXnGOW0d41y3g6jvOFDo3xwi1H++ROjnPuTUc783CjnpBnlrd8Y5ewfjHLVZ3H7cqO883uj/Osio7znR6P8y5dGeV+GUd403ygfyDTKGR8b5arPYvp7RvnobqO87i2jXLjfKK9+wyiXHjHK308zyhXFRnn534yy22WUlzyv9dPR1e/toinH/+x9+cd6Pnvd9ZkC0nQDeVUS+rnO49a6YJ+R6LXWurJM66wlWufvMcpl+UZSPLTNKBcf0nr5X6s/5AV7tf76Ca1zfzbKR3YaH6yq8qFfjQSyf7NR3r9Z63/fVZ2w9200Ek7V8nPStZ5zh7EcrY0DwL/vqo4n+wetP7lH68Jco7zze63n3a918WGjvGOFsf6yfF/5OyNBVf0j2r7MSFCu8urywieN96GqvOS56vdn+zKtv3uxdrkqIVWV171Vo7y8+uBXtf6qg19VfFUJqCr+rQury9mrqhOQ1lrvWl2dgLTWevda4z2psudH46DpL6dVv/daG+9n1XuttdZ7N1QnJK2N9z9vR3U596fqBKW11vs3VR8stTb2e1XC0tpYdtXB0+s19nfVvvB4jHVVJTSPx1hX1cHV4zYOzFWfPbfLWHd5UXU5P6d637krjc9qZWl1uXC/8ZmtKhcdqN637krj8+qqMMquCiO2qoO3q0LrkrzqhOqqMGKtVT5a/dlwVRifq1rlAmO7qsrlhXXLXm+NclHtckXxMeWS6ve2vnLVttdXdldWvxdn4HgJXb4UFUKIAHK8L0XlzhghhAgSktCFECJISEIXQoggIQldCCGChCR0IYQIEpLQhRAiSEhCF0KIICEJXQghgoRpNxYppQ4Bu05z9njgcCOGEyia43Y3x22G5rndzXGb4dS3u6PWumV9FaYl9DOhlEpr6E6pYNYct7s5bjM0z+1ujtsMjbvd0uUihBBBQhK6EEIEiUBN6DPNDsAkzXG7m+M2Q/Pc7ua4zdCI2x2QfehCCCHqCtQzdCGEEMeQhC6EEEEi4BK6UmqUUmqrUipLKfW42fE0BaVUe6XUMqVUplJqs1Jqkm96C6XUt0qpbb7fsWbH2hSUUlal1Aal1Je+cpJSaq1vn89RSjnMjrExKaVilFKfKKW2KKV+UUoNaw77Win1qO/zvUkp9ZFSKjQY97VSarZS6qBSalONafXuX2WY7tv+n5RSA05lXQGV0JVSVuB14CqgJzBeKdXT3KiahBt4TGvdExgKPOjbzseBJVrrrsASXzkYTQJ+qVH+K/Cq1vo84ChwjylRNZ3XgG+01t2BfhjbHtT7WinVFngYGKi17g1YgVsIzn39LjDqmGkN7d+rgK6+n/uBN09lRQGV0IHBQJbWeofWuhL4GLjO5JgandY6V2u93ve6COMfeFuMbf2nr9k/gevNibDpKKXaAaOBt3xlBVwCfOJrElTbrZSKBi4C3gbQWldqrfNpBvsasAFOpZQNCANyCcJ9rbVeARw5ZnJD+/c64D3f40PXADFKqTYnu65AS+htgT01yjm+aUFLKdUJ6A+sBVpprXN9VfuBViaF1ZSmAf8FeH3lOCBfa+32lYNtnycBh4B3fN1Mbymlwgnyfa213gu8BOzGSOQFQDrBva9ramj/nlGOC7SE3qwopSKAecAjWuvCmnW+p38H1TWnSqlrgINa63SzYzmLbMAA4E2tdX+ghGO6V4J0X8dinI0mAYlAOHW7JZqFxty/gZbQ9wLta5Tb+aYFHaWUHSOZf6C1nu+bfKDqzy/f74NmxddEhgNjlFLZGN1pl2D0L8f4/iyH4NvnOUCO1nqtr/wJRoIP9n19GbBTa31Ia+0C5mPs/2De1zU1tH/PKMcFWkL/Eejq+ybcgfElyucmx9TofP3GbwO/aK1fqVH1OXCn7/WdwH/OdmxNSWv9hNa6nda6E8a+Xaq1ngAsA8b5mgXVdmut9wN7lFLdfJMuBTIJ8n2N0dUyVCkV5vu8V2130O7rYzS0fz8H7vBd7TIUKKjRNXNiWuuA+gGuBn4FYTRSTwAAAKRJREFUtgP/bXY8TbSNF2L8CfYTkOH7uRqjP3kJsA1YDLQwO9YmfA9GAF/6XncG1gFZwFwgxOz4Gnlbk4E03/7+DIhtDvsaeBbYAmwC3gdCgnFfAx9hfE/gwviL7J6G9i+gMK7k2w78jHEV0EmvS279F0KIIBFoXS5CCCEaIAldCCGChCR0IYQIEpLQhRAiSEhCF0KIICEJXQghgoQkdCGECBL/HzOrluEcZZk/AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from numpy import linalg as LA\n",
        "print(LA.norm(model.weights, 2))\n",
        "print(LA.norm(model_ridge.weights, 2))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aeUDn_EfId-K",
        "outputId": "6359deab-8946-45ae-979f-6c2d2d894b3d"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "187.17545901615978\n",
            "161.7645420688599\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Liear regression using one layer FCN"
      ],
      "metadata": {
        "id": "CqNxAgqwgg0N"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Without ridge penalty"
      ],
      "metadata": {
        "id": "SFz3g7zwZpRJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class FCN_regression:\n",
        "    def __init__(self, input_size, lr=1e-3, num_epochs=1000, l2_penalty=0):\n",
        "        self.lr = lr\n",
        "        self.num_epochs = num_epochs\n",
        "        self.l2_penalty = l2_penalty\n",
        "        self.train_loss = []\n",
        "        self.test_loss = []\n",
        "        self.model = nn.Linear(input_size, 1)\n",
        "        self.criterion = nn.MSELoss()\n",
        "        self.optimizer = torch.optim.SGD(self.model.parameters(), lr=self.lr)  \n",
        "\n",
        "    def fit(self, X_train, y_train, X_test, y_test):\n",
        "        # initialize weights to be an array of zeros\n",
        "        weights = np.zeros(X_train.shape[1])\n",
        "        # iteration counter\n",
        "        epoch = 0 \n",
        "        # convert to tensor\n",
        "        X_train = torch.from_numpy(X_train).float()\n",
        "        X_test = torch.from_numpy(X_test).float()\n",
        "        y_train = torch.from_numpy(y_train).float()\n",
        "        y_test = torch.from_numpy(y_test).float()\n",
        "        \n",
        "        for epoch in range(self.num_epochs):\n",
        "            # forward pass to calculate prediction\n",
        "            y_predicted = self.model(X_train)\n",
        "            # get current MSE loss\n",
        "            loss_train = self.criterion(y_predicted, y_train)\n",
        "            self.train_loss.append(loss_train.item())\n",
        "            # add L2 penalty\n",
        "            l2_norm = sum(p.pow(2.0).sum() for p in self.model.parameters())\n",
        "            loss_train = loss_train + self.l2_penalty * l2_norm\n",
        "            # backward pass to perform SGD\n",
        "            self.optimizer.zero_grad()\n",
        "            loss_train.backward()\n",
        "            self.optimizer.step()\n",
        "            # calculate and store test loss without gradient\n",
        "            with torch.no_grad():\n",
        "                y_test_pred = self.predict(X_test)\n",
        "                loss_test = self.criterion(y_test_pred, y_test)\n",
        "                self.test_loss.append(loss_test.item())\n",
        "\n",
        "        return self\n",
        "\n",
        "    # make class predictions for test set\n",
        "    def predict(self, x):\n",
        "        model = self.model.eval()\n",
        "        outputs = model(x)\n",
        "        model = self.model.train()\n",
        "        return outputs"
      ],
      "metadata": {
        "id": "_76gW_5NyUGn"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = FCN_regression(input_size=X_train.shape[1], lr=1e-3, num_epochs=10000)\n",
        "model.fit(X_train, y_train, X_test, y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gv0fuKdayUEM",
        "outputId": "77be52ce-8187-4ed3-94e7-0368628da451"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:530: UserWarning: Using a target size (torch.Size([80])) that is different to the input size (torch.Size([80, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n",
            "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:530: UserWarning: Using a target size (torch.Size([20])) that is different to the input size (torch.Size([20, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.mse_loss(input, target, reduction=self.reduction)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<__main__.FCN_regression at 0x7f0d5519a710>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_axis = list(range(model.num_epochs))\n",
        "plt.plot(x_axis, model.train_loss, label = \"train\")\n",
        "plt.plot(x_axis, model.test_loss, ':', label = \"test\")\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "GADrfwBhyUBc",
        "outputId": "3b8c5abb-4d62-4718-f9cf-987310afd6e6"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dfXwV5Z338c8vzyEBAklAIChRWQVRUSKidrutVnmwrXbbtWhdaatld23vut27rXpvrdJu77V3d1fL1oeiYn1GVm1FCyJWWOtdAYOigqAE1BJQiTyE50CS3/5xXSEnSMgBkxyS832/XvPK/K655sw1Z07O78zMNTPm7oiIiGSkugEiInJkUEIQERFACUFERCIlBBERAZQQREQkykp1Aw5XSUmJDxkyJNXNEBHpMpYsWfKRu5e2Nr3LJoQhQ4ZQWVmZ6maIiHQZZvbewabrkJGIiABKCCIiEikhiIgI0IXPIYiIHKq9e/dSXV3N7t27U92UDpWXl0dZWRnZ2dmHNJ8Sgoikjerqanr27MmQIUMws1Q3p0O4Oxs3bqS6upry8vJDmleHjEQkbezevZvi4uJumwwAzIzi4uLD2gtSQhCRtNKdk0GTw13HtEsIU/+wiv9+uybVzRAROeKkXUKY9sIaXlBCEJEU2LJlC7fffvshzzdhwgS2bNnSAS1qKe0SQo+cTHbU1ae6GSKShlpLCPX1B/9Omj17NkVFRR3VrH3SrpdRYW4W25UQRCQFrrvuOlavXs3IkSPJzs4mLy+PPn36sHLlSt5++20uvvhi1q5dy+7du7nmmmuYPHky0Hyrnu3btzN+/Hg+9alP8ac//YlBgwbx5JNPkp+f3y7tS7uEUJCbpT0EEWHKU8t5c/3Wdn3N4QN7ceMXTmp1+s0338yyZctYunQpCxYs4MILL2TZsmX7uodOnz6dvn37smvXLs444wy+/OUvU1xc3OI1Vq1axSOPPMJdd93FJZdcwuOPP87ll1/eLu1Pw4SQyY66hlQ3Q0SE0aNHt7hWYOrUqfz2t78FYO3ataxatepjCaG8vJyRI0cCMGrUKN599912a0/aJYTC3CzWb+neVymKSNsO9ku+sxQUFOwbX7BgAc899xwvvfQSPXr04DOf+cwBryXIzc3dN56ZmcmuXbvarT1pd1K5IDeLHXt0yEhEOl/Pnj3Ztm3bAafV1tbSp08fevTowcqVK1m4cGEnty4N9xB65GTpkJGIpERxcTHnnHMOI0aMID8/n/79+++bNm7cOO68806GDRvGCSecwJgxYzq9fWmXEApz1e1URFLn4YcfPmB5bm4uc+bMOeC0pvMEJSUlLFu2bF/597///XZtW1oeMtq1t4GGRk91U0REjihplxAKc8NOkc4jiIi0lHYJoaApIeiwkYhIC0oIIiICHEJCMLNMM3vVzJ6O8UNm9paZLTOz6WaWHcvNzKaaWZWZvW5mpye8xiQzWxWHSQnlo8zsjTjPVOvA+9MW5mYCsF09jUREWjiUPYRrgBUJ8UPAicDJQD5wVSwfDwyNw2TgDgAz6wvcCJwJjAZuNLM+cZ47gG8lzDfuMNYlKT1ywh7CTu0hiIi0kFRCMLMy4ELg7qYyd5/tEbAYKIuTLgLuj5MWAkVmNgAYC8xz903uvhmYB4yL03q5+8L4WvcDF7fXCu6v6aSybnAnIp3tcG9/DXDrrbeyc+fOdm5RS8nuIdwK/BBo3H9CPFT0t8AzsWgQsDahSnUsO1h59QHKP8bMJptZpZlV1tQc3jMNCtTLSERS5EhPCG1emGZmnwc2uPsSM/vMAarcDrzg7n9s78btz92nAdMAKioqDutCgqY9hG27lRBEpHMl3v76/PPPp1+/fsycOZO6ujq+9KUvMWXKFHbs2MEll1xCdXU1DQ0N3HDDDXz44YesX7+ez372s5SUlDB//vwOaV8yewjnAF80s3eBGcC5ZvYggJndCJQC/5RQfx0wOCEui2UHKy87QHmH6J2fDcDWXXs7ahEi0lXceyG8+lAYb9gb4tceDfGenSFe9niId9eG+M1ZId6xMcRvxauLt33Y5uJuvvlmjjvuOJYuXcr555/PqlWrWLx4MUuXLmXJkiW88MILPPPMMwwcOJDXXnuNZcuWMW7cOL773e8ycOBA5s+f32HJAJJICO5+vbuXufsQYCLwvLtfbmZXEc4LXOruiYeSZgFXxN5GY4Bad38fmAtcYGZ94snkC4C5cdpWMxsTexddATzZrmuZICcrg/zsTGqVEEQkhZ599lmeffZZTjvtNE4//XRWrlzJqlWrOPnkk5k3bx7XXnstf/zjH+ndu3entemT3MvoTuA94KXYS/QJd/8JMBuYAFQBO4FvALj7JjP7KfBynP8n7r4pjl8N/IbQW2lOHDpM7/xsJQQRgW/8vnk8M7tlnNOjZZzXu2VcUNwy7tl8o7pkuDvXX389f/d3f/exaa+88gqzZ8/mRz/6Eeeddx4//vGPD+m1D9chJQR3XwAsiOMHnDf2FPp2K9OmA9MPUF4JjDiUtnwSvfOz2bJTCUFEOlfi7a/Hjh3LDTfcwNe+9jUKCwtZt24d2dnZ1NfX07dvXy6//HKKioq4++67W8xbUlLSYe1Lu7udgvYQRCQ1Em9/PX78eC677DLOOussAAoLC3nwwQepqqriBz/4ARkZGWRnZ3PHHXcAMHnyZMaNG7fvXEJHsPCDvuupqKjwysrKw5r3qvsqqd68k2f+8dPt3CoROZKtWLGCYcOGpboZneJA62pmS9y9orV50u5eRhD2ENTLSESkpbRNCDpkJCLSUtomhB17Gtjb8LELr0Wkm+uqh8kPxeGuY5omhHAuXYeNRNJLXl4eGzdu7NZJwd3ZuHEjeXl5hzxvevYy6hGuVq7dtZfiwtwUt0ZEOktZWRnV1dUc7r3Quoq8vDzKysrarrif9EwI+c0JQUTSR3Z2NuXl5aluxhErTQ8Z5QCwRQlBRGSftEwIxQUhIWzcvifFLREROXKkZUIo6RnOG2zcXpfiloiIHDnSMiEU5GSSl53BR0oIIiL7pGVCMDOKC3J1yEhEJEFaJgQIh41qtIcgIrJP+iaEghztIYiIJEjfhFCYq3MIIiIJ0jYhFBfmsHHHHhobu+8l7CIihyJtE0JJYS4Nja6rlUVEovRNCPFaBJ1YFhEJ0jYhDOwd7gS4fsuuFLdEROTIkL4JoSgfgPVbdqe4JSIiR4a0TQj9euaSmWHaQxARiZJOCGaWaWavmtnTMS43s0VmVmVmj5pZTizPjXFVnD4k4TWuj+VvmdnYhPJxsazKzK5rv9VrXVZmBkf1ylNCEBGJDmUP4RpgRUL8c+AWdz8e2AxcGcuvBDbH8ltiPcxsODAROAkYB9wek0wmcBswHhgOXBrrdriBRXmsU0IQEQGSTAhmVgZcCNwdYwPOBR6LVe4DLo7jF8WYOP28WP8iYIa717n7O0AVMDoOVe6+xt33ADNi3Q43sCif9bVKCCIikPwewq3AD4Gmp9IXA1vcvT7G1cCgOD4IWAsQp9fG+vvK95untfKPMbPJZlZpZpXt8Qi8gUX5fFC7WxeniYiQREIws88DG9x9SSe056DcfZq7V7h7RWlp6Sd+vcF9erC3wbWXICJCcs9UPgf4oplNAPKAXsAvgSIzy4p7AWXAulh/HTAYqDazLKA3sDGhvEniPK2Vd6jjSgsAWF2zg7I+PTpjkSIiR6w29xDc/Xp3L3P3IYSTws+7+9eA+cBXYrVJwJNxfFaMidOfd3eP5RNjL6RyYCiwGHgZGBp7LeXEZcxql7Vrw7GlhQCsqdneGYsTETmiJbOH0JprgRlm9i/Aq8A9sfwe4AEzqwI2Eb7gcfflZjYTeBOoB77t7g0AZvYdYC6QCUx39+WfoF1JKynMoVdeFquVEEREDi0huPsCYEEcX0PoIbR/nd3A37Qy/8+Anx2gfDYw+1Da0h7MjGNLC1lTs6OzFy0icsRJ2yuVmxxXWkjVBu0hiIikfUIYNqAnG7bVsWGb7mkkIukt7RPCKWVFACxbV5viloiIpFbaJ4STBvbCDF6vVkIQkfSW9gmhIDeL40sLlRBEJO2lfUIAOO3oIpa8t5kG3cJCRNKYEgJwzvEl1O7ayxs6jyAiaUwJgZAQAP5/1UcpbomISOooIQAlhbkMH9CL51duSHVTRERSRgkhmnDyUSx5b7MemCMiaUsJIfriqeERDLOWrk9xS0REUkMJITq6uAejh/TloUXvUd/Q2PYMIiLdjBJCgqv+spzqzbv4/Rvvp7opIiKdTgkhweeG9eeE/j35xdy32LWnIdXNERHpVEoICTIyjCkXnUT15l3827Nvpbo5IiKdSglhP2OOLeaKs47hnhff4fEl1alujohIp/kkT0zrtn504XBWfbid7z/2Glt37+XrZw/BzFLdLBGRDqU9hAPIycpg+tfP4LwT+zHlqTeZdO/LLF+v21qISPdm7l3zhm4VFRVeWVnZoctobHTuf+ld/n3e22zbXc/o8r584dSBnHNcMeUlBdprEJEuxcyWuHtFq9OVENpWu2svDy58jydeqWZ1fP5ySWEOJx7Vi+NKCygvKaB/rzxKe+ZS2jOXvgU5FORkkZGhhCEiRw4lhHbk7qyu2cHidzax5L3NVG3YxpqaHWyrqz9g/cLcLApyMynMzaIwN4vc7ExyMjPIycogO9PIjuM5mRlkxyEzI/R2yjAj04wMAzMjMyOMN03LMOLfMA7s22Np2nGxhKApNTVPs5b19pvWNKF52sdfRztIydv3nspB6TPVttysDMaNGHBY87aVENo8qWxmecALQG6s/5i732hm5wG/IJyH2A583d2rzCwXuB8YBWwEvuru78bXuh64EmgAvuvuc2P5OOCXQCZwt7vffFhr28HMjOP7FXJ8v0IuO/NoICSJjTv2sGFrHTXb69iwdTebduxhR1092+sa2F63lx11DWyrq6dubwM799RTu8vZU9/I3oZG9jTEv/WN7G1wGhqdRm8a0DMaRKSFksLcw04IbUmml1EdcK67bzezbOBFM5sD3AFc5O4rzOxq4EfA1wlf+Jvd/Xgzmwj8HPiqmQ0HJgInAQOB58zsL+IybgPOB6qBl81slru/2X6r2XHMjJLCXEoKcztsGZ6QHBKTRaM7jY2OO3hCXQhx086fs2+k1Wnesgruzv47j4nzdNEdy5TQW5Wcrnq0orNlduCh6DYTgoettD2G2XHwOPSK5b2BprvCXQTcFMcfA35l4XjDRcAMd68D3jGzKmB0rFfl7msAzGxGrNslEkJnMDMyrWM/CCIiSV2HYGaZwBLgeOA2d19kZlcBs81sF7AVGBOrDwLWArh7vZnVAsWxfGHCy1bHMprqJ5SfeXirIyIihyup6xDcvcHdRwJlwGgzGwF8D5jg7mXAvcB/dFwzAzObbGaVZlZZU1PT0YsTEUkrh3RhmrtvAeYD44FT3X1RnPQocHYcXwcMBjCzLMLhpI2J5VFZLGut/EDLn+buFe5eUVpaeihNFxGRNrSZEMys1MyK4ng+4eTvCqB3wknhpjKAWcCkOP4V4Pl4HmIWMNHMcs2sHBgKLAZeBoaaWbmZ5RBOPM9ql7UTEZGkJXMOYQBwXzyPkAHMdPenzexbwONm1ghsBr4Z698DPBBPGm8ifMHj7svNbCbhZHE98G13bwAws+8AcwndTqe7+/J2W0MREUmKLkwTEUkTbV2YppvbiYgIoIQgIiKREoKIiABKCCIiEikhiIgIoIQgIiKREoKIiABKCCIiEikhiIgIoIQgIiKREoKIiABKCCIiEikhiIgIoIQgIiKREoKIiABKCCIiEikhiIgIoIQgIiKREoKIiABKCCIiEikhiIgIoIQgIiJRmwnBzPLMbLGZvWZmy81sSiw3M/uZmb1tZivM7LsJ5VPNrMrMXjez0xNea5KZrYrDpITyUWb2RpxnqplZR6ysiIi0LiuJOnXAue6+3cyygRfNbA4wDBgMnOjujWbWL9YfDwyNw5nAHcCZZtYXuBGoABxYYmaz3H1zrPMtYBEwGxgHzGmvlRQRkba1uYfgwfYYZsfBgX8AfuLujbHehljnIuD+ON9CoMjMBgBjgXnuvikmgXnAuDitl7svdHcH7gcubsd1FBGRJCR1DsHMMs1sKbCB8KW+CDgO+KqZVZrZHDMbGqsPAtYmzF4dyw5WXn2A8gO1Y3JcXmVNTU0yTRcRkSQllRDcvcHdRwJlwGgzGwHkArvdvQK4C5jecc3c145p7l7h7hWlpaUdvTgRkbRySL2M3H0LMJ9wjL8aeCJO+i1wShxfRzi30KQslh2svOwA5SIi0omS6WVUamZFcTwfOB9YCfwO+Gys9lfA23F8FnBF7G00Bqh19/eBucAFZtbHzPoAFwBz47StZjYm9i66Aniy/VZRRESSkUwvowHAfWaWSUggM939aTN7EXjIzL4HbAeuivVnAxOAKmAn8A0Ad99kZj8FXo71fuLum+L41cBvgHxC7yL1MBIR6WQWOvZ0PRUVFV5ZWZnqZoiIdBlmtiSe9z0gXaksIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERtJgQzyzOzxWb2mpktN7Mp+02fambbE+JcM3vUzKrMbJGZDUmYdn0sf8vMxiaUj4tlVWZ2XfusmoiIHIpk9hDqgHPd/VRgJDDOzMYAmFkF0Ge/+lcCm939eOAW4Oex7nBgInASMA643cwyzSwTuA0YDwwHLo11RUSkE7WZEDxo2gPIjoPHL/JfAD/cb5aLgPvi+GPAeWZmsXyGu9e5+ztAFTA6DlXuvsbd9wAzYl0REelESZ1DiL/klwIbgHnuvgj4DjDL3d/fr/ogYC2Au9cDtUBxYnlUHctaKz9QOyabWaWZVdbU1CTTdBERSVJSCcHdG9x9JFAGjDazTwN/A/xnRzbuAO2Y5u4V7l5RWlramYsWEen2DqmXkbtvAeYDnwWOB6rM7F2gh5lVxWrrgMEAZpYF9AY2JpZHZbGstXIREelEyfQyKjWzojieD5wPLHH3o9x9iLsPAXbGk8gAs4BJcfwrwPPu7rF8YuyFVA4MBRYDLwNDzazczHIIJ55ntd8qiohIMrKSqDMAuC+eRM4AZrr70wepfw/wQNxj2ET4gsfdl5vZTOBNoB74trs3AJjZd4C5QCYw3d2XH+4KiYjI4bHw473rqaio8MrKylQ3Q0SkyzCzJe5e0dp0XaksIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiKAEoKIiERKCCIiAighiIhIpIQgIiJAEgnBzPLMbLGZvWZmy81sSix/yMzeMrNlZjbdzLJjuZnZVDOrMrPXzez0hNeaZGar4jApoXyUmb0R55lqZtYRKysiIq1LZg+hDjjX3U8FRgLjzGwM8BBwInAykA9cFeuPB4bGYTJwB4CZ9QVuBM4ERgM3mlmfOM8dwLcS5hv3iddMREQOSZsJwYPtMcyOg7v77DjNgcVAWaxzEXB/nLQQKDKzAcBYYJ67b3L3zcA8QnIZAPRy94Xxte4HLm7XtRQRkTYldQ7BzDLNbCmwgfClvihhWjbwt8AzsWgQsDZh9upYdrDy6gOUi4hIJ0oqIbh7g7uPJOwFjDazEQmTbwdecPc/dkQDE5nZZDOrNLPKmpqajl6ciEhaOaReRu6+BZhPPMZvZjcCpcA/JVRbBwxOiMti2cHKyw5QfqDlT3P3CnevKC0tPZSmi4hIG5LpZVRqZkVxPB84H1hpZlcRzgtc6u6NCbPMAq6IvY3GALXu/j4wF7jAzPrEk8kXAHPjtK1mNib2LroCeLI9V1JERNqWlUSdAcB9ZpZJSCAz3f1pM6sH3gNeir1En3D3nwCzgQlAFbAT+AaAu28ys58CL8fX/Ym7b4rjVwO/IfRWmhMHERHpRBY69nQ9FRUVXllZmepmiIh0GWa2xN0rWpuuK5VFRARIx4Tw8Fdhwc2pboWIyBEn/RJCjxLI7ZnqVoiIHHGSOancvVx8W6pbICJyREq/PQQRETmg9EsIrz4Et54M9XWpbomIyBEl/RJCYX8YPAb27kp1S0REjijplxCGfg6+fBfkF6W6JZIKjQ1haFK/Bxr2Nsd7d8He3c1x3faWPx52bYE9O5rjnZugbltzvH0D7N7aHG9dD7tr94vjdHeoXdccNzaGuOn1PhY3QG11aBNAQ/2B46b2NeyFLWub4/o9Md4Z4zrY8ufm9du7e794V4zj+7FnJ2x+r3nvel+8J8Y7WsZ122Hzu83vb922GNeHePfWEDdtj921sOmd5njXlhjHGyHs2gyb1oT3rem937Sm5bbY9E5zvGNjy3h7TVjevnhDaG+TbR/uF38Q1r/J1vdbxrXrwvu5L64OQ2vxlj+HeZpsfi98Hppseme/eE1YZpONq0ObOlD6JYQmXfSCvEPmHv5Bm/6pdtfChhXN/6Sb34MVTzX/k3/wBiya1hz/eSHM/9fm+queg99/v/n9W/YEPH5V8/KW/AYeuqQ5fuk2uO+LzfF//wJ+8/nm+LmbYHrC4y9m/wDuPr85/t3VcPfnmuP/+nrL+OGJcM/Y5vj+i+DeC5vje8a2XP7dnwtdj5vc9dnwmk1+/Wn43d83x3ecDU9d0xzffhbMubY5/tUZMO/G5vg/R8GCf22Op54Gf/z35viWk+BPU8O4N8Itw2HRnSGu3x3il+8Jcd3WEL/yQIh3bgzzvz4jxNs/CPHyJ0Jc++cQr3gqxBtXw60j4O25Ia5ZGeLVz4f4gzfC4dN3Xwzx+ldCvDbezHjtohCvfyXE774IvzwFPlgW4tXPh7hmZYjffibEm+OX8Mqn4ZenQm380lz+2xBvj19qrz8a4p3xhgWvPgRTRzYnwCX3hrghfhYX3xXez6Y75Sy8PbzfTV68JWyfJi/8An79V83xgv/b8rPzhylw7/jmeN4NcH/CZ2XOtfDgV5rj3/9veOSy5vip78J/TWqOf/cPLf8XHr8qlDWZOanlZ2nG18L/UpOHL4G5/6c5fuBL8FzCZ+u+L8DzP6UjZd50000duoCOMm3atJsmT5586DM2NoZ/4h01UP7p9m9YR2hsDF/AZuFXzqp5UNgPsvPhw+XhC6j/CMjrFf5JZ1wGx50L+X3CP920T8MpX4UefeH1mfDgl+H0KyCvNyx7DH7793DGVZBbGL7gn7kWxlwdXn/Z4+FDeM41kJUT/skrp4fpGZmwen74IjjjW6F91S/Dh8tgZPzH+eAN2PY+jPjrENesDL8kh8WksOkdsAwYekGIt30Aub3g2M+EeOcm6DkAjjk7xHt2QN9yGDw6xPW7oeQvYFD8Ymioh/7DYODI+OY5DDgFjjo5hJYBA0+D/sNDnJkd5i09McRZeVB2BpQMDXFOAQw+E4qPC3FeLzh6TGgDhPf4mLOh6OgQF5TAMedA73i/xsKjYMinoNfAEPccEOMB4ECvQSHueRRgYb4h54RDm2bQe3B4vcLS0PY+x4TlFZSEuG85HH0W9CgGy4S+x4b29egbtk/x8SHOLwrrWjwUjj4zbPvMHCg9AcpGh/XKzIF+J4b1z+0JmbnQbxgMqgifjaw86H8SlI2CnB7h89F/RHj/svMhuwccdUp4f7PzQp0BI0OclQs5hWG7DBwZlpXTEwadDgNODW3L7RmWNeDU0Pbc3qEtR50MGRmhzYPPDMs0g7yicPj3qBHN22LwmNBGgB59wrr3i9u6R0mM47YuKA1x6Qkx7gfHnBU+TxC2wTFnNX8WevYP733TZ6Hpc9n32BD3Ghjj8oT4HOgzJMS9y2J8TIiL4rZt+uwUHRM+C02fnT7l4bPQOz4NoM8QGPKXzZ+lwzBlypT3b7rppmmtTU/PW1fMuQ4GnwEjvty+jTpcOzc1/0PUVsOf/jN8Yfc/CdYuDr9iLn88fEm+80L4pTDpaSj/S1izAB67Ei5/LPzjVVeGX0pjfxY+QB++CW/9HkZ9EwqKwxfw+lfDF3BuYUgw29ZD6bDwhb9nRzhUkN83/BM2NoZ/Pj3VVKTLa+vWFemZEFJp91ZY+lD4hTtoFHxUBb8aBRffEX5Vb1wddnO/dAcM+0I4hvjyXXDqpeGXSt122PJe+PWQ0yPVayMiXYjuZdSahr2dcx6hoR6e/DYsfSTEmdkw95+hKh7H7TMELvhZ2DWGsPt5/dqQDCAcWjjvx827rbmFYc9ByUBE2ll6JoQ3n4SfHQUfreqY13/2hnBCCyAzK5zE3Rp7F2Tnww+q4NPfb55+9neav/B1eEZEUiT9bl0B4cTX2f8rfDm3hw/fhLULoeKbId72fugh0uRbz7es36Nv+yxXRKQdpWdC6FsOn7vpk71GU68fCOcEltwHp0wMh3L++i79yheRLic9DxlB+EKvefvw5v1gGdw2Gt5/PcSf+h784+vNx/WVDESkC0rfhPDSbXDbGYd25V/TxVq9y0K/773xis+CEh0GEpEuLz0PGQGcOCFcrJNTkFz92T8MF1Vd8WSY75vPdGz7REQ6WfomhL7HNl9h2JrE8wT9TwoXjjXWh66jIiLdTPoeMoJwLcLrM2Htyx+ftu0DmD4WVs4O8ahJcN4NSgYi0m2leULYA8//S7ifT5OmO0fm9w33W2msT03bREQ6WZsJwczyzGyxmb1mZsvNbEosLzezRWZWZWaPmllOLM+NcVWcPiThta6P5W+Z2diE8nGxrMrMrmv/1WxFTgFcNhPOj3cQfG4K3D4m3L8nKwcmPQXDv3jw1xAR6SaS2UOoA85191OBkcA4MxsD/By4xd2PBzYDV8b6VwKbY/ktsR5mNhyYCJwEjANuN7NMM8sEbgPGA8OBS2PdztHvxPDlD+H2EWdcGfYcQqM7rRkiIqnWZkLwIB5HITsODpwLNB1ruQ+4OI5fFGPi9PPMzGL5DHevc/d3gCpgdByq3H2Nu+8BZsS6ne/ECeGaguy8lI9w+24AAAVqSURBVCxeRCSVkjqHEH/JLwU2APOA1cAWd286wF4NxJt2MwhYCxCn1wLFieX7zdNa+YHaMdnMKs2ssqamJpmmi4hIkpJKCO7e4O4jgTLCL/oTO7RVrbdjmrtXuHtFaWlpKpogItJtHVIvI3ffAswHzgKKzKzpOoYyoOlhoeuAwQBxem9gY2L5fvO0Vi4iIp0omV5GpWZWFMfzgfOBFYTE0PTA0UnAk3F8VoyJ05/38BSeWcDE2AupHBgKLAZeBobGXks5hBPPs9pj5UREJHnJXKk8ALgv9gbKAGa6+9Nm9iYww8z+BXgViE8G5x7gATOrAjYRvuBx9+VmNhN4E6gHvu3uDQBm9h1gLpAJTHf35e22hiIikhQ9QlNEJE3oEZoiIpIUJQQREQG68CEjM6sB3jvM2UuAj9qxOV2B1rn7S7f1Ba3zoTrG3Vvts99lE8InYWaVBzuO1h1pnbu/dFtf0Dq3Nx0yEhERQAlBRESidE0I01LdgBTQOnd/6ba+oHVuV2l5DkFERD4uXfcQRERkP0oIIiICpFlCSNmjOjuAmQ02s/lm9mZ8tOk1sbyvmc0zs1Xxb59YbmY2Na7762Z2esJrTYr1V5nZpNaWeSSIz+Z41cyejnG7Pcr1SGVmRWb2mJmtNLMVZnZWd97OZva9+JleZmaPWHiMb7fbzmY23cw2mNmyhLJ2265mNsrM3ojzTDVL4hGQ7p4WA+HGeauBY4Ec4DVgeKrb9QnWZwBwehzvCbxNeATp/wOui+XXAT+P4xOAOYABY4BFsbwvsCb+7RPH+6R6/Q6y3v8EPAw8HeOZwMQ4fifwD3H8auDOOD4ReDSOD4/bPhcoj5+JzFSvVxvrfB9wVRzPAYq663YmPBzrHSA/Yft+vTtuZ+DTwOnAsoSydtuuhLtJj4nzzAHGt9mmVL8pnfjmnwXMTYivB65Pdbvacf2eJNya/C1gQCwbALwVx38NXJpQ/604/VLg1wnlLeodSQPhWRl/IDy+9en4Qf8IyNp/GxPunntWHM+K9Wz/7Z5Y70gcCM8TeYfYAWT/7dfdtjPNT1DsG7fb08DY7rqdgSH7JYR22a5x2sqE8hb1WhvS6ZBR0o/q7GribvJpwCKgv7u/Hyd9APSP45/4EaZHgFuBHwKNMS6m/R7leqQqB2qAe+OhsrvNrIBuup3dfR3wb8CfgfcJ220J3X87N2mv7Tooju9fflDplBC6JTMrBB4H/tHdtyZO8/DToFv0KzazzwMb3H1JqtvSybIIhxXucPfTgB2EQwn7dLPt3Ae4iJAIBwIFwLiUNipFUrFd0ykhdLtHdZpZNiEZPOTuT8TiD81sQJw+ANgQy7v6I0zPAb5oZu8CMwiHjX5J+z3K9UhVDVS7+6IYP0ZIEN11O38OeMfda9x9L/AEYdt39+3cpL2267o4vn/5QaVTQuhWj+qMPQbuAVa4+38kTEp8hOn+jza9IvZWGAPUxl3TucAFZtYn/jq7IJYdUdz9encvc/chhG33vLt/jfZ7lOsRyd0/ANaa2Qmx6DzCUwe75XYmHCoaY2Y94me8aX279XZO0C7bNU7bamZj4vt4RcJrtS7VJ1U6+QTOBEJvnNXAP6e6PZ9wXT5F2J18HVgahwmE46d/AFYBzwF9Y30Dbovr/gZQkfBa3wSq4vCNVK9bEuv+GZp7GR1L+EevAv4LyI3leTGuitOPTZj/n+P78BZJ9LxI9QCMBCrjtv4doTdJt93OwBRgJbAMeIDQU6jbbWfgEcJ5kr2EPcEr23O7AhXxPVwN/Ir9OiYcaNCtK0REBEivQ0YiInIQSggiIgIoIYiISKSEICIigBKCiIhESggiIgIoIYiISPQ/gRJMK2mdh+wAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### With ridge penalty"
      ],
      "metadata": {
        "id": "PVpVrH7hZrHn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_ridge = FCN_regression(input_size=X_train.shape[1], lr=1e-3, num_epochs=10000, l2_penalty=0.1)\n",
        "model_ridge.fit(X_train, y_train, X_test, y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cu2Gx4UjyT-z",
        "outputId": "55a5e2e8-751f-4bc1-954c-8842e39fa330"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<__main__.FCN_regression at 0x7f0d4e299d10>"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "jDvW3SWahOBF"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_axis = list(range(model.num_epochs))\n",
        "plt.plot(x_axis, model.train_loss, label = \"train-without-ridge\")\n",
        "plt.plot(x_axis, model.test_loss, ':', label = \"test-without-ridge\")\n",
        "plt.plot(x_axis, model_ridge.train_loss, '--', label = \"train-with-ridge\")\n",
        "plt.plot(x_axis, model_ridge.test_loss, '-.', label = \"test-with-ridge\")\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "O-GAT_A1yT8M",
        "outputId": "e96a85d7-d1c9-4ff8-97a5-587135652db8"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD4CAYAAADsKpHdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU5dn48e89SyaTPUAIhAAJsskaICCIUVBBFEWpuCAqrkjVovZXqrQKSGtf+ypqsSovKKLWCi61WkTBBQpSFgOiBQQJEiWALIEA2Zd5fn+ck8m+AIFA5v5c11w5z3bOc+ZM5j7bPEeMMSillFKOxu6AUkqpM4MGBKWUUoAGBKWUUjYNCEoppQANCEoppWyuxu7AiWrRooVJSEho7G4opdRZY/369QeNMTE1lZ+1ASEhIYHU1NTG7oZSSp01ROTH2sr1lJFSSilAA4JSSimbBgSllFLAWXwNQSlVvaKiIjIyMsjPz2/srqhGEhwcTHx8PG63+7jaaUBQqonJyMggPDychIQERKSxu6NOM2MMmZmZZGRkkJiYeFxt9ZSRUk1Mfn4+zZs312AQoESE5s2bn9ARogYEpZogDQaB7US3vwYEpZRSQAAGhGF/v5Upn/21sbuhVJOWlZXFiy++eNztrrjiCrKyshq0L6XzrNyn5cuXc+WVVzbIMpYvX85//vOfete/66672LJlS5X8+fPnc//99zdIn05EwAWEn/N2sPng9sbuhlJNWk0Bobi4uNZ2ixcvJioqqkH7UjrPEw1S9XE8AaGkpISXX36Zbt26nZK+nIyACwhiPOQX5zZ2N5Rq0h555BF27NhBUlIS/fv3JyUlhVGjRvm/BK+55hr69etH9+7dmTNnjr9dQkICBw8eJD09nXPPPZe7776b7t27M3z4cPLy8qos56mnnmLWrFkAPPTQQ1x88cUAfPHFF4wbN67CPMv3afLkyQBkZ2czZswYunbtyrhx4yh9guTnn39Onz596NmzJ3fccQcFBQUV5gWQmprKkCFDSE9PZ/bs2Tz77LMkJSWxcuXKKv0MCwvj//2//0fv3r1ZvXo1Q4YM8Q+98+qrr9K5c2cGDBjAqlWr/G127NjBwIED6dmzJ48++ihhYWEV1rt///706tWLadOmncgmqlbA3XbqlGAKfFU/WEo1RY//azNb9hxt0Hl2i4tg2lXda63z5JNPsmnTJjZu3Mjy5csZOXIkmzZt8t8GOW/ePJo1a0ZeXh79+/fn2muvpXnz5hXmsX37dt566y3mzp3L9ddfz3vvvcfNN99coU5KSgozZ85k0qRJpKamUlBQQFFREStXruTCCy+ssU9g7dV//fXXbN68mbi4OAYPHsyqVatITk7mtttu4/PPP6dz587ceuutvPTSSzz44IPVrmtCQgITJ04kLCyM3/zmN9XWycnJ4bzzzmPmzJkV8vfu3cu0adNYv349kZGRDB06lD59+gDwwAMP8MADDzB27Fhmz57tb7N06VK2b9/OunXrMMYwatQoVqxYUWV9T0TAHSG4CKbQp0cISp1OAwYMqHBP/KxZs+jduzcDBw5k165dbN9e9TRuYmIiSUlJAPTr14/09PQqdfr168f69es5evQoHo+HQYMGkZqaysqVK0lJSalXv+Lj43E4HCQlJZGens62bdtITEykc+fOAIwfP54VK1ac4JpbnE4n1157bZX8tWvXMmTIEGJiYggKCuKGG27wl61evZrrrrsOgJtuusmfv3TpUpYuXUqfPn3o27cvW7durfb9OxEBd4QQSjuKS/QIQQWGuvbkT5fQ0FD/9PLly/nss89YvXo1ISEhDBkypNp75j0ej3/a6XSSl5fHrl27uOqqqwCYOHEiEydOJDExkfnz53P++efTq1cvli1bRlpaGueee26d/aq8jLqucbhcLnw+H0CN9/mXlJTQr18/AEaNGsWMGTMIDg7G6XTW2Z/6MMYwZcoU7rnnngaZX3kBd4TQ0XUz4dk3111RKXXCwsPDOXbsWLVlR44cITo6mpCQELZu3cqaNWvqPd+2bduyceNGNm7cyMSJEwHrtNHTTz/NhRdeSEpKCrNnz6ZPnz5V7sWvrU/ldenShfT0dNLS0gB44403uOiiiwDr9ND69esBeO+996qdt9Pp9PdxxowZtS7rvPPO49///jeZmZkUFRXxzjvv+MsGDhzoX8aCBQv8+Zdddhnz5s0jOzsbgN27d7N///4616s+Ai4ghHmc5BTUvheglDo5zZs3Z/DgwfTo0cN/AbfUiBEjKC4u5txzz+WRRx5h4MCBJ7WslJQU9u7dy6BBg4iNjSU4OLja00W19am84OBgXn31Va677jp69uyJw+HwB59p06bxwAMPkJycXGGP/6qrruL999+v8aJyTVq3bs306dMZNGgQgwcPrnBU89xzz/HMM8/Qq1cv0tLSiIyMBGD48OHcdNNNDBo0iJ49ezJmzJh6Bbr6kNKr6meb5ORkcyIPyBm74Ck2H/2Cbyd8fAp6pVTj++677+p1ukSd2XJzc/F6vYgICxYs4K233uKDDz6od/vqPgcist4Yk1xTm4C7hoAzF+PJwGd8OCTgDpCUUmeJ9evXc//992OMISoqinnz5p3yZQZcQAh1Wxe3juZnE+WNaOTeKKVU9VJSUvjmm29O6zIDbhc5LMgKCAdzG+acm1JKNRUBFxDC/QGhYX+so5RSZ7uACwixIa0ozu5EQVFj90Qppc4s9Q4IIuIUka9FZJGdflNEtonIJhGZJyJuO19EZJaIpInItyLSt9w8xovIdvs1vlx+PxH5r91mlpzCwdy7N0sib9edRLhanapFKKXUWel4jhAeAL4rl34T6Ar0BLzAXXb+5UAn+zUBeAlARJoB04DzgAHANBGJttu8BNxdrt2IE1iXegn1WNfRcwpKTtUilAp4JzOy6HPPPUdu7okNL7Nnzx7GjBkDwMaNG1m8eLG/bPr06Tz99NMnNN/K5s+fz549e+pd//zzz682/7bbbuPdd99tkD41hHoFBBGJB0YCL5fmGWMWGxuwDoi3i64GXreL1gBRItIauAz41BhzyBhzGPgUGGGXRRhj1tjzeh24pqFWsLIc335Cz3mS//y87FQtQqmA11gBIS4uzv8FWzkgNKT6BoTSoTCO51kJjam+RwjPAb8FfJUL7FNFtwCf2FltgF3lqmTYebXlZ1STX4WITBCRVBFJPXDgQD27XlFUsBdHUBaH8jJPqL1Sqm6Vh5qubrjmnJwcRo4cSe/evenRowcLFy5k1qxZ7Nmzh6FDhzJ06NAq8x05ciTffvstAH369PEPDTF16lTmzp1Leno6PXr0oLCwkKlTp7Jw4UKSkpJYuHAhAFu2bGHIkCF06NDBP2w2wDPPPEOPHj3o0aMHzz33HIB/XqWefvpppk+fzrvvvktqairjxo0jKSmpyrDcy5cvrzLcd+nQ1cYY7r//frp06cKll15aYciJxYsX07VrV/r168ekSZP8D+/JycnhjjvuYMCAAfTp0+e4fpx2vOoMCCJyJbDfGLO+hiovAiuMMfX/vfYJMsbMMcYkG2OSY2JiTmgercKbAXCkQO8yUgHi1ZHw9ZvWdEmRlf7G+oKkMNdKb7LH5ck/YqW3fGilczKt9Db7l/3H9tVrkU8++STnnHMOGzduZNiwYf7hmjdu3Mj69etZsWIFn3zyCXFxcXzzzTds2rSJESNGMGnSJOLi4li2bBnLllU9ik9JSWHlypUcOXIEl8vlf35A5eGug4KCmDFjBjfccAMbN270jyK6detWlixZwrp163j88ccpKipi/fr1vPrqq6xdu5Y1a9Ywd+5cvv766xrXbcyYMSQnJ/Pmm2+yceNGvF5vlTobNmzgL3/5C99//32F/Pfff59t27axZcsWXn/9df+RQ35+Pvfccw8ff/wx69evp/wO7xNPPMHFF1/MunXrWLZsGZMnTyYnJ6de2+F41ecIYTAwSkTSgQXAxSLyNwARmQbEAL8uV3830LZcOt7Oqy0/vpr8UyImNBTjc3GkUH+HoNTpUNNwzT179uTTTz/l4YcfZuXKlf6xemqTkpLCihUrWLVqFSNHjiQ7O5vc3Fx27txJly5d6mw/cuRIPB4PLVq0oGXLluzbt48vv/yS0aNHExoaSlhYGL/4xS+Oazyi6lQe7rvUihUrGDt2LE6nk7i4OP8DfbZu3UqHDh38bcaOHetvs3TpUp588kmSkpL8I8P+9NNPJ9W/mtT5S2VjzBRgCoCIDAF+Y4y5WUTuwroucIkxpvyppA+B+0VkAdYF5CPGmL0isgT4U7kLycOBKcaYQyJyVEQGAmuBW4HnG2j9qnA5HeALIbtIjxBUgLj9o7Jpp7tiOiikYjo4smI6tHnFdHjscS++tuGaN2zYwOLFi3n00Ue55JJLmDp1aoXy999/n8cffxyAl19+mf79+5OamkqHDh0YNmwYBw8eZO7cuf7hputyPMNdlx/qGmoe7nrt2rX+dZsxYwYREREVhvs+WcYY3nvvvXoFvJN1Mr9DmA3EAqtFZKOIlG7JxcAPQBowF7gXwBhzCPgD8JX9mmHnYdd52W6zAzilI88FFfQmyBdfd0Wl1AkpPxx0TcM179mzh5CQEG6++WYmT57Mhg0bqrQdPXq0fyjp5ORkgoKCaNu2Le+88w6DBg2qMPR1bX2oTUpKCv/85z/Jzc0lJyeH999/n5SUFGJjY9m/fz+ZmZkUFBSwaNGiaud93nnn+fs4atSoWpd14YUXsnDhQkpKSti7d6//tFiXLl344Ycf/A8BKr3mUfr+Pf/88/7He9Z2OutkHddYRsaY5cBye7ratvadQvfVUDYPqDJCkzEmFehRtcWpEVs0lqii4NO1OKUCTvmhpi+//HL/cM1gXWD929/+RlpaGpMnT8bhcOB2u3nppZcAmDBhAiNGjPBfS6gsJSWFzz//HK/XS0pKChkZGdUOdz106FD/qZYpU6bU2Ne+ffty2223MWDAAADuuusu/2Msp06dyoABA2jTpg1du3b1t7ntttuYOHEiXq+X1atXV3sdoTqjR4/miy++oFu3brRr187/nni9Xl588UVGjBhBaGgo/fv397d57LHHePDBB+nVqxc+n4/ExMQKwakhBdzw1wA3zllNic/wzsTq7w1W6mymw1+fnbKzswkLC8MYw3333UenTp146KGHTnh+JzL8dcANXQGQ6VnA9qCpdVdUSqnTZO7cuSQlJdG9e3eOHDlySh6RWZeAG/4awONyUeLTi8pKqTPHQw89dFJHBA0hII8QwtzhGMnHZ6r8zk4ppQJWQAaESE8kIobM3CON3RWllDpjBGRAiAq2fgCz+5gOX6GUUqUCMiC0j+hA4eHzyC8IyNVXSqlqBeQ3Yvfm3Sn4eTQuE9XYXVGqSTrR0U6vuOIKsrKyGrQvpfOs3Kfly5f7B5A7XmfLcNbHKyADQpTXDfg4kH1qBohSKtDVFBBqGyoCrBE/o6IadketdJ4nMyR3qbNtOOvjFZABwRtcRFjX37N01z8auytKNUnlh7/u379/leGgr7nmGvr160f37t2ZM2eOv11CQgIHDx4kPT2dc889l7vvvpvu3bszfPjwKsNMAzz11FP+Yawfeugh/2BxX3zxBePGjaswz8pDcoP1Y7AxY8bQtWtXxo0bR3U/1D2bh7M+XgH5O4S2kdFgHBzIO9jYXVHqlLv9k9ur5F2WcBk3dr2RvOI87v3s3irlV3e8mms6XsPh/MP8evmvK5S9OuLVOpf55JNPsmnTJjZu3Mjy5csZOXIkmzZt8o/mOW/ePJo1a0ZeXh79+/fn2muvpXnz5hXmsX37dt566y3mzp3L9ddfz3vvvcfNN99coU5KSgozZ85k0qRJpKamUlBQQFFRUZXhsCv3Cawv+q+//prNmzcTFxfH4MGDWbVqFRdccEGV9dmwYUOF/pcqP5z1vn376NatG3fccYd/OOsVK1aQmJhYYfTS0uGs582bR1ZWFgMGDODSSy9t0AHxTlRgHiEEuRBfGFkFhxu7K0oFhMrDQc+aNYvevXszcOBAdu3axfbt26u0SUxMJCkpCYB+/fr5B34rr1+/fqxfv56jR4/i8XgYNGgQqamprFy5strxjarrV3x8PA6Hg6SkpGqXUV3/S53pw1kfr4A8QgBwmQiOFTXsxSulzkS17dF7Xd5ay6ODo+t1RFCX8nu/y5cv57PPPmP16tWEhIT4vxQrqzxUdV5eHrt27eKqq64CYOLEiUycOJHExETmz5/P+eefT69evVi2bBlpaWn1Gs+puuGwm9Jw1scrII8QADyOSHJLNCAodSrUNvT0kSNHiI6OJiQkhK1bt7JmzZp6z7dt27b+oaYnTpwIUGEI7JSUFGbPnk2fPn0QkXr3qbymNJz18QrYgBDnOh933oDG7oZSTVL54a9LL+CWGjFiBMXFxZx77rk88sgjDBw48KSWlZKSwt69exk0aBCxsbEEBwdXe7qotj6dqNGjR9OpUye6devGrbfeWu1w1v369SM8PNz/RLjHHnuMoqIievXqRffu3XnssccapC8NISCHvwZ49J//5aNv9/L11OEN2CulGp8Of31maOjhrI+XDn99HKJDnGQVHiSvqLCxu6KUaoLOhOGsj1fAXlQ+SCphnZ7iv/uSGBDfrbG7o5RqYs6E4ayPV8AeIbSPiAPg+8yMRu6JUkqdGQI2IHSNaQvAjsMaEJRSCgI4IHRr2RZjhF1H9zR2V5RS6oxQ74AgIk4R+VpEFtnpRBFZKyJpIrJQRILsfI+dTrPLE8rNY4qdv01ELiuXP8LOSxORRxpu9WoWE+aF4gj25f58OhanlFJnvOM5QngA+K5c+s/As8aYjsBh4E47/07gsJ3/rF0PEekG3Ah0B0YAL9pBxgm8AFwOdAPG2nVPKREhIv8qwov0twhKNbSTGVn0ueeeIzc394Ta7tmzhzFjxgCwceNGFi9e7C+bPn06Tz/99HHPMzU1lUmTJlVbVjpwXlNRr4AgIvHASOBlOy3AxUDpwN+vAdfY01fbaezyS+z6VwMLjDEFxpidQBowwH6lGWN+MMYUAgvsuqfcOd4h5Bw953QsSqmA0lgBIS4uzv88gsoB4UQUFxeTnJzsH1G1qavvEcJzwG+B0qfSNweyjDGlg5tnAG3s6TbALgC7/Ihd359fqU1N+adcTIQhI3czPuOru7JSqt4qDzX91FNP0b9/f3r16sW0adMAaxjokSNH0rt3b3r06MHChQuZNWsWe/bsYejQoQwdOrTKfEeOHMm3334LQJ8+fZgxYwYAU6dOZe7cuaSnp9OjRw8KCwuZOnUqCxcuJCkpyT90xJYtWxgyZAgdOnSo8Ut++vTp3HLLLQwePJhbbrmlwoN0MjMzGT58ON27d+euu+6qMFz2H/7wB7p06cIFF1zA2LFj/UcjO3bs8P9iOSUlha1btzbQu9zw6vwdgohcCew3xqwXkSGnvku19mUCMAGgXbt2Jz2/vKD1FLacTXrWCDpEn/z8lDoT/XjLrXXWCRsyhOZ33uGvHzl6NFG/GE3x4cPsnvRAhbrt33i9zvmVH2p66dKlvPvuu6xbtw5jDKNGjWLFihUcOHCAuLg4PvroI8Aa4ygyMpJnnnmGZcuW0aJFiyrzTUlJYeXKlbRv3x6Xy8WqVasAWLlyJbNnz/bXCwoKYsaMGaSmpvLXv/4VsL7ot27dyrJlyzh27BhdunThl7/8JW63u8pytmzZwpdffonX62X58uX+/Mcff5wLLriAqVOn8tFHH/HKK68A8NVXX/Hee+/xzTffUFRURN++fenXrx8AEyZMYPbs2XTq1Im1a9dy77338sUXX9T5HjaG+hwhDAZGiUg61umci4G/AFEiUhpQ4oHd9vRuoC2AXR4JZJbPr9SmpvwqjDFzjDHJxpjkmJiYenS9dl2adwDgq93bTnpeSqnqLV26lKVLl9KnTx/69u3L1q1b2b59Oz179uTTTz/l4YcfZuXKlf6xfmqTkpLCihUrWLVqFSNHjiQ7O5vc3Fx27txZr9FDR44cicfjoUWLFrRs2ZJ9+/ZVW2/UqFF4vd4q+StWrPA/k2HkyJFER0cDsGrVKq6++mqCg4MJDw/3j8ianZ3Nf/7zH6677jqSkpK455572Lt3b539bCx1HiEYY6YAUwDsI4TfGGPGicg7wBisIDEeKH3sz4d2erVd/oUxxojIh8DfReQZIA7oBKwDBOgkIolYgeBG4KYGW8NaJMd14ZUd8O2+7dzQY9jpWKRSp1199uhrqu+Kjj7u9pUZY5gyZUq1Qzds2LCBxYsX8+ijj3LJJZcwderUCuXvv/8+jz/+OAAvv/wy/fv3JzU1lQ4dOjBs2DAOHjzI3Llz/XvjdaluuOsXXniBuXPnAvivOTTUcNc+n4+oqCj/Q3nOdCfzO4SHgV+LSBrWNYJX7PxXgOZ2/q+BRwCMMZuBt4EtwCfAfcaYEvs6w/3AEqy7mN62655yfdq0xZQEk5a183QsTqmAUX6o6csuu4x58+aRnZ0NwO7du9m/fz979uwhJCSEm2++mcmTJ7Nhw4YqbUePHu0fijo5OZmgoCDatm3LO++8w6BBgyoMfV1bH2pz3333+ZcRFxdXa90LL7yQv//97wB8/PHHHD5sPWRr8ODB/Otf/yI/P5/s7GwWLVoEQEREBImJibzzzjuAFRy/+eabOvvUWI5rLCNjzHJguT39A9YdQpXr5APX1dD+CeCJavIXAyd3O8AJCPW4cZa05OfcXXVXVkrVW/mhpi+//HJuuukm/9DQYWFh/O1vfyMtLY3JkyfjcDhwu9289NJLgHXOfcSIEcTFxfmfL1BeSkoKn3/+OV6vl5SUFDIyMqod7nro0KH+J5NNmTKlQdZr2rRpjB07lu7du3P++ef7r2X279+fUaNG0atXL2JjY+nZs6f/FNibb77JL3/5S/74xz9SVFTEjTfeSO/evRukPw0tYIe/LjX6lTfIK3Dwyb3jGqBXSjU+Hf66cZQOd52bm8uFF17InDlz6Nu3b6P150SGvw7Y0U5L9Y7py9upuyjxGZwOqbuBUkpVY8KECWzZsoX8/HzGjx/fqMHgRAV8QOjSOpjC4A38e2drLj7nzDyMU0qd+UqvLZzNAnZwu1Ld4sIIjlvA+99/3NhdUarBnK2nglXDONHtH/ABoUfrllDUkq2HtjR2V5RqEMHBwWRmZmpQCFDGGDIzMwkODj7utgF/ysjpEKIdndlf9BXFvmJcjoB/S9RZLj4+noyMDA4cONDYXVGNJDg4mPj4+ONup99+QK8W/VmRtZJ1e77h/Pj6/cBFqTOV2+0mMTGxsbuhzkIBf8oI4OouF2KMsPj7NY3dFaWUajQaEICLOiZQkv47nMcuauyuKKVUo9GAAHhcTgYnnsPSLfvw+fRCnFIqMGlAsI3sFcvh0Pn84csTe6iHUkqd7TQg2C7rHofLncO/0hdQWFLY2N1RSqnTTgOCLSTIxaAWYygwWby5+R+N3R2llDrtNCCU87shoyjJa8uLG1/gWGHdw+YqpVRTogGhnMSYMFKiJ5DnO8KfV7/Q2N1RSqnTSgNCJf9z5RU4M69n7cae5BQUN3Z3lFLqtNGAUEmz0CCev+oedvwM97y5ln9s+xCf8TV2t5RS6pTTgFCNCzvH8OS1vVh38COmrfk94xbdzneZ3zV2t5RS6pTSgFCD65Pb8sLISTgyr+O/B77j+kXXc9Oi21mSvqSxu6aUUqdEwD9Csy4HjhUw87ONfPDD2zgiviKY1lwVO5XkhGjWZM0nMbo1naM70iq0FbGhsYS7wxHRJ68ppc48dT1CUwNCPWVmF/DRf/fw0ebtfPujj7yiAkI7/hmHu+LtqR09V3BxzF14gkpYtO8JwtxhhLnDCfeE4XV56BczkL6xyRSbfP69ewkhQV5C3B68Lg9BThedojsRFxZHblEu3x36Dqc4cYgDp8OJU5y0Dm1NpCeSgpIC9ufsBwFBEBEEITo4Gq/LS0FJAVn5Wf780iAVGRSJ2+mmsKSQ3KLcCsFLRAhxheByuCjyFVFUUlShvSC4HC4c4qDYV1zttRW3w42IUOIrqbbc5XCVlVNNuZSVG6p+NkuHJ/cZX7Xj/TsdTn95dRxiHRTX9LkvfT/qKlfqbKQB4RQoKvGxde8xtu8/xpZ9P7M1M42DefvJKjxIztGWHDuSCI5cvG1fQxz5iDMPceSDo4TCA8MozByCuDMJ6/hUlXkX7rsac3QwDs9e3O2erVLuyhyLK28AeHaSHzOrSnnEkTvxFCZR6N7CkaiXqpRHHrmPoKKuFAR9zdGIeVXKo4/8P4KKE8nzrOZo2JtVyptn/Q53SRy5wcs5GvpulfKWWTNw+ZpzLPgTjoX8q0p566z/xWFCOeL9J9nBn1Ypb5P1PIKTw96F5Hj+XbHQuGh7xFrnQyGvkRO0tkKxwxdK22zrPd3v/T/y3N9UKHf5mhOf/UcAfg75C/murRXK3SVtaJv7KAC7Q/5MgTO9QrmnpANt8n4LwC7vDIqceyqUe4u7EZc/CYAfQ35HseNQhfLQ4j60yr8HgJ2hv8YnuRXKw4sG0bJgPAA7Qu8FqRjUIguH0qLwBnwUsTPsV1QWVTiC5oXXUEI26WG/qVLerOAaootGUCQH+Sn00SrlLQpuILJoKAWODDJC/lilPCZ/PBHFg8hzpLEn5Okq5bF59xBW0odc52b2ep+vUt46bxIhJd3Idq5nn3dulfI2ub8l2NeBo67/cCD49Srl8bmP4fG1Icv9BZmet6uUt8t5ArdpzmH3Yg55PqxSnpA9EyehZAa9T1ZQ1VO/HbJfQHByIOgtjgZV/OyJcdMhx1qn/Z75HHNV+uwRSmKO9Z78HPx/5Dg3Vih3mea0z7Xe0z3BfyHPWfGzF+SLo23eYwBkeP9MgSO9QnmwrwNt8iYDEBXiZv7tA6r0vz7qCgh1Pg9BRIKBFYDHrv+uMWaaiFwCPIV1HSIbuM0YkyYiHuB1oB+QCdxgjEm35zUFuBMoASYZY5bY+SOAvwBO4GVjzJMntLanidvpoGd8JD3jI/kF8UDF97ew2EdOQTHZBSPJLigmp6CYYwXFFBT5KCwuodhnKCgu4nBBd/KLC8kvLiCvOGvGySkAAB6aSURBVJ+ikmK8LVrgcURRWBLLgaJH8ZkSio0PY0ooMT4iwhIIlubklXjYX3wPPmMwxuDD+hvVohteiSTfdOaQ73Z7T9d6GaB583PwSAR5pguHfOPAX2JNxTRrh0fCyTHncsjcYJeVviC2eSvchHHMdCermn2JVtEtcEkIR00vjhBUpbx1dCQOgjhikjhGeJXyuKgQRByEmX5k06xioTiIi/IChmAGkEfrisUSRGy49ZQoF4PIN4kVjjFc4iUm3IMxBmEwBb7OFdq7JILoUKvPJVxEoa9XhXK3NCPK68YAhTKUYl/Fo0OPI4YIr9t6n7iUEl9ehfJgR1xZubkcYyoOkeJ1tPOXtzJXQqWdtRBnByK8bgxCK99VVBbq7EyE100JIdWWh7u6EOZyU0xEteURro6EutwU0Yziasoj3QmEuN0E0xJfNeVRQfF4cRNEa0w15dFBrfHgxk1bpLr2nhiCcOMiAWe15dG4ceOkI+7q+hccgQs3DroSVM0glZFeLw7cQHeCfVU/mxHeIAQHht6E+CIqlTr828ZHX0JNTIVSIchfXiz9CTcVH07jxOsvL5JBFJqOFcpdEuEvbykpFJkeFcrdEu0vDw92V+l7Q6nzCEGsY+RQY0y2iLiBL4EHsL70rzbGfCci9wIDjDG32dO9jDETReRGYLQx5gYR6Qa8BQwA4oDPgNL/yO+BYUAG8BUw1hhT6zMtG/MIQSmlzkZ1HSHUeZeRsWTbSbf9Kt1lLA2jkUDp8fPVwGv29LvAJXZQuRpYYIwpMMbsBNKwgsMAIM0Y84OxdpkW2HWVUkqdRvV6hKaIOIH1QEfgBWPMWhG5C1gsInnAUWCgXb0NsAvAGFMsIkeA5nZ++UeSZdh5lNYvl39eDf2YAEwAaNeuXX26rpRSqp7q9TsEY0yJMSYJiAcGiEgP4CHgCmNMPPAq8Myp66a/H3OMMcnGmOSYmJi6GyillKq34/phmjEmC1gGXA70NsaUXmpfCJxvT+8G2gKIiAvrdFJm+XxbvJ1XU75SSqnTqM6AICIxIhJlT3uxLv5+B0SKSOlF4dI8gA+B8fb0GOALY125/hC4UUQ8IpIIdALWYV1E7iQiiSISBNxo11VKKXUa1ecaQmvgNfs6ggN42xizSETuBt4TER9wGLjDrv8K8IaIpAGHsL7gMcZsFpG3gS1AMXCfMaYEQETuB5Zg3XY6zxizucHWUCmlVL3oD9OUUipAnPRtp0oppQKDBgSllFKABgSllFI2DQhKKaUADQhKKaVsGhCUUkoBGhCUUkrZNCAopZQCNCAopZSyaUBQSikFaEBQSill04CglFIK0ICglFLKpgFBKaUUoAFBKaWUTQOCUkopQAOCUkopmwYEpZRSgAYEpZRSNg0ISimlAA0ISimlbHUGBBEJFpF1IvKNiGwWkcftfBGRJ0TkexH5TkQmlcufJSJpIvKtiPQtN6/xIrLdfo0vl99PRP5rt5klInIqVlYppVTNXPWoUwBcbIzJFhE38KWIfAycC7QFuhpjfCLS0q5/OdDJfp0HvAScJyLNgGlAMmCA9SLyoTHmsF3nbmAtsBgYAXzcUCuplFKqbnUeIRhLtp102y8D/BKYYYzx2fX223WuBl63260BokSkNXAZ8Kkx5pAdBD4FRthlEcaYNcYYA7wOXNOA66iUUqoe6nUNQUScIrIR2I/1pb4WOAe4QURSReRjEelkV28D7CrXPMPOqy0/o5p8pZRSp1G9AoIxpsQYkwTEAwNEpAfgAfKNMcnAXGDeqeumRUQm2AEo9cCBA6d6cUopFVCO6y4jY0wWsAzrHH8G8A+76H2glz29G+vaQql4O6+2/Phq8qtb/hxjTLIxJjkmJuZ4uq6UUqoO9bnLKEZEouxpLzAM2Ar8ExhqV7sI+N6e/hC41b7baCBwxBizF1gCDBeRaBGJBoYDS+yyoyIy0L676Fbgg4ZbRaWUUvVRn7uMWgOviYgTK4C8bYxZJCJfAm+KyENANnCXXX8xcAWQBuQCtwMYYw6JyB+Ar+x6M4wxh+zpe4H5gBfr7iK9w0gppU4zsW7sOfskJyeb1NTUxu6GUkqdNURkvX3dt1r6S2WllFKABgSllFI2DQhKKaUADQhKKaVsGhCUUkoBGhCUUkrZNCAopZQCNCAopZSyaUBQSikFaEBQSill04CglFIK0ICglFLKpgFBKaUUoAFBKaWUTQOCUkopQAOCUkopmwYEpZRSgAYEpZRSNg0ISimlAA0ISimlbBoQlFJKARoQlFJK2eoMCCISLCLrROQbEdksIo9XKp8lItnl0h4RWSgiaSKyVkQSypVNsfO3ichl5fJH2HlpIvJIw6yaUkqp41GfI4QC4GJjTG8gCRghIgMBRCQZiK5U/07gsDGmI/As8Ge7bjfgRqA7MAJ4UUScIuIEXgAuB7oBY+26SimlTqM6A4KxlB4BuO2Xsb/InwJ+W6nJ1cBr9vS7wCUiInb+AmNMgTFmJ5AGDLBfacaYH4wxhcACu65SSqnTqF7XEOw9+Y3AfuBTY8xa4H7gQ2PM3krV2wC7AIwxxcARoHn5fFuGnVdTfnX9mCAiqSKSeuDAgfp0XSmlVD3VKyAYY0qMMUlAPDBARC4ErgOeP5Wdq6Yfc4wxycaY5JiYmNO5aKWUavKO6y4jY0wWsAwYCnQE0kQkHQgRkTS72m6gLYCIuIBIILN8vi3ezqspXyml1GlUn7uMYkQkyp72AsOA9caYVsaYBGNMApBrX0QG+BAYb0+PAb4wxhg7/0b7LqREoBOwDvgK6CQiiSIShHXh+cOGW0WllFL14apHndbAa/ZFZAfwtjFmUS31XwHesI8YDmF9wWOM2SwibwNbgGLgPmNMCYCI3A8sAZzAPGPM5hNdIaWUUidGrJ33s09ycrJJTU1t7G4opdRZQ0TWG2OSayrXXyorpZQCNCAopZSyaUBQSikFaEBQSill04CglFIK0ICglFLKpgFBKaUUoAFBKaWUTQOCUkopQAOCUkopmwYEpZRSgAYEpZRSNg0ISimlAA0ISimlbBoQlFJKARoQlFJK2TQgKKWUAjQgKKWUsmlAUEopBWhAUEopZdOAoJRSCqhHQBCRYBFZJyLfiMhmEXnczn9TRLaJyCYRmScibjtfRGSWiKSJyLci0rfcvMaLyHb7Nb5cfj8R+a/dZpaIyKlYWaWUUjWrzxFCAXCxMaY3kASMEJGBwJtAV6An4AXusutfDnSyXxOAlwBEpBkwDTgPGABME5Fou81LwN3l2o046TVTSil1XOoMCMaSbSfd9ssYYxbbZQZYB8Tbda4GXreL1gBRItIauAz41BhzyBhzGPgUK7i0BiKMMWvseb0OXNOga6mUUqpO9bqGICJOEdkI7Mf6Ul9brswN3AJ8Yme1AXaVa55h59WWn1FNvlJKqdOoXgHBGFNijEnCOgoYICI9yhW/CKwwxqw8FR0sT0QmiEiqiKQeOHDgVC9OKaUCynHdZWSMyQKWYZ/jF5FpQAzw63LVdgNty6Xj7bza8uOrya9u+XOMMcnGmOSYmJjj6bpSSqk61OcuoxgRibKnvcAwYKuI3IV1XWCsMcZXrsmHwK323UYDgSPGmL3AEmC4iETbF5OHA0vssqMiMtC+u+hW4IOGXEmllFJ1c9WjTmvgNRFxYgWQt40xi0SkGPgRWG3fJfoPY8wMYDFwBZAG5AK3AxhjDonIH4Cv7PnOMMYcsqfvBeZj3a30sf1SSil1Gol1Y8/ZJzk52aSmpjZ2N5RS6qwhIuuNMck1lesvlZVSSgEBGBB2XXMBBx6+tbG7oZRSZ5z6XENoUooO5UD6z43dDaWUOuME3BGCq2t/iovCG7sbSil1xgm4gOCObUXRvn2N3Q2llDrjBFxAcDkOUZKZiS83u+7KSikVQAIuILhjmgFQvDu9cTuilFJnmIALCK4+lwFQnJXXyD1RZwWfz3r50yXWq1RJkfUqVVwAxYVl6aL8iunCXKtOqYJsq06p/KNQVO6zmZdltSmVewgKc8rSOZllaWMg52A16dyydakuXbo8X4mdtvtTUgzZB2pJF1np0vWpnC4utNOFZe9N9v6q6dL3ryi/hnSxnc6z0qXvf2EuHNtXLp1jp33VpwuyrXTpb6+qpI/BsXI3nOQftcr96SNV09n7y9J5Wdb6+tOHK5bnHqpYnnvIer9rSudkWi9/+mDF9CkQcAHBHRcHQGFGtcMlNX3FBZC1q+yfOvcQ/LDc+vADZP0EX79p5QMc2AZfPluW3r0ePvldWTr9S3h/ovXhB9j+KSwYVza/LR/A69eUfQl9swBevrTsS+Grl+HFQWX/lKtfgBfOK+vviqfhrwPK0l/8sWL5kt9XTC/6tTW/Uv+8t2L6ndvgxfPL0m+NhdkpZek3Rlv9K/X6KHjtyrL0q5fD335Rln75Umt9S80ZCu/eXpaePRj+ObEs/eJ58K8HytJ/TYaPJ5el/9Iblj5Wln62h7XOpWZ2gRVPlaWf6gCr/mJN+4rhqXNgzYtWuijXSn8110rnZ1npDa9b6ZwDVnrj36300d1WetO7VjrrR3i6I3z3oZXO3G6lv7cHEti32UqnfW6l92y00jvtcS4z1lnpXWus9I+r4OlOsGeDlf5huZX++VsrvX2plT6wzUpvXWSlD/1gpTe/b6WP2IMm//dtmNkZsu0v6Y1/t9J59mcz9VUrXWifHl43x0qXBqzVf7XSpb58Fp7tXpZe8b8wK6ks/cUT8EK5z+Kn0+ClwWXpJb+DuUPL0osnw7zLytKLHqz4WfrnvdbnrdR7d8HfbyhLvzMe3r6lLL3gJnjvDk4l5/Tp00/pAk6VOXPmTJ8wYcJxt3OGhHBwzmw8QZmEXjbmFPTsFDEGRKy9mp0rwB0MnnA4ugf+/WcIi4WwlrD/O+tLrkUXiIyHH1fD832h7XkQnWC1nXMRdLwUotpa6b9dC12vgog465/5vTuhxxgIj4Wd/4ZFD0HSOAhtAemrrOUljQNvNOxaBxvegKSbwBMGe76GbYuh91hwe+HnTfDTf6DXDeAKggNbrVfPMeBwweGd1l5Z99HW+h3ZZe2pdRtlrffRPWBKoMvlVjp7Pzg90Mn+0s49BMERcI79j5h/1HofEi6w0kV5ENUO2g200r5iiOkM8cll72vLrtDG/2A/iO0OcaVfBAKtekLrXnbSYZXF2l8cDhe06QMtu1lppxvaJENMFyvt8kB8f2jRyU57rW3R/Bwr7fZafWvWwUp7wqDdIGtbgbWN2w+21gGs97z9Bda2BfA2t9Y1wtrRITTGqh/R2up7WEtIGAzhrax0eCurPKyl9X6Ht4b250NYjLVuEXFWOrSFlY6Mh3bnQ0gzECdEtrX6G9LMWvfS99YbZaWj21vp4Eg7nWilPeHWe9P8HGg7wFpPZxA07wjx50FQqJVu0ckqd3ut9650W7m94Aq23tc2yVaZK9h63+OTrbYur7Vd2vSzluX2QmwPK+1wQlCItS3j+oLDAe4QaN0b4vpY70VQmLVtS7d9ULg13bp32baI61v2WfCEW5+bVj3tdIS1rNLPRnCkne5Wlo7vDy3PtbddlJWO6Vq2bdsOsNbZnz6v7LMT0hzaDiz77JyAxx9/fO/06dPn1FQekENXZFx7Ad4+vWn+6AsN3KsTVJhjHfYGR1hfaCtnQqfh1j/ykQxrj3bEn6DPzZC5w/qCv2Y2JI219qb+7yL4xRzrSzRzB3z0axj6e+vDlfUTpM6DPrdYH6SjeyHtM2v+4bHWF+r+76wPdXCE1Zecg9YXhSuo7JSIK9j6J1JKnbXqGroiIANCozIG1s+3vpwTL7ROpfwpzvoCv2iydSrnybYw7A8wcKJV/tl06HEttDvPOtWyd6O1ZxXSrJFXxUBJCcbng5ISxONBHA58BQWY/HwcERGICL6cHHz5+da6G2O1MwDGn4cxuFq1QhwOig8fxpeTS1C89Zykov378WXnlNWvjjjwdEi06v/8M6awkKB21l51YcZuTH7t14zE7SaofXu7fgYg/uUX7NyJKSqqpTU4QkIIio/313cEB+Nu3dpKb9+O8dX+f+aMCPfXz//+e5xRUbhbtsSUlFCwfXutbQGc0c1wx5bVd7VsiatZM3wFBRT+8EOd7V2xsRXqu+PicEZGUpKdQ9FPP9bZ3t2mjV0/m8Iff8STkIAjNJTiw4cp2r2nzvZBCQk4w8rqezp1xOHxUHzgAEU/132buKdzJxweD0X791O8bx/B3bohTidFu3dTnFn3effg7t3L6h/OwtvD2ssv/PFHSrKyam/scOLtaT0ipjA9HV9eHsHnWkcBBdu348vJqa014vFUqI8Ino4dAcjfsgVTWFihvqdrVxzBwXWuU7XL0oBQPePzgQj2SK2n1tLHrFM6599vpZ/qBF1GwKjnrfTqF61Dw/h+Vtrnq7A37isooPjnn/Hl5eHLzcOXl4svNxfjT1t54ZdeSnDnzhT88AOHXp1P8zvvICghgZw1azk0fz6msBBTVFT2t6jI/2VufD7azJyJt2cPjn6yhJ+f+CMJby0gKL4Nh157jQN/mVWhLj5flS/nc5YuIahdOw7OncuBmc/Q5esNOLxefv7Tnzj8+ht1vk3l6x95/590+WodABkPPsSxTz6pta0jPLys/kMPUbDte85Z/BEA6eNuJm/9+lrbB3XoUKG+uN20n/8qAGnDL6Pop59qbR8ycGCF+t7evWnz1P8CsDWpDyY/v7bmRFx1VYX60eNuInbyZEqOHuX7AefV2hag2Z13VKgfO+URmo0fT8GOHfww8so621eu3+aZmURccQU5q1fz0+11n7euXL/9394gJDmZIx9+yJ7fPlxn+8r1z1nyCUHt25P58svsf3pmne0r12+wz95DD3Hs4zPrs9dh8Ud4OnSoc52qU1dACLihKwCyF85i9xMvkvDyC3gGXNLwC/jyWetc94j/sdIHt0Nx2ReC77bPKDpWQvGatRTv30fRPjfFn39C2EV5hKVcQOHu3fx0x53EPvxbwi+9lLyvv+an226vYWFlgtq1J7hzZ0qysji2fBmRvxhNUEICpiCf4v37EbcbCQrCERqKBAUhbhc4nIjTAQ4njtAQANytYgkfMhSH19oL8XTpQtR114GzrC4OQRxOcDr8f52RkQCEDhyE43dTEJf18YoYPtza+y4NwCKA/Vfw55fWj7zqKry9evvXq9ktNxN+ySX++tUGcZerXP1b8R076k/H/Op+Sg4frvW9c4SGVqhfPiC3evT3+HJrP8JwNouuUN8ZFeVPxz31v1DHEYK7dSv/dJunn8Ld1jq6cXi9tHl+Vq1tAf/RTWn94M7WeWhXbCvi//p8ne09nTpVqB/cw9rj9XTpQvwLf62zfYX6L75A0DnWee6QAQOIf/HFuvtfqb7LfgBW+LBh/rLa+OsPH05Qx45IUBAA0ddfT9jgwbU1BahY/8KL/Pkt7r6bqNGja2pmcTrL6k+YUOGIIPa3VpCujcPrrVCfcp/vuCf+aB1dl+OOja29PychII8QCr9ZSebM6TR7aBqePheefGcOpll3RFzwoJVe8ns4tBMz5jXE7ab48GH2z5xJYXq6dQh64GCVWTjCw4n51a9odustFB8+zL4//Q/RN95ASL9+FGdmkr1yJQ5vCI6QEBwhXuuv14t4Q3CEhuAIDvZ/oSqlVHX0lNGpYp9yQgTWvIT55DFKbl2Oq0MPjM/HjstGEH7ppcQ+/Ft8+fmkDRtGULv2BCW0J6hde9ytYnHFtsIV2xJ3y5YV9lCVUupU0FNGNTDGULgpFU/P/sff+HA6/P1GfCm/I/vnYI599B3Zqzvg2fYnEt76O+JwEHnVlXi6WLeTOYKD6bxyZcOugFJKNbCADQiH/+dX7Hv9czr+ayHuTr3q16ggGzxhFBwsJHOFg2MvT8WXV4CzWTPCL7mUsJQL/FVjJk06RT1XSqlTI2ADQuhl18Lrn3PsyzU0q09A+HwGef/+gMzD53Pss88Rj4eIK68k8oorCBkwQM/fK6XOegH7LRbUdwieLl3I+uBjom+7u/o7V3w+wIDDSfaBSHa9nYsjbA3N75lAs1tvxdWscX8HoJRSDSlgf3oqIjS7+WYKtm4l+92Xq1bIyyL7sYs4+qJ1D3XotffS8pGH6bjsC1o++KAGA6VUkxOwAQEg8opheJrB3j8/T/627wHwZdq/qgyOJPPrIg4ttQbeEreb5rfdhjNcn7amlGqa6gwIIhIsIutE5BsR2Swij9v5iSKyVkTSRGShiATZ+R47nWaXJ5Sb1xQ7f5uIXFYuf4SdlyYijzT8atawbqGRtJk5E4LC2HnNNaRdMIDvUy6mKCMdRIh79QPav7vodHVHKaUaVX2OEAqAi40xvYEkYISIDAT+DDxrjOkIHAbutOvfCRy285+16yEi3YAbge7ACOBFEXGKiBN4Abgc6AaMteueFp5BV9Bh0SJa/Op+vL2602x4H//46+7YWP8vGJVSqqmr86KysX65Vvq8Sbf9MsDFwE12/mvAdOAl4Gp7GuBd4K9iXbG9GlhgjCkAdopIGlA6uHiaMeYHABFZYNfdcjIrdjxczZoRc++9p2txSil1RqrXNQR7T34jsB/4FNgBZBlj7EcZkQG0safbALsA7PIjQPPy+ZXa1JRfXT8miEiqiKQeOHCguipKKaVOUL0CgjGmxBiTBMRj7dV3PaW9qrkfc4wxycaY5Bh7MCullFIN47juMjLGZAHLgEFAlIiUnnKKB0qfSbkbaAtgl0cCmeXzK7WpKV8ppdRpVJ+7jGJEJMqe9gLDgO+wAkPpMyjHAx/Y0x/aaezyL+zrEB8CN9p3ISUCnYB1wFdAJ/uupSCsC88fNsTKKaWUqr/6/FK5NfCafTeQA3jbGLNIRLYAC0Tkj8DXwCt2/VeAN+yLxoewvuAxxmwWkbexLhYXA/cZY0oAROR+YAngBOYZYzY32BoqpZSqFx3+WimlAkRdw18H9C+VlVJKldGAoJRSCjiLTxmJyAHgxxNs3gKo+hzLpk3XuekLtPUFXefj1d4YU+M9+2dtQDgZIpJa23m0pkjXuekLtPUFXeeGpqeMlFJKARoQlFJK2QI1IMxp7A40Al3npi/Q1hd0nRtUQF5DUEopVVWgHiEopZSqRAOCUkopIMACQmM9qvNUEJG2IrJMRLbYjzZ9wM5vJiKfish2+2+0nS8iMste929FpG+5eY23628XkfE1LfNMYD+b42sRWWSnG+xRrmcqEYkSkXdFZKuIfCcig5rydhaRh+zP9CYReUusx/g2ue0sIvNEZL+IbCqX12DbVUT6ich/7TazRETq7JQxJiBeWAPn7QA6AEHAN0C3xu7XSaxPa6CvPR0OfI/1CNL/BR6x8x8B/mxPXwF8DAgwEFhr5zcDfrD/RtvT0Y29frWs96+BvwOL7PTbwI329Gzgl/b0vcBse/pGYKE93c3e9h4g0f5MOBt7vepY59eAu+zpICCqqW5nrIdj7QS85bbvbU1xOwMXAn2BTeXyGmy7Yo0mPdBu8zFweZ19auw35TS++YOAJeXSU4Apjd2vBly/D7CGJt8GtLbzWgPb7On/A8aWq7/NLh8L/F+5/Ar1zqQX1rMyPsd6fOsi+4N+EHBV3sZYo+cOsqdddj2pvN3L1zsTX1jPE9mJfQNI5e3X1LYzZU9QbGZvt0XAZU11OwMJlQJCg2xXu2xrufwK9Wp6BdIpo3o/qvNsYx8m9wHWArHGmL120c9ArD190o8wPQM8B/wW8Nnp5jTco1zPVInAAeBV+1TZyyISShPdzsaY3cDTwE/AXqzttp6mv51LNdR2bWNPV86vVSAFhCZJRMKA94AHjTFHy5cZa9egSdxXLCJXAvuNMesbuy+nmQvrtMJLxpg+QA7WqQS/Jrado4GrsQJhHBAKjGjUTjWSxtiugRQQmtyjOkXEjRUM3jTG/MPO3icire3y1sB+O/9sf4TpYGCUiKQDC7BOG/2FhnuU65kqA8gwxqy10+9iBYimup0vBXYaYw4YY4qAf2Bt+6a+nUs11HbdbU9Xzq9VIAWEJvWoTvuOgVeA74wxz5QrKv8I08qPNr3VvlthIHDEPjRdAgwXkWh772y4nXdGMcZMMcbEG2MSsLbdF8aYcTTco1zPSMaYn4FdItLFzroE66mDTXI7Y50qGigiIfZnvHR9m/R2LqdBtqtddlREBtrv463l5lWzxr6ocpov4FyBdTfODuD3jd2fk1yXC7AOJ78FNtqvK7DOn34ObAc+A5rZ9QV4wV73/wLJ5eZ1B5Bmv25v7HWrx7oPoewuow5Y/+hpwDuAx84PttNpdnmHcu1/b78P26jHnReN/QKSgFR7W/8T626SJrudgceBrcAm4A2sO4Wa3HYG3sK6TlKEdSR4Z0NuVyDZfg93AH+l0o0J1b106AqllFJAYJ0yUkopVQsNCEoppQANCEoppWwaEJRSSgEaEJRSStk0ICillAI0ICillLL9fxTlS3LwVYveAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "1qmf8tduawE4"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "gEUlZZv3awCi"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "k-vFAXi-av_8"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "PCVyoLCZfJbi"
      },
      "execution_count": 13,
      "outputs": []
    }
  ]
}