08.3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/udlbook/udlbook/blob/main/Notebooks/Chap08/8_3_Double_Descent.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L6chybAVFJW2"
   },
   "source": [
    "# **Notebook 8.3: Double Descent**\n",
    "\n",
    "This notebook investigates double descent as described in section 8.4 of the book.\n",
    "\n",
    "It uses the MNIST-1D database which can be found at https://github.com/greydanus/mnist1d\n",
    "\n",
    "Work through the cells below, running each cell in turn. In various places you will see the words \"TO DO\". Follow the instructions at these places and make predictions about what is going to happen or write code to complete the functions.\n",
    "\n",
    "Contact me at udlbookmail@gmail.com if you find any mistakes or have any suggestions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "fn9BP5N5TguP"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'mnist1d'...\n"
     ]
    }
   ],
   "source": [
    "# Run this if you're in a Colab to make a local copy of the MNIST 1D repository\n",
    "!git clone https://github.com/greydanus/mnist1d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "hFxuHpRqTgri"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using: cuda\n"
     ]
    }
   ],
   "source": [
    "import torch, torch.nn as nn\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import mnist1d\n",
    "import random\n",
    "random.seed(0)\n",
    "\n",
    "# Try attaching to GPU -- Use \"Change Runtime Type to change to GPUT\"\n",
    "DEVICE = str(torch.device('cuda' if torch.cuda.is_available() else 'cpu'))\n",
    "print('Using:', DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "PW2gyXL5UkLU"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Did or could not load data from ./mnist1d_data.pkl. Rebuilding dataset...\n",
      "Examples in training set: 4000\n",
      "Examples in test set: 4000\n",
      "Length of each example: 40\n"
     ]
    }
   ],
   "source": [
    "args = mnist1d.data.get_dataset_args()\n",
    "args.num_samples = 8000\n",
    "args.train_split = 0.5\n",
    "args.corr_noise_scale = 0.25\n",
    "args.iid_noise_scale=2e-2\n",
    "data = mnist1d.data.get_dataset(args, path='./mnist1d_data.pkl', download=False, regenerate=True)\n",
    "\n",
    "# Add 15% noise to training labels\n",
    "for c_y in range(len(data['y'])):\n",
    "    random_number = random.random()\n",
    "    if random_number < 0.15 :\n",
    "        random_int = int(random.random() * 10)\n",
    "        data['y'][c_y] = random_int\n",
    "\n",
    "# The training and test input and outputs are in\n",
    "# data['x'], data['y'], data['x_test'], and data['y_test']\n",
    "print(\"Examples in training set: {}\".format(len(data['y'])))\n",
    "print(\"Examples in test set: {}\".format(len(data['y_test'])))\n",
    "print(\"Length of each example: {}\".format(data['x'].shape[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "hAIvZOAlTnk9"
   },
   "outputs": [],
   "source": [
    "# Initialize the parameters with He initialization\n",
    "def weights_init(layer_in):\n",
    "  if isinstance(layer_in, nn.Linear):\n",
    "    nn.init.kaiming_uniform_(layer_in.weight)\n",
    "    layer_in.bias.data.fill_(0.0)\n",
    "\n",
    "# Return an initialized model with two hidden layers and n_hidden hidden units at each\n",
    "def get_model(n_hidden):\n",
    "\n",
    "  D_i = 40    # Input dimensions\n",
    "  D_k = n_hidden   # Hidden dimensions\n",
    "  D_o = 10    # Output dimensions\n",
    "\n",
    "  # Define a model with two hidden layers\n",
    "  # And ReLU activations between them\n",
    "  model = nn.Sequential(\n",
    "  nn.Linear(D_i, D_k),\n",
    "  nn.ReLU(),\n",
    "  nn.Linear(D_k, D_k),\n",
    "  nn.ReLU(),\n",
    "  nn.Linear(D_k, D_o))\n",
    "\n",
    "  # Call the function you just defined\n",
    "  model.apply(weights_init)\n",
    "\n",
    "  # Return the model\n",
    "  return model ;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "AazlQhheWmHk"
   },
   "outputs": [],
   "source": [
    "def fit_model(model, data):\n",
    "\n",
    "  # choose cross entropy loss function (equation 5.24)\n",
    "  loss_function = torch.nn.CrossEntropyLoss()\n",
    "  # construct SGD optimizer and initialize learning rate and momentum\n",
    "  # optimizer = torch.optim.Adam(model.parameters(), lr=0.01)\n",
    "  optimizer = torch.optim.SGD(model.parameters(), lr = 0.01, momentum=0.9)\n",
    "\n",
    "\n",
    "  x_train = torch.tensor(data['x'].astype('float32')).to(DEVICE)\n",
    "  y_train = torch.tensor(data['y'].transpose().astype('long')).type(torch.LongTensor).to(DEVICE) #added due to error\n",
    "  x_test= torch.tensor(data['x_test'].astype('float32')).to(DEVICE)\n",
    "  y_test = torch.tensor(data['y_test'].astype('long')).type(torch.LongTensor).to(DEVICE)\n",
    "\n",
    "  model.to(DEVICE)\n",
    "\n",
    "  # load the data into a class that creates the batches\n",
    "  data_loader = DataLoader(TensorDataset(x_train,y_train), batch_size=100, shuffle=True, worker_init_fn=np.random.seed(1))\n",
    "\n",
    "  # loop over the dataset n_epoch times\n",
    "  n_epoch = 1000\n",
    "\n",
    "  for epoch in range(n_epoch):\n",
    "    # loop over batches\n",
    "    for i, batch in enumerate(data_loader):\n",
    "      # retrieve inputs and labels for this batch\n",
    "      x_batch, y_batch = batch\n",
    "      # zero the parameter gradients\n",
    "      optimizer.zero_grad()\n",
    "      # forward pass -- calculate model output\n",
    "      pred = model(x_batch)\n",
    "      # compute the loss\n",
    "      loss = loss_function(pred, y_batch)\n",
    "      # backward pass\n",
    "      loss.backward()\n",
    "      # SGD update\n",
    "      optimizer.step()\n",
    "\n",
    "    # Run whole dataset to get statistics -- normally wouldn't do this\n",
    "    pred_train = model(x_train)\n",
    "    pred_test = model(x_test)\n",
    "    _, predicted_train_class = torch.max(pred_train.data, 1)\n",
    "    _, predicted_test_class = torch.max(pred_test.data, 1)\n",
    "    errors_train = 100 - 100 * (predicted_train_class == y_train).float().sum() / len(y_train)\n",
    "    errors_test= 100 - 100 * (predicted_test_class == y_test).float().sum() / len(y_test)\n",
    "    losses_train = loss_function(pred_train, y_train).item()\n",
    "    losses_test= loss_function(pred_test, y_test).item()\n",
    "    if epoch%100 ==0 :\n",
    "      print(f'Epoch {epoch:5d}, train loss {losses_train:.6f}, train error {errors_train:3.2f},  test loss {losses_test:.6f}, test error {errors_test:3.2f}')\n",
    "\n",
    "  return errors_train, errors_test\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "IcP4UPMudxPS"
   },
   "source": [
    "The following code produces the double descent curve by training the model with different numbers of hidden units and plotting the test error.\n",
    "\n",
    "TO DO:\n",
    "\n",
    "*Before* you run the code, and considering that there are 4000 training examples predict:<br>\n",
    "\n",
    "1.    At what capacity do you think the training error will become zero?\n",
    "2.   At what capacity do you expect the first minima of the double descent curve to appear?\n",
    "3. At what capacity do you expect the maximum of the double descent curve to appear?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "K4OmBZGHWXpk"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with   2 hidden variables\n",
      "Epoch     0, train loss 2.302428, train error 89.57,  test loss 2.302762, test error 90.30\n",
      "Epoch   100, train loss 2.302207, train error 89.57,  test loss 2.303494, test error 90.30\n",
      "Epoch   200, train loss 2.302210, train error 89.57,  test loss 2.303369, test error 90.30\n",
      "Epoch   300, train loss 2.302210, train error 89.57,  test loss 2.303474, test error 90.30\n",
      "Epoch   400, train loss 2.302209, train error 89.57,  test loss 2.303424, test error 90.30\n",
      "Epoch   500, train loss 2.302214, train error 89.57,  test loss 2.303358, test error 90.30\n",
      "Epoch   600, train loss 2.302211, train error 89.57,  test loss 2.303411, test error 90.30\n",
      "Epoch   700, train loss 2.302213, train error 89.57,  test loss 2.303418, test error 90.30\n",
      "Epoch   800, train loss 2.302215, train error 89.57,  test loss 2.303526, test error 90.30\n",
      "Epoch   900, train loss 2.302211, train error 89.57,  test loss 2.303417, test error 90.30\n",
      "Training model with   4 hidden variables\n",
      "Epoch     0, train loss 2.301058, train error 90.30,  test loss 2.302223, test error 89.93\n",
      "Epoch   100, train loss 1.810517, train error 66.27,  test loss 1.662408, test error 63.25\n",
      "Epoch   200, train loss 1.759063, train error 63.12,  test loss 1.591103, test error 59.88\n",
      "Epoch   300, train loss 1.753453, train error 62.70,  test loss 1.593768, test error 60.70\n",
      "Epoch   400, train loss 1.753088, train error 62.80,  test loss 1.590046, test error 60.72\n",
      "Epoch   500, train loss 1.751561, train error 62.80,  test loss 1.588622, test error 60.65\n",
      "Epoch   600, train loss 1.750438, train error 63.00,  test loss 1.587184, test error 60.32\n",
      "Epoch   700, train loss 1.749507, train error 62.75,  test loss 1.588432, test error 60.67\n",
      "Epoch   800, train loss 1.751242, train error 62.97,  test loss 1.589767, test error 60.88\n",
      "Epoch   900, train loss 1.750803, train error 62.38,  test loss 1.604721, test error 60.67\n",
      "Training model with   6 hidden variables\n",
      "Epoch     0, train loss 2.258576, train error 85.12,  test loss 2.250971, test error 84.50\n",
      "Epoch   100, train loss 1.720527, train error 61.90,  test loss 1.587443, test error 61.15\n",
      "Epoch   200, train loss 1.688161, train error 60.65,  test loss 1.593536, test error 61.22\n",
      "Epoch   300, train loss 1.674473, train error 59.75,  test loss 1.579422, test error 61.97\n",
      "Epoch   400, train loss 1.638553, train error 57.20,  test loss 1.557047, test error 59.05\n",
      "Epoch   500, train loss 1.612244, train error 56.30,  test loss 1.508286, test error 57.50\n",
      "Epoch   600, train loss 1.599758, train error 55.92,  test loss 1.484365, test error 56.75\n",
      "Epoch   700, train loss 1.596048, train error 55.92,  test loss 1.483226, test error 56.37\n",
      "Epoch   800, train loss 1.599753, train error 56.07,  test loss 1.492900, test error 56.85\n",
      "Epoch   900, train loss 1.587118, train error 55.85,  test loss 1.474656, test error 56.57\n",
      "Training model with   8 hidden variables\n",
      "Epoch     0, train loss 2.272185, train error 84.97,  test loss 2.265079, test error 84.88\n",
      "Epoch   100, train loss 1.599613, train error 55.32,  test loss 1.441871, test error 52.55\n",
      "Epoch   200, train loss 1.542048, train error 53.02,  test loss 1.385667, test error 51.32\n",
      "Epoch   300, train loss 1.525573, train error 52.67,  test loss 1.384527, test error 51.67\n",
      "Epoch   400, train loss 1.519742, train error 52.47,  test loss 1.399220, test error 52.55\n",
      "Epoch   500, train loss 1.508150, train error 51.70,  test loss 1.401612, test error 52.17\n",
      "Epoch   600, train loss 1.503144, train error 51.47,  test loss 1.400557, test error 52.22\n",
      "Epoch   700, train loss 1.504490, train error 51.92,  test loss 1.400035, test error 52.22\n",
      "Epoch   800, train loss 1.503825, train error 51.52,  test loss 1.412950, test error 52.52\n",
      "Epoch   900, train loss 1.501419, train error 51.85,  test loss 1.404065, test error 51.42\n",
      "Training model with  10 hidden variables\n",
      "Epoch     0, train loss 2.303539, train error 88.97,  test loss 2.299653, test error 88.85\n",
      "Epoch   100, train loss 1.599018, train error 54.25,  test loss 1.489865, test error 57.05\n",
      "Epoch   200, train loss 1.515509, train error 51.72,  test loss 1.409123, test error 52.12\n",
      "Epoch   300, train loss 1.500583, train error 50.52,  test loss 1.395473, test error 50.97\n",
      "Epoch   400, train loss 1.489465, train error 50.47,  test loss 1.398352, test error 51.57\n",
      "Epoch   500, train loss 1.489434, train error 50.72,  test loss 1.407100, test error 51.17\n",
      "Epoch   600, train loss 1.488828, train error 51.35,  test loss 1.413012, test error 51.27\n",
      "Epoch   700, train loss 1.481206, train error 51.12,  test loss 1.394002, test error 51.12\n",
      "Epoch   800, train loss 1.482436, train error 50.80,  test loss 1.403963, test error 51.15\n",
      "Epoch   900, train loss 1.475191, train error 50.60,  test loss 1.414818, test error 51.07\n",
      "Training model with  14 hidden variables\n",
      "Epoch     0, train loss 2.282324, train error 84.93,  test loss 2.272361, test error 83.45\n",
      "Epoch   100, train loss 1.531158, train error 52.30,  test loss 1.473066, test error 56.30\n",
      "Epoch   200, train loss 1.356083, train error 44.35,  test loss 1.326831, test error 48.40\n",
      "Epoch   300, train loss 1.327369, train error 43.62,  test loss 1.346715, test error 49.47\n",
      "Epoch   400, train loss 1.301519, train error 43.17,  test loss 1.328543, test error 48.27\n",
      "Epoch   500, train loss 1.281600, train error 42.27,  test loss 1.341742, test error 48.72\n",
      "Epoch   600, train loss 1.278047, train error 42.12,  test loss 1.348951, test error 48.12\n",
      "Epoch   700, train loss 1.275817, train error 41.60,  test loss 1.364583, test error 48.95\n",
      "Epoch   800, train loss 1.269785, train error 41.80,  test loss 1.370775, test error 48.65\n",
      "Epoch   900, train loss 1.263724, train error 41.60,  test loss 1.382596, test error 48.92\n",
      "Training model with  18 hidden variables\n",
      "Epoch     0, train loss 2.159707, train error 81.30,  test loss 2.106550, test error 79.93\n",
      "Epoch   100, train loss 1.348244, train error 44.65,  test loss 1.290830, test error 47.95\n",
      "Epoch   200, train loss 1.258056, train error 41.65,  test loss 1.328868, test error 49.32\n",
      "Epoch   300, train loss 1.215941, train error 40.52,  test loss 1.368378, test error 49.15\n",
      "Epoch   400, train loss 1.201411, train error 40.40,  test loss 1.391212, test error 49.02\n",
      "Epoch   500, train loss 1.174123, train error 39.32,  test loss 1.427231, test error 50.27\n",
      "Epoch   600, train loss 1.158798, train error 38.67,  test loss 1.453201, test error 50.90\n",
      "Epoch   700, train loss 1.155526, train error 39.30,  test loss 1.480765, test error 50.67\n",
      "Epoch   800, train loss 1.149373, train error 39.20,  test loss 1.520569, test error 51.65\n",
      "Epoch   900, train loss 1.131347, train error 38.15,  test loss 1.507576, test error 50.50\n",
      "Training model with  22 hidden variables\n",
      "Epoch     0, train loss 2.192563, train error 80.80,  test loss 2.157186, test error 80.65\n",
      "Epoch   100, train loss 1.296018, train error 42.80,  test loss 1.336031, test error 49.45\n",
      "Epoch   200, train loss 1.174110, train error 39.72,  test loss 1.381397, test error 49.52\n",
      "Epoch   300, train loss 1.121140, train error 38.35,  test loss 1.433221, test error 50.20\n",
      "Epoch   400, train loss 1.090953, train error 37.27,  test loss 1.465806, test error 50.50\n",
      "Epoch   500, train loss 1.076664, train error 36.80,  test loss 1.531259, test error 52.20\n",
      "Epoch   600, train loss 1.052870, train error 35.80,  test loss 1.570552, test error 52.67\n",
      "Epoch   700, train loss 1.040540, train error 35.38,  test loss 1.568357, test error 52.00\n",
      "Epoch   800, train loss 1.036016, train error 35.55,  test loss 1.604399, test error 52.87\n",
      "Epoch   900, train loss 1.040569, train error 36.70,  test loss 1.645137, test error 53.27\n",
      "Training model with  26 hidden variables\n",
      "Epoch     0, train loss 2.115736, train error 78.35,  test loss 2.050428, test error 77.75\n",
      "Epoch   100, train loss 1.171433, train error 38.10,  test loss 1.294036, test error 47.97\n",
      "Epoch   200, train loss 1.034414, train error 34.02,  test loss 1.440920, test error 49.12\n",
      "Epoch   300, train loss 0.954669, train error 30.90,  test loss 1.539135, test error 50.20\n",
      "Epoch   400, train loss 0.912235, train error 30.20,  test loss 1.653815, test error 51.82\n",
      "Epoch   500, train loss 0.892025, train error 29.35,  test loss 1.749812, test error 52.47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   600, train loss 0.862905, train error 28.22,  test loss 1.828865, test error 53.37\n",
      "Epoch   700, train loss 0.862951, train error 28.90,  test loss 1.963266, test error 53.32\n",
      "Epoch   800, train loss 0.841004, train error 28.17,  test loss 1.982211, test error 53.02\n",
      "Epoch   900, train loss 0.841001, train error 28.88,  test loss 2.047128, test error 53.37\n",
      "Training model with  30 hidden variables\n",
      "Epoch     0, train loss 2.186210, train error 80.80,  test loss 2.148356, test error 79.80\n",
      "Epoch   100, train loss 1.181185, train error 39.15,  test loss 1.366492, test error 50.05\n",
      "Epoch   200, train loss 0.971511, train error 31.90,  test loss 1.515349, test error 50.75\n",
      "Epoch   300, train loss 0.882933, train error 30.00,  test loss 1.757398, test error 52.27\n",
      "Epoch   400, train loss 0.809050, train error 26.57,  test loss 1.962890, test error 52.77\n",
      "Epoch   500, train loss 0.780353, train error 25.88,  test loss 2.179576, test error 53.75\n",
      "Epoch   600, train loss 0.750876, train error 24.80,  test loss 2.302116, test error 54.22\n",
      "Epoch   700, train loss 0.724095, train error 24.35,  test loss 2.502676, test error 55.25\n",
      "Epoch   800, train loss 0.707146, train error 23.38,  test loss 2.654762, test error 55.30\n",
      "Epoch   900, train loss 0.699029, train error 23.77,  test loss 2.802953, test error 55.70\n",
      "Training model with  35 hidden variables\n",
      "Epoch     0, train loss 2.113798, train error 78.35,  test loss 2.065448, test error 77.28\n",
      "Epoch   100, train loss 1.047315, train error 34.35,  test loss 1.412071, test error 50.82\n",
      "Epoch   200, train loss 0.818575, train error 26.50,  test loss 1.693381, test error 52.20\n",
      "Epoch   300, train loss 0.710843, train error 23.97,  test loss 2.057861, test error 54.10\n",
      "Epoch   400, train loss 0.655542, train error 22.42,  test loss 2.430780, test error 55.70\n",
      "Epoch   500, train loss 0.600531, train error 20.70,  test loss 2.767156, test error 56.00\n",
      "Epoch   600, train loss 0.537077, train error 18.05,  test loss 3.028693, test error 56.40\n",
      "Epoch   700, train loss 0.509546, train error 17.50,  test loss 3.339022, test error 57.52\n",
      "Epoch   800, train loss 0.545544, train error 19.22,  test loss 3.597382, test error 58.20\n",
      "Epoch   900, train loss 0.485921, train error 16.60,  test loss 3.799197, test error 56.62\n",
      "Training model with  40 hidden variables\n",
      "Epoch     0, train loss 2.127964, train error 78.93,  test loss 2.083750, test error 78.10\n",
      "Epoch   100, train loss 0.964112, train error 31.90,  test loss 1.479250, test error 51.00\n",
      "Epoch   200, train loss 0.684293, train error 23.05,  test loss 1.996392, test error 53.77\n",
      "Epoch   300, train loss 0.551029, train error 18.50,  test loss 2.602575, test error 54.97\n",
      "Epoch   400, train loss 0.478367, train error 16.62,  test loss 3.288060, test error 55.12\n",
      "Epoch   500, train loss 0.360642, train error 12.20,  test loss 4.012337, test error 56.40\n",
      "Epoch   600, train loss 0.322655, train error 10.92,  test loss 4.635782, test error 57.20\n",
      "Epoch   700, train loss 0.387818, train error 12.95,  test loss 5.335463, test error 57.42\n",
      "Epoch   800, train loss 0.344658, train error 12.47,  test loss 5.800427, test error 58.75\n",
      "Epoch   900, train loss 0.239487, train error 8.32,  test loss 6.233307, test error 57.22\n",
      "Training model with  45 hidden variables\n",
      "Epoch     0, train loss 2.076900, train error 77.32,  test loss 2.006456, test error 75.57\n",
      "Epoch   100, train loss 0.817772, train error 26.07,  test loss 1.515265, test error 49.97\n",
      "Epoch   200, train loss 0.481492, train error 14.97,  test loss 2.279838, test error 53.87\n",
      "Epoch   300, train loss 0.343124, train error 11.47,  test loss 3.363905, test error 56.32\n",
      "Epoch   400, train loss 0.220326, train error 7.07,  test loss 4.583739, test error 56.65\n",
      "Epoch   500, train loss 0.146113, train error 4.67,  test loss 5.743219, test error 56.40\n",
      "Epoch   600, train loss 0.128859, train error 4.45,  test loss 6.912613, test error 57.00\n",
      "Epoch   700, train loss 0.013813, train error -0.00,  test loss 7.965572, test error 56.87\n",
      "Epoch   800, train loss 0.008431, train error -0.00,  test loss 8.666749, test error 57.17\n",
      "Epoch   900, train loss 0.006134, train error -0.00,  test loss 9.181439, test error 57.22\n",
      "Training model with  50 hidden variables\n",
      "Epoch     0, train loss 2.037109, train error 76.45,  test loss 1.947618, test error 74.78\n",
      "Epoch   100, train loss 0.724817, train error 23.32,  test loss 1.595425, test error 49.95\n",
      "Epoch   200, train loss 0.348016, train error 9.95,  test loss 2.730633, test error 55.00\n",
      "Epoch   300, train loss 0.151276, train error 3.37,  test loss 4.341858, test error 56.40\n",
      "Epoch   400, train loss 0.029621, train error 0.05,  test loss 6.073876, test error 57.00\n",
      "Epoch   500, train loss 0.013002, train error -0.00,  test loss 7.056958, test error 57.35\n",
      "Epoch   600, train loss 0.008093, train error -0.00,  test loss 7.719812, test error 57.27\n",
      "Epoch   700, train loss 0.005732, train error -0.00,  test loss 8.190256, test error 57.42\n",
      "Epoch   800, train loss 0.004418, train error -0.00,  test loss 8.579594, test error 57.42\n",
      "Epoch   900, train loss 0.003565, train error -0.00,  test loss 8.902440, test error 57.47\n",
      "Training model with  55 hidden variables\n",
      "Epoch     0, train loss 2.067141, train error 77.47,  test loss 2.007289, test error 76.78\n",
      "Epoch   100, train loss 0.664650, train error 20.62,  test loss 1.594685, test error 50.50\n",
      "Epoch   200, train loss 0.248241, train error 6.45,  test loss 2.902361, test error 52.30\n",
      "Epoch   300, train loss 0.049367, train error 0.10,  test loss 4.623705, test error 54.77\n",
      "Epoch   400, train loss 0.016034, train error -0.00,  test loss 5.741310, test error 54.37\n",
      "Epoch   500, train loss 0.009183, train error -0.00,  test loss 6.436931, test error 54.65\n",
      "Epoch   600, train loss 0.006234, train error -0.00,  test loss 6.903138, test error 54.77\n",
      "Epoch   700, train loss 0.004649, train error -0.00,  test loss 7.269921, test error 54.60\n",
      "Epoch   800, train loss 0.003670, train error -0.00,  test loss 7.568997, test error 54.55\n",
      "Epoch   900, train loss 0.003004, train error -0.00,  test loss 7.819682, test error 54.77\n",
      "Training model with  60 hidden variables\n",
      "Epoch     0, train loss 2.068890, train error 77.15,  test loss 2.006988, test error 76.90\n",
      "Epoch   100, train loss 0.536243, train error 15.90,  test loss 1.782736, test error 52.10\n",
      "Epoch   200, train loss 0.096623, train error 0.65,  test loss 3.572114, test error 54.92\n",
      "Epoch   300, train loss 0.018255, train error -0.00,  test loss 4.968001, test error 55.52\n",
      "Epoch   400, train loss 0.008986, train error -0.00,  test loss 5.669069, test error 55.87\n",
      "Epoch   500, train loss 0.005637, train error -0.00,  test loss 6.125247, test error 56.12\n",
      "Epoch   600, train loss 0.004104, train error -0.00,  test loss 6.466606, test error 56.22\n",
      "Epoch   700, train loss 0.003144, train error -0.00,  test loss 6.737769, test error 56.30\n",
      "Epoch   800, train loss 0.002537, train error -0.00,  test loss 6.967058, test error 56.35\n",
      "Epoch   900, train loss 0.002111, train error -0.00,  test loss 7.160599, test error 56.35\n",
      "Training model with  70 hidden variables\n",
      "Epoch     0, train loss 2.052979, train error 76.50,  test loss 1.993640, test error 76.60\n",
      "Epoch   100, train loss 0.366559, train error 9.02,  test loss 2.062217, test error 53.82\n",
      "Epoch   200, train loss 0.033135, train error -0.00,  test loss 3.872143, test error 55.52\n",
      "Epoch   300, train loss 0.011092, train error -0.00,  test loss 4.717029, test error 55.45\n",
      "Epoch   400, train loss 0.006176, train error -0.00,  test loss 5.183798, test error 55.45\n",
      "Epoch   500, train loss 0.004158, train error -0.00,  test loss 5.509650, test error 55.37\n",
      "Epoch   600, train loss 0.003094, train error -0.00,  test loss 5.757762, test error 55.42\n",
      "Epoch   700, train loss 0.002422, train error -0.00,  test loss 5.965644, test error 55.62\n",
      "Epoch   800, train loss 0.001982, train error -0.00,  test loss 6.134915, test error 55.60\n",
      "Epoch   900, train loss 0.001671, train error -0.00,  test loss 6.280475, test error 55.67\n",
      "Training model with  80 hidden variables\n",
      "Epoch     0, train loss 2.030819, train error 75.68,  test loss 1.974363, test error 75.65\n",
      "Epoch   100, train loss 0.237481, train error 4.82,  test loss 2.153792, test error 52.87\n",
      "Epoch   200, train loss 0.019537, train error -0.00,  test loss 3.509802, test error 53.90\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   300, train loss 0.007862, train error -0.00,  test loss 4.065742, test error 53.80\n",
      "Epoch   400, train loss 0.004664, train error -0.00,  test loss 4.405617, test error 53.70\n",
      "Epoch   500, train loss 0.003224, train error -0.00,  test loss 4.643003, test error 53.60\n",
      "Epoch   600, train loss 0.002428, train error -0.00,  test loss 4.826610, test error 53.60\n",
      "Epoch   700, train loss 0.001933, train error -0.00,  test loss 4.974301, test error 53.75\n",
      "Epoch   800, train loss 0.001592, train error -0.00,  test loss 5.104980, test error 53.70\n",
      "Epoch   900, train loss 0.001348, train error -0.00,  test loss 5.214190, test error 53.65\n",
      "Training model with  90 hidden variables\n",
      "Epoch     0, train loss 2.022272, train error 75.40,  test loss 1.933297, test error 74.93\n",
      "Epoch   100, train loss 0.163522, train error 1.17,  test loss 2.283844, test error 53.90\n",
      "Epoch   200, train loss 0.015888, train error -0.00,  test loss 3.535394, test error 54.47\n",
      "Epoch   300, train loss 0.006834, train error -0.00,  test loss 4.016518, test error 54.32\n",
      "Epoch   400, train loss 0.004140, train error -0.00,  test loss 4.323086, test error 54.50\n",
      "Epoch   500, train loss 0.002898, train error -0.00,  test loss 4.536473, test error 54.57\n",
      "Epoch   600, train loss 0.002200, train error -0.00,  test loss 4.704394, test error 54.55\n",
      "Epoch   700, train loss 0.001758, train error -0.00,  test loss 4.844176, test error 54.52\n",
      "Epoch   800, train loss 0.001454, train error -0.00,  test loss 4.960331, test error 54.45\n",
      "Epoch   900, train loss 0.001234, train error -0.00,  test loss 5.059988, test error 54.50\n",
      "Training model with 100 hidden variables\n",
      "Epoch     0, train loss 2.020290, train error 76.00,  test loss 1.955291, test error 75.65\n",
      "Epoch   100, train loss 0.092062, train error 0.07,  test loss 2.235560, test error 52.10\n",
      "Epoch   200, train loss 0.011934, train error -0.00,  test loss 3.043023, test error 51.32\n",
      "Epoch   300, train loss 0.005598, train error -0.00,  test loss 3.399044, test error 51.25\n",
      "Epoch   400, train loss 0.003486, train error -0.00,  test loss 3.618381, test error 51.50\n",
      "Epoch   500, train loss 0.002483, train error -0.00,  test loss 3.780484, test error 51.37\n",
      "Epoch   600, train loss 0.001901, train error -0.00,  test loss 3.906556, test error 51.40\n",
      "Epoch   700, train loss 0.001528, train error -0.00,  test loss 4.011281, test error 51.32\n",
      "Epoch   800, train loss 0.001270, train error -0.00,  test loss 4.099812, test error 51.30\n",
      "Epoch   900, train loss 0.001082, train error -0.00,  test loss 4.177006, test error 51.32\n",
      "Training model with 120 hidden variables\n",
      "Epoch     0, train loss 2.003334, train error 74.18,  test loss 1.897407, test error 73.88\n",
      "Epoch   100, train loss 0.049260, train error -0.00,  test loss 2.263734, test error 49.92\n",
      "Epoch   200, train loss 0.009311, train error -0.00,  test loss 2.867255, test error 49.25\n",
      "Epoch   300, train loss 0.004620, train error -0.00,  test loss 3.145469, test error 49.15\n",
      "Epoch   400, train loss 0.002970, train error -0.00,  test loss 3.328778, test error 49.20\n",
      "Epoch   500, train loss 0.002141, train error -0.00,  test loss 3.460900, test error 49.40\n",
      "Epoch   600, train loss 0.001656, train error -0.00,  test loss 3.566614, test error 49.60\n",
      "Epoch   700, train loss 0.001339, train error -0.00,  test loss 3.653829, test error 49.50\n",
      "Epoch   800, train loss 0.001119, train error -0.00,  test loss 3.728033, test error 49.50\n",
      "Epoch   900, train loss 0.000957, train error -0.00,  test loss 3.791905, test error 49.62\n",
      "Training model with 140 hidden variables\n",
      "Epoch     0, train loss 1.946448, train error 72.57,  test loss 1.864614, test error 71.97\n",
      "Epoch   100, train loss 0.034372, train error -0.00,  test loss 2.248192, test error 50.35\n",
      "Epoch   200, train loss 0.007884, train error -0.00,  test loss 2.759170, test error 50.25\n",
      "Epoch   300, train loss 0.004078, train error -0.00,  test loss 3.002605, test error 50.35\n",
      "Epoch   400, train loss 0.002655, train error -0.00,  test loss 3.162472, test error 50.22\n",
      "Epoch   500, train loss 0.001935, train error -0.00,  test loss 3.281049, test error 50.25\n",
      "Epoch   600, train loss 0.001507, train error -0.00,  test loss 3.375040, test error 50.25\n",
      "Epoch   700, train loss 0.001224, train error -0.00,  test loss 3.453105, test error 50.30\n",
      "Epoch   800, train loss 0.001025, train error -0.00,  test loss 3.520444, test error 50.37\n",
      "Epoch   900, train loss 0.000879, train error -0.00,  test loss 3.578543, test error 50.30\n",
      "Training model with 160 hidden variables\n",
      "Epoch     0, train loss 1.949072, train error 72.05,  test loss 1.852963, test error 73.47\n",
      "Epoch   100, train loss 0.028939, train error -0.00,  test loss 2.233161, test error 50.67\n",
      "Epoch   200, train loss 0.007210, train error -0.00,  test loss 2.672071, test error 50.65\n",
      "Epoch   300, train loss 0.003794, train error -0.00,  test loss 2.887796, test error 50.77\n",
      "Epoch   400, train loss 0.002494, train error -0.00,  test loss 3.033356, test error 50.90\n",
      "Epoch   500, train loss 0.001827, train error -0.00,  test loss 3.140588, test error 51.07\n",
      "Epoch   600, train loss 0.001427, train error -0.00,  test loss 3.225440, test error 50.92\n",
      "Epoch   700, train loss 0.001162, train error -0.00,  test loss 3.297318, test error 50.90\n",
      "Epoch   800, train loss 0.000975, train error -0.00,  test loss 3.358333, test error 50.92\n",
      "Epoch   900, train loss 0.000837, train error -0.00,  test loss 3.411123, test error 50.90\n",
      "Training model with 180 hidden variables\n",
      "Epoch     0, train loss 1.955110, train error 71.32,  test loss 1.877182, test error 71.85\n",
      "Epoch   100, train loss 0.024678, train error -0.00,  test loss 2.059622, test error 47.95\n",
      "Epoch   200, train loss 0.006584, train error -0.00,  test loss 2.422210, test error 48.30\n",
      "Epoch   300, train loss 0.003521, train error -0.00,  test loss 2.601758, test error 48.45\n",
      "Epoch   400, train loss 0.002329, train error -0.00,  test loss 2.724057, test error 48.50\n",
      "Epoch   500, train loss 0.001714, train error -0.00,  test loss 2.814641, test error 48.52\n",
      "Epoch   600, train loss 0.001343, train error -0.00,  test loss 2.887889, test error 48.62\n",
      "Epoch   700, train loss 0.001096, train error -0.00,  test loss 2.948621, test error 48.60\n",
      "Epoch   800, train loss 0.000922, train error -0.00,  test loss 3.000962, test error 48.52\n",
      "Epoch   900, train loss 0.000793, train error -0.00,  test loss 3.047147, test error 48.57\n",
      "Training model with 200 hidden variables\n",
      "Epoch     0, train loss 1.933387, train error 70.65,  test loss 1.858209, test error 71.20\n",
      "Epoch   100, train loss 0.021645, train error -0.00,  test loss 2.031267, test error 48.45\n",
      "Epoch   200, train loss 0.006183, train error -0.00,  test loss 2.346095, test error 48.15\n",
      "Epoch   300, train loss 0.003355, train error -0.00,  test loss 2.509581, test error 48.30\n",
      "Epoch   400, train loss 0.002238, train error -0.00,  test loss 2.620103, test error 48.47\n",
      "Epoch   500, train loss 0.001651, train error -0.00,  test loss 2.702214, test error 48.55\n",
      "Epoch   600, train loss 0.001296, train error -0.00,  test loss 2.768572, test error 48.45\n",
      "Epoch   700, train loss 0.001060, train error -0.00,  test loss 2.824200, test error 48.42\n",
      "Epoch   800, train loss 0.000892, train error -0.00,  test loss 2.872100, test error 48.42\n",
      "Epoch   900, train loss 0.000768, train error -0.00,  test loss 2.913547, test error 48.37\n",
      "Training model with 250 hidden variables\n",
      "Epoch     0, train loss 1.885657, train error 67.77,  test loss 1.805493, test error 69.77\n",
      "Epoch   100, train loss 0.017067, train error -0.00,  test loss 1.962010, test error 48.02\n",
      "Epoch   200, train loss 0.005398, train error -0.00,  test loss 2.222747, test error 47.85\n",
      "Epoch   300, train loss 0.003014, train error -0.00,  test loss 2.364627, test error 47.75\n",
      "Epoch   400, train loss 0.002037, train error -0.00,  test loss 2.458641, test error 47.75\n",
      "Epoch   500, train loss 0.001515, train error -0.00,  test loss 2.531425, test error 47.92\n",
      "Epoch   600, train loss 0.001196, train error -0.00,  test loss 2.589782, test error 47.87\n",
      "Epoch   700, train loss 0.000982, train error -0.00,  test loss 2.637764, test error 47.92\n",
      "Epoch   800, train loss 0.000830, train error -0.00,  test loss 2.679847, test error 48.02\n",
      "Epoch   900, train loss 0.000716, train error -0.00,  test loss 2.716832, test error 48.02\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model with 300 hidden variables\n",
      "Epoch     0, train loss 1.870595, train error 67.02,  test loss 1.787224, test error 68.32\n",
      "Epoch   100, train loss 0.014792, train error -0.00,  test loss 1.956277, test error 47.67\n",
      "Epoch   200, train loss 0.004857, train error -0.00,  test loss 2.192907, test error 47.12\n",
      "Epoch   300, train loss 0.002748, train error -0.00,  test loss 2.321197, test error 47.12\n",
      "Epoch   400, train loss 0.001870, train error -0.00,  test loss 2.409806, test error 47.27\n",
      "Epoch   500, train loss 0.001398, train error -0.00,  test loss 2.476564, test error 47.32\n",
      "Epoch   600, train loss 0.001107, train error -0.00,  test loss 2.531138, test error 47.40\n",
      "Epoch   700, train loss 0.000911, train error -0.00,  test loss 2.576692, test error 47.42\n",
      "Epoch   800, train loss 0.000771, train error -0.00,  test loss 2.616042, test error 47.45\n",
      "Epoch   900, train loss 0.000666, train error -0.00,  test loss 2.650472, test error 47.50\n",
      "Training model with 400 hidden variables\n",
      "Epoch     0, train loss 1.862366, train error 67.07,  test loss 1.776754, test error 68.62\n",
      "Epoch   100, train loss 0.011698, train error -0.00,  test loss 1.772027, test error 46.32\n",
      "Epoch   200, train loss 0.004184, train error -0.00,  test loss 1.950427, test error 46.67\n",
      "Epoch   300, train loss 0.002425, train error -0.00,  test loss 2.049486, test error 46.52\n",
      "Epoch   400, train loss 0.001670, train error -0.00,  test loss 2.118839, test error 46.55\n",
      "Epoch   500, train loss 0.001259, train error -0.00,  test loss 2.171793, test error 46.60\n",
      "Epoch   600, train loss 0.001002, train error -0.00,  test loss 2.214585, test error 46.62\n",
      "Epoch   700, train loss 0.000828, train error -0.00,  test loss 2.250874, test error 46.60\n",
      "Epoch   800, train loss 0.000703, train error -0.00,  test loss 2.281801, test error 46.52\n",
      "Epoch   900, train loss 0.000609, train error -0.00,  test loss 2.309642, test error 46.57\n"
     ]
    }
   ],
   "source": [
    "# This code will take a while (~30 mins on GPU) to run!  Go and make a cup of coffee!\n",
    "\n",
    "hidden_variables = np.array([2,4,6,8,10,14,18,22,26,30,35,40,45,50,55,60,70,80,90,100,120,140,160,180,200,250,300,400]) ;\n",
    "errors_train_all = np.zeros_like(hidden_variables)\n",
    "errors_test_all = np.zeros_like(hidden_variables)\n",
    "\n",
    "# For each hidden variable size\n",
    "for c_hidden in range(len(hidden_variables)):\n",
    "    print(f'Training model with {hidden_variables[c_hidden]:3d} hidden variables')\n",
    "    # Get a model\n",
    "    model = get_model(hidden_variables[c_hidden]) ;\n",
    "    # Train the model\n",
    "    errors_train, errors_test = fit_model(model, data)\n",
    "    # Store the results\n",
    "    errors_train_all[c_hidden] = errors_train\n",
    "    errors_test_all[c_hidden]= errors_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "Rw-iRboTXbck"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAG2CAYAAACZEEfAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABG8ElEQVR4nO3deXxU1f3/8fckmYSELOxZJJKgrIZFiFJABdlUFEGqItCWiFoti8alILUqaiVCK4KiWLUFrFK0CsqvXxdQMIIIZZVVFtkiEsOahC3r/f1xzcCQBALJzJ2583o+HvPIzL137nzOXOy8e8659zoMwzAEAABgU0FWFwAAAOBJhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrhB0AAGBrloadr7/+Wv369VNCQoIcDoc++ugjt/WGYWj8+PFKSEhQeHi4unfvrk2bNrltU1BQoNGjR6tBgwaqXbu2br31Vv34449ebAUAAPBlload48ePq127dpo2bVqF6ydNmqTJkydr2rRpWrlypeLi4tS7d2/l5+e7tklPT9e8efM0Z84cLV26VMeOHdMtt9yikpISbzUDAAD4MIev3AjU4XBo3rx5GjBggCSzVychIUHp6ekaO3asJLMXJzY2VhMnTtT999+v3NxcNWzYUP/61780aNAgSdJPP/2kxMREffLJJ7rhhhusag4AAPARIVYXUJldu3YpOztbffr0cS0LCwtTt27dtGzZMt1///1avXq1ioqK3LZJSEhQSkqKli1bVmnYKSgoUEFBget1aWmpDh8+rPr168vhcHiuUQAAoMYYhqH8/HwlJCQoKKjywSqfDTvZ2dmSpNjYWLflsbGx2rNnj2ub0NBQ1a1bt9w2Ze+vSEZGhp555pkarhgAAFghKytLjRs3rnS9z4adMmf3tBiGcd7el/NtM27cOD3yyCOu17m5ubr00kuVlZWl6Ojo6hUMAAC8Ii8vT4mJiYqKijrndj4bduLi4iSZvTfx8fGu5Tk5Oa7enri4OBUWFurIkSNuvTs5OTnq0qVLpfsOCwtTWFhYueXR0dGEHQAA/Mz5OkF89jo7ycnJiouL08KFC13LCgsLlZmZ6QoyHTt2lNPpdNtm//792rhx4znDDgAACByW9uwcO3ZMO3bscL3etWuX1q1bp3r16unSSy9Venq6JkyYoGbNmqlZs2aaMGGCIiIiNGTIEElSTEyM7rnnHj366KOqX7++6tWrp8cee0xt2rRRr169rGoWAADwIZaGnVWrVun66693vS6bRzNs2DDNnDlTY8aM0cmTJzVixAgdOXJEnTp10oIFC9zG5l566SWFhITozjvv1MmTJ9WzZ0/NnDlTwcHBXm8PAADwPT5znR0r5eXlKSYmRrm5uczZAQDUqJKSEhUVFVldhl9yOp3n7Lyo6u+3z05QBgDAnxmGoezsbB09etTqUvxanTp1FBcXV63r4BF2AADwgLKg06hRI0VERHDR2gtkGIZOnDihnJwcSXI7M/tCEXYAAKhhJSUlrqBTv359q8vxW+Hh4ZLMS8o0atTooufj+uyp5wAA+KuyOToREREWV+L/yr7D6sx7IuwAAOAhDF1VX018h4QdAABga4QdAADgEUlJSZoyZYrVZTBBGQAAnNa9e3e1b9++RkLKypUrVbt27eoXVU2EHQAAUGWGYaikpEQhIeePEA0bNvRCRefHMBYAAJAkpaWlKTMzU1OnTpXD4ZDD4dDMmTPlcDj0+eefKzU1VWFhYVqyZIl++OEH9e/fX7GxsYqMjNRVV12lL774wm1/Zw9jORwOvfXWW7rtttsUERGhZs2aaf78+R5vF2EHAABvMAzp+HFrHlW8M9TUqVPVuXNn3Xfffdq/f7/279+vxMRESdKYMWOUkZGhLVu2qG3btjp27Jj69u2rL774QmvXrtUNN9ygfv36ae/evef8jGeeeUZ33nmn1q9fr759+2ro0KE6fPhwtb/ec2EYCwAAbzhxQoqMtOazjx2TqjB3JiYmRqGhoYqIiFBcXJwk6fvvv5ckPfvss+rdu7dr2/r166tdu3au13/5y180b948zZ8/X6NGjar0M9LS0jR48GBJ0oQJE/TKK6/of//7n2688caLalpV0LMDAADOKzU11e318ePHNWbMGLVu3Vp16tRRZGSkvv/++/P27LRt29b1vHbt2oqKinLdEsJT6NkBAMAbIiLMHharPruazj6r6o9//KM+//xz/e1vf9Pll1+u8PBw3X777SosLDznfpxOp9trh8Oh0tLSatd3LoQdAAC8weGo0lCS1UJDQ1VSUnLe7ZYsWaK0tDTddtttkqRjx45p9+7dHq7u4jCMBQAAXJKSkrRixQrt3r1bBw8erLTX5fLLL9fcuXO1bt06fffddxoyZIjHe2guFmEHAAC4PPbYYwoODlbr1q3VsGHDSufgvPTSS6pbt666dOmifv366YYbblCHDh28XG3VOAyjiuej2VheXp5iYmKUm5ur6Ohoq8sBAPi5U6dOadeuXUpOTlatWrWsLsevneu7rOrvNz07AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADA1gg7AADApXv37kpPT6+x/aWlpWnAgAE1tr+LQdjxoNGjpfBw6YUXrK4EAIDARdjxoJIS6dQp8wEAgK9LS0tTZmampk6dKofDIYfDod27d2vz5s3q27evIiMjFRsbq9/+9rc6ePCg630ffPCB2rRpo/DwcNWvX1+9evXS8ePHNX78eM2aNUsff/yxa39fffWV19sV4vVPDCBOp/m3qMjaOgAA1jMM6cQJaz47IkJyOM6/3dSpU7Vt2zalpKTo2WeflSSVlJSoW7duuu+++zR58mSdPHlSY8eO1Z133qlFixZp//79Gjx4sCZNmqTbbrtN+fn5WrJkiQzD0GOPPaYtW7YoLy9PM2bMkCTVq1fPk02tEGHHg5z5hyXVU9GBo5LqWFsMAMBSJ05IkZHWfPaxY1Lt2uffLiYmRqGhoYqIiFBcXJwk6amnnlKHDh00YcIE13b//Oc/lZiYqG3btunYsWMqLi7WwIED1aRJE0lSmzZtXNuGh4eroKDAtT8rMIzlQc41KyRJRZu2WlwJAAAXZ/Xq1Vq8eLEiIyNdj5YtW0qSfvjhB7Vr1049e/ZUmzZtdMcdd+jNN9/UkSNHLK7aHT07HhTqNCRJhYVV6DsEANhaRITZw2LVZ1+s0tJS9evXTxMnTiy3Lj4+XsHBwVq4cKGWLVumBQsW6JVXXtETTzyhFStWKDk5uRpV1xzCjgcxZwcAUMbhqNpQktVCQ0NVUlLiet2hQwd9+OGHSkpKUkhIxbHB4XCoa9eu6tq1q5566ik1adJE8+bN0yOPPFJuf1ZgGMuDnKHm36Jia+sAAKCqkpKStGLFCu3evVsHDx7UyJEjdfjwYQ0ePFj/+9//tHPnTi1YsEDDhw9XSUmJVqxYoQkTJmjVqlXau3ev5s6dqwMHDqhVq1au/a1fv15bt27VwYMHVWRBDwBhx4OcTnP4qqiYYSwAgH947LHHFBwcrNatW6thw4YqLCzUN998o5KSEt1www1KSUnRQw89pJiYGAUFBSk6Olpff/21+vbtq+bNm+vPf/6zXnzxRd10002SpPvuu08tWrRQamqqGjZsqG+++cbrbWIYy4Ocob+EnSLCDgDAPzRv3lzffvttueVz586tcPtWrVrps88+q3R/DRs21IIFC2qsvotBz44HhYaZfwuL+ZoBALAKv8IexDAWAADWI+x4kDPM/HqLSgg7AABYhbDjQafDTrDFlQAAELgIOx4UGmb26BSW8DUDQCAyDMPqEvxeTXyH/Ap7kLOW2aNTRNgBgIDi/OWqsiesuvOnjZR9h2Xf6cXg1HMPcv7Ss1NUyjAWAASS4OBg1alTRzk5OZKkiIgIOapy23G4GIahEydOKCcnR3Xq1FFw8MX/lhJ2PMgZVtazQ9gBgEBTdpfvssCDi1OnTp1q3zGdsONBZcNYhfTsAEDAcTgcio+PV6NGjSy5RYIdOJ3OavXolCHseFBo+C89O6V8zQAQqIKDg2vkBxsXj5mzHuQMN0NOkcE/cgAArELY8SDX2VgGPTsAAFiFsONBrp4dhrEAALAMYceDQiPMkFOoi782AAAAqB7CjgednrNDzw4AAFYh7HiQM8Ls0SmiZwcAAMsQdjyoLOyUKlglJRYXAwBAgCLseFDZnB1JKiootbASAAACF2HHg8p6diSp6ARXzwQAwAqEHQ9y1g51PSfsAABgDcKOBwWHE3YAALAaYceDHCHBcqpQklR4nLADAIAVCDseFvpL2Ck6SdgBAMAKhB0Pc8oMOUUnii2uBACAwOTTYae4uFh//vOflZycrPDwcDVt2lTPPvusSktPn8ZtGIbGjx+vhIQEhYeHq3v37tq0aZOFVbtzOsyQU3SSsAMAgBV8OuxMnDhRr7/+uqZNm6YtW7Zo0qRJ+utf/6pXXnnFtc2kSZM0efJkTZs2TStXrlRcXJx69+6t/Px8Cys/zRV2TnFVQQAArODTYefbb79V//79dfPNNyspKUm33367+vTpo1WrVkkye3WmTJmiJ554QgMHDlRKSopmzZqlEydOaPbs2RZXbwr9JewUMowFAIAlfDrsXHPNNfryyy+1bds2SdJ3332npUuXqm/fvpKkXbt2KTs7W3369HG9JywsTN26ddOyZcsq3W9BQYHy8vLcHp7iDGIYCwAAK/n07bjHjh2r3NxctWzZUsHBwSopKdHzzz+vwYMHS5Kys7MlSbGxsW7vi42N1Z49eyrdb0ZGhp555hnPFX4GZ5A5fMUwFgAA1vDpnp333ntP77zzjmbPnq01a9Zo1qxZ+tvf/qZZs2a5bedwONxeG4ZRbtmZxo0bp9zcXNcjKyvLI/VLktNB2AEAwEo+3bPzxz/+UY8//rjuuusuSVKbNm20Z88eZWRkaNiwYYqLi5Nk9vDEx8e73peTk1Out+dMYWFhCgsL82zxv3AGmyGn8BQ3AgUAwAo+3bNz4sQJBQW5lxgcHOw69Tw5OVlxcXFauHCha31hYaEyMzPVpUsXr9ZamdCyYSzueg4AgCV8umenX79+ev7553XppZfqiiuu0Nq1azV58mQNHz5ckjl8lZ6ergkTJqhZs2Zq1qyZJkyYoIiICA0ZMsTi6k1lPTsMYwEAYA2fDjuvvPKKnnzySY0YMUI5OTlKSEjQ/fffr6eeesq1zZgxY3Ty5EmNGDFCR44cUadOnbRgwQJFRUVZWPlpzmCzR4eeHQAArOEwDMOwugir5eXlKSYmRrm5uYqOjq7RffeNXaVPc1L1z7RM3T2jW43uGwCAQFbV32+fnrNjB6Gunp2Az5QAAFiCsONhzhAz5BQVEnYAALACYcfDnCG/9OwQdgAAsARhx8Po2QEAwFqEHQ8L/SXsFBZaXAgAAAGKsONhTic9OwAAWImw42FOp/m3iJueAwBgCcKOh7nCTpG1dQAAEKgIOx7m/OUa1YWFld+FHQAAeA5hx8NCQ82/RcWEHQAArEDY8TBnqBlymLMDAIA1CDsedjrs8FUDAGAFfoE9rCzsFDKMBQCAJQg7HhYaRs8OAABW4hfYw1zDWCX07AAAYAXCjoc5w8yvuKiErxoAACvwC+xhhB0AAKzFL7CHOWsFS5IKS0IsrgQAgMBE2PEw1wTlUr5qAACswC+wh5X17BSVBltcCQAAgYmw42GEHQAArEXY8TBnuDlXp7CUOTsAAFiBsONhoeG/9OwYhB0AAKxA2PGwsp4dwg4AANYg7HiYa84OYQcAAEsQdjzMNWfHcFpcCQAAgYmw42Ghtc2QUySnZBgWVwMAQOAh7HiYM+KMsFNUZHE1AAAEHsKOh7mFncJCi6sBACDwEHY8zHU2FmEHAABLEHY8rOxsrFIFq+QkYQcAAG8j7HhY2Y1AJanoOGEHAABvI+x4mPOMM86LTjBBGQAAbyPseJhb2DlZbF0hAAAEKMKOhwWfcbPzwhOEHQAAvI2w42EOhxSqAkn07AAAYAXCjhc4HWbIYc4OAADeR9jxAlfYoWcHAACvI+x4gTOoRJJUVFBqcSUAAAQewo4XlPXsMEEZAADvI+x4QWjQL8NYp0osrgQAgMBD2PEC1zAWYQcAAK8j7HiBM8icq1NUQNgBAMDbCDte4Aw2Q07hSSYoAwDgbYQdLwj9JewUFRoWVwIAQOAh7HiBM7hsGIueHQAAvI2w4wWEHQAArEPY8QJnsDl8VVjAMBYAAN5G2PECZ4gZcooK6dkBAMDbCDteEBryyzBWocWFAAAQgAg7XnC6Z4dhLAAAvI2w4wVO5y9hp8jiQgAACECEHS9whph/CxnGAgDA6wg7XhAaSs8OAABWIex4gdNp/iXsAADgfYQdL3CFnWJr6wAAIBARdrzA6XRIkgqL+LoBAPA2fn29IDTM/EvPDgAA3kfY8QJnqNmzU1TM1w0AgLfx6+sFzjDza2aCMgAA3kfY8QJnVLgkqfBkicWVAAAQeAg7XuCsU1uSVHSKSTsAAHgbYccLQutFSvrl3liMZQEA4FU+H3b27dun3/zmN6pfv74iIiLUvn17rV692rXeMAyNHz9eCQkJCg8PV/fu3bVp0yYLKy7PGRMhSSqSUzpwwOJqAAAILD4ddo4cOaKuXbvK6XTq008/1ebNm/Xiiy+qTp06rm0mTZqkyZMna9q0aVq5cqXi4uLUu3dv5efnW1f4WVwTlOWUfv7Z4moAAAgsIVYXcC4TJ05UYmKiZsyY4VqWlJTkem4YhqZMmaInnnhCAwcOlCTNmjVLsbGxmj17tu6//35vl1yhsisoFypUysmxthgAAAKMT/fszJ8/X6mpqbrjjjvUqFEjXXnllXrzzTdd63ft2qXs7Gz16dPHtSwsLEzdunXTsmXLKt1vQUGB8vLy3B6eFBpq/iXsAADgfT4ddnbu3Knp06erWbNm+vzzz/XAAw/owQcf1Ntvvy1Jys7OliTFxsa6vS82Nta1riIZGRmKiYlxPRITEz3XCEkNGph/f1ICw1gAAHiZT4ed0tJSdejQQRMmTNCVV16p+++/X/fdd5+mT5/utp3D4XB7bRhGuWVnGjdunHJzc12PrKwsj9RfpkUL8+9ONVXh/kMe/SwAAODOp8NOfHy8Wrdu7basVatW2rt3ryQpLi5Oksr14uTk5JTr7TlTWFiYoqOj3R6edMklUu3QQpUoRDt/MDz6WQAAwJ1Ph52uXbtq69atbsu2bdumJk2aSJKSk5MVFxenhQsXutYXFhYqMzNTXbp08Wqt5+JwSC3izXlBW/eGW1wNAACBxafDzsMPP6zly5drwoQJ2rFjh2bPnq033nhDI0eOlGQOX6Wnp2vChAmaN2+eNm7cqLS0NEVERGjIkCEWV++uRVKhJGnrz3WsLQQAgADj06eeX3XVVZo3b57GjRunZ599VsnJyZoyZYqGDh3q2mbMmDE6efKkRowYoSNHjqhTp05asGCBoqKiLKy8vBYtDClT+v5o5cNrAACg5jkMwwj4SSR5eXmKiYlRbm6ux+bvzJl2UINHN1AXxzJ9U9LZHNuyQFGR9OyzUmys9MADUohPx10AACpX1d9vnx7GspOWV5k9TVuN5tLRo5bV8eab0l/+Io0eLXXuLG3YYFkpAAB4BWHHS5qlhEmSDqmBDm09aEkNJ09Kzz9vPg8JkVatkjp2lMaPlwoLLSkJAACPI+x4Se3aUmLIT5KkrWuOW1LDG29IP/0kJSZK27dL/fubw1rPPGOGnpUrLSkLAACPIux4UYva+yRJ328s9vpnnzghZWSYz598UkpKkubNk+bMkRo2lDZulK6+WgoKqtojOFi6+26vNwMAgAtG2PGilvUPSJK27vD+1/7qq+adKpKTpbQ0c5nDIQ0aJG3eLJWd4GYYVXuUlkozZ0rffOP1pgAAcEEIO17UIsGaCwvm50sTJ5rPn3rq9F3YyzRoIL3zjnTkiLR/f9UeZYHpqae82hQAAC4YJx57UYsmBdJS719Y8JVXpEOHpObNpd/8pvLt6tSp+j7Hj5fefVdatEj66iupe/fKtz1wQNqzR7rySnP4CwAAb6Jnx4tatDSvrbMjt6GKirzzmbm50t/+Zj5/+umau65OkybSffeZz5980hzaqsiBA+bk56uuMu8RNmKE9OWXUrH3py0BAAIUYceLGjePUISOq9gI0a5d3vnMl14yh6datzbn59SkP/1JCguTli6Vzrg9mUtJiTkXqOym8j//LE2fLvXqJcXFSffeK332Gae9AwA8i7DjRUHxsWqubZKks+5v6hGHD5thRzKHnWp6COmSS8yrMEsV9+4895wZgiIipDVrpE8/NQNOgwbmsNo//iHddJN5Nedhw6T586VTp2q2RgAACDve1KiRWshMOd4IOy+8IOXlSW3bSr/+tWc+4/HHpfBw6X//kz755PTyzz83b0shSX//uzlf58YbzSs4799vDmX94Q9mD8/Ro9Lbb5vX/WnYUBoyRNq0yTP1AgACD2HHm+LjT4edDQUe+5iSEumPf5T++lfz9TPPmNfG8YS4OGnUKPP5U0+ZvTtZWebwlWGYPT9nT4oOCZF69JBee0368Ufp66+lhx6SGjeWjh2T/v1vMxw9+yxDXACA6iPseFN0tFo0PCJJ+n7NSY98xNGj0i23nJ6U/Oc/mz0mnjRmjBQZaQ5V/ec/0p13msNUHTueHkarTHCwdO210pQp5hlby5dL/fqZV3Z++mkpNdW8rQUAABeLsONlLduZ98jaust5ni0v3PffS506mZN+w8Ol994z5814+gbrDRqYPTOS2YuzfLl5Gvt//iPVqlX1/QQFmfV//LE0e7a53w0bzGVjx5r39gIA4EIRdrys+XVxkqQDx2vryJGa2+8nn5ihYNs26dJLpWXLzB4Wb3nkESk6Wq5T6t9+27xa88VwOKTBg80rO991l3m15kmTpPbtpSVLLnx/+/ZJ06ZJ118vXXON2YMEAAgchB0vi7ymvS7Rj5JqZpKyYZhXR77lFnMy8rXXmjf0bN+++vu+EPXqmWdkSebQWb9+1d9nw4bm/J2PP5bi480gd9115hyh/Pxzv3fPHmnyZKlrV3Mu0OjR5sUPv/lG6tZN2rmz+vUBAPzDBYed4uJihYSEaOPGjZ6ox/46dHBNUv5+5Xl+sc/jxAlzIvDjj5uh5/77pS++kBo1qolCL9yjj5q9KM89V7P7vfVWs5fnnnvM16++KqWkSAsWuG+3fbt5BtpVV5k3On30UbOHS5K6dDF7h5o3N4NQt27Sjh01WycAwDdd8PV0Q0JC1KRJE5WUlHiiHvuLiVHLmGwtypW2Lj0gjY66qN1kZUkDBpiTgkNCpJdfNk/ltpLDISUkeGbfdepIb71lDmvdd5+0e7d0ww3mPbqaNpU++EBav/709kFBZi/X7bdLt91mXhNIMucU9ewpbdli9hItWiS1bOmZmgEAvuGihrH+/Oc/a9y4cTp8+HBN1xMQWlxuBsWtGy7uvOqSEqlPHzPoNGhg9uZYHXS8pVcvc9Lygw+a4WrmTPOU9/XrzTO7evc2r+uzf785bDVq1OmgI5nDYV99ZfYM7d9v3tOLa/oAgL1d1J2SXn75Ze3YsUMJCQlq0qSJateu7bZ+zZo1NVKcXbVIjZJWS1t/jLio93/2mXnmVd265vycpKSarc/XRUZKU6eaE7CfeUYKDTUvmnjrrVL9+ud/f6NG0uLFZjBat86cuPzFF+bFFwEA9nNRYWfAgAE1XEZgadGzsfR3aXt+nIqLL/zmnK+/bv69++7ACzpn6tq1/LydqmrQwLyKc58+0urVpwPPlVfWbI0AAOs5DKOy+1UHjry8PMXExCg3N1fR0dEe/7zS/OOqHR2kUwrX9qU/6/KusVV+75495indhmGezdW8uQcLDQBHj5q3sVixwpwXtGCBOcEZAOD7qvr7Xa1Tz1evXq133nlH7777rtauXVudXQWUoKjaalVrtyRp3oQtF/TeN94wg07PngSdmlAWcLp0MYNPr17St99aXRUAoCZdVNjJyclRjx49dNVVV+nBBx/UqFGj1LFjR/Xs2VMHDhyo6RptadRdByVJz3ySqh//91OV3lNYaN4pXAqcCcneEB1t3rj0uuvMaxX16SMtXWp1VQCAmnJRYWf06NHKy8vTpk2bdPjwYR05ckQbN25UXl6eHnzwwZqu0ZbS3uyqrlHrdVyRSh9YtUv6fvSR9PPP5hlFt97q2foCTWSkeRXqHj3Mm5HeeKN51hYAwP9dVNj57LPPNH36dLVq1cq1rHXr1nr11Vf16aef1lhxdhYUEqTX/llLwSrWh/s667PnV5/3PdOnm3/vvVdy1vyttQJe7drSf/9r9uwcPy717Wve5PR//zOHDgEA/umiwk5paamcFfzaOp1OlZaWVruoQNH29uZ6sO1XkqQXXo0857bff2/2NAQFmRfVg2eEh5u3p+jb17zx6F//at5zLClJevhh83YT/BMHAP9yUWGnR48eeuihh/TTT6fnmuzbt08PP/ywevbsWWPFBYKHf3NQQSpR5v4W2nKOucplp5vfcouUmOid2gJVrVpm4PngA/OKzZGR0t690pQp5o1EGzc2L1a4eLFUXGx1tQCA87moU8+zsrLUv39/bdy4UYmJiXI4HNq7d6/atGmjjz/+WI0bN/ZErR7j7VPP3Xz5pQb0ytfHGqAHHzQvlne2EyfMqwAfPSp9+qk5nwTec/KkecbWhx9K8+dLubmn1zVsaN624/bbzWv1WDm8aBjSwYPmvxdPCAuT4uI8s28AuBhV/f2u1nV2Fi5cqO+//16GYah169bq1avXxe7KUpaGnR079FmzUbpJnykmxtC+fQ6ddUFqTZ8ujRhhXl9nxw5zKAvWKCw0L0b4wQfmhPEz75hSt67Uv795Nefevc1w4AmGYd5wdfNm8x5fmzeffnj6Di6tW5vtu/12qU0b85YdAGAVj4Wd4uJi1apVS+vWrVNKSkq1C/UFloadggKV1gpXM23XTl2mf/xDGj789OqDB6UWLcwfsZdflkaP9m55qFxRkZSZafb4zJ0r5eScXhcdbQ453n67ecPSiIu4M0hpqXkRyTPDTFm4yc+v+D0Oh+dCVmGh+3ylyy832/frX0sdOxJ8AHifR3t2LrvsMs2dO1ft2rWrVpG+wtKwI0mXXKJJPw3VWE1Saqp5v6syw4dLM2ZI7dpJq1Zd+K0l4B0lJebk5Q8+MMPPGdPZFBEh3XyzGQpuvtmcA3Sm4mJp5073ULN5szkp/eTJij8vOFhq1szsaWndWmrVyvzbooU5ydoTjh41z1b74APz/mwFBafXNWlyusenUyd6HwF4h0fDzowZM/Sf//xH77zzjurVq1etQn2B5WGnSxcd+Ha7Godkq7A4WCtXSqmp5oXtrr3W3GTZMqlzZ++XhgtXWmrefqIs+Ow54zJKtWqZPT1t25q3+9i8Wdq2zew1qUhoqNSy5ekwU/a4/HJznVXy883rEn34ofR//+c+T+iSS6SBA83wc801ZjADAE/waNi58sortWPHDhUVFdnirueWh53Bg6U5czT0yk2avba17rnHnKfToYO0caN5qvkbb3i/LFSfYZg3Gv3wQzP87NhR8XYREe6Bpux5crLv9+adOGFegfqDD6T/9//ch9gaNZJuu83s8ene3ffbAsC/VPX3m7ue+4Jfbl3+h+TPNXtta82ebZ71snGjeXfuF16wtjxcPIfD7KVLTZUmTJA2bDCDz/795pBTWbhJTPTfoZ+ICDPQ3HabObS1cKHZxo8/Nucx/f3v5qN+fXMC9+23m/d2s7JnCkBgueCwU/zLhUWGDx+uRC74UjOaNJEkdS1YpCuueFibNknPP2+u+utfJRuMFEJm8Gnb1nzYVViYOTH7llvMCdyLF58+c+3AAemf/zQfMTFSv35m8OnTx3PzjABAushhrKioKG3YsEFJv/RI+DvLh7E++0y66SapTRu9ev96jRplLr72WvNsH85ygb8rLpaWLDF7fD78UMrOPr2udm3p6qsZ4rpYQUFmeKxTx7z8Qd267s/PfF2nDnOoYC8enbMzYMAADRgwQGlpadWp0WdYHna2bDHHMqKjlbf3qJKSHTp2TFqzRrLJ2f2AS2mp9O23pydwZ2VZXVFgiYo6fyiq7HmtWtbWDpzNo2Hn73//u8aPH6+hQ4eqY8eO5SYo3+pnt+S2POwcP376fOTDh7XjUF0VFpr5B7AzwzAvqbBtm9WV+K/iYvOq3kePSkeOmI+Knh87Vv3PqlXrwgNS2SMykl5q1DyPhp2gc8ykdDgcKikpudBdWsrysCOZp60cOCCtXSu1b29NDQBsq6jIDD6VhaLKQlLZ84u/1r4pONh9OK0qAansdVSUeSsWwhLO5tGzsbizuQc0aWKGnd27CTsAapzTad7LrWHDC39vaal5SYHzhaLK1hUWmhfePHTIfFyMoCCzZyk83PxblecXsu25njPPyf9dUNjp27ev/v3vfysmJkaS9Pzzz2vkyJGqU6eOJOnQoUO69tprtXnz5hov1PaSksz+/DOvQAcAPqBsEnRMjOtKGVVmGOaVwKvai3T26+PHzf2UlprXdPLUjW7PxemsXmC62PeFhdGbVVMuKOx8/vnnKjjjGvETJ07U4MGDXWGnuLhYW7durdECA8Yvp58TdgDYicNhXospIkJKSLjw9xcVmfONTp0yHydPuv+tzvNzrf/lKiuuGoqKKr8nnSdVt2fqYt8XEmKvoHVBYefs6T3VuGE6zlYWdnbvtrQMAPAlTqc5b8fbiovNi2TWVHi6kH2c+dNatv7oUe+2PyioZocDw8PNGwY3berddpThyha+oqxvmJ4dALBcSIj5OOtkY48zDLMXyRu9V2c/P/PmvqWl5hBi2TBiTZg+XXrggZrb34W4oLDjcDjkOKtf6+zXuEgMYwFAwHM4zFuphIZK3j45uLTUDDye6r2y8qYLFzyMlZaWprCwMEnSqVOn9MADD7ius3PmfB5coLKwc+iQOUBddt0dAAC8oGzoyo63b7mgsDNs2DC317/5zW/KbfO73/2uehUFqrLrvR89avbuXHGF1RUBAGALFxR2ZsyY4ak6IJm9O4QdAABqVOWXQob3lU1S5owsAABqDGHHlzBJGQCAGkfY8SVcawcAgBpH2PElXGsHAIAaR9jxJQxjAQBQ4wg7vqSsZyc727wCEwAAqDbCji+pV+/0tcn37rW2FgAAbIKw40scDoayAACoYYQdX8O1dgAAqFGEHV9Dzw4AADWKsONrOP0cAIAaRdjxNVxYEACAGkXY8TUMYwEAUKMIO76mbBhr3z6psNDSUgAAsAO/CjsZGRlyOBxKT093LTMMQ+PHj1dCQoLCw8PVvXt3bdq0yboiqys2VqpfXyotlRYutLoaAAD8nt+EnZUrV+qNN95Q27Zt3ZZPmjRJkydP1rRp07Ry5UrFxcWpd+/eys/Pt6jSanI4pGHDzOfTp1tbCwAANuAXYefYsWMaOnSo3nzzTdWtW9e13DAMTZkyRU888YQGDhyolJQUzZo1SydOnNDs2bMtrLiaHnjA/PvJJ0xUBgCgmvwi7IwcOVI333yzevXq5bZ8165dys7OVp8+fVzLwsLC1K1bNy1btqzS/RUUFCgvL8/t4VOaNZN69ZIMQ3rzTaurAQDAr/l82JkzZ47WrFmjjIyMcuuys7MlSbGxsW7LY2NjXesqkpGRoZiYGNcjMTGxZouuCWW9O2+9xURlAACqwafDTlZWlh566CG98847qlWrVqXbORwOt9eGYZRbdqZx48YpNzfX9cjKyqqxmmvMrbdK8fFSTo40b57V1QAA4Ld8OuysXr1aOTk56tixo0JCQhQSEqLMzEy9/PLLCgkJcfXonN2Lk5OTU66350xhYWGKjo52e/gcp1O6917zOROVAQC4aD4ddnr27KkNGzZo3bp1rkdqaqqGDh2qdevWqWnTpoqLi9PCM07RLiwsVGZmprp06WJh5TXkvvukoCApM1PassXqagAA8EshVhdwLlFRUUpJSXFbVrt2bdWvX9+1PD09XRMmTFCzZs3UrFkzTZgwQRERERoyZIgVJdesxESpXz/p44+l11+Xpk61uiIAAPyOT/fsVMWYMWOUnp6uESNGKDU1Vfv27dOCBQsUFRVldWk1o2yi8qxZ0vHj1tYCAIAfchiGYVhdhNXy8vIUExOj3Nxc35u/U1pqnoq+c6d5GnrZPB4AAAJcVX+//b5nx/aCgqQ//MF8Pn685K9XhgYAwCKEHX8wcqTUtKl5c9Dx462uBgAAv0LY8Qfh4dK0aebzqVOl9eutrQcAAD9C2PEXN90k/frXUkmJOWm5tNTqigAA8AuEHX8yZYoUGSl9+630z39aXQ0AAH6BsONPGjeWnnnGfD52rHTwoLX1AADgBwg7/ubBB6W2baXDh83AAwAAzomw429CQk7fK+uf/5R+/NHaegAA8HGEHX/UpYvUurX5fPNma2sBAMDHEXb81WWXmX9/+MHaOgAA8HGEHX/VtKn5l7ADAMA5EXb8VVnPzs6d1tYBAICPI+z4K4axAACoEsKOvzoz7HDjegAAKkXY8VdJSZLDIR0/LuXkWF0NAAA+i7Djr8LCpMRE8zlDWQAAVIqw48+YtwMAwHkRdvwZp58DAHBehB1/Rs8OAADnRdjxZ1xrBwCA8yLs+DN6dgAAOC/Cjj8rCzs//ywdO2ZtLQAA+CjCjj+rU0eqV898zlAWAAAVIuz4O4ayAAA4J8KOv+P0cwAAzomw4+/o2QEA4JwIO/6OsAMAwDkRdvwd19oBAOCcCDv+rizs7NkjFRdbWwsAAD6IsOPvEhLMO6AXF0t791pdDQAAPoew4++CgjgjCwCAcyDs2AFhBwCAShF27IAzsgAAqBRhxw4IOwAAVIqwYwecfg4AQKUIO3ZwZs+OYVhbCwAAPoawYwfJyZLDIR07Jh04YHU1AAD4FMKOHYSFSY0bm8+ZtwMAgBvCjl1w+jkAABUi7NgFZ2QBAFAhwo5dlIWd7dutrQMAAB9D2LGL1FTz76JFUmmptbUAAOBDCDt20a2bFBUl7d8vrV5tdTUAAPgMwo5dhIVJN95oPv/4Y2trAQDAhxB27OTWW82/8+dbWwcAAD6EsGMnfftKwcHShg3Srl1WVwMAgE8g7NhJvXrStdeaz+ndAQBAEmHHfvr3N/8SdgAAkETYsZ+yeTuZmdKRI9bWAgCADyDs2E3TplJKilRSIn36qdXVAABgOcKOHZX17nAKOgAAhB1bKpu38+mnUmGhtbUAAGAxwo4dpaZKcXFSfr701VdWVwMAgKUIO3YUFCT162c+56wsAECAI+zY1ZmnoBuGtbUAAGAhwo5d9eghRURIWVnS+vVWVwMAgGUIO3YVHi51724+//JLS0sBAMBKhB07u/568+/ixdbWAQCAhQg7dtajh/n366+l4mJrawEAwCKEHTtr106qU0fKy5PWrLG6GgAALEHYsbPg4NPzdhjKAgAEKMKO3ZXN21m0yNo6AACwCGHH7srCztKl3DoCABCQfDrsZGRk6KqrrlJUVJQaNWqkAQMGaOvWrW7bGIah8ePHKyEhQeHh4erevbs2bdpkUcU+6IorpIYNpRMnpJUrra4GAACv8+mwk5mZqZEjR2r58uVauHChiouL1adPHx0/fty1zaRJkzR58mRNmzZNK1euVFxcnHr37q38/HwLK/chQUGn5+0wlAUACEAOw/CfewkcOHBAjRo1UmZmpq677joZhqGEhASlp6dr7NixkqSCggLFxsZq4sSJuv/++6u037y8PMXExCg3N1fR0dGebII1pk+XRowwh7QIPAAAm6jq77dP9+ycLTc3V5JUr149SdKuXbuUnZ2tPn36uLYJCwtTt27dtGzZskr3U1BQoLy8PLeHrZVdb2fZMunUKWtrAQDAy/wm7BiGoUceeUTXXHONUlJSJEnZ2dmSpNjYWLdtY2NjXesqkpGRoZiYGNcjMTHRc4X7gubNpfh4qaBA+vZbq6sBAMCr/CbsjBo1SuvXr9e///3vcuscDofba8Mwyi0707hx45Sbm+t6ZGVl1Xi9PsXhON27w/V2AAABxi/CzujRozV//nwtXrxYjRs3di2Pi4uTpHK9ODk5OeV6e84UFham6Ohot4ftcb0dAECA8umwYxiGRo0apblz52rRokVKTk52W5+cnKy4uDgtXLjQtaywsFCZmZnq0qWLt8v1bWVhZ8UK6Yyz2QAAsDufDjsjR47UO++8o9mzZysqKkrZ2dnKzs7WyZMnJZnDV+np6ZowYYLmzZunjRs3Ki0tTRERERoyZIjF1fuY5GSpSRPzhqBLl1pdDQAAXuPTYWf69OnKzc1V9+7dFR8f73q89957rm3GjBmj9PR0jRgxQqmpqdq3b58WLFigqKgoCyv3QQ7H6d4d5u0AAAKIX11nx1Nsf52dMm+/LQ0bJl19tTmcBQCAH7PldXZQTWU9O6tWSb9cswgAALsj7ASSxETp8sul0lJpyRKrqwEAwCsIO4GGU9ABAAGGsBNoyi4uSNgBAAQIwk6gKbsD+nffSYcOWVoKAADeQNgJNHFxUuvW5vPMTGtrAQDACwg7gYjr7QAAAghhJxAxSRkAEEAIO4GobN7O5s3Szz9bWgoAAJ5G2AlE9etL7dqZz7/6ytJSAADwNMJOoOIUdABAgCDsBComKQMAAgRhJ1Bdd50UFCRt3y79+KPV1QAA4DGEnUAVEyN17Gg+p3cHAGBjhJ1AxlAWACAAEHYCGZOUAQABgLATyLp2lUJCpD17pF27rK4GAACPIOwEsshIqVMn8zm9OwAAmyLsBDrm7QAAbI6wE+jODDuGYW0tAAB4AGEn0HXuLIWFST/9JG3bZnU1AADUOMJOoAsPNwOPxFAWAMCWCDs4fQo6YQcAYEOEHTBvBwBga4QdSFdfLUVESAcOSJs2WV0NAAA1irADKTRUuuYa8znX2wEA2AxhB6YuXcy/69dbWwcAADWMsAPT5Zebf3fssLYOAABqGGEHJsIOAMCmCDswlYWdffukkyetrQUAgBpE2IGpXj0pJsZ8vnOntbUAAFCDCDswORwMZQEAbImwg9PKws4PP1hbBwAANYiwg9Muu8z8S88OAMBGCDs4jWEsAIANEXZwGsNYAAAbIuzgtLJhrN27pcJCS0sBAKCmEHZwWny8FB4ulZZKe/ZYXQ0AADWCsIPTzjz9nKEsAIBNEHbgjjOyAAA2Q9iBO87IAgDYDGEH7hjGAgDYDGEH7hjGAgDYDGEH7sp6dnbulEpKrK0FAIAaQNiBu8REyek0r7Ozb5/V1QAAUG2EHbgLDpaSk83nDGUBAGyAsIPyOCMLAGAjhB2UxxlZAAAbIeygPM7IAgDYCGEH5TGMBQCwEcIOyjtzGMswrK0FAIBqIuygvKQkKShIOn5c+vlnq6sBAKBaCDsoLzRUuvRS8zlDWQAAP0fYQcU4IwsAYBOEHVSMM7IAADZB2EHFOCMLAGAThB1UjGEsAIBNEHZQMYaxAAA2QdhBxZo2Nf8eOSIdPmxtLQAAVANhBxWrXVtKSDCfM5QFAPBjhB1UjqEsAIANEHZQOc7IAgDYAGEHleOMLACADdgm7Lz22mtKTk5WrVq11LFjRy1ZssTqkvwfw1gAABuwRdh57733lJ6erieeeEJr167Vtddeq5tuukl79+61ujT/xjAWAMAGHIZhGFYXUV2dOnVShw4dNH36dNeyVq1aacCAAcrIyDjv+/Py8hQTE6Pc3FxFR0d7slT/cvSoVLeu+XzOHMnptLQcAIAfu/JKKTm5RndZ1d/vkBr9VAsUFhZq9erVevzxx92W9+nTR8uWLavwPQUFBSooKHC9zs3NlWR+aThDUJBUr555nZ277rK6GgCAP5syRbr77hrdZdnv9vn6bfw+7Bw8eFAlJSWKjY11Wx4bG6vs7OwK35ORkaFnnnmm3PLExESP1AgAQMBLTzcfHpCfn6+YmJhK1/t92CnjcDjcXhuGUW5ZmXHjxumRRx5xvS4tLdXhw4dVv379St9zIfLy8pSYmKisrCzbDovZvY12b59EG+3A7u2TaKMdeLJ9hmEoPz9fCWUXwa2E34edBg0aKDg4uFwvTk5OTrnenjJhYWEKCwtzW1anTp0ary06OtqW/3DPZPc22r19Em20A7u3T6KNduCp9p2rR6eM35+NFRoaqo4dO2rhwoVuyxcuXKguXbpYVBUAAPAVft+zI0mPPPKIfvvb3yo1NVWdO3fWG2+8ob179+qBBx6wujQAAGAxW4SdQYMG6dChQ3r22We1f/9+paSk6JNPPlGTJk0sqScsLExPP/10uaEyO7F7G+3ePok22oHd2yfRRjvwhfbZ4jo7AAAAlfH7OTsAAADnQtgBAAC2RtgBAAC2RtgBAAC2RtjxgNdee03JycmqVauWOnbsqCVLllhd0kUZP368HA6H2yMuLs613jAMjR8/XgkJCQoPD1f37t21adMmCys+v6+//lr9+vVTQkKCHA6HPvroI7f1VWlTQUGBRo8erQYNGqh27dq69dZb9eOPP3qxFZU7X/vS0tLKHdNf/epXbtv4cvsyMjJ01VVXKSoqSo0aNdKAAQO0detWt238/RhWpY3+fhynT5+utm3bui4y17lzZ3366aeu9f5+DM/XPn8/fhXJyMiQw+FQ+hm3g/Cl40jYqWHvvfee0tPT9cQTT2jt2rW69tprddNNN2nv3r1Wl3ZRrrjiCu3fv9/12LBhg2vdpEmTNHnyZE2bNk0rV65UXFycevfurfz8fAsrPrfjx4+rXbt2mjZtWoXrq9Km9PR0zZs3T3PmzNHSpUt17Ngx3XLLLSopKfFWMyp1vvZJ0o033uh2TD/55BO39b7cvszMTI0cOVLLly/XwoULVVxcrD59+uj48eOubfz9GFaljZJ/H8fGjRvrhRde0KpVq7Rq1Sr16NFD/fv3d/0Q+vsxPF/7JP8+fmdbuXKl3njjDbVt29ZtuU8dRwM16uqrrzYeeOABt2UtW7Y0Hn/8cYsqunhPP/200a5duwrXlZaWGnFxccYLL7zgWnbq1CkjJibGeP31171UYfVIMubNm+d6XZU2HT161HA6ncacOXNc2+zbt88ICgoyPvvsM6/VXhVnt88wDGPYsGFG//79K32PP7XPMAwjJyfHkGRkZmYahmG/Y2gY5dtoGPY7joZhGHXr1jXeeustWx5DwzjdPsOw1/HLz883mjVrZixcuNDo1q2b8dBDDxmG4Xv/LdKzU4MKCwu1evVq9enTx215nz59tGzZMouqqp7t27crISFBycnJuuuuu7Rz505J0q5du5Sdne3W1rCwMHXr1s1v21qVNq1evVpFRUVu2yQkJCglJcVv2v3VV1+pUaNGat68ue677z7l5OS41vlb+3JzcyVJ9erVk2TPY3h2G8vY5TiWlJRozpw5On78uDp37my7Y3h2+8rY5fiNHDlSN998s3r16uW23NeOoy2uoOwrDh48qJKSknI3II2NjS13o1J/0KlTJ7399ttq3ry5fv75Z/3lL39Rly5dtGnTJld7Kmrrnj17rCi32qrSpuzsbIWGhqpu3brltvGHY3zTTTfpjjvuUJMmTbRr1y49+eST6tGjh1avXq2wsDC/ap9hGHrkkUd0zTXXKCUlRZL9jmFFbZTscRw3bNigzp0769SpU4qMjNS8efPUunVr14+cvx/Dyton2eP4SdKcOXO0Zs0arVy5stw6X/tvkbDjAQ6Hw+21YRjllvmDm266yfW8TZs26ty5sy677DLNmjXLNZnOLm0908W0yV/aPWjQINfzlJQUpaamqkmTJvq///s/DRw4sNL3+WL7Ro0apfXr12vp0qXl1tnlGFbWRjscxxYtWmjdunU6evSoPvzwQw0bNkyZmZmu9f5+DCtrX+vWrW1x/LKysvTQQw9pwYIFqlWrVqXb+cpxZBirBjVo0EDBwcHlEmlOTk65dOuPateurTZt2mj79u2us7Ls1NaqtCkuLk6FhYU6cuRIpdv4k/j4eDVp0kTbt2+X5D/tGz16tObPn6/FixercePGruV2OoaVtbEi/ngcQ0NDdfnllys1NVUZGRlq166dpk6daptjWFn7KuKPx2/16tXKyclRx44dFRISopCQEGVmZurll19WSEiIq05fOY6EnRoUGhqqjh07auHChW7LFy5cqC5dulhUVc0pKCjQli1bFB8fr+TkZMXFxbm1tbCwUJmZmX7b1qq0qWPHjnI6nW7b7N+/Xxs3bvTLdh86dEhZWVmKj4+X5PvtMwxDo0aN0ty5c7Vo0SIlJye7rbfDMTxfGyvib8exIoZhqKCgwBbHsCJl7auIPx6/nj17asOGDVq3bp3rkZqaqqFDh2rdunVq2rSpbx3HGp3uDGPOnDmG0+k0/vGPfxibN2820tPTjdq1axu7d++2urQL9uijjxpfffWVsXPnTmP58uXGLbfcYkRFRbna8sILLxgxMTHG3LlzjQ0bNhiDBw824uPjjby8PIsrr1x+fr6xdu1aY+3atYYkY/LkycbatWuNPXv2GIZRtTY98MADRuPGjY0vvvjCWLNmjdGjRw+jXbt2RnFxsVXNcjlX+/Lz841HH33UWLZsmbFr1y5j8eLFRufOnY1LLrnEb9r3hz/8wYiJiTG++uorY//+/a7HiRMnXNv4+zE8XxvtcBzHjRtnfP3118auXbuM9evXG3/605+MoKAgY8GCBYZh+P8xPFf77HD8KnPm2ViG4VvHkbDjAa+++qrRpEkTIzQ01OjQoYPbKaP+ZNCgQUZ8fLzhdDqNhIQEY+DAgcamTZtc60tLS42nn37aiIuLM8LCwozrrrvO2LBhg4UVn9/ixYsNSeUew4YNMwyjam06efKkMWrUKKNevXpGeHi4ccsttxh79+61oDXlnat9J06cMPr06WM0bNjQcDqdxqWXXmoMGzasXO2+3L6K2ibJmDFjhmsbfz+G52ujHY7j8OHDXf8b2bBhQ6Nnz56uoGMY/n8Mz9U+Oxy/ypwddnzpODoMwzBqtq8IAADAdzBnBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphBwAA2BphB8AFGT9+vNq3b3/Obbp376709PRzbpOUlKQpU6accxuHw6GPPvrogurzJVVp45lmzpypOnXqnHObqnz/ANwRdgAbSktLk8Ph0AsvvOC2/KOPPvLKXZPnzp2r5557zuOf4+tWrlyp3//+91aXAQQ8wg5gU7Vq1dLEiRPL3VHYG+rVq6eoqCivf66vKCwslCQ1bNhQERERFlcDgLAD2FSvXr0UFxenjIyMc2734Ycf6oorrlBYWJiSkpL04osvVmn///rXv5SUlKSYmBjdddddys/Pd607exgrJydH/fr1U3h4uJKTk/Xuu++W29/27dt13XXXqVatWmrdurXbnZDL7Nu3T4MGDVLdunVVv3599e/fX7t373atT0tL04ABA/S3v/1N8fHxql+/vkaOHKmioqIK27B161Y5HA59//33bssnT56spKQkGYahkpIS3XPPPUpOTlZ4eLhatGihqVOnum1f9rkZGRlKSEhQ8+bNJZUfxpo8ebLatGmj2rVrKzExUSNGjNCxY8fK1fXRRx+pefPmqlWrlnr37q2srKwK6y8zY8YMtWrVSrVq1VLLli312muvudYVFhZq1KhRio+PV61atZSUlHTefxOA3RB2AJsKDg7WhAkT9Morr+jHH3+scJvVq1frzjvv1F133aUNGzZo/PjxevLJJzVz5sxz7vuHH37QRx99pP/+97/673//q8zMzHJDZmdKS0vT7t27tWjRIn3wwQd67bXXlJOT41pfWlqqgQMHKjg4WMuXL9frr7+usWPHuu3jxIkTuv766xUZGamvv/5aS5cuVWRkpG688UZXT4okLV68WD/88IMWL16sWbNmaebMmZW2p0WLFurYsWO58DV79mwNGTJEDodDpaWlaty4sd5//31t3rxZTz31lP70pz/p/fffd3vPl19+qS1btmjhwoX673//W+HnBQUF6eWXX9bGjRs1a9YsLVq0SGPGjCnXzueff16zZs3SN998o7y8PN11112VfrdvvvmmnnjiCT3//PPasmWLJkyYoCeffFKzZs2SJL388suaP3++3n//fW3dulXvvPOOkpKSKt0fYEs1fmtRAJYbNmyY0b9/f8MwDONXv/qVMXz4cMMwDGPevHnGmf/ZDxkyxOjdu7fbe//4xz8arVu3rnTfTz/9tBEREWHk5eW5vadTp06u12fe/Xjr1q2GJGP58uWu9Vu2bDEkGS+99JJhGIbx+eefG8HBwUZWVpZrm08//dSQZMybN88wDMP4xz/+YbRo0cIoLS11bVNQUGCEh4cbn3/+uavdTZo0MYqLi13b3HHHHcagQYMqbc/kyZONpk2bul6X1btp06ZK3zNixAjj17/+tev1sGHDjNjYWKOgoMBtuyZNmrjaWJH333/fqF+/vuv1jBkzKv2uVqxYYRiG+f23a9fOtT4xMdGYPXu2236fe+45o3PnzoZhGMbo0aONHj16uH1vQKChZwewuYkTJ2rWrFnavHlzuXVbtmxR165d3ZZ17dpV27dvV0lJSaX7TEpKcpuTEx8f79ZTc/ZnhISEKDU11bWsZcuWbmcdbdmyRZdeeqkaN27sWta5c2e3/axevVo7duxQVFSUIiMjFRkZqXr16unUqVP64YcfXNtdccUVCg4OrlJtknTXXXdpz549Wr58uSTp3XffVfv27dW6dWvXNq+//rpSU1PVsGFDRUZG6s0339TevXvd9tOmTRuFhoZW+jmS2evUu3dvXXLJJYqKitLvfvc7HTp0SMePH3dtU9l3tWXLlnL7O3DggLKysnTPPfe4vpPIyEj95S9/cX0naWlpWrdunVq0aKEHH3xQCxYsOGeNgB0RdgCbu+6663TDDTfoT3/6U7l1hmGUOzvLMIzz7tPpdLq9LhvuqUjZ/s51FlhFn3n29qWlperYsaPWrVvn9ti2bZuGDBlyUbVJZhi6/vrrNXv2bEnSv//9b/3mN79xrX///ff18MMPa/jw4VqwYIHWrVunu+++223oTJJq165d6WdI0p49e9S3b1+lpKToww8/1OrVq/Xqq69KUrk5RRV9VxUtK2vXm2++6fadbNy40RXeOnTooF27dum5557TyZMndeedd+r2228/Z62A3YRYXQAAz3vhhRfUvn1718TZMq1bt9bSpUvdli1btkzNmzd36x2pjlatWqm4uFirVq3S1VdfLcmcGHz06FG3Ovbu3auffvpJCQkJkqRvv/3WbT8dOnTQe++9p0aNGik6OrpGaiszdOhQjR07VoMHD9YPP/zgNkdmyZIl6tKli0aMGOFadmZPUlWtWrVKxcXFevHFFxUUZP7/zLPn/Uiq9Ltq2bJluW1jY2N1ySWXaOfOnRo6dGilnx0dHa1BgwZp0KBBuv3223XjjTfq8OHDqlev3gW3A/BH9OwAAaBNmzYaOnSoXnnlFbfljz76qL788ks999xz2rZtm2bNmqVp06bpscceq7HPbtGihW688Ubdd999WrFihVavXq17771X4eHhrm169eqlFi1a6He/+52+++47LVmyRE888YTbfoYOHaoGDRqof//+WrJkiXbt2qXMzEw99NBDlU7ArqqBAwcqLy9Pf/jDH3T99dfrkksuca27/PLLtWrVKn3++efatm2bnnzySa1cufKCP+Oyyy5TcXGxXnnlFe3cuVP/+te/9Prrr5fbzul0avTo0VqxYoXWrFmju+++W7/61a9c4eds48ePV0ZGhqZOnapt27Zpw4YNmjFjhiZPnixJeumllzRnzhx9//332rZtm/7zn/8oLi7uvBcvBOyEsAMEiOeee67ccFGHDh30/vvva86cOUpJSdFTTz2lZ599VmlpaTX62TNmzFBiYqK6deumgQMH6ve//70aNWrkWh8UFKR58+apoKBAV199te699149//zzbvuIiIjQ119/rUsvvVQDBw5Uq1atNHz4cJ08ebLaPT3R0dHq16+fvvvuu3I9JA888IAGDhyoQYMGqVOnTjp06JBbL09VtW/fXpMnT9bEiROVkpKid999t8JTwCMiIjR27FgNGTJEnTt3Vnh4uObMmVPpfu+991699dZbmjlzptq0aaNu3bpp5syZSk5OliRFRkZq4sSJSk1N1VVXXaXdu3frk08+cfUuAYHAYVRlgB4AAMBPEe0BAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICtEXYAAICt/X8an4dceq2IuAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the results\n",
    "fig, ax = plt.subplots()\n",
    "ax.plot(hidden_variables, errors_train_all,'r-',label='train')\n",
    "ax.plot(hidden_variables, errors_test_all,'b-',label='test')\n",
    "ax.set_ylim(0,100);\n",
    "ax.set_xlabel('No hidden variables'); ax.set_ylabel('Error')\n",
    "ax.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyN/KUpEObCKnHZ/4Onp5sHG",
   "gpuType": "T4",
   "include_colab_link": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}