pytorch-lightning_ipynb/ordinal/baseline-light_cement.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66951a8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch            : 1.10.1\n",
      "pytorch_lightning: 1.6.0.dev0\n",
      "torchmetrics     : 0.6.2\n",
      "matplotlib       : 3.3.4\n",
      "coral_pytorch    : 1.2.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -p torch,pytorch_lightning,torchmetrics,matplotlib,coral_pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "60a1d929",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext pycodestyle_magic\n",
    "%flake8_on --ignore W291,W293,E703"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dc0efd6",
   "metadata": {},
   "source": [
    "<a href=\"https://pytorch.org\"><img src=\"https://raw.githubusercontent.com/pytorch/pytorch/master/docs/source/_static/img/pytorch-logo-dark.svg\" width=\"90\"/></a> &nbsp; &nbsp;&nbsp;&nbsp;<a href=\"https://www.pytorchlightning.ai\"><img src=\"https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/docs/source/_static/images/logo.svg\" width=\"150\"/></a>\n",
    "\n",
    "# Cross entropy baseline model for ordinal regression and deep learning -- cement strength dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47f22067",
   "metadata": {},
   "source": [
    "This is a regular cross entropy classifier as a baseline for comparison with ordinal regression methods."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c528557",
   "metadata": {},
   "source": [
    "## General settings and hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2527be84",
   "metadata": {},
   "source": [
    "- Here, we specify some general hyperparameter values and general settings\n",
    "- Note that for small datatsets, it is not necessary and better not to use multiple workers as it can sometimes cause issues with too many open files in PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "37d461ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "NUM_EPOCHS = 200\n",
    "LEARNING_RATE = 0.005\n",
    "NUM_WORKERS = 0\n",
    "\n",
    "DATA_BASEPATH = \".\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddfe5704",
   "metadata": {},
   "source": [
    "## Implementing a `MultiLayerPerceptron` using PyTorch Lightning's `LightningModule`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "829a6664",
   "metadata": {},
   "source": [
    "- In this section, we set up the main model architecture using the `LightningModule` from PyTorch Lightning.\n",
    "- We start with defining our `MultiLayerPerceptron` model in pure PyTorch, and then we use it in the `LightningModule` to get all the extra benefits that PyTorch Lightning provides."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c03faadc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "# Regular PyTorch Module\n",
    "class MultiLayerPerceptron(torch.nn.Module):\n",
    "    def __init__(self, input_size, hidden_units, num_classes):\n",
    "        super().__init__()\n",
    "\n",
    "        # num_classes is used by the corn loss function\n",
    "        self.num_classes = num_classes\n",
    "        \n",
    "        # Initialize MLP layers\n",
    "        all_layers = []\n",
    "        for hidden_unit in hidden_units:\n",
    "            layer = torch.nn.Linear(input_size, hidden_unit)\n",
    "            all_layers.append(layer)\n",
    "            all_layers.append(torch.nn.ReLU())\n",
    "            input_size = hidden_unit\n",
    "\n",
    "        output_layer = torch.nn.Linear(hidden_units[-1], num_classes)\n",
    "        \n",
    "        all_layers.append(output_layer)\n",
    "        self.model = torch.nn.Sequential(*all_layers)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ff4cacb",
   "metadata": {},
   "source": [
    "- In our `LightningModule` we use loggers to track mean absolute errors for both the training and validation set during training; this allows us to select the best model based on validation set performance later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0c14a021",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pytorch_lightning as pl\n",
    "import torchmetrics\n",
    "\n",
    "\n",
    "# LightningModule that receives a PyTorch model as input\n",
    "class LightningMLP(pl.LightningModule):\n",
    "    def __init__(self, model, learning_rate):\n",
    "        super().__init__()\n",
    "\n",
    "        self.learning_rate = learning_rate\n",
    "        # The inherited PyTorch module\n",
    "        self.model = model\n",
    "\n",
    "        # Save settings and hyperparameters to the log directory\n",
    "        # but skip the model parameters\n",
    "        self.save_hyperparameters(ignore=['model'])\n",
    "\n",
    "        # Set up attributes for computing the MAE\n",
    "        self.train_mae = torchmetrics.MeanAbsoluteError()\n",
    "        self.valid_mae = torchmetrics.MeanAbsoluteError()\n",
    "        self.test_mae = torchmetrics.MeanAbsoluteError()\n",
    "        \n",
    "    # Defining the forward method is only necessary \n",
    "    # if you want to use a Trainer's .predict() method (optional)\n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "        \n",
    "    # A common forward step to compute the loss and labels\n",
    "    # this is used for training, validation, and testing below\n",
    "    def _shared_step(self, batch):\n",
    "        features, true_labels = batch\n",
    "        logits = self(features)\n",
    "        loss = torch.nn.functional.cross_entropy(logits, true_labels)\n",
    "        predicted_labels = torch.argmax(logits, dim=1)\n",
    "\n",
    "        return loss, true_labels, predicted_labels\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        loss, true_labels, predicted_labels = self._shared_step(batch)\n",
    "        self.log(\"train_loss\", loss)\n",
    "        self.train_mae(predicted_labels, true_labels)\n",
    "        self.log(\"train_mae\", self.train_mae, on_epoch=True, on_step=False)\n",
    "        return loss  # this is passed to the optimzer for training\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        loss, true_labels, predicted_labels = self._shared_step(batch)\n",
    "        self.log(\"valid_loss\", loss)\n",
    "        self.valid_mae(predicted_labels, true_labels)\n",
    "        self.log(\"valid_mae\", self.valid_mae,\n",
    "                 on_epoch=True, on_step=False, prog_bar=True)\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        loss, true_labels, predicted_labels = self._shared_step(batch)\n",
    "        self.test_mae(predicted_labels, true_labels)\n",
    "        self.log(\"test_mae\", self.test_mae, on_epoch=True, on_step=False)\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
    "        return optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ec9d753",
   "metadata": {},
   "source": [
    "## Setting up the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5621120c",
   "metadata": {},
   "source": [
    "- In this section, we are going to set up our dataset.\n",
    "- We start by downloading and taking a look at the Cement dataset:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5311a22d",
   "metadata": {},
   "source": [
    "### Inspecting the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "27c9ba77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>response</th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>V5</th>\n",
       "      <th>V6</th>\n",
       "      <th>V7</th>\n",
       "      <th>V8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1040.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>1055.0</td>\n",
       "      <td>676.0</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>332.5</td>\n",
       "      <td>142.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>932.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>198.6</td>\n",
       "      <td>132.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978.4</td>\n",
       "      <td>825.5</td>\n",
       "      <td>360</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   response     V1     V2   V3     V4   V5      V6     V7   V8\n",
       "0         4  540.0    0.0  0.0  162.0  2.5  1040.0  676.0   28\n",
       "1         4  540.0    0.0  0.0  162.0  2.5  1055.0  676.0   28\n",
       "2         2  332.5  142.5  0.0  228.0  0.0   932.0  594.0  270\n",
       "3         2  332.5  142.5  0.0  228.0  0.0   932.0  594.0  365\n",
       "4         2  198.6  132.4  0.0  192.0  0.0   978.4  825.5  360"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "data_df = pd.read_csv(\"https://raw.githubusercontent.com/gagolews/\"\n",
    "                      \"ordinal_regression_data/master/cement_strength.csv\")\n",
    "data_df[\"response\"] = data_df[\"response\"]-1  # labels should start at 0\n",
    "\n",
    "data_labels = data_df[\"response\"]\n",
    "data_features = data_df.loc[:, [\n",
    "    \"V1\", \"V2\", \"V3\", \"V4\", \"V5\", \"V6\", \"V7\", \"V8\"]]\n",
    "\n",
    "data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f9611a79",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of features: 8\n",
      "Number of examples: 998\n",
      "Labels: [0 1 2 3 4]\n",
      "Label distribution: [196 310 244 152  96]\n"
     ]
    }
   ],
   "source": [
    "print('Number of features:', data_features.shape[1])\n",
    "print('Number of examples:', data_features.shape[0])\n",
    "print('Labels:', np.unique(data_labels.values))\n",
    "print('Label distribution:', np.bincount(data_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62085741",
   "metadata": {},
   "source": [
    "- Above, we can see that the dataset consists of 8 features, and there are 998 examples in total.\n",
    "- The labels are in range from 1 (weakest) to 5 (strongest), and we normalize them to start at zero (hence, the normalized labels are in the range 0 to 4). \n",
    "- Notice also that the dataset is quite imbalanced."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "293b9496",
   "metadata": {},
   "source": [
    "### Performance baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "603b070d",
   "metadata": {},
   "source": [
    "- Especially for imbalanced datasets, it's quite useful to compute a performance baseline.\n",
    "- In classification contexts, a useful baseline is to compute the accuracy for a scenario where the model always predicts the majority class -- you want your model to be better than that!\n",
    "- Note that if you are intersted in a single number that minimized the dataset mean squared error (MSE), that's the mean; similary, the median is a number that minimzes the mean absolute error (MAE).\n",
    "- So, if we use the mean absolute error, $\\mathrm{MAE}=\\frac{1}{N} \\sum_{i=1}^{N}\\left|y_{i}-\\hat{y}_{i}\\right|$, to evaluate the model, it is useful to compute the MAE pretending the predicted label is always the median:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a5625136",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline MAE: 1.03\n"
     ]
    }
   ],
   "source": [
    "avg_prediction = np.median(data_labels.values)  # median minimizes MAE\n",
    "baseline_mae = np.mean(np.abs(data_labels.values - avg_prediction))\n",
    "print(f'Baseline MAE: {baseline_mae:.2f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "260494f7",
   "metadata": {},
   "source": [
    "- In other words, a model that would always predict the dataset median would achieve a MAE of 1.03. A model that has an MAE of > 1 is certainly a bad model."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8155016",
   "metadata": {},
   "source": [
    "### Creating a `Dataset` class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "039d15f6",
   "metadata": {},
   "source": [
    "- Next, let us set up a data loading mechanism for our model.\n",
    "- Note that the Cement dataset is a relatively small dataset that fits into memory quite comfortably so this may seem like overkill. However, the following steps are useful as a template since you can use those for arbitrarily-sized datatsets.\n",
    "- First, we define a PyTorch `Dataset` class that returns the features (inputs) and labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6cb989de",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "\n",
    "class MyDataset(Dataset):\n",
    "\n",
    "    def __init__(self, feature_array, label_array, dtype=np.float32):\n",
    "        self.features = feature_array.astype(dtype)\n",
    "        self.labels = label_array\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        inputs = self.features[index]\n",
    "        label = self.labels[index]\n",
    "        return inputs, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.features.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "754c85d2",
   "metadata": {},
   "source": [
    "### Setting up a `DataModule`"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d6eb8dc",
   "metadata": {},
   "source": [
    "- There are three main ways we can prepare the dataset for Lightning. We can\n",
    "  1. make the dataset part of the model;\n",
    "  2. set up the data loaders as usual and feed them to the fit method of a Lightning Trainer -- the Trainer is introduced in the next subsection;\n",
    "  3. create a LightningDataModule.\n",
    "- Here, we are going to use approach 3, which is the most organized approach. The `LightningDataModule` consists of several self-explanatory methods as we can see below:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "91332326",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "class DataModule(pl.LightningDataModule):\n",
    "    def __init__(self, data_path='./'):\n",
    "        super().__init__()\n",
    "        self.data_path = data_path\n",
    "        \n",
    "    def prepare_data(self):\n",
    "        data_df = pd.read_csv(\n",
    "            'https://raw.githubusercontent.com/gagolews/'\n",
    "            'ordinal_regression_data/master/cement_strength.csv')\n",
    "        data_df.to_csv(\n",
    "            os.path.join(self.data_path, 'cement_strength.csv'), index=None)\n",
    "        return\n",
    "\n",
    "    def setup(self, stage=None):\n",
    "        data_df = pd.read_csv(\n",
    "            os.path.join(self.data_path, 'cement_strength.csv'))\n",
    "        data_df[\"response\"] = data_df[\"response\"]-1  # labels should start at 0\n",
    "        self.data_labels = data_df[\"response\"]\n",
    "        self.data_features = data_df.loc[:, [\n",
    "            \"V1\", \"V2\", \"V3\", \"V4\", \"V5\", \"V6\", \"V7\", \"V8\"]]\n",
    "        \n",
    "        # Split into\n",
    "        # 70% train, 10% validation, 20% testing\n",
    "        \n",
    "        X_temp, X_test, y_temp, y_test = train_test_split(\n",
    "            self.data_features.values,\n",
    "            self.data_labels.values,\n",
    "            test_size=0.2,\n",
    "            random_state=1,\n",
    "            stratify=self.data_labels.values)\n",
    "\n",
    "        X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "            X_temp,\n",
    "            y_temp,\n",
    "            test_size=0.1,\n",
    "            random_state=1,\n",
    "            stratify=y_temp)\n",
    "        \n",
    "        # Standardize features\n",
    "        sc = StandardScaler()\n",
    "        X_train_std = sc.fit_transform(X_train)\n",
    "        X_valid_std = sc.transform(X_valid)\n",
    "        X_test_std = sc.transform(X_test)\n",
    "\n",
    "        self.train = MyDataset(X_train_std, y_train)\n",
    "        self.valid = MyDataset(X_valid_std, y_valid)\n",
    "        self.test = MyDataset(X_test_std, y_test)\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        return DataLoader(self.train, batch_size=BATCH_SIZE,\n",
    "                          num_workers=NUM_WORKERS,\n",
    "                          drop_last=True)\n",
    "\n",
    "    def val_dataloader(self):\n",
    "        return DataLoader(self.valid, batch_size=BATCH_SIZE,\n",
    "                          num_workers=NUM_WORKERS)\n",
    "\n",
    "    def test_dataloader(self):\n",
    "        return DataLoader(self.test, batch_size=BATCH_SIZE,\n",
    "                          num_workers=NUM_WORKERS)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce22b8f5",
   "metadata": {},
   "source": [
    "- Note that the `prepare_data` method is usually used for steps that only need to be executed once, for example, downloading the dataset; the `setup` method defines the the dataset loading -- if you run your code in a distributed setting, this will be called on each node / GPU. \n",
    "- Next, lets initialize the `DataModule`; we use a random seed for reproducibility (so that the data set is shuffled the same way when we re-execute this code):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4c1280d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(1) \n",
    "data_module = DataModule(data_path=DATA_BASEPATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f060cf94",
   "metadata": {},
   "source": [
    "## Training the model using the PyTorch Lightning Trainer class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8687bf17",
   "metadata": {},
   "source": [
    "- Next, we initialize our multilayer perceptron model (here, a 2-layer MLP with 24 units in the first hidden layer, and 16 units in the second hidden layer).\n",
    "- We wrap the model in our `LightningMLP` so that we can use PyTorch Lightning's powerful `Trainer` API.\n",
    "- Also, we define a callback so that we can obtain the model with the best validation set performance after training.\n",
    "- Note PyTorch Lightning offers [many advanced logging services](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) like Weights & Biases. However, here, we will keep things simple and use the `CSVLogger`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fc71caab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
    "from pytorch_lightning.loggers import CSVLogger\n",
    "\n",
    "\n",
    "pytorch_model = MultiLayerPerceptron(\n",
    "    input_size=data_features.shape[1],\n",
    "    hidden_units=(40, 20),\n",
    "    num_classes=np.bincount(data_labels).shape[0])\n",
    "\n",
    "lightning_model = LightningMLP(\n",
    "    model=pytorch_model,\n",
    "    learning_rate=LEARNING_RATE)\n",
    "\n",
    "\n",
    "callbacks = [ModelCheckpoint(\n",
    "    save_top_k=1, mode=\"min\", monitor=\"valid_mae\")]  # save top 1 model \n",
    "logger = CSVLogger(save_dir=\"logs/\", name=\"mlp-corn-cement\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd8cdbfe",
   "metadata": {},
   "source": [
    "- Now it's time to train our model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e531b3dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:90: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=50)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.\n",
      "  rank_zero_deprecation(\n",
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name      | Type                 | Params\n",
      "---------------------------------------------------\n",
      "0 | model     | MultiLayerPerceptron | 1.3 K \n",
      "1 | train_mae | MeanAbsoluteError    | 0     \n",
      "2 | valid_mae | MeanAbsoluteError    | 0     \n",
      "3 | test_mae  | MeanAbsoluteError    | 0     \n",
      "---------------------------------------------------\n",
      "1.3 K     Trainable params\n",
      "0         Non-trainable params\n",
      "1.3 K     Total params\n",
      "0.005     Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation sanity check: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:258: PossibleUserWarning: The dataloader, val_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 4 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:258: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 4 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n",
      "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1850: PossibleUserWarning: The number of training samples (5) is smaller than the logging interval Trainer(log_every_n_steps=10). Set a lower value for log_every_n_steps if you want to see logs for the training epoch.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "386ac3987ac84863a7845e29626e3f7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training took 0.29 min in total.\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "\n",
    "trainer = pl.Trainer(\n",
    "    max_epochs=NUM_EPOCHS,\n",
    "    callbacks=callbacks,\n",
    "    progress_bar_refresh_rate=50,  # recommended for notebooks\n",
    "    accelerator=\"auto\",  # Uses GPUs or TPUs if available\n",
    "    devices=\"auto\",  # Uses all available GPUs/TPUs if applicable\n",
    "    logger=logger,\n",
    "    deterministic=True,\n",
    "    log_every_n_steps=10)\n",
    "\n",
    "start_time = time.time()\n",
    "trainer.fit(model=lightning_model, datamodule=data_module)\n",
    "\n",
    "runtime = (time.time() - start_time)/60\n",
    "print(f\"Training took {runtime:.2f} min in total.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f66903b8",
   "metadata": {},
   "source": [
    "## Evaluating the model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb4bd0b8",
   "metadata": {},
   "source": [
    "- After training, let's plot our training MAE and validation MAE using pandas, which, in turn, uses matplotlib for plotting (you may want to consider a [more advanced logger](https://pytorch-lightning.readthedocs.io/en/latest/extensions/logging.html) that does that for you):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1abca75f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='Epoch', ylabel='MAE'>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEHCAYAAACjh0HiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqvUlEQVR4nO3de3xU5b3v8c+PEEgghFsg3ISggshdwkWLF6itolaxXrHWS7duaqu2bo89YrtPdbf7nLbbXtzWC9t2K9VaoxWp1mJ1a0mxVVSogIAiICgICAS5DBCuv/PHM2gIM0kmsDKD6/t+veaVmTVrrflmEdZv1rOe9Sxzd0REJL6aZTuAiIhklwqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzDWPasVm9iDwJWCduw9MM88Y4C4gH9jg7qfVt96SkhIvKytrVKZt27bRunXrRi0btVzNplyZydVckLvZlCszjc01Z86cDe7eKeWb7h7JAzgVGAYsSPN+O2AR0DP5unND1lteXu6NNWPGjEYvG7VczaZcmcnVXO65m025MtPYXMBsT7NfjaxpyN1nAhvrmOUrwFPu/kFy/nVRZRERkfSyeY6gL9DezCrNbI6ZXZnFLCIisWUe4RATZlYGPOspzhGY2T3AcOB0oBB4FTjH3d9NMe9EYCJAaWlpeUVFRaPyJBIJioqKGrVs1HI1m3JlJldzQe5mU67MNDbX2LFj57j78FTvRXayuAFWEU4QbwO2mdlMYAhwUCFw9weABwCGDx/uY8aMadQHVlZW0thlo5ar2ZQrM7maC3I3W2VlJaNHj2bVqlVUV1dnO84n2rZtS0FBQbZjHKS+XAUFBfTo0YP8/PwGrzObheBp4B4zaw60AEYBv8hiHhHJklWrVtGmTRvKysows2zHAWDr1q20adMm2zEOUlcud6eqqopVq1bRu3fvBq8zyu6jjwFjgBIzWwXcTugmirtPdve3zezPwHxgH/Brd18QVR4RyV3V1dU5VQSOVGZGx44dWb9+fUbLRVYI3P2yBsxzJ3BnVBlE5MihInB4NGY7xufK4o8WcfSy30D1lmwnERHJKfEpBJvep+fKp2D9O9lOIiKSU+JTCDr1Cz/XvZ3dHCKSczZt2sR9992X8XJnn302mzZtyni5q6++mieffDLj5aISn0LQrhd7m7XUEYGIHCRdIdi7d2+dy02fPp127dpFlKrpZLP7aNNq1oztrXrQRkcEIjnt3/64kEWrD++5vP7dirn93AFp3580aRLLli1j6NCh5OfnU1RURElJCQsXLmTRokWcf/75rFy5kurqar797W8zceJEAMrKypg9ezaJRIKzzjqLk08+mVdeeYXu3bvz9NNPU1hYWG+2l156iVtuuYU9e/YwYsQI7r//flq2bMmkSZN45plnaN68OWeccQY//elP+f3vf8/tt99Ofn4+bdu2ZebMmYdl+8SnEADbWvdUIRCRg/z4xz9mwYIFzJ07l8rKSs455xxmzZrFoEGDAHjwwQfp0KEDO3bsYMSIEVx44YV07NjxgHUsWbKExx57jF/96ldccsklTJ06la9+9at1fm51dTVXX301L730En379uXKK6/k/vvv58orr2TatGm88847mNknzU8/+MEPmDZtGscdd1yjmqTSiV0h4KMZsONjKGyf7TgikkJd39ybysiRI6k53P3dd9/NtGnTAFi5ciVLliw5qBD07t2boUOHAlBeXs6KFSvq/ZzFixfTu3dv+vbtC8BVV13Fvffeyw033EBBQQHXXnst55xzDl/60pcAGD16NN/4xje47LLLuOCCCw79F02KzzkCkoUAYJ3OE4hIejXH+6+srOTFF1/k1VdfZd68eZxwwgkph8Jo2bLlJ8/z8vLYs2dPvZ+Tbqy35s2b8/rrr3PhhRfyhz/8gXHjxgEwefJk/vVf/5WVK1cydOhQqqqqMv3VUopVIdje6qjwZL2ah0TkU23atGHr1q0p39u8eTPt27enVatWvPPOO8yaNeuwfW6/fv1YsWIFS5cuBeCRRx7htNNOI5FIsHnzZs4++2zuuusu5s6dC8CyZcsYMWIEP/jBDygpKWHlypWHJUesmoaqCzpBfmsdEYjIATp27Mjo0aMZOHAghYWFlJaWfvLeuHHjmDx5MoMHD+a4447jxBNPPGyfW1BQwEMPPcTFF1/8ycni6667jo0bNzJ+/Hiqq6txd37xizAM23e+8x0WL16MmXH66aczZMiQw5IjVoUAawad++mIQEQO8rvf/e6A1/uPEFq2bMlzzz2Xcpn95wFKSkpYsODTodJuueWWOj9rypQpnzw//fTTefPNNw94v2vXrrz++usHLffUU09FMhherJqGAOh0vI4IRERqiF8h6NwPtq2D7XXdRVNE5NBdf/31DB069IDHQw89lO1YB4lX0xCEIwIIQ02Ujc5uFhH5TLv33nuzHaFB4nlEADpPICKSFL9CUNwdWhbrPIGISFL8CoEZdDpOg8+JiCTFrxBAGJJ63aJspxCRI1RRUREAq1ev5qKLLko5z5gxY5g9e3badZSVlbFhw4ZI8mUqnoWg8/GwvQoSmd3XU0Skpm7duuXUfQUaK7JCYGYPmtk6M6vzhvRmNsLM9ppZ6rIahf03qVHzkIgAt9566wH3I7jjjjv40Y9+xOmnn86wYcMYNGgQTz/99EHLrVixgoEDBwKwY8cOJkyYwODBg7n00kvZsWNHgz//5z//OQMHDmTgwIHcddddAGzbto1zzjmHIUOGMHDgQB5//HEAbr/9dvr378/gwYPrvXCtoaLsPjoFuAd4ON0MZpYH/AR4PsIcBysJI/1RtQR6n9KkHy0i9XhuEqx96/Cus8sgOOvHad+eMGECN910E9/85jcBeOKJJ3jyySeZNGkSxcXFbNiwgRNPPJHzzjsv7c3h77//flq1asX8+fOZP38+w4YNa1C0OXPm8NBDD/Haa6/h7owaNYrTTjuN9957j27duvGnP/0JCGMebdy4kT/+8Y+8++67BwxPfagiOyJw95lAfVdt3QhMBdZFlSOl4u7QvBA2LG3SjxWR3HTCCSewbt06Vq9ezbx582jfvj1dunThu9/9LoMHD+YLX/gCH374IR999FHadcycOfOT+w8MHjyYwYMHN+iz//a3v/HlL3+Z1q1bU1RUxAUXXMDLL7/MoEGDePHFF7n11lt5+eWXadu2LcXFxZ8MT/3UU0/RqlWrw/L7Z+2CMjPrDnwZ+Dwwokk/vFkz6HhsOCIQkdxSxzf3KF100UU8+eSTrF27lgkTJvDEE0+wfv165syZQ35+PmVlZSmHn64p3dFCXdINRd23b1/mzJnD9OnTue222zjjjDP4/ve/z4wZM3j99depqKjgnnvu4S9/+UvGn1lbNq8svgu41d331rfxzGwiMBGgtLSUysrKRn1gIpH4ZNn++4pps3I+rzVyXYdbzWy5RLkyk6u5IHezJRIJ2rZtm3YY6KZy7rnncuONN1JVVcVzzz3H1KlTadeuHdXV1bzwwgu8//77JBKJT3Ju3bqVRCLBvn372Lp1K6NGjWLKlCkMHz6cRYsWMX/+fLZt25b293J3EokE5eXlfOMb3+D666/H3Zk6dSoPPPAA7777Lu3bt2f8+PHk5eXx6KOPsmbNGhKJBKeccgoDBgxg6NChKddfXV2d2b+1u0f2AMqABWneWw6sSD4ShOah8+tbZ3l5uTfWjBkzPn3x0r+739HOfXd1o9d3OB2QLYcoV2ZyNZd77mabMWOGL1q0KNsx3N194MCBPmbMGHd3X758uZ944oleXl7u11xzjffr18+XL1/u7u6tW7f+ZJ4BAwa4u/v27dv90ksv9UGDBvkVV1zhJ510kr/xxhtpP6tXr16+fv16d3f/2c9+5gMGDPABAwb4L37xC3d3//Of/+yDBg3yIUOG+PDhw/2NN97w1atX+7Bhw3zQoEE+cOBAnzJlSsp1p9qewGxPs1/N2hGBu/fe/9zMpgDPuvsfmixASR/wfbBx+afDTohIrL311qcnqTt27Mirr76acr5EIgGEawH2Dz9dWFhIRUVFgz+r5q0sb775Zm6++eYD3j/zzDM588wzD1qusrLysA9DHVkhMLPHgDFAiZmtAm4H8gHcfXJUn9tgHY8NP6uWqBCISKxFVgjc/bIM5r06qhxp7S8EG3TCWESiM2rUKHbu3HnAtEceeYRBgwZlKdHB4jcM9X4FxVDUBarUhVREovPaa69lO0K94jnExH4lfXREIJIjPE03SslMY7ZjvAuBriUQyQkFBQVUVVWpGBwid6eqqoqCgoKMlotv0xCEI4IdH8O2KmjdMdtpRGKrR48erFq1ivXrc2cgyOrq6ox3qE2hvlwFBQX06NEjo3XGuxB07BN+Vi1RIRDJovz8fHr37l3/jE2osrKSE044IdsxDhJFrng3DZWo55CISLwLQbtekNdC5wlEJNbiXQia5UGHozUKqYjEWrwLAajnkIjEngpBSZ8w3tDePdlOIiKSFSoEHfvAvt2w6f1sJxERyQoVgpJkF9L1i7ObQ0QkS1QI9t/Ift2i7OYQEckSFYKCYmjXU4VARGJLhQCg8wD4aGG2U4iIZIUKAUDpgHB18Z6d9c8rIvIZo0IAUNoffC9seDfbSUREmpwKAYSmIVDzkIjEkgoBhKuL81qoEIhILEVWCMzsQTNbZ2YL0rx/uZnNTz5eMbMhUWWpV15z6HScCoGIxFKURwRTgHF1vL8cOM3dBwM/BB6IMEv9SgfB2rdAd0gSkZiJrBC4+0xgYx3vv+LuHydfzgIyu6XO4dZtKGxbB1vXZDWGiEhTy5VzBNcAz2U1Qdeh4efqudlMISLS5CzKm0WbWRnwrLsPrGOescB9wMnuXpVmnonARIDS0tLyioqKRuVJJBIUFRWlfK/Z3p2c8vIE3u91MSt6f6VR6z8UdWXLJuXKTK7mgtzNplyZaWyusWPHznH34SnfdPfIHkAZsKCO9wcDy4C+DV1neXm5N9aMGTPqnuGeUe6/vbjR6z8U9WbLEuXKTK7mcs/dbMqVmcbmAmZ7mv1q1pqGzKwn8BRwhbvnxpVc3YbCmrnZTiEi0qSi7D76GPAqcJyZrTKza8zsOjO7LjnL94GOwH1mNtfMZkeVpcG6DoXER7BFJ4xFJD6aR7Vid7+snvevBa6N6vMbpdvQ8HP1m1DcNatRRESaSq70GsoNXQaBNVPzkIjEigpBTS1aQ8lx6kIqIrGiQlCbThiLSMyoENSmE8YiEjMqBLXtP2GsowIRiQkVgtr2nzDWeQIRiQkVgtpatIaSvjoiEJHYUCFIpdsJ8OEcDUktIrGgQpBKzxNh23qoWpbtJCIikVMhSKXX6PDz/b9nN4eISBNQIUil47HQuhO8/0q2k4iIRE6FIBUz6PU5FQIRiQUVgnR6jYbNH8CmD7KdREQkUioE6fQ8KfzUUYGIfMapEKRTOhBalcDSF7OdREQkUioE6TRrBn2+GArBvr3ZTiMiEhkVgrr0+SLs+DhcXCYi8hmlQlCXYz4PlgdLXsh2EhGRyKgQ1KWwPRw1Ct59PttJREQiE+XN6x80s3VmtiDN+2Zmd5vZUjObb2bDospySI4bB2vnw8fvZzuJiEgkojwimAKMq+P9s4A+ycdE4P4IszRe//Hh56Kns5tDRCQikRUCd58JbKxjlvHAwx7MAtqZWdeo8jRa+7Jw17JFf8hyEBGRaJhHONSymZUBz7r7wBTvPQv82N3/lnz9EnCru89OMe9EwlEDpaWl5RUVFY3Kk0gkKCoqyni5oz6YyjHvPcysUQ9QXVjaqM+uT2OzRU25MpOruSB3sylXZhqba+zYsXPcfXjKN909sgdQBixI896fgJNrvH4JKK9vneXl5d5YM2bMaNyCVe+5317s/rf/bPRn16fR2SKmXJnJ1VzuuZtNuTLT2FzAbE+zX81mr6FVwFE1XvcAVmcpS9069FbzkIh8ZmWzEDwDXJnsPXQisNnd12QxT90GnB8uLFPvIRH5jGke1YrN7DFgDFBiZquA24F8AHefDEwHzgaWAtuBr0WV5bDofz68eEfoPTT6W9lOIyKfNRuWwvp3oE1XWDsPEuvCvVHeehLWvw3lX4MR10Ty0ZEVAne/rJ73Hbg+qs8/7Dr0hq5DQvOQCoGINNSWNVDcNdwDfd0i6HR8mP7eDLBmYYe/6GlYPB1I0XmnsD107g8v3h6GxS8677BHjKwQfCYN+HI4Kvh4RehWKiJS246PYcmLMPBCePMR+OO34LLHYcuH8KebYfCloSi89cSny7TuDKd+B/qeCVvXQqfjoLhbOEIo6Qst24Sm6VYlMG/5YY+sQpCJAReEQrBwGpz8L9lOIyK56Nl/CfuIxFqYNTlMe+47sHMrFHeH+Y+HaWO+C2WjoUVr6DIkjHhcW/fyFM9VCLKrfS/oMQIWTFUhEIm7HZvg4+VhJ770RVgxE9r1CkWgoB288K9hvtE3wd/vCgNYXv0n2Lgc9u0JHVByhApBpgZeCH+eBOvfhU59s51GRJrappWw4En4211QvQkKO8COGoMotO8NX50K/3UqdD4evnAHNC8Ibf2lA8Ijx6gQZKr/+fDn22B+BZz+/WynEZEo7NxK4fbVoS1/727Y+B6smQf/+A28//cwzzGfD2ORLftLaLYZ+lVYNA2OOhE6HgPXvRyODMxg7G1Z/XXqo0KQqeKu0O8cmP0gnPK/QvueiBzZdm0P9x3xvaH75pP/xKita2D+JNiZCNMhNP2cfjscfx6UHBumlV/96XpGXPvp8w5HN1n8Q6VC0Bif+xa88yy8+VsY9fVspxGRdLZtgK1roE230J6/8jXY8G7os59YG/rp79oWeuTs3v7pcm2PYsmx/0yftrtDj56SvlDSB7oMgmZ52ft9ItKgQmBmrYEd7r7PzPoC/YDn3H13pOlyVc9R4fDvlXvCRR7NW2Q7kchny5p50CwfSvsf/N6u7aEP/lGjwhH5W09Cs+ahj/2bj4RmmS6DQt/8TR8cvHxhe+jYJ7TfVy2D5i3hhCvg+C+F9Xz4Dxh8CR/OXkSfMWMi/1VzQUOPCGYCp5hZe8LgcLOBS4HLowqW8069BR69CN74FZx05FwXJ5Jr2m5aCFMfhYLicNHmvj0w/Tvg+8IOet+e0Ad/d3WYZ/WbsG09tGgDBW1hy6pPV3b0mLBzXz4ztOGP/Hroj79ldWjW7XUytKlnBOFen0s+WRTVr5xzGloIzN23m9k1wC/d/T/M7M0og+W8Y78QHpU/gUGXQFGnbCcSaRr79oYdc9vkmJGbV4Z29cS60LW6uBt06gcti8K3900fwNq3wk575xaY+2j45t2xD+QXMHTBNChsG3b8b/w6rLP3qaGNfc4UaN0pXMCZXwiJj5InZi8PPXe2bYDz7wv98/Oah/n27Q3NPC3bZGkDHXkaXAjM7CTCEcD+wS7ifX7BDM78Edx/Evzlh3De3dlOJHJo9u2D5ZWha/Sgi6B1yYHvrZwFsx+CJc9D9eawI2+WF65+zW8Ne3fBvga0FpedEorCxvdg2wbWdhlL16/9BloUhYJRtQT6nRuaXMf9BPILUq+nf5qhFprlqQhkqKE785uA24Bp7r7QzI4GZkSW6kjRqW849Jx1XxgMquuQbCcSabi9u+GDV2H7xrAzf/NR2JxsU3/x9vBNfOcWaNsTtq6G7VXQsi30PzeMl7N4evgWf8b/DcOuNG8Jw/8JdiXCRVO7toVv8W26hjb7XYlQLNr1PCDG4spKuu7fcXcdHB77pSsCclg1qBC4+1+BvwKYWTNgg7tr5DWA0/53uKbguUnwtenhSEEkm3Ymwk67uFvoIbOnGjocTfHmt+G552D+E5DfKjSf1LwQ6ugx8MU7QrPO7IfCjrxFq9C002UgHD02nFDd32X6czekz5DqS1HL3LvblwQN7TX0O+A6YC8wB2hrZj939zujDHdEKGwXLiz747dD++bIf852IokT99BMU5BsY1/0NEy/JRSCWoZB6Ilz/Lnh2zuE5+17Q6uOB55EPeenTRJfckNDm4b6u/sWM7uccB+BWwkFQYUAYNhV8PYfw9givU8NIweKRGX3jtDF8f1XYOFTYWjjotLQq2bn5nA3vbHfha0fhX7yLVpB1VIWrKlm4Dlfh1Ydsv0bSI5paCHIN7N84HzgHnffbWbR3fX+SGMG4+8LJ44f/ypc80LoqyxyqHYmwg5/0/uhHX7la7B67qcnZXuMgLHfCyde81qELyL9zw89aGrZUFmpIiApNbQQ/BewApgHzDSzXsCWqEIdkdqUwiWPwMPj4Ykr4fInPz38FqnNHda9HcatadMFqpaGK9V7jITuw8JQxZs/hG3rQj96CDv6bsPCdSs9T4KjRmrHLodFQ08W3w3U7B/5vpmNjSbSEaxsNIy/B6Z9PRwZXPKIej3E0dKXQht9QVuY+7vQht++DPbsDM93bIT1iw88UQuhCCx8Cub9DkoHwrGfh6IuyebGfqEdP8U3fZFD1dCTxW0J9xw+NTnpr8APgM31LDcO+E8gD/i1u/84xXp/C/RMZvmpuz+UyS+Qc4ZMCG24z94EFV+BCY+GLnRyZHKHDUvCBYOWF5pp8gtCP/WPVwAWduhv/xFadaTf+iqorPx0+cIOobvk2vmhr31BcRiR8vhzwzf/o8fAtqqwztIB4aKsrWtDd0v1QJMm0tCvFw8CC4BLkq+vAB4CLki3gJnlAfcCXwRWAW+Y2TPuXvO67euBRe5+rpl1Ahab2aPuvivD3yO3DP9auHLymRvhsQkw4XcapfRIsa0qDEC2Zl4YlGzF38ONwyH8m+5vpqmtYx9Y+xal2zeGUWkHXBB26GUn139UWPO2p0Wdw0OkCTW0EBzj7hfWeP1vZja3nmVGAkvd/T0AM6sAxnPgAB4OtDEzA4qAjUCa/2lHmGFXhB3H09+Eh8+Hrzyu9txcsbs6DFq2a1vobbNuEax6A1bNDiNU7lfYIYw6efZPw20GdyVCX3ocqrdAh97h5uOWF0am3LeXV16azujTk1e8dhmYlV9PJFMNLQQ7zOxkd/8bgJmNBnbUs0x3YGWN16uAUbXmuQd4BlgNtAEudfd9DcyU+4ZeFo4Epl4D//1FuOThnLw70WeKexgHZ+PyMNDYlg/DY/OHYUyc6s2h/b7mkMMQrn7tXg7lV0H34dDthMwvgMprzu4WxYfvdxFpIuZefy9QMxsCPAy0TU76GLjK3efXsczFwJnufm3y9RXASHe/scY8FwGjgZuBY4D/AYa4+5Za65oITAQoLS0tr6ioaPAvWFMikaCoqOmvbmy7aSH9F91J8z3bWNLn66zt+oWcyVafXM21betmjtq1jGb7dlJQvZ4OG+eQv3srLXZtouWuAy+m2t28iJ0tO1Jd0Jnd+W3Ym9eaqo7l7GrRnha7NrK9VU92FpSk+aTM5Or2gtzNplyZaWyusWPHznH34anea2ivoXnAEDMrTr7eYmY3AWkLAeEI4Kgar3sQvvnX9DXgxx6q0VIzW06418HrtT7/AeABgOHDh/uYRo4RXllZSWOXPTRj4AsXw9Rr6Lf4l/RrsRbO/H8HtAVnL1vdsp7LPTThVG8Og569+zzs28OOpS9TWL3u0/k69YOSY8KJ2B4jwjhQxd2huBv5LVqTT2h73K9HRHGzvr3qkKvZlCszUeTKqC9arW/qNwN31TH7G0AfM+sNfAhMAL5Sa54PgNOBl82sFDgOeC+TTEeMos5wxR9g5p0w86dhhzbq62HQurgPYb1mHuzZFfrTL3gyDIPQbShU/jgMilbzBG3rTlDQluqCzhSO/3kYHqFlG2jfK2vxRY50h9Ipuc6+be6+x8xuAJ4ndB99MDly6XXJ9ycDPwSmmNlbyfXd6u4bDiFTbmuWB2MmwcCL4KU7QkF45Zcw9HLa7jka9pwUj4vQ9u2FhdPC1bCrZodhjVNp1RFO/GYYDrmgbRguoedJ0CyPeZWVjDl+TJPGFvmsOpRCUO/JBXefThibqOa0yTWerwbOOIQMR6aSY+HS34b+6a/cDW8+wgl7d8GCH4arRctODRendRl85I/YuGt76JWz4d3QRz6xLvTYWZfsPFbYHj7/f0Kvm49XQL8vhUKxclZ4rp5WIpGrsxCY2VZS7/AN0FVSh6qkD5z3S/jiD3nrT//FoKJNsPxlmPHvyRksdF/sOjj87HhsaALJaxmGGM6lneTePbBxWdjBr3sbPloYfn68PIyKuV9+K+hwDFz8m7CjT3elbKe+TZNbROouBO6u2/w0hcJ2VJWMgv0ngLZvhJWvw5q5YYCxD2bBW78/eLlWJWHYguYtQmHoMjicNC3uljxR2jV0izyczU3uoRtmzZ39urdhw+Jw0xEIfes7HBO6yg66KOTqfHzos3+kH+GIfAZp4JJc1KoDHDcuPPbbtT184960MuxwP14R2tjzC0Mx2LwynICe++jB62vdqUZx6BbuNdvpuPBo1yucu6ht9w7Yu5vC7WvgnemhOWf1m2Gnvyvx6XzFPcJO/tjPQ+f+4XlJXw2rIXIEUSE4UrRoFcaf6TIo/Tzu4daCW9YkL6RaXeOiqtXhTlMfvAo7Pv50GWsGLYvDydiC4vB80wehsJC8AvB1QpNOt2Ew9CvJHX5/6NwvLCciRzQVgs8Ss+QOvW3YSaezY1M4Ub3+nTDOffWW0E9//6PHiHCzneYteeeD9fT73Fnhqts49GgSiSEVgjgqbAdHjQiPeqzdXUm/Xp+LPpOIZE2zbAcQEZHsUiEQEYk5FQIRkZhTIRARiTkVAhGRmFMhEBGJORUCEZGYUyEQEYk5FQIRkZhTIRARiTkVAhGRmFMhEBGJORUCEZGYi7QQmNk4M1tsZkvNbFKaecaY2VwzW2hmf40yj4iIHCyyYajNLA+4F/gisAp4w8yecfdFNeZpB9wHjHP3D8ysc1R5REQktSiPCEYCS939PXffBVQA42vN8xXgKXf/AMDd10WYR0REUoiyEHQHVtZ4vSo5raa+QHszqzSzOWZ2ZYR5REQkBXP3aFZsdjFwprtfm3x9BTDS3W+sMc89wHDgdKAQeBU4x93frbWuicBEgNLS0vKKiopGZUokEhQVFTVq2ajlajblykyu5oLczaZcmWlsrrFjx85x9+Ep33T3SB7AScDzNV7fBtxWa55JwB01Xv83cHFd6y0vL/fGmjFjRqOXjVquZlOuzORqLvfczaZcmWlsLmC2p9mvRtk09AbQx8x6m1kLYALwTK15ngZOMbPmZtYKGAW8HWEmERGpJbJeQ+6+x8xuAJ4H8oAH3X2hmV2XfH+yu79tZn8G5gP7gF+7+4KoMomIyMEiKwQA7j4dmF5r2uRar+8E7owyh4iIpKcri0VEYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYi7SQmBm48xssZktNbNJdcw3wsz2mtlFUeYREZGDRVYIzCwPuBc4C+gPXGZm/dPM9xPg+aiyiIhIelEeEYwElrr7e+6+C6gAxqeY70ZgKrAuwiwiIpJGlIWgO7CyxutVyWmfMLPuwJeByRHmEBGROpi7R7Nis4uBM9392uTrK4CR7n5jjXl+D/zM3WeZ2RTgWXd/MsW6JgITAUpLS8srKioalSmRSFBUVNSoZaOWq9mUKzO5mgtyN5tyZaaxucaOHTvH3YenfNPdI3kAJwHP13h9G3BbrXmWAyuSjwSheej8utZbXl7ujTVjxoxGLxu1XM2mXJnJ1VzuuZtNuTLT2FzAbE+zX22ecVlpuDeAPmbWG/gQmAB8pVYR6r3/eY0jgj9EmElERGqJrBC4+x4zu4HQGygPeNDdF5rZdcn3dV5ARCQHRHlEgLtPB6bXmpayALj71VFmERGR1HRlsYhIzKkQiIjEnAqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzKkQiIjEnAqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzKkQiIjEnAqBiEjMqRCIiMScCoGISMypEIiIxJwKgYhIzEVaCMxsnJktNrOlZjYpxfuXm9n85OMVMxsSZR4RETlYZIXAzPKAe4GzgP7AZWbWv9Zsy4HT3H0w8EPggajyiIhIalEeEYwElrr7e+6+C6gAxtecwd1fcfePky9nAT0izCMiIilEWQi6AytrvF6VnJbONcBzEeYREZEUzN2jWbHZxcCZ7n5t8vUVwEh3vzHFvGOB+4CT3b0qxfsTgYkApaWl5RUVFY3KlEgkKCoqatSyUcvVbMqVmVzNBbmbTbky09hcY8eOnePuw1O+6e6RPICTgOdrvL4NuC3FfIOBZUDfhqy3vLzcG2vGjBmNXjZquZpNuTKTq7ncczebcmWmsbmA2Z5mvxpl09AbQB8z621mLYAJwDM1ZzCznsBTwBXu/m6EWUREJI3mUa3Y3feY2Q3A80Ae8KC7LzSz65LvTwa+D3QE7jMzgD2e7tBFREQiEVkhAHD36cD0WtMm13h+LXBtlBlERKRuurJYRCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGJOhUBEJOZUCEREYk6FQEQk5lQIRERiToVARCTmVAhERGIusltVRsXM1gPvN3LxEmDDYYxzOOVqNuXKTK7mgtzNplyZaWyuXu7eKdUbR1whOBRmNjtXb3yTq9mUKzO5mgtyN5tyZSaKXGoaEhGJORUCEZGYi1sheCDbAeqQq9mUKzO5mgtyN5tyZeaw54rVOQIRETlY3I4IRESkltgUAjMbZ2aLzWypmU3KYo6jzGyGmb1tZgvN7NvJ6XeY2YdmNjf5ODsL2VaY2VvJz5+dnNbBzP7HzJYkf7bPQq7jamyXuWa2xcxuysY2M7MHzWydmS2oMS3tNjKz25J/c4vN7MwmznWnmb1jZvPNbJqZtUtOLzOzHTW22+QmzpX2362ptlcd2R6vkWuFmc1NTm+SbVbH/iHavzF3/8w/gDxgGXA00AKYB/TPUpauwLDk8zbAu0B/4A7glixvpxVASa1p/wFMSj6fBPwkB/4t1wK9srHNgFOBYcCC+rZR8t91HtAS6J38G8xrwlxnAM2Tz39SI1dZzfmysL1S/rs15fZKl63W+z8Dvt+U26yO/UOkf2NxOSIYCSx19/fcfRdQAYzPRhB3X+Pu/0g+3wq8DXTPRpYGGg/8Jvn8N8D52YsCwOnAMndv7EWFh8TdZwIba01Ot43GAxXuvtPdlwNLCX+LTZLL3V9w9z3Jl7OAHlF8dqa56tBk26u+bGZmwCXAY1F9fppM6fYPkf6NxaUQdAdW1ni9ihzY+ZpZGXAC8Fpy0g3Jw/gHs9EEAzjwgpnNMbOJyWml7r4Gwh8p0DkLuWqawIH/ObO9zSD9Nsqlv7t/Ap6r8bq3mb1pZn81s1OykCfVv1suba9TgI/cfUmNaU26zWrtHyL9G4tLIbAU07LaXcrMioCpwE3uvgW4HzgGGAqsIRyWNrXR7j4MOAu43sxOzUKGtMysBXAe8PvkpFzYZnXJib87M/sesAd4NDlpDdDT3U8AbgZ+Z2bFTRgp3b9bTmyvpMs48AtHk26zFPuHtLOmmJbxNotLIVgFHFXjdQ9gdZayYGb5hH/kR939KQB3/8jd97r7PuBXRHhInI67r07+XAdMS2b4yMy6JnN3BdY1da4azgL+4e4fQW5ss6R02yjrf3dmdhXwJeByTzYqJ5sRqpLP5xDalfs2VaY6/t2yvr0AzKw5cAHw+P5pTbnNUu0fiPhvLC6F4A2gj5n1Tn6rnAA8k40gybbH/wbedvef15jetcZsXwYW1F424lytzazN/ueEE40LCNvpquRsVwFPN2WuWg74lpbtbVZDum30DDDBzFqaWW+gD/B6U4Uys3HArcB57r69xvROZpaXfH50Mtd7TZgr3b9bVrdXDV8A3nH3VfsnNNU2S7d/IOq/sajPgufKAzibcAZ+GfC9LOY4mXDoNh+Ym3ycDTwCvJWc/gzQtYlzHU3ofTAPWLh/GwEdgZeAJcmfHbK03VoBVUDbGtOafJsRCtEaYDfh29g1dW0j4HvJv7nFwFlNnGspof14/9/Z5OS8Fyb/jecB/wDObeJcaf/dmmp7pcuWnD4FuK7WvE2yzerYP0T6N6Yri0VEYi4uTUMiIpKGCoGISMypEIiIxJwKgYhIzKkQiIjEnAqBSC1mttcOHO30sI1WmxzFMlvXO4ik1DzbAURy0A53H5rtECJNRUcEIg2UHJ/+J2b2evJxbHJ6LzN7KTmI2ktm1jM5vdTCfQDmJR+fS64qz8x+lRxv/gUzK8zaLyWCCoFIKoW1moYurfHeFncfCdwD3JWcdg/wsLsPJgzsdndy+t3AX919CGHc+4XJ6X2Ae919ALCJcNWqSNboymKRWsws4e5FKaavAD7v7u8lBwZb6+4dzWwDYZiE3cnpa9y9xMzWAz3cfWeNdZQB/+PufZKvbwXy3f3fm+BXE0lJRwQimfE0z9PNk8rOGs/3onN1kmUqBCKZubTGz1eTz18hjGgLcDnwt+Tzl4BvAJhZXhOP+S/SYPomInKwQkvetDzpz+6+vwtpSzN7jfAl6rLktG8BD5rZd4D1wNeS078NPGBm1xC++X+DMNqlSE7ROQKRBkqeIxju7huynUXkcFLTkIhIzOmIQEQk5nREICIScyoEIiIxp0IgIhJzKgQiIjGnQiAiEnMqBCIiMff/AdXtfE+eYoaHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA+p0lEQVR4nO3dd3hUZdr48e8zk8mkTHqHAEnoVUooKmBiBfUVXVFRFhXXZW27uu6+q1t+6r4v++rq7lp2VRa7LhoVGxYQQSIo0oKU0AkghJpQEia9PL8/nkkhJCEJmZmQuT/XNRdnzjlzzp2TcO556lFaa4QQQvgui7cDEEII4V2SCIQQwsdJIhBCCB8niUAIIXycJAIhhPBxft4OoLWio6N1UlJSmz5bVFREcHBw+wbUDjpqXNBxY5O4Wkfiap3OGFdWVla+1jqm0Y1a63PqNWLECN1WS5YsafNn3amjxqV1x41N4modiat1OmNcwBrdxH3VbVVDSqlXlVJHlFLZZ9hvpFKqSik12V2xCCGEaJo72wheByY0t4NSygr8FfjSjXEIIYRohtsSgdZ6KXDsDLv9EvgAOOKuOIQQQjTPa43FSqmuwHXAxcBIb8UhhPCOiooKHA4HW7Zs8XYopwkLCztn4woICCAxMRGbzdbi4yrtxrmGlFJJwGda60GNbHsf+LvWeoVS6nXXfnObOM4MYAZAXFzciIyMjDbF43Q6cTgcbfqsO3XUuKDjxiZxtU5HjMvhcBATE0NERARKKW+Hc4qqqiqsVqu3wzjNmeLSWlNQUMDhw4dxOp2nbEtPT8/SWqc2+UF3vYAkILuJbbuBPa6XE1M9dO2Zjim9hjyro8YmcbVOR4xr8+bNuqCgwNthNKqwsNDbITSqJXFVV1frzZs3n7aeZnoNea1qSGudXLNcr0TwsbfiEUJ4XkcrCXQGbbmmbksESql3gDQgWimVCzwK2AC01rPcdd4zeev7PYQG2pg0tKu3QhBCiA7FbYlAa31zK/a93V1xNPTOqn10CQ+QRCCEEC4+N9eQ3WahrLLa22EIIbzsxIkTvPDCC63+3JVXXsmJEyfaPyAv8rlE4G+VRCCEaDoRVFVVNfu5L774gvDwcDdF5R3n3KRzZ8tus1JQUuHtMIQQ9fz5001sPlDYrscc0CWUR/9rYJPbH374YXJychg6dCg2mw2Hw0FCQgLr1q1j5cqVXHvttezbt4/S0lLuv/9+ZsyYAUBSUhJr1qzB6XQyceJExo4dy/Lly+natSuffPIJgYGBjZ4vLS2NYcOGkZWVRV5eHm+++SaPP/44Gzdu5KabbmLmzJkATZ534cKF/OlPf6KyspKePXvy2muvtVuXYJ8sEZRLiUAIn/fEE0/Qs2dP1q1bx1NPPcWqVav4y1/+wubNmwF49dVXycrKYs2aNTz33HMcPXr0tGPs2LGDe++9l02bNhEeHs4HH3zQ7Dn9/f1ZunQpd911F5MmTeL5558nOzub119/vfb4jZ03Pz+fmTNnMm/ePNauXUtqair/+Mc/2u1a+F6JwM9CeWXzRT8hhGc1983dU0aNGkVycm2vdp577jk++ugjAPbt28eOHTuIioo65TPJyckMHToUgBEjRrBnz55mz3HNNdcAMHjwYAYOHEhCQgIAKSkp7Nu3j6ioqEbPm5+fz+bNm7n88suxWCyUl5dz/vnnt8ePDfhoIpA2AiFEQ/Xn+V+2bBmLFi3i+++/JygoiLS0NEpLS0/7jN1ur122Wq2UlJQ0e46a/S0WyymftVgsVFZWkpmZ2eh5tdZcdtllzJ49m5CQkLP9UU/je1VDflI1JISAkJAQTp482ei2wsJCIiIiCAoKYuvWraxYscIjMRUUFDR63jFjxvDdd9+Rk5MDQHFxMdu3b2+380qJQAjhk6KiorjwwgsZNGgQgYGBxMXF1W679NJLeeONNxgyZAh9+/ZlzJgxHolpwoQJzJo167TzxsTE8Prrr3PHHXdQWVkJwMyZM+nTp0+7nNfnEoGUCIQQNd5+++1G19vtdubPn9/otpp2gOjoaLKz65679dvf/rbZc2VmZtYup6WlkZaW1ui2ps578cUX880330jVUHvw97NQXiWJQAghavhcicDuZ6WqWlNZVY2f1efyoBDCze69916+++67U9bdf//9TJ8+3UsRnZnPJQJ/P3PzL5dEIIRwg+eff97bIbSaz90J7TWJQNoJhBAC8MFEUFMikJ5DQghh+F4isEqJQAgh6vO5RGC3med9lsk0E0IIAfhgIqgpEUjVkBCiNWpm+jxw4ACTJ09udJ+0tDTWrFnjybDahc8lArtNqoaEEG3XpUsX5s6d6+0w2pXPdR+1S4lAiI5n/sNwaGP7HjN+MEx8osnNDz30ED169OCee+4B4LHHHkMpxdKlSzl69ChVVVXMnDmTSZMmnfK5PXv2cPXVV5OdnU1JSQnTp09n8+bN9O/f/4yTzjkcDu69914WLVpEREQE//d//8fvfvc79u7dyzPPPMM111zDnj17mDZtGkVFRQD861//4oILLgDg2Wef5ZNPPqGsrIzrrruOP//5z2dzhWr5XInAX7qPCiGAKVOm8O6779a+f++995g+fTofffQRy5YtY8mSJfzmN79Ba93kMV588UWCgoLYsGEDf/zjH8nKymr2nEVFRaSlpZGVlUVISAh/+tOf+Oqrr/joo4945JFHAIiNjeWrr75i7dq1vPvuu/zqV78CzINpcnJyWLVqFevWrSMrK4ulS5e2w5XwxRKBX01jsSQCITqMZr65u8uwYcM4cuQIBw4cIC8vj4iICBISEvj1r39NZmYmfn5+7N+/n8OHDxMfH9/oMZYuXVp7ox4yZAhDhgxp9pz+/v5MmDABMM8ksNvt2Gw2Bg8eXDuHUUVFBffddx/r1q3DarXWzjK6cOFCvv76a4YNGwaA0+lkx44djB8//qyvhc8lAikRCCFqTJ48mblz53Lo0CGmTJnCnDlzyMvLY+nSpURGRpKUlNTocwjqU0q1+Hw2m612//rPJKh5HgHA008/TVxcHOvXr6e6upqAgAAAtNY8+OCD3H///W35UZvltqohpdSrSqkjSqnsJrZPVUptcL2WK6XOc1cs9dWOLD7DA6qFEJ3flClTyMjIYO7cuUyePJmCggJiY2Ox2WwsWbKEH3/8sdnPjx8/njlz5gCQnZ3Nhg0bzjqmgoICEhISsFgsvPXWW1S57lVXXHEFb731Fk6nE4D9+/dz5MiRsz4fuLeN4HVgQjPbdwMXaa2HAP8LzHZjLLVqRxZXSIlACF83cOBATp48SdeuXUlISGDq1KmsWbOGiy66iDlz5tCvX79mP3/33XfjdDoZMmQITz75JKNGjTrrmO655x7eeOMNxowZw/bt22ufnHb55Zdzww03cP755zN48GAmT57c5IN1WsttVUNa66VKqaRmti+v93YFkOiuWOqrP+mcEEJs3FjXWyk6Oprvv/+ekydPnjbvf8038aSkpNrnEAQGBpKRkdHic9UcA0wvpca29e7d+5SSxeOPP167fM899/DQQw+1+HwtpZprET/rg5tE8JnWetAZ9vst0E9rfWcT22cAMwDi4uJGtObC1+d0OrEGBHP3omJu6uvPxGRbm47T3pxOZ+1glY6mo8YmcbVOR4wrLCyM5ORkrFart0M5TVVV1Tkd186dOykoKDhlXXp6epbWOrWx/b3eWKyUSgd+Boxtah+t9WxcVUepqam6/pN9WiMzM5Pzx46DRQvonpRMWlqvNh2nvWVmZtLWn8ndOmpsElfrdMS4tmzZgtVqdcsTt85WYyWC1hg9ejRlZWWnrHvrrbcYPHiwR+IKCAio7V3UEl5NBEqpIcDLwESt9VFPnFOmmBCi43BnjYQ3rVy50mvnbss19dqAMqVUd+BDYJrWersHz4u/n0UmnRPCywICAigoKOi0ycAbtNYcPXq0tstpS7mtRKCUegdIA6KVUrnAo4ANQGs9C3gEiAJecPWrrWyq/qq92a3yAHshvC0xMZH169ef0oDaUZSWlrb6ZuoJLYkrICCAxMTW9b1xZ6+hm8+w/U6g0cZhd7PbJBEI4W02mw2n00lqqke+/7VKZmZmq+rYPcVdcfncXENg2gmkjUAIIQyfTAR2m1VKBEII4eKTicCUCKSxWAghwFcTgZ+0EQghRA2fTAR2P4tMMSGEEC4+mQj8/Swy6ZwQQrj4bCKQEoEQQhg+mQjsUiIQQohaPpkI/P2sUiIQQggXn0oElqoyqK42jcXSa0gIIQBfSgQb5zJu2U1wYo9MOieEEPX4TiII745CQ952mWJCCCHq8Z1EEN3H/Ju/DbtNEoEQQtTwnUQQGE6ZfwTkbaudhlrmQRdCCF9KBEBxUKJJBDbzzM+KKkkEQgjhY4mgG+Rvx9+iAKTBWAgh8LFEUBScCGWFhFflA0gXUiGEwMcSQXFQNwCiS/cAyKAyIYTA5xKBeY5nRPEeAJlmQggh8LFEUO4fAQFhRBTtNu+lRCCEEL6VCFAKYvoTVrgNgJJyaSwWQgi3JQKl1KtKqSNKqewmtiul1HNKqZ1KqQ1KqeHuiuUUXYYSWrAFC9VsyD3hkVMKIURH5s4SwevAhGa2TwR6u14zgBfdGEudLsOwVJYwLuIYX2894pFTCiFER+a2RKC1Xgoca2aXScCb2lgBhCulEtwVT62EoQBcG5vH8pyjUj0khPB5yp3TLCilkoDPtNaDGtn2GfCE1vpb1/vFwENa6zWN7DsDU2ogLi5uREZGRpvicTqdOIIDGbfsFrLDL2bSgZ/ywHA7Q2P92nS89uJ0OnE4HF6NoSkdNTaJq3UkrtbpjHGlp6dnaa1TG9vmzTugamRdo1lJaz0bmA2Qmpqq09LS2nTCzMxM0tLSYPcwBlfnE+RvJd8/jrS0wW06XnupjasD6qixSVytI3G1jq/F5c1eQ7lAt3rvE4EDHjlzl2FYDm1kRLcQNu4v9MgphRCio/JmIpgH3OrqPTQGKNBaH/TImROGQmUJw4Ly2X+8xCOnFEKIjsptVUNKqXeANCBaKZULPArYALTWs4AvgCuBnUAxMN1dsZwmMgWAXv7HyHf6UVpRRYBrRlIhhPA1bksEWuubz7BdA/e66/zNCokDoKtfIRDLgRMlpMR0vIYhIYTwBN8aWVzDYRJBrDoOQK5UDwkhfJhvJgI/OwRFEVF1FID9JyQRCCF8l28mAoCQBILK8rBalDQYCyF8mg8ngngszkPEhwZIiUAI4dN8NxE44uHkYbpGBEqJQAjh03w3EYTEg/Mw3cL8pUQghPBpvp0IdBW9HKUcLCihQh5SI4TwUT6cCMxEpyn2k1RrOFRQ6uWAhBDCO3w+EXS1mbmGpHpICOGrfDgRmEFlcZhBZdJgLITwVb6bCFyji8NlUJkQwsf5biKw2iA4BlvxYaIddikRCCF8lu8mAjA9hwoPkhgRKCUCIYTP8u1EEDcIcleTGG4n93ixt6MRQgiv8O1EkJIOJccY7r+PAydKqa523/ObhRCio/LxRHARAOeV/0B5VTX5zjIvBySEEJ7n24kgJB5iB5BcuAqAXGknEEL4IN9OBAAp6YTnr8VOufQcEkL4JEkEKRdhqSpjmGWn9BwSQvgkSQTxgwEY7H9ASgRCCJ8kiSAkAfxDOM9+WLqQCiF8klsTgVJqglJqm1Jqp1Lq4Ua2hymlPlVKrVdKbVJKTXdnPE0ECTF96an2c1BmIBVC+CC3JQKllBV4HpgIDABuVkoNaLDbvcBmrfV5QBrwd6WUv7tialJMXxIr93GoUBKBEML3uLNEMArYqbXepbUuBzKASQ320UCIUkoBDuAYUOnGmBoX3YeQyqNUFx+ntKLK46cXQghvUlq7ZzStUmoyMEFrfafr/TRgtNb6vnr7hADzgH5ACHCT1vrzRo41A5gBEBcXNyIjI6NNMTmdThwOx2nro/JXMzh7Jj8pe4ybxp5HXLBnm06aiqsj6KixSVytI3G1TmeMKz09PUtrndroRq21W17ADcDL9d5PA/7ZYJ/JwNOAAnoBu4HQ5o47YsQI3VZLlixpfMPRHK0fDdX//Yff6OU789t8/LZqMq4OoKPGJnG1jsTVOp0xLmCNbuK+6s6vvrlAt3rvE4EDDfaZDnzoinOnKxH0c2NMjQvvQbXVTi91gMPSTiCE8DHuTASrgd5KqWRXA/AUTDVQfXuBSwCUUnFAX2CXG2NqnMWKjupNb5UrPYeEED7HbYlAa10J3Ad8CWwB3tNab1JK3aWUusu12/8CFyilNgKLgYe01vnuiqk51viBDLDslRKBEMLn+DW3USkVqrUubGJbd6313uY+r7X+AviiwbpZ9ZYPAJe3PFw3ShhK3IZ3KTqaCwz0djRCCOExZyoRZNYsKKUWN9j2cXsH41VdhgEQenyTlwMRQgjPOlMiUPWWI5vZdu6LH0w1injnFm9HIoQQHnWmRKCbWG7s/bnN7uBYYBIpFTuorKr2djRCCOExzbYRALFKqQcx3/5rlnG9j3FrZF5QGDGQwcXfkucsIyEs0NvhCCGER5ypRPASZsSvo95yzfuX3Rua51XEDSVWnSD/4I/eDkUIITym2RKB1vrPTW1TSo1s/3C8y6/bCPgBqncthX6eH9cmhBDe0KpxBEqpAUqp/1FK7QBedFNMXhOcMoZ91THE7XzP26EIIYTHnDERKKV6KKUeVkqtB94C7gEu001NXnQOi3QE8E5VOvHHVkP+Tm+HI4QQHtFsIlBKLccMCLMBk7XWI4CTWus9HojN4/z9LMy3XUoVVsh6zdvhCCGER5ypRJCHaRyOo66XUOfqNtqAcsSxOXgkbF/g7VCEEMIjmk0EWutJwGBgLfBnpdRuIEIpNcoTwXlDlMOfXao7HP8RquUhNUKIzu+MbQRa6wKt9ata68uAMcCjwDNKqX1uj84LooLt5FTGQHUFFDacNVsIITqfVvUa0lof1lo/p7W+ABjrppi8Ksrhz/byKPPm+B6vxiKEEJ5wptlHGz4/oKFr2jGWDiHKYWdpSSTYgeO7IXmct0MSQgi3OtMUE+cD+4B3gJV0tonmGhHt8OeAjkJb/FBSIhBC+IAzJYJ44DLgZuAW4HPgHa11p52rOSrYThVWKhxd8ZdEIITwAWfqNVSltV6gtb4N01C8E8hUSv3SI9F5QZTDH4Ci4G5wbLeXoxFCCPc7U4kApZQduApTKkgCngM+dG9Y3hPtSgQn7F2JOPyVl6MRQgj3O1Nj8RvAIGA+8GetdbZHovKiqGA7AHl+CSSXHIPSAggI83JUQgjhPmcqEUwDioA+wK+Uqm0rVoDWWoe6MTavCAu0YbUoDlrizIrjeyDhPK/GJIQQ7nSmaahbNc6gM7BYFFHB/uyplkQghPANbr3RK6UmKKW2KaV2KqUebmKfNKXUOqXUJqXUN+6Mp6WiHHZ2VESbN9JgLITo5M7YWNxWSikr8Dym+2kusFopNU9rvbnePuHAC8AErfVepVSsu+JpjWiHP7nFlRAUJaOLhRCdnjtLBKOAnVrrXVrrciADmNRgn1uAD7XWewG01kfcGE+LJUYEsSvPiQ5PkkQghOj0lNbumVVaKTUZ803/Ttf7acBorfV99fZ5BvOsg4GY6a6f1Vq/2cixZgAzAOLi4kZkZGS0KSan04nD4TjjfktzK3g1u5xvurxAbPFOVo75d5vO195xeUNHjU3iah2Jq3U6Y1zp6elZTT5QTGvtlhdwA/ByvffTgH822OdfwAogGIgGdgB9mjvuiBEjdFstWbKkRfttO1Soezz0md78n99q/ViE1pUVbT5ne8blDR01NomrdSSu1umMcQFrdBP3VXdWDeUC3eq9TwQazuucCyzQWhdprfOBpYDXu+j0jHHgsPuxpTQSdBUU5no7JCGEcBt3JoLVQG+lVLJSyh+YAjSczfQTYJxSyk8pFQSMBra4MaYWsVoUQxLDWF3gGiYhPYeEEJ2Y23oNaa0rlVL3AV8CVuBVrfUmpdRdru2ztNZblFILgA1ANaYqqUOMXh7aLZxPl4aCP9JgLITo1NyWCAC01l8AXzRYN6vB+6eAp9wZR1uc1y2cWdXhVFv8sRyXEoEQovPyuZHDLTW8ewTVWCgI6CIlAiFEpyaJoAkxIXaSo4PJ1bGQtw3c1M1WCCG8TRJBM0YlRfJxyVDI2wrL/ubtcIQQwi0kETRjZHIkr5ReRGHv6+Drv8Ceb70dkhBCtDtJBM0YlRQJKD5LehiCo+H7570dkhBCtDtJBM3oFhlIXKidFXtLYNhPYfsCKNjv7bCEEKJdSSJohlKKMSlRfLM9j8IBt4CuhrWnTYUkhBDnNEkEZ3B3Wk9Ollbwt9Xl0PNikwiqKr0dlhBCtBtJBGfQLz6UaWN68J8VP7K/181w8gDsWNj4zs48OLAOCmRuIiHEuUMSQQvcf2kfNPDByUHgiIes107f6cA6eGYwzL4I/jkCSo57OkwhhGgTSQQtEBnsT9+4ENbkOmH4NNjxFWx4D7Z+bl6b50HGVPNEs8tnQmWpdDUVQpwz3DrXUGcyrHsEn204QPW107B8+wx8+PNTd/ALhDvmQ9wgyHwCcpZA///ySqxCCNEakghaaHj3cN5ZtZecikh6/+oHKDl26g4hXcARY5aTxsKuJZ4PUggh2kASQQsN7xEBwNq9x+k9sjuEd2t655R0M+bg+I8Q0cNDEQohRNtIImih5KhgwgJtrP3xBDeN7N78zilp5t8v/wDRvc3ywJ9AwhC3xiiEEG0hiaCFLBbFsO7hZO1tQW+gmL7QNdV0M92xEKoqTOPxnYvcH6gQQrSS9BpqhbG9otl5xMnu/KLmd1QKfr4Y/l+eeV0+E3JXw6EO8fA1IYQ4hSSCVrhycAIAn2840LoPDr0FrPbGxx8IIYSXSSJohS7hgaT2iOCzDQdb98GgSBh4LfwwB167Cta9febPlByHeb+EvO1tilUIIVpKEkErXT0kga2HTrJu34nWffDCB6D7aHAehk/uhZ2Lm98/6w0zr9E7N0FJK88lhBCtIImgla4cnECQv5Vrn/+O6a+toqq6hY+wjBsAt34Cv/gGYvqbkcjPDoVnh5K6+oFTp7eurjbVSJE94cQ++OwBN/wkQghhSCJopdjQAL58YDwzxqewZFsen29sZTWRfzDckgFDboTEkZCYSnDRj7Dmlbp9dmfC8T2Q9nsY9XMzjUV5cXv+GEIIUcutiUApNUEptU0ptVMp9XAz+41USlUppSa7M5720i0yiIcm9KNXrIN/Lt5BdUtLBTXCu8M1z8H1L8H1L3M0agT88B/TzbS6Cr59BgIjzRQVvS6FqnL4cblbfhYhhHBbIlBKWYHngYnAAOBmpdSAJvb7K/Clu2JxB6tF8cuLe7HjiJO3V+09q2MdTLjCtB1snAtf/y/s/gYu/hPYAqDHBabHkUxZIYRwE3cOKBsF7NRa7wJQSmUAk4DNDfb7JfABMNKNsbjF1UO68O7qffy/T7LRWjPt/KQ2Hedo1HAITYSP7zIrht8GqXeYZVugaWTeldkuMQshRENK61ZWa7T0wKaaZ4LW+k7X+2nAaK31ffX26Qq8DVwMvAJ8prWe28ixZgAzAOLi4kZkZGS0KSan04nD4WjTZ5tSXqV5fl0ZG/KqePT8AJLCrG2KK14fIbRwC1XWYI7EXoi22Gq3d/9xLim73+K7C96gwj8ce2k+torCU45RZo+mwj+06ZNoTVDxPizVlRQHJVJt9W9xbO19zdqDxNU6ElfrdMa40tPTs7TWqY1tc2eJQDWyrmHWeQZ4SGtdpVRju7s+pPVsYDZAamqqTktLa1NAmZmZtPWzzRlzYQUX/y2TefsD+OCaC2juZ2kqrtS0q2vf92+4w8EI+PdbXGhZD13GwDs/N89Prs/fAT/7yvROasznv4HVL5vl826B615scWzuuGZnS+JqHYmrdXwtLncmglyg/hSdiUDDIbmpQIbrxhkNXKmUqtRaf+zGuNpdaICN313Rj999sIF3Vu3jltFnmJSuteKHwLBp8O0/wBZknnmQVq/tvboSvvhvyLgZLv8LqAZNP4c2miQw4nYzJiH7A7jiL2agmxDC57kzEawGeiulkoH9wBTglvo7aK2Ta5aVUq9jqoY+dmNMbjN5RCLz1h/gkU+ySQgLIL1fbPsdXCm46u+Qtw2O7YIpc0zPo/oc8fDG1fDu1MaP0fMSuOof5hibPzajmy+4r/F9hRA+xW2JQGtdqZS6D9MbyAq8qrXepJS6y7V9lrvO7Q0Wi+LFnw7n5pdWcM+ctSx4YBw9ooLb7wR+drj9c6gohsDw07d3Hw0PbISTh07fpiwQOwAsVlN11G00rH7JHLO+5PFm5lQhhE9x6zTUWusvgC8arGs0AWitb3dnLJ4QEmDj5VtHctk/vuH3H25kzp2jW91e0Cw/f/NqMoB48zqT0XfB3OnwxW9PXR87EO7+zpRAhBA+Q55H0M7iwwL4/ZX9+cNHG3nl293cOS7F2yGdbtBPoGc6VFXWrcueCwseNtNldxvlvdiEEB4nU0y4wZSR3bi0fxwzP9/CrG9yvB1O4wIjzDOWa17Dfmp6Hq2RqbKF8DVSInADi0XxwtThPPjeOp6Yv5VRyZEM7x7h7bCaZw+BwTfA+ndAV9Wu7nf4MEQchvNugm0LzAjoEbed/vnqalj82OltFJE9YfxvTfuEEKJDkkTgJv5+Fv56/RCW5xzlucU7eH36OVDdMuYe2Ps97FtZuyqiqAA+yoSjO+Hbp6G6wnRhHXLDqZ/N+Rq+exZCu4LVNRhOV8OGd6GqDC55xHM/hxCiVSQRuFGw3Y87xyXz5IJtrNt3gqHdwr0dUvNi+sC9K09ZtXLxQsbnPA5Ln4TIFHDEwbz7YO0bkJgKlzxqGpezXoPgGPjVulMbtD+9H5b9HeIHw8Dr6tZXV8OiR+DAulNjCIqEiU9B2UlY+Ccod0K/q2DM3bD6FZNcRv389NhLTphBc87D5r3VBhc9BN3HtMeVEaJTkzYCN7v1/CQig/25/bVVzFn5I86yyjN/qAOptvqbcQtDpsDNGXDjW9D7cnOD/vZpWPY3KDwA2+bD0Kmn92qa+KTprvrxPac+sznzcVj+T9MdVlfXvbYvhPemwTtTYO9yKD5qGrE/uRc+fxC+/CMUH2sQZBV88DPY/EndcQ5vhnduNtN5CyGaJSUCN3PY/XjvF2P4w4fZ/PGjbP48bzPXDevKQxP7ERncsvl+vC60C/zk33Xvb3oLtIYPZ8DXf4FVL5l2hcbaDvzscOObMDsNMm6BGZmw51tTwhj6U5j0r1O7q2Z/AHPvAIufeZBPt9Hw5iQzTXdMP8jbCuvfoceejfDcr81nKsuhMBeufgZSp5t1R3PgpXT490VNj6BWFlMdNvJn7XCRhDh3SSLwgF6xIWTMGMOqPcf4fMNB3lm1ly83H2L6Bcncen4Pb4fXNkqZZyo4Yk11TMJ5puqoMSHxcNN/4LWJ8PaN5tt611QzWrrhmIVB15uH8ASGQ9JYs+7GN+H75031UMZUyHyC5LJC6DEWQhPMPokj65IAQFRPuOV988CfhvMy1TiaY6qTwntA70vP6nIIcS6TROAhFotiTEoUY1KimDqmO3+dv5WnF23n30tzGJug6DO0hC7hgd4Os3VsgWbOopZITIWrnzZVPI44U6qwBTS+7/Bpp74PjoZLHzXLqdPh47s5FnEekbd+AtZm/oS7jzavppQXwSuXw5zJ0MLZWM9kfHU1LGtQ42q1mevU+3KYc4Ppqjvm7nY5nxDtQRKBF/SLD+W16aPYeqiQf3+zi0/W7efrJ5dwzdAuzBifQt+4kPYdkdxRDPsp+AVA3EBT3dQWg2+A6ko2H41kbHNJoCX8g2Hq+5D1OlSWnd2xXHL37qV79wbzQO1bCZ89CFG9IH8bLPg9hHUzDx1y58R/Wpv2k6auU3VV27r1tvVznZHWdaXa+stN6aDXThKBF/WLD+Xpm4ZyYegxsivieHf1Pj5cu5/QAD9GJUfxi4tSGJnUyWYIHXyWTyO12mD4rVRmZrZLOIR2gfQ/tM+xgF2ZmXRvOE1waSG8fIlJAj95Cb57rm5ywFEz4Mqn2u38taqr4f3b4MhmuGPh6dvztptJCi/6HYy8s+XHXfsmfPUo3PGl6WXmy3K+hnenwX2r4dhu+M/1cN+q0yeErHE0B168EG5+x4zs70AkEXQA0YEWHps4kPsv6c2nGw6w9dBJFmQf4oZZ3zOiRwTTL0wirW8sDrv8us5JAaFw26eQvwOSx0HPi00Pp9zVsGo2hCVCylneGCJTwO4wJZu8bbDxfdgyzzSIz70dR+R1cND1paK60jT0Ow+bpDTiDig9AQHhYLFAyXGwh57+zXXvClOyqa4w05pf+SQUHYXgKLO95ETdhIhNLZ/rtIaifDMaf+di03tu5yLTI66yxDxbvGEicB4xbWnbvzT7rHjRJILiY3UlwuJjZrS/Uqcul5yAgDC3z/8ld5YOJCLYn1tdj7v8f1cN4P2sfcxeuov73v4Bfz8L1w7twkV9YjlYUMLALmGM6BGBn0VhsXTCaqTOpv6EgMHRpqfSiNvNTeWrdhhsF94dps83jekH15l1w35qGtQ/vovU3Ushq97+Fj9TGlk1G5Y/B988aW5OF/8/026SmGqqzWqSQeEB8+03vBtE94H1GeZm+PVMuOafpsrj8wdh6lyTUF69HNL/aKrD3r8Nrvs3DLnx7H9Ob1s3xyTD+9fXjYHJWQKHXV2jD/wA502p2z83C16+GKZ+UPfc8R0LTTJY8HsY9xvocT7MuRFG/8KMmXnzWhh6Mwy/HV6/0uxz0e/c+mNJIuigAv2t3Hp+EreM6k7Wj8eZt/4Ac7NyeW9N7in7WZQZq/CHK/vj7yfDQs4pFqvpTbX7G6iqaPtxSk+Ym9Pzo00D+IQnILo3JKeZ9oHo3mR/v4hBgwbVfSaqpylFZH8Aix41o8W3fmaeja2rIWex6VHV5wqz/9KnzJiP2+aZsR3bF5gkYAsy5wbzuZX/Nt9yqyth8f+YNiFdbW58Q1w9xqJ7144+t1SVQv5OiO7V9p+/vZQXQeFBiO6FparMVJ81rP7a+oUZKZ+zGA6uN+t2LDQlAzDJQWuTjLsMg+3zzfrv/wX7VkGfCaZksOBhsAWbcTgrgk036xUvmKo3q838m/0hVJaa7tljf103Yt8NJBF0cH5WC6NTohidEsVvLu/L/uMlxIXZWfvjcbYdcrLveDGvL9/DguxDhAfZuHpIAjPG95SkcK6wBdTdbM+GssDHd5tv9A17JCWmkh/jhP5pp38u9WemRHDbZ6ar7Yb3zPiNTR+a6p+smkkIlUlasf3NjS5+sCkF3PIuvHGNOX/Pi81nrP7mcahHNsHJwzD0LjP4cOGfzCDCQZPh+pehqpzz1j8C3+WYqrMeF5z9dWirynJ46zrYnwXTPmLwxpnwbbYpFfVydS2uqoQ9y8zy6leg/CQkjatblzTOfH7j+/Dhz+Hmd01pAepKA8OmmRv6vtXwsy9NFV3+DrhzEXz2a1OiuHOxSc67l5mpWRb/D2z9HAZe67YfXxLBOSQy2L92ENqEQQlMcH3Bu7R/HPPW7+dYUTl/W7idD9fu5660nlzWPw6LRbFy11EGdAklMSLIi9ELtxp6C/S6zFTXtEba703iCIqErsPhsv8xVVc9LjBJosrVmyooqq7uWylTDWW1m5Hkd39n1hcfNYmgqgzOv8dUIVWWmsSxYpZJAo54M+V5YDgU5RFWuM10J37vVjMliFKgrNDvavOz7F1hxnmEJpibZ0i8qZ4CqCgxN8tel5gSyK5M8yQ+tGnITUk3pa6di81Dl6z+5pt8jwtMSWbnYtO92B4C839nenc54uDNa4nQVWZ57h2miisi2dTVlxVCYCQcWGtiGPuASQTBMeZ3sGcZLPqz2fbt02a/oVPNZI4o00bU61IzANM/2CTgiiLTJvDTD83xgyJhyttQWmDOueY1k5AlEYjmTBgUz4RBpv55ydYjPPnlNn43dwNg/l9pDSF2P351SW925TvpFx/K1NHd8bNKqaFTaW0SANM4XNNgqZRJAjXLcQOa/pw9pG7ZP7ju3wGTTEKIH2zW1TwFL3W6qUKZvgC+/INJGMCeHjeRdPVv4dUrTn1Q0qrZJjG8f7upwrr8f00bRWgXMzo9KMqUgDZ9BGMfNA3f6+bA6LvNTXbVbPPtOyjSTIY4aLJJTJn/B32vNMlg4Z9Msuh3lbnRjn0QzrsZXpvAvshxdPvJYyau+a76+fghgIJxD5rP+gWY6rfu55snAHYZbvYrzIXQRNi3wrwf9lNTrVZaaG7s9dV/2JTVr+53YbHWLQ+/FZb+zbQnuYkkgk4mvV8saX1jWLn7GNn7CzhZWsnQ7uH8Y+F2/vLFFgJsFkor9vGfFT8yMjmS8b1juHxAnDQ4i/Yx+dXG11/xf3D5THOD+8ls895iZc+qDSTF9IEHN5uJBsFUr2TcYhqZI3vCiR/N+4gkM8352zea5LDpI4jpD9/+w3wupj+sfLFu+Ye36paz59Ytb/vCvGL6myqbXUtMaeriP5n4frONnGXf0S0yGe7fYL6Zf3y3KU10GQYDf2ISQfxgc/O+/XNAAdrU+1dXwJT/wOx0U/LomgrXj3bt0wajfm5KZzW9s9xAEkEnpFTdKOYaF/SMYldeEb1iHSzcdJjXvtvNZ+sP8PbKvaREBzNjfAoDuoRysKCUruGBpMQEE+Qvfx6ilZoaLFVT5VOz3LD0Ygs0L4C+E81khateMnX0u5fCd8+YOveD6+DTB0xD7YjbzX4ZU82352v+ZR7Bqixw/SvwyT3mJn7Tf0xp4/iPpk3jq0fh0AbTw2npU6Y65/qX62Kv3yhrCzCvya+YOa+G3ARhXaH3FXVToNT/mQdea7redhkGw6bWVZ+djUD3P8tE/qf7CLuflf4JoQBcNSSBq4YkUFWtmZ99kFnf5PDwhxtP+0yPqCCu7VHFRVpTUFJBWKCtc454Fh3PqJ/XTTceMa1u2pHoXqcPSvzp3LrlKXPqlq9/uW75mn/WLV/5ZN3yZX9uWTyBEfCLpXXvp77X+H7XvlC3POn5lh27A5BE4MOsFsXVQ7pw1eAEVu4+xonichLCAtl/ooScI07mZx/i2bXFvJfzNQcLSrl+eCIzrx1EoH/HGyIvhGg7tyYCpdQE4FnACrystX6iwfapwEOut07gbq31enfGJE5XU5VU4zzXA3R+Pj6FB15ZRGVgGJf0j2XOyr1sPljIi1OH0yMqCK2RtgUhOgG3JQKllBV4HrgMyAVWK6Xmaa0319ttN3CR1vq4UmoiMBtoZrpI4UkBNitT+tlJS0sF4JL+cTyQsY6Jzy7DoqCiSpMUHURKtIOBXUK5pH8c/RM66YR5QnRi7iwRjAJ2aq13ASilMoBJQG0i0Fovr7f/CiDRjfGIs5TeN5bPfjmWZxfvwGH3w+5nISeviO2HT/Ll5kP8/avtJEYEcuXgBG67IImu59q02kL4KKW1ds+BlZoMTNBa3+l6Pw0YrbW+r4n9fwv0q9m/wbYZwAyAuLi4ERkZGW2Kyel04nA42vRZd+qocUHLYyso06w7UsnaI1VszK9CAYkhFnqGWbiml41we/uOWeio10ziah2Jq3XOJq709PQsrXVqY9vcWSJorH6g0ayjlEoHfgaMbWy71no2ptqI1NRUndZwmt8WyszMpK2fdaeOGhe0LrZJrn9zjxczZ+VesvcX8O2uY6zOq2Bot3DsfhaSooJJTYpkfJ/os+qe2lGvmcTVOhJX67grLncmglygW733icCBhjsppYYALwMTtdZH3RiP8JDEiCAemtAPgF15Tv62cBuHCkrJO1nFsh35vPztbux+Fsb1jmZkUiQxIXZyj5cwqGsoF/eL83L0QvgedyaC1UBvpVQysB+YAtxSfwelVHfgQ2Ca1nq7G2MRXpIS4+CFqSNq31dUVbN69zEWbj7Moi2HWbTlyCn7X9QnhnG9o0mJCaZnjIPEiCCs0jNJCLdyWyLQWlcqpe4DvsR0H31Va71JKXWXa/ss4BEgCnjB1dOksqk6LNE52KwWLugVzQW9onnsmoEUlFSQd7KMuFA7b6/cy0vLdvHN9rza/f39LCRHBdMjKqh2RlU/iyKsvIJBzjKiHXZv/ShCdBpuHUegtf4C+KLBuln1lu8EWvGcPNHZhAXaCAs0Q/p/cVFPfnFRT44VlbMrz0lOnpNdeUXk5DnZc7SIymrTxFRUVsnhwnLe2fY114/oyozxPUmODvbmjyHEOU1GFosOx0y3HUlqE89r1lrzn8+WsLUyhvezcslYvY+L+8YyYVA8Vw5OIFge6SlEq8j/GHHOUUrRLcTCtLTBPHBpH177bjcf/bCfxVuPMPPzLVwxMA67n5X4sAB6xjjoFRtMSrRDRkEL0QRJBOKcFhNi53cT+vHfV/Rl7d7jzF66i8VbjlClNSeK6x7/GO2wM75PNH3iQrjmvC50kcFuQtSSRCA6BaUUI3pE8u9pddVJJ0sr2J1fxLZDJ8ncnsfS7fl8uHY/s77J4ZcX92bLwUKOF5UTGmjjltHdGdlEVZQQnZ0kAtFphQTYGJIYzpDEcG5INUNacvKc3DtnLf/72WbCg2wkRgSStfc4H/2wnz5xDlKTIjlSWEpMiJ2UaAdzs3KJCLbVjosoKKnA38/CyKRIbPKEN9FJSCIQPqVnjIOP772QXXlF9Ilz4Ge1UFJexdy1uXyx4SCfrjtAl/BAVuw6hrOskoFdQtl+2Ml1Lyw/5ThdwgIY1j2CfceLqazSOAL86BnjgMIK1i/awXc5+YQF2hjYJZS0vrGUV1bj72dhaLdwjhWVszvfyfDuETJBn+gQJBEInxNgszKgS2jt+0B/K9PG9GDamB6168oqq9h/vITk6GBOFFcwP/sQsSF2IoL9yTtZyuvL97BxfwE9ooKw+1k5XlzOguyDHC+uQG3bzuCuYRwrKmfxlsM8s2hH7XEv6hPD+twTnCiu4JJ+sfRPCMVZVsn4PtF0jwwm2G4lIUzaL4RnSSIQohF2PyspMWZyr4hgf24Z3f2U7RMGJTT6uU8XLuHCCy8kMtg8nvCos4zlOUcJCfAje38B//x6J0MSwxjbK4ZZ3+SQuT0Pm1Xx+vI9tcfoHWuqqGJD7Ow7Xozdz0JsSAD7T5TgZ1F0jwrC31o3uG50ShRdwgPZd6yYympNXKhdkoloFUkEQrSjEH9VmwQAohx2/uu8LgCk9Y3ljrHJBNqsKKWYMT4Fq0Wh0azefZxjxeXknSzj662HmZ99kBPFFcSHBlBeVc2xonJiQ+xUa02+s/yMcQxICGVw1zACbBZyj5cQVFFOYcQBDhWUcHG/WHrGONidX0RCWCCB/lb25BcRbPcjJkRGavsiSQRCeFD9GVfrP/JzbO/o2uWfjU0GqG1XaLhcXF5JlWuU9cnSSpZuz+NESQVJrmk4dh5xsmRrHou3Hqa0opr4sAByjlTwac4PADz15TZ6x4aw+WAhXcMDGdEjgnnrD6AUDO4aRp+4EEIDbGZ6j+ggUmIcxLomBuweGUS3yCAqqqqxKiVjMzoJSQRCdFA1N/6Gy/WTSUiAjSmjTq22urhfHDPG9zxl3acLl9Bz0AhCA/14+qsdbD5YyH9f0Zd56w7w+caDzBifQrC/H9/vymfp9jxKyqsoq6ymvKr6lONYFIzoEcGmA4WEBdq4/YIkhveIYO2Px/lq82HS+8USYLOyIPsgY3vFcNWQBOx+FrqGB2KxKE4UlxMWaEMpxfGicsqr3PM8FNE6kgiE8AEh/qq2gfzvN55Xu37G+BROFFfUVgndT+/abVXVmgMnStiZ5ySvsIwu4YF8uzOfzG1HmDS0K7vynDw+f2vt/inRwTz15bba5acXbefpRWZS4bhQO2GBNrYfdtIvPoQeUUEs3HwYNAzYtIxHrh7A6HrPzRaeJYlACB9ms1qabBewWhTdXFVBNcb2jubhif1q3+87VszOI05iQ+0M7BLGtkMnqaiqZlDXMHYcPkn2gQKKy6v4dkc+haUVTBiUwPyNB1m+8ygzxqVw+MA+1hyt4KbZK0iODiYlOpiUmGBCAmy15+gTF0L/hBC+23mUfGcZNquFpKggwoP8CfK3MrhrmNurqLTWnbqrryQCIUSbNUwUfeNDapd7x4XQO868nzq6rmvug5f1qV3OzDzM47eO443v97Ah9wS78opYtjOf8spTq6Sa0yfOQUSQPxtyCxjeI5zLB8QzqGsYH/2QS3llNTeP6k5VtWZXXhH7jhcTG2KnZ4yD7lFB5DtNA31VtSb3eDEAk0ckkp1fxa5vdzNxcDzPL9nJFxsP8cClvSkqq+KHvcf5yfCuWC0WNu4vIDE8kIhg/9oeXd0jg/CzKDYdKOT15XtYkH2IgV1M431xRRXLduShNUwZ2Y09R4spLKkgrW/saQk50GYlMSKQDfsLyD1ezPjeMbjr0cKSCIQQXhXob+Wui+raNLTWuNrCqayuJmvPcXYccXJBzyhSYhyUVFSxJ7+Ik6WV5B4v5s3vf6SgpIJrh3Vl1e6jPDpvEwB2PwtWi+K9Nbmtiufx+VtdjfGb+Z/PNgMm2TzyiTluZLC/qdZqgp9FERpo41hROQE2C1cMjGfrwZPMWbkXP4tiZHIkztJK/rZwO6EBfoQE2Jo9Xo0nF2zjsh5+pKe36sdpEUkEQogORSmF1VULY7VYax9kVMNh92NQ1zDXu6ja6UNq5OQ5Wb/vBOP7xGCzWPhqy2Eig230jHHQNTyQfGc5OXlO9h4rJirYn/iwACxK0SU8kCMnS3lvdS76xH5uvHQ0767ex9he0VzSP5Zvd+YTEeRP/4RQMrcdqZ1q5HBhKSdLKymrrGJPfjE5eU6OnCxjVFIklw2II6Jed+L69p8oITbEjp9FsTu/iOLyqlO2nyytZO+xInrFOugeGUzmtiMU5LrnQY6SCIQQnUrPGIeZ7sNl8ojEU7bHhwUQHxbAhY18NjLYn0f+awCZmUfonxDKY9cMrN02rndM7fIl/euerd0jqu6hSCN6tHziwq71ZsBNqRdvfef3rGtAvyG1G5nOnBYfvzVk1iwhhPBxkgiEEMLHSSIQQggfJ4lACCF8nFsTgVJqglJqm1Jqp1Lq4Ua2K6XUc67tG5RSw90ZjxBCiNO5LREopazA88BEYABws1JqQIPdJgK9Xa8ZwIvuikcIIUTj3FkiGAXs1Frv0lqXAxnApAb7TALe1MYKIFwp1fhE70IIIdxCuWvIslJqMjBBa32n6/00YLTW+r56+3wGPKG1/tb1fjHwkNZ6TYNjzcCUGIiLixuRkZHRppicTicOR+P9db2po8YFHTc2iat1JK7W6YxxpaenZ2mtUxvb5s4BZY3N0NQw67RkH7TWs4HZAEqpvPT09B/bGFM0kN/Gz7pTR40LOm5sElfrSFyt0xnj6tHUBncmglyg/tjvROBAG/Y5hdY6prntzVFKrWkqI3pTR40LOm5sElfrSFyt42txubONYDXQWymVrJTyB6YA8xrsMw+41dV7aAxQoLU+6MaYhBBCNOC2EoHWulIpdR/wJWAFXtVab1JK3eXaPgv4ArgS2AkUA9PdFY8QQojGuXXSOa31F5ibff11s+ota+Bed8bQwGwPnqs1Ompc0HFjk7haR+JqHZ+Ky229hoQQQpwbZIoJIYTwcZIIhBDCx/lMIjjTvEcejKObUmqJUmqLUmqTUup+1/rHlFL7lVLrXK8rvRDbHqXURtf517jWRSqlvlJK7XD9G+HhmPrWuybrlFKFSqkHvHG9lFKvKqWOKKWy661r8voopX7v+nvbppS6wsNxPaWU2uqaw+sjpVS4a32SUqqk3nWb1eSB3RNXk783L1+vd+vFtEcptc613pPXq6l7g/v/xrTWnf6F6bWUA6QA/sB6YICXYkkAhruWQ4DtmLmYHgN+6+XrtAeIbrDuSeBh1/LDwF+9/Hs8hBkY4/HrBYwHhgPZZ7o+rt/pesAOJLv+/qwejOtywM+1/Nd6cSXV388L16vR35u3r1eD7X8HHvHC9Wrq3uD2vzFfKRG0ZN4jj9BaH9Rar3UtnwS2AF29EUsLTQLecC2/AVzrvVC4BMjRWrd1ZPlZ0VovBY41WN3U9ZkEZGity7TWuzFdpEd5Ki6t9UKtdaXr7QrMYE2PauJ6NcWr16uGUkoBNwLvuOPczWnm3uD2vzFfSQRdgX313ufSAW6+SqkkYBiw0rXqPldR/lVPV8G4aGChUirLNb8TQJx2DfJz/RvrhbhqTOHU/6Devl7Q9PXpSH9zdwDz671PVkr9oJT6Rik1zgvxNPZ76yjXaxxwWGu9o946j1+vBvcGt/+N+UoiaNGcRp6klHIAHwAPaK0LMVNw9wSGAgcxxVNPu1BrPRwzPfi9SqnxXoihUcqMTr8GeN+1qiNcr+Z0iL85pdQfgUpgjmvVQaC71noY8CDwtlIq1IMhNfV76xDXC7iZU79sePx6NXJvaHLXRta16Zr5SiJo9ZxG7qSUsmF+0XO01h8CaK0Pa62rtNbVwEu4qVjcHK31Ade/R4CPXDEcVq6pwV3/HvF0XC4TgbVa68OuGL1+vVyauj5e/5tTSt0GXA1M1a5KZVc1wlHXchamXrmPp2Jq5vfWEa6XH/AT4N2adZ6+Xo3dG/DA35ivJIKWzHvkEa46yFeALVrrf9RbX/85DNcB2Q0/6+a4gpVSITXLmMbGbMx1us21223AJ56Mq55Tvql5+3rV09T1mQdMUUrZlVLJmIcvrfJUUEqpCcBDwDVa6+J662OUeWgUSqkUV1y7PBhXU783r14vl0uBrVrr3JoVnrxeTd0b8MTfmCdawzvCCzOn0XZMRv+jF+MYiym+bQDWuV5XAm8BG13r5wEJHo4rBdMDYT2wqeYaAVHAYmCH699IL1yzIOAoEFZvncevFyYRHQQqMN/Gftbc9QH+6Pp72wZM9HBcOzH1xzV/Y7Nc+17v+v2uB9YC/+XhuJr8vXnzernWvw7c1WBfT16vpu4Nbv8bkykmhBDCx/lK1ZAQQogmSCIQQggfJ4lACCF8nCQCIYTwcZIIhBDCx0kiEKIBpVSVOnXG03abrdY1m6W3xjwI0Si3PqpSiHNUidZ6qLeDEMJTpEQgRAu55qn/q1JqlevVy7W+h1JqsWsitcVKqe6u9XHKPAtgvet1getQVqXUS6455xcqpQK99kMJgSQCIRoT2KBq6KZ62wq11qOAfwHPuNb9C3hTaz0EM7nbc671zwHfaK3Pw8x/v8m1vjfwvNZ6IHACM3pVCK+RkcVCNKCUcmqtHY2s3wNcrLXe5Zoc7JDWOkoplY+ZKqHCtf6g1jpaKZUHJGqty+odIwn4Smvd2/X+IcCmtZ7pgR9NiEZJiUCI1tFNLDe1T2PK6i1XIW11wsskEQjROjfV+/d71/JyzIy2AFOBb13Li4G7AZRSVg/P+y9Ei8k3ESFOF6hcDy93WaC1rulCaldKrcR8ibrZte5XwKtKqf8G8oDprvX3A7OVUj/DfPO/GzPrpRAdirQRCNFCrjaCVK11vrdjEaI9SdWQEEL4OCkRCCGEj5MSgRBC+DhJBEII4eMkEQghhI+TRCCEED5OEoEQQvi4/w8O84tffDu1rgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics = pd.read_csv(f\"{trainer.logger.log_dir}/metrics.csv\")\n",
    "\n",
    "aggreg_metrics = []\n",
    "agg_col = \"epoch\"\n",
    "for i, dfg in metrics.groupby(agg_col):\n",
    "    agg = dict(dfg.mean())\n",
    "    agg[agg_col] = i\n",
    "    aggreg_metrics.append(agg)\n",
    "\n",
    "df_metrics = pd.DataFrame(aggreg_metrics)\n",
    "df_metrics[[\"train_loss\", \"valid_loss\"]].plot(\n",
    "    grid=True, legend=True, xlabel='Epoch', ylabel='Loss')\n",
    "df_metrics[[\"train_mae\", \"valid_mae\"]].plot(\n",
    "    grid=True, legend=True, xlabel='Epoch', ylabel='MAE')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d73be89",
   "metadata": {},
   "source": [
    "- As we can see from the loss plot above, the model starts overfitting pretty quickly; however the validation set MAE keeps improving. Based on the MAE plot, we can see that the best model, based on the validation set MAE, may be around epoch 175.\n",
    "- The `trainer` saved this model automatically for us, we which we can load from the checkpoint via the `ckpt_path='best'` argument; below we use the `trainer` instance to evaluate the best model on the test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d60a0ce3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Restoring states from the checkpoint path at logs/mlp-corn-cement/version_2/checkpoints/epoch=155-step=779.ckpt\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "Loaded model weights from checkpoint at logs/mlp-corn-cement/version_2/checkpoints/epoch=155-step=779.ckpt\n",
      "/home/jovyan/conda/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/data_connector.py:258: PossibleUserWarning: The dataloader, test_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 4 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88b0f73bc1154f74898737527129654f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">        Test metric        </span>┃<span style=\"font-weight: bold\">       DataLoader 0        </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">         test_mae          </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.28999999165534973    </span>│\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m        test_mae         \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.28999999165534973   \u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────────┴───────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'test_mae': 0.28999999165534973}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.test(model=lightning_model, datamodule=data_module, ckpt_path='best')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec7d93a1",
   "metadata": {},
   "source": [
    "- The MAE of our model is quite good, especially compared to the 1.03 MAE baseline earlier."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}