examples/transfer_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transfer Learning With Poutyne"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --upgrade poutyne # to install lastest poutyne version, if it's not already\n",
    "import os\n",
    "import tarfile\n",
    "import urllib.request\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.models as models\n",
    "from torch.utils import model_zoo\n",
    "from torch.utils.data import Subset, DataLoader\n",
    "from torchvision import transforms\n",
    "from torchvision.datasets import ImageFolder\n",
    "\n",
    "from poutyne import set_seeds, Model, ModelCheckpoint, CSVLogger, Experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Also, we need to set Pythons's, NumPy's and PyTorch's seeds by using Poutyne function so that our training is (almost) reproducible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "set_seeds(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We download the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_and_extract_dataset(path):\n",
    "    os.makedirs(path, exist_ok=True)\n",
    "    tgz_filename = os.path.join(path, \"images.tgz\")\n",
    "\n",
    "    print(\"Downloading dataset...\")\n",
    "    urllib.request.urlretrieve(\"https://graal.ift.ulaval.ca/public/CUB200.tgz\", tgz_filename)\n",
    "    print(\"Extracting archive...\")\n",
    "    archive = tarfile.open(tgz_filename)\n",
    "    archive.extractall(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading dataset...\n",
      "Extracting archive...\n"
     ]
    }
   ],
   "source": [
    "base_path = './datasets/CUB200'\n",
    "extract_dest = os.path.join(base_path, 'images')\n",
    "download_and_extract_dataset(base_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We create our dataset object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "norm_coefs = {}\n",
    "norm_coefs['cub200'] = [(0.47421962, 0.4914721, 0.42382449), (0.22846779, 0.22387765, 0.26495799)]\n",
    "norm_coefs['imagenet'] = [(0.485, 0.456, 0.406), (0.229, 0.224, 0.225)]\n",
    "transform = transforms.Compose(\n",
    "    [\n",
    "        transforms.Resize((224, 224)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(*norm_coefs['cub200']),\n",
    "    ]\n",
    ")\n",
    "\n",
    "# is_valid_file removes hidden files from the dataset.\n",
    "dataset = ImageFolder(\n",
    "    extract_dest,\n",
    "    transform=transform,\n",
    "    is_valid_file=lambda path: not os.path.split(path)[1].startswith('.'),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "We do the split train/valid/test with a 60/20/20 split respectively. We do a *stratified* split with scikit-learn in order to get examples of every class in every split."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We take 60% of the dataset for the training dataset\n",
    "train_indices, valid_test_indices = train_test_split(\n",
    "    np.arange(len(dataset)),\n",
    "    train_size=0.6,\n",
    "    stratify=dataset.targets,\n",
    "    random_state=42,\n",
    ")\n",
    "# We take 20% for the validation dataset and 20% for the test dataset\n",
    "# (i.e. 50% of the remaining 40%).\n",
    "valid_indices, test_indices = train_test_split(\n",
    "    valid_test_indices,\n",
    "    train_size=0.5,\n",
    "    stratify=np.asarray(dataset.targets)[valid_test_indices],\n",
    "    random_state=42,\n",
    ")\n",
    "\n",
    "train_dataset = Subset(dataset, train_indices)\n",
    "valid_dataset = Subset(dataset, valid_indices)\n",
    "test_dataset = Subset(dataset, test_indices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Now, let's set our training constants. We first have the CUDA device used for training if one is present. Secondly, we set the number of classes (i.e. one for each number). Finally, we set the batch size (i.e. the number of elements to see before updating the model), the learning rate for the optimizer, and the number of epochs (i.e. the number of times we see the full dataset).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Training constants\n",
    "cuda_device = 0\n",
    "device = torch.device(\"cuda:%d\" % cuda_device if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "num_classes = 200\n",
    "batch_size = 64\n",
    "learning_rate = 0.1\n",
    "n_epoch = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creation of the PyTorch's dataloader to split our data into batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=8, shuffle=True)\n",
    "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, num_workers=8)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, num_workers=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We load a pretrained `ResNet-18` networks and replace the head with the number of neurons equal to our number of classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "resnet18 = models.resnet18(pretrained=True)\n",
    "resnet18.fc = nn.Linear(resnet18.fc.in_features, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We freeze the network except for its head."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_weights(resnet18):\n",
    "    for name, param in resnet18.named_parameters():\n",
    "        if not name.startswith('fc.'):\n",
    "            param.requires_grad = False\n",
    "\n",
    "\n",
    "freeze_weights(resnet18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "We define callbacks for saving last epoch, best epoch and logging the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# We are saving everything into ./saves/cub200.\n",
    "save_base_dir = 'saves'\n",
    "save_path = os.path.join(save_base_dir, 'cub200')\n",
    "os.makedirs(save_path, exist_ok=True)\n",
    "\n",
    "callbacks = [\n",
    "    # Save the latest weights to be able to resume the optimization at the end for more epochs.\n",
    "    ModelCheckpoint(os.path.join(save_path, 'last_epoch.ckpt')),\n",
    "    # Save the weights in a new file when the current model is better than all previous models.\n",
    "    ModelCheckpoint(\n",
    "        os.path.join(save_path, 'best_epoch_{epoch}.ckpt'),\n",
    "        monitor='val_acc',\n",
    "        mode='max',\n",
    "        save_best_only=True,\n",
    "        restore_best=True,\n",
    "        verbose=True,\n",
    "    ),\n",
    "    # Save the losses and accuracies for each epoch in a TSV.\n",
    "    CSVLogger(os.path.join(save_path, 'log.tsv'), separator='\\t'),\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we start the training and output its final test loss, accuracy, and micro F1-score.\n",
    "\n",
    "> Note that the F1-score is quite similar to the accuracy since the dataset is very balanced."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[35mEpoch: \u001b[36m 1/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.26s \u001b[35mloss:\u001b[94m 4.984878\u001b[35m acc:\u001b[94m 4.614534\u001b[35m fscore_micro:\u001b[94m 0.046145\u001b[35m val_loss:\u001b[94m 4.329983\u001b[35m val_acc:\u001b[94m 10.356255\u001b[35m val_fscore_micro:\u001b[94m 0.103563\u001b[0m\n",
      "Epoch 1: val_acc improved from -inf to 10.35626, saving file to saves/cub200/best_epoch_1.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 2/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.25s \u001b[35mloss:\u001b[94m 3.730639\u001b[35m acc:\u001b[94m 23.984526\u001b[35m fscore_micro:\u001b[94m 0.239845\u001b[35m val_loss:\u001b[94m 3.651718\u001b[35m val_acc:\u001b[94m 21.209611\u001b[35m val_fscore_micro:\u001b[94m 0.212096\u001b[0m\n",
      "Epoch 2: val_acc improved from 10.35626 to 21.20961, saving file to saves/cub200/best_epoch_2.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 3/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.28s \u001b[35mloss:\u001b[94m 2.916003\u001b[35m acc:\u001b[94m 43.188726\u001b[35m fscore_micro:\u001b[94m 0.431887\u001b[35m val_loss:\u001b[94m 3.165393\u001b[35m val_acc:\u001b[94m 30.571665\u001b[35m val_fscore_micro:\u001b[94m 0.305717\u001b[0m\n",
      "Epoch 3: val_acc improved from 21.20961 to 30.57167, saving file to saves/cub200/best_epoch_3.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 4/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.14s \u001b[35mloss:\u001b[94m 2.359695\u001b[35m acc:\u001b[94m 55.457309\u001b[35m fscore_micro:\u001b[94m 0.554573\u001b[35m val_loss:\u001b[94m 2.882293\u001b[35m val_acc:\u001b[94m 34.548467\u001b[35m val_fscore_micro:\u001b[94m 0.345485\u001b[0m\n",
      "Epoch 4: val_acc improved from 30.57167 to 34.54847, saving file to saves/cub200/best_epoch_4.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 5/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.01s \u001b[35mloss:\u001b[94m 1.985980\u001b[35m acc:\u001b[94m 64.548218\u001b[35m fscore_micro:\u001b[94m 0.645482\u001b[35m val_loss:\u001b[94m 2.743506\u001b[35m val_acc:\u001b[94m 36.454018\u001b[35m val_fscore_micro:\u001b[94m 0.364540\u001b[0m\n",
      "Epoch 5: val_acc improved from 34.54847 to 36.45402, saving file to saves/cub200/best_epoch_5.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 6/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.02s \u001b[35mloss:\u001b[94m 1.687166\u001b[35m acc:\u001b[94m 71.566731\u001b[35m fscore_micro:\u001b[94m 0.715667\u001b[35m val_loss:\u001b[94m 2.572532\u001b[35m val_acc:\u001b[94m 40.347970\u001b[35m val_fscore_micro:\u001b[94m 0.403480\u001b[0m\n",
      "Epoch 6: val_acc improved from 36.45402 to 40.34797, saving file to saves/cub200/best_epoch_6.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 7/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m4.92s \u001b[35mloss:\u001b[94m 1.463140\u001b[35m acc:\u001b[94m 76.291793\u001b[35m fscore_micro:\u001b[94m 0.762918\u001b[35m val_loss:\u001b[94m 2.474046\u001b[35m val_acc:\u001b[94m 42.170671\u001b[35m val_fscore_micro:\u001b[94m 0.421707\u001b[0m\n",
      "Epoch 7: val_acc improved from 40.34797 to 42.17067, saving file to saves/cub200/best_epoch_7.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 8/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.15s \u001b[35mloss:\u001b[94m 1.296519\u001b[35m acc:\u001b[94m 79.856314\u001b[35m fscore_micro:\u001b[94m 0.798563\u001b[35m val_loss:\u001b[94m 2.370366\u001b[35m val_acc:\u001b[94m 43.330572\u001b[35m val_fscore_micro:\u001b[94m 0.433306\u001b[0m\n",
      "Epoch 8: val_acc improved from 42.17067 to 43.33057, saving file to saves/cub200/best_epoch_8.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 9/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m4.92s \u001b[35mloss:\u001b[94m 1.146529\u001b[35m acc:\u001b[94m 82.508980\u001b[35m fscore_micro:\u001b[94m 0.825090\u001b[35m val_loss:\u001b[94m 2.339533\u001b[35m val_acc:\u001b[94m 43.744822\u001b[35m val_fscore_micro:\u001b[94m 0.437448\u001b[0m\n",
      "Epoch 9: val_acc improved from 43.33057 to 43.74482, saving file to saves/cub200/best_epoch_9.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m10/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.33s \u001b[35mloss:\u001b[94m 1.041320\u001b[35m acc:\u001b[94m 86.156397\u001b[35m fscore_micro:\u001b[94m 0.861564\u001b[35m val_loss:\u001b[94m 2.260914\u001b[35m val_acc:\u001b[94m 47.307374\u001b[35m val_fscore_micro:\u001b[94m 0.473074\u001b[0m\n",
      "Epoch 10: val_acc improved from 43.74482 to 47.30737, saving file to saves/cub200/best_epoch_10.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m11/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.07s \u001b[35mloss:\u001b[94m 0.938370\u001b[35m acc:\u001b[94m 87.759049\u001b[35m fscore_micro:\u001b[94m 0.877590\u001b[35m val_loss:\u001b[94m 2.215440\u001b[35m val_acc:\u001b[94m 46.810273\u001b[35m val_fscore_micro:\u001b[94m 0.468103\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m12/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.47s \u001b[35mloss:\u001b[94m 0.857947\u001b[35m acc:\u001b[94m 89.195910\u001b[35m fscore_micro:\u001b[94m 0.891959\u001b[35m val_loss:\u001b[94m 2.191947\u001b[35m val_acc:\u001b[94m 47.224523\u001b[35m val_fscore_micro:\u001b[94m 0.472245\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m13/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.79s \u001b[35mloss:\u001b[94m 0.783668\u001b[35m acc:\u001b[94m 91.710417\u001b[35m fscore_micro:\u001b[94m 0.917104\u001b[35m val_loss:\u001b[94m 2.191301\u001b[35m val_acc:\u001b[94m 47.473074\u001b[35m val_fscore_micro:\u001b[94m 0.474731\u001b[0m\n",
      "Epoch 13: val_acc improved from 47.30737 to 47.47307, saving file to saves/cub200/best_epoch_13.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m14/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.51s \u001b[35mloss:\u001b[94m 0.726757\u001b[35m acc:\u001b[94m 92.567008\u001b[35m fscore_micro:\u001b[94m 0.925670\u001b[35m val_loss:\u001b[94m 2.163284\u001b[35m val_acc:\u001b[94m 47.555924\u001b[35m val_fscore_micro:\u001b[94m 0.475559\u001b[0m\n",
      "Epoch 14: val_acc improved from 47.47307 to 47.55592, saving file to saves/cub200/best_epoch_14.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m15/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.41s \u001b[35mloss:\u001b[94m 0.671790\u001b[35m acc:\u001b[94m 93.561757\u001b[35m fscore_micro:\u001b[94m 0.935618\u001b[35m val_loss:\u001b[94m 2.143682\u001b[35m val_acc:\u001b[94m 47.638774\u001b[35m val_fscore_micro:\u001b[94m 0.476388\u001b[0m\n",
      "Epoch 15: val_acc improved from 47.55592 to 47.63877, saving file to saves/cub200/best_epoch_15.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m16/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.38s \u001b[35mloss:\u001b[94m 0.629114\u001b[35m acc:\u001b[94m 94.667035\u001b[35m fscore_micro:\u001b[94m 0.946670\u001b[35m val_loss:\u001b[94m 2.130498\u001b[35m val_acc:\u001b[94m 49.047224\u001b[35m val_fscore_micro:\u001b[94m 0.490472\u001b[0m\n",
      "Epoch 16: val_acc improved from 47.63877 to 49.04722, saving file to saves/cub200/best_epoch_16.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m17/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.76s \u001b[35mloss:\u001b[94m 0.580824\u001b[35m acc:\u001b[94m 95.717049\u001b[35m fscore_micro:\u001b[94m 0.957170\u001b[35m val_loss:\u001b[94m 2.116311\u001b[35m val_acc:\u001b[94m 47.224524\u001b[35m val_fscore_micro:\u001b[94m 0.472245\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m18/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.37s \u001b[35mloss:\u001b[94m 0.547148\u001b[35m acc:\u001b[94m 96.076264\u001b[35m fscore_micro:\u001b[94m 0.960763\u001b[35m val_loss:\u001b[94m 2.112233\u001b[35m val_acc:\u001b[94m 47.638774\u001b[35m val_fscore_micro:\u001b[94m 0.476388\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m19/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.20s \u001b[35mloss:\u001b[94m 0.512744\u001b[35m acc:\u001b[94m 96.822327\u001b[35m fscore_micro:\u001b[94m 0.968223\u001b[35m val_loss:\u001b[94m 2.075002\u001b[35m val_acc:\u001b[94m 49.295775\u001b[35m val_fscore_micro:\u001b[94m 0.492958\u001b[0m\n",
      "Epoch 19: val_acc improved from 49.04722 to 49.29577, saving file to saves/cub200/best_epoch_19.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m20/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.42s \u001b[35mloss:\u001b[94m 0.489004\u001b[35m acc:\u001b[94m 97.236806\u001b[35m fscore_micro:\u001b[94m 0.972368\u001b[35m val_loss:\u001b[94m 2.077726\u001b[35m val_acc:\u001b[94m 50.041425\u001b[35m val_fscore_micro:\u001b[94m 0.500414\u001b[0m\n",
      "Epoch 20: val_acc improved from 49.29577 to 50.04142, saving file to saves/cub200/best_epoch_20.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m21/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.49s \u001b[35mloss:\u001b[94m 0.460475\u001b[35m acc:\u001b[94m 97.927604\u001b[35m fscore_micro:\u001b[94m 0.979276\u001b[35m val_loss:\u001b[94m 2.081020\u001b[35m val_acc:\u001b[94m 48.881524\u001b[35m val_fscore_micro:\u001b[94m 0.488815\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m22/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.49s \u001b[35mloss:\u001b[94m 0.434759\u001b[35m acc:\u001b[94m 98.038132\u001b[35m fscore_micro:\u001b[94m 0.980381\u001b[35m val_loss:\u001b[94m 2.059778\u001b[35m val_acc:\u001b[94m 49.461475\u001b[35m val_fscore_micro:\u001b[94m 0.494615\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m23/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.32s \u001b[35mloss:\u001b[94m 0.414827\u001b[35m acc:\u001b[94m 98.535507\u001b[35m fscore_micro:\u001b[94m 0.985355\u001b[35m val_loss:\u001b[94m 2.064994\u001b[35m val_acc:\u001b[94m 49.710025\u001b[35m val_fscore_micro:\u001b[94m 0.497100\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m24/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.73s \u001b[35mloss:\u001b[94m 0.397676\u001b[35m acc:\u001b[94m 98.618403\u001b[35m fscore_micro:\u001b[94m 0.986184\u001b[35m val_loss:\u001b[94m 2.051294\u001b[35m val_acc:\u001b[94m 50.041425\u001b[35m val_fscore_micro:\u001b[94m 0.500414\u001b[0m\n",
      "Epoch 24: val_acc improved from 50.04142 to 50.04143, saving file to saves/cub200/best_epoch_24.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m25/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.46s \u001b[35mloss:\u001b[94m 0.379117\u001b[35m acc:\u001b[94m 98.839458\u001b[35m fscore_micro:\u001b[94m 0.988395\u001b[35m val_loss:\u001b[94m 2.049864\u001b[35m val_acc:\u001b[94m 50.372825\u001b[35m val_fscore_micro:\u001b[94m 0.503728\u001b[0m\n",
      "Epoch 25: val_acc improved from 50.04143 to 50.37283, saving file to saves/cub200/best_epoch_25.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m26/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.28s \u001b[35mloss:\u001b[94m 0.355746\u001b[35m acc:\u001b[94m 99.253938\u001b[35m fscore_micro:\u001b[94m 0.992539\u001b[35m val_loss:\u001b[94m 2.050693\u001b[35m val_acc:\u001b[94m 50.041425\u001b[35m val_fscore_micro:\u001b[94m 0.500414\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m27/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.43s \u001b[35mloss:\u001b[94m 0.346945\u001b[35m acc:\u001b[94m 99.253938\u001b[35m fscore_micro:\u001b[94m 0.992539\u001b[35m val_loss:\u001b[94m 2.038936\u001b[35m val_acc:\u001b[94m 50.372825\u001b[35m val_fscore_micro:\u001b[94m 0.503728\u001b[0m\n",
      "Epoch 27: val_acc improved from 50.37283 to 50.37283, saving file to saves/cub200/best_epoch_27.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m28/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.39s \u001b[35mloss:\u001b[94m 0.330237\u001b[35m acc:\u001b[94m 99.419729\u001b[35m fscore_micro:\u001b[94m 0.994197\u001b[35m val_loss:\u001b[94m 2.027896\u001b[35m val_acc:\u001b[94m 49.710025\u001b[35m val_fscore_micro:\u001b[94m 0.497100\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m29/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.60s \u001b[35mloss:\u001b[94m 0.317088\u001b[35m acc:\u001b[94m 99.447361\u001b[35m fscore_micro:\u001b[94m 0.994474\u001b[35m val_loss:\u001b[94m 2.044203\u001b[35m val_acc:\u001b[94m 50.041425\u001b[35m val_fscore_micro:\u001b[94m 0.500414\u001b[0m\n",
      "\u001b[35mEpoch: \u001b[36m30/30 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.35s \u001b[35mloss:\u001b[94m 0.312700\u001b[35m acc:\u001b[94m 99.585521\u001b[35m fscore_micro:\u001b[94m 0.995855\u001b[35m val_loss:\u001b[94m 2.036675\u001b[35m val_acc:\u001b[94m 50.207125\u001b[35m val_fscore_micro:\u001b[94m 0.502071\u001b[0m\n",
      "Restoring model from saves/cub200/best_epoch_27.ckpt\n",
      "\u001b[35mTest steps: \u001b[36m19 \u001b[32m1.78s \u001b[35mtest_loss:\u001b[94m 2.002988\u001b[35m test_acc:\u001b[94m 51.118476\u001b[35m test_fscore_micro:\u001b[94m 0.511185\u001b[0m           \n"
     ]
    }
   ],
   "source": [
    "optimizer = optim.SGD(resnet18.fc.parameters(), lr=learning_rate, weight_decay=0.001)\n",
    "loss_function = nn.CrossEntropyLoss()\n",
    "\n",
    "model = Model(\n",
    "    resnet18,\n",
    "    optimizer,\n",
    "    loss_function,\n",
    "    batch_metrics=['accuracy'],\n",
    "    epoch_metrics=['f1'],\n",
    "    device=device,\n",
    ")\n",
    "\n",
    "model.fit_generator(\n",
    "    train_loader,\n",
    "    valid_loader,\n",
    "    epochs=n_epoch,\n",
    "    callbacks=callbacks,\n",
    ")\n",
    "\n",
    "test_loss, (test_acc, test_f1) = model.evaluate_generator(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    epoch      time   lr      loss        acc  fscore_micro  val_loss  \\\n",
      "0       1  5.261212  0.1  4.984878   4.614534      0.046145  4.329983   \n",
      "1       2  5.254282  0.1  3.730639  23.984526      0.239845  3.651718   \n",
      "2       3  5.275054  0.1  2.916003  43.188726      0.431887  3.165393   \n",
      "3       4  5.139137  0.1  2.359695  55.457309      0.554573  2.882293   \n",
      "4       5  5.009642  0.1  1.985980  64.548218      0.645482  2.743506   \n",
      "5       6  5.019162  0.1  1.687166  71.566731      0.715667  2.572532   \n",
      "6       7  4.915594  0.1  1.463140  76.291793      0.762918  2.474046   \n",
      "7       8  5.151500  0.1  1.296519  79.856314      0.798563  2.370366   \n",
      "8       9  4.922735  0.1  1.146529  82.508980      0.825090  2.339533   \n",
      "9      10  5.330284  0.1  1.041320  86.156397      0.861564  2.260914   \n",
      "10     11  5.067723  0.1  0.938370  87.759049      0.877590  2.215440   \n",
      "11     12  5.465988  0.1  0.857947  89.195910      0.891959  2.191947   \n",
      "12     13  5.788359  0.1  0.783668  91.710417      0.917104  2.191301   \n",
      "13     14  5.510693  0.1  0.726757  92.567008      0.925670  2.163284   \n",
      "14     15  5.409089  0.1  0.671790  93.561757      0.935618  2.143682   \n",
      "15     16  5.383382  0.1  0.629114  94.667035      0.946670  2.130498   \n",
      "16     17  5.758362  0.1  0.580824  95.717049      0.957170  2.116311   \n",
      "17     18  5.369471  0.1  0.547148  96.076264      0.960763  2.112233   \n",
      "18     19  5.204324  0.1  0.512744  96.822327      0.968223  2.075002   \n",
      "19     20  5.417673  0.1  0.489004  97.236806      0.972368  2.077726   \n",
      "20     21  5.486404  0.1  0.460475  97.927604      0.979276  2.081020   \n",
      "21     22  5.488888  0.1  0.434759  98.038132      0.980381  2.059778   \n",
      "22     23  5.315572  0.1  0.414827  98.535507      0.985355  2.064994   \n",
      "23     24  5.730207  0.1  0.397676  98.618403      0.986184  2.051294   \n",
      "24     25  5.460709  0.1  0.379117  98.839458      0.988395  2.049864   \n",
      "25     26  5.279990  0.1  0.355746  99.253938      0.992539  2.050693   \n",
      "26     27  5.429697  0.1  0.346945  99.253938      0.992539  2.038936   \n",
      "27     28  5.394186  0.1  0.330237  99.419729      0.994197  2.027896   \n",
      "28     29  5.602181  0.1  0.317088  99.447361      0.994474  2.044203   \n",
      "29     30  5.350323  0.1  0.312700  99.585521      0.995855  2.036675   \n",
      "\n",
      "      val_acc  val_fscore_micro  \n",
      "0   10.356255          0.103563  \n",
      "1   21.209611          0.212096  \n",
      "2   30.571665          0.305717  \n",
      "3   34.548467          0.345485  \n",
      "4   36.454018          0.364540  \n",
      "5   40.347970          0.403480  \n",
      "6   42.170671          0.421707  \n",
      "7   43.330572          0.433306  \n",
      "8   43.744822          0.437448  \n",
      "9   47.307374          0.473074  \n",
      "10  46.810273          0.468103  \n",
      "11  47.224523          0.472245  \n",
      "12  47.473074          0.474731  \n",
      "13  47.555924          0.475559  \n",
      "14  47.638774          0.476388  \n",
      "15  49.047224          0.490472  \n",
      "16  47.224524          0.472245  \n",
      "17  47.638774          0.476388  \n",
      "18  49.295775          0.492958  \n",
      "19  50.041425          0.500414  \n",
      "20  48.881524          0.488815  \n",
      "21  49.461475          0.494615  \n",
      "22  49.710025          0.497100  \n",
      "23  50.041425          0.500414  \n",
      "24  50.372825          0.503728  \n",
      "25  50.041425          0.500414  \n",
      "26  50.372825          0.503728  \n",
      "27  49.710025          0.497100  \n",
      "28  50.041425          0.500414  \n",
      "29  50.207125          0.502071  \n",
      "Best epoch: 27\n"
     ]
    }
   ],
   "source": [
    "logs = pd.read_csv(os.path.join(save_path, 'log.tsv'), sep='\\t')\n",
    "print(logs)\n",
    "\n",
    "best_epoch_idx = logs['val_acc'].idxmax()\n",
    "best_epoch = int(logs.loc[best_epoch_idx]['epoch'])\n",
    "print(\"Best epoch: %d\" % best_epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAp1UlEQVR4nO3dd3gc5bn38e+zRbvqvctyw11yFQYDtunV9GKDqYFASKEkhyRvTnJCcsFJTkhIo4UACRDAOEAIoYRq4kJxtyUXbFxkS7J6r9ue949ZycKWbZVdz+7q/lzXXDM72+7xwm9H9zwzq7TWCCGECF0WswsQQghxdBLUQggR4iSohRAixElQCyFEiJOgFkKIEGcLxoumpaXpUaNGBeOlhRAiIq1bt65Wa53e131BCepRo0axdu3aYLy0EEJEJKVU6ZHuk9aHEEKEOAlqIYQIcRLUQggR4oLSoxZCDD9ut5uysjI6OzvNLiWkOZ1O8vLysNvt/X5Ov4JaKbUXaAG8gEdrXTSoCoUQEausrIz4+HhGjRqFUsrsckKS1pq6ujrKysoYPXp0v583kD3qM7TWtQMvTQgxHHR2dkpIH4NSitTUVGpqagb0POlRCyECRkL62Abzb9TfoNbAe0qpdUqp2wf8Lv3g8vh4/ONdrNg5sG8aIYSIdP0N6tO01jOBC4BvKaXmHfoApdTtSqm1Sqm1A92tB7BbFX9avou3Nh8Y8HOFEAIgLi7O7BKCol9BrbUu98+rgX8As/t4zJNa6yKtdVF6ep9nQR6VUorC3ESKy5sG/FwhhIhkxwxqpVSsUiq+exk4FygJRjFTchLZUdVCl8cbjJcXQgwTWmvuu+8+CgoKKCws5OWXXwbgwIEDzJs3j+nTp1NQUMCKFSvwer3cfPPNPY/97W9/a3L1h+vPqI9M4B/+BrgNeFFr/e9gFFOYm4jbq9lR2UphXmIw3kIIcRz87F9b2FrRHNDXnJyTwE8vntKvx7722mts3LiRTZs2UVtby4knnsi8efN48cUXOe+88/jv//5vvF4v7e3tbNy4kfLyckpKjP3PxsbGgNYdCMcMaq31bmDacaiFwlwjnIvLmySohRCDtnLlSq699lqsViuZmZnMnz+fNWvWcOKJJ/K1r30Nt9vNZZddxvTp0xkzZgy7d+/mO9/5DhdddBHnnnuu2eUfJqTOTByREk2C0yZ9aiHCXH/3fI+3efPmsXz5ct566y1uvvlmvvvd73LjjTeyadMm3n33XZ544gmWLl3KM888Y3apXxFS46iVUhTkJrKlQoJaCDF4c+fO5eWXX8br9VJTU8Py5cuZPXs2paWlZGZm8vWvf53bbruN9evXU1tbi8/n48orr+SBBx5g/fr1Zpd/mJDaowaj/fGXVXtxeXxE2ULqe0QIESYuv/xyPv30U6ZNm4ZSil/96ldkZWXx7LPP8tBDD2G324mLi+O5556jvLycW265BZ/PB8AvfvELk6s/nNJaB/xFi4qK9GB/OOCNTRXc9dIG3rrrNKbkSJ9aiHCxbds2Jk2aZHYZYaGvfyul1LojXUcp5HZZuw8olkifWgghgBAM6pEpMcQ55ICiEEJ0C7mgtlgUU3ISKCkP7BhMIYQIVyEX1GC0P7YdaMbj9ZldihBCmC4kg7ogN5Euj4+d1a1mlyKEEKYL2aAGOaAohBAQokE9Ji2W2CirBLUQQhCiQW2xKCbnJFAS4Iu6CCFEt6Ndu3rv3r0UFBQcx2qOLiSDGoz2x9aKZry+wJ+QI4QQ4STkTiHvVpCTSId7L7tqWhmfGW92OUKIgXjnh1BZHNjXzCqEC355xLt/+MMfMmLECL71rW8BcP/992Oz2Vi2bBkNDQ243W4eeOABLr300gG9bWdnJ3feeSdr167FZrPx8MMPc8YZZ7BlyxZuueUWXC4XPp+PV199lZycHK655hrKysrwer385Cc/YeHChUPabAjhoO6+zGlJeZMEtRDimBYuXMg999zTE9RLly7l3Xff5a677iIhIYHa2lpOPvlkLrnkkgH9wOyjjz6KUori4mK2b9/Oueeey44dO3jiiSe4++67Wbx4MS6XC6/Xy9tvv01OTg5vvfUWAE1NgTnOFrJBPTY9DqfdQnF5E1fMzDO7HCHEQBxlzzdYZsyYQXV1NRUVFdTU1JCcnExWVhb33nsvy5cvx2KxUF5eTlVVFVlZWf1+3ZUrV/Kd73wHgIkTJzJy5Eh27NjBnDlzePDBBykrK+OKK65g3LhxFBYW8r3vfY8f/OAHLFiwgLlz5wZk20K2R221KCZnJ8jIDyFEv1199dW88sorvPzyyyxcuJAXXniBmpoa1q1bx8aNG8nMzKSzszMg73XdddfxxhtvEB0dzYUXXshHH33E+PHjWb9+PYWFhfz4xz/m5z//eUDeK2SDGowzFLdUNOOTA4pCiH5YuHAhS5Ys4ZVXXuHqq6+mqamJjIwM7HY7y5Yto7S0dMCvOXfuXF544QUAduzYwb59+5gwYQK7d+9mzJgx3HXXXVx66aVs3ryZiooKYmJiuP7667nvvvsCdm3rkG19gDHy49lPS9ld28YJGZH5M/BCiMCZMmUKLS0t5Obmkp2dzeLFi7n44ospLCykqKiIiRMnDvg1v/nNb3LnnXdSWFiIzWbjr3/9Kw6Hg6VLl/L8889jt9vJysriRz/6EWvWrOG+++7DYrFgt9t5/PHHA7JdIXc96t62HWjmgt+v4HcLp3PZjNwAVCaECBa5HnX/hf31qHsblxGHw2aRPrUQYlgLvdaH1uAfOmOzWpiUnSDXphZCBEVxcTE33HDDV9Y5HA4+//xzkyrqW+gEdVcrvLQIJl8Ks7/es7ogN4F/bqjA59NYLP0f+yiEOP601gMao2y2wsJCNm7ceFzfczDt5tBpfTjioLMJNr74ldWFuYm0dHkorW83qTAhRH84nU7q6uoGFUTDhdaauro6nE7ngJ4XOnvUANMWwbs/gpodkD4eoOcHbovLmxidFmtmdUKIo8jLy6OsrIyamhqzSwlpTqeTvLyBncQXWkFdcBW89xPYvATO+h8AxmfGE2W1sKW8iUum5ZhcoBDiSOx2O6NHjza7jIgUOq0PgPhMGHsmbHoZfMbPcEXZLEzMjpcDikKIYSu0ghqM9kdzGZSu7Fk1JSeRkvIm6X0JIYal0AvqiReBI8HYq/YrzE2kudPD/voOEwsTQghzhF5Q26Nh8iWw9XVwGSM9CnMPHlAUQojhJvSCGmDateBqhe3GNV3HZ8VhtypKKiSohRDDT2gGdf4pkJhvjP4AHDYr4zPj5VRyIcSwFJpBbbHA1Gtg10fQUgkYP81VLAcUhRDDUGgGNRijP7QPiv8OQEFeIo3tbsob5YCiEGJ46XdQK6WsSqkNSqk3g1lQj7RxkFvUM/qj+4CitD+EEMPNQPao7wa2BauQPk1bBFXFUFnCxKx4rBYlIz+EEMNOv4JaKZUHXAQ8FdxyDjHlCrDYYfMSnHYr4zLiKClvPq4lCCGE2fq7R/074PuA70gPUErdrpRaq5RaG7CLssSmwrhzYfNS8HoozJUzFIUQw88xg1optQCo1lqvO9rjtNZPaq2LtNZF6enpASuQaYugtQr2fExBbiJ1bS4qmwPzK8JCCBEO+rNHfSpwiVJqL7AEOFMp9begVtXb+PPAmQSbXqag+wzFMulTCyGGj2MGtdb6/2mt87TWo4BFwEda6+uDXlk3mwMKroBt/2JyisKiZOSHEGJ4Cd1x1L1NuxY8HUR/+RYnZMRRUiEHFIUQw8eAglpr/bHWekGwijmivBMhZQxsXkJBbqIM0RNCDCvhsUetFExdBHtWcFJKOzUtXVTJAUUhxDARHkENxrU/0Jza/hEgfWohxPARPkGdMhry55BT+k+U0tL+EEIMG+ET1ADTFmGp28H5yQdkj1oIMWyEV1BPvgysDhY5PpVTyYUQw0Z4BXV0Eky4gJNaP6K2uZV9de1mVySEEEEXXkENMO1anO4G5ls28cG2KrOrEUKIoAu/oD7hLIhJ48aYzySohRDDQvgFtdUOhVdxqnc1u/fsoqndbXZFQggRVOEX1ACzb8eKj9stb/DxjmqzqxFCiKAKz6BOHYuefh2LbR+ydnOJ2dUIIURQhWdQA5Z592FVmim7n8LlOeLvGQghRNgL26AmeSQHRl/FFfpDNpcUm12NEEIETfgGNZB24Y8AhWXlQ2aXIoQQQRPWQR2dNpLlCQuYWvsWum632eUIIURQhHVQA7SceBcebaX53f81uxQhhAiKsA/qU6dP4XnvOcTveBVqvzS7HCGECLiwD+qMBCcrMq/HhR3+80uzyxFCiIAL+6AGmD1lPH/xnIsufgWqt5ldjhBCBFREBPXZkzP5k2cBHlsMfCx71UKIyBIRQT0hM5645Az+HXc5bH0dKmVctRAickREUCulOHtSJj+vPR3tSJC9aiFERImIoAY4Z3ImNZ4Ydp1wC2x/Eyo2mF2SEEIERMQE9ezRKcQ7bTznuwCik2GZjKsWQkSGiAlqu9XC6RMyeHtnK745d8HO92D/GrPLEkKIIYuYoAY4e1IGta0uNudeAzFpsOxBs0sSQoghi6igPn18BjaL4r2drXDaPbB7GZR+YnZZQggxJBEV1IkxdmaPTjF+S7HoVojLlF61ECLsRVRQA5w9KZMdVa2Utmg47buwdwXsWmZ2WUIIMWgRGdQAH2yrhlk3Q9JI+Mcd0FBqbmFCCDFIERfU+akxjM+M44OtVWB3wnVLwd0JL1wNHQ1mlyeEEAMWcUENxl716r31NLW7IWMiLHoB6nfDkuvB02V2eUIIMSCRGdSTM/H6NB/vqDZWjJ4Llz0OpSvh9W+CT34MVwgRPo4Z1Eopp1JqtVJqk1Jqi1LqZ8ejsKGYnpdEWlwU72+tOrhy6tVw1k+h5BX46OfmFSeEEANk68djuoAztdatSik7sFIp9Y7W+rMg1zZoFovirImZvF18AJfHR5TN/3102r3QuA9W/hYSR8CJt5pbqBBC9MMx96i1odV/0+6fdFCrCoCzJ2fS0uVh9Z76gyuVggt/DePOg7f/C774t3kFCiFEP/WrR62UsiqlNgLVwPta68/7eMztSqm1Sqm1NTU1AS5z4E47IQ2HzWKc/NKb1QZXPQNZU+GVW6B8vTkFCiFEP/UrqLXWXq31dCAPmK2UKujjMU9qrYu01kXp6ekBLnPgoqOszB2Xxvtbq9D6kD8AHHHGsL2YNHjxGmjYa0qNQgjRHwMa9aG1bgSWAecHpZoAO3tSJuWNHWyvbDn8zvhMuP4V8LqNMdbt9Yc/RgghQkB/Rn2kK6WS/MvRwDnA9iDXFRBnTsoA4J2Syr4fkD4BFr1o7FEvWWycGCOEECGmP3vU2cAypdRmYA1Gj/rN4JYVGBnxTuaPT+el1fvo8nj7ftCoU40x1vs+gddug86m41ukEEIcQ39GfWzWWs/QWk/VWhdorcNqEPJtc0dT09LFvzYdOPKDCq+Ccx+EbW/CH2bA6j8bLREhhAgBEXlmYm+nnZDGhMx4nlqx+/CDir2d8m244z+QMdkYuvfYHGP43tGeI4QQx0HEB7VSilvnjmZ7ZQurvqw7+oOzp8FN/4JFLwEaXloIz10CBzYfl1qFEKIvER/UAJdOzyEtzsFTK3cf+8FKwcQL4ZufwQUPQWUJ/GmecY2Q5orgFyuEEIcYFkHtsFm5cc5IPv6ihp1VfQzV64vVDifdDndtMNoixX+HP84yfjGmq/XYzxdCiAAZFkENsPikfBw2C8+s2jOwJ0YnwbkPwLfXwPjz4D//ZwR2yWtBqVMIIQ41bII6Nc7BlbPyeHV9OXWtg7gmdfIouPqvcOv7kJBtnH7+yq1yoowQIuiGTVADfO3U0bg8Pv722b7Bv8iI2XDrB3DGj2Hr6/D4KfDlhwGrUQghDjWsgvqEjDjOnJjB85/tpdN9hBNg+sNqg/n3wW0fgCMB/nYFvPU9cLUFrlghhPAbVkENcNtpo6ltdfHPjeVDf7GcGcbY6znfhjVPwxNzYf+aob+uEEL0MuyCes7YVCZlJ/DUij1HPwGmv+zRcN6DxvhrrwueORc+egA8rqG/thBCMAyDWinFbaeNZmd1K8t31gbuhUfPhTtXwbRrYflD8NRZUL0tcK8vhBi2hl1QA1w8LYeMeAdPrejHCTAD4UyEyx6DhS8YJ8f8aT6seFgu9CSEGJJhGdRRNgs3nTKKFTtr+aKva1UP1aQFxpmNJ5wNH/4MfjMRXv+W0b+Wa4cIIQZoWAY1GCfARNutPN2f08oHIy4dFr0AX18GhVfDln/A02cbw/k+/xN0NATnfYUQEWfYBnVSTBRXzcrj9Q0V1LQM4gSY/lAKcmfCJX+A//oCLv492BzwzveNvezX7oDST2UvWwhxVMM2qAFuOXUUbp+P5z/dG/w3c8TDrJvh9o/hjuUwfTFsfwv+cj48ehJ8+qic5SiE6NOwDuox6XGcNTGT5z8rHdoJMAOVPQ0WPGzsZV/yCDgT4N0fwcOTjF52xYbjV4sQIuQN66AG4xdgGtrdvLY+ACfADFRULMy8wTjD8Rv+oX1bXoMnT4c/nwWblsjvOAohJKhPGp1CYW4iT6/cjc9nYq84qwAu/h18bzuc/3/GkL5/3AG/nQwf3A+NQ7g+iRAirA37oFZKcdvc0eyqaeM/O2rMLscYi33yN4zLqt7wOuTPgVW/h99Pg5euNS4A5fOZXaUQ4jiymV1AKLiwMJtfvL2dP6/YzRkTM8wux6AUjD3DmBr3w7q/wLpn4Yu3IT4bUk+ApPyvTokjICHXuGiUECJiyP/RgN1q4dbTRvPg29t4f2sV50zONLukr0oaAWf9D8z/AWz9J+x8z2iF7PoIWiqBXi0bZTXCuju880+GCRca47qFEGFJBeTCRIcoKirSa9euDfjrBpPL4+OSR1ZS1+bi/XvnkRQTZXZJ/ePpgqYyI7gb90HT/oPL9buhtQqUxWihTFxgnDWZlG921UKIQyil1mmti/q8T4L6oJLyJi59dBWXTs/h4Wumm13O0GkNlcWw/U3Y9iZUbzHWZ02FSRcbwZ0xyWizCCFMJUE9AA+/9wV/+OhLnr6piLMmhVgLZKjqdh0M7bLVxrqUMUZgjz8fsqcaJ+YIIY47CeoB6G6BNLS7eO+e+STG2M0uKThaKo0zI7e/CXuWg89jrE8eBZkFkFXonxdA0kjZ6xYiyCSoB6i4rInLHlvF5TNy+fXV08wuJ/g6GmDf51BVDJUlUFVi7H13H6R0JEDmlF7BnQ9xmRCXBdHJYBn2ozyFGLKjBbWM+uhDYV4id84fyyPLvuSiwuzQGbIXLNHJMOF8Y+rmajN++KCy2Ajuqi3GmZJrDrksrMUGsRkQn+kP7wz/PNN4XaUA/974EZctEJMKibnG0ENrhP4VI8QgyR71EXR5vFz8x5U0d3h49955JEZLeKC1MZqkuQJaK6G12hhV0lJlzLunthrQgz0pRxkhn5hrDDNMzPPPcyEhDxKyISYN7M6AbpoQZpPWxyBt2t/I5Y+t4qpZefzqqmHQAgkUnxfa64yWSs9/X/5579vdy9pnhHtzOTSVQ3OZf+6/7e7j193tscZeeEwKxKb5l/23Y/y3nQnGwdGoeGPuiDOeJ60aEYKk9TFI00Ykccf8sTz+8S4uLMzm9AkR3gIJFIvV3wIJwL+X1tDZeDC4Ww4YXwLt9ca8rdaY1+401rmO9Ys9CqLiDga3I77XlHgw3B0JhywnGnNblNHu6ZmsYLF/9bYceBUBJnvUx9Dp9rLgjytp6zJaIAlOaYGENHcndPhDvLMZXK3Q1QJdzdDVvdxiBHpXi39ds7Hc2XzwvqFQVuNLIGUspI0zTvfvmcYaV00U4hDS+hiiDfsauPLxT7imaAS/vHKq2eWIYPN5DwZ6V/PBAO9qBq/LGMro84DXc3DZ5zGe53Mbyx2NUL8Lar80Wjm9JeQagZ06zpjbnEb7x+c15trrX/bf9h2yzucx1vk8B9f7PP7H+ufORIhNN1pAsem9pjRwJkn7JwQNqfWhlBoBPAdkYjQan9Ra/z6wJYa2GfnJfH3eGP70n91cWJjNvPFy3YyIZrFCdJIxBYKr3Qjtui+NqdY/L3llYL9QryxGe0VZ/W0WS69l/1xZjKmzyfjLoi8Wm9HHj0039vy9buMLxus+fLn7i8nrBnu00f7p3S5y9r7tX7bHGO0fZQHUIcsW/+3uUT+HHLvQmoPHL3qts1iNv0Si4vxTbK8pzmhJBYPXbXxRWyxgdRhfqiZ8yR1zj1oplQ1ka63XK6XigXXAZVrrrUd6TqTtUYPRArnoDyvocHl59955xEsLRAyV1sYBV6/LH7jWg0FrsR6+bqC9b6/H38+vNQ7WtnXPaw7e7moxhkNa7Ma8ZznKuAqjNcrfg7eCp/PgXxqdTb3+6vBPno7g/Dv1hzXqYGjbYw6GeM9yTK/7/MsoYzs6G42/gA6bN/V9INtiNwLbFuWf+wPcGmWMWFq8dFCbMKQ9aq31AeCAf7lFKbUNyAWOGNSRyGm38tDV07jq8U/437e384srCs0uSYQ7pYxRKsFitRnj2+OP06UQvG4jsN3tvfaMfV/dS9b+dd33de9xQ6/lXuu6b/s8xtj+nqml13Krcayhe9nVZtTgajcOPncvu9qM4O0+C7dbVLzRKopOMtpCKWOMefc6R4JRr6fTuAhaz+S/7e21HBUTlH/aAY36UEqNAmYAnwelmhA3Mz+Z2+aO4cnluzm/IIv50gIR4iCr3f/FE8Qvn0DwuIxA19oI4zC4fnu/my1KqTjgVeAerXVzH/ffrpRaq5RaW1MTAr+UEiTfPWc84zPjuHvJBvbW9vFnkRAitNmi/OPvU8MipKGfQa2UsmOE9Ata69f6eozW+kmtdZHWuig9PXL3NJ12K3++0Wgj3fbcWpo73SZXJISIdMcMaqWUAp4GtmmtHw5+SaFvZGosjy+exd7aNu56aQNeM38UVwgR8fqzR30qcANwplJqo3+6MMh1hbw5Y1P5+aUFfPxFDb94e5vZ5QghIlh/Rn2spOcyZ6K3607KZ0dVC0+t3MO4zDgWnig/cSWECDw5PWmIfnzRJOaOS+PHr5ewes8RTjAQQoghkKAeIpvVwiPXzWREcgzf+Ns69te3m12SECLCSFAHQGK0naduKsLj9XHbs2tp7fIc+0lCCNFPEtQBMiY9jscWz+LLmlbuWSIjQYQQgSNBHUCnjUvjpxdP5oNt1Tz07hdmlyOEiBDhcVpOGLlxzih2VLXwxH92MS4jjitn5ZldkhAizMkedRD89OIpnDI2lf/3WjHrShvMLkcIEeYkqIPAbrXw2OKZZCc5uf25tZSUD+Caw0IIcQgJ6iBJioniLzefiMNmYdGTn/HJl7VmlySECFMS1EE0Jj2O1755KrlJ0dz0l9W8ubnC7JKEEGFIgjrIshKdLL1jDjNGJPOdlzbw11V7zC5JCBFmJKiPg8QYO8/dOptzJmVy/7+28qt/bycYPyoshIhMEtTHidNu5bHFM7l2dj6PfbyL77+yGY/XZ3ZZQogwIOOojyOb1cL/Xl5ARryD33+4k7o2F49eN5PoKKvZpQkhQpjsUR9nSinuPWc8D1xWwLIvqrnuqc9oaHOZXZYQIoRJUJvk+pNH8vjimWypaOaqJz6hvLHD7JKEECFKgtpE5xdk89zXZlPd0sUVj62SE2OEEH2SoDbZyWNS+fs35qBQXPboKn73wQ7ccpBRCNGLBHUImJiVwL/vmcuCqdn87oOdXPrIKrYdaDa7LCFEiJCgDhFJMVH8btEMnrh+FtUtnVzyyEoe+WinDOETQkhQh5rzC7J47975nDcli1+/t4PLH/uEHVUtZpclhDCRBHUISomN4pHrZvLodTMpb+xgwR9W8vjHu2TvWohhSoI6hF00NZv37p3HWZMy+L9/b+eqJz7ly+pWs8sSQhxnEtQhLi3OwWOLZ/KHa2ewt66NC/+wgj/9R/auhRhOJKjDgFKKS6bl8N6985g/Pp1fvLOdBX9cyZq99WaXJoQ4DiSow0hGvJMnb5jFE9fPornDzdVPfMp3l26kpqXL7NKEEEEkQR1mlFKcX5DFB9+bzzdPH8u/NlVw5m8+5rlP9+L1yaVThYhEEtRhKibKxvfPn8g7d89jal4i//PPLVzyyErW75Mf0xUi0khQh7kTMuL4260n8cdrZ1Db2sUVj33CD1/dLFfkEyKCSFBHAKUUF0/L4cPvnc7X547m7+vKOOM3H/PS6n34pB0iRNhTwfhJqKKiIr127dqAv67ony8qW/jJ6yWs3lvPqNQYbpgziqtm5ZEYbTe7NCHEESil1mmti/q8T4I6Mmmtebu4kmdW7WFdaQMxUVYun5HLjXNGMSEr3uzyhBCHkKAe5krKm3j2k738c1MFLo+POWNSuemUkZw9KRObVbpfQoQCCWoBQH2bi5fX7Odvn5VS3thBblI0i0/OZ9GJ+aTERpldnhDD2pCCWin1DLAAqNZaF/TnDSWoQ5vH6+PD7dU8+8lePtlVR5TNwsVTc7hxzkimjUgyuzwhhqWhBvU8oBV4ToI68uysauHZT/fy2vpy2l1epuYlcsPJI7l4Wg5Ou/w6uhDHy5BbH0qpUcCbEtSRq6XTzT82lPP8p6XsrG4lMdrONUV5XH/ySEamxppdnhAR77gEtVLqduB2gPz8/FmlpaWDq1aYSmvNZ7vr+dtnpby7pRKPTzN/fDo3nDySMyZmYLUos0sUIiLJHrUYlKrmTpas3s+Lq0upau4iNyma607K58qZeWQlOs0uT4iIIkEthsTt9fHB1iqe/6yUT3bVATAjP4kLCrK4oCCbESkxJlcoRPiToBYBs7umlXdKKnmn5AAl5cYvpU/JSeCCgizOL8jmhIw4kysUIjwNddTHS8DpQBpQBfxUa/300Z4jQT087K9v59/+0F6/rxGAcRlxPaE9KTsepaSnLUR/yAkvIugqmzp5d4sR2qv31OPTMDI1hrMmZnLWpAxOHJVClE3OghTiSCSoxXFV29rF+1ureG9LJat21eHy+Ih32Jg3IZ2zJmZwxoQMkuVMSCG+QoJamKbd5WHVl3V8uK2KD7dXU9PShUXBrJHJnDkxk7MnZXBCRpy0SMSwJ0EtQoLPpympaOKDbdV8uK2KLRXGwcj8lBhOGZvKzJHJFI1MZnRarAS3GHYkqEVIqmzq5MPtVSzbXs2avQ00dbgBSI2NYubIZGb5g7sgN1FOZxcRT4JahDyfT7OrppV1pQ2sLW1gXWkDe2rbAIiyWijMS2SWP7xn5ieTHu8wuWIhAkuCWoSl2tYu1vtDe21pA8VlTbi8PgDykqOZmZ/MzPwkZo5MZlJ2Ana5trYIYxLUIiJ0ur1sqWhiw75G1u9rYH1pI5XNnQA4bBam5iUyMz+ZGflJzMxPJiNBTnMX4UOCWkSsisYO1u9r6AnvLeXNPXvdaXEOJmTFMT4zngmZ8YzPimd8ZjxxDpvJVQtxuKMFtfwXK8JaTlI0OUnRLJiaA0CXx0tJeTMb9jXwRWULO6paWLJ6Px1ub89z8pKje4J7QmY8E7LiOSEjTlonImRJUIuI4rBZew46dvP5NGUNHXxRZQR3d4Av31mD22v8RRllszAxK54pOYkU5CYwJSeRiVnxMtpEhARpfYhhy+31sbe2ja0Hmtla0UxJRRMl5c09wwStFsW4jLivhPfY9FhSYqNknLcIOOlRC9FPWht731sqmthS0UxJeRPF5c3Utnb1PCbOYSM/JYb8lBhGpsaQnxrDyJRY8lNiyElyyi+7i0GRHrUQ/aSUYkRKDCNSYji/ILtnfXVzJ1sqmtlT28a++nZK69rYWd3CR9urew5eAtgsitzk6J4QH5VqBPioNGMurRQxGBLUQvRDRoKTjAQnZxyy3ufTVDZ3UlrXzv76dkrr2yita2dffTtvbKygudPzlcdnJTjJT41hVGoMI/0hnp3oJCPeSUaCQ4Jc9EmCWoghsFhUz8iTOWNTD7u/sd1FaV07e+uMADemNpZ9UUNNS9lhj4932EhPcJAe5yAjwemfG7czE5xkJznJSYwmOkoCfTiRoBYiiJJiokiKiWLaiKTD7mvr8rCvvp2q5k6qW7qo6TVVt3RSXNZIdUsX7S7vYc9NjrGTnRjt/5JwkpMUTXaik9ykaLKTosmId8hwwwgiQS2ESWIdNiZlJzApO+Goj2vr8lDT0kVlcycHmjqoaOykorGDisYOyhra+XxPHS2HtFiUgtRYB5kJDrL8bZusBCdZiY6e5cwEJ8kxdhnBEgYkqIUIcbEOG7EOG6PSYo/4mNYuDwcaOyhvNIK8qvngVNHUyYb9jdS3uQ57nt2qSI6JIiX22FNyTBSJ0Xbpo5tAglqICBDnsDEuM55xmfFHfEyXx0t1s9FWqWzq6mm5NLS5qGtz0dDuYktFM/Vtrp6x5H1x2CwkxdhJijaCOzHGTlK0naQYu/92FMkxdpJjokiKsfeEvAT84ElQCzFMOGzWnqGHx+L2+mhsd1Pf5uqZGjtcNLa7ae5w09ju7rm9v76dEv+63qfqH8ppt/jD2x/ksVGkxzlIj3ccnPun1NgoGY/eiwS1EOIwdqulJzQHosvjpandTUO7m4Z2F43tLurbDi43tLt75lsrjBOJDu2vQ3ePPYo0f4CnxEbhtFlx2i047VYcdv+yzYrTfnB9tN1KvNNm7NlH20mIkFaNBLUQImAcNisZCdYBXWK20+01Rru0fnXkS+/b++vb6XT76PR46XR76XT7jv3Cfk67pSe4jSnKH+I24v39/zinjTjHwSnWYSPe6b/PYSPKasFiMe+gqwS1EMJUTnv/WzLdtNZ0eXw9od3p9tLp8dLh8tLS6aGpw/3Vqd2YN3a4KG/sYNuBZpo73LS6PPT3KhpKgd1iwWZVWC0Ku9VizC0Kq1Vht1hIjYvi7984ZZD/EkcmQS2ECDtKKX/LY2htDZ9P0+H20trlMaZOD21dHlq6l10eWjo9uL0+vD6N26vx+nz+ucbj8+Hxajw+Y4oN0olIEtRCiGHLYlE9wx8zzS7mKOSwqhBChDgJaiGECHES1EIIEeIkqIUQIsRJUAshRIiToBZCiBAnQS2EECFOgloIIUJcUH6FXClVA5T2WpUG1Ab8jcwVadsUadsDkbdNkbY9EHnbNJTtGam1Tu/rjqAE9WFvotTaI/0MeriKtG2KtO2ByNumSNseiLxtCtb2SOtDCCFCnAS1EEKEuOMV1E8ep/c5niJtmyJteyDytinStgcib5uCsj3HpUcthBBi8KT1IYQQIU6CWgghQlzQg1opdb5S6gul1JdKqR8G+/2CTSm1VylVrJTaqJRaa3Y9g6GUekYpVa2UKum1LkUp9b5Saqd/nmxmjQNxhO25XylV7v+cNiqlLjSzxoFSSo1QSi1TSm1VSm1RSt3tXx+Wn9NRtidsPyellFMptVoptcm/TT/zrx+tlPrcn3kvK6WihvxewexRK6WswA7gHKAMWANcq7XeGrQ3DTKl1F6gSGsdtoP0lVLzgFbgOa11gX/dr4B6rfUv/V+oyVrrH5hZZ38dYXvuB1q11r82s7bBUkplA9la6/VKqXhgHXAZcDNh+DkdZXuuIUw/J6WUAmK11q1KKTuwErgb+C7wmtZ6iVLqCWCT1vrxobxXsPeoZwNfaq13a61dwBLg0iC/pzgGrfVyoP6Q1ZcCz/qXn8X4nygsHGF7wprW+oDWer1/uQXYBuQSpp/TUbYnbGlDq/+m3T9p4EzgFf/6gHxGwQ7qXGB/r9tlhPmHg/FBvKeUWqeUut3sYgIoU2t9wL9cCSH9E3L99W2l1GZ/ayQsWgR9UUqNAmYAnxMBn9Mh2wNh/DkppaxKqY1ANfA+sAto1Fp7/A8JSObJwcSBO01rPRO4APiW/8/uiKKNfli4j9t8HBgLTAcOAL8xtZpBUkrFAa8C92itm3vfF46fUx/bE9afk9baq7WeDuRhdBAmBuN9gh3U5cCIXrfz/OvClta63D+vBv6B8eFEgip/H7G7n1htcj1DorWu8v9P5AP+TBh+Tv6+56vAC1rr1/yrw/Zz6mt7IuFzAtBaNwLLgDlAklLK5r8rIJkX7KBeA4zzHwWNAhYBbwT5PYNGKRXrPxCCUioWOBcoOfqzwsYbwE3+5ZuAf5pYy5B1h5nf5YTZ5+Q/UPU0sE1r/XCvu8LyczrS9oTz56SUSldKJfmXozEGTWzDCOyr/A8LyGcU9DMT/cNtfgdYgWe01g8G9Q2DSCk1BmMvGsAGvBiO26OUegk4HeOSjFXAT4HXgaVAPsYlaq/RWofFAbojbM/pGH9Oa2AvcEev3m7IU0qdBqwAigGff/WPMPq6Yfc5HWV7riVMPyel1FSMg4VWjJ3epVrrn/tzYgmQAmwArtdadw3pveQUciGECG1yMFEIIUKcBLUQQoQ4CWohhAhxEtRCCBHiJKiFECLESVALIUSIk6AWQogQ9/8BxiM6t4EH/fIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics = ['loss', 'val_loss']\n",
    "plt.plot(logs['epoch'], logs[metrics])\n",
    "plt.legend(metrics)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqmklEQVR4nO3deXxcdb3/8dc3kz1p9nRN03Rf6QKRtoC0UHaFIsomcFGRiqIsrog+brlXvD/vvV4FFOstohRBSwUrXEUKFLDSDVq6UbqlS5p0SbM0eyaZzHx/f5xpSUtS2kySM8v7+XjM4ywzc87ndODd0+/5nu8x1lpERCS6xLldgIiI9DyFu4hIFFK4i4hEIYW7iEgUUriLiESheLcLAMjLy7NFRUVulyEiElHWr19fZa3N7+y9sAj3oqIi1q1b53YZIiIRxRhT2tV7apYREYlCCncRkSikcBcRiUJh0ebeGZ/PR3l5OV6v1+1SwlJycjIFBQUkJCS4XYqIhKGPDXdjzG+BTwNHrLWTgutygOeAImAfcIO19qgxxgCPAlcBzcAXrLXvdaew8vJy+vXrR1FREc5m5RhrLdXV1ZSXlzN8+HC3yxGRMHQ6zTJPAVectO4BYLm1djSwPLgMcCUwOviaByzobmFer5fc3FwFeyeMMeTm5upfNSLSpY8Nd2vtCqDmpNVzgUXB+UXAtR3WP20da4AsY8yg7hanYO+a/mxE5FS62+Y+wFp7KDh/GBgQnB8ClHX4XHlw3SFOYoyZh3N2T2FhYTfLEBHpedZa2vwBWtsDeH1+Wn3OfMBarOWEKXy4bIPzgYClrd35Tmt7gDZ/gLb2Yy//Cctzxg9gytCsHj+GkC+oWmutMeaMB4W31i4EFgIUFxdrUHkR+ViBgKXe66O6qY2apjaqG51pTVPr8XU1TW0cbW6j3X96seIP2GAI+/H6nGlre4C+etRF/4zksAr3CmPMIGvtoWCzy5Hg+gPA0A6fKwiuE5EoZ62l3ttOZYOXivpW6lp8tAcs/kAAn9/iD1hn2R+g/dh8wOLzB/D6AjS3tdPc5qelzU9zWztNHeZb2vw0tflpam2nPdB56qYleshNTyInLZH+/ZKJjzu9pss4Y0hOiCM5wUNSfBxJCR6Sg9OkjtP4ODxxhjhjMDhNo8bQYdmZx4DHGBLj45yXx/lux+XE+DiS4j0keEyvNbF2N9xfAm4HfhKcvthh/deNMYuB6UBdh+abiHTttddSVlaG1+vl3nvvZd68ebzyyis8+OCD+P1+8vLyWL58OY2NjXzjG99g3bp1GGOYP38+n/3sZ90uX+RjBQKWhtZ2fP4APn+Adr/TJHHCfLsTyG3tAaqb2qio91LZ0MqRBi9H6lupCE5b2wPdqiHRE0dKoofU4694UhI95KYnUpiYevy99KR4ctISyU1PJCctidy0RHKCr+QETw//yUS20+kK+UdgNpBnjCkH5uOE+hJjzB1AKXBD8OMv43SDLMHpCvnFnijy3/5vKx8crO+JTR03YXAG86+e+LGf++1vf0tOTg4tLS184hOfYO7cudx5552sWLGC4cOHU1PjXGv+0Y9+RGZmJlu2bAHg6NGjPVqvSCja2gOUH22mtKaZ/dXNlFY3s7+mKTht7lYo90uKJz8jif79kji7MJv+/ZLo3y+Z/hnONCs1gQRPHPFxBk+cId5jiI8LLnvMh+vjnDNi6VkfG+7W2pu7eGtOJ5+1wN2hFhVOHnvsMZYuXQpAWVkZCxcu5MILLzzevzwnJweA119/ncWLFx//XnZ2dt8XKzHH6/Mfb2euPtb23Phhe3TZUSfID9W10LE1IyXBw7DcVIbnpTF7bD4DMpJJio9zwtgTR4LHkOA5tmxIDIZ0QnwcOamJ9M9IIjUxbO+BFML4DtWOTucMuze89dZbvP7666xevZrU1FRmz57N1KlT2b59uyv1SOypbW5jT1UT+6qa2FvVxJ6qJsqPtlDT1EpNYxtNbf5OvxcfZ8hOS2RIVgrFRdkMyxnCsNw0huWmUpibSn56krrTRrmICHe31NXVkZ2dTWpqKtu3b2fNmjV4vV5WrFjB3r17jzfL5OTkcOmll/L444/zyCOPAE6zjM7e5VSstTS1+altbuNok4+yo81OgFc2sbeqkb1VTRxt9h3/vCfOUJCdwtDsVIpyU8lNSwq2PTuv3OPTJDJS4hXeMU7hfgpXXHEFv/71rxk/fjxjx45lxowZ5Ofns3DhQq677joCgQD9+/fntdde44c//CF33303kyZNwuPxMH/+fK677jq3D0FcYK3lYJ2XzWW17K5spLbZR22Lz5k2tx2fr2tpw9dJd72BGckMz0vjikmDGJGXxvC8NIbnpzE0O5XEeI31J6dH4X4KSUlJ/P3vf+/0vSuvvPKE5fT0dBYtWtTpZyW61TS1sam8ls1ldWwur2VTeR1Vja3H309N9JCVkkBmaiLZqQmMGZBOZkoiWakJZKcmkJWSSGZqAgXZKRTlppGWpP8tJXT6r0jkDDS3tbOlvI5NwRDfXF5LWU0L4PRzHpmfzqwx+UwZmsnkgizGDeynLnriCoW7SBesteyrbmbD/qO8t/8oG/bXsv1wA/5gt5MhWSlMGZrJLdOHMaUgi0lDMuiXrCGYJTwo3EWC6r0+NpXVsmF/LRv2H2VDWS21wQua6UnxTBmayVdnjWTq0CymFmaRl57kcsUiXVO4S0yq9/rYeqCe9w/U8f7BOrYcqGNvVRPWOs0ro/unc/mEgUwrzGJaYTaj+qfrRhuJKAp3iXq1zW28f6D+eIhvPVDHvurm4+8Pykxm0pBMrp06hGmFWUwZmkWGmlckwincJWp4fX5KjjSys6KBHRUN7KpoZMfhBg7Uthz/TEF2CpMGZ3J98VAmDs5g0pBMNa9IVFK4S8QJBCy7jjSyo6KBnYcb2FnhvPbXNB+/xT7BYxiZn87Zw7K5dcYwzhqSycTBGWSnJbpbvEgfUbj3kPT0dBobG90uI6r5A5aXtxziseW72HXE+bP2xBmKclOZMDiDuVOHMGZAP8YOTGdYbhoJHt3wI7FL4S5h7+RQH90/nZ9cdxZThmYxIj+NpHj1Ixc5WWSE+98fgMNbenabA8+CK3/S5dsPPPAAQ4cO5e67nUEuH3roIeLj43nzzTc5evQoPp+Phx9+mLlz537srhobG5k7d26n33v66af56U9/ijGGyZMn8/vf/56Kigruuusu9uzZA8CCBQs477zzeuCgI4s/YPlbMNRLgqH+y89P46pJg4hTzxWRU4qMcHfBjTfeyH333Xc83JcsWcKyZcu45557yMjIoKqqihkzZnDNNdd87ABNycnJLF269CPf++CDD3j44YdZtWoVeXl5x8eGv+eee5g1axZLly7F7/fHXHOPP2D56+aDPLZ8F7srmxgzIJ3HP382V04aqFAXOU2REe6nOMPuLdOmTePIkSMcPHiQyspKsrOzGThwIPfffz8rVqwgLi6OAwcOUFFRwcCBA0+5LWstDz744Ee+98Ybb3D99deTl5cHfDg2/BtvvMHTTz8NgMfjITMzs3cPNkycHOpjB/TjV7eczRUTFeoiZyoywt0l119/Pc8//zyHDx/mxhtv5Nlnn6WyspL169eTkJBAUVERXq/3Y7fT3e/FAq/Pzzt7a3hrRyWvb6tgf00z4wYq1EVCpXA/hRtvvJE777yTqqoq/vGPf7BkyRL69+9PQkICb775JqWlpae1nbq6uk6/d/HFF/OZz3yGb37zm+Tm5h4fG37OnDksWLCA++6773izTDSdvZdWN/HWjkre2nGE1Xuq8foCJMXHMWNELg9cOU6hLtIDFO6nMHHiRBoaGhgyZAiDBg3illtu4eqrr+ass86iuLiYcePGndZ2uvrexIkT+cEPfsCsWbPweDxMmzaNp556ikcffZR58+bx5JNP4vF4WLBgATNnzuzNQ+1VXp+fNXuqeWtHJf/YWcneqiYAinJTuekThcwam8+M4bmkJKrXi0hPMc5jT91VXFxs161bd8K6bdu2MX78eJcqigzh/Gd0tKmN17dVsGzrYd4uqTp+dj5zZC6zx+Qze2x/ivLS3C5TJKIZY9Zba4s7e09n7tJjDtW18OrWCl55/zDv7KvBH7AMzkzmpk8UMntsPjNG5Gpsc5E+onDvQVu2bOG22247YV1SUhJr1651qaLet7uykWVbD7Ps/cNsKq8DYFT/dO6aNYIrJg5i0pAMPctTxAVhHe7W2ogKhrPOOouNGzf2yb7cbE5rbffzxIo9vLjx4PFhAKYUZPKdy8dy+cSBjOqf7lptIuII23BPTk6murqa3NzciAr4vmCtpbq6muTk5D7f9+7KRu754wa2Hqxn+vAcHrp6ApdNHMjgrJQ+r0VEuha24V5QUEB5eTmVlZVulxKWkpOTKSgo6LP9WWtZsq6Mh176gOSEOJ74l2IunTCgz/YvImcmbMM9ISGB4cOHu12GAHUtPh5cuoW/bT7EzBG5/PzGqQzM7Pt/NYjI6QvbcJfwsL60hnv+uJHD9V6+c/lY7po1Uo+bE4kACnfplD9gefzNEh5dvovBWck8f9dMphVmu12WiJwmhbt8xMHaFu57biPv7K3h2qmD+dG1k+inZ4qKRBSFu5zglfcP8b0XttDuD/CzG6Zw3dl9d9FWRHqOwl0AKDnSyM9e28HLWw4zpSCTR2+apuEBRCKYwj3GHaht4dHXd/L8+nJSEjzcf8kYvjp7JInxev6oSCRTuMeoqsZWHn+zhGfX7AcDXzx/OF+bPZLc9CS3SxORHhBSuBtj7ge+DFhgC/BFYBCwGMgF1gO3WWvbQqxTeki918cTK/bw5Nt7aW0PcP05BdwzZ7TuMBWJMt0Od2PMEOAeYIK1tsUYswS4CbgK+Lm1drEx5tfAHcCCHqlWuq2lzc/Tq/ex4B+7qW328anJg/jmpWMYma9xYESiUajNMvFAijHGB6QCh4CLgc8H318EPITC3TWBgOW5dWX8/LWdHGloZfbYfL592VgmDYmeJzuJyEd1O9yttQeMMT8F9gMtwKs4zTC11tr24MfKgSGdfd8YMw+YB1BYWNjdMuQUjtR7+fbzm1mxs5LiYdn88vNnc+7wHLfLEpE+EEqzTDYwFxgO1AJ/Aq443e9baxcCC8F5ElN365DOLdt6mAde2EyLz8+Prp3ErdMLNbqmSAwJpVnmEmCvtbYSwBjzZ+B8IMsYEx88ey8ADoReppyuptZ2fvTXD1j8bhmThmTwyI3TNL66SAwKJdz3AzOMMak4zTJzgHXAm8DncHrM3A68GGqRcno2ltVy3+INlNY089XZI7n/kjHqry4So0Jpc19rjHkeeA9oBzbgNLP8DVhsjHk4uO7JnihUuuYPWH71ZgmPLN/FgH5J/PHOGcwYket2WSLiopB6y1hr5wPzT1q9Bzg3lO3K6Suraeb+5zayrvQo10xxBvnKTNEgXyKxTneoRihrLUs3HOBfX9yKAR65cSrXTuu0Y5KIxCCFewQKBCzzX9rK79eUcm5RDv9zwxSG5qS6XZaIhBGFe4Tx+QN8+0+beHHjQeZdOILvXTFOT0YSkY9QuEcQr8/P1//wHq9vO8J3rxjL12aPcrskEQlTCvcI0djazp2L1rFmbzU/unYSt80Y5nZJIhLGFO4R4GhTG1/43Tu8f7CeR26cytypunAqIqemcA9zFfVebntyLfuqm/nfW8/hkgkD3C5JRCKAwj2M7a9u5pYn11DT2MaiL57LzJG6MUlETo/CPUztrGjg1t+spc0f4A93zmDK0Cy3SxKRCKJwD0Obymq5/XfvkOiJY8lXZjJmQD+3SxKRCKNwDzOrd1fz5UXvkpOeyLN3zKAwVzcniciZU7iHkZIjjXzpqXcpyE7hmS9PZ0BGstsliUiE0niwYaK13c+9izeQnBCnYBeRkOnMPUz8dNkOth6s54l/KVawi0jIdOYeBlbsrOSJf+7lthnDuFT92EWkByjcXVbV2Mo3l2xidP90fvCp8W6XIyJRQs0yLrLW8t3nN1Pv9fH7O84lOcHjdkkiEiV05u6ip1eX8sb2Izx45TjGD8pwuxwRiSIKd5dsP1zPj1/exkVj87n9vCK3yxGRKKNwd4HX5+eeP24gIzmB/75+CsboYRsi0rPU5u6C/3h5GzsrGln0pXPJS09yuxwRiUI6c+9jr39QwdOrS7njguHMGpPvdjkiEqUU7n2oot7Ld57fxPhBGXz3irFulyMiUUzh3kcCAcu3lmyixefnFzdPJSle3R5FpPco3PvIb97ew9slVfzrpycyqr+G8BWR3qVw7wPvH6jjv5ft4PKJA7j53KFulyMiMUDh3sustcx/aSuZKYn85LrJ6vYoIn1C4d7LXnn/MOtLj/Kty8aQnZbodjkiEiMU7r2orT3AT17ZzpgB6Vx/ToHb5YhIDFG496Jn1pRSWt3M968aT7xHf9Qi0neUOL2krtnHY2/s4oJReczWzUoi0scU7r3k8bdKqGvx8f2rxukiqoj0uZDC3RiTZYx53hiz3RizzRgz0xiTY4x5zRizKzjN7qliI0VZTTNPrdzHZ88uYOLgTLfLEZEYFOqZ+6PAK9baccAUYBvwALDcWjsaWB5cjin/tWwHcXHwrcvGuF2KiMSoboe7MSYTuBB4EsBa22atrQXmAouCH1sEXBtaiZFlw/6j/N+mg9z5yREMykxxuxwRiVGhnLkPByqB3xljNhhjfmOMSQMGWGsPBT9zGIiZJz5ba/mPl7eRl57IV2aNdLscEYlhoYR7PHA2sMBaOw1o4qQmGGutBWxnXzbGzDPGrDPGrKusrAyhjPCxbGsF7+47yv2XjiE9SUPli4h7Qgn3cqDcWrs2uPw8TthXGGMGAQSnRzr7srV2obW22FpbnJ8f+V0F29oD/OTv2xjVP50bizV+jIi4q9vhbq09DJQZY44NTD4H+AB4Cbg9uO524MWQKowQf1hbyr7qZh68apxuWBIR14XadvAN4FljTCKwB/gizl8YS4wxdwClwA0h7iPs1bX4eHT5Ls4bmctFY/u7XY6ISGjhbq3dCBR38tacULYbaX71Vgm1LT4evGq8blgSkbCg9oMQlR9t5ncr9/GZaUOYNEQ3LIlIeFC4h+iny3ZggG9fpmeiikj4ULiHYHN5LX/ZeJAvf3I4g7N0w5KIhA+Fewj+4+Vt5KYlcpduWBKRMKNw76bN5bWs2VPD1y4aRb/kBLfLERE5gcK9m55ZU0pKgofri/WEJREJPwr3bqhr9vHSpoNcO20wGTprF5EwpHDvhhfeK8frC3DL9GFulyIi0imF+xmy1vLM2lKmFWapX7uIhC2F+xlavbuaPZVN3KqzdhEJYwr3M/TM2lKyUhP41ORBbpciItIlhfsZOFLv5dWtFVx/TgHJCR63yxER6ZLC/QwsfreM9oDVhVQRCXsK99PU7g/wh7X7+eToPIry0twuRyT2+FrAWweBQM9sz1pnm23NPbO9MKNnwZ2m5duPcLjey7/Nneh2KSKn1lwD1bvh6D7IKoTBUyE+ye2qnDBta4SWWiekvbXB+eDysflOp3Xgb3W2Y+IgKQNSsiA566RppjOfnAE+74nf72zb/jZnm2n5kF0EWcOcaXYRZAfnM4ZAXBfNsH5f57W3NkB7K7S3OFNfy4nL7V6nvnYvTL8Lxl7RM3/GHSjcT9Mza0oZlJnMnHF6GIeEgbYmqNkD1SXBV4f5lpoTP+tJgsHToHA6DJ0BQ6dDWu6Z77O1AeoPOmHW1gitjc60rcl57/h8I7Q1OOs6Bqu3DgLtp9iBccK5Y0hnDDoxwD0JnYdp/cGPBvbHbfPYehuA2v3OX4bl78LWpWD9H24iLh4yh0LWUAj4T9y3r+n0/uw8iRCf4vwlG58MCcnB+RQI+E5vG2dI4X4a9lU18c9dVdx/yRg9Qi/WtTXDgfWQNxr6Dey7/fpaoOR1+OBFKF0F9QdOfL/fYMgdCROugdxRziur0DmDL1sL+9fA6l/Bykedz+eOPjHss4dBwyGoOwB15VBf7kw7LnvrTl2jiYPEfpCYBknpkJjuhGd20anPso+tS8qAuBD//zrW1NJa74Rod7bp9znHXFvqBP7R4LSuzAnp0zmepH7BEE9x/nIN9bi6QeF+Gp5dW0p8nOGmc/Xg65hjLVRud4K1ZLkTrP5WJ8hGzIbJN8G4Tzlh1tPammDXa06g71zmnCWmZMOoSyB/LOSMdEI8Z0TX+x8w0Ql8cELv4AYn6MvWwva/wYZnut5/Sg5kDnGCf9h5znxGgVPD8QBPcwI9Kd0JM7efRGYMJKY6r+7yJEDOcOcVwRTuH8Pr8/On9eVcNnEAAzKS3S5H+kLLUdjzlhPmJcuh4aCzPn8cfOLLUHQ+HHgPNi+BpfMgIQ3Gfxom3wDDZ4MnhP+tWhucIP/gRSfY21sgNc/Z9oS5UHSBEz7dkZDihPSw85zlQACqdzlh33AYMgY7AZ451JlPVMeBSKZw/xh/23yI2maf7kgNJwG/cxYaaHfmA+1Ou+UJy8GXv/3E5YDvxM/4OyzXlTlhfmCd0w6blAkjZjlnyqPmQGaHEUDHfQou+gGUrYHNzznttJufg/QBcNb1ThgPnNz1mWxbEzRVQXOVM204BDtfdf6F4G91tjPtVifQh53X9QW9UMTFOf8CyNdTxKKRsda6XQPFxcV23bp1bpfRqc/8aiV1LT6Wf3OWHn59TFMVvPX/nIBKHwD9BkG/AZA+8MPpqf5Z7PdBc/WJ4dZc7by89c7FuOMX5ho/evHO11td14xz4XHUHCfQhxSf/lm4zwu7XnUCfucy5y+R/PEw/EKnrbrjcTZVOWfkJ+s32GlCmTDXaQfvjUCXqGKMWW+tLe7sPZ25n8L7B+rYsL+Wf/30BAX7Mbteg798zektkNYfGis6v9qflBkM+gHOxaWOYd7lhTnjfDYx/cSLchkFHdp3053PJKRAXILTkyHOE5zGO00WHZe7enlOXuf58GJYdyQkB4P5Gqcr4gd/gU3PwcZnnbbrtFynu13/8ZCaC2l5TnNLx2nWMFcuvEl0UrifwrNrS0lOiOOz5+iBHPha4LV/hXcWQv8JcNtSGDjJabdtOQqNh52mhYaK4HyFs9xY4XQzS82BQVM6hFnuR8MtJTs6zlZTc6D4S85LxCUK9y7Ue338ZcNBrpkymMyUGH8gx6HN8Oc7nV4j078KlzzknKmCc6aZluu8BugGL5FwoXDvwtL3DtDi83PrjBi+kBoIwOpfwvJ/d85Gb/2z0x4tImFP4d4Jay3PrCllSkEmkwuy3C7HHXXlsPQu2PdPGPdpuPqx7t3VKCKuULh3Yu3eGnYdaeS/PjfZ7VLc8f6f4a/3Od0Ir/kFTLvN/ZtTROSMKNw78cyaUjKS47l68mC3S+l91ga76gV7s6z/HWz6Iww5B657wrmlXUQijsL9JJUNrSzbepjbZhSRkhjhPTesdQaSKn/XGSPkeF/rKmiq/nC5Y1dGEwcXfhdmfbf7d0KKiOsU7if5y4YD+PyWW2YUul3KmfN54dDGD8cOKVvrnJEfk5TxYR/rzAIYPOWj3RFzRzpjlYhIRFO4n2TFrkrGDEhnZH4vDATV05qqTgzygxs+HO40ZySMucK503HodGcQpHAY01tE+oTCvYPWdj/v7qvhpk+E8Vl7cw28/4IzaFX5O846TyIMmgrTv/LhEK7p+a6WKSLuUrh38F5pLV5fgPNH5bldyol8Xtj5ijNuya5XnUGu+k+Ai38Iwy5wxkNJ0IiVIvKhkMPdGOMB1gEHrLWfNsYMBxYDucB64DZrbdupthEuVu2uIs7A9BE5bpfi3EC0fxVsWgwfvAStdc4AXTO+6owhPnCS2xWKSBjriTP3e4FtQEZw+T+Bn1trFxtjfg3cASzogf30upUlVUwuyCIj2cVeIlUlzmBTW/7kDEGbkOYMRjX5RmeEwWgYe0VEel1I4W6MKQA+BfwY+KZxhk68GPh88COLgIeIgHBv8PrYVF7HV2e51K/bWlj7v/DqD5z5kRfDnPkw7io9NEFEzlioZ+6PAN8F+gWXc4Faa+2xp+CWA0M6+6IxZh4wD6Cw0P0LmGv31OAPWM4b5cIt9q2N8NI3YOufYcyVcPUjfft8ThGJOt0ePNoY82ngiLV2fXe+b61daK0tttYW5+e737Nj5e4qkuLjOLswu293fGQ7PHGRM/73nPlw0x8U7CISslDO3M8HrjHGXAUk47S5PwpkGWPig2fvBcCBU2wjbKwqqeYTRTkkJ/Rhm/aW5+Gle5ynFv3Li06buohID+j2mbu19vvW2gJrbRFwE/CGtfYW4E3gc8GP3Q68GHKVvexIg5cdFQ191yTT3gYvfxdeuMPp9fKVFQp2EelRvfFMr+/hXFwtwWmDf7IX9tGjVu92btG/oC/6t9cdgKeugnf+F2bcDV/4m/OkeRGRHtQjNzFZa98C3grO7wHO7Ynt9pWVJVVkJMczcXBm7+5oz1vw/JegvRWufwomfqZ39yciMSvm71C11rKypJqZI3PxxPXSmOWBALz9M3jzx5A3Bm74PeSP6Z19iYigcGd/TTMHalv4yqxeGAmxtswZG33js3B0H0z6HFz9KCRFwKBkIhLRYj7cV5Y47e09Np5MWzNs/ytseAb2rgCsc7F0znynGUZPNBKRPqBwL6liYEYyI/JCuAvUWueBGBufdR5R11oPWYUw+wGYcjNkx/BDtkXEFTEd7oGAZdXuKi4a1x/TnTPqhsPBZpc/QNVOSEiFCXNh6i0w7HyI643OSCIiHy+mw33b4XqONvs4f2Q3mmTefwFe/Dr4mqFwpvMg6YmfgaR+H/9dEZFeFtPhvqo77e3+dnh9Pqz+pfNgjLm/hLzRvVShiEj3xHS4v11Sxcj8NAZmnuaDLhor4fkvwr5/wrnz4LIfQ3xi7xYpItINMRvube0B3tlbw/XFBaf3hfL1sOQ254HT1/4apt7cuwWKiIQgZsN9Y1ktLT4/551Oe/v6p+Dl7zijNd7xKgya0uv1iYiEImbDfWWJ80i9mSNOMViYzwt//w689zSMnAOf/Q2khsEj+EREPkbMhvuq3VVMGpJJZmoXj9SrK4fnboOD78Envw0XPahH3IlIxIjJcG9qbWfD/lruvLCLIQf2roA/fdEZ4OvGZ2H8p/u2QBGREMVkuL+zt4b2gO28f/u2vzoXTnNHw03PqpujiESkmAz3lSVVJMbHUVx00iP12lvhle9D/4nwpb/rhiQRiVixGe67qzmnMPujj9R790mo2w/XLFWwi0hEi7nBT6obW9l2qJ7zT36knrcOVvw3jJgNIy92pTYRkZ4Sc+G+ancXQw6s+gW01MAlD/V9USIiPSwGw72KfknxnDWkwyP1Gg7D6sdh0mdh8DT3ihMR6SExF+4rS6qZPiKXeE+HQ//Hf4K/DS7+oXuFiYj0oJgK97KaZvbXNJ/Y3l5VAusXQfGXIKcXHrUnIuKCmAr3lSVVwEnt7W/8OySkwIXfdakqEZGeF1vhvrua/H5JjO4ffEB1+Tr44EU47xuQnu9ucSIiPShmwt1ay+rdVZw/Mtd5pJ618Np8SMuHmXe7XZ6ISI+KmXDfUdFAVWMb5x1rkil5HUrfdppjdMOSiESZmAn3lR0fqRfwO2ft2UVwzhdcrUtEpDfEzPADK0uqKMpNZUhWCmxaDEe2wmef1GPyRCQqxcSZu88fYO2eaqdJxueFNx52nqY08Tq3SxMR6RUxcea+ubyOpjY/F4zKg3VPQl0ZXPMLiIuJv9tEJAbFRLqtCvZvnzkkAVb8FEZcBCMvcrkqEZHeExPhvnJ3FRMGZZC9YYEGBxORmBD14e71+XmvtJbLC21wcLDPweCpbpclItKroj7c1+07Sps/wOcan4WADy7+gdsliYj0um6HuzFmqDHmTWPMB8aYrcaYe4Prc4wxrxljdgWn2R+3rd60cncVhXFVDN77PJzzRQ0OJiIxIZQz93bgW9baCcAM4G5jzATgAWC5tXY0sDy47JpVu6v5XuZyDMAF97lZiohIn+l2uFtrD1lr3wvONwDbgCHAXGBR8GOLgGtDrLHb6lp8lJWXcVnrMjjresgscKsUEZE+1SNt7saYImAasBYYYK09FHzrMDCgi+/MM8asM8asq6ys7IkyPmLtnmpui3uVhIAXzr+3V/YhIhKOQg53Y0w68AJwn7W2vuN71loL2M6+Z61daK0tttYW5+f3znC77+4s5wvxr+IffTn0H98r+xARCUchhbsxJgEn2J+11v45uLrCGDMo+P4g4EhoJXZf5o7nyDYNeC64360SRERcEUpvGQM8CWyz1v6sw1svAbcH528HXux+ed1XWdfE3OalHMqYAsNmulGCiIhrQjlzPx+4DbjYGLMx+LoK+AlwqTFmF3BJcLnPla54hqFxlbRO/4YbuxcRcVW3Bw6z1r4NTg/DTszp7nZ7hLUM3rqQEgoYPkMjP4pI7InOO1RLljPYW8I/82/G4/G4XY2ISJ+LyiF/vW/9DzU2B8/kG9wuRUTEFdF35l6+nuQDq3iy/UpmjhnkdjUiIq6IvnBf+XOa49J5PeVKRvVPd7saERFXRFe4V+3Cbvsrf7SXM3VUAU5vTRGR2BNd4b7qMawnkQUtl3D+yDy3qxERcU30hHvDYdi0mJ2DrqGKTM4blet2RSIiromecF/zKwi0s4hrKMxJpSA71e2KRERcEx3h7q2Ddb8jMH4ufy1L4nydtYtIjIuOcF/3W2itZ9foO2hobec8tbeLSIyL/HD3eWHNAhgxm9drnX7tM0fqzF1EYlvkh/vmxdBYAeffx+rd1Ywb2I+89CS3qxIRcVVkh3vADysfg0FT8A79JO/uq1GTjIgIkR7u2/8GNbvh/Pt4r6yW1vYA56lJRkQkwsMdYPgsmDCX1bur8cQZpo/IcbsiERHXRfaokBOucV7AypIqJhdk0i85weWiRETcF/ln7kCD18em8jo1yYiIBEVFuL+7rwZ/wGo8GRGRoKgI95Ul1STGx3H2sGy3SxERCQtREe6rdldTPCyb5AQ9Uk9EBKIg3KsbW9l2qJ7zR6lJRkTkmIgP99V7qgF0MVVEpIOID/dVu6vplxTPWUMy3S5FRCRsRH64l1QxfUQO8Z6IPxQRkR4T0Yl4oLaFfdXNGk9GROQkER3uq0qqAPRIPRGRk0R0uGelJnLphAGMHdDP7VJERMJKRI8tc+mEAVw6YYDbZYiIhJ2IPnMXEZHOKdxFRKKQwl1EJAop3EVEolCvhLsx5gpjzA5jTIkx5oHe2IeIiHStx8PdGOMBHgeuBCYANxtjJvT0fkREpGu9ceZ+LlBird1jrW0DFgNze2E/IiLShd4I9yFAWYfl8uA6ERHpI67dxGSMmQfMCy42GmN2nPSRPKCqb6vqVdF2PBB9xxRtxwPRd0zRdjwQ2jEN6+qN3gj3A8DQDssFwXUnsNYuBBZ2tRFjzDprbXHPl+eOaDseiL5jirbjgeg7pmg7Hui9Y+qNZpl3gdHGmOHGmETgJuClXtiPiIh0ocfP3K217caYrwPLAA/wW2vt1p7ej4iIdK1X2tyttS8DL4e4mS6bbCJUtB0PRN8xRdvxQPQdU7QdD/TSMRlrbW9sV0REXKThB0REopDCXUQkCoVduEfjuDTGmH3GmC3GmI3GmHVu19MdxpjfGmOOGGPe77AuxxjzmjFmV3Ca7WaNZ6KL43nIGHMg+DttNMZc5WaNZ8IYM9QY86Yx5gNjzFZjzL3B9ZH8G3V1TBH5Oxljko0x7xhjNgWP59+C64cbY9YGM++5YC/D0PcXTm3uwXFpdgKX4tzZ+i5ws7X2A1cLC5ExZh9QbK2N2JsvjDEXAo3A09baScF1/wXUWGt/EvyLONta+z036zxdXRzPQ0CjtfanbtbWHcaYQcAga+17xph+wHrgWuALRO5v1NUx3UAE/k7GGAOkWWsbjTEJwNvAvcA3gT9baxcbY34NbLLWLgh1f+F25q5xacKUtXYFUHPS6rnAouD8Ipz/8SJCF8cTsay1h6y17wXnG4BtOMN+RPJv1NUxRSTraAwuJgRfFrgYeD64vsd+o3AL92gdl8YCrxpj1geHXYgWA6y1h4Lzh4FoeKDt140xm4PNNhHThNGRMaYImAasJUp+o5OOCSL0dzLGeIwxG4EjwGvAbqDWWtse/EiPZV64hXu0usBaezbOMMh3B5sEoop12vfCp42vexYAI4GpwCHgf1ytphuMMenAC8B91tr6ju9F6m/UyTFF7O9krfVba6fiDMtyLjCut/YVbuF+WuPSRBpr7YHg9AiwFOdHjQYVwXbRY+2jR1yuJyTW2org/3wB4Aki7HcKtuO+ADxrrf1zcHVE/0adHVOk/04A1tpa4E1gJpBljDl2Q2mPZV64hXvUjUtjjEkLXgzCGJMGXAa8f+pvRYyXgNuD87cDL7pYS8iOhWDQZ4ig3yl4se5JYJu19mcd3orY36irY4rU38kYk2+MyQrOp+B0HNmGE/KfC36sx36jsOotAxDs1vQIH45L82N3KwqNMWYEztk6OMM9/CESj8kY80dgNs7wpBXAfOAvwBKgECgFbrDWRsRFyi6OZzbOP/UtsA/4Sof26rBmjLkA+CewBQgEVz+I00Ydqb9RV8d0MxH4OxljJuNcMPXgnFgvsdb+ezAjFgM5wAbgVmtta8j7C7dwFxGR0IVbs4yIiPQAhbuISBRSuIuIRCGFu4hIFFK4i4hEIYW7iEgUUriLiESh/w/XlWbhKcppHgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics = ['acc', 'val_acc']\n",
    "plt.plot(logs['epoch'], logs[metrics])\n",
    "plt.legend(metrics)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since we have created checkpoints using callbacks, we can restore the best model from those checkpoints and test it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[35mTest steps: \u001b[36m19 \u001b[32m1.59s \u001b[35mtest_loss:\u001b[94m 2.002988\u001b[35m test_acc:\u001b[94m 51.118476\u001b[35m test_fscore_micro:\u001b[94m 0.511185\u001b[0m           \n"
     ]
    }
   ],
   "source": [
    "resnet18 = models.resnet18(pretrained=False, num_classes=num_classes)\n",
    "\n",
    "model = Model(\n",
    "    resnet18,\n",
    "    'sgd',\n",
    "    'cross_entropy',\n",
    "    batch_metrics=['accuracy'],\n",
    "    epoch_metrics=['f1'],\n",
    "    device=device,\n",
    ")\n",
    "\n",
    "model.load_weights(os.path.join(save_path, 'best_epoch_{epoch}.ckpt').format(epoch=best_epoch))\n",
    "\n",
    "test_loss, (test_acc, test_f1) = model.evaluate_generator(test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also use the Experiment class to train our network. This class saves checkpoints and logs as above in a directory and allows to stop and resume optimization at will. See documentation for details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def experiment_train(epochs):\n",
    "    # Reload the pretrained network and freeze it except for its head.\n",
    "    resnet18 = models.resnet18(pretrained=True)\n",
    "    resnet18.fc = nn.Linear(resnet18.fc.in_features, num_classes)\n",
    "    freeze_weights(resnet18)\n",
    "\n",
    "    # Saves everything into ./saves/cub200_resnet18_experiment\n",
    "    save_path = os.path.join(save_base_dir, 'cub200_resnet18_experiment')\n",
    "\n",
    "    optimizer = optim.SGD(resnet18.fc.parameters(), lr=learning_rate, weight_decay=0.001)\n",
    "\n",
    "    # Poutyne Experiment\n",
    "    exp = Experiment(\n",
    "        save_path,\n",
    "        resnet18,\n",
    "        device=device,\n",
    "        optimizer=optimizer,\n",
    "        task='classif',\n",
    "    )\n",
    "\n",
    "    # Train\n",
    "    exp.train(train_loader, valid_loader, epochs=epochs)\n",
    "\n",
    "    # Test\n",
    "    exp.test(test_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let’s train for 5 epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[35mEpoch: \u001b[36m1/5 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.18s \u001b[35mloss:\u001b[94m 4.972403\u001b[35m acc:\u001b[94m 4.448743\u001b[35m fscore_micro:\u001b[94m 0.044487\u001b[35m val_loss:\u001b[94m 4.354537\u001b[35m val_acc:\u001b[94m 11.350456\u001b[35m val_fscore_micro:\u001b[94m 0.113505\u001b[0m\n",
      "Epoch 1: val_acc improved from -inf to 11.35046, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_1.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m2/5 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.03s \u001b[35mloss:\u001b[94m 3.739010\u001b[35m acc:\u001b[94m 23.984526\u001b[35m fscore_micro:\u001b[94m 0.239845\u001b[35m val_loss:\u001b[94m 3.638358\u001b[35m val_acc:\u001b[94m 21.458161\u001b[35m val_fscore_micro:\u001b[94m 0.214582\u001b[0m\n",
      "Epoch 2: val_acc improved from 11.35046 to 21.45816, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_2.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m3/5 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.13s \u001b[35mloss:\u001b[94m 2.918689\u001b[35m acc:\u001b[94m 41.503178\u001b[35m fscore_micro:\u001b[94m 0.415032\u001b[35m val_loss:\u001b[94m 3.234509\u001b[35m val_acc:\u001b[94m 28.417564\u001b[35m val_fscore_micro:\u001b[94m 0.284176\u001b[0m\n",
      "Epoch 3: val_acc improved from 21.45816 to 28.41756, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_3.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m4/5 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.18s \u001b[35mloss:\u001b[94m 2.376595\u001b[35m acc:\u001b[94m 55.153357\u001b[35m fscore_micro:\u001b[94m 0.551534\u001b[35m val_loss:\u001b[94m 2.913982\u001b[35m val_acc:\u001b[94m 36.205468\u001b[35m val_fscore_micro:\u001b[94m 0.362055\u001b[0m\n",
      "Epoch 4: val_acc improved from 28.41756 to 36.20547, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_4.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m5/5 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.08s \u001b[35mloss:\u001b[94m 1.976730\u001b[35m acc:\u001b[94m 65.349544\u001b[35m fscore_micro:\u001b[94m 0.653495\u001b[35m val_loss:\u001b[94m 2.722522\u001b[35m val_acc:\u001b[94m 37.531069\u001b[35m val_fscore_micro:\u001b[94m 0.375311\u001b[0m\n",
      "Epoch 5: val_acc improved from 36.20547 to 37.53107, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_5.ckpt\n",
      "Restoring model from saves/cub200_resnet18_experiment/checkpoint_epoch_5.ckpt\n",
      "Found best checkpoint at epoch: 5\n",
      "lr: 0.1, loss: 1.97673, acc: 65.3495, fscore_micro: 0.653495, val_loss: 2.72252, val_acc: 37.5311, val_fscore_micro: 0.375311\n",
      "Loading checkpoint saves/cub200_resnet18_experiment/checkpoint_epoch_5.ckpt\n",
      "Running test\n",
      "\u001b[35mTest steps: \u001b[36m19 \u001b[32m1.56s \u001b[35mtest_loss:\u001b[94m 2.696897\u001b[35m test_acc:\u001b[94m 39.768020\u001b[35m test_fscore_micro:\u001b[94m 0.397680\u001b[0m           \n"
     ]
    }
   ],
   "source": [
    "experiment_train(epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let’s train for 5 more epochs (10 epochs total)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading weights from saves/cub200_resnet18_experiment/checkpoint.ckpt and starting at epoch 6.\n",
      "Loading optimizer state from saves/cub200_resnet18_experiment/checkpoint.optim and starting at epoch 6.\n",
      "\u001b[35mEpoch: \u001b[36m 6/10 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.16s \u001b[35mloss:\u001b[94m 1.668047\u001b[35m acc:\u001b[94m 71.677259\u001b[35m fscore_micro:\u001b[94m 0.716773\u001b[35m val_loss:\u001b[94m 2.573154\u001b[35m val_acc:\u001b[94m 41.425021\u001b[35m val_fscore_micro:\u001b[94m 0.414250\u001b[0m\n",
      "Epoch 6: val_acc improved from 37.53107 to 41.42502, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_6.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 7/10 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.01s \u001b[35mloss:\u001b[94m 1.463776\u001b[35m acc:\u001b[94m 76.844432\u001b[35m fscore_micro:\u001b[94m 0.768444\u001b[35m val_loss:\u001b[94m 2.463882\u001b[35m val_acc:\u001b[94m 43.744822\u001b[35m val_fscore_micro:\u001b[94m 0.437448\u001b[0m\n",
      "Epoch 7: val_acc improved from 41.42502 to 43.74482, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_7.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 8/10 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.41s \u001b[35mloss:\u001b[94m 1.282330\u001b[35m acc:\u001b[94m 80.851064\u001b[35m fscore_micro:\u001b[94m 0.808511\u001b[35m val_loss:\u001b[94m 2.420289\u001b[35m val_acc:\u001b[94m 44.159072\u001b[35m val_fscore_micro:\u001b[94m 0.441591\u001b[0m\n",
      "Epoch 8: val_acc improved from 43.74482 to 44.15907, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_8.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m 9/10 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m5.02s \u001b[35mloss:\u001b[94m 1.149077\u001b[35m acc:\u001b[94m 83.337939\u001b[35m fscore_micro:\u001b[94m 0.833379\u001b[35m val_loss:\u001b[94m 2.343573\u001b[35m val_acc:\u001b[94m 44.656172\u001b[35m val_fscore_micro:\u001b[94m 0.446562\u001b[0m\n",
      "Epoch 9: val_acc improved from 44.15907 to 44.65617, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_9.ckpt\n",
      "\u001b[35mEpoch: \u001b[36m10/10 \u001b[35mTrain steps: \u001b[36m57 \u001b[35mVal steps: \u001b[36m19 \u001b[32m4.98s \u001b[35mloss:\u001b[94m 1.028025\u001b[35m acc:\u001b[94m 86.681404\u001b[35m fscore_micro:\u001b[94m 0.866814\u001b[35m val_loss:\u001b[94m 2.283656\u001b[35m val_acc:\u001b[94m 46.147473\u001b[35m val_fscore_micro:\u001b[94m 0.461475\u001b[0m\n",
      "Epoch 10: val_acc improved from 44.65617 to 46.14747, saving file to saves/cub200_resnet18_experiment/checkpoint_epoch_10.ckpt\n",
      "Restoring model from saves/cub200_resnet18_experiment/checkpoint_epoch_10.ckpt\n",
      "Found best checkpoint at epoch: 10\n",
      "lr: 0.1, loss: 1.02802, acc: 86.6814, fscore_micro: 0.866814, val_loss: 2.28366, val_acc: 46.1475, val_fscore_micro: 0.461475\n",
      "Loading checkpoint saves/cub200_resnet18_experiment/checkpoint_epoch_10.ckpt\n",
      "Running test\n",
      "\u001b[35mTest steps: \u001b[36m19 \u001b[32m1.49s \u001b[35mtest_loss:\u001b[94m 2.245021\u001b[35m test_acc:\u001b[94m 47.224523\u001b[35m test_fscore_micro:\u001b[94m 0.472245\u001b[0m           \n"
     ]
    }
   ],
   "source": [
    "# Retrain for 5 more epochs\n",
    "experiment_train(epochs=10)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}