flower-classifier-resnet.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nq21KSBUeKMt",
        "outputId": "01ff84c8-d72a-4b94-fbb9-91b40ce44608"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7fe212620410>"
            ]
          },
          "execution_count": 42,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tqdm\n",
        "from torchvision import datasets, transforms, models\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "import matplotlib.pyplot as plt\n",
        "from tabulate import tabulate\n",
        "\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "torch.manual_seed(42)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "jizKX4rYeqDa"
      },
      "outputs": [],
      "source": [
        "CROPPED_IMAGES_SIZE = 224\n",
        "N_CLASSES = 102\n",
        "BATCH_SIZE = 64\n",
        "LEARNING_RATE = 0.0001\n",
        "N_EPOCHS = 40"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "na-4B7TueSFU"
      },
      "outputs": [],
      "source": [
        "normalize_mean = np.array([0.485, 0.456, 0.406])\n",
        "normalize_std = np.array([0.229, 0.224, 0.225])\n",
        "\n",
        "\n",
        "# Training\n",
        "# transform_train = transforms.Compose([\n",
        "#   transforms.RandomRotation(45),\n",
        "#   transforms.RandomHorizontalFlip(),\n",
        "#   transforms.Resize((CROPPED_IMAGES_SIZE, CROPPED_IMAGES_SIZE)),\n",
        "#   transforms.ToTensor(),\n",
        "#   transforms.Normalize(normalize_mean, normalize_std)\n",
        "# ])\n",
        "transform_train = transforms.Compose([\n",
        "  transforms.RandomHorizontalFlip(p=0.5),\n",
        "  transforms.RandomVerticalFlip(p=0.5),\n",
        "  transforms.RandomRotation(180),\n",
        "  transforms.RandomResizedCrop(CROPPED_IMAGES_SIZE),\n",
        "  transforms.ToTensor(),\n",
        "  transforms.Normalize(normalize_mean, normalize_std)\n",
        "])\n",
        "dataset_train = datasets.Flowers102(root=\"dataset/train\", download=True, transform=transform_train)\n",
        "train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n",
        "\n",
        "# Validation\n",
        "transform_validation = transforms.Compose([\n",
        "  transforms.Resize(256),\n",
        "  transforms.CenterCrop(CROPPED_IMAGES_SIZE),\n",
        "  transforms.ToTensor(),\n",
        "  transforms.Normalize(normalize_mean, normalize_std)\n",
        "])\n",
        "dataset_validation = datasets.Flowers102(root=\"dataset/validation\", download=True, transform=transform_validation, split=\"val\")\n",
        "validation_loader = torch.utils.data.DataLoader(dataset_validation, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n",
        "\n",
        "# Test\n",
        "transform_test = transforms.Compose([\n",
        "  transforms.Resize(256),\n",
        "  transforms.CenterCrop(CROPPED_IMAGES_SIZE),\n",
        "  transforms.ToTensor(),\n",
        "  transforms.Normalize(normalize_mean, normalize_std)\n",
        "])\n",
        "\n",
        "dataset_test = datasets.Flowers102(root=\"dataset/test\", download=True, transform=transform_test, split=\"test\")\n",
        "test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r1nKx7Z_RUcC",
        "outputId": "c8fcef6d-990d-4ea3-f8fd-c77d1d867661"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "ResNet(\n",
              "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
              "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  (relu): ReLU(inplace=True)\n",
              "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
              "  (layer1): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer2): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (3): Bottleneck(\n",
              "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer3): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (3): Bottleneck(\n",
              "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (4): Bottleneck(\n",
              "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (5): Bottleneck(\n",
              "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer4): Sequential(\n",
              "    (0): Bottleneck(\n",
              "      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): Bottleneck(\n",
              "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "    (2): Bottleneck(\n",
              "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
              "  (fc): Sequential(\n",
              "    (0): Linear(in_features=2048, out_features=512, bias=True)\n",
              "    (1): ReLU()\n",
              "    (2): Dropout(p=0.2, inplace=False)\n",
              "    (3): Linear(in_features=512, out_features=102, bias=True)\n",
              "    (4): LogSoftmax(dim=1)\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Download the model\n",
        "# model = models.resnet18(weights='ResNet18_Weights.DEFAULT')\n",
        "# Replace the model classifier\n",
        "# classifier = nn.Sequential(nn.Linear(512, 256),\n",
        "#                         nn.ReLU(),\n",
        "#                         nn.Dropout(p=0.2),\n",
        "#                         nn.Linear(256, 102),\n",
        "#                         nn.LogSoftmax(dim=1)\n",
        "#                         )\n",
        "\n",
        "model = models.resnet50(weights='ResNet50_Weights.DEFAULT')\n",
        "classifier = nn.Sequential(nn.Linear(2048, 512),\n",
        "                        nn.ReLU(),\n",
        "                        nn.Dropout(p=0.2),\n",
        "                        nn.Linear(512, 102),\n",
        "                        nn.LogSoftmax(dim=1)\n",
        "                        )\n",
        "\n",
        "# model = models.resnet152(weights='ResNet152_Weights.DEFAULT')\n",
        "# classifier = nn.Sequential(nn.Linear(2048, 1024),\n",
        "#                         nn.ReLU(),\n",
        "#                         nn.Dropout(p=0.2),\n",
        "#                         nn.Linear(1024, 102),\n",
        "#                         nn.LogSoftmax(dim=1)\n",
        "#                         )\n",
        "\n",
        "model.fc = classifier\n",
        "\n",
        "# Move model to device\n",
        "model.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iwS8EpAIgVPF",
        "outputId": "623cd297-b98d-486f-cc5d-8e6a12dca725"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.33batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1, loss: 73.7702784538269, lr: 0.0001\n",
            "Accuracy: 78/1020 = 7.65%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.35batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2, loss: 72.11170959472656, lr: 0.0001\n",
            "Accuracy: 268/1020 = 26.27%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.31batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3, loss: 67.4869978427887, lr: 0.0001\n",
            "Accuracy: 386/1020 = 37.84%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4, loss: 57.490894079208374, lr: 0.0001\n",
            "Accuracy: 534/1020 = 52.35%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.35batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5, loss: 43.895432472229004, lr: 0.0001\n",
            "Accuracy: 663/1020 = 65.00%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.35batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6, loss: 31.42710053920746, lr: 0.0001\n",
            "Accuracy: 764/1020 = 74.90%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.33batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7, loss: 21.042168378829956, lr: 0.0001\n",
            "Accuracy: 846/1020 = 82.94%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.33batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8, loss: 15.392122864723206, lr: 0.0001\n",
            "Accuracy: 857/1020 = 84.02%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.26batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9, loss: 11.640070021152496, lr: 0.0001\n",
            "Accuracy: 882/1020 = 86.47%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.34batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10, loss: 8.724350541830063, lr: 0.0001\n",
            "Accuracy: 895/1020 = 87.75%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.33batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 11, loss: 6.570531338453293, lr: 0.0001\n",
            "Accuracy: 920/1020 = 90.20%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.34batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 12, loss: 6.457452401518822, lr: 0.0001\n",
            "Accuracy: 909/1020 = 89.12%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.34batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 13, loss: 5.552673161029816, lr: 0.0001\n",
            "Accuracy: 918/1020 = 90.00%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.31batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 14, loss: 4.716696545481682, lr: 0.0001\n",
            "Accuracy: 922/1020 = 90.39%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 15, loss: 4.5405828058719635, lr: 0.0001\n",
            "Accuracy: 923/1020 = 90.49%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 16, loss: 3.431905321776867, lr: 0.0001\n",
            "Accuracy: 929/1020 = 91.08%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 17, loss: 4.038203299045563, lr: 0.0001\n",
            "Accuracy: 927/1020 = 90.88%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 18, loss: 3.86802376806736, lr: 0.0001\n",
            "Accuracy: 934/1020 = 91.57%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.34batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 19, loss: 3.5586897283792496, lr: 0.0001\n",
            "Accuracy: 927/1020 = 90.88%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.34batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 20, loss: 3.0219541490077972, lr: 1e-05\n",
            "Accuracy: 933/1020 = 91.47%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.34batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 21, loss: 3.055841848254204, lr: 1e-05\n",
            "Accuracy: 932/1020 = 91.37%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.33batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 22, loss: 3.9040060192346573, lr: 1e-05\n",
            "Accuracy: 934/1020 = 91.57%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.34batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 23, loss: 3.2636860832571983, lr: 1e-05\n",
            "Accuracy: 935/1020 = 91.67%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.24batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 24, loss: 2.772924728691578, lr: 1e-05\n",
            "Accuracy: 939/1020 = 92.06%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:13<00:00,  1.20batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 25, loss: 2.741037029772997, lr: 1e-05\n",
            "Accuracy: 939/1020 = 92.06%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:13<00:00,  1.18batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 26, loss: 2.158935822546482, lr: 1e-05\n",
            "Accuracy: 943/1020 = 92.45%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:11<00:00,  1.33batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 27, loss: 2.0577730648219585, lr: 1e-05\n",
            "Accuracy: 943/1020 = 92.45%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.29batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 28, loss: 2.992121711373329, lr: 1e-05\n",
            "Accuracy: 944/1020 = 92.55%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:13<00:00,  1.21batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 29, loss: 2.4469929337501526, lr: 1e-05\n",
            "Accuracy: 944/1020 = 92.55%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:13<00:00,  1.19batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 30, loss: 2.5749445110559464, lr: 1e-05\n",
            "Accuracy: 943/1020 = 92.45%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:13<00:00,  1.21batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 31, loss: 1.7362560257315636, lr: 1e-05\n",
            "Accuracy: 944/1020 = 92.55%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.33batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 32, loss: 2.0105949118733406, lr: 1e-05\n",
            "Accuracy: 943/1020 = 92.45%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:14<00:00,  1.13batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 33, loss: 2.208995819091797, lr: 1e-05\n",
            "Accuracy: 940/1020 = 92.16%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 34, loss: 2.369050722569227, lr: 1e-05\n",
            "Accuracy: 942/1020 = 92.35%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.26batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 35, loss: 2.262709517031908, lr: 1e-05\n",
            "Accuracy: 946/1020 = 92.75%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.31batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 36, loss: 2.7734922245144844, lr: 1e-05\n",
            "Accuracy: 944/1020 = 92.55%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 37, loss: 2.571748446673155, lr: 1e-05\n",
            "Accuracy: 946/1020 = 92.75%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.23batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 38, loss: 2.3326503187417984, lr: 1e-05\n",
            "Accuracy: 946/1020 = 92.75%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 39, loss: 2.230727653950453, lr: 1e-05\n",
            "Accuracy: 946/1020 = 92.75%\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 16/16 [00:12<00:00,  1.32batch/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 40, loss: 2.2911267802119255, lr: 1.0000000000000002e-06\n",
            "Accuracy: 944/1020 = 92.55%\n",
            "Finished Training\n"
          ]
        }
      ],
      "source": [
        "def check_accuracy(data_loader, model):\n",
        "    n_corrects = 0\n",
        "    n_samples = 0\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for x, y in data_loader:\n",
        "            # Sending data to device\n",
        "            x = x.to(device)\n",
        "            y = y.to(device)\n",
        "\n",
        "            # Forward propagation\n",
        "            y_hat = model(x)\n",
        "\n",
        "            # Calculate accuracy\n",
        "            _, predictions = y_hat.max(1)\n",
        "            n_corrects += (predictions == y).sum()\n",
        "            n_samples += predictions.size(0)\n",
        "\n",
        "        perc = (n_corrects.item() / n_samples) * 100\n",
        "        return (n_corrects.item(), n_samples, perc)\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)\n",
        "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1)\n",
        "\n",
        "training_results_data = pd.DataFrame(\n",
        "    {\"Epoch\": [], \"Predictions\": [], \"Samples\": [], \"Accuracy\": [], \"Loss\": []}\n",
        ")\n",
        "\n",
        "# Tensorboard writer\n",
        "writer = SummaryWriter()\n",
        "\n",
        "for epoch in range(N_EPOCHS):  # loop over the dataset multiple times\n",
        "    running_loss = 0.0\n",
        "    model.train()\n",
        "    with tqdm.tqdm(train_loader, unit=\"batch\") as tepoch:\n",
        "      for i, data in enumerate(tepoch):\n",
        "          # get the inputs; data is a list of [inputs, labels]\n",
        "          inputs, labels = data\n",
        "          inputs = inputs.to(device)\n",
        "          labels = labels.to(device)\n",
        "\n",
        "          # zero the parameter gradients\n",
        "          optimizer.zero_grad()\n",
        "\n",
        "          # forward + backward + optimize\n",
        "          outputs = model(inputs)\n",
        "          loss = criterion(outputs, labels)\n",
        "          loss.backward()\n",
        "          optimizer.step()\n",
        "          \n",
        "\n",
        "          # print statistics\n",
        "          running_loss += loss.item()\n",
        "    scheduler.step()\n",
        "    current_lr = optimizer.state_dict()['param_groups'][0]['lr']\n",
        "    n_corrects, n_samples, accuracy = check_accuracy(validation_loader, model)\n",
        "    training_results_data.loc[len(training_results_data.index)] = [\n",
        "        int(epoch + 1),\n",
        "        int(n_corrects),\n",
        "        int(n_samples),\n",
        "        accuracy,\n",
        "        running_loss,\n",
        "    ]\n",
        "    writer.add_scalar(\"Loss/train\", running_loss, epoch)\n",
        "    writer.add_scalar('Accuracy/validation', accuracy, epoch)\n",
        "    print(f\"Epoch {epoch + 1}, loss: {running_loss}, lr: {current_lr}\")\n",
        "    print(f\"Accuracy: {n_corrects}/{n_samples} = {accuracy:.2f}%\")\n",
        "    \n",
        "writer.flush()\n",
        "print('Finished Training')\n",
        "\n",
        "torch.save(model.state_dict(), 'model.pt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "-mERX70sgO2O",
        "outputId": "762941fa-9153-4493-d0c0-82aa7f7b7e2a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Training/Validation results\n",
            "╒═════════╤═══════════════════════╤════════════╤══════════╕\n",
            "│   Epoch │ Predictions/Samples   │ Accuracy   │     Loss │\n",
            "╞═════════╪═══════════════════════╪════════════╪══════════╡\n",
            "│       1 │ 78/1020               │ 7.65%      │ 73.7703  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       2 │ 268/1020              │ 26.27%     │ 72.1117  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       3 │ 386/1020              │ 37.84%     │ 67.487   │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       4 │ 534/1020              │ 52.35%     │ 57.4909  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       5 │ 663/1020              │ 65.00%     │ 43.8954  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       6 │ 764/1020              │ 74.90%     │ 31.4271  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       7 │ 846/1020              │ 82.94%     │ 21.0422  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       8 │ 857/1020              │ 84.02%     │ 15.3921  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│       9 │ 882/1020              │ 86.47%     │ 11.6401  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      10 │ 895/1020              │ 87.75%     │  8.72435 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      11 │ 920/1020              │ 90.20%     │  6.57053 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      12 │ 909/1020              │ 89.12%     │  6.45745 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      13 │ 918/1020              │ 90.00%     │  5.55267 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      14 │ 922/1020              │ 90.39%     │  4.7167  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      15 │ 923/1020              │ 90.49%     │  4.54058 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      16 │ 929/1020              │ 91.08%     │  3.43191 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      17 │ 927/1020              │ 90.88%     │  4.0382  │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      18 │ 934/1020              │ 91.57%     │  3.86802 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      19 │ 927/1020              │ 90.88%     │  3.55869 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      20 │ 933/1020              │ 91.47%     │  3.02195 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      21 │ 932/1020              │ 91.37%     │  3.05584 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      22 │ 934/1020              │ 91.57%     │  3.90401 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      23 │ 935/1020              │ 91.67%     │  3.26369 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      24 │ 939/1020              │ 92.06%     │  2.77292 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      25 │ 939/1020              │ 92.06%     │  2.74104 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      26 │ 943/1020              │ 92.45%     │  2.15894 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      27 │ 943/1020              │ 92.45%     │  2.05777 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      28 │ 944/1020              │ 92.55%     │  2.99212 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      29 │ 944/1020              │ 92.55%     │  2.44699 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      30 │ 943/1020              │ 92.45%     │  2.57494 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      31 │ 944/1020              │ 92.55%     │  1.73626 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      32 │ 943/1020              │ 92.45%     │  2.01059 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      33 │ 940/1020              │ 92.16%     │  2.209   │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      34 │ 942/1020              │ 92.35%     │  2.36905 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      35 │ 946/1020              │ 92.75%     │  2.26271 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      36 │ 944/1020              │ 92.55%     │  2.77349 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      37 │ 946/1020              │ 92.75%     │  2.57175 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      38 │ 946/1020              │ 92.75%     │  2.33265 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      39 │ 946/1020              │ 92.75%     │  2.23073 │\n",
            "├─────────┼───────────────────────┼────────────┼──────────┤\n",
            "│      40 │ 944/1020              │ 92.55%     │  2.29113 │\n",
            "╘═════════╧═══════════════════════╧════════════╧══════════╛\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEWCAYAAABmE+CbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xcdX3/8ddnZu+b2242CZAEkhAggkLAiKIBFG2LF0R/Kki18mspKVatVlvF1rbqT3+1tv5QvGEQQYsCgopCrXKXBEEIcmkkQEISICFkd7OXJHub2ZnP74/z3c1k2VuSnTlzeT8fj3nsOd8zc85nzu6ez/l+v+d8j7k7IiIiibgDEBGR4qCEICIigBKCiIgESggiIgIoIYiISKCEICIigBKCiEySmW01szfFHYfkjxKC5I2Z3WNmnWZWG3csIjIxJQTJCzNbBJwOOPD2Am+7qpDby7dy+z5SvJQQJF8+ADwAXANcmLvAzBaa2U/NrM3MdpnZN3KWXWxmG8xsj5k9YWanhHI3s6U577vGzL4Qpl9vZtvM7FNm9iJwtZk1mdmtYRudYXpBzuebzexqM3shLL85lK83s3Ny3ldtZu1mdvJoXzLEu8nMOszsF2Z2RCj/tpn9x4j3/tzMPh6mjzCzn4T4tpjZ3+S877NmdpOZXWtmu4H/Pcp2a83sP8zsOTPbaWZXmFn9iP3xDyH2rWb2vpzPzjSzH4RtP2tmnzGzRM7yUX8HwXIze9zMus3sBjOrC59pCfu4K+yLNbnrlNKgX5jkyweAH4bXn5jZPAAzSwK3As8Ci4D5wPVh2XuAz4bPziCqWeya5PYOA5qBo4BVRH/bV4f5I4E+4Bs57/9PoAE4AZgLXBbKfwC8P+d9bwF2uPsjIzdoZmcB/wqcBxwevtP1YfF1wPlmZuG9TcAfA9eHA+UtwGPh+78R+JiZ/UnO6s8FbgJmEe3Dkb4EHAssB5aG9fzziP3REsovBFab2XFh2deBmcAS4Eyi/f3nIc6JfgfnAWcDi4ET2ZesPgFsA+YA84B/IKodSilxd730mtIXsBJIAy1h/kngb8P0aUAbUDXK534NfHSMdTqwNGf+GuALYfr1QAqoGyem5UBnmD4cyAJNo7zvCGAPMCPM3wR8cox1XgV8OWd+WvjeiwADngPOCMsuBu4K068Gnhuxrk8DV4fpzwL3jvNdDOgBjs4pOw3YkrM/BoHGnOU/Bv4JSIZ9dXzOsr8C7pnE72Ar8P6c+S8DV4TpzwM/z/0d6VV6L9UQJB8uBG5z9/Yw/yP2NRstBJ5198FRPrcQeOYgt9nm7v1DM2bWYGbfCU0iu4F7gVmhhrIQ6HD3zpErcfcXgPuAd5nZLODNjH6GDlHyeDbns3uJzqbne3SUvB64ICz+05z1HAUcEZpXusysi+iMel7Oup8f57vOIardPJzz+V+F8iGd7t6TM/9siLcFqM6NO0zPD9MT/Q5ezJnuJUqCAP8ObAJuM7PNZnbpOOuQIqXOKplSoR37PCAZ2vMBaokOxicRHeiONLOqUZLC88DRY6y6l+ggOOQwoiaKISObJz4BHAe82t1fNLPlwCNEZ9fPA81mNsvdu0bZ1veBvyT6/7jf3bePEdMLRAd3AMysEZgNDL3/OqID5JeIagXvzPmeW9z9mDHWO9r3ydVO1AR2wjixNZlZY05SOBJYHz6bDnE/kbNsaD3j/Q7GDtZ9D9E+/4SZvRy4y8wecvc7D3RdEh/VEGSqvQPIAMcTNdMsB14GrCFql34Q2AF8ycwazazOzF4XPvtd4O/M7JUWWWpmQwfcR4E/NbOkmZ1N1PY9nulEB80uM2sG/mVogbvvAP4b+FbofK42szNyPnszcArwUaI+hbFcB/y5mS236NLa/wv8zt23hu08QnQA/i7w65zk8yCwJ3SC14fv9HIze9UE32ko/ixwJXCZmc0FMLP5I/ogAD5nZjVmdjrwNuBGd88QNR990cymh/37ceDa8JnxfgdjMrO3hfca0E30N5CdzPeR4qGEIFPtQqK28Ofc/cWhF1GH7vuIztDPIeoIfY7oLP98AHe/EfgiURPTHqIDc3NY70fD57rCem6eII6vAvVEB+QHiJpUcv0Z0Znyk0Ar8LGhBe7eB/yEqOP0p2NtwN3vIGqX/wlRkjsaeO+It/0IeFP4OfS5DNEBejmwhX1JY+YE3ynXp4iaaB4ITWJ3ENWIhrwIdBLVYn4IXOLuT4ZlHyHqg9gMrA2xfS/ENt7vYDzHhBj2AvcD33L3uw/g+0gRsKipU0Rymdk/A8e6+/snfHORMbPXA9e6+4KJ3iuSS30IIiOEJqaLiGoRIhVDTUYiOczsYqKO1f9293vjjkekkNRkJCIigGoIIiISlHQfQktLiy9atCjuMERESsrDDz/c7u5zRpYXTUIws3cAbyUaP+Uqd79tos8sWrSIdevW5T02EZFyYmbPjlae1yYjM/uembWa2foR5Web2VMWjRJ5KYC73+zuFwOXEK5LFxGRwsl3H8I1RCMjDgtjyXyTaIyY44ELzOz4nLd8JiwXEZECymtCCJftdYwoPhXY5O6b3T1FNADYueE2+X8jutzv92Ot08xWmdk6M1vX1taWv+BFRCpMHFcZzWf/kRy3hbKPEN3i/24zu2SsD7v7andf4e4r5sx5SZ+IiIgcpKLpVHb3y4HL445DRKRSxVFD2E405vqQBewbeldERGISR0J4CDjGzBabWQ3R6JC/iCEOERHJkdcmIzO7juhxfi1mtg34F3e/ysw+TPSoviTwPXf/Qz7jEBEYzGTpTWfoS2XoTWXoTQ3Sl8qQyozz2AKPntSTyToZd7JZJ5N1su5kPSqfSMKMZALMjKQZyYSRSETTCYOM71tnJku07rA9d6L3h/fmfjaZMKqTCWqrEtRWJ6itSlJblaCuOvpZW5Ug6zCYzZIedNLZLOlMlsGMk8pE032pnP2RztCXGgz7JsPAYJaqsI3qZPSzKrlvPmFGOuMMhnWlcqbTmeHHjB60ZCJBdZVRnYi2V5VMUJMTw5nHzWFGXfUhbWOkvCYEd79gjPJfAr/M57alsvSno3/g4X/chBGeb7+fgcEMXb1pOntTdPSk6OpN09GTorsvTW1VgubGGpoaamhqrKGpoZqmxhqm11YNr6s/naG7L83uvjTd4bW7P03PQAYzooNcYv8DXyKEkXsgGsx4OIhE04PZfQfB4QNiODhmsk5/OpNzMB/MOahn6E9nxn28WibrEx/4ZcqN8uc3aZPJJXd8/MzSSghSPtydTa17ads7wPxZ9Rw+s56aqsm3OKYzWdr3DrCnf3C/s9Oh6d5Uhr50hkxm6KDIvgNkOCtNZ7I5B+HB4enuvjSpwZce7HKTQ01Vgr5Uhp5U5oC/e1XCmF5XRU8qM+p2psrwGXBIJkPJJWFQV52kviZJQ02ShuoqZjXUcMSsqKy+OklinKNPwqC+pir6bM2+9dRXR2XVycS4B6+hs/P9Et3wz/EPfO5Ev8v9agH7ahjZrIfvmPudGZ43CH8D+9aR+zcxmHUGBjMMpLP0h58Dg1kGBjP0p7MkjHBmn6AmnGXnnvHX5+7XmmS0n0JZbVVieBupnCQ+VAPIukdn71VGVWL/s/fq5OgnJAdi6G8+9wQinXXSg1kGs1kWNtcf0vpHo4QgY+ruS3PfpnbufbqN3zzdxo7u4WfYYwbzptcxv6me+bPqh3+6Ozt3D9C6pz/8HKB1dz+7elIHvP3cA0TSjOqqBDPqqplZX82M+irmzZgWpqOy2qrkmNX3VCZLfXVy+Ky/qWGoJlBNc0MNM+qrGUhn6ehN0dmborMnRWdvmq5Qk9jTP0hDbXJ4+0OvoW031iT3Na2MaFLJhtO93APRfs0QiehAJcWnKmlUJaOEXGjJhJFMJAu67ZIc/trMzgHOWbp06cUbN26MO5yyMTCYYcOOPcMJ4NHnu8hknel1Vaxc2sIZx87hyOYGtnf1sb2zj22dfWzv6mV7Vx87uvoZDO3JyYTRMq2GeTPqmDu9lrlDP6fXMaO+avjstLF231lZY030h1+VCGeGh3h2JSJjM7OH3X3FS8pLMSEMWbFihWtwuwOTzTovdPexpb2HLe09bG7rGZ7e1tlL1qOz/xPnz+SMY+dw5rFzWL5w1oRnsJms07qnn2TCmN1YSzKhA7pIsRorIajJqAJs6+zl7idbufupNu5/Zhd96X3t6A01SRa3NHLigpm8Y/kRHDNvOq89ejazp9Ue0DaSCePwmVPfpikihaOEUIbSmSzrtnZyz1Ot3PVkKxtb9wJw1OwG3rNiAcsOm8HilkaWzGlk7vRaNc+ICKCEUNJSg1le6OrjuY5enuvo5fnOXja39fDA5l3s6R+kOmmcuriZ81+1kLOWzWVxS6MO/iIyJiWEEuLufOueZ1izsY3nO/rY0d1H7n1BNckEC5rqecvLD+cNy+ay8pgWptXqVywik6OjRYlwdz53yxNc89utnLhgJqcubmZhcwNHNjewsKmeI2c3MG96HQl15orIQVJCKAHuzpd+9STX/HYrF61czGfe+jI1/YjIlNPdMCXgq3ds5Du/2cz7X3OkkoGI5E1JJgQzO8fMVnd3d8cdSt59655NfO3OjbznlQv4/NtfrmQgInlTkgnB3W9x91UzZ86MO5S8umrtFr78q6c4d/kRfOldJ6p/QETyqiQTQiW49oFn+T+3PsHZJxzGV95zku78FZG8U0IoQjeue57P3Lyes5bN5fILTtbAZyJSEDrSFJn/enwHn/rJ45x+TAvfet8pBzTEtIjIodDRpog8t6uXT970GCcf2cTqP1sRy5C7IlK5lBCKRCbrfPzHj5Iw4/ILTqa+RslARApLN6YVie/c+wzrnu3ksvNPYv4sjRoqIoWnGkIRWL+9m8tuf5q3vuJw3rF8ftzhiEiFUkKIWX86w9/e8ChNDTV84R268UxE4qMmo5j9+6+fYmPrXr7/F6fS1FgTdzgiUsFKsoZQLkNX3LepnavWbuEDpx3FmcfOiTscEalwJZkQymHoiu6+NH9342MsaWnk029+WdzhiIioySgu//Lz9bTuGeCnH3ytLjEVkaJQkjWEUnfr4y9w86Mv8DdnHcNJC2fFHY6ICKCEUHA7d/fzjz9bz0kLZ/GhNxwddzgiIsOUEArsK7c9RX86w2XnnaRB60SkqOiIVECtu/u5+ZEXOG/FQpbMmRZ3OCIi+1FCKKBrfruVdDbLRSsXxx2KiMhLKCEUSM/AINc+8Cxnn3AYi1oa4w5HROQllBAK5IaHnmd3/yCrzlgSdygiIqNSQiiAwUyWq9Zu4VWLmjj5yKa4wxERGZUSQgH8cv2LbO/qY9UZusxURIpXSSaEUhrLyN1Zfe8zLJnTyBuXzY07HBGRMZVkQiilsYzu37yL9dt3c/HpS0gkNLS1iBSvkkwIpWT1vZtpmVbDO0/Wg29EpLgpIeTRUy/u4Z6n2rjwtEXUVWsAOxEpbkoIeXTlms3UVyd5/2uOijsUEZEJKSHkyc7d/fz80e2ct2KBnoQmIiVBCSFPrr5vK5msc9FK3YgmIqVBCSEP9g4M8sPfPcubX344R85uiDscEZFJUULIg+sffI49GqZCREqMEsIUS2eyfG/tFk5d3KynoYlISVFCmGJ3PdnKC939rDpdtQMRKS1KCFPs9id2MqOuijOPmxN3KCIiB0QJYQplss5dT7byhmVzqdbjMUWkxOioNYUeea6Tjp4Ub3rZvLhDERE5YCWZEIp1tNM7NrRSlTA1F4lISSrJhFCso53esWEnr17SzIy66rhDERE5YCWZEIrR1vYeNrXuVXORiJQsJYQpcseGnQBKCCJSspQQpsgdG3Zy3LzpLGzWUBUiUpqUEKZAd2+ah7Z28qbj9YhMESldSghT4J6nW8lkXc1FIlLSlBCmwO1P7KRlWi0nLdDYRSJSupQQDlFqMMtvnmrjjcvmkkhY3OGIiBw0JYRD9NDWDvYMDPKm49VcJCKlTQnhEN3+xE5qqxKsXNoSdygiIodECeEQuDt3bNjJyqUt1Nck4w5HROSQKCEcgqd37mVbZ5+ai0SkLCghHIKhu5PfuEz3H4hI6VNCOAS3P7GTkxbMZO6MurhDERE5ZEoIB6l1Tz+PbevSzWgiUjaUEA7S3U+24o76D0SkbCghHKTbn2hl/qx6lh02Pe5QRESmREkmhLifmNafzrB2UxtvetlczHR3soiUh0klBDNrMrMTzGyJmcWeROJ+Ytp9m9rpT2fVXCQiZaVqrAVmNhP4EHABUAO0AXXAPDN7APiWu99dkCiLzB0bdjKttopXL54ddygiIlNmzIQA3AT8ADjd3btyF5jZK4E/M7Ml7n5VPgMsNtmsc+eGVs48dg41VbFXlkREpsyYCcHd/2icZQ8DD+cloiL3xI7dtO4Z4CzdjCYiZWa8GsJ+zGwO8FGgHrjC3TfmLaoitmZjOwCnH6PB7ESkvBxIm8dXgF8DPwN+lJ9wit/aTW0cN2+67k4WkbIzZkIws1+b2Rk5RTXA1vCqzW9Yxak/neGhrZ2qHYhIWRqvhnAecI6ZXWdmRwP/BPwr8DXgrwsRXLF5cEsHqcEsK5UQRKQMjdep3A38vZktAb4IvAB8eOQVR5Vk7aZ2apIJXW4qImVpvPsQjgY+CKSATwBHAzeY2X8B33T3TGFCLB5rNrbzyqOa9DAcESlL4zUZXQf8FLgb+E93X+PufwJ0AbcVIrhi0rZngA07dqu5SETK1niXndYCW4BpQMNQobv/wMxuzHdgxea+TbrcVETK23gJ4a+BbxA1GV2Su8Dd+/IZVDFas7GdWQ3VnHBEPOMniYjk23idyvcB9xUwlqLl7qzd1MbrlraQTGh0UxEpT+Pdh3CLmb3NzKpHWbbEzD5vZn+R3/CKw6bWvezcPcDpS9VcJCLla7wmo4uBjwNfM7MO9o12ugh4BviGu/887xEWgaHhKtShLCLlbLwmoxeBTwKfNLNFwOFAH/C0u/cWJLoisXZTO4tbGlnQ1DDxm0VEStSkBrdz961EQ1ZUnNRglgc27+JdpyyIOxQRkbzSgP4T+P1znfSmMmouEpGyp4QwgbUb20kmjNOO1nAVIlLeJkwI4YH2FZs41mxqZ/nCWcyoe8nFViIiZWUyB/rzgY1m9mUzW5bvgIpJd2+a/9nWxUpdbioiFWDChODu7wdOJrrU9Bozu9/MVpnZ9LxHN4ZQa1nd3d2d1+389pl2sq7hKkSkMkyqKcjddwM3AdcTXX76TuD3ZvaRPMY2Xjy3uPuqmTPzO4zEmk3tTKut4qSFs/K6HRGRYjCZPoS3m9nPgHuAauBUd38zcBLRsNhla83GNl6zZDbVyYrtQhGRCjKZ+xDeBVzm7vfmFrp7r5ldlJ+w4vfsrh6e7+jjL1cuiTsUEZGCmExC+CywY2jGzOqBee6+1d3vzFdgcRsarkL9ByJSKSbTFnIjkM2Zz4SysrZ2YzvzZ9WzuKUx7lBERApiMgmhyt1TQzNhuiZ/IcUvk3V++0w7K5e2YKbhrkWkMkwmIbSZ2duHZszsXKA9fyHF7/FtXezuH9RwFSJSUSbTh3AJ8EMz+wZgwPPAB/IaVczWbmzHDF6nG9JEpIJMmBDc/RngNWY2LczvzXtUMVuzsZ0TjphBc2NZt4yJiOxnUsNfm9lbgROAuqE2dXf/fB7jitWTL+7m3OXz4w5DRKSgJnNj2hVE4xl9hKjJ6D3AUXmOKzbpTJbd/YO0TKuNOxQRkYKaTKfya939A0Cnu38OOA04Nr9hxaezN7qgqnmamotEpLJMJiH0h5+9ZnYEkCYaz6gsdfakAWhuUEIQkcoymT6EW8xsFvDvwO8BB67Ma1Qx2tUzAKAOZRGpOOMmhPBgnDvdvQv4iZndCtS5e37HnY7RcA1BCUFEKsy4TUbungW+mTM/UM7JAKAj1BCaGvWENBGpLJPpQ7jTzN5lFTKGQ0eoITSpD0FEKsxkEsJfEQ1mN2Bmu81sj5ntznNcsenoGWBGXZWegSAiFWcydyrH9qjMOHT0ppmtexBEpAJNmBDM7IzRykc+MKdcdPakaGpQ/4GIVJ7JXHb69znTdcCpwMPAWXmJKGa7elLMn1UfdxgiIgU3mSajc3LnzWwh8NW8RRSzzp4Ur5g/I+4wREQK7mB6TrcBL5vqQIqBu9PRk6JJ9yCISAWaTB/C14nuToYogSwnumO57PSkMqQyWWYrIYhIBZpMH8K6nOlB4Dp3vy9P8cSqY280sJ3uQRCRSjSZhHAT0O/uGQAzS5pZg7v35je0wusII53O1kinIlKBJnWnMpB72U09cEd+wolXZ49qCCJSuSaTEOpyH5sZphvyF1J8doWEMLtRN6aJSOWZTELoMbNThmbM7JVAX/5Cis9wDUED24lIBZpMH8LHgBvN7AWiR2geRvRIzbKzqydFddKYVjupR02LiJSVydyY9pCZLQOOC0VPuXs6v2HFo7MnRXNjDRUysKuIyH4mbDIysw8Bje6+3t3XA9PM7K/zH9q4MZ1jZqu7u6f20Qy7elLqUBaRijWZPoSLwxPTAHD3TuDi/IU0MXe/xd1XzZw5c0rX29mb0iWnIlKxJpMQkrkPxzGzJFCWR81O1RBEpIJNpvf0V8ANZvadMP9Xoazs7OpJadgKEalYk0kInwJWAR8M87cDV+YtopgMZrJ096U1sJ2IVKwJm4zcPevuV7j7u9393cATwNfzH1phdfZGF041KyGISIWa1AX3ZnYycAFwHrAF+Gk+g4pDZxjHSAlBRCrVmAnBzI4lSgIXAO3ADYC5+xsKFFtB7QojnTarU1lEKtR4NYQngTXA29x9E4CZ/W1BoorBcA1Bl52KSIUarw/hfwE7gLvN7EozeyPR0BVlaWhgO9UQRKRSjZkQ3P1md38vsAy4m2hMo7lm9m0z++NCBVgo+wa2U0IQkco0mauMetz9R+5+DrAAeIToUtSy0tGTYnpdFdXJg3nMtIhI6Tugo5+7d7r7and/Y74CiktHGNhORKRS6XQ46OxVQhCRyqaEEOzam1KHsohUNCWEQDUEEal0SgiAu7NLfQgiUuGUEIDeVIbUYFYJQUQqmhIC0RVGoHsQRKSyKSGwLyGoU1lEKpkSAtChcYxERJQQADo00qmIiBICaKRTERFQQgCikU6rk8b02kk9L0hEpCwpIRCNdNrUUINZ2Y7uLSIyISUENLCdiAgoIQBRQmhSh7KIVDglBKLLTtWhLCKVTgmB0GSkGoKIVLiKTwiDmSzdfWn1IYhIxav4hNDVl8YdJQQRqXgVnxA6NbCdiAighDA8sN1sJQQRqXBKCEM1BHUqi0iFU0II4xjN1mWnIlLhlBDCSKezGqpjjkREJF5KCL0pptdWUVuVjDsUEZFYKSH0pHSFkYgISghKCCIigRJCT0qXnIqIoIQw/CwEEZFKV/EJoaM3pUtORUSo8ITQmxqkP51VDUFEhApPCBq2QkRkHyUENLCdiAgoIQDQ3Ki7lEVElBCA5sbamCMREYmfEgLo8ZkiIlR4QujsTZFMGDPqq+IORUQkdhWdEDrCTWlmFncoIiKxq/iEoEtORUQiRZMQzGyJmV1lZjcVapvRwHa6wkhEBPKcEMzse2bWambrR5SfbWZPmdkmM7sUwN03u/tF+YxnpI6eFM2qIYiIAPmvIVwDnJ1bYGZJ4JvAm4HjgQvM7Pg8xzEqJQQRkX3ymhDc/V6gY0TxqcCmUCNIAdcD5+YzjtFksk5XX1qXnIqIBHH0IcwHns+Z3wbMN7PZZnYFcLKZfXqsD5vZKjNbZ2br2traDjqI7r407qiGICISFM0F+O6+C7hkEu9bDawGWLFihR/s9jp6BgCNYyQiMiSOGsJ2YGHO/IJQVlAdPWkAZmvYChERIJ6E8BBwjJktNrMa4L3ALwodxL4agi47FRGB/F92eh1wP3CcmW0zs4vcfRD4MPBrYAPwY3f/Qz7jGM1QDUF9CCIikbz2Ibj7BWOU/xL4ZT63PZHhGoKuMhIRAYroTuVC6+hJ01iTpK46GXcoIiJFoWITQmdviuZpqh2IiAwpyYRgZueY2eru7u6DXseunpRuShMRyVGSCcHdb3H3VTNnzjzodXRq2AoRkf2UZEKYCtFIp0oIIiJDKjohqMlIRGSfikwIfakMfemMOpVFRHJUZELo6E0BqIYgIpKjMhPC3pAQ1IcgIjKsMhNCrxKCiMhIFZkQOnuUEERERirJhHCoN6btUkIQEXmJkkwIh3pjWmdPimTCmFGnoa9FRIaUZEI4VLt6UjQ1VJNIWNyhiIgUjYpMCJ09KQ17LSIyQtE8U7mQXrFgJotaGuMOQ0SkqFRkQvjQG5bGHYKISNGpyCYjERF5KSUEEREBlBBERCRQQhAREUAJQUREgpJMCFPxTGUREdlfSSaEqXimsoiI7K8kE4KIiEw9c/e4YzhoZtYGPDvG4hagvYDhHAjFdnAU28FRbAennGM7yt3njCws6YQwHjNb5+4r4o5jNIrt4Ci2g6PYDk4lxqYmIxERAZQQREQkKOeEsDruAMah2A6OYjs4iu3gVFxsZduHICIiB6acawgiInIAlBBERAQo04RgZmeb2VNmtsnMLo07nlxmttXM/sfMHjWzdTHH8j0zazWz9TllzWZ2u5ltDD+biii2z5rZ9rDvHjWzt8QU20Izu9vMnjCzP5jZR0N57PtunNhi33dmVmdmD5rZYyG2z4XyxWb2u/D/eoOZFfz5tuPEdo2ZbcnZb8sLHVtOjEkze8TMbg3zU7/f3L2sXkASeAZYAtQAjwHHxx1XTnxbgZa44wixnAGcAqzPKfsycGmYvhT4tyKK7bPA3xXBfjscOCVMTweeBo4vhn03Tmyx7zvAgGlhuhr4HfAa4MfAe0P5FcAHiyi2a4B3x/03F+L6OPAj4NYwP+X7rRxrCKcCm9x9s7ungOuBc2OOqSi5+71Ax4jic4Hvh+nvA+8oaFDBGLEVBXff4e6/D9N7gA3AfIpg340TW+w8sjfMVoeXA2cBN4XyuPbbWLEVBTNbALwV+G6YN/Kw38oxIcwHns+Z30aR/EMEDtxmZg+b2aq4gxnFPHffEaZfBObFGcwoPmxmj4cmpVias3KZ2SLgZKIzyqLadyNig31+xogAAAPzSURBVCLYd6HZ41GgFbidqDbf5e6D4S2x/b+OjM3dh/bbF8N+u8zMauOIDfgq8EkgG+Znk4f9Vo4JoditdPdTgDcDHzKzM+IOaCwe1UWL5iwJ+DZwNLAc2AF8Jc5gzGwa8BPgY+6+O3dZ3PtulNiKYt+5e8bdlwMLiGrzy+KIYzQjYzOzlwOfJorxVUAz8KlCx2VmbwNa3f3hfG+rHBPCdmBhzvyCUFYU3H17+NkK/Izon6KY7DSzwwHCz9aY4xnm7jvDP20WuJIY952ZVRMdcH/o7j8NxUWx70aLrZj2XYinC7gbOA2YZWZVYVHs/685sZ0dmuDc3QeAq4lnv70OeLuZbSVqAj8L+Bp52G/lmBAeAo4JPfA1wHuBX8QcEwBm1mhm04emgT8G1o//qYL7BXBhmL4Q+HmMsexn6GAbvJOY9l1ov70K2ODu/y9nUez7bqzYimHfmdkcM5sVpuuBPyLq47gbeHd4W1z7bbTYnsxJ8EbURl/w/ebun3b3Be6+iOh4dpe7v4987Le4e87z8QLeQnR1xTPAP8YdT05cS4iuenoM+EPcsQHXETUfpInaIC8iapu8E9gI3AE0F1Fs/wn8D/A40cH38JhiW0nUHPQ48Gh4vaUY9t04scW+74ATgUdCDOuBfw7lS4AHgU3AjUBtEcV2V9hv64FrCVcixfUCXs++q4ymfL9p6AoREQHKs8lIREQOghKCiIgASggiIhIoIYiICKCEICIigRKCyAhmlskZ3fJRm8IRc81sUe4IriLFpGrit4hUnD6PhjAQqSiqIYhMkkXPsviyRc+zeNDMlobyRWZ2VxgA7U4zOzKUzzOzn4Ux9h8zs9eGVSXN7Mow7v5t4c5YzOxvwnMMHjez62P6mlLBlBBEXqp+RJPR+TnLut39FcA3iEagBPg68H13PxH4IXB5KL8c+I27n0T0bIc/hPJjgG+6+wlAF/CuUH4pcHJYzyX5+nIiY9GdyiIjmNled582SvlW4Cx33xwGkHvR3WebWTvRUBDpUL7D3VvMrA1Y4NHAaEPrWEQ0tPIxYf5TQLW7f8HMfgXsBW4GbvZ94/OLFIRqCCIHxseYPhADOdMZ9vXlvRX4JlFt4qGckSxFCkIJQeTAnJ/z8/4w/VuiUSgB3gesCdN3Ah+E4YevzBxrpWaWABa6+91EY+7PBF5SSxHJJ52BiLxUfXhy1pBfufvQpadNZvY40Vn+BaHsI8DVZvb3QBvw56H8o8BqM7uIqCbwQaIRXEeTBK4NScOAyz0al1+kYNSHIDJJoQ9hhbu3xx2LSD6oyUhERADVEEREJFANQUREACUEEREJlBBERARQQhARkUAJQUREAPj//O3PJV+bSskAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEWCAYAAABmE+CbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xV9f3H8dcnm4QQIANCwkrCENlbQBkiKG5R0bZO1Iqr1lX9dQ/b2lpXbaUqilvqqFsrIiJO9t4jzEAIgQCBkPX9/XEPNiJgEpKce5P38/G4D+49995z3zma+85Z32POOURERML8DiAiIsFBhSAiIoAKQUREPCoEEREBVAgiIuJRIYiICKBCEGlQzGyYmW32O4cEJxWCBA0zyzazkX7nEGmoVAgidcDMIvzOIPJ9VAgS9Mws2sweMrOt3u0hM4v2nksys3fMbLeZ5ZvZTDML8577mZltMbO9ZrbSzE49yvwTzOxZM9thZhvM7BdmFuZ97m4z61rhtclmdsDMUrzHZ5nZAu91X5hZ9wqvzfYyLAIKj1QKZtbZzKZ62Vea2cUVnptsZhO95/ea2Qwza1vh+UFmNtvMCrx/B1V4rrmZPe0tr11m9sZhn3u7meWaWY6ZXVVh+hgzW+Z93hYzu6NK/7EktDnndNMtKG5ANjDyCNN/B3wFpADJwBfA773n/gRMBCK928mAAZ2ATUAr73XtgMyjfO6zwJtAvPe6VcB477mngHsrvPZG4APvfi8gFxgAhANXeD9DdIWfZwHQGmh0hM+N8zJeBUR488sDunjPTwb2AqcA0cDDwGfec82BXcBl3nsv9R4nes+/C0wBmnnLZag3fRhQ6i3TSGAMsB9o5j2fA5zs3W8G9Pb7/wvd6u7mewDddDt0O0YhrAXGVHg8Gsj27v/O+zLPOuw9Wd6X9Ugg8hifGQ4UH/oS9qb9GPjEuz8SWFvhuc+By737jx0qpgrPr6zw5ZsNXH2Mzx4HzDxs2r+AX3v3JwMvV3iuMVDmFcxlwKzD3vslcCWQCpQf+pI/7DXDgANARIVpucBA7/5G7+dv4vf/D7rV/U2bjCQUtAI2VHi8wZsG8FdgDfChma0zs7sBnHNrgFuB3wC5ZvaymbXiu5II/KV8+PzTvPvTgVgzG2Bm7YCewH+859oCt3ubi3ab2W4CX9YVP2fTMX6utsCAw97/Q6Dlkd7vnNsH5HvzP3yZVMzdGsh3zu06yufudM6VVni8n0DZAIwlsNawwdtEddIx8ks9o0KQULCVwJfnIW28aTjn9jrnbnfOZQDnALcd2lfgnHvROTfEe68D7jvCvPOAkiPMf4s3jzLg3wQ2yVwKvOOc2+u9bhOBzUlNK9xinXMvVZjXsYYT3gTMOOz9jZ1zEyq8pvWhO2bWmMCmoq1HWCYVc28CmptZ02N89hE552Y7584lsHnuDe9nlwZChSDBJtLMYircIoCXgF94O3STgF8Bz8M3O3WzzMyAAgKbVMrNrJOZjfB2PhcR2ExSfviHVfjCv9fM4r2dtrcdmr/nRQKbd37o3T/kCeB6b+3BzCzOzM40s/hK/qzvAB3N7DIzi/Ru/czshAqvGWNmQ8wsCvg98JVzbhPwnvfeH5hZhJmNA7oQKKwc4H3gn2bWzJvvKd8XxsyizOyHZpbgnCsB9hxpmUn9pUKQYPMegS/vQ7ffAH8A5gCLgMXAPG8aQAfgI2AfgW3o/3TOTSewE/bPBNYAthH4i/eeo3zmzUAhsA74jMCX/lOHnnTOfe0934rAF+2h6XOAa4FHCezQXUNgG36leGsao4BLCPzFv43AWkx0hZe9CPyawKaiPsCPvPfuBM4Cbgd2AncBZznn8rz3XUZgzWcFgX0Et1Yy1mVAtpntAa4nUILSQJhzukCOSDAys8nAZufcL/zOIg2D1hBERARQIYiIiEebjEREBNAagoiIeEJ6wK2kpCTXrl07v2OIiISUuXPn5jnnkg+fHtKF0K5dO+bMmeN3DBGRkGJmh5/lDmiTkYiIeFQIIiICqBBERMSjQhARESBEC8HMzjazxwsKCvyOIiJSb4RkITjn3nbOXZeQkOB3FBGReiMkC0FERGpeSJ+HUF2vz9vMzn3FZCTHkZncmPRmjYgIVzeKSMPWIAvh3UU5TFuR+83jyHCjXWLcNwWRkdyYwVmJpCY08jGliEjdapCFMOnKfuwqLGZd3j7W7ihk3Y5C1u7Yx5rcfUxbnktpuSMizDizeyrXnpxB1zTtqxCR+q9BFgJAs7go+sQ1p0/b5t+aXlJWzvq8QqbM3sSU2Zt4c8FWBmY059qTMxjeKYWwMPMpsYhI7QrJ4a/N7Gzg7KysrGtXr15da5+zp6iEKbM28fTn69laUERGchzjh7RnbO90YiLDa+1zRURqk5nNdc71/c70UCyEQ/r27evqYnC7krJy3lucw5Mz17N4SwHN46J44OIeDOuUUuufLSJS045WCDq0phIiw8M4t2cab900mCnXDaRFkxiuf34us7Pz/Y4mIlJjVAhVYGYMyEjk+fH9aZXQiKsnz2Z5zh6/Y4mI1AgVQjUkNo7muWsG0Dg6gssmzWLDzkK/I4mIHDcVQjWlNW3Ec+P7U1Zezo8mfU3uniK/I4mIHBcVwnHISoln8lX92bmvmMsmzaJgf4nfkUREqk2FcJx6tG7K45f1ZX1eIVc/M5v9xaV+RxIRqZaQLIRgG/56SIckHr6kJ/M37mLC8/MoLi33O5KISJWFZCEE4/DXZ3RL5Y/nd2PGqh3c/spCystD9/wOEWmYQrIQgtUl/dtw5+hOvL1wK6/N2+x3HBGRKlEh1LAbhmXSIz2BB6auoqikzO84IiKVpkKoYWbGPWNOIKegiKc+X+93HBGRSlMh1IKBGYmMPCGFx6avJb+w2O84IiKVokKoJT87vTOFxaU8Mq32RmMVEalJKoRa0qFFPOP6teH5rzaQnaehLUQk+KkQatFPR3YgKiKMv/53pd9RRES+lwqhFqU0ieHakzN4d3EO8zfu8juOiMgxhWQhBNuZysdy3SkZJDWO5o/vLSeUL0YkIvVfSBZCMJ6pfDRx0RH89LQOzM7exdRl2/2OIyJyVCFZCKFmXN/WZCbH8ecPVlBSpnGORCQ4qRDqQER4GHefcQLrdhQyZfYmv+OIiByRCqGOjDwhhf7tmvPQR6vYd1BDZItI8FEh1BEz4//OPIG8fcU8/uk6v+OIiHyHCqEO9WzdlDO7p/LEp+vYpSEtRCTIqBDq2C0jOnCgpIwXZ230O4qIyLeoEOpYp5bxnNwhiWe/zNaV1UQkqKgQfHD1kPZs33OQdxdv9TuKiMg3VAg+GNohmayUxkz6bL3OXhaRoKFC8EFYmHH14PYs2bKHr9fn+x1HRAQI0UIIpbGMjuaC3mk0i41k0me6qpqIBIeQLIRQGsvoaGIiw/nRwLZ8tHy7rpcgIkEhJAuhvrhsYFsiwoynde1lEQkCKgQfpTSJ4ZweabwydzMF+0v8jiMiDZwKwWfjh7Rnf3EZL83WiWoi4i8Vgs+6tGrCoMxEnvkiW0Nji4ivVAhBYPyQ9uQUFPHe4hy/o4hIA6ZCCALDO6WQkRSnE9VExFcqhCAQFmZcNaQ9izYXMGfDLr/jiEgDpUIIEmN7p5HQKJJJM3UIqoj4Q4UQJGKjIvjhgDZ8uGwbG3fu9zuOiDRAKoQgcvlJ7Qgz4+kvtJYgInVPhRBEWibEMKZbKq/N3UxRSZnfcUSkgVEhBJmL+qazp6iUactz/Y4iIg2MCiHIDMpMomWTGF6bt9nvKCLSwIRkIdSH4a+PJjzMOK9XGjNW7WDH3oN+xxGRBiQkC6E+DH99LGN7p1FW7nhzwRa/o4hIAxKShVDfdWgRT/f0BF6fp0IQkbqjQghSY3unsyxnD8tz9vgdRUQaCBVCkDq7Rysiw43XtXNZROqICiFINY+LYninFN5YsJVSDYstInVAhRDELuidzo69B5m5Js/vKCLSAKgQgtiIzik0jY3UzmURqRMqhCAWFRHGOT1a8eHSbewp0jWXRaR2qRCC3Nje6RwsLee9RbqamojULhVCkOuenkBmcpyGshCRWqdCCHJmxtg+6czO3sWGnYV+xxGRekyFEALO75WGGdq5LCK1SoUQAlITGjE4M4nX52/GOed3HBGpp1QIIeKC3mlsyj/A7OxdfkcRkXpKhRAiTu/aktiocA1lISK1RoUQImKjIjijayrvLsrR5TVFpFaoEELI2N5p7D1YyofLtvsdRUTqIRVCCBmYkUha00Y8/+UGv6OISD2kQgghYWHG+CHtmZWdz5zsfL/jiEg9o0IIMZf0b02z2Ej++clav6OISD0TkoVgZmeb2eMFBQV+R6lzsVERXDW4PR+vyNXV1ESkRoVkITjn3nbOXZeQkOB3FF9ccVI74qLCeUxrCSJSg0KyEBq6hNhIfjSwLe8s2qrxjUSkxqgQQtT4Ie2JCA9j4ox1fkcRkXpChRCiUprEcGGfdF6bu5nte4r8jiMi9YAKIYT9+JQMSsvLmfTZer+jiEg9oEIIYW0T4zireyte+GoDBft1iU0ROT4qhBA3YVgmhcVlPPNltt9RRCTEqRBC3AmpTTi1cwpPf76e/cWlfscRkRCmQqgHbhieya79Jbw0a5PfUUQkhKkQ6oE+bZvTv31znpy5juLScr/jiEiIUiHUEzcMyySnoIg35uu6yyJSPSqEemJox2RObNWEiTPWUlau6y6LSNWpEOoJM+OGYVmsyyvk/SU5fscRkRCkQqhHTu/akg4pjXlg6ipKy7QvQUSqRoVQj4SHGXeM7sS6HYW8Nm+z33FEJMSoEOqZUV1a0LN1Ux76aDVFJWV+xxGREKJCqGfMjLtO70ROQRHPf6VrL4tI5akQ6qFBmUmc3CGJf0xfw94ijXEkIpWjQqin7hrdmV37S3hipkZCFZHKUSHUU93SEzizWypPzlxH3r6DfscRkRCgQqjHbhvVkYOl5Tz68Rq/o4hICKhUIZhZnJmFefc7mtk5ZhZZu9HkeGUmN+aiPum88PUGNuXv9zuOiAS5yq4hfArEmFka8CFwGTC5tkJJzfnJyA6YGQ99tNrvKCIS5CpbCOac2w9cAPzTOXcRcGLtxZKakprQiCsHteP1+ZtZuW2v33FEJIhVuhDM7CTgh8C73rTw2okkNW3C0EwaR0Vw/4cr/Y4iIkGssoVwK3AP8B/n3FIzywCm114sqUnN4qK47pQMpi7bzryNu/yOIyJBqlKF4Jyb4Zw7xzl3n7dzOc85d0stZ5MadPWQ9iQ1juK+91fgnIbHFpHvquxRRi+aWRMziwOWAMvM7M7ajSY1KS46gpuGZ/H1+nxmrs7zO46IBKHKbjLq4pzbA5wHvA+0J3CkkYSQSwe0oVVCDH+bukprCSLyHZUthEjvvIPzgLeccyWAvlFCTHREOLec2oGFm3bz8Ypcv+OISJCpbCH8C8gG4oBPzawtsKe2QkntGdsnnTbNY/nbh6so16U2RaSCyu5UfsQ5l+acG+MCNgDDazmb1ILI8DBuHdmBZTl7+O/SbX7HEZEgUtmdyglm9oCZzfFufyOwtlBjzCzDzCaZ2as1OV/5rnN7ppGZHMcDU1dRprUEEfFUdpPRU8Be4GLvtgd4+vveZGZPmVmumS05bPrpZrbSzNaY2d0Azrl1zrnxVYsv1REeZtw6siOrc/fxzqKtfscRkSBR2ULIdM792vvSXuec+y2QUYn3TQZOrzjBzMKBfwBnAF2AS82sSxUySw04s1sqnVvG89BHqyktK/c7jogEgcoWwgEzG3LogZkNBg5835ucc58C+YdN7g+s8YqlGHgZOLeSOaSGhIUZPz2tI+vzCnl9/ha/44hIEKhsIVwP/MPMss0sG3gU+HE1PzMN2FTh8WYgzcwSzWwi0MvM7jnam83sukP7Mnbs2FHNCAIwqksLuqUl8Mi01RSXai1BpKGr7FFGC51zPYDuQHfnXC9gRE0Gcc7tdM5d75zLdM796Rive9w519c51zc5ObkmIzQ4ZsZtozqyedcB/j1n0/e/QUTqtSpdMc05t8c7Yxngtmp+5hagdYXH6d408cGwjsn0aduMRz9eQ1FJmd9xRMRHx3MJTavm+2YDHcysvZlFAZcAbx1HDjkOZsbtp3Vk254iXpq10e84IuKj4ymE7z2A3cxeAr4EOpnZZjMb75wrBW4C/gssB/7tnFt6HDnkOA3KSuKkjET+MX0tB4q1liDSUB2zEMxsr5ntOcJtL9Dq+2bunLvUOZfqnIt0zqU75yZ5099zznX09hfcW9XQZna2mT1eUFBQ1bfKUdw+qiN5+w7y7JfZfkcREZ8csxCcc/HOuSZHuMU75yLqKuQRcr3tnLsuISHBrwj1Tt92zRnaMZmJM9ayt6jE7zgi4oPj2WQk9cztozqya38JT32W7XcUEfGBCkG+0T29KaNPbMETM9exq7DY7zgiUsdUCPItt4/qRGFxKRM/Xet3FBGpYyoE+ZaOLeI5v2caz3yRzfY9RX7HEZE6FJKFoKOMatetIztSWuZ49OM1fkcRkToUkoWgo4xqV5vEWMb1a83LszeyKX+/33FEpI6EZCFI7bt5RAfCzHjoo9V+RxGROqJCkCNqmRDD5Se15T/zN7Mmd6/fcUSkDqgQ5KgmDMuiUWQ4D0xd5XcUEakDKgQ5quZxUVxzcgbvLd7Gki3agS9S36kQ5JiuObk9TWMjuf/DlX5HEZFaFpKFoMNO6058TCQThmbyycodzM4+/GqoIlKfhGQh6LDTunX5Se1IiY/mrx+sxLnvHfVcREJUSBaC1K1GUeHcPCKLWdn5fLo6z+84IlJLfBvCWkLLuH5t+Nen67j22TmkN2tEWtNGtEpoRKumjWjVNCbwuGkj2jSPJSysuhfTExE/qRCkUqIiwph0RT9embOJrQUH2LK7iBXbctmx9+C3Xjc4K5EnL+9Ho6hwn5KKSHVZKG8T7tu3r5szZ47fMRq0g6VlbCsoYsvuA8zfuJv7P1zJ4MwknryiLzGRKgWRYGRmc51zfQ+frn0IclyiI8JpmxjHoMwkbhyexV8v7MHna/P48XNzKSrR9ZlFQokKQWrUhX3S+fMF3Zixagc3vDCP4tJyvyOJSCWpEKTGjevXhj+c15WPV+Ry04vzKClTKYiEgpAsBJ2YFvx+NLAtvz3nRD5ctp2fvDyfUpWCSNALyULQiWmh4YpB7fjFmSfw3uJt/PTfC1UKIkFOh51Krbrm5AxKyx1/fn8FEWHG/Rf1IFznKYgEJRWC1Lrrh2ZSWlbO/R+uIiYynD+e3xUzlYJIsFEhSJ24aUQHCovLeOyTtSTHR3PbaR39jiQih1EhSJ25a3Qndu47yCPTVpPcOIrLTmrndyQRqUCFIHXGzPjj+d3ILyzmV28tJbFxNGO6pfodS0Q8IXmUkYSuiPAw/n5pb/q0acatLy/gi7UaPVUkWKgQpM41igrnySv60i4pluuenavLc4oEiZAsBJ2YFvqaxkbxzNX9aRITwZVPz2bDzkK/I4k0eCFZCDoxrX5ITWjEs+P7U1pezuVPzfrOUNoiUrdCshCk/shKiefpK/uRu+cgVzw1i49XbGdvUYnfsUQaJF0PQYLC9JW53PjCPPYXlxEeZnRLS2BQZiKDMpPo07aZLrgjUoOOdj0EFYIEjaKSMuZt2MUXa3fy5bqdLNy0m9JyR1R4GD3bNGVM15ZcMaidznIWOU5HKwSdhyBBIyYynEFZSQzKSgJg38FSZmfn89XancxYtYPfvL2MtolxDO+c4nNSkfpJ+xAkaDWOjmB4pxTuGXMCb988hLaJsfzlvyspLw/dtVqRYKZCkJAQGR7Gbad1ZHnOHt5etNXvOCL1kgpBQsbZ3VtxQmoTHpi6SldhE6kFKgQJGWFhxl2jO7Fh536mzN7kdxyRekeFICFlWKdk+rVrxiPTVnOguKzS71u1fS9LthRo/4PIMegoIwkpZsZdp3fmoolfMvmLbCYMy/ze98zdkM8Pn/yaopJykuOjGdYxmRGdUxjSIYn4mMg6SC0SGkKyEMzsbODsrKwsv6OID/q1a86Izik89skaftC/DQmxR/9SX7V9L1dPnkPLJjHcMCyLT1fv4L9Lt/HK3M1EhBn92jVneOdAQWQmN/blHIdpy7fTpFEk/do1r/PPFqlIJ6ZJSFq2dQ9jHpnJDcMyuev0zkd8zZbdBxj7zy8oc47XJwyidfNYAErLypm3cTfTV+YyfUUuK7btBWBQZiI/P/METmxVd2NkrcndxxkPf0p4mPHKjwfRLV3jc0nt05nKUu/c8tJ8pi7bzow7h5HSJOZbz+UXFnPRxC/I3XOQKT8+iS6tmhx1Plt3H+DdRTn845M1FBwo4eI+rbl9VMfvzLOmOef40aSvWby5gPiYSMrKHW/dNLjWP1fkaIWgncoSsm47rSMlZeX8/eM135q+v7iUqyfPZtOuAzx5Rd9jlgFAq6aNuPaUDGbcMZxrhrTn9fmbGXb/J/y9ijuuq+qdRTl8vmYnd47uxOOX96HgQAnXPTeXopLa+0yRY1EhSMhqlxTHuH6teWnWRjbu3A9ASVk5E56fx6LNu/n7pb0YkJFY6fklxEby8zO78NFtQzmlQzJ/m7qKEX/7hDfmb6nxo5P2HSzlD+8uo2taE34woC0ntkrgwXE9WLBpN/e8vphQXnOX0KVCkJB2y6kdiAg3HpgaGNLizlcWMmPVDv54fjdGn9iyWvNsmxjHxMv6MOW6gSQ1jubWKQs4/5+fs2X3gRrL/fBHq8jde5Dfn9uV8LDAjuzTu6Zy+2kd+c/8Lfzr03U19lkilaVCkJDWokkMVw5qz5sLt3LzS/N5Y8FW7hjVkUv6tznueQ/ISOTNGwfzwMU9WJdXyOWTvmZXYfFxz3fltr089Xk2l/RrTa82zb713E0jsjireyr3fbCCj5ZtP+7PEqkKFYKEvAlDM2kcHcG7i3O4clA7bhxec4cjh4UZF/RO58nL+7Jp1wGumjyb/cWl1Z6fc45fvrmE+JgI7hz93aOjzIy/XtiDrq0S+MnL81npHQElUhdUCBLyEmIjuW9sd24ansWvzupSK+cSDMhI5O+X9mLR5t3c+MK8ao+l9MaCLcxan8/PTu9M87ioI76mUVQ4T1zel9joCK55djb5NbBWIlIZKgSpF8Z0S+WO0Z0IC6u9E8tGn9iSP5zXjekrd3D3a1Xf8VtwoIR7311Bj9ZNGde39TFf2zIhhscv68P2PQeZ8Pxciks1mJ/UPhWCSBX8YEAbfjqyI6/N28x9H6ys0nsfnLqKnYUH+cO5XStVXL3aNOMvY7vz9fp8HXkkdSIkh64Q8dMtp2axY18RE2esJalxFNecnPG971m6tYBnv8zmRwPaVuls5PN6pZG9s5CHPlpNcnw0d59x5LOyRWqCCkGkisyM357TlZ37ivnDu8tJahzNeb3Sjvr68nLHL99YQrPYKO4Y1anKn/eTUzuwY+/BKhWQSHWoEESqITzMeHBcT3btn8UdrywkNiqcrJTG7Npfwu79xezaX8KuwmJ27S9mfV4h8zbu5q8Xdj/mQHxHY2b87tyu5BdWroBEqkuFIFJNMZHhPH55X8b96yuue27uEV8THmY0i43kgt5pjO2dXu3POryAmsVFMbRjcrXnV1PeW5xDt7SEbwYOlNAWkoPbVRj++trVq1f7HUcauPzCYt5fkkNsVDhNY6NoFhtFs9hImsVFER8dUaOHwe4pKmHcv75iw85CXrx2ID1bN62xeVfVqu17GfXgp2QkxfH2zUOIi9bfl6FCo52K1BO5e4sY+9gX7Csq5dUJg8hMbuxLjt+9vYxnv8ym3DnO65XGAxf39CWHVJ1GOxWpJ1LiY3ju6gGEhxmXT5rFtoKiOs9QVFLG6/M3M/rEltw8ogOvz9vCa3M313kOqVlaxxMJQe2S4ph8VX/G/etLLn/qay7u25pmsVE0j4uiWVwUzWOjaBYXSeMa3mR1yH+XbmP3/hIu6d+aQZlJfLVuJ798cwk92zT1bY2lLq3avpev1+cztncasVH152tUm4xEQtjna/KY8Pxc9hQdeXylyHCjQ0o8z1zdn+T46Br73Ese/5Ituw8w447hhIUZOQUHGPPwTFITGvH6DYOIiQyvsc8KNgUHShjz8Ey27D5AUuMorh+ayY8Gtg2pn1n7EETqKeccew+WsquwmHzvUNf8wsBhr3n7DvLU5+s5q3srHhxXM9v41+cVMvz+T7hzdKdvDSQ4bfl2xj8zhytOastvz+1aI58VbJxz3PjiPD5cup17z+/K2wtz+GxNHinx0dw4PItL+rcmOiL4i+FohVB/1nVEGigzo0lMJE1iImmbGPed5yPDw3h0+hou7JPO4Kyk4/68l2dvJDzMuKjPtw+jPfWEFowf0p5Jn61nUFZSta9HEcxenr2J9xZv4+4zOjOuXxvG9WvD1+t28sDUVfz6raVMnLGWm0ZkcVGf1kRFhN4u2tBLLCJVctOILNomxvKLN5Yc9+U5i0vLeW3uZkZ0TjnitZ/vOr0T3dISuOvVRce8oNC+g6X8Z/5mHpy6itJqjhxb11Zv38tv317KyR2SuK7C2eIDMhJ5+bqBvHDNAFITYvj5f5Z8c6W9UNsCo0IQqediIsP53bldWZ9XyMQZa49rXh8t307evmIu7X/k0VqjI8L5+6W9KCt33PLS/G992ReVlPH+4hwmPD+XPr+fyk+nLOThaauZvnLHcWWqC0UlZdz80nzioiL428U9vjM4oZkxOCuJ1yYMYvJV/WgWG8WtUxYw/pk55BTU3JX2apsKQaQBGNoxmbO6p/LP6WtZt2Nftefz0qyNpCbEMLRjylFf0y4pjnvP78rcDbu4/8NVfLxiOz+dsoA+v5/KhBfmMTs7n0v6tfYuURrFq3M3VTtPXfnTe8tZsW0v91/cg5T4764ZHWJmDOuUwhs3DuZXZ3Xhi7V5jHrgU6bM3hgSawvahyDSQPzqrC7MWLmDX765hOfHD6jy4aib8vfz2Zo8bhnR4ZvrQB/NuT3T+HxNHhNnrGXiDEhoFMnZPVpxdo9WDGjfnIjwwN+i5/VMY/IX2ezcd5DExjV3FFRNmrpsO898uYFrhrRneKejF2FF4WHG1SsEuaEAAAyASURBVEPac+oJKdz16iJ+9tpi3lmUw58u6EZ6s+Ad5kNrCCINREqTGO46vROfr9nJmwu2Vvn9/54T+Ev+4n7HvrjPIb89pys/HdmRp67sy+yfj+TPY7szOCvpmzIAuLBvOqXljrcWVj3P8XDOsT6vkIOlx96nklNwgDtfXUjXtCbceXrVR6ptmxjHS9cO5PfnnsjcDbsY/eCnPPfVBsrLg3NtQYedijQgZeWOCx77gi279jPttmGVHn21tKycwfd9zAmpTZh8Vf8azXT23z+j3DneveXkGp3vkZSXOz5ctp3HPlnDws0FNI6OYGinZEZ1acGwTikkNPrf8igrd/zgia9YvKWAd24eQsZxnnC3KX8/97y+mM/W5DEwozl3jOpErzbNvndtqzbosFMRITzMuPe8rpzz6Gfc998V/PH8bpV63/SVO9i+5yC/O7dNjWe6sE86v35rKcu27qFLqyY1Pn+AkrJy3lywlYkz1rImdx9tmsfyf2M6sz6vkKnLcnl3UQ4RYcbAjERO69KC07q04LW5m/l6fT5/u6jHcZcBQOvmsTw3vj9TZm/i3neXc+HEL2keF8XwTimc1iWFkzskH3OAQOccm3cdYMW2vazI2cM1J2fQKKpmz3nQGoJIA/T7d5Yx6bP1vDZhEH3aNvve14+fPJtFWwr44u4RRIbX7JbmXYXF9P/jR1x+Ujt+eVaXGp33geIypszeyBMz17Nl9wE6t4xnwrBMzuyW+s2mq/Jyx/xNu5m6bDsfLtvGuh2FAJjBuT0CJ/TV9PAfBQdKmLFqB9OWb2f6ilz2FJUSFR7GwMxERp4QKIf8wmJWbNvDipy93/y79+D/zkh/95YhnNiq8lffq0hnKovINwoPljLygRkkNIrk7ZuHHPNLPqfgAIP//DHXD83krtNr5xKeE56fy6z1+Xz1f6fWSOEcLC1j0mfrmTRzPTsLi+nbthk3DM9keKeU7/1yX7tjH1OXbWfdjn388qwuxMdU/aJGVVFSVs6c7F1MW76daStyWZ9X+K3n46Mj6JwazwmpTejcsgmdU+Pp1CL+uIYb1yYjEflGXHQEvznnRH783Fz+OT1wdu3RtmW/Mmcz5Q7GVXJncnVc2Ced95dsY/qKXEYd5xnOc7Lzufv1xazJ3cewTsncMCyL/u2bV/r9mcmNyRxadwP0RYaHcVJmIidlJvKLs7qwdsc+vlq3kxbxMXROjSetaaNaGaDwSFQIIg3U6BNbMvrEFjz40Sqe/TKbUSe24PSuqQzKTPzmr/SycseU2ZsYnJV4xGExasrQjskkNY7m1bmbq10Ie4pKuO/9Fbzw9UbSmjbi6Sv7Mbxz5Q4TDSaZyY19GzFWhSDSgD1yaS+mLc/l/SXbeGvBVl6atYkmMRGM7NKCM7qmUu4cW3Yf4J4xtbOp6JCI8DAu6J3GU5+tr9Y5CR8s2cav31rCjr0HGT+kPbed1lFXcKsGLTGRBiw6Ipwx3VIZ0y2VopIyZq7O4/0lOXy0bDuvz9sCQPO4KE7r0qLWs4ztnc7jn67jzQVbuXpI+0q9Z/ueIn715hL+u3Q7J6Q24fHL+tLDx8uKhjoVgogAgTGPDh1yWVxazpfrdjJ12Tb6tm1eJ0M6d2oZT/f0BF6du7lShfDa3M385q2lFJeVc/cZnRk/pH2NHwHV0IRkIZjZ2cDZWVlZ3/taEam6qIgwhnZMZmjH5Dr93Av7pPOrN5eydGvBMQ+pfOaLbH791lIGtG/OXy7sXqv7NxqSkKxT59zbzrnrEhKqdwyuiASns7u3Iio8jFePcX3mpz5bz6/fWsppXVrw3PgBKoMaFJKFICL1U7O4KEZ2SeHNBVspLv3udRKenLmO372zjNEntuAfP+gdkhehCWZamiISVC7sk05+YTHTV+Z+a/oTn67jD+8u54yuLXlUZVArtERFJKic0iGZ5Pjob202+teMtdz73nLO7JbKI5f20s7jWqKlKiJBJSI8jAt6pTF9RS55+w7y2Cdr+dP7KzireyoPX9JTZVCLtGRFJOiM7RO4TsI1z8zhvg9WcHaPVjw0rue3rqUgNU9LV0SCTscW8fRIT2DBpt2c27MVD17cQ2VQB0LyPAQRqf9+dXYXvlqXz/VDM325iExDpEIQkaDUp21z+rSt/Cilcvy0DiYiIoAKQUREPCoEEREBVAgiIuJRIYiICKBCEBERjwpBREQAFYKIiHjMOed3hmozsx3AhqM8nQTk1WGcqlC26lG26lG26qnP2do6575zObyQLoRjMbM5zrm+fuc4EmWrHmWrHmWrnoaYTZuMREQEUCGIiIinPhfC434HOAZlqx5lqx5lq54Gl63e7kMQEZGqqc9rCCIiUgUqBBERAeppIZjZ6Wa20szWmNndfuepyMyyzWyxmS0wszk+Z3nKzHLNbEmFac3NbKqZrfb+bRZE2X5jZlu8ZbfAzMb4lK21mU03s2VmttTMfuJN933ZHSOb78vOzGLMbJaZLfSy/dab3t7MvvZ+X6eYWVQQZZtsZusrLLeedZ2tQsZwM5tvZu94j2t+uTnn6tUNCAfWAhlAFLAQ6OJ3rgr5soEkv3N4WU4BegNLKkz7C3C3d/9u4L4gyvYb4I4gWG6pQG/vfjywCugSDMvuGNl8X3aAAY29+5HA18BA4N/AJd70icCEIMo2GbjQ7//nvFy3AS8C73iPa3y51cc1hP7AGufcOudcMfAycK7PmYKSc+5TIP+wyecCz3j3nwHOq9NQnqNkCwrOuRzn3Dzv/l5gOZBGECy7Y2TznQvY5z2M9G4OGAG86k33a7kdLVtQMLN04EzgSe+xUQvLrT4WQhqwqcLjzQTJL4THAR+a2Vwzu87vMEfQwjmX493fBrTwM8wR3GRmi7xNSr5szqrIzNoBvQj8RRlUy+6wbBAEy87b7LEAyAWmElib3+2cK/Ve4tvv6+HZnHOHltu93nJ70Myi/cgGPATcBZR7jxOpheVWHwsh2A1xzvUGzgBuNLNT/A50NC6wLho0fyUBjwGZQE8gB/ibn2HMrDHwGnCrc25Pxef8XnZHyBYUy845V+ac6wmkE1ib7+xHjiM5PJuZdQXuIZCxH9Ac+Fld5zKzs4Bc59zc2v6s+lgIW4DWFR6ne9OCgnNui/dvLvAfAr8UwWS7maUCeP/m+pznG8657d4vbTnwBD4uOzOLJPCF+4Jz7nVvclAsuyNlC6Zl5+XZDUwHTgKamlmE95Tvv68Vsp3ubYJzzrmDwNP4s9wGA+eYWTaBTeAjgIepheVWHwthNtDB2wMfBVwCvOVzJgDMLM7M4g/dB0YBS479rjr3FnCFd/8K4E0fs3zLoS9bz/n4tOy87beTgOXOuQcqPOX7sjtatmBYdmaWbGZNvfuNgNMI7OOYDlzovcyv5XakbCsqFLwR2EZf58vNOXePcy7dOdeOwPfZx865H1Iby83vPee1cQPGEDi6Yi3wc7/zVMiVQeCop4XAUr+zAS8R2HxQQmAb5HgC2yanAauBj4DmQZTtOWAxsIjAl2+qT9mGENgctAhY4N3GBMOyO0Y235cd0B2Y72VYAvzKm54BzALWAK8A0UGU7WNvuS0Bnsc7EsmvGzCM/x1lVOPLTUNXiIgIUD83GYmISDWoEEREBFAhiIiIR4UgIiKACkFERDwqBJHDmFlZhdEtF1gNjphrZu0qjuAqEkwivv8lIg3OARcYwkCkQdEagkglWeBaFn+xwPUsZplZlje9nZl97A2ANs3M2njTW5jZf7wx9hea2SBvVuFm9oQ37v6H3pmxmNkt3nUMFpnZyz79mNKAqRBEvqvRYZuMxlV4rsA51w14lMAIlAB/B55xznUHXgAe8aY/AsxwzvUgcG2Hpd70DsA/nHMnAruBsd70u4Fe3nyur60fTuRodKayyGHMbJ9zrvERpmcDI5xz67wB5LY55xLNLI/AUBAl3vQc51ySme0A0l1gYLRD82hHYGjlDt7jnwGRzrk/mNkHwD7gDeAN97/x+UXqhNYQRKrGHeV+VRyscL+M/+3LOxP4B4G1idkVRrIUqRMqBJGqGVfh3y+9+18QGIUS4IfATO/+NGACfHPxlYSjzdTMwoDWzrnpBMbcTwC+s5YiUpv0F4jIdzXyrpx1yAfOuUOHnjYzs0UE/sq/1Jt2M/C0md0J7ACu8qb/BHjczMYTWBOYQGAE1yMJB573SsOAR1xgXH6ROqN9CCKV5O1D6Oucy/M7i0ht0CYjEREBtIYgIiIerSGIiAigQhAREY8KQUREABWCiIh4VAgiIgLA/wPpQSfv6+NC/QAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Testing results\n",
            "Testing accuracy: 5596/6149 (91.01%)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<Figure size 18000x18000 with 0 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def createConfusionMatrix(data_loader, model):\n",
        "    confusion_matrix = torch.zeros((102, 102))\n",
        "    with torch.no_grad():\n",
        "        for i, (inputs, classes) in enumerate(data_loader):\n",
        "            inputs = inputs.to(device)\n",
        "            classes = classes.to(device)\n",
        "            outputs = model(inputs)\n",
        "            _, preds = torch.max(outputs, 1)\n",
        "            for t, p in zip(classes.view(-1), preds.view(-1)):\n",
        "                    confusion_matrix[t.long(), p.long()] += 1\n",
        "\n",
        "    fig = plt.figure(figsize=(250,250))\n",
        "\n",
        "    # confusion_matrix_np = confusion_matrix.numpy()\n",
        "    # np.savetxt(\"confusion_matrix.csv\", confusion_matrix_np, delimiter=\",\")\n",
        "    \n",
        "    # class_names = list(labelReference.values())\n",
        "    df_cm = pd.DataFrame(confusion_matrix).astype(int)\n",
        "    df_cm.to_csv('confusion_matrix.csv')\n",
        "\n",
        "    # heatmap = sn.heatmap(df_cm, annot=True, fmt=\"d\")\n",
        "\n",
        "    # heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right',fontsize=10)\n",
        "    # heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right',fontsize=10)\n",
        "    # plt.ylabel('True label')\n",
        "    # plt.xlabel('Predicted label')\n",
        "    # return fig\n",
        "\n",
        "print(\"Training/Validation results\")\n",
        "formatted_training_results_data = pd.DataFrame(\n",
        "    {\"Epoch\": [], \"Predictions/Samples\": [], \"Accuracy\": [], \"Loss\": []}\n",
        ")\n",
        "for index, row in training_results_data.iterrows():\n",
        "    formatted_training_results_data.loc[len(formatted_training_results_data.index)] = [\n",
        "            row[0],\n",
        "            f\"{int(row[1])}/{int(row[2])}\",\n",
        "            f\"{row[3]:.2f}%\",\n",
        "            f\"{row[4]}\",\n",
        "        ]\n",
        "\n",
        "print(tabulate(formatted_training_results_data, headers = 'keys', tablefmt = 'fancy_grid', showindex=\"never\"))\n",
        "\n",
        "plt.plot(training_results_data[\"Epoch\"], training_results_data[\"Accuracy\"])\n",
        "plt.title('Accuracy over epochs')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy (%)')\n",
        "plt.yscale('log')\n",
        "# plt.grid(True)\n",
        "plt.show()\n",
        "\n",
        "plt.plot(training_results_data[\"Epoch\"], training_results_data[\"Loss\"])\n",
        "plt.title('Loss over epochs')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.yscale('log')\n",
        "# plt.grid(True)\n",
        "plt.show()\n",
        "\n",
        "print(\"Testing results\")\n",
        "n_corrects, n_samples, accuracy = check_accuracy(test_loader, model)\n",
        "print(f\"Testing accuracy: {n_corrects}/{n_samples} ({accuracy:.2f}%)\")\n",
        "writer.add_figure(\"Confusion matrix\", createConfusionMatrix(test_loader, model))\n",
        "writer.flush()\n",
        "writer.close()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}