train_with_vgg16.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5-final"
    },
    "orig_nbformat": 2,
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.8.5 64-bit",
      "metadata": {
        "interpreter": {
          "hash": "0e0d1c7ef0f381ce9c31735005e25185fd13b9c57d8e85878ff9ff982cb55e39"
        }
      }
    },
    "colab": {
      "name": "train_with_vgg16.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "SfIDt-ZZsIHm"
      },
      "source": [
        "# import libraries\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data.sampler import SubsetRandomSampler\n",
        "\n",
        "import numpy as np\n",
        "from tqdm import tqdm\n",
        "\n",
        "from torch.utils.tensorboard import SummaryWriter\n",
        "from utils import device, get_num_correct\n",
        "from vgg16modified import Network"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BrVtDiKMMQoG"
      },
      "source": [
        "# declare the transforms\n",
        "data_transforms = {\n",
        "    'train': transforms.Compose([\n",
        "        # add augmentations\n",
        "        transforms.ColorJitter(brightness=0.25, saturation=0.1),\n",
        "        transforms.RandomHorizontalFlip(p=0.5),\n",
        "        transforms.RandomRotation(10),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize(\n",
        "            mean=[0.485, 0.456, 0.406],\n",
        "            std=[0.229, 0.224, 0.225]\n",
        "        )\n",
        "    ]),\n",
        "    'test': transforms.Compose([\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize(\n",
        "            mean=[0.485, 0.456, 0.406],\n",
        "            std=[0.229, 0.224, 0.225]\n",
        "        )\n",
        "    ])\n",
        "}"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JGmTRbzEsIHr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "32eca5b0-9f07-4d7a-8c62-f906def18313"
      },
      "source": [
        "# choose the training and test datasets\n",
        "train_set = torchvision.datasets.CIFAR10(\n",
        "    root='./data/',\n",
        "    train=True,\n",
        "    download=True,\n",
        "    transform=data_transforms['train']\n",
        ")\n",
        "test_set = torchvision.datasets.CIFAR10(\n",
        "    root='./data/',\n",
        "    train=False,\n",
        "    download=True,\n",
        "    transform=data_transforms['test']\n",
        ")"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uoWPNbbVsIH0"
      },
      "source": [
        "batch_size = 256\n",
        "valid_size = 0.5  # percentage of test_set to be used as validation\n",
        "\n",
        "# obtain training indices that will be used for validation\n",
        "num_test = len(test_set)\n",
        "indices = list(range(num_test))\n",
        "np.random.shuffle(indices)\n",
        "split = int(np.floor(valid_size * num_test))\n",
        "test_idx, valid_idx = indices[split:], indices[:split]\n",
        "\n",
        "# define samplers for obtaining training and validation batches\n",
        "valid_sampler = SubsetRandomSampler(valid_idx)\n",
        "test_sampler = SubsetRandomSampler(test_idx)\n",
        "\n",
        "# prepare the data loaders\n",
        "train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=1)\n",
        "valid_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, sampler=valid_sampler, num_workers=1)\n",
        "test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, sampler=test_sampler, num_workers=1)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wIQ8mJNfPSeD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6a5b7cc9-0e09-4478-8fc7-a3031b50b38f"
      },
      "source": [
        "vgg16 = torchvision.models.vgg16(pretrained=True)\n",
        "vgg16"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VGG(\n",
              "  (features): Sequential(\n",
              "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (3): ReLU(inplace=True)\n",
              "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (6): ReLU(inplace=True)\n",
              "    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (8): ReLU(inplace=True)\n",
              "    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (11): ReLU(inplace=True)\n",
              "    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (13): ReLU(inplace=True)\n",
              "    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (15): ReLU(inplace=True)\n",
              "    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (18): ReLU(inplace=True)\n",
              "    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (20): ReLU(inplace=True)\n",
              "    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (22): ReLU(inplace=True)\n",
              "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (25): ReLU(inplace=True)\n",
              "    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (27): ReLU(inplace=True)\n",
              "    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (29): ReLU(inplace=True)\n",
              "    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  )\n",
              "  (avgpool): AdaptiveAvgPool2d(output_size=(7, 7))\n",
              "  (classifier): Sequential(\n",
              "    (0): Linear(in_features=25088, out_features=4096, bias=True)\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Dropout(p=0.5, inplace=False)\n",
              "    (3): Linear(in_features=4096, out_features=4096, bias=True)\n",
              "    (4): ReLU(inplace=True)\n",
              "    (5): Dropout(p=0.5, inplace=False)\n",
              "    (6): Linear(in_features=4096, out_features=1000, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "umFYw1-cW_jM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47cf0700-e5ca-48f6-a124-fc0eb3506904"
      },
      "source": [
        "# replace the vgg16 classifier\n",
        "model = Network(vgg16)\n",
        "model"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Network(\n",
              "  (features): Sequential(\n",
              "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (3): ReLU(inplace=True)\n",
              "    (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (6): ReLU(inplace=True)\n",
              "    (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (8): ReLU(inplace=True)\n",
              "    (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (11): ReLU(inplace=True)\n",
              "    (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (13): ReLU(inplace=True)\n",
              "    (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (15): ReLU(inplace=True)\n",
              "    (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (18): ReLU(inplace=True)\n",
              "    (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (20): ReLU(inplace=True)\n",
              "    (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (22): ReLU(inplace=True)\n",
              "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "    (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (25): ReLU(inplace=True)\n",
              "    (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (27): ReLU(inplace=True)\n",
              "    (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (29): ReLU(inplace=True)\n",
              "    (30): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
              "  )\n",
              "  (classifier): Sequential(\n",
              "    (0): Linear(in_features=512, out_features=256, bias=True)\n",
              "    (1): ReLU(inplace=True)\n",
              "    (2): Linear(in_features=256, out_features=128, bias=True)\n",
              "    (3): ReLU(inplace=True)\n",
              "    (4): Linear(in_features=128, out_features=10, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4pDgSH74Xlhp"
      },
      "source": [
        "# transfer learning (first 8 layers of vgg16)\n",
        "# freeze the transferred weights which won't be trained\n",
        "for layer_num, child in enumerate(model.features.children()):\n",
        "    if layer_num < 19:\n",
        "        for param in child.parameters():\n",
        "            param.requires_grad_(False)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iUq5gbZRsIH6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "33d85345-d312-4815-cad4-c033a3c0f0bb"
      },
      "source": [
        "model.to(device)\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()  # loss function (categorical cross-entropy)\n",
        "optimizer = optim.SGD(\n",
        "    [      # parameters which need optimization\n",
        "        {'params':model.features[19:].parameters(), 'lr':0.001},\n",
        "        {'params':model.classifier.parameters()}\n",
        "    ], lr=0.01, momentum=0.9, weight_decay=1e-3)\n",
        "\n",
        "scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=1/3, patience=5, verbose=True) # lr scheduler\n",
        "\n",
        "comment = f'-transferlr_vgg16'  # will be used for naming the run\n",
        "tb = SummaryWriter(comment=comment)\n",
        "\n",
        "# initialize tracker for minimum validation loss\n",
        "valid_loss_min = np.Inf  # set initial minimum to infinity\n",
        "num_epochs = 30  # number of epochs used for training\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "    train_loss, train_correct = 0, 0  # wil be used to track the running loss and correct\n",
        "    ###################\n",
        "    # train the model #\n",
        "    ###################\n",
        "    train_loop = tqdm(train_loader)\n",
        "    model.train()  # set the model to train mode\n",
        "    for batch in train_loop:\n",
        "        images, labels = batch[0].to(device), batch[1].to(device)  # load the batch to the available device (cpu/gpu)\n",
        "        preds = model(images)  # forward pass\n",
        "        loss = criterion(preds, labels)  # calculate loss\n",
        "        optimizer.zero_grad()  # clear accumulated gradients from the previous pass\n",
        "        loss.backward()  # backward pass\n",
        "        optimizer.step()  # perform a single optimization step\n",
        "\n",
        "        train_loss += loss.item() * labels.size(0) # update the running loss\n",
        "        train_correct += get_num_correct(preds, labels)  # update running num correct\n",
        "\n",
        "        train_loop.set_description(f'Epoch [{epoch+1:2d}/{num_epochs}]')\n",
        "        train_loop.set_postfix(loss=loss.item(), acc=train_correct/len(train_set))\n",
        "\n",
        "    # add train loss and train accuracy for the current epoch to tensorboard\n",
        "    tb.add_scalar('Train Loss', train_loss, epoch)\n",
        "    tb.add_scalar('Train Accuracy', train_correct/len(train_set), epoch)\n",
        "\n",
        "    model.eval()  # set the model to evaluation mode\n",
        "    with torch.no_grad():  # turn off grad tracking, as we don't need gradients for validation\n",
        "        valid_loss, valid_correct = 0, 0  # will be used to track the running validation loss and correct\n",
        "        ######################\n",
        "        # validate the model #\n",
        "        ######################\n",
        "        for batch in valid_loader:\n",
        "            images, labels = batch[0].to(device), batch[1].to(device)  # load the batch to the available device\n",
        "            preds = model(images)  # forward pass\n",
        "            loss = criterion(preds, labels)  # calculate the loss\n",
        "\n",
        "            valid_loss += loss.item() * labels.size(0)  # update the running loss\n",
        "            valid_correct += get_num_correct(preds, labels)  # update running num correct\n",
        "            \n",
        "\n",
        "        # add validation loss and validation accuracy for the current epoch to tensorboard\n",
        "        tb.add_scalar('Validation Loss', valid_loss, epoch)\n",
        "        tb.add_scalar('Validation Accuracy', valid_correct/len(valid_loader.sampler), epoch)\n",
        "\n",
        "        # print training/validation statistics\n",
        "        # calculate average loss over an epoch\n",
        "        train_loss = train_loss/len(train_set)\n",
        "        valid_loss = valid_loss/len(valid_loader.sampler)\n",
        "        train_loop.write(f'\\t\\tAvg training loss: {train_loss:.6f}\\tAvg validation loss: {valid_loss:.6f}')\n",
        "        scheduler.step(valid_loss)\n",
        "\n",
        "        # save model if validation loss has decreased\n",
        "        if valid_loss <= valid_loss_min:\n",
        "            train_loop.write(f'\\t\\tvalid_loss decreased ({valid_loss_min:.6f} --> {valid_loss:.6f})  saving model...')\n",
        "            torch.save(model.state_dict(), f'./models/model{comment}.pth')\n",
        "            valid_loss_min = valid_loss\n",
        "\n",
        "\n",
        "        test_loss, test_correct = 0, 0  # will be used to track the running test loss and correct\n",
        "        ##################\n",
        "        # test the model #\n",
        "        ##################\n",
        "        for batch in test_loader:\n",
        "            images, labels = batch[0].to(device), batch[1].to(device)  # load the batch to the available device\n",
        "            preds = model(images)  # forward pass\n",
        "            loss = criterion(preds, labels)  # calculate the loss\n",
        "\n",
        "            test_loss += loss.item() * labels.size(0)  # update the running loss\n",
        "            test_correct += get_num_correct(preds, labels)  # update running num correct\n",
        "\n",
        "        # add test loss and test accuracy for the current epoch to tensorboard\n",
        "        tb.add_scalar('Test Loss', test_loss, epoch)\n",
        "        tb.add_scalar('Test Accuracy', test_correct/len(test_loader.sampler), epoch)\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 1/30]: 100%|██████████| 196/196 [00:42<00:00,  4.58it/s, acc=0.555, loss=0.899]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 1.258526\tAvg validation loss: 0.798657\n",
            "\t\tvalid_loss decreased (inf --> 0.798657)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 2/30]: 100%|██████████| 196/196 [00:42<00:00,  4.61it/s, acc=0.716, loss=0.674]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.808466\tAvg validation loss: 0.695511\n",
            "\t\tvalid_loss decreased (0.798657 --> 0.695511)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 3/30]: 100%|██████████| 196/196 [00:42<00:00,  4.60it/s, acc=0.749, loss=0.709]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.716494\tAvg validation loss: 0.607289\n",
            "\t\tvalid_loss decreased (0.695511 --> 0.607289)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 4/30]: 100%|██████████| 196/196 [00:41<00:00,  4.68it/s, acc=0.767, loss=0.586]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.663401\tAvg validation loss: 0.590412\n",
            "\t\tvalid_loss decreased (0.607289 --> 0.590412)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 5/30]: 100%|██████████| 196/196 [00:41<00:00,  4.69it/s, acc=0.782, loss=0.676]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.621336\tAvg validation loss: 0.545776\n",
            "\t\tvalid_loss decreased (0.590412 --> 0.545776)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 6/30]: 100%|██████████| 196/196 [00:43<00:00,  4.54it/s, acc=0.796, loss=0.637]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.581307\tAvg validation loss: 0.540139\n",
            "\t\tvalid_loss decreased (0.545776 --> 0.540139)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 7/30]: 100%|██████████| 196/196 [00:43<00:00,  4.55it/s, acc=0.802, loss=0.393]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.555703\tAvg validation loss: 0.506904\n",
            "\t\tvalid_loss decreased (0.540139 --> 0.506904)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 8/30]: 100%|██████████| 196/196 [00:42<00:00,  4.62it/s, acc=0.81, loss=0.413]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.540095\tAvg validation loss: 0.504078\n",
            "\t\tvalid_loss decreased (0.506904 --> 0.504078)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [ 9/30]: 100%|██████████| 196/196 [00:42<00:00,  4.62it/s, acc=0.819, loss=0.516]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.511801\tAvg validation loss: 0.499604\n",
            "\t\tvalid_loss decreased (0.504078 --> 0.499604)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [10/30]: 100%|██████████| 196/196 [00:42<00:00,  4.65it/s, acc=0.823, loss=0.419]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.498117\tAvg validation loss: 0.485179\n",
            "\t\tvalid_loss decreased (0.499604 --> 0.485179)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [11/30]: 100%|██████████| 196/196 [00:42<00:00,  4.64it/s, acc=0.831, loss=0.484]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.475245\tAvg validation loss: 0.477229\n",
            "\t\tvalid_loss decreased (0.485179 --> 0.477229)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [12/30]: 100%|██████████| 196/196 [00:42<00:00,  4.60it/s, acc=0.835, loss=0.579]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.464521\tAvg validation loss: 0.477346\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [13/30]: 100%|██████████| 196/196 [00:42<00:00,  4.60it/s, acc=0.841, loss=0.462]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.446889\tAvg validation loss: 0.463188\n",
            "\t\tvalid_loss decreased (0.477229 --> 0.463188)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [14/30]: 100%|██████████| 196/196 [00:42<00:00,  4.59it/s, acc=0.843, loss=0.351]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.445140\tAvg validation loss: 0.451549\n",
            "\t\tvalid_loss decreased (0.463188 --> 0.451549)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [15/30]: 100%|██████████| 196/196 [00:42<00:00,  4.61it/s, acc=0.849, loss=0.343]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.425136\tAvg validation loss: 0.450946\n",
            "\t\tvalid_loss decreased (0.451549 --> 0.450946)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [16/30]: 100%|██████████| 196/196 [00:42<00:00,  4.58it/s, acc=0.855, loss=0.377]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.411279\tAvg validation loss: 0.452803\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [17/30]: 100%|██████████| 196/196 [00:42<00:00,  4.58it/s, acc=0.857, loss=0.362]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.399843\tAvg validation loss: 0.453718\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [18/30]: 100%|██████████| 196/196 [00:42<00:00,  4.58it/s, acc=0.86, loss=0.502]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.392495\tAvg validation loss: 0.435896\n",
            "\t\tvalid_loss decreased (0.450946 --> 0.435896)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [19/30]: 100%|██████████| 196/196 [00:42<00:00,  4.58it/s, acc=0.865, loss=0.334]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.378240\tAvg validation loss: 0.444111\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [20/30]: 100%|██████████| 196/196 [00:42<00:00,  4.63it/s, acc=0.867, loss=0.592]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.373910\tAvg validation loss: 0.432857\n",
            "\t\tvalid_loss decreased (0.435896 --> 0.432857)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [21/30]: 100%|██████████| 196/196 [00:42<00:00,  4.65it/s, acc=0.869, loss=0.319]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.363137\tAvg validation loss: 0.427644\n",
            "\t\tvalid_loss decreased (0.432857 --> 0.427644)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [22/30]: 100%|██████████| 196/196 [00:42<00:00,  4.64it/s, acc=0.875, loss=0.406]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.350984\tAvg validation loss: 0.448720\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [23/30]: 100%|██████████| 196/196 [00:42<00:00,  4.63it/s, acc=0.875, loss=0.373]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.347980\tAvg validation loss: 0.439049\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [24/30]: 100%|██████████| 196/196 [00:42<00:00,  4.66it/s, acc=0.883, loss=0.419]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.326903\tAvg validation loss: 0.424339\n",
            "\t\tvalid_loss decreased (0.427644 --> 0.424339)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [25/30]: 100%|██████████| 196/196 [00:42<00:00,  4.62it/s, acc=0.885, loss=0.2]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.319745\tAvg validation loss: 0.440760\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [26/30]: 100%|██████████| 196/196 [00:42<00:00,  4.61it/s, acc=0.886, loss=0.363]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.322240\tAvg validation loss: 0.426965\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [27/30]: 100%|██████████| 196/196 [00:42<00:00,  4.59it/s, acc=0.889, loss=0.302]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.307315\tAvg validation loss: 0.415051\n",
            "\t\tvalid_loss decreased (0.424339 --> 0.415051)  saving model...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [28/30]: 100%|██████████| 196/196 [00:43<00:00,  4.56it/s, acc=0.893, loss=0.279]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.298390\tAvg validation loss: 0.443596\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [29/30]: 100%|██████████| 196/196 [00:42<00:00,  4.57it/s, acc=0.895, loss=0.263]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.294750\tAvg validation loss: 0.443056\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [30/30]: 100%|██████████| 196/196 [00:43<00:00,  4.55it/s, acc=0.895, loss=0.193]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\t\tAvg training loss: 0.290253\tAvg validation loss: 0.441863\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}