vit_example.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](http://colab.research.google.com/github/RustamyF/vision-transformer/blob/master/vit_example.ipynb)\n"
      ],
      "metadata": {
        "id": "6ORCnWA_30f2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir ~/.kaggle\n",
        "!cp kaggle.json ~/.kaggle/\n",
        "!chmod 600 ~/.kaggle/kaggle.json"
      ],
      "metadata": {
        "id": "NM_jcZ0myOQG",
        "outputId": "6f6a4237-a3fd-4fc3-a5f5-54b78eb790c7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mkdir: cannot create directory ‘/root/.kaggle’: File exists\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install kaggle einops"
      ],
      "metadata": {
        "id": "5QWVOKRbxwkm",
        "outputId": "9c1ff6ca-4b96-4dfa-dbc7-c45d97e72dc6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.8/dist-packages (1.5.12)\n",
            "Collecting einops\n",
            "  Downloading einops-0.6.0-py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.6/41.6 KB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil in /usr/local/lib/python3.8/dist-packages (from kaggle) (2.8.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from kaggle) (4.64.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from kaggle) (2.25.1)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.8/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.8/dist-packages (from kaggle) (2022.12.7)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.8/dist-packages (from kaggle) (8.0.0)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.8/dist-packages (from kaggle) (1.15.0)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.8/dist-packages (from python-slugify->kaggle) (1.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->kaggle) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->kaggle) (4.0.0)\n",
            "Installing collected packages: einops\n",
            "Successfully installed einops-0.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from kaggle.api.kaggle_api_extended import KaggleApi\n",
        "\n",
        "api = KaggleApi()\n",
        "api.authenticate()\n",
        "\n",
        "# we write to the current directory with './'\n",
        "api.dataset_download_files('karakaggle/kaggle-cat-vs-dog-dataset', path='./')"
      ],
      "metadata": {
        "id": "w5qnikB9xu3v"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip -qq kaggle-cat-vs-dog-dataset.zip\n",
        "!rm -r kaggle-cat-vs-dog-dataset.zip"
      ],
      "metadata": {
        "id": "BVokL2zoyGn3"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "X8VbaIvlyHVB"
      },
      "outputs": [],
      "source": [
        "# !wget --no-check-certificate \\\n",
        "#     https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip \\\n",
        "#     -O pets\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/RustamyF/vision-transformer.git\n",
        "!mv vision-transformer/vision_tr ."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AfnUeUsMibys",
        "outputId": "c5daec59-bd10-4402-f2d2-f48b5c7fe6a7"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'vision-transformer'...\n",
            "remote: Enumerating objects: 40, done.\u001b[K\n",
            "remote: Counting objects: 100% (40/40), done.\u001b[K\n",
            "remote: Compressing objects: 100% (30/30), done.\u001b[K\n",
            "remote: Total 40 (delta 12), reused 35 (delta 7), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (40/40), 125.46 KiB | 3.92 MiB/s, done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !unzip pets\n",
        "# !rm pets\n",
        "# !mv -v cats_and_dogs_filtered/validation/dogs/* cats_and_dogs_filtered/train/dogs/\n",
        "# !mv -v cats_and_dogs_filtered/validation/cats/* cats_and_dogs_filtered/train/cats/"
      ],
      "metadata": {
        "id": "q9AFhZQKkfI9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from vision_tr.simple_vit import Transformer"
      ],
      "metadata": {
        "id": "KrYTcGVxjYE_"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "\n",
        "from torchvision import datasets, models, transforms\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from PIL import Image\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import os\n",
        "\n",
        "\n",
        "class LoadData:\n",
        "    def __init__(self):\n",
        "        self.cat_path = 'kagglecatsanddogs_3367a/PetImages/Cat'\n",
        "        self.dog_path = 'kagglecatsanddogs_3367a/PetImages/Dog'\n",
        "\n",
        "    def delete_non_jpeg_files(self, directory):\n",
        "        for filename in os.listdir(directory):\n",
        "            if not filename.endswith('.jpg') and not filename.endswith('.jpeg'):\n",
        "                file_path = os.path.join(directory, filename)\n",
        "                try:\n",
        "                    if os.path.isfile(file_path) or os.path.islink(file_path):\n",
        "                        os.unlink(file_path)\n",
        "                    elif os.path.isdir(file_path):\n",
        "                        shutil.rmtree(file_path)\n",
        "                    print('deleted', file_path)\n",
        "                except Exception as e:\n",
        "                    print('Failed to delete %s. Reason: %s' % (file_path, e))\n",
        "\n",
        "    def data(self):\n",
        "        self.delete_non_jpeg_files(self.dog_path)\n",
        "        self.delete_non_jpeg_files(self.cat_path)\n",
        "\n",
        "        dog_list = os.listdir(self.dog_path)\n",
        "        dog_list = [(os.path.join(self.dog_path, i), 1) for i in dog_list]\n",
        "\n",
        "        cat_list = os.listdir(self.cat_path)\n",
        "        cat_list = [(os.path.join(self.cat_path, i), 0) for i in cat_list]\n",
        "\n",
        "        total_list = cat_list + dog_list\n",
        "\n",
        "        train_list, test_list = train_test_split(total_list, test_size=0.2)\n",
        "        train_list, val_list = train_test_split(train_list, test_size=0.2)\n",
        "        print('train list', len(train_list))\n",
        "        print('test list', len(test_list))\n",
        "        print('val list', len(val_list))\n",
        "        return train_list, test_list, val_list\n",
        "\n",
        "\n",
        "# data Augumentation\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),\n",
        "    transforms.RandomResizedCrop(224),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "\n",
        "class dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, file_list, transform=None):\n",
        "        self.file_list = file_list\n",
        "        self.transform = transform\n",
        "\n",
        "    # dataset length\n",
        "    def __len__(self):\n",
        "        self.filelength = len(self.file_list)\n",
        "        return self.filelength\n",
        "\n",
        "    # load an one of images\n",
        "    def __getitem__(self, idx):\n",
        "        img_path, label = self.file_list[idx]\n",
        "        img = Image.open(img_path).convert('RGB')\n",
        "        img_transformed = self.transform(img)\n",
        "        return img_transformed, label\n",
        "\n",
        "\n",
        "class Cnn(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Cnn, self).__init__()\n",
        "\n",
        "        self.layer1 = nn.Sequential(\n",
        "            nn.Conv2d(3, 16, kernel_size=3, padding=0, stride=2),\n",
        "            nn.BatchNorm2d(16),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2)\n",
        "        )\n",
        "\n",
        "        self.layer2 = nn.Sequential(\n",
        "            nn.Conv2d(16, 32, kernel_size=3, padding=0, stride=2),\n",
        "            nn.BatchNorm2d(32),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2)\n",
        "        )\n",
        "\n",
        "        self.layer3 = nn.Sequential(\n",
        "            nn.Conv2d(32, 64, kernel_size=3, padding=0, stride=2),\n",
        "            nn.BatchNorm2d(64),\n",
        "            nn.ReLU(),\n",
        "            nn.MaxPool2d(2)\n",
        "        )\n",
        "\n",
        "        self.fc1 = nn.Linear(3 * 3 * 64, 10)\n",
        "        self.dropout = nn.Dropout(0.5)\n",
        "        self.fc2 = nn.Linear(10, 2)\n",
        "        self.relu = nn.ReLU()\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = self.layer1(x)\n",
        "        out = self.layer2(out)\n",
        "        out = self.layer3(out)\n",
        "        out = out.view(out.size(0), -1)\n",
        "        out = self.relu(self.fc1(out))\n",
        "        out = self.fc2(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    lr = 0.001  # learning_rate\n",
        "    batch_size = 800  # we will use mini-batch method\n",
        "    epochs = 10  # How much to train a model\n",
        "\n",
        "    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "    torch.manual_seed(1234)\n",
        "    if device == 'cuda':\n",
        "        torch.cuda.manual_seed_all(1234)\n",
        "\n",
        "    print(device)\n",
        "\n",
        "    load_data = LoadData()\n",
        "\n",
        "    train_list, test_list, val_list = load_data.data()\n",
        "\n",
        "    train_data = dataset(train_list, transform=transform)\n",
        "    test_data = dataset(test_list, transform=transform)\n",
        "    val_data = dataset(val_list, transform=transform)\n",
        "\n",
        "    train_loader = torch.utils.data.DataLoader(dataset=train_data, batch_size=batch_size, shuffle=False)\n",
        "    test_loader = torch.utils.data.DataLoader(dataset=test_data, batch_size=batch_size, shuffle=True)\n",
        "    val_loader = torch.utils.data.DataLoader(dataset=val_data, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "    model = Cnn().to(device)\n",
        "    model.train()\n",
        "\n",
        "    optimizer = optim.Adam(params=model.parameters(), lr=0.001)\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        epoch_loss = 0\n",
        "        epoch_accuracy = 0\n",
        "\n",
        "        for data, label in train_loader:\n",
        "            data = data.to(device)\n",
        "            label = label.to(device)\n",
        "\n",
        "            output = model(data)\n",
        "            loss = criterion(output, label)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            acc = ((output.argmax(dim=1) == label).float().mean())\n",
        "            epoch_accuracy += acc / len(train_loader)\n",
        "            epoch_loss += loss / len(train_loader)\n",
        "\n",
        "        print('Epoch : {}, train accuracy : {}, train loss : {}'.format(epoch + 1, epoch_accuracy, epoch_loss))\n",
        "\n",
        "        with torch.no_grad():\n",
        "            epoch_val_accuracy = 0\n",
        "            epoch_val_loss = 0\n",
        "            for data, label in val_loader:\n",
        "                data = data.to(device)\n",
        "                label = label.to(device)\n",
        "\n",
        "                val_output = model(data)\n",
        "                val_loss = criterion(val_output, label)\n",
        "\n",
        "                acc = ((val_output.argmax(dim=1) == label).float().mean())\n",
        "                epoch_val_accuracy += acc / len(val_loader)\n",
        "                epoch_val_loss += val_loss / len(val_loader)\n",
        "\n",
        "            print('Epoch : {}, val_accuracy : {}, val_loss : {}'.format(epoch + 1, epoch_val_accuracy, epoch_val_loss))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TChHpDa1jtRD",
        "outputId": "002368c2-759d-4b88-ff95-2c184b5de826"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n",
            "train list 15973\n",
            "test list 4992\n",
            "val list 3994\n",
            "Epoch : 1, train accuracy : 0.5923538208007812, train loss : 0.6718765497207642\n",
            "Epoch : 1, val_accuracy : 0.6094502806663513, val_loss : 0.6586422920227051\n",
            "Epoch : 2, train accuracy : 0.6331198215484619, train loss : 0.6380239725112915\n",
            "Epoch : 2, val_accuracy : 0.6572360992431641, val_loss : 0.6225108504295349\n",
            "Epoch : 3, train accuracy : 0.6697515249252319, train loss : 0.6039386987686157\n",
            "Epoch : 3, val_accuracy : 0.684025764465332, val_loss : 0.5946614146232605\n",
            "Epoch : 4, train accuracy : 0.6884258985519409, train loss : 0.5855448842048645\n",
            "Epoch : 4, val_accuracy : 0.6938022375106812, val_loss : 0.5771335363388062\n",
            "Epoch : 5, train accuracy : 0.7091054916381836, train loss : 0.562760591506958\n",
            "Epoch : 5, val_accuracy : 0.7087947130203247, val_loss : 0.5653967261314392\n",
            "Epoch : 6, train accuracy : 0.7210022211074829, train loss : 0.547177255153656\n",
            "Epoch : 6, val_accuracy : 0.7123268842697144, val_loss : 0.5627478957176208\n",
            "Epoch : 7, train accuracy : 0.7299484610557556, train loss : 0.5322500467300415\n",
            "Epoch : 7, val_accuracy : 0.731610894203186, val_loss : 0.531222403049469\n",
            "Epoch : 8, train accuracy : 0.7374877333641052, train loss : 0.5186564922332764\n",
            "Epoch : 8, val_accuracy : 0.7358759641647339, val_loss : 0.5283315777778625\n",
            "Epoch : 9, train accuracy : 0.7431643009185791, train loss : 0.5115637183189392\n",
            "Epoch : 9, val_accuracy : 0.7303627729415894, val_loss : 0.5311689972877502\n",
            "Epoch : 10, train accuracy : 0.7496491074562073, train loss : 0.5001087784767151\n",
            "Epoch : 10, val_accuracy : 0.748603343963623, val_loss : 0.5130950808525085\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.nn as nn\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "from vision_tr.simple_vit import ViT\n",
        "# from vit_pytorch.efficient import ViT\n",
        "from torchvision import datasets, models, transforms\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from PIL import Image\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import os\n",
        "\n",
        "# from linformer import Linformer\n",
        "from torch.optim.lr_scheduler import StepLR\n",
        "# from vit_pytorch.efficient import ViT\n",
        "\n",
        "\n",
        "class LoadData:\n",
        "\n",
        "    def __init__(self):\n",
        "        self.cat_path = 'kagglecatsanddogs_3367a/PetImages/Cat'\n",
        "        self.dog_path = 'kagglecatsanddogs_3367a/PetImages/Dog'\n",
        "\n",
        "    def delete_non_jpeg_files(self, directory):\n",
        "        for filename in os.listdir(directory):\n",
        "            if not filename.endswith('.jpg') and not filename.endswith('.jpeg'):\n",
        "                file_path = os.path.join(directory, filename)\n",
        "                try:\n",
        "                    if os.path.isfile(file_path) or os.path.islink(file_path):\n",
        "                        os.unlink(file_path)\n",
        "                    elif os.path.isdir(file_path):\n",
        "                        shutil.rmtree(file_path)\n",
        "                    print('deleted', file_path)\n",
        "                except Exception as e:\n",
        "                    print('Failed to delete %s. Reason: %s' % (file_path, e))\n",
        "\n",
        "    def data(self):\n",
        "        self.delete_non_jpeg_files(self.dog_path)\n",
        "        self.delete_non_jpeg_files(self.cat_path)\n",
        "\n",
        "        dog_list = os.listdir(self.dog_path)\n",
        "        dog_list = [(os.path.join(self.dog_path, i), 1) for i in dog_list]\n",
        "\n",
        "        cat_list = os.listdir(self.cat_path)\n",
        "        cat_list = [(os.path.join(self.cat_path, i), 0) for i in cat_list]\n",
        "\n",
        "        total_list = cat_list + dog_list\n",
        "\n",
        "        train_list, test_list = train_test_split(total_list, test_size=0.2)\n",
        "        train_list, val_list = train_test_split(train_list, test_size=0.2)\n",
        "        print('train list', len(train_list))\n",
        "        print('test list', len(test_list))\n",
        "        print('val list', len(val_list))\n",
        "        return train_list, test_list, val_list\n",
        "\n",
        "\n",
        "# data Augumentation\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),\n",
        "    transforms.RandomResizedCrop(224),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "\n",
        "class dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, file_list, transform=None):\n",
        "        self.file_list = file_list\n",
        "        self.transform = transform\n",
        "\n",
        "    # dataset length\n",
        "    def __len__(self):\n",
        "        self.filelength = len(self.file_list)\n",
        "        return self.filelength\n",
        "\n",
        "    # load an one of images\n",
        "    def __getitem__(self, idx):\n",
        "        img_path, label = self.file_list[idx]\n",
        "        img = Image.open(img_path).convert('RGB')\n",
        "        img_transformed = self.transform(img)\n",
        "        return img_transformed, label\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    # Training settings\n",
        "    batch_size = 64\n",
        "    epochs = 20\n",
        "    lr = 3e-5\n",
        "    gamma = 0.7\n",
        "    seed = 42\n",
        "\n",
        "    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "    torch.manual_seed(1234)\n",
        "    if device == 'cuda':\n",
        "        torch.cuda.manual_seed_all(1234)\n",
        "\n",
        "    print(device)\n",
        "\n",
        "    load_data = LoadData()\n",
        "\n",
        "    train_list, test_list, val_list = load_data.data()\n",
        "\n",
        "    train_data = dataset(train_list, transform=transform)\n",
        "    test_data = dataset(test_list, transform=transform)\n",
        "    val_data = dataset(val_list, transform=transform)\n",
        "\n",
        "    train_loader = torch.utils.data.DataLoader(dataset=train_data, batch_size=batch_size, shuffle=False)\n",
        "    test_loader = torch.utils.data.DataLoader(dataset=test_data, batch_size=batch_size, shuffle=True)\n",
        "    val_loader = torch.utils.data.DataLoader(dataset=val_data, batch_size=batch_size, shuffle=True)\n",
        "    model = ViT(\n",
        "        image_size=224,\n",
        "        patch_size=32,\n",
        "        num_classes=2,\n",
        "        dim=128,\n",
        "        depth=12,\n",
        "        heads=8,\n",
        "        mlp_dim=1024,\n",
        "        dropout=0.1,\n",
        "        emb_dropout=0.1,\n",
        "    ).to(device)\n",
        "\n",
        "    # loss function\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    # optimizer\n",
        "    optimizer = optim.Adam(model.parameters(), lr=lr)\n",
        "    # scheduler\n",
        "    scheduler = StepLR(optimizer, step_size=1, gamma=gamma)\n",
        "\n",
        "    epochs = 20\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        epoch_loss = 0\n",
        "        epoch_accuracy = 0\n",
        "\n",
        "        for data, label in train_loader:\n",
        "            data = data.to(device)\n",
        "            label = label.to(device)\n",
        "\n",
        "            output = model(data)\n",
        "            loss = criterion(output, label)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            acc = ((output.argmax(dim=1) == label).float().mean())\n",
        "            epoch_accuracy += acc / len(train_loader)\n",
        "            epoch_loss += loss / len(train_loader)\n",
        "\n",
        "        print('Epoch : {}, train accuracy : {}, train loss : {}'.format(epoch + 1, epoch_accuracy, epoch_loss))\n",
        "\n",
        "        with torch.no_grad():\n",
        "            epoch_val_accuracy = 0\n",
        "            epoch_val_loss = 0\n",
        "            for data, label in val_loader:\n",
        "                data = data.to(device)\n",
        "                label = label.to(device)\n",
        "\n",
        "                val_output = model(data)\n",
        "                val_loss = criterion(val_output, label)\n",
        "\n",
        "                acc = ((val_output.argmax(dim=1) == label).float().mean())\n",
        "                epoch_val_accuracy += acc / len(val_loader)\n",
        "                epoch_val_loss += val_loss / len(val_loader)\n",
        "\n",
        "            print('Epoch : {}, val_accuracy : {}, val_loss : {}'.format(epoch + 1, epoch_val_accuracy, epoch_val_loss))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M3CfkEoQjwfr",
        "outputId": "00cc48ad-7671-45f2-d00a-1da2d5a37b08"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n",
            "train list 15973\n",
            "test list 4992\n",
            "val list 3994\n",
            "Epoch : 1, train accuracy : 0.5375151634216309, train loss : 0.6935465931892395\n",
            "Epoch : 1, val_accuracy : 0.5417619943618774, val_loss : 0.6855644583702087\n",
            "Epoch : 2, train accuracy : 0.5730372667312622, train loss : 0.6744303703308105\n",
            "Epoch : 2, val_accuracy : 0.5649802088737488, val_loss : 0.6711758971214294\n",
            "Epoch : 3, train accuracy : 0.601486325263977, train loss : 0.6573463678359985\n",
            "Epoch : 3, val_accuracy : 0.5747672915458679, val_loss : 0.6639878153800964\n",
            "Epoch : 4, train accuracy : 0.6121062636375427, train loss : 0.6453043818473816\n",
            "Epoch : 4, val_accuracy : 0.5804525017738342, val_loss : 0.6615283489227295\n",
            "Epoch : 5, train accuracy : 0.629486083984375, train loss : 0.6355270147323608\n",
            "Epoch : 5, val_accuracy : 0.6207266449928284, val_loss : 0.6372857093811035\n",
            "Epoch : 6, train accuracy : 0.6291114091873169, train loss : 0.6319907307624817\n",
            "Epoch : 6, val_accuracy : 0.6100619435310364, val_loss : 0.643184244632721\n",
            "Epoch : 7, train accuracy : 0.6467196941375732, train loss : 0.624461829662323\n",
            "Epoch : 7, val_accuracy : 0.6397665143013, val_loss : 0.6221678256988525\n",
            "Epoch : 8, train accuracy : 0.64268559217453, train loss : 0.6215488314628601\n",
            "Epoch : 8, val_accuracy : 0.6418079733848572, val_loss : 0.6235150098800659\n",
            "Epoch : 9, train accuracy : 0.6501232385635376, train loss : 0.6183606386184692\n",
            "Epoch : 9, val_accuracy : 0.6290255784988403, val_loss : 0.6331229209899902\n",
            "Epoch : 10, train accuracy : 0.6568276882171631, train loss : 0.6106156706809998\n",
            "Epoch : 10, val_accuracy : 0.6661136150360107, val_loss : 0.6032187938690186\n",
            "Epoch : 11, train accuracy : 0.6600776314735413, train loss : 0.607439398765564\n",
            "Epoch : 11, val_accuracy : 0.6626794934272766, val_loss : 0.6119657754898071\n",
            "Epoch : 12, train accuracy : 0.6662142872810364, train loss : 0.6049618124961853\n",
            "Epoch : 12, val_accuracy : 0.6790485382080078, val_loss : 0.5934984683990479\n",
            "Epoch : 13, train accuracy : 0.674236536026001, train loss : 0.6004449725151062\n",
            "Epoch : 13, val_accuracy : 0.6843140721321106, val_loss : 0.5900341272354126\n",
            "Epoch : 14, train accuracy : 0.6758108735084534, train loss : 0.5967960357666016\n",
            "Epoch : 14, val_accuracy : 0.6757097840309143, val_loss : 0.5940932631492615\n",
            "Epoch : 15, train accuracy : 0.6763781905174255, train loss : 0.5928819179534912\n",
            "Epoch : 15, val_accuracy : 0.680880069732666, val_loss : 0.5945420861244202\n",
            "Epoch : 16, train accuracy : 0.6798441410064697, train loss : 0.5915115475654602\n",
            "Epoch : 16, val_accuracy : 0.694024920463562, val_loss : 0.5839072465896606\n",
            "Epoch : 17, train accuracy : 0.6873230338096619, train loss : 0.583565890789032\n",
            "Epoch : 17, val_accuracy : 0.6809945106506348, val_loss : 0.585138738155365\n",
            "Epoch : 18, train accuracy : 0.6885657906532288, train loss : 0.5838371515274048\n",
            "Epoch : 18, val_accuracy : 0.6986036896705627, val_loss : 0.5794425010681152\n",
            "Epoch : 19, train accuracy : 0.6872652769088745, train loss : 0.5805392861366272\n",
            "Epoch : 19, val_accuracy : 0.6978405714035034, val_loss : 0.576508104801178\n",
            "Epoch : 20, train accuracy : 0.6974021196365356, train loss : 0.573847234249115\n",
            "Epoch : 20, val_accuracy : 0.6885876655578613, val_loss : 0.5791235566139221\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2gNOGSRX5N5l"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}