unit08-large-language-models/8.7-distilbert-finetuning/part3_distilbert-finetuning-whole.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3c5d72f4",
   "metadata": {},
   "source": [
    "# Finetuning a DistilBERT Classifier in Lightning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0873756d-262a-4525-b809-4e0b3a1e63dd",
   "metadata": {},
   "source": [
    "![](figures/finetuning-ii.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6fd9cda8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "92ea5612",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fe7191cf-62ed-4793-8358-bee70b233d05",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install lightning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "033b75c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch       : 2.0.0\n",
      "transformers: 4.27.4\n",
      "datasets    : 2.11.0\n",
      "lightning   : 2.0.0\n",
      "\n",
      "conda environment: dl-fundamentals\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark --conda -p torch,transformers,datasets,lightning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09213821-b2b4-402e-adf8-7c7fe4ec57cb",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 1 Loading the dataset into DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e39e2228-5f0b-4fb9-b762-df26c2052b45",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install datasets\n",
    "\n",
    "import os.path as op\n",
    "\n",
    "from datasets import load_dataset\n",
    "\n",
    "import lightning as L\n",
    "from lightning.pytorch.loggers import CSVLogger\n",
    "from lightning.pytorch.callbacks import ModelCheckpoint\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "from local_dataset_utilities import download_dataset, load_dataset_into_to_dataframe, partition_dataset\n",
    "from local_dataset_utilities import IMDBDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fb31ac90-9e3a-41d0-baf1-8e613043924b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████| 50000/50000 [00:27<00:00, 1814.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class distribution:\n"
     ]
    }
   ],
   "source": [
    "download_dataset()\n",
    "\n",
    "df = load_dataset_into_to_dataframe()\n",
    "partition_dataset(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "221f30a1-b433-4304-a18d-8d03abd42b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"train.csv\")\n",
    "df_val = pd.read_csv(\"val.csv\")\n",
    "df_test = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "876736c1-ae27-491c-850b-050507fa02b5",
   "metadata": {},
   "source": [
    "# 2 Tokenization and Numericalization"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afe0cca0-bac4-49ed-982c-14c998e578d1",
   "metadata": {},
   "source": [
    "**Load the dataset via `load_dataset`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a1aa66c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset csv/default to /home/sebastian/.cache/huggingface/datasets/csv/default-17ef1f917b4946aa/0.0.0/6954658bab30a358235fa864b05cf819af0e179325c740e4bc853bcc7ec513e1...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9687802551b241858b2b74f8eb0362f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5162503aac64cceabeeb24a031c14d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating validation split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating test split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset csv downloaded and prepared to /home/sebastian/.cache/huggingface/datasets/csv/default-17ef1f917b4946aa/0.0.0/6954658bab30a358235fa864b05cf819af0e179325c740e4bc853bcc7ec513e1. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f733df6dc43c49f1bf3fd8b6835931cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DatasetDict({\n",
      "    train: Dataset({\n",
      "        features: ['index', 'text', 'label'],\n",
      "        num_rows: 35000\n",
      "    })\n",
      "    validation: Dataset({\n",
      "        features: ['index', 'text', 'label'],\n",
      "        num_rows: 5000\n",
      "    })\n",
      "    test: Dataset({\n",
      "        features: ['index', 'text', 'label'],\n",
      "        num_rows: 10000\n",
      "    })\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "imdb_dataset = load_dataset(\n",
    "    \"csv\",\n",
    "    data_files={\n",
    "        \"train\": \"train.csv\",\n",
    "        \"validation\": \"val.csv\",\n",
    "        \"test\": \"test.csv\",\n",
    "    },\n",
    ")\n",
    "\n",
    "print(imdb_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b201159-f3fa-4649-8076-eff8bc5535d3",
   "metadata": {},
   "source": [
    "**Tokenize the dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5ea762ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenizer input max length: 512\n",
      "Tokenizer vocabulary size: 30522\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
    "print(\"Tokenizer input max length:\", tokenizer.model_max_length)\n",
    "print(\"Tokenizer vocabulary size:\", tokenizer.vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8432c15c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_text(batch):\n",
    "    return tokenizer(batch[\"text\"], truncation=True, padding=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0bb392cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/35000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/5000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/10000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imdb_tokenized = imdb_dataset.map(tokenize_text, batched=True, batch_size=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6d4103c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "del imdb_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "89ef894c-978f-47f2-9d61-cb6a9f38e745",
   "metadata": {},
   "outputs": [],
   "source": [
    "imdb_tokenized.set_format(\"torch\", columns=[\"input_ids\", \"attention_mask\", \"label\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0ea67091-aeb7-46c1-871f-638ce58d8a0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ff16488-abe6-48af-9b03-868b457b0ea3",
   "metadata": {},
   "source": [
    "# 3 Set Up DataLoaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0807b068-7d8f-4055-a26a-177e07dea4c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "\n",
    "class IMDBDataset(Dataset):\n",
    "    def __init__(self, dataset_dict, partition_key=\"train\"):\n",
    "        self.partition = dataset_dict[partition_key]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self.partition[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.partition.num_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "90cb08f3-ef77-4351-8b19-42d99dd24f98",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = IMDBDataset(imdb_tokenized, partition_key=\"train\")\n",
    "val_dataset = IMDBDataset(imdb_tokenized, partition_key=\"validation\")\n",
    "test_dataset = IMDBDataset(imdb_tokenized, partition_key=\"test\")\n",
    "\n",
    "train_loader = DataLoader(\n",
    "    dataset=train_dataset,\n",
    "    batch_size=12,\n",
    "    shuffle=True, \n",
    "    num_workers=4\n",
    ")\n",
    "\n",
    "val_loader = DataLoader(\n",
    "    dataset=val_dataset,\n",
    "    batch_size=12,\n",
    "    num_workers=4\n",
    ")\n",
    "\n",
    "test_loader = DataLoader(\n",
    "    dataset=test_dataset,\n",
    "    batch_size=12,\n",
    "    num_workers=4\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78e774ab-45a0-4c48-ad61-a3d0e1927ef4",
   "metadata": {},
   "source": [
    "# 4 Initializing DistilBERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dc28ddbe-1a96-4c24-9f5c-40ffdca4a572",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_transform.weight']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'pre_classifier.weight', 'pre_classifier.bias', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    \"distilbert-base-uncased\", num_labels=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "def1cf25-0a7d-4bb2-9419-b7a8fe1c1eab",
   "metadata": {},
   "source": [
    "## 5 Finetuning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "534f7a59-2c86-4895-ad7c-2cdd675b003a",
   "metadata": {},
   "source": [
    "**Wrap in LightningModule for Training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9f2c474d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightning as L\n",
    "import torch\n",
    "import torchmetrics\n",
    "\n",
    "\n",
    "class LightningModel(L.LightningModule):\n",
    "    def __init__(self, model, learning_rate=5e-5):\n",
    "        super().__init__()\n",
    "\n",
    "        self.learning_rate = learning_rate\n",
    "        self.model = model\n",
    "\n",
    "        self.val_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2)\n",
    "        self.test_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=2)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask, labels):\n",
    "        return self.model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        \n",
    "    def training_step(self, batch, batch_idx):\n",
    "        outputs = self(batch[\"input_ids\"], attention_mask=batch[\"attention_mask\"],\n",
    "                       labels=batch[\"label\"])        \n",
    "        self.log(\"train_loss\", outputs[\"loss\"])\n",
    "        return outputs[\"loss\"]  # this is passed to the optimizer for training\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        outputs = self(batch[\"input_ids\"], attention_mask=batch[\"attention_mask\"],\n",
    "                       labels=batch[\"label\"])        \n",
    "        self.log(\"val_loss\", outputs[\"loss\"], prog_bar=True)\n",
    "        \n",
    "        logits = outputs[\"logits\"]\n",
    "        predicted_labels = torch.argmax(logits, 1)\n",
    "        self.val_acc(predicted_labels, batch[\"label\"])\n",
    "        self.log(\"val_acc\", self.val_acc, prog_bar=True)\n",
    "        \n",
    "    def test_step(self, batch, batch_idx):\n",
    "        outputs = self(batch[\"input_ids\"], attention_mask=batch[\"attention_mask\"],\n",
    "                       labels=batch[\"label\"])        \n",
    "        \n",
    "        logits = outputs[\"logits\"]\n",
    "        predicted_labels = torch.argmax(logits, 1)\n",
    "        self.test_acc(predicted_labels, batch[\"label\"])\n",
    "        self.log(\"accuracy\", self.test_acc, prog_bar=True)\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)\n",
    "        return optimizer\n",
    "    \n",
    "\n",
    "lightning_model = LightningModel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e6dab813-e1fc-47cd-87a1-5eb8070699c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightning.pytorch.callbacks import ModelCheckpoint\n",
    "from lightning.pytorch.loggers import CSVLogger\n",
    "\n",
    "\n",
    "callbacks = [\n",
    "    ModelCheckpoint(\n",
    "        save_top_k=1, mode=\"max\", monitor=\"val_acc\"\n",
    "    )  # save top 1 model\n",
    "]\n",
    "logger = CSVLogger(save_dir=\"logs/\", name=\"my-model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "492aa043-02da-459e-a266-091b34254ac6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using 16bit Automatic Mixed Precision (AMP)\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "/home/sebastian/miniforge3/envs/dl-fundamentals/lib/python3.9/site-packages/lightning/pytorch/callbacks/model_checkpoint.py:612: UserWarning: Checkpoint directory logs/my-model/version_0/checkpoints exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
      "\n",
      "  | Name     | Type                                | Params\n",
      "-----------------------------------------------------------------\n",
      "0 | model    | DistilBertForSequenceClassification | 67.0 M\n",
      "1 | val_acc  | MulticlassAccuracy                  | 0     \n",
      "2 | test_acc | MulticlassAccuracy                  | 0     \n",
      "-----------------------------------------------------------------\n",
      "67.0 M    Trainable params\n",
      "0         Non-trainable params\n",
      "67.0 M    Total params\n",
      "267.820   Total estimated model params size (MB)\n",
      "/home/sebastian/miniforge3/envs/dl-fundamentals/lib/python3.9/site-packages/lightning/fabric/loggers/csv_logs.py:188: UserWarning: Experiment logs directory logs/my-model/version_0 exists and is not empty. Previous log files in this directory will be deleted when the new ones are saved!\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Sanity Checking: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e3f37c7b5284164ae801f91ce63f6d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "`Trainer.fit` stopped: `max_epochs=3` reached.\n"
     ]
    }
   ],
   "source": [
    "trainer = L.Trainer(\n",
    "    max_epochs=3,\n",
    "    callbacks=callbacks,\n",
    "    accelerator=\"gpu\",\n",
    "    precision=\"16-mixed\",\n",
    "    devices=[1],\n",
    "    logger=logger,\n",
    "    log_every_n_steps=10,\n",
    ")\n",
    "\n",
    "trainer.fit(model=lightning_model,\n",
    "            train_dataloaders=train_loader,\n",
    "            val_dataloaders=val_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d795778a-70d2-4b04-96fb-598eccbcd1be",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "Restoring states from the checkpoint path at logs/my-model/version_0/checkpoints/epoch=0-step=2917.ckpt\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
      "Loaded model weights from the checkpoint at logs/my-model/version_0/checkpoints/epoch=0-step=2917.ckpt\n",
      "/home/sebastian/miniforge3/envs/dl-fundamentals/lib/python3.9/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:478: PossibleUserWarning: Your `test_dataloader`'s sampler has shuffling enabled, it is strongly recommended that you turn shuffling off for val/test dataloaders.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "024e5eb0b8b946beb1b658880d48d873",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">        Test metric        </span>┃<span style=\"font-weight: bold\">       DataLoader 0        </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">         accuracy          </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.9659714102745056     </span>│\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m        accuracy         \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.9659714102745056    \u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────────┴───────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'accuracy': 0.9659714102745056}]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.test(lightning_model, dataloaders=train_loader, ckpt_path=\"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "10ca0af1-106e-4ef7-9793-478d580af827",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "Restoring states from the checkpoint path at logs/my-model/version_0/checkpoints/epoch=0-step=2917.ckpt\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
      "Loaded model weights from the checkpoint at logs/my-model/version_0/checkpoints/epoch=0-step=2917.ckpt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6b8e406e41b4c548756d4d91af741ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">        Test metric        </span>┃<span style=\"font-weight: bold\">       DataLoader 0        </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">         accuracy          </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.9300000071525574     </span>│\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m        accuracy         \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.9300000071525574    \u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────────┴───────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'accuracy': 0.9300000071525574}]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.test(lightning_model, dataloaders=val_loader, ckpt_path=\"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "eeb92de4-d483-4627-b9f3-f0bba0cddd9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a CUDA device ('NVIDIA A100-SXM4-40GB') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n",
      "Restoring states from the checkpoint path at logs/my-model/version_0/checkpoints/epoch=0-step=2917.ckpt\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]\n",
      "Loaded model weights from the checkpoint at logs/my-model/version_0/checkpoints/epoch=0-step=2917.ckpt\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d80a8d5b13e499c8ec8806bba3c2dba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">        Test metric        </span>┃<span style=\"font-weight: bold\">       DataLoader 0        </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">         accuracy          </span>│<span style=\"color: #800080; text-decoration-color: #800080\">    0.9264000058174133     </span>│\n",
       "└───────────────────────────┴───────────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m       Test metric       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m      DataLoader 0       \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m        accuracy         \u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m   0.9264000058174133    \u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────────┴───────────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'accuracy': 0.9264000058174133}]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.test(lightning_model, dataloaders=test_loader, ckpt_path=\"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3515de14-6243-4cec-a554-e55e8d10dcc4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d34b9bce-b269-420a-8533-cff862e0b85d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}