multiclass_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "massive-puppy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-10T03:30:23.476096Z",
     "iopub.status.busy": "2021-05-10T03:30:23.475841Z",
     "iopub.status.idle": "2021-05-10T03:30:26.101941Z",
     "shell.execute_reply": "2021-05-10T03:30:26.101176Z",
     "shell.execute_reply.started": "2021-05-10T03:30:23.476031Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pytorch_lightning as pl\n",
    "import torch\n",
    "from core.data import PneumoniaDataModule\n",
    "from core.model import PneumoniaDetector\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score,\n",
    "    confusion_matrix,\n",
    "    f1_score,\n",
    "    precision_score,\n",
    "    recall_score,\n",
    ")\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "mounted-straight",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-10T03:30:26.103302Z",
     "iopub.status.busy": "2021-05-10T03:30:26.103074Z",
     "iopub.status.idle": "2021-05-10T03:30:29.920124Z",
     "shell.execute_reply": "2021-05-10T03:30:29.919192Z",
     "shell.execute_reply.started": "2021-05-10T03:30:26.103272Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_dir = Path(\"./chest_xray\").expanduser().absolute()\n",
    "checkpoint_file = (\n",
    "    Path(\"./model_weights/multiclass_model.pt.ckpt\").expanduser().resolve()\n",
    ")\n",
    "\n",
    "dm = PneumoniaDataModule(\n",
    "    data_dir,\n",
    "    batch_size=256,\n",
    "    num_workers=16,\n",
    "    augment_minority=True,\n",
    "    pneumonia_subclass=True,\n",
    "    val_ratio=0.2,\n",
    ")\n",
    "\n",
    "if not checkpoint_file.is_file():\n",
    "    pl.seed_everything(42)\n",
    "\n",
    "    model = PneumoniaDetector(lr=5e-4, class_num=3)\n",
    "    logger = TensorBoardLogger(\n",
    "        \"tb_logs\", name=\"pneumonia_multi_classifier_augment_train\"\n",
    "    )\n",
    "    trainer = pl.Trainer(\n",
    "        max_epochs=50,\n",
    "        gpus=[1],\n",
    "        logger=logger,\n",
    "        deterministic=True,\n",
    "    )\n",
    "    trainer.fit(model, dm)\n",
    "    test_metrics = trainer.test(model=model, datamodule=dm)\n",
    "    trainer.save_checkpoint(checkpoint_file)\n",
    "else:\n",
    "    model = PneumoniaDetector.load_from_checkpoint(checkpoint_file).cuda()\n",
    "    dm.setup(\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b753566d-9323-471e-bd31-f279ed89f67e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-10T03:30:29.921670Z",
     "iopub.status.busy": "2021-05-10T03:30:29.921433Z",
     "iopub.status.idle": "2021-05-10T03:30:44.160352Z",
     "shell.execute_reply": "2021-05-10T03:30:44.159634Z",
     "shell.execute_reply.started": "2021-05-10T03:30:29.921639Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Predicted NORMAL</th>\n",
       "      <th>Predicted bacteria</th>\n",
       "      <th>Predicted virus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>True NORMAL</th>\n",
       "      <td>133</td>\n",
       "      <td>44</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True bacteria</th>\n",
       "      <td>4</td>\n",
       "      <td>231</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True virus</th>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Predicted NORMAL  Predicted bacteria  Predicted virus\n",
       "True NORMAL                 133                  44               57\n",
       "True bacteria                 4                 231                7\n",
       "True virus                    0                  44              104"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Evaluate on the test set: confusion matrix\n",
    "def show_confusion_mat(model, datamod):\n",
    "    model.freeze()\n",
    "    y_pred, y_true = [], []\n",
    "\n",
    "    for batch in datamod.test_dataloader():\n",
    "        x, y = batch\n",
    "        y_hat = model(x.cuda())\n",
    "        pred = F.log_softmax(y_hat, dim=1).argmax(dim=1)\n",
    "        y_pred.append(pred)\n",
    "        y_true.append(y)\n",
    "\n",
    "    y_pred = torch.cat(y_pred).cpu()\n",
    "    y_true = torch.cat(y_true).cpu()\n",
    "\n",
    "    labels = sorted(datamod.class_to_idx.keys())\n",
    "    mat = confusion_matrix(y_true, y_pred, labels=sorted(datamod.class_to_idx.values()))\n",
    "    res = pd.DataFrame(mat, columns=labels, index=labels)\n",
    "    res.index = [f\"True {x}\" for x in res.index]\n",
    "    res.columns = [f\"Predicted {x}\" for x in res.columns]\n",
    "    return res\n",
    "\n",
    "\n",
    "show_confusion_mat(model, dm)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b876e542-fee3-48ed-bbb6-6ab7d8060874",
   "metadata": {},
   "source": [
    "|                | Predicted normal | Predicted bacterial | Predicted viral |\n",
    "|:---------------|-----------------:|--------------------:|----------------:|\n",
    "| True normal    |              133 |                  44 |              57 |\n",
    "| True bacterial |                4 |                 231 |               7 |\n",
    "| True viral     |                0 |                  44 |             104 |\n",
    "\n",
    "The performance is similar to the binary classifier in terms of differentiating normal and pneumonia images."
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "encoding": "# -*- coding: utf-8 -*-",
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc-autonumbering": false,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 5
}