evaluate_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cdd18257",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Evaluate a CNN classifier on holdout test set.\n",
    "Copyright (c) 2023 Global Health Labs, Inc\n",
    "The above copyright notice and this permission notice shall be included in all\n",
    "copies or substantial portions of the Software.\n",
    "\"\"\"\n",
    "\n",
    "import os\n",
    "import time\n",
    "import sys\n",
    "import glob\n",
    "import random\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import tensorflow.keras.backend as K\n",
    "from CNN.data_generator import DataGeneratorMemory,DataGenerator\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, accuracy_score, precision_score, recall_score, confusion_matrix,f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6fafc146",
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Configurations #####\n",
    "val_pos_dir='/workspace/data/EndorTeamArea/skulhare/lrs_ped_con/frames/holdout/consolidation' # directory with positive images\n",
    "val_neg_dir='/workspace/data/EndorTeamArea/skulhare/lrs_ped_con/frames/holdout/no_consolidation' # directory with negative images\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\" # it will occupy GPU randomly if it's not limited by the system\n",
    "IMG_SHAPE = (256, 256, 1)\n",
    "batch_size = 1\n",
    "experiment_dir = 'logs/75/855/combine'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "de1e1bc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Data Setup #####\n",
    "validation_images_1 = glob.glob(val_pos_dir + '/*.jpg')+glob.glob(val_pos_dir + '/**/*.jpg')\n",
    "validation_images_2 = glob.glob(val_neg_dir + '/*.jpg')+glob.glob(val_neg_dir + '/**/*.jpg')\n",
    "validation_images=validation_images_1+validation_images_2\n",
    "\n",
    "random.shuffle(validation_images)\n",
    "validation_labels = [1 if val_pos_dir in image else 0 for image in validation_images]\n",
    "\n",
    "validation_datagen = DataGeneratorMemory(validation_images,\n",
    "                                    validation_labels,\n",
    "                                    IMG_SHAPE,\n",
    "                                    batch_size=batch_size,\n",
    "                                    n_classes=2,\n",
    "                                    shuffle=True,\n",
    "                                    balanced=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5163b5f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_best(basedir):\n",
    "    \"\"\" find the best CNN classifier with the highest validation accuracy\n",
    "    Args:\n",
    "      basedir:  a folder path which follows the underlining structure (default results of CNN classifier)\n",
    "     └──basedir\n",
    "          ├── modelA      (hyper-parameter setting A)\n",
    "          │   ├── saved-model-01-0.xxxx.hdf5\n",
    "          │   ├── saved-model-02-0.yyyy.hdf5\n",
    "          :   :\n",
    "          └── modelB     (hyper-parameter setting A)\n",
    "              ├── saved-model-01-0.zzzz.hdf5\n",
    "              :\n",
    "              └── saved-model-100-0.aaaa.hdf5\n",
    "    :return: model checkpoint path with the highest validation accuracy\n",
    "    \"\"\"\n",
    "    model_list = glob.glob(basedir +'/**/*.hdf5')\n",
    "    maxi = 0.0 \n",
    "    best = None\n",
    "    for model in model_list:\n",
    "        acc = float(model.split('.')[-2].split('-')[-1])\n",
    "        if acc>maxi:\n",
    "            maxi = acc\n",
    "            best = model\n",
    "    return best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0169f237",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = find_best(experiment_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "06361819",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-08-24 23:36:09.521818: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
      "2023-08-24 23:36:09.570822: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:09.571014: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1666] Found device 0 with properties: \n",
      "name: NVIDIA RTX A6000 major: 8 minor: 6 memoryClockRate(GHz): 1.8\n",
      "pciBusID: 0000:4b:00.0\n",
      "2023-08-24 23:36:09.571264: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2023-08-24 23:36:09.586343: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2023-08-24 23:36:09.620620: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
      "2023-08-24 23:36:09.622229: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
      "2023-08-24 23:36:09.623556: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n",
      "2023-08-24 23:36:09.627291: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
      "2023-08-24 23:36:09.627389: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
      "2023-08-24 23:36:09.627515: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:09.627737: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:09.627872: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1794] Adding visible gpu devices: 0\n",
      "2023-08-24 23:36:09.657343: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 3799965000 Hz\n",
      "2023-08-24 23:36:09.659887: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2d03fda20 initialized for platform Host (this does not guarantee that XLA will be used). Devices:\n",
      "2023-08-24 23:36:09.659920: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): Host, Default Version\n",
      "2023-08-24 23:36:09.739976: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:09.740203: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2cfc6c1e0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "2023-08-24 23:36:09.740228: I tensorflow/compiler/xla/service/service.cc:176]   StreamExecutor device (0): NVIDIA RTX A6000, Compute Capability 8.6\n",
      "2023-08-24 23:36:09.740507: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:09.740742: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1666] Found device 0 with properties: \n",
      "name: NVIDIA RTX A6000 major: 8 minor: 6 memoryClockRate(GHz): 1.8\n",
      "pciBusID: 0000:4b:00.0\n",
      "2023-08-24 23:36:09.740774: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2023-08-24 23:36:09.740805: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2023-08-24 23:36:09.740824: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcufft.so.10\n",
      "2023-08-24 23:36:09.740841: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcurand.so.10\n",
      "2023-08-24 23:36:09.740859: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusolver.so.11\n",
      "2023-08-24 23:36:09.740875: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcusparse.so.11\n",
      "2023-08-24 23:36:09.740892: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n",
      "2023-08-24 23:36:09.740983: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:09.741268: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:09.741468: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1794] Adding visible gpu devices: 0\n",
      "2023-08-24 23:36:09.741855: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
      "2023-08-24 23:36:10.823571: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1206] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2023-08-24 23:36:10.823622: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1212]      0 \n",
      "2023-08-24 23:36:10.823630: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1225] 0:   N \n",
      "2023-08-24 23:36:10.824158: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:10.824436: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1082] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-08-24 23:36:10.824644: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1351] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 46475 MB memory) -> physical GPU (device: 0, name: NVIDIA RTX A6000, pci bus id: 0000:4b:00.0, compute capability: 8.6)\n",
      "2023-08-24 23:36:12.821757: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcublas.so.11\n",
      "2023-08-24 23:36:13.806995: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudnn.so.8\n"
     ]
    }
   ],
   "source": [
    "model = keras.models.load_model(model_path)\n",
    "        \n",
    "# if we want to train with other values:\n",
    "model.training = False # to shut down the drop out and batch normalization\n",
    "scores = model.predict_generator(validation_datagen)\n",
    "\n",
    "# Convert predictions to binary values (0 or 1)\n",
    "y_pred_binary = [1 if pred[1] > 0.5 else 0 for pred in scores]\n",
    "consolidation_score = [score[1] for score in scores]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1a8c7e97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC AUC: 0.9693253441938101\n",
      "Accuracy: 0.9299134612220723\n",
      "F1 Score: 0.8953865295831832\n"
     ]
    }
   ],
   "source": [
    "# Compute ROC and AUROC\n",
    "fpr, tpr, thresholds = roc_curve(validation_datagen.labels, consolidation_score)\n",
    "roc_auc = roc_auc_score(validation_datagen.labels, consolidation_score)\n",
    "\n",
    "# Compute accuracy, and f1\n",
    "accuracy = accuracy_score(validation_datagen.labels, y_pred_binary)\n",
    "f1 = f1_score(validation_datagen.labels, y_pred_binary, average='binary')\n",
    "\n",
    "# Print the evaluation metrics\n",
    "print(\"ROC AUC:\", roc_auc)\n",
    "print(\"Accuracy:\", accuracy)\n",
    "print(\"F1 Score:\", f1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}