PAN_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a47f42d5-9942-4351-b4a5-83803320e534",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import sys\n",
    "sys.path.append(str(Path('../WACV2022/PAN-PyTorch').resolve()))\n",
    "\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from ops.dataset import PANDataSet\n",
    "from ops.models import PAN\n",
    "from ops.transforms import *\n",
    "from opts import parser\n",
    "from ops import dataset_config\n",
    "from ops.utils import AverageMeter, accuracy\n",
    "from ops.temporal_shift import make_temporal_pool\n",
    "\n",
    "\n",
    "from methods import gradcamPAN as gradcam\n",
    "from methods import risePAN as rise\n",
    "from methods import siduPAN as sidu\n",
    "\n",
    "from metrics import insertionPAN as insertion\n",
    "from metrics import deletionPAN as deletion\n",
    "from util import normalize\n",
    "from util import groupNormalize as norm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c20979da-c143-4ee8-8057-5c260283f281",
   "metadata": {},
   "outputs": [],
   "source": [
    "spectrum = 'TV'\n",
    "model_spectrum = 'IRTV'\n",
    "\n",
    "num_classes = 8\n",
    "cfg_LITE = {'dataset': f'{spectrum.lower()}', 'modality': 'Lite', 'train_list': f'/data/SOIB/file_lists/{spectrum}_train.txt', 'val_list': f'/data/SOIB/file_lists/{spectrum}_test.txt', 'root_path': '/data/', 'store_name': 'PAN_Lite_irtv_resnet50_shift8_blockres_avg_segment8_e50', 'lmdb': False, 'arch': 'resnet50', 'num_segments': 8, 'consensus_type': 'avg', 'k': 3, 'dropout': 0.5, 'loss_type': 'nll', 'img_feature_dim': 256, 'suffix': None, 'pretrain': 'imagenet', 'tune_from': None, 'base': 'TSM', 'epochs': 50, 'batch_size': 22, 'lr': 0.001, 'lr_type': 'step', 'lr_steps': [30.0, 40.0], 'momentum': 0.9, 'weight_decay': 0.0001, 'clip_gradient': 20, 'no_partialbn': True, 'iter_size': 1, 'print_freq': 20, 'eval_freq': 1, 'workers': 18, 'resume': '', 'evaluate': False, 'snapshot_pref': '', 'start_epoch': 0, 'gpus': None, 'flow_prefix': '', 'root_log': 'log', 'root_model': 'checkpoint', 'shift': True, 'shift_div': 8, 'shift_place': 'blockres', 'temporal_pool': False, 'non_local': False, 'dense_sample': False, 'VAP': True}\n",
    "\n",
    "cfg = cfg_LITE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d363e65-0454-4df7-9961-7274e573d780",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_length = 1 if cfg['modality'] == 'RGB' else 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a18f38a8-1599-4e27-ba64-26bc813127af",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = PAN(num_classes, cfg['num_segments'], cfg['modality'],\n",
    "                base_model=cfg['arch'],\n",
    "                consensus_type=cfg['consensus_type'],\n",
    "                dropout=cfg['dropout'],\n",
    "                img_feature_dim=cfg['img_feature_dim'],\n",
    "                partial_bn=not cfg['no_partialbn'],\n",
    "                pretrain=cfg['pretrain'],\n",
    "                is_shift=cfg['shift'], shift_div=cfg['shift_div'], shift_place=cfg['shift_place'],\n",
    "                fc_lr5=not (cfg['tune_from'] and cfg['dataset'] in cfg['tune_from']),\n",
    "                temporal_pool=cfg['temporal_pool'],\n",
    "                non_local=cfg['non_local'], data_length=data_length, has_VAP=cfg['VAP']).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53fa97b7-513d-4f14-a7bd-453f1e04a51b",
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = torch.load(f'../WACV2022/PAN-PyTorch/checkpoint/PAN_Lite_{model_spectrum.lower()}_resnet50_shift8_blockres_avg_segment8_e50/ckpt.best.pth.tar')\n",
    "model.load_state_dict({k.split('.',1)[1]: v for k, v in weights['state_dict'].items()})\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51930a23-c7a0-4075-b4c7-7b4ef05ac40c",
   "metadata": {},
   "outputs": [],
   "source": [
    "val_loader = torch.utils.data.DataLoader(\n",
    "        PANDataSet('/data/SOIB', f'/data/SOIB/file_lists/{spectrum}_test.txt', num_segments=cfg['num_segments'],\n",
    "                   new_length=data_length,\n",
    "                   modality=cfg['modality'],\n",
    "                   image_tmpl='{:04d}.png',\n",
    "                   random_shift=False,\n",
    "                   transform=torchvision.transforms.Compose([\n",
    "                       GroupScale(int(model.scale_size)),\n",
    "                       GroupCenterCrop(model.crop_size),\n",
    "                       Stack(roll=(cfg['arch'] in ['BNInception', 'InceptionV3'])),\n",
    "                       ToTorchFormatTensor(div=(cfg['arch'] not in ['BNInception', 'InceptionV3'])),\n",
    "                   ]), dense_sample=False, is_lmdb=False),\n",
    "                    batch_size=1, shuffle=False,\n",
    "                    num_workers=4, pin_memory=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3697926-b75b-4e38-aec2-5ff5601b2aad",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "def evaluate(input, target=None):\n",
    "    if target is None:\n",
    "        output = model(norm(input).cuda())\n",
    "        target = output.argmax().item()\n",
    "    gcam = gradcam(model, model.base_model.avgpool, vid, target)\n",
    "    rcams = rise(model, vid)\n",
    "    scams = sidu(model, model.base_model.avgpool, vid)\n",
    "    scores = {}\n",
    "    values = {}\n",
    "    \n",
    "    with torch.cuda.amp.autocast():\n",
    "        for desc, cam in [('gcam', gcam), ('rcam', rcams[target]), ('scam', scams[target])]:\n",
    "            dels, dscore = deletion(model, vid, cam, target)\n",
    "            inss, iscore = insertion(model, vid, cam, target, factor=4)\n",
    "\n",
    "            scores[desc] = {'insertion': iscore, 'deletion': dscore}\n",
    "            values[desc] = {'insertion': inss, 'deletion': dels}\n",
    "    \n",
    "    return scores, values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7517113-08fd-46e4-8a82-6ec4df5860ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = []\n",
    "values = []\n",
    "for i, (vid, target) in enumerate(tqdm(val_loader)):\n",
    "    s, v = evaluate(vid)\n",
    "    scores.append(s)\n",
    "    values.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a956db0-2538-4f1a-98ac-6f33b8252846",
   "metadata": {},
   "outputs": [],
   "source": [
    "content = [(\n",
    "{\n",
    "    'experiment': i,\n",
    "    'cam_type': 'gcam',\n",
    "    'step': s,\n",
    "    'insertion': v['gcam']['insertion'][s].item(),\n",
    "    'deletion': v['gcam']['deletion'][s].item(),\n",
    "},\n",
    "{\n",
    "    'experiment': i,\n",
    "    'cam_type': 'scam',\n",
    "    'step': s,\n",
    "    'insertion': v['scam']['insertion'][s].item(),\n",
    "    'deletion': v['scam']['deletion'][s].item(),\n",
    "},\n",
    "{\n",
    "    'experiment': i,\n",
    "    'cam_type': 'rcam',\n",
    "    'step': s,\n",
    "    'insertion': v['rcam']['insertion'][s].item(),\n",
    "    'deletion': v['rcam']['deletion'][s].item(),\n",
    "},\n",
    ") for i, v in enumerate(values) for s in range(len(values[0]['gcam']['insertion']))]\n",
    "\n",
    "df = []\n",
    "for c in content:\n",
    "    df += c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b56b562-6f14-472f-8b75-bb4f955d0404",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "import seaborn as sns\n",
    "\n",
    "df = pandas.DataFrame(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f26aab3c-6a7d-4b17-a9bb-cd67b18c1d8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import auc\n",
    "aucs = {}\n",
    "for j, cam in df.groupby('cam_type'):\n",
    "    aucs[j] = {'deletion': [], 'insertion': []}\n",
    "    for i, g in cam.groupby('experiment'):\n",
    "        aucs[j]['deletion'].append(auc(g['step'], g['deletion']))\n",
    "        aucs[j]['insertion'].append(auc(g['step'], g['insertion']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a195c5b-068a-41b7-be10-defcf79f7d4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,8))\n",
    "ax1.set_title('Insertion')\n",
    "ax2.set_title('Deletion')\n",
    "\n",
    "sns.lineplot(y='insertion', x='step', data=df[df.cam_type=='gcam'], ax=ax1, label=f'Grad-CAM AUC=${np.mean(aucs[\"gcam\"][\"insertion\"]):0.2f}\\pm {np.std(aucs[\"gcam\"][\"insertion\"]):0.2f}$')\n",
    "sns.lineplot(y='insertion', x='step', data=df[df.cam_type=='scam'], ax=ax1, label=f'SIDU AUC=${np.mean(aucs[\"scam\"][\"insertion\"]):0.2f}\\pm {np.std(aucs[\"scam\"][\"insertion\"]):0.2f}$')\n",
    "sns.lineplot(y='insertion', x='step', data=df[df.cam_type=='rcam'], ax=ax1, label=f'RISE AUC=${np.mean(aucs[\"rcam\"][\"insertion\"]):0.2f}\\pm {np.std(aucs[\"rcam\"][\"insertion\"]):0.2f}$')\n",
    "\n",
    "sns.lineplot(y='deletion', x='step', data=df[df.cam_type=='gcam'], ax=ax2, label=f'Grad-CAM AUC=${np.mean(aucs[\"gcam\"][\"deletion\"]):0.2f}\\pm {np.std(aucs[\"gcam\"][\"deletion\"]):0.2f}$')\n",
    "sns.lineplot(y='deletion', x='step', data=df[df.cam_type=='scam'], ax=ax2, label=f'SIDU AUC=${np.mean(aucs[\"scam\"][\"deletion\"]):0.2f}\\pm {np.std(aucs[\"scam\"][\"deletion\"]):0.2f}$')\n",
    "sns.lineplot(y='deletion', x='step', data=df[df.cam_type=='rcam'], ax=ax2, label=f'RISE AUC=${np.mean(aucs[\"rcam\"][\"deletion\"]):0.2f}\\pm {np.std(aucs[\"rcam\"][\"deletion\"]):0.2f}$')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}