[NC] Infection Evaluation Fidelity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c15c756b",
   "metadata": {},
   "source": [
    "# Evaluate the fidelity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8ebffdac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "from Datasets.synthetics import Infection\n",
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd65078c",
   "metadata": {},
   "source": [
    "## Setup PyG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "95ed4f81",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.set_num_threads(4)\n",
    "device = 'cuda'\n",
    "\n",
    "from metrics.utils_nc import set_seeds"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cbb7ca0",
   "metadata": {},
   "source": [
    "# Compute the fidelity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e01376e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* * * * * * * * GCN * * * * * * * *\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/anaconda3/envs/geo_new/lib/python3.9/site-packages/numpy/core/fromnumeric.py:3440: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "/home/antonio/anaconda3/envs/geo_new/lib/python3.9/site-packages/numpy/core/_methods.py:189: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.997  0.760  -0.198\tsubgraphX\n",
      "* * * * * * * * GAT * * * * * * * *\n",
      " -0.297  0.771  0.657\tsubgraphX\n",
      "* * * * * * * * GIN * * * * * * * *\n",
      " 0.988  0.771  -0.200\tsubgraphX\n",
      "* * * * * * * * GraphSAGE * * * * * * * *\n",
      " 0.642  -0.191  0.774\tsubgraphX\n",
      "* * * * * * * * Cheb * * * * * * * *\n",
      " -----  -----  -----\tsubgraphX\n"
     ]
    }
   ],
   "source": [
    "from metrics.utils_nc import build_expl, compute_fidelity\n",
    "from models.models_Infection import GCN_framework, GraphSAGE_framework, GAT_framework, GIN_framework, CHEB_framework\n",
    "\n",
    "\n",
    "# Define the parameters\n",
    "DATASET = 'Infection'\n",
    "MODELS = ['GCN', 'GAT', 'GIN', 'GraphSAGE', 'Cheb']\n",
    "\n",
    "EXPLS = ['cam', 'grad_cam', 'grad_exp', 'guided_bp', 'ig_node', 'pgmexplainer', 'gnnexpl', 'pgexplainer']\n",
    "EXPLS = [\"subgraphX\"]\n",
    "MODES = ['train']\n",
    "\n",
    "IGNORE = [('Infection', 'Cheb', 'gnnexpl', 'train'),\n",
    "          ('Infection', 'Cheb', 'pgexplainer', 'train'),\n",
    "          ('Infection', 'GIN', 'gnnexpl', 'train')]\n",
    "\n",
    "GNN_NUM_LAYERS = {'GCN': 2, 'GAT': 2, 'GIN': 2, 'GraphSAGE': 2, 'Cheb': 2}\n",
    "\n",
    "FRAMEWORKS = {'GCN': GCN_framework, \n",
    "              'GAT': GAT_framework, \n",
    "              'GIN': GIN_framework,\n",
    "              'GraphSAGE': GraphSAGE_framework, \n",
    "              'Cheb': CHEB_framework\n",
    "             }\n",
    "\n",
    "\n",
    "# Load the dataset\n",
    "set_seeds()\n",
    "dataset = Infection()\n",
    "\n",
    "\n",
    "# Define history variables\n",
    "suff = {0: [], 1: [], 2: []}\n",
    "model, expl = [], []\n",
    "\n",
    "\n",
    "# Compute the metrics\n",
    "for MODE in MODES:\n",
    "    for MODEL in MODELS:\n",
    "        print(8 * '* ' + MODEL + 8 * ' *')\n",
    "        \n",
    "        # Define and load the trained model\n",
    "        gcn = FRAMEWORKS[MODEL](dataset, device='cuda')\n",
    "        path = 'models/' + DATASET + '_' + MODEL\n",
    "        gcn.load_model(path)\n",
    "\n",
    "        # Loop over the explainers\n",
    "        for EXPL in EXPLS:\n",
    "            # Define the setting and store it\n",
    "            ID = (DATASET, MODEL, EXPL, MODE)\n",
    "            model += [MODEL]\n",
    "            expl += [EXPL]\n",
    "            \n",
    "            # Compute the sufficiency\n",
    "            if ID in IGNORE:\n",
    "                # Return nan metrics\n",
    "                suff[0].append(float('nan'))\n",
    "                suff[1].append(float('nan'))\n",
    "                suff[2].append(float('nan'))\n",
    "            else:\n",
    "                # Load and process the explanations\n",
    "                graphs = build_expl(DATASET, MODEL, EXPL, GNN_NUM_LAYERS, num_features=2, cut_ego=False)\n",
    "\n",
    "                # Loop over the class labels\n",
    "                for label in graphs:\n",
    "                    if not graphs[label] == None:\n",
    "                        # If there are valid explanations compute suff and comp    \n",
    "                        suff[label] += [compute_fidelity(gcn, dataset.data, graphs, num_features=2, y=label)]\n",
    "                    else:\n",
    "                        # Otherwise return nan metrics\n",
    "                        suff[label].append(float('nan'))\n",
    "                    \n",
    "            # Print the partial results                    \n",
    "            print(' '.join([' {:.3f}'.format(suff[label][-1]) if ~np.isnan(suff[label][-1]) else ' -----' for label in suff]) + '\\t' + EXPL)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f18eafb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>expl</th>\n",
       "      <th>class 0</th>\n",
       "      <th>class 1</th>\n",
       "      <th>class 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GCN</td>\n",
       "      <td>subgraphX</td>\n",
       "      <td>0.997460</td>\n",
       "      <td>0.759996</td>\n",
       "      <td>-0.197984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GAT</td>\n",
       "      <td>subgraphX</td>\n",
       "      <td>-0.296510</td>\n",
       "      <td>0.771209</td>\n",
       "      <td>0.657493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GIN</td>\n",
       "      <td>subgraphX</td>\n",
       "      <td>0.988424</td>\n",
       "      <td>0.770655</td>\n",
       "      <td>-0.199946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GraphSAGE</td>\n",
       "      <td>subgraphX</td>\n",
       "      <td>0.641560</td>\n",
       "      <td>-0.190920</td>\n",
       "      <td>0.774214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cheb</td>\n",
       "      <td>subgraphX</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       model       expl   class 0   class 1   class 2\n",
       "0        GCN  subgraphX  0.997460  0.759996 -0.197984\n",
       "1        GAT  subgraphX -0.296510  0.771209  0.657493\n",
       "2        GIN  subgraphX  0.988424  0.770655 -0.199946\n",
       "3  GraphSAGE  subgraphX  0.641560 -0.190920  0.774214\n",
       "4       Cheb  subgraphX       NaN       NaN       NaN"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "results = pd.DataFrame({'model': model, 'expl': expl, 'class 0': suff[0], 'class 1': suff[1], 'class 2': suff[2]})\n",
    "results.fillna(-100).to_csv('./metrics/fidelity_sufficiency_' + '_'.join(MODELS) + '.csv')\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "decb4849",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}