notebooks/ncg_on_all.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "43eb1eff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../\")\n",
    "from os.path import join\n",
    "import os\n",
    "\n",
    "import matplotlib\n",
    "matplotlib.rcParams['pdf.fonttype'] = 42\n",
    "matplotlib.rcParams['ps.fonttype'] = 42\n",
    "matplotlib.rc('text', usetex=False)\n",
    "\n",
    "import scipy\n",
    "import faiss\n",
    "import matplotlib.pyplot as plt\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from tqdm.notebook import tqdm\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from lolip.models.torch_utils import archs\n",
    "from lolip.variables import auto_var"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6c782df",
   "metadata": {},
   "source": [
    "# Pixel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3672b46a",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [f\"mnistwo{i}\" for i in range(10)] + [f\"cifar10wo{i}\" for i in range(10)] + [f\"cifar100coarsewo{i}\" for i in range(10)] + [f\"aug10-imgnet100wo{i}\" for i in range(10)]\n",
    "paths = [f\"./results/oos_repr/cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/oos_repr/cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/oos_repr/cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/oos_repr/cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl\" for i in range(10)]\n",
    "n_classes = [9] * 10 + [9] * 10 + [19] * 10 + [99] * 10\n",
    "\n",
    "test_data = {}\n",
    "data = {}\n",
    "for dset, path, n_class in zip(datasets, paths, n_classes):\n",
    "    if not os.path.exists(path):\n",
    "        continue\n",
    "    try:\n",
    "        res = joblib.load(path)\n",
    "    except:\n",
    "        print(f\"problem with {path}\")\n",
    "        continue\n",
    "    nn_preds = joblib.load(f\"./out_of_sample/{dset}.pkl\")\n",
    "    nn_preds = np.concatenate((nn_preds['miss_trn_knn_y_2'].reshape(-1), nn_preds['miss_tst_knn_y_2'].reshape(-1)))\n",
    "    ood_preds = np.concatenate((res['oos_trn_pred'].reshape(-1), res['oos_tst_pred'].reshape(-1)))\n",
    "    ncg_acc = (ood_preds == nn_preds)\n",
    "    test_data[dset] = scipy.stats.chisquare(np.bincount(ncg_acc), [len(ncg_acc) * (n_class - 1) / n_class, len(ncg_acc) / n_class])\n",
    "    data[dset] = ncg_acc.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "870d7f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [f\"mnistwo{i}\" for i in range(10)] + [f\"cifar10wo{i}\" for i in range(10)] + [f\"cifar100coarsewo{i}\" for i in range(10)] + [f\"aug10-imgnet100wo{i}\" for i in range(10)]\n",
    "paths = [f\"./results/oos_repr/cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/oos_repr/cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/oos_repr/cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/oos_repr/cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl\" for i in range(10)]\n",
    "n_classes = [9] * 10 + [9] * 10 + [19] * 10 + [99] * 10\n",
    "\n",
    "test_data = {}\n",
    "data = {}\n",
    "for dset, path, n_class in zip(datasets, paths, n_classes):\n",
    "    if not os.path.exists(path):\n",
    "        continue\n",
    "    try:\n",
    "        res = joblib.load(path)\n",
    "    except:\n",
    "        print(f\"problem with {path}\")\n",
    "        continue\n",
    "    nn_preds = joblib.load(f\"./out_of_sample/{dset}.pkl\")\n",
    "    nn_preds = np.concatenate((nn_preds['miss_trn_knn_y_2'].reshape(-1), nn_preds['miss_tst_knn_y_2'].reshape(-1)))\n",
    "    ood_preds = np.concatenate((res['oos_trn_pred'].reshape(-1), res['oos_tst_pred'].reshape(-1)))\n",
    "    ncg_acc = (ood_preds == nn_preds)\n",
    "    test_data[dset] = scipy.stats.chisquare(np.bincount(ncg_acc), [len(ncg_acc) * (n_class - 1) / n_class, len(ncg_acc) / n_class])\n",
    "    data[dset] = ncg_acc.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bfee1ca",
   "metadata": {},
   "source": [
    "# Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2298e71f",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = [\n",
    "    f\"calcedrepr-mnistwo{i}-cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl\" for i in range(10)\n",
    "] + [f\"calcedrepr-cifar10wo{i}-cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in range(10)\n",
    "] + [f\"calcedrepr-cifar100coarsewo{i}-cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in range(10)\n",
    "] + [f\"calcedrepr-aug10-imgnet100wo{i}-cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl\" for i in range(10)]\n",
    "\n",
    "paths = [f\"./results/out_of_sample/cwl2-256-calcedrepr-mnistwo{i}-cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/out_of_sample/cwl2-128-calcedrepr-cifar10wo{i}-cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/out_of_sample/cwl2-128-calcedrepr-cifar100coarsewo{i}-cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0.pkl\" for i in range(10)]\n",
    "paths += [f\"./results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo{i}-cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLPv4-0.9-2-sgd-0-0.0.pkl\" for i in range(10)]\n",
    "n_classes = [9] * 10 + [9] * 10 + [19] * 10 + [99] * 10\n",
    "\n",
    "data = {}\n",
    "for dset, path, n_class in zip(datasets, paths, n_classes):\n",
    "    if not os.path.exists(path):\n",
    "        continue\n",
    "    try:\n",
    "        res = joblib.load(path)\n",
    "    except:\n",
    "        print(f\"problem with {path}\")\n",
    "        continue\n",
    "    nn_preds = joblib.load(f\"./out_of_sample/{dset}.pkl\")\n",
    "    nn_preds = np.concatenate((nn_preds['miss_trn_knn_y_2'].reshape(-1), nn_preds['miss_tst_knn_y_2'].reshape(-1)))\n",
    "    ood_preds = np.concatenate((res['oos_trn_pred'].reshape(-1), res['oos_tst_pred'].reshape(-1)))\n",
    "    ncg_acc = (ood_preds == nn_preds)\n",
    "    data[dset] = scipy.stats.chisquare(np.bincount(ncg_acc), [len(ncg_acc) * (n_class - 1) / n_class, len(ncg_acc) / n_class])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a3c7c0de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'calcedrepr-mnistwo0-cwl2-128-mnistwo0-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=1902.8502281616688, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo1-cwl2-128-mnistwo1-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=0.029341754475053956, pvalue=0.8639923225491427),\n",
       " 'calcedrepr-mnistwo2-cwl2-128-mnistwo2-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=6506.0349785407725, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo3-cwl2-128-mnistwo3-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=23597.26676936003, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo4-cwl2-128-mnistwo4-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=30943.951513042197, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo5-cwl2-128-mnistwo5-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=15909.546095358786, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo6-cwl2-128-mnistwo6-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=12546.63759816754, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo7-cwl2-128-mnistwo7-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=12533.761209378856, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo8-cwl2-128-mnistwo8-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=8457.456428571428, pvalue=0.0),\n",
       " 'calcedrepr-mnistwo9-cwl2-128-mnistwo9-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl': Power_divergenceResult(statistic=21478.315051020407, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo0-cwl2-64-cifar10wo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=28426.869187499997, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo1-cwl2-64-cifar10wo1-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=39372.9526875, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo2-cwl2-64-cifar10wo2-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=30184.751999999997, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo3-cwl2-64-cifar10wo3-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=32393.388, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo4-cwl2-64-cifar10wo4-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=30528.299999999996, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo5-cwl2-64-cifar10wo5-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=36603.147000000004, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo6-cwl2-64-cifar10wo6-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=31032.7921875, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo7-cwl2-64-cifar10wo7-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=32349.0421875, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo8-cwl2-64-cifar10wo8-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=34420.3876875, pvalue=0.0),\n",
       " 'calcedrepr-cifar10wo9-cwl2-64-cifar10wo9-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=32689.803, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo0-cwl2-64-cifar100coarsewo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=20242.51562962963, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo1-cwl2-64-cifar100coarsewo1-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=21710.696296296297, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo2-cwl2-64-cifar100coarsewo2-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=24085.409185185188, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo3-cwl2-64-cifar100coarsewo3-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=24365.38016666667, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo4-cwl2-64-cifar100coarsewo4-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=11526.900018518521, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo5-cwl2-64-cifar100coarsewo5-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=20993.86851851852, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo6-cwl2-64-cifar100coarsewo6-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=21017.56890740741, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo7-cwl2-64-cifar100coarsewo7-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=19918.104, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo8-cwl2-64-cifar100coarsewo8-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=19779.87266666667, pvalue=0.0),\n",
       " 'calcedrepr-cifar100coarsewo9-cwl2-64-cifar100coarsewo9-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=22268.413500000002, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo0-cwl2-128-aug10-imgnet100wo0-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=2124.960612244898, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=2463.695510204082, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=2001.345306122449, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo3-cwl2-128-aug10-imgnet100wo3-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=2150.1281632653063, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo4-cwl2-128-aug10-imgnet100wo4-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=3372.4953061224496, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo5-cwl2-128-aug10-imgnet100wo5-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=2050.3469387755104, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo6-cwl2-128-aug10-imgnet100wo6-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=1834.506734693878, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo7-cwl2-128-aug10-imgnet100wo7-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=2572.9591836734694, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo8-cwl2-128-aug10-imgnet100wo8-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=2409.9526530612247, pvalue=0.0),\n",
       " 'calcedrepr-aug10-imgnet100wo9-cwl2-128-aug10-imgnet100wo9-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl': Power_divergenceResult(statistic=1565.3883673469388, pvalue=0.0)}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}