notebooks/feature_space_ncg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../\")\n",
    "import re\n",
    "from os.path import join\n",
    "from IPython.display import display\n",
    "import gc\n",
    "from collections.abc import Iterable\n",
    "\n",
    "\n",
    "import matplotlib\n",
    "matplotlib.rcParams['pdf.fonttype'] = 42\n",
    "matplotlib.rcParams['ps.fonttype'] = 42\n",
    "matplotlib.rc('text', usetex=False)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from tqdm.notebook import tqdm\n",
    "import faiss\n",
    "\n",
    "from lolip.models.torch_utils.archs import *\n",
    "from lolip.variables import auto_var\n",
    "from params import *\n",
    "from utils import params_to_dataframe\n",
    "from notebook_utils import get_model_name, get_arch, get_ds_name\n",
    "\n",
    "fontsize=15\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "2022-06-29 10:01:24 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar10wo0-cwl2-64-cifar10wo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:24 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar10wo0-cwl2-64-cifar10wo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 4.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar10wo0-cwl2-64-cifar10wo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar10wo0-cwl2-64-cifar10wo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-4.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 10:01:27 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:27 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 4.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "problem with ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-4.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-8.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 10:01:27 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 8.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:27 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 32.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:27 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 4.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-32.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-4.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-8.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 8.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 32.0, 'learning_rate': 0.01, 'model': 'trades6ce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar10wo0-cwl2-64-cifar10wo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar10wo4-cwl2-64-cifar10wo4-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-32.0-0.01-trades6ce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar10wo0-cwl2-64-cifar10wo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar10wo4-cwl2-64-cifar10wo4-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar10wo9-cwl2-64-cifar10wo9-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar10wo9-cwl2-64-cifar10wo9-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar100coarsewo0-cwl2-64-cifar100coarsewo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar100coarsewo4-cwl2-64-cifar100coarsewo4-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar100coarsewo0-cwl2-64-cifar100coarsewo0-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar100coarsewo4-cwl2-64-cifar100coarsewo4-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-cifar100coarsewo9-cwl2-64-cifar100coarsewo9-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-cifar100coarsewo9-cwl2-64-cifar100coarsewo9-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:28 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo0-cwl2-128-aug10-imgnet100wo0-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "2022-06-29 10:01:29 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo0-cwl2-128-aug10-imgnet100wo0-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo1-cwl2-128-aug10-imgnet100wo1-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n",
      "Missing ../results/out_of_sample/cwl2-128-calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-advce-vtor2-VggMLP-0.0-2-adam-0-0.0.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 10:01:29 ERROR    Error with {'attack': 'cwl2', 'batch_size': 128, 'dataset': 'calcedrepr-aug10-imgnet100wo2-cwl2-128-aug10-imgnet100wo2-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl', 'epochs': 70, 'eps': 2.0, 'learning_rate': 0.01, 'model': 'advce-vtor2-VggMLP', 'momentum': 0.0, 'norm': '2', 'optimizer': 'adam', 'random_seed': 0, 'weight_decay': 0.0}\n",
      "[Parallel(n_jobs=1)]: Done 419 out of 419 | elapsed:   41.0s finished\n"
     ]
    }
   ],
   "source": [
    "columns = ['trn_acc', 'tst_acc', 'oos_trn_pred', 'oos_tst_pred', 'model_path']\n",
    "result_file_dir = \"../results/out_of_sample/\"\n",
    "#result_file_dir = \"../results/out_of_sample_mnist_bak/\"\n",
    "_, _, grid_params, _ = OutOfSampleReprClf()()\n",
    "df = params_to_dataframe(grid_params,\n",
    "                         columns=columns,\n",
    "                         result_file_dir=result_file_dir,\n",
    "                         verbose=1, logging_level=0)\n",
    "df['arch'] = df['model'].apply(get_arch)\n",
    "df['model name'] = [get_model_name(d['model'], d['eps']) for _, d in df.iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f5d75e2bfe134e06b402fd16bbc36b4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ncg_acc = []\n",
    "for _, di in tqdm(df.iterrows()):\n",
    "    if isinstance(di['oos_trn_pred'], Iterable):\n",
    "        net_pred = np.concatenate((di['oos_trn_pred'], di['oos_tst_pred']))\n",
    "        res = joblib.load(f\"./out_of_sample/{di['dataset']}.pkl\")\n",
    "        nn_pred = np.concatenate((\n",
    "            res[f'miss_trn_knn_y_{di[\"norm\"]}'].reshape(-1),\n",
    "            res[f'miss_tst_knn_y_{di[\"norm\"]}'].reshape(-1)))\n",
    "        ncg_acc.append((net_pred == nn_pred).mean())\n",
    "    else:\n",
    "        ncg_acc.append(-1)\n",
    "df['ncg acc'] = ncg_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets, ds_names, archs = [], [], []\n",
    "accepted_models = ['natural', 'mixup', 'AT(0.5)', 'AT(1)', 'AT(2)', 'TRADES(2)', 'TRADES(4)', 'TRADES(8)']\n",
    "\n",
    "datasets += [f'calcedreprold-mnistwo{i}-cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl' for i in range(10)]\n",
    "ds_names += [f\"MNIST-wo{i}\" for i in range(10)]\n",
    "archs += [\"LargeMLP\"] * 10\n",
    "#accepted_models += ['natural', 'AT(2)', 'TRADES(2)', 'DBS6($\\lambda$=1)', 'ellipsoid', 'sub-Voronoi']\n",
    "\n",
    "datasets += [f\"calcedrepr-cifar10wo{i}-cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in [0, 4, 9]]\n",
    "ds_names += [f\"CIFAR10-wo{i}\" for i in [0, 4, 9]]\n",
    "archs += [\"LargeMLP\"] * 3\n",
    "##accepted_models = ['natural', 'AT(2)', 'TRADES(2)', 'TRADES(4)', 'DBS6($\\lambda$=1)', 'ellipsoid', 'sub-Voronoi']\n",
    "\n",
    "datasets += [f\"calcedrepr-cifar100coarsewo{i}-cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in [0, 4, 9]]\n",
    "ds_names += [f\"CIFAR100-wo{i}\" for i in [0, 4, 9]]\n",
    "archs += [\"LargeMLP\"] * 3\n",
    "\n",
    "datasets += [f'calcedrepr-aug10-imgnet100wo{i}-cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl' for i in range(3)]\n",
    "ds_names += [f\"ImgNet100-wo{i}\" for i in range(3)]\n",
    "archs += [\"LargeMLPv4\"] * 3\n",
    "##accepted_models = ['natural', 'AT', 'TRADES', 'DBS6($\\lambda$=1)']\n",
    "\n",
    "#columns = ['natural', 'AT(2)', 'TRADES(2)', 'DBS']\n",
    "\n",
    "dfs = []\n",
    "all_data = {}\n",
    "for ds_name, arch in zip(datasets, archs):\n",
    "    td = df[df['dataset'].str.contains(ds_name) & df['arch'].str.contains(arch)]\n",
    "    td = td[['model name', 'arch', 'trn_acc', 'tst_acc', 'ncg acc', 'eps', 'learning_rate', 'batch_size', 'dataset', 'random_seed']]\n",
    "    short_ds_name = ds_names[datasets.index(ds_name)]\n",
    "    \n",
    "    ddfs = []\n",
    "    used_model_names = []\n",
    "    for i, (_, d) in enumerate(td.iterrows()):\n",
    "        if d['model name'] in ['TRADES', 'AT']:\n",
    "            if d['eps'] >= 1:\n",
    "                d['model name'] = f\"{d['model name']}({int(d['eps'])})\"\n",
    "            else:\n",
    "                d['model name'] = f\"{d['model name']}({float(d['eps'])})\"\n",
    "        if \"imgnet\" in ds_name:\n",
    "            if d['model name'] == \"TRADES(2)\":\n",
    "                if d['learning_rate'] != 0.01:\n",
    "                    continue\n",
    "            elif d['learning_rate'] != 0.1:\n",
    "                continue\n",
    "        if \"cifar\" in ds_name:\n",
    "            if d['learning_rate'] != 0.01:\n",
    "                continue\n",
    "            if d['arch'] == 'LargeMLPv3':\n",
    "                continue\n",
    "            \n",
    "        if d['model name'] not in accepted_models:\n",
    "            continue\n",
    "        if d['model name'] in used_model_names:\n",
    "            continue\n",
    "        data = []\n",
    "            \n",
    "        all_data[(short_ds_name, d['model name'])] = (d['trn_acc'], d['tst_acc'], d['ncg acc'])\n",
    "        used_model_names.append(d['model name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.11975308641975309 0.008054077249530917\n",
      "0.13827160493827162 0.011826103718348685\n",
      "0.16074074074074074 0.014299686158587909\n",
      "0.13111111111111112 0.010066082523772602\n",
      "0.13753086419753088 0.005811655454785095\n"
     ]
    }
   ],
   "source": [
    "#accepted_models = ['natural', 'mixup', 'TRADES(2)', 'TRADES(4)', 'TRADES(8)', 'AT(2)']\n",
    "accepted_models = ['natural', 'mixup', 'TRADES(2)', 'TRADES(4)', 'TRADES(8)']\n",
    "#accepted_models = ['natural', 'TRADES(2)', 'TRADES(4)', 'TRADES(8)', 'AT(0.5)']\n",
    "for model in accepted_models:\n",
    "    temp = []\n",
    "    #for ds_name in [f'MNIST-wo{i}' for i in range(10)]:\n",
    "    #for ds_name in [f\"CIFAR10-wo{i}\" for i in [0, 4, 9]]:\n",
    "    #for ds_name in [f\"CIFAR100-wo{i}\" for i in [0, 4, 9]]:\n",
    "    for ds_name in [f\"ImgNet100-wo{i}\" for i in range(3)]:\n",
    "        temp.append(all_data[(ds_name, model)][2])\n",
    "    print(np.mean(temp), np.std(temp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>attack</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>dataset</th>\n",
       "      <th>epochs</th>\n",
       "      <th>eps</th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>model</th>\n",
       "      <th>momentum</th>\n",
       "      <th>norm</th>\n",
       "      <th>optimizer</th>\n",
       "      <th>random_seed</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>trn_acc</th>\n",
       "      <th>tst_acc</th>\n",
       "      <th>oos_trn_pred</th>\n",
       "      <th>oos_tst_pred</th>\n",
       "      <th>model_path</th>\n",
       "      <th>arch</th>\n",
       "      <th>model name</th>\n",
       "      <th>ncg acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>cwl2</td>\n",
       "      <td>256</td>\n",
       "      <td>calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....</td>\n",
       "      <td>70</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>ce-vtor2-LargeMLP</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.993792</td>\n",
       "      <td>[5, 7, 1, 5, 7, 5, 7, 1, 1, 5, 1, 5, 5, 5, 1, ...</td>\n",
       "      <td>[5, 1, 5, 5, 1, 1, 8, 5, 5, 1, 1, 5, 1, 1, 5, ...</td>\n",
       "      <td>./models/out_of_sample/pgd-256-calcedrepr-mnis...</td>\n",
       "      <td>LargeMLP</td>\n",
       "      <td>natural</td>\n",
       "      <td>0.276112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>cwl2</td>\n",
       "      <td>256</td>\n",
       "      <td>calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....</td>\n",
       "      <td>70</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>trades6ce-vtor2-LargeMLP</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999186</td>\n",
       "      <td>0.994568</td>\n",
       "      <td>[5, 5, 5, 5, 7, 5, 7, 5, 7, 7, 7, 5, 5, 5, 1, ...</td>\n",
       "      <td>[5, 5, 5, 5, 1, 7, 8, 5, 5, 1, 1, 5, 5, 1, 5, ...</td>\n",
       "      <td>./models/out_of_sample/pgd-256-calcedrepr-mnis...</td>\n",
       "      <td>LargeMLP</td>\n",
       "      <td>TRADES</td>\n",
       "      <td>0.389975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>cwl2</td>\n",
       "      <td>256</td>\n",
       "      <td>calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....</td>\n",
       "      <td>70</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>trades6ce-vtor2-LargeMLP</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.997670</td>\n",
       "      <td>0.994568</td>\n",
       "      <td>[5, 5, 5, 5, 8, 5, 7, 1, 7, 5, 1, 5, 5, 5, 5, ...</td>\n",
       "      <td>[5, 5, 5, 5, 1, 7, 5, 5, 5, 1, 1, 5, 5, 8, 5, ...</td>\n",
       "      <td>./models/out_of_sample/pgd-256-calcedrepr-mnis...</td>\n",
       "      <td>LargeMLP</td>\n",
       "      <td>TRADES</td>\n",
       "      <td>0.485876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>cwl2</td>\n",
       "      <td>256</td>\n",
       "      <td>calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....</td>\n",
       "      <td>70</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>trades6ce-vtor2-LargeMLP</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.996043</td>\n",
       "      <td>0.994457</td>\n",
       "      <td>[5, 5, 5, 5, 4, 5, 7, 5, 7, 7, 7, 5, 5, 5, 5, ...</td>\n",
       "      <td>[5, 8, 8, 5, 7, 7, 5, 5, 5, 8, 1, 5, 5, 8, 6, ...</td>\n",
       "      <td>./models/out_of_sample/pgd-256-calcedrepr-mnis...</td>\n",
       "      <td>LargeMLP</td>\n",
       "      <td>TRADES</td>\n",
       "      <td>0.547733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>cwl2</td>\n",
       "      <td>256</td>\n",
       "      <td>calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....</td>\n",
       "      <td>70</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.01</td>\n",
       "      <td>advce-vtor2-LargeMLP</td>\n",
       "      <td>0.9</td>\n",
       "      <td>2</td>\n",
       "      <td>sgd</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.999982</td>\n",
       "      <td>0.994900</td>\n",
       "      <td>[5, 5, 5, 5, 7, 5, 7, 1, 7, 5, 1, 5, 5, 5, 7, ...</td>\n",
       "      <td>[5, 1, 5, 5, 1, 1, 5, 5, 5, 1, 1, 5, 5, 1, 5, ...</td>\n",
       "      <td>./models/out_of_sample/pgd-256-calcedrepr-mnis...</td>\n",
       "      <td>LargeMLP</td>\n",
       "      <td>AT</td>\n",
       "      <td>0.324207</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   attack  batch_size                                            dataset  \\\n",
       "18   cwl2         256  calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....   \n",
       "28   cwl2         256  calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....   \n",
       "29   cwl2         256  calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....   \n",
       "30   cwl2         256  calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....   \n",
       "58   cwl2         256  calcedreprold-mnistwo0-cwl2-128-mnistwo0-70-1....   \n",
       "\n",
       "    epochs  eps  learning_rate                     model  momentum norm  \\\n",
       "18      70  1.0           0.01         ce-vtor2-LargeMLP       0.9    2   \n",
       "28      70  2.0           0.01  trades6ce-vtor2-LargeMLP       0.9    2   \n",
       "29      70  4.0           0.01  trades6ce-vtor2-LargeMLP       0.9    2   \n",
       "30      70  8.0           0.01  trades6ce-vtor2-LargeMLP       0.9    2   \n",
       "58      70  2.0           0.01      advce-vtor2-LargeMLP       0.9    2   \n",
       "\n",
       "   optimizer  random_seed  weight_decay   trn_acc   tst_acc  \\\n",
       "18       sgd            0           0.0  1.000000  0.993792   \n",
       "28       sgd            0           0.0  0.999186  0.994568   \n",
       "29       sgd            0           0.0  0.997670  0.994568   \n",
       "30       sgd            0           0.0  0.996043  0.994457   \n",
       "58       sgd            0           0.0  0.999982  0.994900   \n",
       "\n",
       "                                         oos_trn_pred  \\\n",
       "18  [5, 7, 1, 5, 7, 5, 7, 1, 1, 5, 1, 5, 5, 5, 1, ...   \n",
       "28  [5, 5, 5, 5, 7, 5, 7, 5, 7, 7, 7, 5, 5, 5, 1, ...   \n",
       "29  [5, 5, 5, 5, 8, 5, 7, 1, 7, 5, 1, 5, 5, 5, 5, ...   \n",
       "30  [5, 5, 5, 5, 4, 5, 7, 5, 7, 7, 7, 5, 5, 5, 5, ...   \n",
       "58  [5, 5, 5, 5, 7, 5, 7, 1, 7, 5, 1, 5, 5, 5, 7, ...   \n",
       "\n",
       "                                         oos_tst_pred  \\\n",
       "18  [5, 1, 5, 5, 1, 1, 8, 5, 5, 1, 1, 5, 1, 1, 5, ...   \n",
       "28  [5, 5, 5, 5, 1, 7, 8, 5, 5, 1, 1, 5, 5, 1, 5, ...   \n",
       "29  [5, 5, 5, 5, 1, 7, 5, 5, 5, 1, 1, 5, 5, 8, 5, ...   \n",
       "30  [5, 8, 8, 5, 7, 7, 5, 5, 5, 8, 1, 5, 5, 8, 6, ...   \n",
       "58  [5, 1, 5, 5, 1, 1, 5, 5, 5, 1, 1, 5, 5, 1, 5, ...   \n",
       "\n",
       "                                           model_path      arch model name  \\\n",
       "18  ./models/out_of_sample/pgd-256-calcedrepr-mnis...  LargeMLP    natural   \n",
       "28  ./models/out_of_sample/pgd-256-calcedrepr-mnis...  LargeMLP     TRADES   \n",
       "29  ./models/out_of_sample/pgd-256-calcedrepr-mnis...  LargeMLP     TRADES   \n",
       "30  ./models/out_of_sample/pgd-256-calcedrepr-mnis...  LargeMLP     TRADES   \n",
       "58  ./models/out_of_sample/pgd-256-calcedrepr-mnis...  LargeMLP         AT   \n",
       "\n",
       "     ncg acc  \n",
       "18  0.276112  \n",
       "28  0.389975  \n",
       "29  0.485876  \n",
       "30  0.547733  \n",
       "58  0.324207  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['dataset'].str.contains(\"mnistwo0\") & df['arch'].str.contains(\"MLP\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llccccc}\n",
      "\\toprule\n",
      "          &          &  natural &  mixup &  TRADES(2) &  TRADES(4) &  TRADES(8) \\\\\n",
      "\\midrule\n",
      "\\multirow{3}{*}{CIFAR10-wo0} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.89 &   0.90 &       0.89 &       0.90 &       0.90 \\\\\n",
      "          & NCG acc. &     0.80 &   0.78 &       0.81 &       0.83 &       0.83 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{CIFAR10-wo4} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.88 &   0.88 &       0.88 &       0.89 &       0.88 \\\\\n",
      "          & NCG acc. &     0.82 &   0.78 &       0.82 &       0.85 &       0.85 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{CIFAR10-wo9} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.88 &   0.88 &       0.88 &       0.89 &       0.89 \\\\\n",
      "          & NCG acc. &     0.84 &   0.82 &       0.83 &       0.88 &       0.87 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{CIFAR100-wo0} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.72 &   0.73 &       0.73 &       0.74 &       0.74 \\\\\n",
      "          & NCG acc. &     0.63 &   0.64 &       0.69 &       0.68 &       0.68 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{CIFAR100-wo4} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.72 &   0.73 &       0.73 &       0.74 &       0.74 \\\\\n",
      "          & NCG acc. &     0.69 &   0.70 &       0.75 &       0.73 &       0.74 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{CIFAR100-wo9} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.70 &   0.72 &       0.72 &       0.73 &       0.73 \\\\\n",
      "          & NCG acc. &     0.66 &   0.64 &       0.72 &       0.71 &       0.71 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{ImgNet100-wo0} & train acc. &     0.99 &   0.53 &       0.33 &       0.98 &       0.98 \\\\\n",
      "          & test acc. &     0.22 &   0.27 &       0.26 &       0.26 &       0.26 \\\\\n",
      "          & NCG acc. &     0.11 &   0.12 &       0.15 &       0.12 &       0.13 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{ImgNet100-wo1} & train acc. &     1.00 &   0.55 &       0.32 &       0.98 &       0.98 \\\\\n",
      "          & test acc. &     0.22 &   0.27 &       0.27 &       0.26 &       0.25 \\\\\n",
      "          & NCG acc. &     0.13 &   0.15 &       0.18 &       0.14 &       0.15 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{ImgNet100-wo2} & train acc. &     1.00 &   0.57 &       0.33 &       0.98 &       0.98 \\\\\n",
      "          & test acc. &     0.22 &   0.27 &       0.26 &       0.26 &       0.26 \\\\\n",
      "          & NCG acc. &     0.11 &   0.14 &       0.15 &       0.14 &       0.14 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo0} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   0.99 &       0.99 &       0.99 &       0.99 \\\\\n",
      "          & NCG acc. &     0.28 &   0.35 &       0.39 &       0.49 &       0.55 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo1} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   0.99 &       0.99 &       0.99 &       0.99 \\\\\n",
      "          & NCG acc. &     0.13 &   0.25 &       0.26 &       0.51 &       0.50 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo2} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   0.99 &       0.99 &       0.99 &       1.00 \\\\\n",
      "          & NCG acc. &     0.41 &   0.54 &       0.53 &       0.59 &       0.62 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo3} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   0.99 &       0.99 &       1.00 &       0.99 \\\\\n",
      "          & NCG acc. &     0.68 &   0.58 &       0.73 &       0.73 &       0.74 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo4} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   0.99 &       0.99 &       1.00 &       1.00 \\\\\n",
      "          & NCG acc. &     0.78 &   0.81 &       0.77 &       0.81 &       0.86 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo5} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   0.99 &       1.00 &       1.00 &       0.99 \\\\\n",
      "          & NCG acc. &     0.61 &   0.64 &       0.65 &       0.68 &       0.69 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo6} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & NCG acc. &     0.54 &   0.57 &       0.60 &       0.65 &       0.66 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo7} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & NCG acc. &     0.53 &   0.55 &       0.61 &       0.68 &       0.67 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo8} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   0.99 &       0.99 &       1.00 &       0.99 \\\\\n",
      "          & NCG acc. &     0.46 &   0.56 &       0.51 &       0.56 &       0.59 \\\\\n",
      "\\cline{1-7}\n",
      "\\multirow{3}{*}{MNIST-wo9} & train acc. &     1.00 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & test acc. &     0.99 &   1.00 &       1.00 &       1.00 &       1.00 \\\\\n",
      "          & NCG acc. &     0.61 &   0.77 &       0.71 &       0.74 &       0.80 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ddff = pd.DataFrame.from_dict(all_data, orient=\"index\", columns=['train acc.', 'test acc.', 'NCG acc.'])\n",
    "ddff.index = pd.MultiIndex.from_tuples(ddff.index)\n",
    "ddff = ddff.stack(0).unstack(1)[accepted_models]\n",
    "text = ddff.to_latex(float_format=\"%.2f\", multirow=True)\n",
    "text = text.replace(\"DBS6(\\$\\\\textbackslash lambda\\$=1)\", \"ball\")\n",
    "text = text.replace(\"llrrrrr\", \"llccccc\")\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### t-test (rob vs. nat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('MNIST', 'mixup'): 9,\n",
       " ('CIFAR10', 'mixup'): 0,\n",
       " ('CIFAR100', 'mixup'): 2,\n",
       " ('ImgNet100', 'mixup'): 3,\n",
       " ('MNIST', 'TRADES(2)'): 9,\n",
       " ('CIFAR10', 'TRADES(2)'): 2,\n",
       " ('CIFAR100', 'TRADES(2)'): 3,\n",
       " ('ImgNet100', 'TRADES(2)'): 3,\n",
       " ('MNIST', 'TRADES(4)'): 10,\n",
       " ('CIFAR10', 'TRADES(4)'): 3,\n",
       " ('CIFAR100', 'TRADES(4)'): 3,\n",
       " ('ImgNet100', 'TRADES(4)'): 3,\n",
       " ('MNIST', 'TRADES(8)'): 10,\n",
       " ('CIFAR10', 'TRADES(8)'): 3,\n",
       " ('CIFAR100', 'TRADES(8)'): 3,\n",
       " ('ImgNet100', 'TRADES(8)'): 3}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = {}\n",
    "for model_name in accepted_models[1:]:\n",
    "    tt = ddff['natural'] - ddff[model_name]\n",
    "    for ds_name in ds_names:\n",
    "        key = (ds_name.split('-')[0], model_name)\n",
    "        results[key] = results.setdefault(key, 0) + (tt.loc[(ds_name, 'NCG acc.')] < 0)\n",
    "#pd.DataFrame.from_dict(results)\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### small NCG table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lrrrr}\n",
      "\\toprule\n",
      "{} &  M-0 &  M-9 &  C10-0 &  C100-0 \\\\\n",
      "\\midrule\n",
      "natural   &       .28 &       .61 &         .80 &          .63 \\\\\n",
      "mixup     &       .35 &       .77 &          NaN &           NaN \\\\\n",
      "TRADES(2) &       .39 &       .71 &         .81 &          .69 \\\\\n",
      "TRADES(4) &       .49 &       .74 &         .83 &          .68 \\\\\n",
      "TRADES(8) &       .55 &       .80 &         .83 &          .68 \\\\\n",
      "AT(2)     &       .32 &       .71 &         .39 &          .09 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#small_df = ddff.loc[[(\"MNIST-wo0\", \"NCG acc.\"), (\"MNIST-wo9\", \"NCG acc.\"), (\"CIFAR10-wo0\", \"NCG acc.\"), (\"CIFAR100-wo0\", \"NCG acc.\"), (\"ImgNet100-wo0\", \"NCG acc.\")]]\n",
    "small_df = ddff.loc[[(\"MNIST-wo0\", \"NCG acc.\"), (\"MNIST-wo9\", \"NCG acc.\"), (\"CIFAR10-wo0\", \"NCG acc.\"), (\"CIFAR100-wo0\", \"NCG acc.\")]]\n",
    "small_df.index = small_df.index.droplevel(1)\n",
    "text = small_df.transpose().to_latex(float_format=\"%.2f\")\n",
    "text = text.replace(\"MNIST-wo\", \"M-\")\n",
    "text = text.replace(\"CIFAR10-wo\", \"C10-\")\n",
    "text = text.replace(\"CIFAR100-wo\", \"C100-\")\n",
    "text = text.replace(\"ImgNet100-wo\", \"I-\")\n",
    "text = text.replace(\"0.\", \".\")\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"natural\":{\"MNIST-wo0\":0.2761118354,\"MNIST-wo9\":0.6632653061,\"CIFAR10-wo0\":0.7951666667,\"CIFAR100-wo0\":0.6326666667,\"ImgNet100-wo0\":0.0992592593},\"AT(2)\":{\"MNIST-wo0\":0.3221787629,\"MNIST-wo9\":0.7009198045,\"CIFAR10-wo0\":0.3866666667,\"CIFAR100-wo0\":0.0903333333,\"ImgNet100-wo0\":0.0155555556},\"TRADES(2)\":{\"MNIST-wo0\":0.3905548312,\"MNIST-wo9\":0.7053751078,\"CIFAR10-wo0\":0.8111666667,\"CIFAR100-wo0\":0.6923333333,\"ImgNet100-wo0\":0.122962963},\"TRADES(4)\":{\"MNIST-wo0\":0.4482109228,\"MNIST-wo9\":0.7298074159,\"CIFAR10-wo0\":0.8318333333,\"CIFAR100-wo0\":0.6786666667,\"ImgNet100-wo0\":0.1377777778},\"TRADES(8)\":{\"MNIST-wo0\":0.5769955092,\"MNIST-wo9\":0.7786720322,\"CIFAR10-wo0\":0.8258333333,\"CIFAR100-wo0\":0.6843333333,\"ImgNet100-wo0\":0.1214814815}}'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "small_df.to_json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## in-distribution NCG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lolip.models.torch_utils import archs\n",
    "from lolip.variables import auto_var\n",
    "\n",
    "def get_preds(model, dset, batch_size=64, device=\"cuda\"):\n",
    "    model.eval().to(device)\n",
    "    if isinstance(dset, np.ndarray):\n",
    "        dset = torch.utils.data.TensorDataset(\n",
    "            torch.from_numpy(dset).float(),\n",
    "            torch.ones(len(dset))\n",
    "        )\n",
    "    loader = torch.utils.data.DataLoader(dset, batch_size=batch_size, shuffle=False, num_workers=24)\n",
    "\n",
    "    ret = []\n",
    "    for (x, _) in tqdm(loader, desc=\"[get_preds]\"):\n",
    "        with torch.no_grad():\n",
    "            output = F.softmax(model(x.to(device)), dim=1)\n",
    "        ret.append(output.cpu().numpy())\n",
    "    del loader\n",
    "    return np.concatenate(ret, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "inncg_acc = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b766c476429e4645a2e1f1c50f3a0aa1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60678f6d7c4048b7812d466c8db2cb65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/141 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d00da1992fb0483ead05c569e2c31957",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/141 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85d0029add4440e1acb5ef810fc846a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/141 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e99dd3465e94ef6b7ea18403b11aa05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/141 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6d4cf20daaa41b19a0774cab530eb3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/141 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "955de73e98f645a9985d47a314870d6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/141 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5c6ecf22b5d4644935fb1cc2052b9e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/149 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fe6e5a2922447a9960e38ea2bfe4f1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/149 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33e919ea5fed4b3eb6f4e73b12a5c4b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/149 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4e2bd2f29444bccbf1fc2f50f8bb426",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/149 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83d35549d79a49dd8931fb9642d00d2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/149 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39beaa49415c4891a72df8bd37bee5fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "[get_preds]:   0%|          | 0/149 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "model_paths = []\n",
    "#model_paths += [(\n",
    "#    (f\"MNIST-wo{i}\", \"TRADES(2)\"),\n",
    "#    f\"calcedrepr-mnistwo{i}-cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl\",\n",
    "#    f\"../models/out_of_sample/mnist_bak/pgd-256-calcedrepr-mnistwo{i}-pgd-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt\",\n",
    "#) for i in range(10)]\n",
    "#model_paths += [(\n",
    "#    (f\"MNIST-wo{i}\", \"natural\"),\n",
    "#    f\"calcedrepr-mnistwo{i}-cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl\",\n",
    "#    f\"../models/out_of_sample/mnist_bak/pgd-256-calcedrepr-mnistwo{i}-pgd-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl-70-2.0-0.01-trades6ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt\",\n",
    "#) for i in range(10)]\n",
    "model_paths += [(\n",
    "    (f\"CIFAR10-wo{i}\", \"natural\"),\n",
    "    f\"calcedrepr-cifar10wo{i}-cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\",\n",
    "    f'../models/out_of_sample/pgd-128-calcedrepr-cifar10wo{i}-pgd-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt',\n",
    ") for i in [0, 4, 9]]\n",
    "model_paths += [(\n",
    "    (f\"CIFAR10-wo{i}\", \"TRADES(2)\"),\n",
    "    f\"calcedrepr-cifar10wo{i}-cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\",\n",
    "    f'../models/out_of_sample/pgd-128-calcedrepr-cifar10wo{i}-pgd-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-trades6ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt',\n",
    ") for i in [0, 4, 9]]\n",
    "model_paths += [(\n",
    "    (f\"CIFAR100-wo{i}\", \"natural\"),\n",
    "    f\"calcedrepr-cifar100coarsewo{i}-cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\",\n",
    "    f'../models/out_of_sample/pgd-128-calcedrepr-cifar100coarsewo{i}-pgd-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt',\n",
    ") for i in [0, 4, 9]]\n",
    "model_paths += [(\n",
    "    (f\"CIFAR100-wo{i}\", \"TRADES(2)\"),\n",
    "    f\"calcedrepr-cifar100coarsewo{i}-cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\",\n",
    "    f'../models/out_of_sample/pgd-128-calcedrepr-cifar100coarsewo{i}-pgd-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-trades6ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt',\n",
    ") for i in [0, 4, 9]]\n",
    "model_paths += [(\n",
    "    (f\"ImgNet100-wo{i}\", \"natural\"),\n",
    "    f\"calcedrepr-aug10-imgnet100wo{i}-cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl\",\n",
    "    f'../models/out_of_sample/pgd-128-calcedrepr-cifar100coarsewo{i}-pgd-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-1.0-0.01-ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt',\n",
    ") for i in [0, 1, 2]]\n",
    "model_paths += [(\n",
    "    (f\"ImgNet100-wo{i}\", \"TRADES(2)\"),\n",
    "    f\"calcedrepr-aug10-imgnet100wo{i}-cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl\",\n",
    "    f'../models/out_of_sample/pgd-128-calcedrepr-cifar100coarsewo{i}-pgd-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl-70-2.0-0.01-trades6ce-vtor2-LargeMLP-0.9-2-sgd-0-0.0-ep0070.pt',\n",
    ") for i in [0, 1, 2]]\n",
    "\n",
    "\n",
    "\n",
    "for key, dataset, model_path in tqdm(model_paths):\n",
    "    if key in inncg_acc:\n",
    "        continue\n",
    "    _, trny, tstX, tsty, _ = auto_var.get_var_with_argument(\"dataset\", dataset)\n",
    "    arch_name = model_path.split(\"-\")[-7]\n",
    "    model = getattr(archs, arch_name)(n_classes=len(np.unique(tsty)), n_features=(tstX.shape[1], ))\n",
    "    model.load_state_dict(torch.load(model_path)['model_state_dict'])\n",
    "\n",
    "    pred = get_preds(model, tstX).argmax(1)\n",
    "\n",
    "    res = joblib.load(f\"./out_of_sample/tst_k5/{dataset}.pkl\")\n",
    "    nn_pred = trny[res['l2_ind'][:, 0].reshape(-1)]\n",
    "    inncg_acc[key] = [(pred == tsty).mean(), (pred == nn_pred).mean()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{('CIFAR10-wo0', 'natural'): [0.8945555555555555, 0.9523333333333334],\n",
       " ('CIFAR10-wo4', 'natural'): [0.8803333333333333, 0.949],\n",
       " ('CIFAR10-wo9', 'natural'): [0.8801111111111111, 0.9542222222222222],\n",
       " ('CIFAR10-wo0', 'TRADES(2)'): [0.8913333333333333, 0.9563333333333334],\n",
       " ('CIFAR10-wo4', 'TRADES(2)'): [0.8782222222222222, 0.9521111111111111],\n",
       " ('CIFAR10-wo9', 'TRADES(2)'): [0.8803333333333333, 0.9552222222222222],\n",
       " ('CIFAR100-wo0', 'natural'): [0.7176842105263158, 0.8208421052631579],\n",
       " ('CIFAR100-wo4', 'natural'): [0.7246315789473684, 0.818],\n",
       " ('CIFAR100-wo9', 'natural'): [0.7031578947368421, 0.8088421052631579],\n",
       " ('CIFAR100-wo0', 'TRADES(2)'): [0.7327368421052631, 0.8517894736842105],\n",
       " ('CIFAR100-wo4', 'TRADES(2)'): [0.7318947368421053, 0.8483157894736842],\n",
       " ('CIFAR100-wo9', 'TRADES(2)'): [0.7152631578947368, 0.8431578947368421]}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inncg_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = joblib.load(\"../results/oos_repr/cwl2-128-aug10-imgnet100-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5266"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[\"tst_acc\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "534638b011344cb8bc4a0244e3805cda",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[0.9876940133037694, 0.9894655134176092, 0.9903236569903237]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datasets = []\n",
    "datasets += [f'calcedrepr-mnistwo{i}-cwl2-128-mnistwo{i}-70-1.0-0.01-ce-vtor2-CNN002-0.9-2-sgd-0-0.0.pkl' for i in [0, 4, 9]]\n",
    "#datasets += [f\"calcedrepr-cifar10wo{i}-cwl2-64-cifar10wo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in [0, 4, 9]]\n",
    "#datasets += [f\"calcedrepr-cifar100coarsewo{i}-cwl2-64-cifar100coarsewo{i}-70-1.0-0.01-ce-vtor2-WRN_40_10-0.0-2-adam-0-0.0.pkl\" for i in [0, 4, 9]]\n",
    "#datasets += [f'calcedrepr-aug10-imgnet100wo{i}-cwl2-128-aug10-imgnet100wo{i}-70-1.0-0.01-ce-vtor2-ResNet50Norm01-0.0-2-adam-0-0.0.pkl' for i in [0, 1, 2]]\n",
    "\n",
    "nn_accs = []\n",
    "for dataset in tqdm(datasets):\n",
    "    _, trny, tstX, tsty, _ = auto_var.get_var_with_argument(\"dataset\", dataset)\n",
    "\n",
    "    res = joblib.load(f\"./out_of_sample/tst_k5/{dataset}.pkl\")\n",
    "    nn_pred = trny[res['l2_ind'][:, 0].reshape(-1)]\n",
    "    nn_accs.append((nn_pred == tsty).mean())\n",
    "nn_accs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([24, 19, 36, ..., 61, 25,  0])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  0,  0, ..., 98, 98, 98])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tsty"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}