aggregate_result.ipynb

{
 "cells": [
  {
   "source": [],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.stats import ks_2samp, ttest_ind\n",
    "import pickle\n",
    "\n",
    "metrics = ['correctness', 'confidence', 'entropy']\n",
    "klist = [0, 10, 20, 30, 40, 50]\n",
    "\n",
    "mixup = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_member_ratio(datadf, thre_cnt=1, skip=[], mode='ks'):\n",
    "    sample2mem = []\n",
    "    for idx, row in datadf.iterrows():\n",
    "        sample2mem.append(np.sum([row[m] for m in (set(metrics) - set(skip))]))\n",
    "    \n",
    "    members_bool = np.asarray(sample2mem) >= thre_cnt\n",
    "\n",
    "    if mode == 'ks':\n",
    "        _, pvalue = ks_2samp(members_bool,[1 for i in range(10000)] , mode='asymp')\n",
    "    elif mode == 't':\n",
    "        if members_bool.mean() == 1:\n",
    "            return 1, 1\n",
    "        else:\n",
    "            _, pvalue = ttest_ind(members_bool,[1 for i in range(len(members_bool))], equal_var = True, nan_policy='raise')\n",
    "\n",
    "    return np.sum(members_bool) / len(datadf), pvalue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "-----  Query_size:2000 \t Cal data:MNIST  -----\n",
      "      M1 M2 M3 M4 M5   M6 SVHN\n",
      "k=100  1  1  1  1  1  0.0  0.0\n",
      "k=90   1  1  1  1  1  0.0  0.0\n",
      "k=80   1  1  1  1  1  0.0  0.0\n",
      "k=70   1  1  1  1  1  0.0  0.0\n",
      "k=60   1  1  1  1  1  0.0  0.0\n",
      "k=50   1  1  1  1  1  0.0  0.0 \n",
      "\n",
      "-----  Query_size:500 \t Cal data:MNIST  -----\n",
      "      M1 M2 M3 M4 M5   M6 SVHN\n",
      "k=100  1  1  1  1  1  0.0  0.0\n",
      "k=90   1  1  1  1  1  0.0  0.0\n",
      "k=80   1  1  1  1  1  0.0  0.0\n",
      "k=70   1  1  1  1  1  0.0  0.0\n",
      "k=60   1  1  1  1  1  0.0  0.0\n",
      "k=50   1  1  1  1  1  0.0  0.0 \n",
      "\n",
      "-----  Query_size:200 \t Cal data:MNIST  -----\n",
      "      M1 M2 M3 M4 M5    M6 SVHN\n",
      "k=100  1  1  1  1  1  0.01  0.0\n",
      "k=90   1  1  1  1  1   0.0  0.0\n",
      "k=80   1  1  1  1  1  0.01  0.0\n",
      "k=70   1  1  1  1  1  0.08  0.0\n",
      "k=60   1  1  1  1  1  0.32  0.0\n",
      "k=50   1  1  1  1  1  0.01  0.0 \n",
      "\n",
      "-----  Query_size:50 \t Cal data:MNIST  -----\n",
      "      M1 M2 M3 M4 M5    M6 SVHN\n",
      "k=100  1  1  1  1  1  0.16  0.0\n",
      "k=90   1  1  1  1  1  0.16  0.0\n",
      "k=80   1  1  1  1  1  0.32  0.0\n",
      "k=70   1  1  1  1  1     1  0.0\n",
      "k=60   1  1  1  1  1     1  0.0\n",
      "k=50   1  1  1  1  1  0.32  0.0 \n",
      "\n",
      "-----  Query_size:20 \t Cal data:MNIST  -----\n",
      "      M1 M2 M3 M4 M5    M6 SVHN\n",
      "k=100  1  1  1  1  1  0.32  0.0\n",
      "k=90   1  1  1  1  1  0.32  0.0\n",
      "k=80   1  1  1  1  1     1  0.0\n",
      "k=70   1  1  1  1  1     1  0.0\n",
      "k=60   1  1  1  1  1     1  0.0\n",
      "k=50   1  1  1  1  1     1  0.0 \n",
      "\n",
      "-----  Query_size:5 \t Cal data:MNIST  -----\n",
      "      M1 M2 M3 M4 M5    M6 SVHN\n",
      "k=100  1  1  1  1  1  0.35  0.0\n",
      "k=90   1  1  1  1  1     1  0.0\n",
      "k=80   1  1  1  1  1     1  0.0\n",
      "k=70   1  1  1  1  1     1  0.0\n",
      "k=60   1  1  1  1  1     1  0.0\n",
      "k=50   1  1  1  1  1     1  0.0 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Run EMA on the benchmark datasets, Table 2.b and Table 6\n",
    "querydata = 'MNIST'\n",
    "epoch = 50\n",
    "EMA_res_table = pd.DataFrame(columns=['M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'SVHN'], index=[f'k={100-k}' for k in klist])\n",
    "for caldata in ['MNIST']:\n",
    "    for size in [2000, 500, 200, 50, 20, 5]:\n",
    "        for mode in ['t']:\n",
    "            print(f'-----  Query_size:{size} \\t Cal data:{caldata}  -----')\n",
    "            for k in klist:\n",
    "                logname = f'caldata={caldata}_epoch={epoch}_k={k}_calsize=10000'\n",
    "                for fold in range(7):\n",
    "                    queryset = pd.read_csv(f'./saves_new/EMA_{querydata}/query_set/binarized_{logname}_fold{fold}.csv')  \n",
    "\n",
    "                    if fold == 0 or fold == 6:\n",
    "                        ground_truth = 'Query is not in base'\n",
    "                    else:\n",
    "                        ground_truth = 'Query is in base'\n",
    "                    t, pv = get_member_ratio(queryset[:size], skip=['modified entropy'], mode=mode)\n",
    "\n",
    "                    EMA_res = np.around(pv, decimals=2)\n",
    "                    if fold == 0:\n",
    "                        EMA_res_table['SVHN'][f'k={100-k}'] = EMA_res\n",
    "                    else:\n",
    "                        EMA_res_table[f'M{fold}'][f'k={100-k}'] = EMA_res\n",
    "\n",
    "            print(EMA_res_table, '\\n')\n",
    "            EMA_res_table = pd.DataFrame(columns=['M1', 'M2', 'M3', 'M4', 'M5', 'M6', 'SVHN'], index=[f'k={100-k}' for k in klist])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "-----  Query_size:800 \t Cal data:COVIDx  -----\n",
      "Using t-test\n",
      "      C1 C2 C3 C4 C5   C6  CXR\n",
      "k=100  1  1  1  1  1  0.0  0.0\n",
      "k=90   1  1  1  1  1  0.0  0.0\n",
      "k=80   1  1  1  1  1  0.0  0.0\n",
      "k=70   1  1  1  1  1  0.0  0.0\n",
      "k=60   1  1  1  1  1  0.0  0.0\n",
      "k=50   1  1  1  1  1  0.0  0.0 \n",
      "\n",
      "-----  Query_size:500 \t Cal data:COVIDx  -----\n",
      "Using t-test\n",
      "      C1 C2 C3 C4 C5   C6  CXR\n",
      "k=100  1  1  1  1  1  0.0  0.0\n",
      "k=90   1  1  1  1  1  0.0  0.0\n",
      "k=80   1  1  1  1  1  0.0  0.0\n",
      "k=70   1  1  1  1  1  0.0  0.0\n",
      "k=60   1  1  1  1  1  0.0  0.0\n",
      "k=50   1  1  1  1  1  0.0  0.0 \n",
      "\n",
      "-----  Query_size:200 \t Cal data:COVIDx  -----\n",
      "Using t-test\n",
      "      C1 C2 C3 C4 C5   C6  CXR\n",
      "k=100  1  1  1  1  1  0.0  0.0\n",
      "k=90   1  1  1  1  1  0.0  0.0\n",
      "k=80   1  1  1  1  1  0.0  0.0\n",
      "k=70   1  1  1  1  1  0.0  0.0\n",
      "k=60   1  1  1  1  1  0.0  0.0\n",
      "k=50   1  1  1  1  1  0.0  0.0 \n",
      "\n",
      "-----  Query_size:50 \t Cal data:COVIDx  -----\n",
      "Using t-test\n",
      "      C1 C2 C3 C4 C5    C6   CXR\n",
      "k=100  1  1  1  1  1  0.01  0.04\n",
      "k=90   1  1  1  1  1   0.0   0.0\n",
      "k=80   1  1  1  1  1  0.01   0.0\n",
      "k=70   1  1  1  1  1  0.01   0.0\n",
      "k=60   1  1  1  1  1  0.01   0.0\n",
      "k=50   1  1  1  1  1   0.0   0.0 \n",
      "\n",
      "-----  Query_size:20 \t Cal data:COVIDx  -----\n",
      "Using t-test\n",
      "      C1 C2 C3 C4 C5    C6   CXR\n",
      "k=100  1  1  1  1  1  0.15  0.07\n",
      "k=90   1  1  1  1  1  0.02  0.04\n",
      "k=80   1  1  1  1  1  0.07  0.01\n",
      "k=70   1  1  1  1  1  0.32  0.04\n",
      "k=60   1  1  1  1  1  0.04  0.04\n",
      "k=50   1  1  1  1  1  0.04  0.04 \n",
      "\n",
      "-----  Query_size:5 \t Cal data:COVIDx  -----\n",
      "Using t-test\n",
      "      C1 C2 C3 C4 C5    C6   CXR\n",
      "k=100  1  1  1  1  1     1     1\n",
      "k=90   1  1  1  1  1  0.35  0.14\n",
      "k=80   1  1  1  1  1     1  0.14\n",
      "k=70   1  1  1  1  1     1  0.35\n",
      "k=60   1  1  1  1  1  0.14     1\n",
      "k=50   1  1  1  1  1  0.14  0.35 \n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Run EMA on chest X-ray datasets, Table 3.b and Table 7\n",
    "querydata = 'COVIDx'\n",
    "epoch = 30\n",
    "\n",
    "EMA_res_table = pd.DataFrame(columns=['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'CXR'], index=[f'k={100-k}' for k in klist])\n",
    "for caldata in ['COVIDx']:\n",
    "    for size in [800, 500, 200, 50, 20, 5]:\n",
    "        print(f'-----  Query_size:{size} \\t Cal data:{caldata}  -----')\n",
    "        for mode in ['t']:\n",
    "            print(f'Using {mode}-test')\n",
    "            for k in klist:\n",
    "                logname = f'caldata={caldata}_epoch={epoch}_k={k}_calsize=4000'\n",
    "                for fold in range(7):\n",
    "                    queryset = pd.read_csv(f'./saves_new/EMA_{querydata}/query_set/binarized_{logname}_fold{fold}.csv')  \n",
    "\n",
    "                    if fold == 0 or fold == 6:\n",
    "                        ground_truth = 'Query is not in base'\n",
    "                    else:\n",
    "                        ground_truth = 'Query is in base'\n",
    "                    t, pv = get_member_ratio(queryset[:size], skip=['modified entropy'], mode=mode)\n",
    "\n",
    "                    EMA_res = np.around(pv, decimals=2)\n",
    "                    if fold == 0:\n",
    "                        EMA_res_table['CXR'][f'k={100-k}'] = EMA_res\n",
    "                    else:\n",
    "                        EMA_res_table[f'C{fold}'][f'k={100-k}'] = EMA_res\n",
    "\n",
    "            print(EMA_res_table, '\\n')\n",
    "            EMA_res_table = pd.DataFrame(columns=['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'CXR'], index=[f'k={100-k}' for k in klist])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.9 64-bit ('audit': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "interpreter": {
   "hash": "7857d3388344c3796d2292e683a2eba131089bb20b038b40443d31522808b21c"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}