evaluate_attacks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n"
     ]
    }
   ],
   "source": [
    "import os,random\n",
    "import csv\n",
    "os.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'\n",
    "import numpy as np\n",
    "from copy import deepcopy\n",
    "from tensorflow.keras.models import load_model\n",
    "import tensorflow.keras.models as models\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Input, AlphaDropout, GlobalAveragePooling1D, BatchNormalization, add\n",
    "from tensorflow.python.keras.layers.core import Flatten, Dense, Dropout, Reshape, Activation,Lambda,Permute\n",
    "from tensorflow.python.keras.layers.convolutional import Conv1D,Conv2D, AveragePooling1D, ZeroPadding2D, Convolution2D, MaxPooling1D, MaxPooling2D\n",
    "from tensorflow.keras.regularizers import *\n",
    "from tensorflow.keras import layers\n",
    "import matplotlib\n",
    "matplotlib.use('Agg')\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle,time\n",
    "import random, sys\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import backend as K\n",
    "import tensorflow as tf\n",
    "mirrored_strategy = tf.distribute.MirroredStrategy()\n",
    "from art.attacks.evasion import CarliniL2Method, CarliniLInfMethod\n",
    "from art.estimators.classification import KerasClassifier\n",
    "tf.compat.v1.disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1260000, 2, 128)\n"
     ]
    }
   ],
   "source": [
    "# Dataset setup\n",
    "Xd = pickle.load(open('RML22.pickle','rb'))\n",
    "\n",
    "snrs,mods = map(lambda j: sorted( list( set( map( lambda x: x[j], Xd.keys() ) ) ) ), [1,0])\n",
    "\n",
    "X = []\n",
    "lbl = []\n",
    "for mod in mods:\n",
    "    for snr in snrs:\n",
    "        X.append(Xd[(mod,snr)][0:6000])\n",
    "        for i in range(6000):  lbl.append((mod,snr))\n",
    "X = np.vstack(X)\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_onehot(yin):\n",
    "    yy = list(yin)\n",
    "    yy1 = np.zeros([len(yy), max(yy)+1])\n",
    "    yy1[np.arange(len(yy)),yy] = 1\n",
    "    return yy1\n",
    "Y_snr = to_onehot(map(lambda x: mods.index(lbl[x][0]), range(X.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network input shape in variable input-shp: [2, 128]\n",
      "Network output shape in variable input-shp: 10\n",
      "(630000, 2, 128) [2, 128]\n"
     ]
    }
   ],
   "source": [
    "# Partition the dataset into training and testing datasets\n",
    "np.random.seed(2022)\n",
    "n_examples = X.shape[0]\n",
    "n_train    = int(round(n_examples * 0.5))\n",
    "train_idx  = np.random.choice(range(0,n_examples), size=n_train, replace=False)\n",
    "test_idx   = list(set(range(0,n_examples))-set(train_idx))\n",
    "X_train    = X[train_idx]\n",
    "X_test     = X[test_idx]\n",
    "\n",
    "Y_train = to_onehot(list(map(lambda x: mods.index(lbl[x][0]), train_idx)))\n",
    "Y_test = to_onehot(list(map(lambda x: mods.index(lbl[x][0]), test_idx)))\n",
    "\n",
    "input_shp = list(X_train.shape[1:])\n",
    "output_shp = Y_train.shape[1]\n",
    "print(f'Network input shape in variable input-shp: {input_shp}')\n",
    "print(f'Network output shape in variable input-shp: {output_shp}')\n",
    "print(X_train.shape, input_shp)\n",
    "classes = mods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_orig = deepcopy(X_train)\n",
    "X_test_orig = deepcopy(X_test)\n",
    "\n",
    "num_samples_list = [16,32,64]\n",
    "field_names= ['num_subsample'] + list(snrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# versions\n",
    "targetModels = [\"complex_cnn\", \"resnet\", \"cldnn\", \"holistic\", \"uniform\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for targetModel in targetModels:\n",
    "    def residual_stack(x):\n",
    "        def residual_unit1(y,_strides=1):\n",
    "            shortcut_unit=y\n",
    "            # 1x1 conv linear\n",
    "            y = layers.Conv1D(8, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                         kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "\n",
    "            y = layers.Conv1D(8, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                         kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "            # add batch normalization\n",
    "            y = layers.add([shortcut_unit,y])\n",
    "            return y\n",
    "    \n",
    "        def residual_unit2(y,_strides=1):\n",
    "            shortcut_unit=y\n",
    "            # 1x1 conv linear\n",
    "            y = layers.Conv1D(16, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                         kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "            y = layers.Conv1D(16, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                          kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "            # add batch normalization\n",
    "            y = layers.add([shortcut_unit,y])\n",
    "            return y\n",
    "    \n",
    "        def residual_unit3(y,_strides=1):\n",
    "            shortcut_unit=y\n",
    "            # 1x1 conv linear\n",
    "            y = layers.Conv1D(32, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                         kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "            y = layers.Conv1D(32, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                         kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "            # add batch normalization\n",
    "            y = layers.add([shortcut_unit,y])\n",
    "            return y\n",
    "    \n",
    "        def residual_unit4(y,_strides=1):\n",
    "            shortcut_unit=y\n",
    "            # 1x1 conv linear\n",
    "            y = layers.Conv1D(64, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                         kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "            y = layers.Conv1D(64, kernel_size=7,data_format='channels_first',strides=_strides,padding='same',activation='relu',\n",
    "                         kernel_initializer='glorot_uniform')(y)\n",
    "            y = layers.BatchNormalization()(y)\n",
    "            # add batch normalization\n",
    "            y = layers.add([shortcut_unit,y])\n",
    "            return y\n",
    "    \n",
    "        x = layers.Conv1D(8, data_format='channels_first',kernel_size=1, padding='same',activation='relu',\n",
    "                      kernel_initializer='glorot_uniform')(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "\n",
    "        x = residual_unit1(x)\n",
    "        x = layers.Conv1D(16, data_format='channels_first',kernel_size=1, padding='same',activation='relu',\n",
    "                     kernel_initializer='glorot_uniform')(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = residual_unit2(x)\n",
    "        x = layers.Conv1D(32, data_format='channels_first',kernel_size=1, padding='same',activation='relu',\n",
    "                     kernel_initializer='glorot_uniform')(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = residual_unit3(x)\n",
    "        x = layers.Conv1D(64, data_format='channels_first',kernel_size=1, padding='same',activation='relu',\n",
    "                     kernel_initializer='glorot_uniform')(x)\n",
    "        x = layers.BatchNormalization()(x)\n",
    "        x = residual_unit4(x)\n",
    "        x = layers.AveragePooling1D(data_format='channels_first',pool_size=in_shp[1])(x)\n",
    "        return x\n",
    "\n",
    "    acc_list = []\n",
    "    acc_attack_list = []\n",
    "\n",
    "    for num_samples in num_samples_list:\n",
    "\n",
    "        X_train = deepcopy(X_train_orig)\n",
    "        X_test =  deepcopy(X_test_orig)\n",
    "\n",
    "        X_train_sub = np.zeros((X_train.shape[0], X_train.shape[1], num_samples))\n",
    "        X_test_sub = np.zeros((X_test.shape[0], X_test.shape[1], num_samples))\n",
    "\n",
    "\n",
    "        X_train = X_train_sub\n",
    "        X_test = X_test_sub\n",
    "        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
    "\n",
    "        print(X_train.shape)\n",
    "        print(Y_train.shape)\n",
    "\n",
    "        in_shp = list(X_train.shape[1:])\n",
    "        print(X_train.shape, in_shp, snrs)\n",
    "        classes = mods\n",
    "\n",
    "        inputs=layers.Input(shape=in_shp)\n",
    "        x = residual_stack(inputs)\n",
    "        x = Flatten()(x)\n",
    "        x = Dense(256,kernel_initializer=\"he_normal\", activation=\"selu\", name=\"dense1\")(x)\n",
    "        x = AlphaDropout(0.5)(x)\n",
    "        x = Dense(len(classes),kernel_initializer=\"he_normal\", activation=\"softmax\", name=\"dense3\")(x)\n",
    "        model = models.Model(inputs,x)\n",
    "        model.compile(loss=keras.losses.categorical_crossentropy,optimizer=keras.optimizers.Adam(lr=0.001),metrics=['accuracy'])\n",
    "        model.summary()\n",
    "\n",
    "\n",
    "        nb_epoch = 200     # number of epochs to train on\n",
    "        batch_size = 512  # training batch size\n",
    "\n",
    "        filepath = './data/model/subsample_' + targetModel + '_' + str(num_samples) + '.wts.h5'\n",
    "        model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "                    optimizer=keras.optimizers.Adam(lr=0.001),\n",
    "                    metrics=['accuracy'])\n",
    "\n",
    "        model.load_weights(filepath)\n",
    "\n",
    "        SNR_vec =  np.arange(-20, 21, 2)\n",
    "        X_train = deepcopy(X_train_orig)\n",
    "        X_test =  deepcopy(X_test_orig)\n",
    "\n",
    "        acc_attack_cnn = {}\n",
    "        acc_attack_cldnn = {}\n",
    "        acc_attack_holistic = {}\n",
    "        acc_attack_resnet = {}\n",
    "        acc_attack_uniform = {}\n",
    "\n",
    "        acc_attack_cnn['num_subsample'] = 'complex_cnn'\n",
    "        acc_attack_cldnn['num_subsample'] = 'cldnn'\n",
    "        acc_attack_holistic['num_subsample'] = 'holistic'\n",
    "        acc_attack_resnet['num_subsample'] = 'resnet'\n",
    "        acc_attack_uniform['num_subsample'] = 'uniform'\n",
    "\n",
    "        startIdx = 0\n",
    "        for i, SNR in enumerate(snrs):\n",
    "            #==============================================================================\n",
    "            print('SNR = ', SNR)\n",
    "            # # Here we load the DATA for modulation classification\n",
    "    ###############################################################################################################\n",
    "            with open('./data/resnet_index/indexList_' + str(SNR)+ '.data', 'rb') as fp:\n",
    "                resnet_list = pickle.load(fp)\n",
    "            with open('./data/complex_cnn_index/indexList_' + str(SNR)+ '.data', 'rb') as fp:\n",
    "                cnn_list = pickle.load(fp)\n",
    "            with open('./data/cldnn_index/indexList_' + str(SNR)+ '.data', 'rb') as fp:\n",
    "                cldnn_list = pickle.load(fp)\n",
    "\n",
    "            holistic_list = []\n",
    "            snr_idx_uniform = list(range(1, 128, int(128/num_samples)))\n",
    "\n",
    "            cldnn_list.sort(key=lambda x: x[1])\n",
    "            cldnn_list = cldnn_list[:num_samples]\n",
    "            snr_idx_cldnn = [ele[0] for ele in cldnn_list]\n",
    "            snr_idx_cldnn.sort()\n",
    "            cldnn_dict = {k: v for k, v in cldnn_list}\n",
    "\n",
    "            cnn_list.sort(key=lambda x: x[1])\n",
    "            cnn_list = cnn_list[:num_samples]\n",
    "            snr_idx_cnn = [ele[0] for ele in cnn_list]\n",
    "            snr_idx_cnn.sort()\n",
    "            cnn_dict = {k: v for k, v in cnn_list}\n",
    "\n",
    "            resnet_list.sort(key=lambda x: x[1])\n",
    "            resnet_list = resnet_list[:num_samples]\n",
    "            snr_idx_resnet = [ele[0] for ele in resnet_list]\n",
    "            snr_idx_resnet.sort()\n",
    "            resnet_dict = {k: v for k, v in resnet_list}\n",
    "\n",
    "            cldnn_cnn_list = [(k, cldnn_dict[k] + cnn_dict[k]) for k in set(cldnn_dict).intersection(cnn_dict)]\n",
    "            cldnn_cnn_dict = {k: v for k, v in cldnn_cnn_list}\n",
    "            cnn_resnet_list = [(k, cnn_dict[k] + resnet_dict[k]) for k in set(cnn_dict).intersection(resnet_dict)]\n",
    "            cnn_resnet_dict = {k: v for k, v in cnn_resnet_list}\n",
    "            cldnn_resnet_list = [(k, cldnn_dict[k] + resnet_dict[k]) for k in set(cldnn_dict).intersection(resnet_dict)]\n",
    "            cldnn_resnet_dict = {k: v for k, v in cldnn_resnet_list}\n",
    "\n",
    "            cldnn_cnn_resnet_list = [(k, cldnn_cnn_dict[k] + resnet_dict[k]) for k in set(cldnn_cnn_dict).intersection(resnet_dict)]\n",
    "            cldnn_cnn_resnet_list.sort(key=lambda x: x[1])\n",
    "\n",
    "            tier_1_samples = cldnn_cnn_resnet_list\n",
    "            tier_1 = [ele[0] for ele in tier_1_samples]\n",
    "\n",
    "            for ele in tier_1:\n",
    "                del cldnn_cnn_dict[ele]\n",
    "                del cnn_resnet_dict[ele]\n",
    "                del cldnn_resnet_dict[ele]\n",
    "\n",
    "            cldnn_cnn_list = [(k, v) for k, v in cldnn_cnn_dict.items()]\n",
    "            cnn_resnet_list = [(k, v) for k, v in cnn_resnet_dict.items()]\n",
    "            cldnn_resnet_list = [(k, v) for k, v in cldnn_resnet_dict.items()]\n",
    "\n",
    "            tier_2_samples = cldnn_cnn_list + cnn_resnet_list + cldnn_resnet_list\n",
    "            tier_2_samples.sort(key=lambda x: x[1])\n",
    "\n",
    "            tier_2 = [ele[0] for ele in tier_2_samples]\n",
    "\n",
    "            for ele in tier_1:\n",
    "                del cldnn_dict[ele]\n",
    "                del cnn_dict[ele]\n",
    "                del resnet_dict[ele]\n",
    "\n",
    "            for ele in tier_2:\n",
    "                try: del cldnn_dict[ele]\n",
    "                except: pass\n",
    "                try: del cnn_dict[ele]\n",
    "                except: pass\n",
    "                try: del resnet_dict[ele]\n",
    "                except: pass\n",
    "\n",
    "            cldnn_list = [(k, v) for k, v in cldnn_dict.items()]\n",
    "            cnn_list = [(k, v) for k, v in cnn_dict.items()]\n",
    "            resnet_list = [(k, v) for k, v in resnet_dict.items()]\n",
    "\n",
    "            tier_3_samples = cldnn_list + cnn_list + resnet_list\n",
    "            tier_3_samples.sort(key=lambda x: x[1])\n",
    "\n",
    "            holistic_list = holistic_list + tier_1_samples\n",
    "            holistic_list = holistic_list + tier_2_samples\n",
    "            holistic_list = holistic_list + tier_3_samples\n",
    "            holistic_list = holistic_list[:num_samples]\n",
    "            holistic_list.sort(key=lambda x: x[1])\n",
    "            holistic_list.sort(key=lambda x: x[0])\n",
    "\n",
    "            snr_idx_holistic = [ele[0] for ele in holistic_list]\n",
    "            snr_idx_holistic.sort()\n",
    "    ##################################################################################\n",
    "\n",
    "            #TEST\n",
    "            test_SNRs = list(map(lambda x: lbl[x][1], test_idx))\n",
    "            test_X_i = X_test[np.where(np.array(test_SNRs)== SNR)]\n",
    "            test_Y_i = Y_test[np.where(np.array(test_SNRs)== SNR)]    \n",
    "\n",
    "            def makeBackBoxExample(x_test_adv, attacked_idx, target_idx):\n",
    "                x_test_adv_128 = deepcopy(test_X_i)\n",
    "                x_test_adv_128[:,:,attacked_idx] = x_test_adv\n",
    "                x_test_adv_64 = deepcopy(x_test_adv_128[:,:,target_idx])\n",
    "                return x_test_adv_128, x_test_adv_64\n",
    "\n",
    "            targetIDX = []\n",
    "            if targetModel == 'complex_cnn' : targetIDX = snr_idx_cnn\n",
    "            if targetModel == 'cldnn' : targetIDX = snr_idx_cldnn\n",
    "            if targetModel == 'resnet' : targetIDX = snr_idx_resnet\n",
    "            if targetModel == 'holistic' : targetIDX = snr_idx_holistic\n",
    "            if targetModel == 'uniform' : targetIDX = snr_idx_uniform\n",
    "\n",
    "            x_test_adv_cnn = np.load('./data/attacked_' + str(num_samples) + '/attacked_'+str(num_samples)+'_' + str(SNR) + 'complex_cnn.npy')\n",
    "            x_test_adv_cnn_128, x_test_adv_cnn_64 = makeBackBoxExample(x_test_adv_cnn, snr_idx_cnn, targetIDX)\n",
    "            \n",
    "            x_test_adv_cldnn = np.load('./data/attacked_' + str(num_samples) + '/attacked_'+str(num_samples)+'_' + str(SNR) + 'cldnn.npy')\n",
    "            x_test_adv_cldnn_128, x_test_adv_cldnn_64 = makeBackBoxExample(x_test_adv_cldnn, snr_idx_cldnn, targetIDX)\n",
    "\n",
    "            x_test_adv_holistic = np.load('./data/attacked_' + str(num_samples) + '/attacked_'+str(num_samples)+'_' + str(SNR) + 'holistic.npy')\n",
    "            x_test_adv_holistic_128, x_test_adv_holistic_64 = makeBackBoxExample(x_test_adv_holistic, snr_idx_holistic, targetIDX)\n",
    "\n",
    "            x_test_adv_resnet = np.load('./data/attacked_' + str(num_samples) + '/attacked_'+str(num_samples)+'_' + str(SNR) + 'resnet.npy')\n",
    "            x_test_adv_resnet_128, x_test_adv_resnet_64 = makeBackBoxExample(x_test_adv_resnet, snr_idx_resnet, targetIDX)\n",
    "\n",
    "            x_test_adv_uniform = np.load('./data/attacked_' + str(num_samples) + '/attacked_'+str(num_samples)+'_' + str(SNR) + 'uniform.npy')\n",
    "            x_test_adv_uniform_128, x_test_adv_uniform_64 = makeBackBoxExample(x_test_adv_uniform, snr_idx_uniform, targetIDX)\n",
    "\n",
    "            def makeEvaluation(x_test_adv, acc_given):\n",
    "                test_Y_i_hat = model.predict(x_test_adv, batch_size=batch_size)\n",
    "                conf = np.zeros([len(classes),len(classes)])\n",
    "                confnorm = np.zeros([len(classes),len(classes)])\n",
    "                for i in range(0,x_test_adv.shape[0]):\n",
    "                    j = list(test_Y_i[i,:]).index(1)\n",
    "                    k = int(np.argmax(test_Y_i_hat[i,:]))\n",
    "                    conf[j,k] = conf[j,k] + 1\n",
    "                for i in range(0,len(classes)):\n",
    "                    confnorm[i,:] = conf[i,:] / np.sum(conf[i,:])   \n",
    "                cor = np.sum(np.diag(conf))\n",
    "                ncor = np.sum(conf) - cor\n",
    "                print (\"Overall Accuracy: \", cor / (cor+ncor))\n",
    "                acc_given[SNR] = 1.0*cor/(cor+ncor)\n",
    "            \n",
    "            print('CNN      ', end='')\n",
    "            makeEvaluation(x_test_adv_cnn_64, acc_attack_cnn)\n",
    "            print('CLDNN    ', end='')\n",
    "            makeEvaluation(x_test_adv_cldnn_64, acc_attack_cldnn)\n",
    "            print('Resnet   ', end='')\n",
    "            makeEvaluation(x_test_adv_resnet_64, acc_attack_resnet)\n",
    "            print('Holistic ', end='')\n",
    "            makeEvaluation(x_test_adv_holistic_64, acc_attack_holistic)\n",
    "            print('Uniform  ', end='')\n",
    "            makeEvaluation(x_test_adv_uniform_64, acc_attack_uniform)\n",
    "\n",
    "        with open('./results/acc_' + str(num_samples) + '_attacked_' + targetModel + '_by_CNN.csv', 'w') as csvfile:\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "            writer.writeheader()\n",
    "            writer.writerow(acc_attack_cnn)\n",
    "\n",
    "        with open('./results/acc_' + str(num_samples) + '_attacked_' + targetModel + '_by_CLDNN.csv', 'w') as csvfile:\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "            writer.writeheader()\n",
    "            writer.writerow(acc_attack_cldnn)\n",
    "\n",
    "        with open('./results/acc_' + str(num_samples) + '_attacked_' + targetModel + '_by_ResNet.csv', 'w') as csvfile:\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "            writer.writeheader()\n",
    "            writer.writerow(acc_attack_resnet)\n",
    "\n",
    "        with open('./results/acc_' + str(num_samples) + '_attacked_' + targetModel + '_by_HOLISTIC.csv', 'w') as csvfile:\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "            writer.writeheader()\n",
    "            writer.writerow(acc_attack_holistic)\n",
    "\n",
    "        with open('./results/acc_' + str(num_samples) + '_attacked_' + targetModel + '_by_uniform.csv', 'w') as csvfile:\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "            writer.writeheader()\n",
    "            writer.writerow(acc_attack_uniform)\n",
    "\n",
    "        with open('./results/acc_' + str(num_samples) + '_attacked_' + targetModel + '_by_all.csv', 'w') as csvfile:\n",
    "            writer = csv.DictWriter(csvfile, fieldnames=field_names)\n",
    "            writer.writeheader()\n",
    "            writer.writerow(acc_attack_cnn)\n",
    "            writer.writerow(acc_attack_cldnn)\n",
    "            writer.writerow(acc_attack_resnet)\n",
    "            writer.writerow(acc_attack_holistic)\n",
    "            writer.writerow(acc_attack_uniform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-gpu",
   "language": "python",
   "name": "tf-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}