notebooks/poisoning_defense_deep_partition_aggregation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fc895cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "import os, sys\n",
    "from os.path import abspath\n",
    "\n",
    "module_path = os.path.abspath(os.path.join('..'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import keras.backend as k\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Activation, Dropout\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.compat.v1.disable_eager_execution()\n",
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from art.estimators.classification import KerasClassifier\n",
    "from art.attacks.poisoning import PoisoningAttackBackdoor, PoisoningAttackCleanLabelBackdoor\n",
    "from art.attacks.poisoning.perturbations import add_pattern_bd\n",
    "from art.utils import load_mnist, preprocess, to_categorical\n",
    "from art.defences.trainer import AdversarialTrainerMadryPGD\n",
    "\n",
    "\n",
    "from art.estimators.classification.deep_partition_ensemble import DeepPartitionEnsemble"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1816da6f",
   "metadata": {},
   "source": [
    "# Load the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fdd77e03",
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_raw, y_raw), (x_raw_test, y_raw_test), min_, max_ = load_mnist(raw=True)\n",
    "\n",
    "# Random Selection:\n",
    "n_train = np.shape(x_raw)[0]\n",
    "num_selection = 10000\n",
    "random_selection_indices = np.random.choice(n_train, num_selection)\n",
    "x_raw = x_raw[random_selection_indices]\n",
    "y_raw = y_raw[random_selection_indices]\n",
    "\n",
    "# Poison training data\n",
    "percent_poison = .33\n",
    "x_train, y_train = preprocess(x_raw, y_raw)\n",
    "x_train = np.expand_dims(x_train, axis=3)\n",
    "\n",
    "x_test, y_test = preprocess(x_raw_test, y_raw_test)\n",
    "x_test = np.expand_dims(x_test, axis=3)\n",
    "\n",
    "# Shuffle training data\n",
    "n_train = np.shape(y_train)[0]\n",
    "shuffled_indices = np.arange(n_train)\n",
    "np.random.shuffle(shuffled_indices)\n",
    "x_train = x_train[shuffled_indices]\n",
    "y_train = y_train[shuffled_indices]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b1eb1c0",
   "metadata": {},
   "source": [
    "# Initialize the Model Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c0c7c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Keras convolutional neural network - basic architecture from Keras examples\n",
    "# Source here: https://github.com/keras-team/keras/blob/master/examples/mnist_cnn.py\n",
    "def create_model():    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=x_train.shape[1:]))\n",
    "    model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(10, activation='softmax'))\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b6b105a",
   "metadata": {},
   "source": [
    "# Set up the Model Backdoor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "43b4b127",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fd1381b56d0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAANoElEQVR4nO3df6zddX3H8dfL/sJeYFKwtZZKFdFYndTlCppuSw1DAUOKUTaajLCEWbJBAovZRliMJFscIyJh05lU6awEYSoQiHZq07gRMla5kFIKZSuyDmvveoG6tQjctvS9P+6X5QL3fO7lfL/nfA99Px/JzTnn+z7f833n2/vq99zz+X7PxxEhAEe/N7XdAID+IOxAEoQdSIKwA0kQdiCJ2f3c2FzPi2M01M9NAqm8qF/pYIx7qlqtsNs+R9JNkmZJ+kZEXFd6/jEa0pk+q84mARRsic0da12/jbc9S9JXJZ0rabmkNbaXd/t6AHqrzt/sZ0h6IiKejIiDkm6XtLqZtgA0rU7Yl0j6+aTHu6tlr2B7re0R2yOHNF5jcwDqqBP2qT4EeM25txGxLiKGI2J4jubV2ByAOuqEfbekpZMenyxpT712APRKnbA/IOk02++0PVfSRZLuaaYtAE3reugtIg7bvkLSjzQx9LY+Ih5trDMAjao1zh4RGyVtbKgXAD3E6bJAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGpN2Wx7l6QDkl6SdDgihptoCkDzaoW98rGIeKaB1wHQQ7yNB5KoG/aQ9GPbD9peO9UTbK+1PWJ75JDGa24OQLfqvo1fGRF7bC+UtMn24xFx7+QnRMQ6Sesk6XgviJrbA9ClWkf2iNhT3Y5JukvSGU00BaB5XYfd9pDt416+L+njkrY31RiAZtV5G79I0l22X36db0fEDxvpCkDjug57RDwp6fQGewHQQwy9AUkQdiAJwg4kQdiBJAg7kEQTF8Kk8OxnP9qx9o6Lnyiu+/jYomL94PicYn3JbeX6/N3Pdawd2fpYcV3kwZEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnH2G/uxPv92x9umhX5ZXPrXmxleVy7sOP9+xdtPTH6u58Teun46d0rE2dMOvFdedvfnBpttpHUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEf2bpOV4L4gzfVbfttekX33mzI61Zz5Y/j/zhB3lffzL97lYn/vB/ynWr//AnR1rZ7/5heK6P3j+2GL9k/M7Xytf1wtxsFjfMj5UrK865lDX2373Dy4r1t+z9oGuX7tNW2Kz9se+KX+hOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJczz5DQ9/bUqjVe+3j662uv3vbqo61v1q5rLztfyl/5/31q97dRUczM/uFI8X60LbRYv3Ee+8o1n99bufv25+/q/xd/EejaY/sttfbHrO9fdKyBbY32d5Z3Z7Q2zYB1DWTt/HflHTOq5ZdLWlzRJwmaXP1GMAAmzbsEXGvpH2vWrxa0obq/gZJFzTbFoCmdfsB3aKIGJWk6nZhpyfaXmt7xPbIIY13uTkAdfX80/iIWBcRwxExPEfzer05AB10G/a9thdLUnU71lxLAHqh27DfI+mS6v4lku5uph0AvTLtOLvt2zTxzeUn2d4t6QuSrpP0HduXSnpK0oW9bBJlh/97b8fa0B2da5L00jSvPfS9Z7voqBl7//Cjxfr755Z/fb+0770da8v+4cniuoeL1TemacMeEWs6lN6Y30IBJMXpskAShB1IgrADSRB2IAnCDiTBJa5ozexTlhbrX7nmK8X6HM8q1r970+90rJ04en9x3aMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJxdrTm8T9ZUqx/eF55KutHD5ano17w2POvu6ejGUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcXb01PgnP9yx9tBnbpxm7fIMQn905ZXF+pv/9afTvH4uHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2dFTT53b+XhyrMvj6Gv+8+xiff4PHy7Wo1jNZ9oju+31tsdsb5+07Frbv7C9tfo5r7dtAqhrJm/jvynpnCmW3xgRK6qfjc22BaBp04Y9Iu6VtK8PvQDooTof0F1he1v1Nv+ETk+yvdb2iO2RQxqvsTkAdXQb9q9JOlXSCkmjkm7o9MSIWBcRwxExPGeaCxsA9E5XYY+IvRHxUkQckfR1SWc02xaApnUVdtuLJz38lKTtnZ4LYDBMO85u+zZJqySdZHu3pC9IWmV7hSaGMndJuqx3LWKQvem444r1i3/rvo61/UdeLK479sV3Fevzxh8o1vFK04Y9ItZMsfjmHvQCoIc4XRZIgrADSRB2IAnCDiRB2IEkuMQVtey89v3F+vdP+vuOtdU7P11cd95GhtaaxJEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1F//v7HynWt/3e3xbrPzt8qGPtub85ubjuPI0W63h9OLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsyc3e8nbi/WrPv+Pxfo8l3+FLnr44o61t/4T16v3E0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfajnGeX/4lP//7uYv3CY58t1m89sLBYX/T5zseTI8U10bRpj+y2l9r+ie0dth+1fWW1fIHtTbZ3Vrcn9L5dAN2aydv4w5I+FxHvk/QRSZfbXi7pakmbI+I0SZurxwAG1LRhj4jRiHioun9A0g5JSyStlrShetoGSRf0qEcADXhdH9DZXibpQ5K2SFoUEaPSxH8Ikqb84832WtsjtkcOabxmuwC6NeOw2z5W0h2SroqI/TNdLyLWRcRwRAzP0bxuegTQgBmF3fYcTQT91oi4s1q81/biqr5Y0lhvWgTQhGmH3mxb0s2SdkTElyeV7pF0iaTrqtu7e9Ih6jn9vcXyXy68pdbLf/WLFxbrb3n4/lqvj+bMZJx9paSLJT1ie2u17BpNhPw7ti+V9JSk8r86gFZNG/aIuE+SO5TParYdAL3C6bJAEoQdSIKwA0kQdiAJwg4kwSWuR4FZy9/Tsbb29nqnPyxff3mxvuyWf6v1+ugfjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7EeBx/+48xf7nj9/xl8qNKWT//lg+QkRtV4f/cORHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJz9DeDF888o1jeff0OhOr/ZZtBzP9qztVj/xNtXdPW6HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IImZzM++VNK3JL1N0hFJ6yLiJtvXSvqspKerp14TERt71Whme1bOKtbfMbv7sfRbDyws1ufsL1/PztXs3SmNpXc7jj6dmZxUc1jS5yLiIdvHSXrQ9qaqdmNEfKknnQFo1EzmZx+VNFrdP2B7h6QlvW4MQLNe19/stpdJ+pCkLdWiK2xvs73e9pTfjWR7re0R2yOHNF6vWwBdm3HYbR8r6Q5JV0XEfklfk3SqpBWaOPJPeYJ2RKyLiOGIGJ6jefU7BtCVGYXd9hxNBP3WiLhTkiJib0S8FBFHJH1dUvlqDQCtmjbsti3pZkk7IuLLk5YvnvS0T0na3nx7AJoyk0/jV0q6WNIjtrdWy66RtMb2Ck2MvuySdFkP+kNNf/3s8mL9/k8sK9Zj9JEGu8mjzmWqvbrEdSafxt8nyVOUGFMH3kA4gw5IgrADSRB2IAnCDiRB2IEkCDuQhKOPU+4e7wVxps/q2/aAbLbEZu2PfVMNlXNkB7Ig7EAShB1IgrADSRB2IAnCDiRB2IEk+jrObvtpSf81adFJkp7pWwOvz6D2Nqh9SfTWrSZ7OyUi3jpVoa9hf83G7ZGIGG6tgYJB7W1Q+5LorVv96o238UAShB1Iou2wr2t5+yWD2tug9iXRW7f60lurf7MD6J+2j+wA+oSwA0m0Enbb59j+d9tP2L66jR46sb3L9iO2t9oeabmX9bbHbG+ftGyB7U22d1a3U86x11Jv19r+RbXvtto+r6Xeltr+ie0dth+1fWW1vNV9V+irL/ut73+z254l6T8knS1pt6QHJK2JiMf62kgHtndJGo6I1k/AsP3bkp6T9K2I+EC17HpJ+yLiuuo/yhMi4s8HpLdrJT3X9jTe1WxFiydPMy7pAkl/oBb3XaGv31Uf9lsbR/YzJD0REU9GxEFJt0ta3UIfAy8i7pW071WLV0vaUN3foIlflr7r0NtAiIjRiHioun9A0svTjLe67wp99UUbYV8i6eeTHu/WYM33HpJ+bPtB22vbbmYKiyJiVJr45ZG0sOV+Xm3aabz76VXTjA/Mvutm+vO62gj7VN+PNUjjfysj4jcknSvp8urtKmZmRtN498sU04wPhG6nP6+rjbDvlrR00uOTJe1poY8pRcSe6nZM0l0avKmo9748g251O9ZyP/9vkKbxnmqacQ3Avmtz+vM2wv6ApNNsv9P2XEkXSbqnhT5ew/ZQ9cGJbA9J+rgGbyrqeyRdUt2/RNLdLfbyCoMyjXenacbV8r5rffrziOj7j6TzNPGJ/M8k/UUbPXTo612SHq5+Hm27N0m3aeJt3SFNvCO6VNKJkjZL2lndLhig3m6R9IikbZoI1uKWevtNTfxpuE3S1urnvLb3XaGvvuw3TpcFkuAMOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4v8A9RUAV2lh2uIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "backdoor = PoisoningAttackBackdoor(add_pattern_bd)\n",
    "example_target = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1])\n",
    "pdata, plabels = backdoor.poison(x_test, y=example_target)\n",
    "\n",
    "plt.imshow(pdata[0].squeeze())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ffec3ec",
   "metadata": {},
   "source": [
    "# Create the poison data\n",
    "For this example, we will select 9 as the target class. Thus, the adversary's goal is to poison the model so adding a trigger will result in the trained model misclassifying the triggered input as a 9.\n",
    "\n",
    "First, the adversary will create a proxy classifier (i.e., a classifier that is similar to the target classifier). As the clean label attack generates noise using PGD in order to encourage the trained classifier to rely on the trigger, it is important that the generated noise be transferable. Thus, adversarial training is used."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7aac5f5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b224203c8b894d19b2f4b4cddb0e3cb3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Precompute adv samples:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f2d06a600a747ac8e6c43814f0ebe9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Adversarial training epochs:   0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Poison some percentage of all non-nines to nines\n",
    "targets = to_categorical([9], 10)[0] \n",
    "\n",
    "proxy = AdversarialTrainerMadryPGD(KerasClassifier(create_model()), nb_epochs=10, eps=0.15, eps_step=0.001)\n",
    "proxy.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "46364ff1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ee1e87251cc4743b6f4e661e67e315c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6188f5aa629a47409ff277c273181157",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d62d2efe8ab4bf68eb2c429a5a9b408",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f37cb71fff34c37b296b9a804e7c50b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f85e53136d8548f9938044cc2de19a7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1dd5b9297308421eb0cc8049f3d8fea9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "315b992141574587a9bb76be8ac9a839",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bff862107f14cd29183473ccbbdcd96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73149654d8324fce8faf026ebfe0cc39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f241593d62ab4c61aadf6df288705816",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a737ca80f23448aa37901edfb8cc2a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Random Initializations:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Iterations:   0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "attack = PoisoningAttackCleanLabelBackdoor(backdoor=backdoor, proxy_classifier=proxy.get_classifier(),\n",
    "                                           target=targets, pp_poison=percent_poison, norm=2, eps=5,\n",
    "                                           eps_step=0.1, max_iter=200)\n",
    "pdata, plabels = attack.poison(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a61a524c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1005\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAASIElEQVR4nO3db2yd5XkG8OvysR07iY2dPxA3ZCGkjJZWI6yGDjFNWUs7oJWAD0zlQ5VJbOmHIrVSNQ2xD+Uj2taiatoqpQM1RR1VpRaFSWhrxlqitqzDsIwkNX+yJDQhJs4fEttJHNvH9z74MLng937Mec973uPc10+K7Jz7vOc8fu3Lr+37PM9DM4OIXP7ayh6AiDSHwi4ShMIuEoTCLhKEwi4SRHszn6yzsty6O3qb+ZQioVycHsNU9QIXquUKO8k7AHwLQAXAP5nZo979uzt6cevGbXmeUkQcL7y5M7NW94/xJCsA/gHAnQBuAHA/yRvqfTwRKVae39lvAXDQzA6Z2RSAHwC4uzHDEpFGyxP29QCOzvv/sdptv4XkdpJDJIemqhdzPJ2I5JEn7Av9EeB9r701sx1mNmhmg52V7hxPJyJ55An7MQAb5v3/agDH8w1HRIqSJ+wvAriO5CaSnQC+AOCZxgxLRBqt7tabmc2QfBDAv2Gu9faEmR1o2MhEpKFy9dnN7FkAzzZoLCJSIL1cViQIhV0kCIVdJAiFXSQIhV0kCIVdJIimzme/bLXyCr1ccGqzpM5L3vM2O5vv+ALoyi4ShMIuEoTCLhKEwi4ShMIuEoTCLhKEWm+LxGr9rRRb1pF4cL/Nw8kp//HbnONTbcFUi6nAtiJnE4+dal8lxmYdzpd36uNur+R67lznraBzriu7SBAKu0gQCrtIEAq7SBAKu0gQCrtIEAq7SBDqsy+W1/NN9EVnu1a49eoKvw/fedzvs7uKnsqZUmS/uZLohTsfm3X557y6cplbtw7/Otl+2t/qjJOXnGIxnxNd2UWCUNhFglDYRYJQ2EWCUNhFglDYRYJQ2EWCUJ99sbye7fiEe+j05ivd+viGTre+9vhZt27HT2TW2lav8o/t8p8bbYnrQWLOOadnsp/b6zUDYEe+dQDgzPO/uKHHPXRsgx+NlW9X3XrHqQtuvQy5wk7yCIBxAFUAM2Y22IhBiUjjNeLK/sdmdqoBjyMiBdLv7CJB5A27AfgJyZdIbl/oDiS3kxwiOTRV9V8vLCLFyftj/G1mdpzklQB2k3zVzPbMv4OZ7QCwAwCu6FrXwpuiiVzecl3Zzex47e0ogKcB3NKIQYlI49UddpIrSPa8+z6AzwLY36iBiUhj5fkx/ioAT3Ou19kO4J/N7F8bMqpW5PV02xLzqhPGNvv1/td6/TscfjOz1FZJfD/31lYHYIk+O6em/cefye5Hs7vLPzbRR59d0e0/dV/245+5PtXD98srDo/7h5/3/z5lnYnndw+u77fhusNuZocA3Fjv8SLSXGq9iQShsIsEobCLBKGwiwShsIsEoSmui+RuizzrT3fsOnTSrbfderVbHx1c6dYHzn04uziRWNJ47LxbR2IK7NSGfrdeXZY9xbb7pSPusTY56dbRs94tn7khuzU3/rvZU28BYN3ziZbj26fduvX4y4eXQVd2kSAUdpEgFHaRIBR2kSAUdpEgFHaRIBR2kSDUZ18sZ3tgdvtTLe3MWbe+enjArZ/b5E+hvXBNX2Zt2Wl/Gmlqu2hO+0tFd476fXqey15me+b0GffYyc99wq2fuNk/L7/3qdezH/vZ691jVx71l4Lm8sTnPLUEd56trOukK7tIEAq7SBAKu0gQCrtIEAq7SBAKu0gQCrtIEOqzvyu1/a/TF7XEksas+vPde395xK1fXH2tWz/2qexP46Zdfi962fBbbt1WLnfrqaWoZ8+NZdborREA4OQW/7Ff/fN/dOt/Mvz5zNrV/+Fvs1057dct8XGX0UdP0ZVdJAiFXSQIhV0kCIVdJAiFXSQIhV0kCIVdJAj12RshtS1yYmvi6ip/XfhLfX4/es3e7DnnHaP+1sK2us+tp/Bs4vGdWttKf2319VuPuvVfTPpz7Scf+1BmbcUbB91j2eN/TlKvy7DU10TqdR0FSF7ZST5BcpTk/nm3rSK5m+Qbtbf+TgEiUrrF/Bj/XQB3vOe2hwA8Z2bXAXiu9n8RaWHJsJvZHgDvXT/obgA7a+/vBHBPY4clIo1W7x/orjKzEQCovb0y644kt5McIjk0VfX3HROR4hT+13gz22Fmg2Y22FnxJ4yISHHqDfsJkgMAUHs72rghiUgR6g37MwC21d7fBmBXY4YjIkVJ9tlJPgVgK4A1JI8B+DqARwH8kOQDAH4D4L6GjCbHnPLkse3+vG5rT+zHfXHKf3zvsRM912O397r13sN+P7n3oNPrTp2XqWm/nlj/PLWH+ux49tgu3PtJ99g71/7Mrf/9yO1uffnzw9nFZf6+88mPOzEXvxUlw25m92eUPt3gsYhIgfRyWZEgFHaRIBR2kSAUdpEgFHaRIFprimuO5Xc5PePfIVVPtOZYzW5/2dhZ99izW/2loM9f67e/uk4nlmvuzK5Xpv0tlXHBfwnz7Jg/hZWJ6buXPnez//yO/3xnk1vf9+oGt/6R6X2ZNSam1yZbayVMUc1LV3aRIBR2kSAUdpEgFHaRIBR2kSAUdpEgFHaRIFqrz57HrD8NFDP+tsmYuOCWzZzH7/OnqI7c7j83L/nfc3vf9Pvw1pl9/MTHMlcMAwBULvpja5vxX/uQ6kcvf+1UZm1mrX/eVi/zXyOw+r/8L9+2Nauzi4nXVSzFPnqKruwiQSjsIkEo7CJBKOwiQSjsIkEo7CJBKOwiQbRWnz3HUtLWkfhQupb5D93tLy184tYrMmupLZU7eyfcesrh+/yx96zNXs75kwOvu8eevrTcrR94/sNufdOuxHz389nz5U98fsA99vCej7v1za8kzmtnR2YpOV89tbbCEuzD68ouEoTCLhKEwi4ShMIuEoTCLhKEwi4ShMIuEkRr9dnzcHqqADC9yu8nT/X5ffY2Z0q5VwOA/n/xn7tyye/pjv+OP/e6erg/s7b/SPbrAwBgss//ft/mT4fHyU/0+I//mew565M3+GvWb3zS/7grx0+7dTivvbDelf6xifUROJn4pKfWVyihT5+8spN8guQoyf3zbnuE5Fsk99b+3VXsMEUkr8X8GP9dAHcscPtjZral9u/Zxg5LRBotGXYz2wPgTBPGIiIFyvMHugdJvlL7MT/zl0aS20kOkRyaqvq/o4lIceoN+7cBbAawBcAIgG9k3dHMdpjZoJkNdla663w6EcmrrrCb2Qkzq9rckqvfAXBLY4clIo1WV9hJzp+beC+A/Vn3FZHWkOyzk3wKwFYAa0geA/B1AFtJbgFgAI4A+FJDRpNjf/bUuvCV835ftPuSf/zyY9n1tnOJPdCnEj3ZNv977hW7/TnjNnkps8YVfo8f69a65fHr+9z66Y/5vfAtdwxn1la0T7nHvnDTjW69a+NGtz7Tld3Lbp/0v9Z6js649e6jY24d0/7xub7W65QMu5ndv8DNjxcwFhEpkF4uKxKEwi4ShMIuEoTCLhKEwi4SxNKa4uq1KxKtjLYJ/6W6TLVKvHpqOmOitYaKX2efP02VznRJS21NnGhZrjzkt/0urPHHduDkuszaxV/3uce2+7OOMbnanyba4Qy9+5T/OesY89uCS5Gu7CJBKOwiQSjsIkEo7CJBKOwiQSjsIkEo7CJBLK0+u7f8bmrKYN4teL3tfxN98txSffoCWbv/3Jf6/fPW92T2UtLrn/e3k0Z/4vUF1cTrG6r+awhcFf/1CcktwkuYwpqiK7tIEAq7SBAKu0gQCrtIEAq7SBAKu0gQCrtIEEurz+7JuQVuct53mVIfm9dPThw72+vv0vPW1uw+OeDPGQeA3j2H/Dt4xibcsiX66Gx3vrw7/C2+kfpySK1h0IJ0ZRcJQmEXCUJhFwlCYRcJQmEXCUJhFwlCYRcJ4vLps1/OEj1dd837xLzs0Zt73PrEtf56+lf/u9/H5zJn8XdnjQAg/zoBVuT6B0tQ8myS3EDypySHSR4g+ZXa7atI7ib5Ru1tf/HDFZF6LeZb5wyAr5nZRwH8AYAvk7wBwEMAnjOz6wA8V/u/iLSoZNjNbMTMXq69Pw5gGMB6AHcD2Fm7204A9xQ0RhFpgA/0SxHJawDcBOBXAK4ysxFg7hsCgCszjtlOcojk0FTV329NRIqz6LCTXAngRwC+amZjiz3OzHaY2aCZDXZW/EkXIlKcRYWdZAfmgv59M/tx7eYTJAdq9QEAo8UMUUQaIdl649x+wI8DGDazb84rPQNgG4BHa293FTLCpSDvMtWJ4znr171ljS9ct8Y9duxat4yeg/6XSO9/v+XWrctpvaWWyM553nK1z3JOmW5Fi+mz3wbgiwD2kdxbu+1hzIX8hyQfAPAbAPcVMkIRaYhk2M3s5wCyvs19urHDEZGi6OWyIkEo7CJBKOwiQSjsIkEo7CJBaIrrEmDdy/x6JbsnPHaN/ynuPOv3kwd+mVjOObEEt7etsqUuNQGnoRZJV3aRIBR2kSAUdpEgFHaRIBR2kSAUdpEgFHaRINRnb4TE3Gev1wwg2S+e7u1y6+98ZHlm7cI6f2xr9/rbHhcqte1xar67fCA6myJBKOwiQSjsIkEo7CJBKOwiQSjsIkEo7CJBqM/eCHnnVSf69Nbuf08+vz77+P5X/V5215kpt9424ddTrM352PKuC3+ZGv7L1W79o397uq7H1ZVdJAiFXSQIhV0kCIVdJAiFXSQIhV0kCIVdJIjF7M++AcD3AKwDMAtgh5l9i+QjAP4CwMnaXR82s2eLGmhLc/ZHBwCbScwZH/PXZj9144fcev/r2b303lfP+s+dmlJ+YdKtWyVxvfDmpAftowN+L73ePnrKYl5UMwPga2b2MskeAC+R3F2rPWZmf1fIyESkoRazP/sIgJHa++MkhwGsL3pgItJYH+h3dpLXALgJwK9qNz1I8hWST5DszzhmO8khkkNT1Yv5RisidVt02EmuBPAjAF81szEA3wawGcAWzF35v7HQcWa2w8wGzWyws9Kdf8QiUpdFhZ1kB+aC/n0z+zEAmNkJM6ua2SyA7wC4pbhhikheybCTJIDHAQyb2Tfn3T4w7273Atjf+OGJSKMs5q/xtwH4IoB9JPfWbnsYwP0ktwAwAEcAfKmA8S0NOZeSHrl3s1vvmPBbVFcceCe7eMmfospEWzC1JXMueae4puqpxy9QnmmqRU1xXcxf438OYKGzFrOnLrJE6RV0IkEo7CJBKOwiQSjsIkEo7CJBKOwiQdCaOM3wiq51duvGbU17vobyerapPvv0jP/Yic+BJabQesenevzuUs9Asb1qLSXdcC+8uRPnJt9e8MTqyi4ShMIuEoTCLhKEwi4ShMIuEoTCLhKEwi4SRFP77CRPAnhz3k1rAJxq2gA+mFYdW6uOC9DY6tXIsW00s7ULFZoa9vc9OTlkZoOlDcDRqmNr1XEBGlu9mjU2/RgvEoTCLhJE2WHfUfLze1p1bK06LkBjq1dTxlbq7+wi0jxlX9lFpEkUdpEgSgk7yTtIvkbyIMmHyhhDFpJHSO4juZfkUMljeYLkKMn9825bRXI3yTdqbxfcY6+ksT1C8q3audtL8q6SxraB5E9JDpM8QPIrtdtLPXfOuJpy3pr+OzvJCoDXAXwGwDEALwK438x+3dSBZCB5BMCgmZX+AgySfwRgAsD3zOzjtdv+BsAZM3u09o2y38z+qkXG9giAibK38a7tVjQwf5txAPcA+DOUeO6ccf0pmnDeyriy3wLgoJkdMrMpAD8AcHcJ42h5ZrYHwJn33Hw3gJ2193di7oul6TLG1hLMbMTMXq69Pw7g3W3GSz13zriaooywrwdwdN7/j6G19ns3AD8h+RLJ7WUPZgFXmdkIMPfFA+DKksfzXsltvJvpPduMt8y5q2f787zKCPtC62O1Uv/vNjP7fQB3Avhy7cdVWZxFbePdLAtsM94S6t3+PK8ywn4MwIZ5/78awPESxrEgMzteezsK4Gm03lbUJ97dQbf2drTk8fy/VtrGe6FtxtEC567M7c/LCPuLAK4juYlkJ4AvAHimhHG8D8kVtT+cgOQKAJ9F621F/QyAd5fo3QZgV4lj+S2tso131jbjKPnclb79uZk1/R+AuzD3F/n/BfDXZYwhY1zXAvif2r8DZY8NwFOY+7FuGnM/ET0AYDWA5wC8UXu7qoXG9iSAfQBewVywBkoa2x9i7lfDVwDsrf27q+xz54yrKedNL5cVCUKvoBMJQmEXCUJhFwlCYRcJQmEXCUJhFwlCYRcJ4v8AHVFzojW9/kIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index: 1 Label: 9\n"
     ]
    }
   ],
   "source": [
    "poisoned = pdata[np.all(plabels == targets, axis=1)]\n",
    "poisoned_labels = plabels[np.all(plabels == targets, axis=1)]\n",
    "print(len(poisoned))\n",
    "for i in range(len(poisoned)):\n",
    "    if poisoned[i][0][0] != 0:\n",
    "        plt.imshow(poisoned[i].squeeze())\n",
    "        plt.show()\n",
    "        print(f\"Index: {i} Label: {np.argmax(poisoned_labels[i])}\")\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e8bbe9c",
   "metadata": {},
   "source": [
    "# Initialize the classification models\n",
    "We will initialize four models. The first is a single model architecture. The other three are DPA models with varying ensemble sizes to demonstrate the tradeoff between clean accuracy and poison accuracy. This make take some time because of the model copying."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "005f0374",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = KerasClassifier(create_model())\n",
    "dpa_model_10 = DeepPartitionEnsemble(model, ensemble_size=10)\n",
    "dpa_model_20 = DeepPartitionEnsemble(model, ensemble_size=20)\n",
    "dpa_model_30 = DeepPartitionEnsemble(model, ensemble_size=30)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0668c63f",
   "metadata": {},
   "source": [
    "Train the models on the poisoned data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "80f2917b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10000 samples\n",
      "Epoch 1/10\n",
      "10000/10000 [==============================] - 1s 106us/sample - loss: 0.6702 - accuracy: 0.7852\n",
      "Epoch 2/10\n",
      "10000/10000 [==============================] - 1s 52us/sample - loss: 0.2179 - accuracy: 0.9353\n",
      "Epoch 3/10\n",
      "10000/10000 [==============================] - 1s 53us/sample - loss: 0.1370 - accuracy: 0.9567\n",
      "Epoch 4/10\n",
      "10000/10000 [==============================] - 1s 51us/sample - loss: 0.1018 - accuracy: 0.9684\n",
      "Epoch 5/10\n",
      "10000/10000 [==============================] - 1s 51us/sample - loss: 0.0778 - accuracy: 0.9766\n",
      "Epoch 6/10\n",
      "10000/10000 [==============================] - 1s 51us/sample - loss: 0.0661 - accuracy: 0.9783\n",
      "Epoch 7/10\n",
      "10000/10000 [==============================] - 1s 51us/sample - loss: 0.0558 - accuracy: 0.9822\n",
      "Epoch 8/10\n",
      "10000/10000 [==============================] - 1s 50us/sample - loss: 0.0510 - accuracy: 0.9820\n",
      "Epoch 9/10\n",
      "10000/10000 [==============================] - 1s 51us/sample - loss: 0.0435 - accuracy: 0.9862\n",
      "Epoch 10/10\n",
      "10000/10000 [==============================] - 1s 50us/sample - loss: 0.0375 - accuracy: 0.9885\n",
      "Train on 977 samples\n",
      "Epoch 1/10\n",
      "977/977 [==============================] - 7s 7ms/sample - loss: 1.9011 - accuracy: 0.3429\n",
      "Epoch 2/10\n",
      "977/977 [==============================] - 0s 53us/sample - loss: 1.0015 - accuracy: 0.6817\n",
      "Epoch 3/10\n",
      "977/977 [==============================] - 0s 52us/sample - loss: 0.6282 - accuracy: 0.7892\n",
      "Epoch 4/10\n",
      "977/977 [==============================] - 0s 51us/sample - loss: 0.4957 - accuracy: 0.8352\n",
      "Epoch 5/10\n",
      "977/977 [==============================] - 0s 52us/sample - loss: 0.3679 - accuracy: 0.8731\n",
      "Epoch 6/10\n",
      "977/977 [==============================] - 0s 51us/sample - loss: 0.3122 - accuracy: 0.8864\n",
      "Epoch 7/10\n",
      "977/977 [==============================] - 0s 54us/sample - loss: 0.2489 - accuracy: 0.9161\n",
      "Epoch 8/10\n",
      "977/977 [==============================] - 0s 56us/sample - loss: 0.2032 - accuracy: 0.9355\n",
      "Epoch 9/10\n",
      "977/977 [==============================] - 0s 54us/sample - loss: 0.1624 - accuracy: 0.9427\n",
      "Epoch 10/10\n",
      "977/977 [==============================] - 0s 52us/sample - loss: 0.1518 - accuracy: 0.9509\n",
      "Train on 992 samples\n",
      "Epoch 1/10\n",
      "992/992 [==============================] - 7s 7ms/sample - loss: 1.9655 - accuracy: 0.3276\n",
      "Epoch 2/10\n",
      "992/992 [==============================] - 0s 54us/sample - loss: 0.9691 - accuracy: 0.6905\n",
      "Epoch 3/10\n",
      "992/992 [==============================] - 0s 52us/sample - loss: 0.6556 - accuracy: 0.7671\n",
      "Epoch 4/10\n",
      "992/992 [==============================] - 0s 52us/sample - loss: 0.4811 - accuracy: 0.8367\n",
      "Epoch 5/10\n",
      "992/992 [==============================] - 0s 52us/sample - loss: 0.3704 - accuracy: 0.8810\n",
      "Epoch 6/10\n",
      "992/992 [==============================] - 0s 52us/sample - loss: 0.3440 - accuracy: 0.8901\n",
      "Epoch 7/10\n",
      "992/992 [==============================] - 0s 52us/sample - loss: 0.2679 - accuracy: 0.9173\n",
      "Epoch 8/10\n",
      "992/992 [==============================] - 0s 53us/sample - loss: 0.2276 - accuracy: 0.9284\n",
      "Epoch 9/10\n",
      "992/992 [==============================] - 0s 52us/sample - loss: 0.1983 - accuracy: 0.9294\n",
      "Epoch 10/10\n",
      "992/992 [==============================] - 0s 52us/sample - loss: 0.1449 - accuracy: 0.9607\n",
      "Train on 1005 samples\n",
      "Epoch 1/10\n",
      "1005/1005 [==============================] - 7s 7ms/sample - loss: 1.8687 - accuracy: 0.4070\n",
      "Epoch 2/10\n",
      "1005/1005 [==============================] - 0s 59us/sample - loss: 0.9425 - accuracy: 0.7045\n",
      "Epoch 3/10\n",
      "1005/1005 [==============================] - 0s 51us/sample - loss: 0.6594 - accuracy: 0.7930\n",
      "Epoch 4/10\n",
      "1005/1005 [==============================] - 0s 60us/sample - loss: 0.5096 - accuracy: 0.8378\n",
      "Epoch 5/10\n",
      "1005/1005 [==============================] - 0s 52us/sample - loss: 0.4274 - accuracy: 0.8527\n",
      "Epoch 6/10\n",
      "1005/1005 [==============================] - 0s 52us/sample - loss: 0.3317 - accuracy: 0.8866\n",
      "Epoch 7/10\n",
      "1005/1005 [==============================] - 0s 51us/sample - loss: 0.3122 - accuracy: 0.9045\n",
      "Epoch 8/10\n",
      "1005/1005 [==============================] - 0s 53us/sample - loss: 0.2434 - accuracy: 0.9244\n",
      "Epoch 9/10\n",
      "1005/1005 [==============================] - 0s 52us/sample - loss: 0.1971 - accuracy: 0.9343\n",
      "Epoch 10/10\n",
      "1005/1005 [==============================] - 0s 52us/sample - loss: 0.1557 - accuracy: 0.9483\n",
      "Train on 1087 samples\n",
      "Epoch 1/10\n",
      "1087/1087 [==============================] - 7s 6ms/sample - loss: 1.8404 - accuracy: 0.3818\n",
      "Epoch 2/10\n",
      "1087/1087 [==============================] - 0s 55us/sample - loss: 0.9119 - accuracy: 0.7001\n",
      "Epoch 3/10\n",
      "1087/1087 [==============================] - 0s 52us/sample - loss: 0.5826 - accuracy: 0.8086\n",
      "Epoch 4/10\n",
      "1087/1087 [==============================] - 0s 55us/sample - loss: 0.4819 - accuracy: 0.8638\n",
      "Epoch 5/10\n",
      "1087/1087 [==============================] - 0s 54us/sample - loss: 0.3799 - accuracy: 0.8832\n",
      "Epoch 6/10\n",
      "1087/1087 [==============================] - 0s 53us/sample - loss: 0.2969 - accuracy: 0.9006\n",
      "Epoch 7/10\n",
      "1087/1087 [==============================] - 0s 51us/sample - loss: 0.2559 - accuracy: 0.9190\n",
      "Epoch 8/10\n",
      "1087/1087 [==============================] - 0s 53us/sample - loss: 0.2213 - accuracy: 0.9255\n",
      "Epoch 9/10\n",
      "1087/1087 [==============================] - 0s 52us/sample - loss: 0.1901 - accuracy: 0.9476\n",
      "Epoch 10/10\n",
      "1087/1087 [==============================] - 0s 52us/sample - loss: 0.1290 - accuracy: 0.9586\n",
      "Train on 1000 samples\n",
      "Epoch 1/10\n",
      "1000/1000 [==============================] - 7s 7ms/sample - loss: 1.8514 - accuracy: 0.3930\n",
      "Epoch 2/10\n",
      "1000/1000 [==============================] - 0s 54us/sample - loss: 0.9959 - accuracy: 0.6640\n",
      "Epoch 3/10\n",
      "1000/1000 [==============================] - 0s 53us/sample - loss: 0.6432 - accuracy: 0.7970\n",
      "Epoch 4/10\n",
      "1000/1000 [==============================] - 0s 66us/sample - loss: 0.5205 - accuracy: 0.8440\n",
      "Epoch 5/10\n",
      "1000/1000 [==============================] - 0s 54us/sample - loss: 0.4049 - accuracy: 0.8600\n",
      "Epoch 6/10\n",
      "1000/1000 [==============================] - 0s 52us/sample - loss: 0.3443 - accuracy: 0.8750\n",
      "Epoch 7/10\n",
      "1000/1000 [==============================] - 0s 53us/sample - loss: 0.2732 - accuracy: 0.9150\n",
      "Epoch 8/10\n",
      "1000/1000 [==============================] - 0s 55us/sample - loss: 0.2527 - accuracy: 0.9230\n",
      "Epoch 9/10\n",
      "1000/1000 [==============================] - 0s 51us/sample - loss: 0.2045 - accuracy: 0.9360\n",
      "Epoch 10/10\n",
      "1000/1000 [==============================] - 0s 62us/sample - loss: 0.1627 - accuracy: 0.9570\n",
      "Train on 976 samples\n",
      "Epoch 1/10\n",
      "976/976 [==============================] - 7s 7ms/sample - loss: 1.8847 - accuracy: 0.3945\n",
      "Epoch 2/10\n",
      "976/976 [==============================] - 0s 54us/sample - loss: 0.9502 - accuracy: 0.6957\n",
      "Epoch 3/10\n",
      "976/976 [==============================] - 0s 53us/sample - loss: 0.6427 - accuracy: 0.7859\n",
      "Epoch 4/10\n",
      "976/976 [==============================] - 0s 59us/sample - loss: 0.4635 - accuracy: 0.8545\n",
      "Epoch 5/10\n",
      "976/976 [==============================] - 0s 53us/sample - loss: 0.3932 - accuracy: 0.8842\n",
      "Epoch 6/10\n",
      "976/976 [==============================] - 0s 52us/sample - loss: 0.3295 - accuracy: 0.8975\n",
      "Epoch 7/10\n",
      "976/976 [==============================] - 0s 54us/sample - loss: 0.2788 - accuracy: 0.9109\n",
      "Epoch 8/10\n",
      "976/976 [==============================] - 0s 53us/sample - loss: 0.1914 - accuracy: 0.9457\n",
      "Epoch 9/10\n",
      "976/976 [==============================] - 0s 52us/sample - loss: 0.1786 - accuracy: 0.9508\n",
      "Epoch 10/10\n",
      "976/976 [==============================] - 0s 54us/sample - loss: 0.1445 - accuracy: 0.9559\n",
      "Train on 964 samples\n",
      "Epoch 1/10\n",
      "964/964 [==============================] - 7s 7ms/sample - loss: 1.9007 - accuracy: 0.3444\n",
      "Epoch 2/10\n",
      "964/964 [==============================] - 0s 55us/sample - loss: 0.9441 - accuracy: 0.6836\n",
      "Epoch 3/10\n",
      "964/964 [==============================] - 0s 53us/sample - loss: 0.6456 - accuracy: 0.7801\n",
      "Epoch 4/10\n",
      "964/964 [==============================] - 0s 53us/sample - loss: 0.5009 - accuracy: 0.8330\n",
      "Epoch 5/10\n",
      "964/964 [==============================] - 0s 53us/sample - loss: 0.4053 - accuracy: 0.8859\n",
      "Epoch 6/10\n",
      "964/964 [==============================] - 0s 56us/sample - loss: 0.3883 - accuracy: 0.8900\n",
      "Epoch 7/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "964/964 [==============================] - 0s 53us/sample - loss: 0.2921 - accuracy: 0.9077\n",
      "Epoch 8/10\n",
      "964/964 [==============================] - 0s 52us/sample - loss: 0.2208 - accuracy: 0.9315\n",
      "Epoch 9/10\n",
      "964/964 [==============================] - 0s 53us/sample - loss: 0.2089 - accuracy: 0.9305\n",
      "Epoch 10/10\n",
      "964/964 [==============================] - 0s 52us/sample - loss: 0.1801 - accuracy: 0.9492\n",
      "Train on 1014 samples\n",
      "Epoch 1/10\n",
      "1014/1014 [==============================] - 7s 7ms/sample - loss: 1.8226 - accuracy: 0.4162\n",
      "Epoch 2/10\n",
      "1014/1014 [==============================] - 0s 56us/sample - loss: 0.9429 - accuracy: 0.6874\n",
      "Epoch 3/10\n",
      "1014/1014 [==============================] - 0s 52us/sample - loss: 0.6000 - accuracy: 0.8116\n",
      "Epoch 4/10\n",
      "1014/1014 [==============================] - 0s 50us/sample - loss: 0.4649 - accuracy: 0.8462\n",
      "Epoch 5/10\n",
      "1014/1014 [==============================] - 0s 50us/sample - loss: 0.3840 - accuracy: 0.8738\n",
      "Epoch 6/10\n",
      "1014/1014 [==============================] - 0s 52us/sample - loss: 0.3062 - accuracy: 0.9004\n",
      "Epoch 7/10\n",
      "1014/1014 [==============================] - 0s 50us/sample - loss: 0.2574 - accuracy: 0.9181\n",
      "Epoch 8/10\n",
      "1014/1014 [==============================] - 0s 51us/sample - loss: 0.2208 - accuracy: 0.9320\n",
      "Epoch 9/10\n",
      "1014/1014 [==============================] - 0s 51us/sample - loss: 0.1951 - accuracy: 0.9310\n",
      "Epoch 10/10\n",
      "1014/1014 [==============================] - 0s 52us/sample - loss: 0.1333 - accuracy: 0.9596\n",
      "Train on 1029 samples\n",
      "Epoch 1/10\n",
      "1029/1029 [==============================] - 7s 7ms/sample - loss: 1.8513 - accuracy: 0.3965\n",
      "Epoch 2/10\n",
      "1029/1029 [==============================] - 0s 57us/sample - loss: 0.9767 - accuracy: 0.6812\n",
      "Epoch 3/10\n",
      "1029/1029 [==============================] - 0s 55us/sample - loss: 0.6929 - accuracy: 0.7697\n",
      "Epoch 4/10\n",
      "1029/1029 [==============================] - 0s 55us/sample - loss: 0.5566 - accuracy: 0.8241\n",
      "Epoch 5/10\n",
      "1029/1029 [==============================] - 0s 53us/sample - loss: 0.4355 - accuracy: 0.8591\n",
      "Epoch 6/10\n",
      "1029/1029 [==============================] - 0s 54us/sample - loss: 0.3800 - accuracy: 0.8785\n",
      "Epoch 7/10\n",
      "1029/1029 [==============================] - 0s 54us/sample - loss: 0.3067 - accuracy: 0.9116\n",
      "Epoch 8/10\n",
      "1029/1029 [==============================] - 0s 53us/sample - loss: 0.2842 - accuracy: 0.9116\n",
      "Epoch 9/10\n",
      "1029/1029 [==============================] - 0s 54us/sample - loss: 0.2307 - accuracy: 0.9329\n",
      "Epoch 10/10\n",
      "1029/1029 [==============================] - 0s 56us/sample - loss: 0.2048 - accuracy: 0.9320\n",
      "Train on 956 samples\n",
      "Epoch 1/10\n",
      "956/956 [==============================] - 7s 8ms/sample - loss: 1.8804 - accuracy: 0.3954\n",
      "Epoch 2/10\n",
      "956/956 [==============================] - 0s 54us/sample - loss: 0.9341 - accuracy: 0.6799\n",
      "Epoch 3/10\n",
      "956/956 [==============================] - 0s 52us/sample - loss: 0.6392 - accuracy: 0.7887\n",
      "Epoch 4/10\n",
      "956/956 [==============================] - 0s 52us/sample - loss: 0.5189 - accuracy: 0.8274\n",
      "Epoch 5/10\n",
      "956/956 [==============================] - 0s 53us/sample - loss: 0.4177 - accuracy: 0.8797\n",
      "Epoch 6/10\n",
      "956/956 [==============================] - 0s 53us/sample - loss: 0.3175 - accuracy: 0.9006\n",
      "Epoch 7/10\n",
      "956/956 [==============================] - 0s 53us/sample - loss: 0.2653 - accuracy: 0.9226\n",
      "Epoch 8/10\n",
      "956/956 [==============================] - 0s 53us/sample - loss: 0.2252 - accuracy: 0.9289\n",
      "Epoch 9/10\n",
      "956/956 [==============================] - 0s 52us/sample - loss: 0.1760 - accuracy: 0.9498\n",
      "Epoch 10/10\n",
      "956/956 [==============================] - 0s 52us/sample - loss: 0.1257 - accuracy: 0.9676\n",
      "Train on 473 samples\n",
      "Epoch 1/10\n",
      "473/473 [==============================] - 7s 16ms/sample - loss: 2.1809 - accuracy: 0.1649\n",
      "Epoch 2/10\n",
      "473/473 [==============================] - 0s 58us/sample - loss: 1.5440 - accuracy: 0.5814\n",
      "Epoch 3/10\n",
      "473/473 [==============================] - 0s 54us/sample - loss: 1.0021 - accuracy: 0.6892\n",
      "Epoch 4/10\n",
      "473/473 [==============================] - 0s 56us/sample - loss: 0.7427 - accuracy: 0.7696\n",
      "Epoch 5/10\n",
      "473/473 [==============================] - 0s 53us/sample - loss: 0.5753 - accuracy: 0.8245\n",
      "Epoch 6/10\n",
      "473/473 [==============================] - 0s 55us/sample - loss: 0.4381 - accuracy: 0.8605\n",
      "Epoch 7/10\n",
      "473/473 [==============================] - 0s 54us/sample - loss: 0.3848 - accuracy: 0.8689\n",
      "Epoch 8/10\n",
      "473/473 [==============================] - 0s 54us/sample - loss: 0.3077 - accuracy: 0.9049\n",
      "Epoch 9/10\n",
      "473/473 [==============================] - 0s 54us/sample - loss: 0.2377 - accuracy: 0.9260\n",
      "Epoch 10/10\n",
      "473/473 [==============================] - 0s 52us/sample - loss: 0.2671 - accuracy: 0.9006\n",
      "Train on 493 samples\n",
      "Epoch 1/10\n",
      "493/493 [==============================] - 1s 1ms/sample - loss: 2.1469 - accuracy: 0.2272\n",
      "Epoch 2/10\n",
      "493/493 [==============================] - 0s 55us/sample - loss: 1.4709 - accuracy: 0.5943\n",
      "Epoch 3/10\n",
      "493/493 [==============================] - 0s 53us/sample - loss: 0.9855 - accuracy: 0.6876\n",
      "Epoch 4/10\n",
      "493/493 [==============================] - 0s 54us/sample - loss: 0.7909 - accuracy: 0.7465\n",
      "Epoch 5/10\n",
      "493/493 [==============================] - 0s 54us/sample - loss: 0.6112 - accuracy: 0.8073\n",
      "Epoch 6/10\n",
      "493/493 [==============================] - 0s 53us/sample - loss: 0.5063 - accuracy: 0.8499\n",
      "Epoch 7/10\n",
      "493/493 [==============================] - 0s 54us/sample - loss: 0.3887 - accuracy: 0.8905\n",
      "Epoch 8/10\n",
      "493/493 [==============================] - 0s 57us/sample - loss: 0.3509 - accuracy: 0.8884\n",
      "Epoch 9/10\n",
      "493/493 [==============================] - 0s 53us/sample - loss: 0.3062 - accuracy: 0.9087\n",
      "Epoch 10/10\n",
      "493/493 [==============================] - 0s 53us/sample - loss: 0.2519 - accuracy: 0.9310\n",
      "Train on 511 samples\n",
      "Epoch 1/10\n",
      "511/511 [==============================] - 8s 15ms/sample - loss: 2.1587 - accuracy: 0.2524\n",
      "Epoch 2/10\n",
      "511/511 [==============================] - 0s 56us/sample - loss: 1.5182 - accuracy: 0.5910\n",
      "Epoch 3/10\n",
      "511/511 [==============================] - 0s 56us/sample - loss: 0.9522 - accuracy: 0.7025\n",
      "Epoch 4/10\n",
      "511/511 [==============================] - 0s 53us/sample - loss: 0.7103 - accuracy: 0.7573\n",
      "Epoch 5/10\n",
      "511/511 [==============================] - 0s 53us/sample - loss: 0.6426 - accuracy: 0.8043\n",
      "Epoch 6/10\n",
      "511/511 [==============================] - 0s 53us/sample - loss: 0.5414 - accuracy: 0.8376\n",
      "Epoch 7/10\n",
      "511/511 [==============================] - 0s 53us/sample - loss: 0.4553 - accuracy: 0.8415\n",
      "Epoch 8/10\n",
      "511/511 [==============================] - 0s 54us/sample - loss: 0.4236 - accuracy: 0.8650\n",
      "Epoch 9/10\n",
      "511/511 [==============================] - 0s 54us/sample - loss: 0.3126 - accuracy: 0.8904\n",
      "Epoch 10/10\n",
      "511/511 [==============================] - 0s 60us/sample - loss: 0.3113 - accuracy: 0.8982\n",
      "Train on 562 samples\n",
      "Epoch 1/10\n",
      "562/562 [==============================] - 8s 14ms/sample - loss: 2.1335 - accuracy: 0.3007\n",
      "Epoch 2/10\n",
      "562/562 [==============================] - 0s 59us/sample - loss: 1.3796 - accuracy: 0.6103\n",
      "Epoch 3/10\n",
      "562/562 [==============================] - 0s 56us/sample - loss: 0.9748 - accuracy: 0.6922\n",
      "Epoch 4/10\n",
      "562/562 [==============================] - 0s 57us/sample - loss: 0.7408 - accuracy: 0.7562\n",
      "Epoch 5/10\n",
      "562/562 [==============================] - 0s 57us/sample - loss: 0.5678 - accuracy: 0.8114\n",
      "Epoch 6/10\n",
      "562/562 [==============================] - 0s 55us/sample - loss: 0.4847 - accuracy: 0.8470\n",
      "Epoch 7/10\n",
      "562/562 [==============================] - 0s 55us/sample - loss: 0.4089 - accuracy: 0.8683\n",
      "Epoch 8/10\n",
      "562/562 [==============================] - 0s 57us/sample - loss: 0.3535 - accuracy: 0.8843\n",
      "Epoch 9/10\n",
      "562/562 [==============================] - 0s 57us/sample - loss: 0.2846 - accuracy: 0.9075\n",
      "Epoch 10/10\n",
      "562/562 [==============================] - 0s 57us/sample - loss: 0.2524 - accuracy: 0.9181\n",
      "Train on 510 samples\n",
      "Epoch 1/10\n",
      "510/510 [==============================] - 8s 15ms/sample - loss: 2.1443 - accuracy: 0.2451\n",
      "Epoch 2/10\n",
      "510/510 [==============================] - 0s 55us/sample - loss: 1.4927 - accuracy: 0.5804\n",
      "Epoch 3/10\n",
      "510/510 [==============================] - 0s 53us/sample - loss: 1.0005 - accuracy: 0.6765\n",
      "Epoch 4/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "510/510 [==============================] - 0s 52us/sample - loss: 0.7577 - accuracy: 0.7373\n",
      "Epoch 5/10\n",
      "510/510 [==============================] - 0s 52us/sample - loss: 0.5855 - accuracy: 0.8275\n",
      "Epoch 6/10\n",
      "510/510 [==============================] - 0s 53us/sample - loss: 0.4825 - accuracy: 0.8373\n",
      "Epoch 7/10\n",
      "510/510 [==============================] - 0s 58us/sample - loss: 0.4021 - accuracy: 0.8765\n",
      "Epoch 8/10\n",
      "510/510 [==============================] - 0s 53us/sample - loss: 0.3385 - accuracy: 0.8784\n",
      "Epoch 9/10\n",
      "510/510 [==============================] - 0s 52us/sample - loss: 0.2920 - accuracy: 0.9118\n",
      "Epoch 10/10\n",
      "510/510 [==============================] - 0s 52us/sample - loss: 0.2465 - accuracy: 0.9275\n",
      "Train on 540 samples\n",
      "Epoch 1/10\n",
      "540/540 [==============================] - 8s 14ms/sample - loss: 2.1399 - accuracy: 0.2907\n",
      "Epoch 2/10\n",
      "540/540 [==============================] - 0s 61us/sample - loss: 1.3855 - accuracy: 0.6037\n",
      "Epoch 3/10\n",
      "540/540 [==============================] - 0s 57us/sample - loss: 0.8968 - accuracy: 0.6889\n",
      "Epoch 4/10\n",
      "540/540 [==============================] - 0s 59us/sample - loss: 0.7184 - accuracy: 0.7667\n",
      "Epoch 5/10\n",
      "540/540 [==============================] - 0s 65us/sample - loss: 0.5614 - accuracy: 0.8278\n",
      "Epoch 6/10\n",
      "540/540 [==============================] - 0s 57us/sample - loss: 0.5385 - accuracy: 0.8093\n",
      "Epoch 7/10\n",
      "540/540 [==============================] - 0s 68us/sample - loss: 0.3907 - accuracy: 0.8667\n",
      "Epoch 8/10\n",
      "540/540 [==============================] - 0s 57us/sample - loss: 0.3501 - accuracy: 0.8889\n",
      "Epoch 9/10\n",
      "540/540 [==============================] - 0s 56us/sample - loss: 0.3064 - accuracy: 0.9074\n",
      "Epoch 10/10\n",
      "540/540 [==============================] - 0s 57us/sample - loss: 0.2511 - accuracy: 0.9167\n",
      "Train on 458 samples\n",
      "Epoch 1/10\n",
      "458/458 [==============================] - 8s 17ms/sample - loss: 2.1754 - accuracy: 0.2183\n",
      "Epoch 2/10\n",
      "458/458 [==============================] - 0s 59us/sample - loss: 1.5343 - accuracy: 0.5633\n",
      "Epoch 3/10\n",
      "458/458 [==============================] - 0s 56us/sample - loss: 1.0270 - accuracy: 0.6703\n",
      "Epoch 4/10\n",
      "458/458 [==============================] - 0s 57us/sample - loss: 0.8076 - accuracy: 0.7598\n",
      "Epoch 5/10\n",
      "458/458 [==============================] - 0s 57us/sample - loss: 0.6061 - accuracy: 0.8035\n",
      "Epoch 6/10\n",
      "458/458 [==============================] - 0s 55us/sample - loss: 0.5397 - accuracy: 0.8057\n",
      "Epoch 7/10\n",
      "458/458 [==============================] - 0s 55us/sample - loss: 0.4747 - accuracy: 0.8472\n",
      "Epoch 8/10\n",
      "458/458 [==============================] - 0s 55us/sample - loss: 0.4171 - accuracy: 0.8755\n",
      "Epoch 9/10\n",
      "458/458 [==============================] - 0s 55us/sample - loss: 0.3737 - accuracy: 0.8886\n",
      "Epoch 10/10\n",
      "458/458 [==============================] - 0s 55us/sample - loss: 0.3178 - accuracy: 0.9061\n",
      "Train on 504 samples\n",
      "Epoch 1/10\n",
      "504/504 [==============================] - 8s 15ms/sample - loss: 2.1662 - accuracy: 0.2361\n",
      "Epoch 2/10\n",
      "504/504 [==============================] - 0s 59us/sample - loss: 1.4684 - accuracy: 0.6468\n",
      "Epoch 3/10\n",
      "504/504 [==============================] - 0s 52us/sample - loss: 0.9344 - accuracy: 0.7103\n",
      "Epoch 4/10\n",
      "504/504 [==============================] - 0s 54us/sample - loss: 0.7055 - accuracy: 0.7778\n",
      "Epoch 5/10\n",
      "504/504 [==============================] - 0s 53us/sample - loss: 0.5763 - accuracy: 0.8175\n",
      "Epoch 6/10\n",
      "504/504 [==============================] - 0s 54us/sample - loss: 0.4725 - accuracy: 0.8552\n",
      "Epoch 7/10\n",
      "504/504 [==============================] - 0s 53us/sample - loss: 0.3873 - accuracy: 0.8968\n",
      "Epoch 8/10\n",
      "504/504 [==============================] - 0s 53us/sample - loss: 0.3294 - accuracy: 0.8929\n",
      "Epoch 9/10\n",
      "504/504 [==============================] - 0s 54us/sample - loss: 0.3092 - accuracy: 0.9067\n",
      "Epoch 10/10\n",
      "504/504 [==============================] - 0s 53us/sample - loss: 0.2785 - accuracy: 0.9008\n",
      "Train on 515 samples\n",
      "Epoch 1/10\n",
      "515/515 [==============================] - 8s 15ms/sample - loss: 2.1411 - accuracy: 0.2660\n",
      "Epoch 2/10\n",
      "515/515 [==============================] - 0s 61us/sample - loss: 1.4938 - accuracy: 0.5553\n",
      "Epoch 3/10\n",
      "515/515 [==============================] - 0s 60us/sample - loss: 1.1121 - accuracy: 0.6447\n",
      "Epoch 4/10\n",
      "515/515 [==============================] - 0s 59us/sample - loss: 0.8340 - accuracy: 0.7223\n",
      "Epoch 5/10\n",
      "515/515 [==============================] - 0s 61us/sample - loss: 0.6983 - accuracy: 0.7767\n",
      "Epoch 6/10\n",
      "515/515 [==============================] - 0s 60us/sample - loss: 0.5879 - accuracy: 0.8039\n",
      "Epoch 7/10\n",
      "515/515 [==============================] - 0s 63us/sample - loss: 0.5509 - accuracy: 0.8252\n",
      "Epoch 8/10\n",
      "515/515 [==============================] - 0s 59us/sample - loss: 0.5238 - accuracy: 0.8388\n",
      "Epoch 9/10\n",
      "515/515 [==============================] - 0s 59us/sample - loss: 0.4933 - accuracy: 0.8524\n",
      "Epoch 10/10\n",
      "515/515 [==============================] - 0s 59us/sample - loss: 0.3861 - accuracy: 0.8718\n",
      "Train on 474 samples\n",
      "Epoch 1/10\n",
      "474/474 [==============================] - 8s 17ms/sample - loss: 2.1533 - accuracy: 0.3038\n",
      "Epoch 2/10\n",
      "474/474 [==============================] - 0s 59us/sample - loss: 1.4620 - accuracy: 0.5992\n",
      "Epoch 3/10\n",
      "474/474 [==============================] - 0s 55us/sample - loss: 0.9472 - accuracy: 0.6899\n",
      "Epoch 4/10\n",
      "474/474 [==============================] - 0s 55us/sample - loss: 0.7755 - accuracy: 0.7384\n",
      "Epoch 5/10\n",
      "474/474 [==============================] - 0s 54us/sample - loss: 0.5711 - accuracy: 0.7932\n",
      "Epoch 6/10\n",
      "474/474 [==============================] - 0s 55us/sample - loss: 0.5215 - accuracy: 0.8439\n",
      "Epoch 7/10\n",
      "474/474 [==============================] - 0s 56us/sample - loss: 0.4137 - accuracy: 0.8797\n",
      "Epoch 8/10\n",
      "474/474 [==============================] - 0s 72us/sample - loss: 0.3107 - accuracy: 0.9072\n",
      "Epoch 9/10\n",
      "474/474 [==============================] - 0s 55us/sample - loss: 0.2880 - accuracy: 0.8987\n",
      "Epoch 10/10\n",
      "474/474 [==============================] - 0s 56us/sample - loss: 0.2661 - accuracy: 0.8945\n",
      "Train on 504 samples\n",
      "Epoch 1/10\n",
      "504/504 [==============================] - 1s 1ms/sample - loss: 2.1567 - accuracy: 0.2183\n",
      "Epoch 2/10\n",
      "504/504 [==============================] - 0s 54us/sample - loss: 1.5629 - accuracy: 0.5238\n",
      "Epoch 3/10\n",
      "504/504 [==============================] - 0s 64us/sample - loss: 0.9954 - accuracy: 0.6806\n",
      "Epoch 4/10\n",
      "504/504 [==============================] - 0s 53us/sample - loss: 0.7676 - accuracy: 0.7381\n",
      "Epoch 5/10\n",
      "504/504 [==============================] - 0s 54us/sample - loss: 0.6315 - accuracy: 0.7738\n",
      "Epoch 6/10\n",
      "504/504 [==============================] - 0s 54us/sample - loss: 0.4747 - accuracy: 0.8413\n",
      "Epoch 7/10\n",
      "504/504 [==============================] - 0s 53us/sample - loss: 0.4441 - accuracy: 0.8532\n",
      "Epoch 8/10\n",
      "504/504 [==============================] - 0s 54us/sample - loss: 0.3398 - accuracy: 0.8869\n",
      "Epoch 9/10\n",
      "504/504 [==============================] - 0s 55us/sample - loss: 0.3249 - accuracy: 0.8889\n",
      "Epoch 10/10\n",
      "504/504 [==============================] - 0s 55us/sample - loss: 0.2866 - accuracy: 0.9028\n",
      "Train on 499 samples\n",
      "Epoch 1/10\n",
      "499/499 [==============================] - 8s 16ms/sample - loss: 2.1409 - accuracy: 0.2766\n",
      "Epoch 2/10\n",
      "499/499 [==============================] - 0s 57us/sample - loss: 1.4840 - accuracy: 0.5671\n",
      "Epoch 3/10\n",
      "499/499 [==============================] - 0s 53us/sample - loss: 1.0183 - accuracy: 0.6713\n",
      "Epoch 4/10\n",
      "499/499 [==============================] - 0s 54us/sample - loss: 0.7745 - accuracy: 0.7415\n",
      "Epoch 5/10\n",
      "499/499 [==============================] - 0s 53us/sample - loss: 0.5848 - accuracy: 0.8176\n",
      "Epoch 6/10\n",
      "499/499 [==============================] - 0s 54us/sample - loss: 0.4338 - accuracy: 0.8838\n",
      "Epoch 7/10\n",
      "499/499 [==============================] - 0s 54us/sample - loss: 0.3783 - accuracy: 0.8878\n",
      "Epoch 8/10\n",
      "499/499 [==============================] - 0s 53us/sample - loss: 0.3259 - accuracy: 0.8918\n",
      "Epoch 9/10\n",
      "499/499 [==============================] - 0s 54us/sample - loss: 0.2636 - accuracy: 0.9218\n",
      "Epoch 10/10\n",
      "499/499 [==============================] - 0s 56us/sample - loss: 0.2292 - accuracy: 0.9259\n",
      "Train on 494 samples\n",
      "Epoch 1/10\n",
      "494/494 [==============================] - 8s 16ms/sample - loss: 2.1955 - accuracy: 0.1680\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/10\n",
      "494/494 [==============================] - 0s 58us/sample - loss: 1.5831 - accuracy: 0.5445\n",
      "Epoch 3/10\n",
      "494/494 [==============================] - 0s 55us/sample - loss: 0.9799 - accuracy: 0.6842\n",
      "Epoch 4/10\n",
      "494/494 [==============================] - 0s 55us/sample - loss: 0.7087 - accuracy: 0.7490\n",
      "Epoch 5/10\n",
      "494/494 [==============================] - 0s 57us/sample - loss: 0.5977 - accuracy: 0.8259\n",
      "Epoch 6/10\n",
      "494/494 [==============================] - 0s 54us/sample - loss: 0.4985 - accuracy: 0.8462\n",
      "Epoch 7/10\n",
      "494/494 [==============================] - 0s 55us/sample - loss: 0.4028 - accuracy: 0.8583\n",
      "Epoch 8/10\n",
      "494/494 [==============================] - 0s 55us/sample - loss: 0.3595 - accuracy: 0.8887\n",
      "Epoch 9/10\n",
      "494/494 [==============================] - 0s 55us/sample - loss: 0.3189 - accuracy: 0.8947\n",
      "Epoch 10/10\n",
      "494/494 [==============================] - 0s 56us/sample - loss: 0.2715 - accuracy: 0.9069\n",
      "Train on 525 samples\n",
      "Epoch 1/10\n",
      "525/525 [==============================] - 8s 16ms/sample - loss: 2.1822 - accuracy: 0.1981\n",
      "Epoch 2/10\n",
      "525/525 [==============================] - 0s 61us/sample - loss: 1.5962 - accuracy: 0.5410\n",
      "Epoch 3/10\n",
      "525/525 [==============================] - 0s 61us/sample - loss: 1.0687 - accuracy: 0.6552\n",
      "Epoch 4/10\n",
      "525/525 [==============================] - 0s 60us/sample - loss: 0.8098 - accuracy: 0.7333\n",
      "Epoch 5/10\n",
      "525/525 [==============================] - 0s 59us/sample - loss: 0.7363 - accuracy: 0.7695\n",
      "Epoch 6/10\n",
      "525/525 [==============================] - 0s 58us/sample - loss: 0.5887 - accuracy: 0.8190\n",
      "Epoch 7/10\n",
      "525/525 [==============================] - 0s 59us/sample - loss: 0.5066 - accuracy: 0.8419\n",
      "Epoch 8/10\n",
      "525/525 [==============================] - 0s 57us/sample - loss: 0.4558 - accuracy: 0.8667\n",
      "Epoch 9/10\n",
      "525/525 [==============================] - 0s 58us/sample - loss: 0.3741 - accuracy: 0.8933\n",
      "Epoch 10/10\n",
      "525/525 [==============================] - 0s 63us/sample - loss: 0.3409 - accuracy: 0.8914\n",
      "Train on 490 samples\n",
      "Epoch 1/10\n",
      "490/490 [==============================] - 8s 17ms/sample - loss: 2.1478 - accuracy: 0.2673\n",
      "Epoch 2/10\n",
      "490/490 [==============================] - 0s 60us/sample - loss: 1.4726 - accuracy: 0.5694\n",
      "Epoch 3/10\n",
      "490/490 [==============================] - 0s 54us/sample - loss: 1.0106 - accuracy: 0.6673\n",
      "Epoch 4/10\n",
      "490/490 [==============================] - 0s 56us/sample - loss: 0.6747 - accuracy: 0.7653\n",
      "Epoch 5/10\n",
      "490/490 [==============================] - 0s 55us/sample - loss: 0.5685 - accuracy: 0.7959\n",
      "Epoch 6/10\n",
      "490/490 [==============================] - 0s 55us/sample - loss: 0.4641 - accuracy: 0.8510\n",
      "Epoch 7/10\n",
      "490/490 [==============================] - 0s 56us/sample - loss: 0.3946 - accuracy: 0.8694\n",
      "Epoch 8/10\n",
      "490/490 [==============================] - 0s 54us/sample - loss: 0.3273 - accuracy: 0.8816\n",
      "Epoch 9/10\n",
      "490/490 [==============================] - 0s 55us/sample - loss: 0.2801 - accuracy: 0.9245\n",
      "Epoch 10/10\n",
      "490/490 [==============================] - 0s 54us/sample - loss: 0.2627 - accuracy: 0.9163\n",
      "Train on 436 samples\n",
      "Epoch 1/10\n",
      "436/436 [==============================] - 8s 19ms/sample - loss: 2.2061 - accuracy: 0.2339\n",
      "Epoch 2/10\n",
      "436/436 [==============================] - 0s 61us/sample - loss: 1.6643 - accuracy: 0.4908\n",
      "Epoch 3/10\n",
      "436/436 [==============================] - 0s 58us/sample - loss: 1.1070 - accuracy: 0.6697\n",
      "Epoch 4/10\n",
      "436/436 [==============================] - 0s 59us/sample - loss: 0.8987 - accuracy: 0.7225\n",
      "Epoch 5/10\n",
      "436/436 [==============================] - 0s 57us/sample - loss: 0.6979 - accuracy: 0.7615\n",
      "Epoch 6/10\n",
      "436/436 [==============================] - 0s 60us/sample - loss: 0.5736 - accuracy: 0.8303\n",
      "Epoch 7/10\n",
      "436/436 [==============================] - 0s 58us/sample - loss: 0.4599 - accuracy: 0.8647\n",
      "Epoch 8/10\n",
      "436/436 [==============================] - 0s 64us/sample - loss: 0.4453 - accuracy: 0.8624\n",
      "Epoch 9/10\n",
      "436/436 [==============================] - 0s 60us/sample - loss: 0.3667 - accuracy: 0.8739\n",
      "Epoch 10/10\n",
      "436/436 [==============================] - 0s 58us/sample - loss: 0.2991 - accuracy: 0.9014\n",
      "Train on 506 samples\n",
      "Epoch 1/10\n",
      "506/506 [==============================] - 8s 17ms/sample - loss: 2.1515 - accuracy: 0.2273\n",
      "Epoch 2/10\n",
      "506/506 [==============================] - 0s 58us/sample - loss: 1.4947 - accuracy: 0.5771\n",
      "Epoch 3/10\n",
      "506/506 [==============================] - 0s 56us/sample - loss: 1.0101 - accuracy: 0.6798\n",
      "Epoch 4/10\n",
      "506/506 [==============================] - 0s 55us/sample - loss: 0.7194 - accuracy: 0.7727\n",
      "Epoch 5/10\n",
      "506/506 [==============================] - 0s 56us/sample - loss: 0.4982 - accuracy: 0.8300\n",
      "Epoch 6/10\n",
      "506/506 [==============================] - 0s 54us/sample - loss: 0.4229 - accuracy: 0.8676\n",
      "Epoch 7/10\n",
      "506/506 [==============================] - 0s 54us/sample - loss: 0.4085 - accuracy: 0.8735\n",
      "Epoch 8/10\n",
      "506/506 [==============================] - 0s 69us/sample - loss: 0.3329 - accuracy: 0.8992\n",
      "Epoch 9/10\n",
      "506/506 [==============================] - 0s 53us/sample - loss: 0.2588 - accuracy: 0.9229\n",
      "Epoch 10/10\n",
      "506/506 [==============================] - 0s 54us/sample - loss: 0.2258 - accuracy: 0.9348\n",
      "Train on 510 samples\n",
      "Epoch 1/10\n",
      "510/510 [==============================] - 1s 1ms/sample - loss: 2.1504 - accuracy: 0.2706\n",
      "Epoch 2/10\n",
      "510/510 [==============================] - 0s 54us/sample - loss: 1.4914 - accuracy: 0.5765\n",
      "Epoch 3/10\n",
      "510/510 [==============================] - 0s 53us/sample - loss: 0.9789 - accuracy: 0.7078\n",
      "Epoch 4/10\n",
      "510/510 [==============================] - 0s 52us/sample - loss: 0.7227 - accuracy: 0.7510\n",
      "Epoch 5/10\n",
      "510/510 [==============================] - 0s 54us/sample - loss: 0.5537 - accuracy: 0.8157\n",
      "Epoch 6/10\n",
      "510/510 [==============================] - 0s 54us/sample - loss: 0.4117 - accuracy: 0.8863\n",
      "Epoch 7/10\n",
      "510/510 [==============================] - 0s 55us/sample - loss: 0.3780 - accuracy: 0.8706\n",
      "Epoch 8/10\n",
      "510/510 [==============================] - 0s 54us/sample - loss: 0.2962 - accuracy: 0.9000\n",
      "Epoch 9/10\n",
      "510/510 [==============================] - 0s 54us/sample - loss: 0.2469 - accuracy: 0.9176\n",
      "Epoch 10/10\n",
      "510/510 [==============================] - 0s 53us/sample - loss: 0.2382 - accuracy: 0.9118\n",
      "Train on 514 samples\n",
      "Epoch 1/10\n",
      "514/514 [==============================] - 8s 17ms/sample - loss: 2.1311 - accuracy: 0.2529\n",
      "Epoch 2/10\n",
      "514/514 [==============================] - 0s 66us/sample - loss: 1.5174 - accuracy: 0.4883\n",
      "Epoch 3/10\n",
      "514/514 [==============================] - 0s 61us/sample - loss: 1.1737 - accuracy: 0.6051\n",
      "Epoch 4/10\n",
      "514/514 [==============================] - 0s 60us/sample - loss: 0.9564 - accuracy: 0.6751\n",
      "Epoch 5/10\n",
      "514/514 [==============================] - 0s 59us/sample - loss: 0.7675 - accuracy: 0.7549\n",
      "Epoch 6/10\n",
      "514/514 [==============================] - 0s 61us/sample - loss: 0.7090 - accuracy: 0.7626\n",
      "Epoch 7/10\n",
      "514/514 [==============================] - 0s 82us/sample - loss: 0.5601 - accuracy: 0.8093\n",
      "Epoch 8/10\n",
      "514/514 [==============================] - 0s 61us/sample - loss: 0.4879 - accuracy: 0.8541\n",
      "Epoch 9/10\n",
      "514/514 [==============================] - 0s 59us/sample - loss: 0.4532 - accuracy: 0.8580\n",
      "Epoch 10/10\n",
      "514/514 [==============================] - 0s 61us/sample - loss: 0.4233 - accuracy: 0.8813\n",
      "Train on 482 samples\n",
      "Epoch 1/10\n",
      "482/482 [==============================] - 9s 18ms/sample - loss: 2.1805 - accuracy: 0.2531\n",
      "Epoch 2/10\n",
      "482/482 [==============================] - 0s 60us/sample - loss: 1.5490 - accuracy: 0.5581\n",
      "Epoch 3/10\n",
      "482/482 [==============================] - 0s 57us/sample - loss: 1.0985 - accuracy: 0.6286\n",
      "Epoch 4/10\n",
      "482/482 [==============================] - 0s 57us/sample - loss: 0.8298 - accuracy: 0.7241\n",
      "Epoch 5/10\n",
      "482/482 [==============================] - 0s 56us/sample - loss: 0.5945 - accuracy: 0.8008\n",
      "Epoch 6/10\n",
      "482/482 [==============================] - 0s 57us/sample - loss: 0.5225 - accuracy: 0.8423\n",
      "Epoch 7/10\n",
      "482/482 [==============================] - 0s 58us/sample - loss: 0.4254 - accuracy: 0.8693\n",
      "Epoch 8/10\n",
      "482/482 [==============================] - 0s 56us/sample - loss: 0.3584 - accuracy: 0.8776\n",
      "Epoch 9/10\n",
      "482/482 [==============================] - 0s 57us/sample - loss: 0.3316 - accuracy: 0.8880\n",
      "Epoch 10/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "482/482 [==============================] - 0s 55us/sample - loss: 0.2440 - accuracy: 0.9378\n",
      "Train on 338 samples\n",
      "Epoch 1/10\n",
      "338/338 [==============================] - 9s 25ms/sample - loss: 2.2088 - accuracy: 0.2189\n",
      "Epoch 2/10\n",
      "338/338 [==============================] - 0s 65us/sample - loss: 1.7714 - accuracy: 0.4438\n",
      "Epoch 3/10\n",
      "338/338 [==============================] - 0s 59us/sample - loss: 1.2951 - accuracy: 0.5947\n",
      "Epoch 4/10\n",
      "338/338 [==============================] - 0s 59us/sample - loss: 0.9404 - accuracy: 0.7160\n",
      "Epoch 5/10\n",
      "338/338 [==============================] - 0s 59us/sample - loss: 0.7767 - accuracy: 0.7456\n",
      "Epoch 6/10\n",
      "338/338 [==============================] - 0s 58us/sample - loss: 0.5287 - accuracy: 0.8314\n",
      "Epoch 7/10\n",
      "338/338 [==============================] - 0s 58us/sample - loss: 0.4792 - accuracy: 0.8284\n",
      "Epoch 8/10\n",
      "338/338 [==============================] - 0s 58us/sample - loss: 0.3464 - accuracy: 0.8846\n",
      "Epoch 9/10\n",
      "338/338 [==============================] - 0s 58us/sample - loss: 0.3205 - accuracy: 0.8905\n",
      "Epoch 10/10\n",
      "338/338 [==============================] - 0s 60us/sample - loss: 0.3031 - accuracy: 0.8817\n",
      "Train on 318 samples\n",
      "Epoch 1/10\n",
      "318/318 [==============================] - 9s 27ms/sample - loss: 2.2445 - accuracy: 0.1761\n",
      "Epoch 2/10\n",
      "318/318 [==============================] - 0s 67us/sample - loss: 1.7898 - accuracy: 0.4969\n",
      "Epoch 3/10\n",
      "318/318 [==============================] - 0s 62us/sample - loss: 1.2530 - accuracy: 0.6478\n",
      "Epoch 4/10\n",
      "318/318 [==============================] - 0s 61us/sample - loss: 1.0756 - accuracy: 0.6289\n",
      "Epoch 5/10\n",
      "318/318 [==============================] - 0s 60us/sample - loss: 0.8333 - accuracy: 0.7233\n",
      "Epoch 6/10\n",
      "318/318 [==============================] - 0s 64us/sample - loss: 0.6580 - accuracy: 0.7704\n",
      "Epoch 7/10\n",
      "318/318 [==============================] - 0s 61us/sample - loss: 0.5656 - accuracy: 0.8113\n",
      "Epoch 8/10\n",
      "318/318 [==============================] - 0s 63us/sample - loss: 0.4813 - accuracy: 0.8396\n",
      "Epoch 9/10\n",
      "318/318 [==============================] - 0s 67us/sample - loss: 0.4491 - accuracy: 0.8711\n",
      "Epoch 10/10\n",
      "318/318 [==============================] - 0s 63us/sample - loss: 0.3618 - accuracy: 0.8962\n",
      "Train on 335 samples\n",
      "Epoch 1/10\n",
      "335/335 [==============================] - 9s 26ms/sample - loss: 2.2123 - accuracy: 0.1970\n",
      "Epoch 2/10\n",
      "335/335 [==============================] - 0s 65us/sample - loss: 1.8210 - accuracy: 0.3821\n",
      "Epoch 3/10\n",
      "335/335 [==============================] - 0s 59us/sample - loss: 1.3212 - accuracy: 0.6179\n",
      "Epoch 4/10\n",
      "335/335 [==============================] - 0s 71us/sample - loss: 0.9346 - accuracy: 0.6896\n",
      "Epoch 5/10\n",
      "335/335 [==============================] - 0s 60us/sample - loss: 0.7993 - accuracy: 0.7343\n",
      "Epoch 6/10\n",
      "335/335 [==============================] - 0s 61us/sample - loss: 0.6540 - accuracy: 0.7970\n",
      "Epoch 7/10\n",
      "335/335 [==============================] - 0s 59us/sample - loss: 0.5595 - accuracy: 0.8179\n",
      "Epoch 8/10\n",
      "335/335 [==============================] - 0s 69us/sample - loss: 0.4867 - accuracy: 0.8716\n",
      "Epoch 9/10\n",
      "335/335 [==============================] - 0s 59us/sample - loss: 0.4078 - accuracy: 0.8627\n",
      "Epoch 10/10\n",
      "335/335 [==============================] - 0s 64us/sample - loss: 0.3882 - accuracy: 0.8896\n",
      "Train on 372 samples\n",
      "Epoch 1/10\n",
      "372/372 [==============================] - 9s 24ms/sample - loss: 2.2522 - accuracy: 0.1720\n",
      "Epoch 2/10\n",
      "372/372 [==============================] - 0s 60us/sample - loss: 1.8264 - accuracy: 0.5054\n",
      "Epoch 3/10\n",
      "372/372 [==============================] - 0s 56us/sample - loss: 1.3457 - accuracy: 0.6183\n",
      "Epoch 4/10\n",
      "372/372 [==============================] - 0s 56us/sample - loss: 1.0196 - accuracy: 0.6694\n",
      "Epoch 5/10\n",
      "372/372 [==============================] - 0s 56us/sample - loss: 0.7542 - accuracy: 0.7446\n",
      "Epoch 6/10\n",
      "372/372 [==============================] - 0s 56us/sample - loss: 0.6572 - accuracy: 0.7876\n",
      "Epoch 7/10\n",
      "372/372 [==============================] - 0s 56us/sample - loss: 0.4672 - accuracy: 0.8763\n",
      "Epoch 8/10\n",
      "372/372 [==============================] - 0s 85us/sample - loss: 0.4436 - accuracy: 0.8629\n",
      "Epoch 9/10\n",
      "372/372 [==============================] - 0s 57us/sample - loss: 0.3556 - accuracy: 0.9005\n",
      "Epoch 10/10\n",
      "372/372 [==============================] - 0s 57us/sample - loss: 0.3038 - accuracy: 0.9167\n",
      "Train on 342 samples\n",
      "Epoch 1/10\n",
      "342/342 [==============================] - 9s 26ms/sample - loss: 2.2220 - accuracy: 0.2368\n",
      "Epoch 2/10\n",
      "342/342 [==============================] - 0s 64us/sample - loss: 1.7808 - accuracy: 0.5146\n",
      "Epoch 3/10\n",
      "342/342 [==============================] - 0s 58us/sample - loss: 1.2565 - accuracy: 0.6433\n",
      "Epoch 4/10\n",
      "342/342 [==============================] - 0s 60us/sample - loss: 0.9451 - accuracy: 0.6842\n",
      "Epoch 5/10\n",
      "342/342 [==============================] - 0s 59us/sample - loss: 0.7607 - accuracy: 0.7339\n",
      "Epoch 6/10\n",
      "342/342 [==============================] - 0s 57us/sample - loss: 0.6507 - accuracy: 0.7953\n",
      "Epoch 7/10\n",
      "342/342 [==============================] - 0s 57us/sample - loss: 0.5249 - accuracy: 0.8216\n",
      "Epoch 8/10\n",
      "342/342 [==============================] - 0s 58us/sample - loss: 0.4956 - accuracy: 0.8216\n",
      "Epoch 9/10\n",
      "342/342 [==============================] - 0s 58us/sample - loss: 0.3944 - accuracy: 0.8509\n",
      "Epoch 10/10\n",
      "342/342 [==============================] - 0s 58us/sample - loss: 0.3334 - accuracy: 0.8918\n",
      "Train on 325 samples\n",
      "Epoch 1/10\n",
      "325/325 [==============================] - 9s 27ms/sample - loss: 2.2378 - accuracy: 0.2215\n",
      "Epoch 2/10\n",
      "325/325 [==============================] - 0s 65us/sample - loss: 1.7701 - accuracy: 0.5108\n",
      "Epoch 3/10\n",
      "325/325 [==============================] - 0s 62us/sample - loss: 1.2641 - accuracy: 0.6646\n",
      "Epoch 4/10\n",
      "325/325 [==============================] - 0s 60us/sample - loss: 0.9989 - accuracy: 0.6738\n",
      "Epoch 5/10\n",
      "325/325 [==============================] - 0s 61us/sample - loss: 0.7344 - accuracy: 0.7662\n",
      "Epoch 6/10\n",
      "325/325 [==============================] - 0s 60us/sample - loss: 0.6490 - accuracy: 0.7877\n",
      "Epoch 7/10\n",
      "325/325 [==============================] - 0s 62us/sample - loss: 0.5303 - accuracy: 0.8554\n",
      "Epoch 8/10\n",
      "325/325 [==============================] - 0s 60us/sample - loss: 0.4490 - accuracy: 0.8677\n",
      "Epoch 9/10\n",
      "325/325 [==============================] - 0s 63us/sample - loss: 0.4094 - accuracy: 0.8769\n",
      "Epoch 10/10\n",
      "325/325 [==============================] - 0s 61us/sample - loss: 0.3876 - accuracy: 0.8769\n",
      "Train on 302 samples\n",
      "Epoch 1/10\n",
      "302/302 [==============================] - 9s 30ms/sample - loss: 2.2317 - accuracy: 0.1490\n",
      "Epoch 2/10\n",
      "302/302 [==============================] - 0s 73us/sample - loss: 1.8106 - accuracy: 0.3907\n",
      "Epoch 3/10\n",
      "302/302 [==============================] - 0s 62us/sample - loss: 1.2261 - accuracy: 0.6689\n",
      "Epoch 4/10\n",
      "302/302 [==============================] - 0s 61us/sample - loss: 0.9943 - accuracy: 0.7053\n",
      "Epoch 5/10\n",
      "302/302 [==============================] - 0s 64us/sample - loss: 0.7917 - accuracy: 0.7318\n",
      "Epoch 6/10\n",
      "302/302 [==============================] - 0s 101us/sample - loss: 0.6734 - accuracy: 0.7848\n",
      "Epoch 7/10\n",
      "302/302 [==============================] - 0s 62us/sample - loss: 0.5752 - accuracy: 0.8212\n",
      "Epoch 8/10\n",
      "302/302 [==============================] - 0s 63us/sample - loss: 0.4797 - accuracy: 0.8742\n",
      "Epoch 9/10\n",
      "302/302 [==============================] - 0s 66us/sample - loss: 0.4404 - accuracy: 0.8642\n",
      "Epoch 10/10\n",
      "302/302 [==============================] - 0s 65us/sample - loss: 0.3985 - accuracy: 0.8742\n",
      "Train on 353 samples\n",
      "Epoch 1/10\n",
      "353/353 [==============================] - 9s 26ms/sample - loss: 2.2223 - accuracy: 0.2521\n",
      "Epoch 2/10\n",
      "353/353 [==============================] - 0s 61us/sample - loss: 1.7660 - accuracy: 0.5581\n",
      "Epoch 3/10\n",
      "353/353 [==============================] - 0s 59us/sample - loss: 1.2050 - accuracy: 0.6827\n",
      "Epoch 4/10\n",
      "353/353 [==============================] - 0s 62us/sample - loss: 0.9564 - accuracy: 0.6884\n",
      "Epoch 5/10\n",
      "353/353 [==============================] - 0s 62us/sample - loss: 0.7825 - accuracy: 0.7422\n",
      "Epoch 6/10\n",
      "353/353 [==============================] - 0s 59us/sample - loss: 0.6460 - accuracy: 0.8074\n",
      "Epoch 7/10\n",
      "353/353 [==============================] - 0s 58us/sample - loss: 0.4682 - accuracy: 0.8612\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/10\n",
      "353/353 [==============================] - 0s 58us/sample - loss: 0.4133 - accuracy: 0.8640\n",
      "Epoch 9/10\n",
      "353/353 [==============================] - 0s 57us/sample - loss: 0.4101 - accuracy: 0.8754\n",
      "Epoch 10/10\n",
      "353/353 [==============================] - 0s 59us/sample - loss: 0.2985 - accuracy: 0.8980\n",
      "Train on 372 samples\n",
      "Epoch 1/10\n",
      "372/372 [==============================] - 1s 2ms/sample - loss: 2.2181 - accuracy: 0.1909\n",
      "Epoch 2/10\n",
      "372/372 [==============================] - 0s 57us/sample - loss: 1.7898 - accuracy: 0.5054\n",
      "Epoch 3/10\n",
      "372/372 [==============================] - 0s 57us/sample - loss: 1.2460 - accuracy: 0.6344\n",
      "Epoch 4/10\n",
      "372/372 [==============================] - 0s 78us/sample - loss: 0.8714 - accuracy: 0.7070\n",
      "Epoch 5/10\n",
      "372/372 [==============================] - 0s 58us/sample - loss: 0.7682 - accuracy: 0.7446\n",
      "Epoch 6/10\n",
      "372/372 [==============================] - 0s 58us/sample - loss: 0.6201 - accuracy: 0.8145\n",
      "Epoch 7/10\n",
      "372/372 [==============================] - 0s 74us/sample - loss: 0.4564 - accuracy: 0.8575\n",
      "Epoch 8/10\n",
      "372/372 [==============================] - 0s 57us/sample - loss: 0.4243 - accuracy: 0.8710\n",
      "Epoch 9/10\n",
      "372/372 [==============================] - 0s 57us/sample - loss: 0.3629 - accuracy: 0.8817\n",
      "Epoch 10/10\n",
      "372/372 [==============================] - 0s 57us/sample - loss: 0.3140 - accuracy: 0.9005\n",
      "Train on 316 samples\n",
      "Epoch 1/10\n",
      "316/316 [==============================] - 1s 3ms/sample - loss: 2.2441 - accuracy: 0.1930\n",
      "Epoch 2/10\n",
      "316/316 [==============================] - 0s 62us/sample - loss: 1.8004 - accuracy: 0.5411\n",
      "Epoch 3/10\n",
      "316/316 [==============================] - 0s 71us/sample - loss: 1.2554 - accuracy: 0.6551\n",
      "Epoch 4/10\n",
      "316/316 [==============================] - 0s 61us/sample - loss: 0.9692 - accuracy: 0.6741\n",
      "Epoch 5/10\n",
      "316/316 [==============================] - 0s 59us/sample - loss: 0.6988 - accuracy: 0.7500\n",
      "Epoch 6/10\n",
      "316/316 [==============================] - 0s 62us/sample - loss: 0.6024 - accuracy: 0.7880\n",
      "Epoch 7/10\n",
      "316/316 [==============================] - 0s 61us/sample - loss: 0.4752 - accuracy: 0.8386\n",
      "Epoch 8/10\n",
      "316/316 [==============================] - 0s 61us/sample - loss: 0.4254 - accuracy: 0.8481\n",
      "Epoch 9/10\n",
      "316/316 [==============================] - 0s 60us/sample - loss: 0.3399 - accuracy: 0.8956\n",
      "Epoch 10/10\n",
      "316/316 [==============================] - 0s 76us/sample - loss: 0.2933 - accuracy: 0.9051\n",
      "Train on 303 samples\n",
      "Epoch 1/10\n",
      "303/303 [==============================] - 9s 31ms/sample - loss: 2.2222 - accuracy: 0.2112\n",
      "Epoch 2/10\n",
      "303/303 [==============================] - 0s 66us/sample - loss: 1.8643 - accuracy: 0.3894\n",
      "Epoch 3/10\n",
      "303/303 [==============================] - 0s 62us/sample - loss: 1.3138 - accuracy: 0.6073\n",
      "Epoch 4/10\n",
      "303/303 [==============================] - 0s 63us/sample - loss: 1.0035 - accuracy: 0.6502\n",
      "Epoch 5/10\n",
      "303/303 [==============================] - 0s 64us/sample - loss: 0.7907 - accuracy: 0.7393\n",
      "Epoch 6/10\n",
      "303/303 [==============================] - 0s 65us/sample - loss: 0.7577 - accuracy: 0.7360\n",
      "Epoch 7/10\n",
      "303/303 [==============================] - 0s 61us/sample - loss: 0.5716 - accuracy: 0.7921\n",
      "Epoch 8/10\n",
      "303/303 [==============================] - 0s 62us/sample - loss: 0.4851 - accuracy: 0.8185\n",
      "Epoch 9/10\n",
      "303/303 [==============================] - 0s 62us/sample - loss: 0.4291 - accuracy: 0.8680\n",
      "Epoch 10/10\n",
      "303/303 [==============================] - 0s 61us/sample - loss: 0.3950 - accuracy: 0.8647\n",
      "Train on 358 samples\n",
      "Epoch 1/10\n",
      "358/358 [==============================] - 9s 26ms/sample - loss: 2.2115 - accuracy: 0.2151\n",
      "Epoch 2/10\n",
      "358/358 [==============================] - 0s 60us/sample - loss: 1.7202 - accuracy: 0.4749\n",
      "Epoch 3/10\n",
      "358/358 [==============================] - 0s 59us/sample - loss: 1.1902 - accuracy: 0.6648\n",
      "Epoch 4/10\n",
      "358/358 [==============================] - 0s 60us/sample - loss: 0.8767 - accuracy: 0.7151\n",
      "Epoch 5/10\n",
      "358/358 [==============================] - 0s 58us/sample - loss: 0.7464 - accuracy: 0.7374\n",
      "Epoch 6/10\n",
      "358/358 [==============================] - 0s 58us/sample - loss: 0.5313 - accuracy: 0.8101\n",
      "Epoch 7/10\n",
      "358/358 [==============================] - 0s 56us/sample - loss: 0.4247 - accuracy: 0.8743\n",
      "Epoch 8/10\n",
      "358/358 [==============================] - 0s 57us/sample - loss: 0.2820 - accuracy: 0.9190\n",
      "Epoch 9/10\n",
      "358/358 [==============================] - 0s 57us/sample - loss: 0.2371 - accuracy: 0.9162\n",
      "Epoch 10/10\n",
      "358/358 [==============================] - 0s 57us/sample - loss: 0.2120 - accuracy: 0.9246\n",
      "Train on 346 samples\n",
      "Epoch 1/10\n",
      "346/346 [==============================] - 1s 2ms/sample - loss: 2.2227 - accuracy: 0.2197\n",
      "Epoch 2/10\n",
      "346/346 [==============================] - 0s 60us/sample - loss: 1.7149 - accuracy: 0.5665\n",
      "Epoch 3/10\n",
      "346/346 [==============================] - 0s 58us/sample - loss: 1.2180 - accuracy: 0.6532\n",
      "Epoch 4/10\n",
      "346/346 [==============================] - 0s 58us/sample - loss: 0.9042 - accuracy: 0.7081\n",
      "Epoch 5/10\n",
      "346/346 [==============================] - 0s 57us/sample - loss: 0.6513 - accuracy: 0.7659\n",
      "Epoch 6/10\n",
      "346/346 [==============================] - 0s 59us/sample - loss: 0.6006 - accuracy: 0.8035\n",
      "Epoch 7/10\n",
      "346/346 [==============================] - 0s 59us/sample - loss: 0.4825 - accuracy: 0.8526\n",
      "Epoch 8/10\n",
      "346/346 [==============================] - 0s 58us/sample - loss: 0.4154 - accuracy: 0.8613\n",
      "Epoch 9/10\n",
      "346/346 [==============================] - 0s 58us/sample - loss: 0.3753 - accuracy: 0.8786\n",
      "Epoch 10/10\n",
      "346/346 [==============================] - 0s 59us/sample - loss: 0.3321 - accuracy: 0.8844\n",
      "Train on 338 samples\n",
      "Epoch 1/10\n",
      "338/338 [==============================] - 1s 2ms/sample - loss: 2.2238 - accuracy: 0.2249\n",
      "Epoch 2/10\n",
      "338/338 [==============================] - 0s 61us/sample - loss: 1.7818 - accuracy: 0.5207\n",
      "Epoch 3/10\n",
      "338/338 [==============================] - 0s 73us/sample - loss: 1.2761 - accuracy: 0.6361\n",
      "Epoch 4/10\n",
      "338/338 [==============================] - 0s 62us/sample - loss: 0.8656 - accuracy: 0.7367\n",
      "Epoch 5/10\n",
      "338/338 [==============================] - 0s 76us/sample - loss: 0.7510 - accuracy: 0.7574\n",
      "Epoch 6/10\n",
      "338/338 [==============================] - 0s 60us/sample - loss: 0.6583 - accuracy: 0.7988\n",
      "Epoch 7/10\n",
      "338/338 [==============================] - 0s 69us/sample - loss: 0.5113 - accuracy: 0.8373\n",
      "Epoch 8/10\n",
      "338/338 [==============================] - 0s 60us/sample - loss: 0.4455 - accuracy: 0.8521\n",
      "Epoch 9/10\n",
      "338/338 [==============================] - 0s 58us/sample - loss: 0.4286 - accuracy: 0.8521\n",
      "Epoch 10/10\n",
      "338/338 [==============================] - 0s 59us/sample - loss: 0.3545 - accuracy: 0.8846\n",
      "Train on 310 samples\n",
      "Epoch 1/10\n",
      "310/310 [==============================] - 9s 31ms/sample - loss: 2.2454 - accuracy: 0.1387\n",
      "Epoch 2/10\n",
      "310/310 [==============================] - 0s 68us/sample - loss: 1.8529 - accuracy: 0.4968\n",
      "Epoch 3/10\n",
      "310/310 [==============================] - 0s 64us/sample - loss: 1.3008 - accuracy: 0.6387\n",
      "Epoch 4/10\n",
      "310/310 [==============================] - 0s 62us/sample - loss: 1.0638 - accuracy: 0.6806\n",
      "Epoch 5/10\n",
      "310/310 [==============================] - 0s 64us/sample - loss: 0.8146 - accuracy: 0.7484\n",
      "Epoch 6/10\n",
      "310/310 [==============================] - 0s 62us/sample - loss: 0.6949 - accuracy: 0.7613\n",
      "Epoch 7/10\n",
      "310/310 [==============================] - 0s 64us/sample - loss: 0.5716 - accuracy: 0.8032\n",
      "Epoch 8/10\n",
      "310/310 [==============================] - 0s 63us/sample - loss: 0.5127 - accuracy: 0.8323\n",
      "Epoch 9/10\n",
      "310/310 [==============================] - 0s 64us/sample - loss: 0.4184 - accuracy: 0.8710\n",
      "Epoch 10/10\n",
      "310/310 [==============================] - 0s 63us/sample - loss: 0.3473 - accuracy: 0.8871\n",
      "Train on 308 samples\n",
      "Epoch 1/10\n",
      "308/308 [==============================] - 1s 3ms/sample - loss: 2.2314 - accuracy: 0.1851\n",
      "Epoch 2/10\n",
      "308/308 [==============================] - 0s 61us/sample - loss: 1.7775 - accuracy: 0.5162\n",
      "Epoch 3/10\n",
      "308/308 [==============================] - 0s 63us/sample - loss: 1.2734 - accuracy: 0.6396\n",
      "Epoch 4/10\n",
      "308/308 [==============================] - 0s 61us/sample - loss: 0.9276 - accuracy: 0.6981\n",
      "Epoch 5/10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "308/308 [==============================] - 0s 61us/sample - loss: 0.6764 - accuracy: 0.7857\n",
      "Epoch 6/10\n",
      "308/308 [==============================] - 0s 63us/sample - loss: 0.6220 - accuracy: 0.7500\n",
      "Epoch 7/10\n",
      "308/308 [==============================] - 0s 63us/sample - loss: 0.4707 - accuracy: 0.8442\n",
      "Epoch 8/10\n",
      "308/308 [==============================] - 0s 62us/sample - loss: 0.3935 - accuracy: 0.8636\n",
      "Epoch 9/10\n",
      "308/308 [==============================] - 0s 73us/sample - loss: 0.3467 - accuracy: 0.8994\n",
      "Epoch 10/10\n",
      "308/308 [==============================] - 0s 61us/sample - loss: 0.2742 - accuracy: 0.9123\n",
      "Train on 345 samples\n",
      "Epoch 1/10\n",
      "345/345 [==============================] - 1s 2ms/sample - loss: 2.2134 - accuracy: 0.2116\n",
      "Epoch 2/10\n",
      "345/345 [==============================] - 0s 60us/sample - loss: 1.7026 - accuracy: 0.5391\n",
      "Epoch 3/10\n",
      "345/345 [==============================] - 0s 58us/sample - loss: 1.0985 - accuracy: 0.6870\n",
      "Epoch 4/10\n",
      "345/345 [==============================] - 0s 61us/sample - loss: 0.7737 - accuracy: 0.7507\n",
      "Epoch 5/10\n",
      "345/345 [==============================] - 0s 58us/sample - loss: 0.6047 - accuracy: 0.8029\n",
      "Epoch 6/10\n",
      "345/345 [==============================] - 0s 59us/sample - loss: 0.4624 - accuracy: 0.8290\n",
      "Epoch 7/10\n",
      "345/345 [==============================] - 0s 60us/sample - loss: 0.3929 - accuracy: 0.8638\n",
      "Epoch 8/10\n",
      "345/345 [==============================] - 0s 59us/sample - loss: 0.3704 - accuracy: 0.8783\n",
      "Epoch 9/10\n",
      "345/345 [==============================] - 0s 58us/sample - loss: 0.2887 - accuracy: 0.9159\n",
      "Epoch 10/10\n",
      "345/345 [==============================] - 0s 59us/sample - loss: 0.2398 - accuracy: 0.9188\n",
      "Train on 350 samples\n",
      "Epoch 1/10\n",
      "350/350 [==============================] - 10s 28ms/sample - loss: 2.2163 - accuracy: 0.2143\n",
      "Epoch 2/10\n",
      "350/350 [==============================] - 0s 63us/sample - loss: 1.7309 - accuracy: 0.5457\n",
      "Epoch 3/10\n",
      "350/350 [==============================] - 0s 58us/sample - loss: 1.1825 - accuracy: 0.6314\n",
      "Epoch 4/10\n",
      "350/350 [==============================] - 0s 60us/sample - loss: 0.9737 - accuracy: 0.6543\n",
      "Epoch 5/10\n",
      "350/350 [==============================] - 0s 59us/sample - loss: 0.6912 - accuracy: 0.7800\n",
      "Epoch 6/10\n",
      "350/350 [==============================] - 0s 60us/sample - loss: 0.5510 - accuracy: 0.8171\n",
      "Epoch 7/10\n",
      "350/350 [==============================] - 0s 62us/sample - loss: 0.4774 - accuracy: 0.8457\n",
      "Epoch 8/10\n",
      "350/350 [==============================] - 0s 61us/sample - loss: 0.4374 - accuracy: 0.8657\n",
      "Epoch 9/10\n",
      "350/350 [==============================] - 0s 61us/sample - loss: 0.3833 - accuracy: 0.8714\n",
      "Epoch 10/10\n",
      "350/350 [==============================] - 0s 59us/sample - loss: 0.2975 - accuracy: 0.9057\n",
      "Train on 321 samples\n",
      "Epoch 1/10\n",
      "321/321 [==============================] - 10s 30ms/sample - loss: 2.2323 - accuracy: 0.2087\n",
      "Epoch 2/10\n",
      "321/321 [==============================] - 0s 68us/sample - loss: 1.7762 - accuracy: 0.4860\n",
      "Epoch 3/10\n",
      "321/321 [==============================] - 0s 63us/sample - loss: 1.3014 - accuracy: 0.6604\n",
      "Epoch 4/10\n",
      "321/321 [==============================] - 0s 62us/sample - loss: 0.9509 - accuracy: 0.7165\n",
      "Epoch 5/10\n",
      "321/321 [==============================] - 0s 62us/sample - loss: 0.7085 - accuracy: 0.7601\n",
      "Epoch 6/10\n",
      "321/321 [==============================] - 0s 60us/sample - loss: 0.6970 - accuracy: 0.7570\n",
      "Epoch 7/10\n",
      "321/321 [==============================] - 0s 61us/sample - loss: 0.4915 - accuracy: 0.8442\n",
      "Epoch 8/10\n",
      "321/321 [==============================] - 0s 66us/sample - loss: 0.4861 - accuracy: 0.8442\n",
      "Epoch 9/10\n",
      "321/321 [==============================] - 0s 63us/sample - loss: 0.4149 - accuracy: 0.8692\n",
      "Epoch 10/10\n",
      "321/321 [==============================] - 0s 62us/sample - loss: 0.3980 - accuracy: 0.8879\n",
      "Train on 316 samples\n",
      "Epoch 1/10\n",
      "316/316 [==============================] - 1s 3ms/sample - loss: 2.2252 - accuracy: 0.1551\n",
      "Epoch 2/10\n",
      "316/316 [==============================] - 0s 63us/sample - loss: 1.8059 - accuracy: 0.4462\n",
      "Epoch 3/10\n",
      "316/316 [==============================] - 0s 62us/sample - loss: 1.2688 - accuracy: 0.6519\n",
      "Epoch 4/10\n",
      "316/316 [==============================] - 0s 64us/sample - loss: 0.8890 - accuracy: 0.7342\n",
      "Epoch 5/10\n",
      "316/316 [==============================] - 0s 64us/sample - loss: 0.7274 - accuracy: 0.7437\n",
      "Epoch 6/10\n",
      "316/316 [==============================] - 0s 63us/sample - loss: 0.5688 - accuracy: 0.8259\n",
      "Epoch 7/10\n",
      "316/316 [==============================] - 0s 64us/sample - loss: 0.4736 - accuracy: 0.8354\n",
      "Epoch 8/10\n",
      "316/316 [==============================] - 0s 62us/sample - loss: 0.4139 - accuracy: 0.8386\n",
      "Epoch 9/10\n",
      "316/316 [==============================] - 0s 62us/sample - loss: 0.3449 - accuracy: 0.8829\n",
      "Epoch 10/10\n",
      "316/316 [==============================] - 0s 63us/sample - loss: 0.2655 - accuracy: 0.9241\n",
      "Train on 336 samples\n",
      "Epoch 1/10\n",
      "336/336 [==============================] - 1s 2ms/sample - loss: 2.2340 - accuracy: 0.1935\n",
      "Epoch 2/10\n",
      "336/336 [==============================] - 0s 61us/sample - loss: 1.8119 - accuracy: 0.4196\n",
      "Epoch 3/10\n",
      "336/336 [==============================] - 0s 61us/sample - loss: 1.2897 - accuracy: 0.6429\n",
      "Epoch 4/10\n",
      "336/336 [==============================] - 0s 60us/sample - loss: 0.9084 - accuracy: 0.7054\n",
      "Epoch 5/10\n",
      "336/336 [==============================] - 0s 59us/sample - loss: 0.7190 - accuracy: 0.8006\n",
      "Epoch 6/10\n",
      "336/336 [==============================] - 0s 59us/sample - loss: 0.5532 - accuracy: 0.8333\n",
      "Epoch 7/10\n",
      "336/336 [==============================] - 0s 62us/sample - loss: 0.5134 - accuracy: 0.8125\n",
      "Epoch 8/10\n",
      "336/336 [==============================] - 0s 60us/sample - loss: 0.4692 - accuracy: 0.8512\n",
      "Epoch 9/10\n",
      "336/336 [==============================] - 0s 62us/sample - loss: 0.3049 - accuracy: 0.9256\n",
      "Epoch 10/10\n",
      "336/336 [==============================] - 0s 65us/sample - loss: 0.3225 - accuracy: 0.8988\n",
      "Train on 316 samples\n",
      "Epoch 1/10\n",
      "316/316 [==============================] - 1s 3ms/sample - loss: 2.2395 - accuracy: 0.1487\n",
      "Epoch 2/10\n",
      "316/316 [==============================] - 0s 64us/sample - loss: 1.8217 - accuracy: 0.4146\n",
      "Epoch 3/10\n",
      "316/316 [==============================] - 0s 63us/sample - loss: 1.2942 - accuracy: 0.6108\n",
      "Epoch 4/10\n",
      "316/316 [==============================] - 0s 64us/sample - loss: 0.9850 - accuracy: 0.6867\n",
      "Epoch 5/10\n",
      "316/316 [==============================] - 0s 62us/sample - loss: 0.7828 - accuracy: 0.7532\n",
      "Epoch 6/10\n",
      "316/316 [==============================] - 0s 63us/sample - loss: 0.6011 - accuracy: 0.8101\n",
      "Epoch 7/10\n",
      "316/316 [==============================] - 0s 62us/sample - loss: 0.5086 - accuracy: 0.8418\n",
      "Epoch 8/10\n",
      "316/316 [==============================] - 0s 63us/sample - loss: 0.4250 - accuracy: 0.8734\n",
      "Epoch 9/10\n",
      "316/316 [==============================] - 0s 65us/sample - loss: 0.3876 - accuracy: 0.8734\n",
      "Epoch 10/10\n",
      "316/316 [==============================] - 0s 65us/sample - loss: 0.3333 - accuracy: 0.9082\n",
      "Train on 324 samples\n",
      "Epoch 1/10\n",
      "324/324 [==============================] - 1s 3ms/sample - loss: 2.2485 - accuracy: 0.1358\n",
      "Epoch 2/10\n",
      "324/324 [==============================] - 0s 64us/sample - loss: 1.8273 - accuracy: 0.4907\n",
      "Epoch 3/10\n",
      "324/324 [==============================] - 0s 62us/sample - loss: 1.3083 - accuracy: 0.6235\n",
      "Epoch 4/10\n",
      "324/324 [==============================] - 0s 60us/sample - loss: 0.8855 - accuracy: 0.7438\n",
      "Epoch 5/10\n",
      "324/324 [==============================] - 0s 60us/sample - loss: 0.7348 - accuracy: 0.7469\n",
      "Epoch 6/10\n",
      "324/324 [==============================] - 0s 65us/sample - loss: 0.6567 - accuracy: 0.7685\n",
      "Epoch 7/10\n",
      "324/324 [==============================] - 0s 63us/sample - loss: 0.4475 - accuracy: 0.8549\n",
      "Epoch 8/10\n",
      "324/324 [==============================] - 0s 61us/sample - loss: 0.4722 - accuracy: 0.8302\n",
      "Epoch 9/10\n",
      "324/324 [==============================] - 0s 61us/sample - loss: 0.3700 - accuracy: 0.8951\n",
      "Epoch 10/10\n",
      "324/324 [==============================] - 0s 59us/sample - loss: 0.3261 - accuracy: 0.9198\n",
      "Train on 377 samples\n",
      "Epoch 1/10\n",
      "377/377 [==============================] - 10s 27ms/sample - loss: 2.2325 - accuracy: 0.1777\n",
      "Epoch 2/10\n",
      "377/377 [==============================] - 0s 62us/sample - loss: 1.8040 - accuracy: 0.4775\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/10\n",
      "377/377 [==============================] - 0s 57us/sample - loss: 1.2591 - accuracy: 0.6605\n",
      "Epoch 4/10\n",
      "377/377 [==============================] - 0s 60us/sample - loss: 0.9453 - accuracy: 0.7003\n",
      "Epoch 5/10\n",
      "377/377 [==============================] - 0s 57us/sample - loss: 0.7164 - accuracy: 0.7241\n",
      "Epoch 6/10\n",
      "377/377 [==============================] - 0s 59us/sample - loss: 0.5858 - accuracy: 0.8276\n",
      "Epoch 7/10\n",
      "377/377 [==============================] - 0s 57us/sample - loss: 0.4412 - accuracy: 0.8621\n",
      "Epoch 8/10\n",
      "377/377 [==============================] - 0s 59us/sample - loss: 0.3762 - accuracy: 0.8647\n",
      "Epoch 9/10\n",
      "377/377 [==============================] - 0s 59us/sample - loss: 0.3873 - accuracy: 0.8568\n",
      "Epoch 10/10\n",
      "377/377 [==============================] - 0s 57us/sample - loss: 0.2809 - accuracy: 0.9072\n",
      "Train on 348 samples\n",
      "Epoch 1/10\n",
      "348/348 [==============================] - 10s 29ms/sample - loss: 2.2198 - accuracy: 0.2126\n",
      "Epoch 2/10\n",
      "348/348 [==============================] - 0s 64us/sample - loss: 1.7543 - accuracy: 0.4741\n",
      "Epoch 3/10\n",
      "348/348 [==============================] - 0s 61us/sample - loss: 1.2262 - accuracy: 0.6437\n",
      "Epoch 4/10\n",
      "348/348 [==============================] - 0s 62us/sample - loss: 0.9269 - accuracy: 0.6925\n",
      "Epoch 5/10\n",
      "348/348 [==============================] - 0s 63us/sample - loss: 0.7221 - accuracy: 0.7443\n",
      "Epoch 6/10\n",
      "348/348 [==============================] - 0s 63us/sample - loss: 0.6049 - accuracy: 0.7730\n",
      "Epoch 7/10\n",
      "348/348 [==============================] - 0s 61us/sample - loss: 0.5129 - accuracy: 0.8333\n",
      "Epoch 8/10\n",
      "348/348 [==============================] - 0s 60us/sample - loss: 0.4578 - accuracy: 0.8362\n",
      "Epoch 9/10\n",
      "348/348 [==============================] - 0s 80us/sample - loss: 0.3557 - accuracy: 0.8994\n",
      "Epoch 10/10\n",
      "348/348 [==============================] - 0s 60us/sample - loss: 0.2937 - accuracy: 0.9080\n",
      "Train on 343 samples\n",
      "Epoch 1/10\n",
      "343/343 [==============================] - 10s 30ms/sample - loss: 2.2447 - accuracy: 0.1691\n",
      "Epoch 2/10\n",
      "343/343 [==============================] - 0s 66us/sample - loss: 1.8162 - accuracy: 0.5044\n",
      "Epoch 3/10\n",
      "343/343 [==============================] - 0s 62us/sample - loss: 1.2891 - accuracy: 0.6443\n",
      "Epoch 4/10\n",
      "343/343 [==============================] - 0s 66us/sample - loss: 0.9185 - accuracy: 0.7114\n",
      "Epoch 5/10\n",
      "343/343 [==============================] - 0s 63us/sample - loss: 0.7428 - accuracy: 0.7755\n",
      "Epoch 6/10\n",
      "343/343 [==============================] - 0s 60us/sample - loss: 0.6220 - accuracy: 0.7843\n",
      "Epoch 7/10\n",
      "343/343 [==============================] - 0s 60us/sample - loss: 0.5590 - accuracy: 0.8134\n",
      "Epoch 8/10\n",
      "343/343 [==============================] - 0s 59us/sample - loss: 0.4537 - accuracy: 0.8338\n",
      "Epoch 9/10\n",
      "343/343 [==============================] - 0s 61us/sample - loss: 0.3820 - accuracy: 0.8892\n",
      "Epoch 10/10\n",
      "343/343 [==============================] - 0s 62us/sample - loss: 0.3269 - accuracy: 0.9009\n",
      "Train on 317 samples\n",
      "Epoch 1/10\n",
      "317/317 [==============================] - 10s 33ms/sample - loss: 2.2410 - accuracy: 0.1609\n",
      "Epoch 2/10\n",
      "317/317 [==============================] - 0s 70us/sample - loss: 1.8336 - accuracy: 0.4038\n",
      "Epoch 3/10\n",
      "317/317 [==============================] - 0s 65us/sample - loss: 1.3884 - accuracy: 0.6215\n",
      "Epoch 4/10\n",
      "317/317 [==============================] - 0s 65us/sample - loss: 1.0278 - accuracy: 0.6877\n",
      "Epoch 5/10\n",
      "317/317 [==============================] - 0s 63us/sample - loss: 0.7994 - accuracy: 0.7224\n",
      "Epoch 6/10\n",
      "317/317 [==============================] - 0s 63us/sample - loss: 0.6030 - accuracy: 0.8044\n",
      "Epoch 7/10\n",
      "317/317 [==============================] - 0s 65us/sample - loss: 0.5187 - accuracy: 0.8328\n",
      "Epoch 8/10\n",
      "317/317 [==============================] - 0s 66us/sample - loss: 0.4410 - accuracy: 0.8423\n",
      "Epoch 9/10\n",
      "317/317 [==============================] - 0s 64us/sample - loss: 0.4011 - accuracy: 0.8644\n",
      "Epoch 10/10\n",
      "317/317 [==============================] - 0s 65us/sample - loss: 0.3498 - accuracy: 0.8770\n",
      "Train on 311 samples\n",
      "Epoch 1/10\n",
      "311/311 [==============================] - 10s 34ms/sample - loss: 2.2341 - accuracy: 0.1833\n",
      "Epoch 2/10\n",
      "311/311 [==============================] - 0s 69us/sample - loss: 1.7578 - accuracy: 0.5370\n",
      "Epoch 3/10\n",
      "311/311 [==============================] - 0s 67us/sample - loss: 1.1892 - accuracy: 0.7170\n",
      "Epoch 4/10\n",
      "311/311 [==============================] - 0s 67us/sample - loss: 0.8591 - accuracy: 0.7460\n",
      "Epoch 5/10\n",
      "311/311 [==============================] - 0s 66us/sample - loss: 0.6534 - accuracy: 0.7974\n",
      "Epoch 6/10\n",
      "311/311 [==============================] - 0s 65us/sample - loss: 0.4622 - accuracy: 0.8553\n",
      "Epoch 7/10\n",
      "311/311 [==============================] - 0s 103us/sample - loss: 0.4083 - accuracy: 0.8650\n",
      "Epoch 8/10\n",
      "311/311 [==============================] - 0s 64us/sample - loss: 0.3908 - accuracy: 0.8617\n",
      "Epoch 9/10\n",
      "311/311 [==============================] - 0s 64us/sample - loss: 0.3142 - accuracy: 0.8682\n",
      "Epoch 10/10\n",
      "311/311 [==============================] - 0s 64us/sample - loss: 0.2398 - accuracy: 0.9228\n",
      "Train on 336 samples\n",
      "Epoch 1/10\n",
      "336/336 [==============================] - 1s 3ms/sample - loss: 2.2314 - accuracy: 0.1607\n",
      "Epoch 2/10\n",
      "336/336 [==============================] - 0s 62us/sample - loss: 1.7711 - accuracy: 0.5238\n",
      "Epoch 3/10\n",
      "336/336 [==============================] - 0s 61us/sample - loss: 1.2727 - accuracy: 0.6310\n",
      "Epoch 4/10\n",
      "336/336 [==============================] - 0s 61us/sample - loss: 0.9643 - accuracy: 0.7083\n",
      "Epoch 5/10\n",
      "336/336 [==============================] - 0s 60us/sample - loss: 0.7248 - accuracy: 0.7411\n",
      "Epoch 6/10\n",
      "336/336 [==============================] - 0s 61us/sample - loss: 0.5790 - accuracy: 0.8125\n",
      "Epoch 7/10\n",
      "336/336 [==============================] - 0s 63us/sample - loss: 0.5265 - accuracy: 0.8185\n",
      "Epoch 8/10\n",
      "336/336 [==============================] - 0s 63us/sample - loss: 0.4250 - accuracy: 0.8780\n",
      "Epoch 9/10\n",
      "336/336 [==============================] - 0s 64us/sample - loss: 0.3255 - accuracy: 0.9107\n",
      "Epoch 10/10\n",
      "336/336 [==============================] - 0s 63us/sample - loss: 0.3201 - accuracy: 0.9107\n",
      "Train on 324 samples\n",
      "Epoch 1/10\n",
      "324/324 [==============================] - 1s 3ms/sample - loss: 2.2470 - accuracy: 0.1543\n",
      "Epoch 2/10\n",
      "324/324 [==============================] - 0s 61us/sample - loss: 1.8273 - accuracy: 0.5000\n",
      "Epoch 3/10\n",
      "324/324 [==============================] - 0s 61us/sample - loss: 1.3491 - accuracy: 0.6235\n",
      "Epoch 4/10\n",
      "324/324 [==============================] - 0s 63us/sample - loss: 0.9773 - accuracy: 0.6759\n",
      "Epoch 5/10\n",
      "324/324 [==============================] - 0s 63us/sample - loss: 0.7357 - accuracy: 0.7716\n",
      "Epoch 6/10\n",
      "324/324 [==============================] - 0s 62us/sample - loss: 0.5878 - accuracy: 0.8241\n",
      "Epoch 7/10\n",
      "324/324 [==============================] - 0s 62us/sample - loss: 0.6669 - accuracy: 0.7840\n",
      "Epoch 8/10\n",
      "324/324 [==============================] - 0s 63us/sample - loss: 0.4734 - accuracy: 0.8642\n",
      "Epoch 9/10\n",
      "324/324 [==============================] - 0s 63us/sample - loss: 0.4445 - accuracy: 0.8611\n",
      "Epoch 10/10\n",
      "324/324 [==============================] - 0s 62us/sample - loss: 0.3273 - accuracy: 0.9074\n"
     ]
    }
   ],
   "source": [
    "model.fit(pdata, plabels, nb_epochs=10)\n",
    "dpa_model_10.fit(pdata, plabels, nb_epochs=10)\n",
    "dpa_model_20.fit(pdata, plabels, nb_epochs=10)\n",
    "dpa_model_30.fit(pdata, plabels, nb_epochs=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7389971",
   "metadata": {},
   "source": [
    "# Evaluate the performance of the trained models on unpoisoned data\n",
    "The performance of the models appears normal. We see that for the DPA models, the performance drops slightly as the ensemble size increases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2f63b588",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Clean test set accuracy (model): 98.04%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOJElEQVR4nO3df4wc9XnH8c8n5rCpSVocfl3AKhBBGoIECScHQZtCUS1AbW2aQnHbyEmpTBKoEilVSigpUKWNRRuStkloLsGyG1FCWkAQiaZBLohGkTAHdWyDAVPqgLFrg6hqEwX7bD/948bkYm6/e96d3dnjeb+k0+7Os7Pz3OIPszffnfk6IgTgze8tTTcAoD8IO5AEYQeSIOxAEoQdSOKwfm7scM+OOZrbz00CqbymH2tP7PZUta7CbvsiSX8raZakb0TE8tLz52iu3u8Lu9kkgIJHYnXLWscf423PkvQVSRdLOl3SEtund/p6AHqrm7/ZF0h6NiKei4g9kr4laVE9bQGoWzdhP0HSC5Meb6mW/Qzby2yP2R4b1+4uNgegG92EfaqDAG/47m1EjEbESESMDGl2F5sD0I1uwr5F0vxJj0+UtLW7dgD0Sjdhf1TSqbZPtn24pCsk3VdPWwDq1vHQW0TstX2NpH/TxNDbioh4orbOANSqq3H2iLhf0v019QKgh/i6LJAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HXKZvTfrF/4+WL96S+fUqw/dcE3ivXrd5xdrK///dNa1vY9+UxxXdSLPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4+5vc/pNPLNbXn/+1Yn08yq//uWMfK9bPvPTclrX5jLP3VVdht71Z0i5J+yTtjYiROpoCUL869uwXRMTLNbwOgB7ib3YgiW7DHpK+Z/sx28umeoLtZbbHbI+Na3eXmwPQqW4/xp8XEVttHyvpAdtPRcTDk58QEaOSRiXpbZ7X5nAPgF7pas8eEVur2x2S7pG0oI6mANSv47Dbnmv7rQfuS1ooaUNdjQGoVzcf44+TdI/tA6/zTxHx3Vq6wiE5bH7rsfSTR5/tYycYZB2HPSKek3Rmjb0A6CGG3oAkCDuQBGEHkiDsQBKEHUiCU1xngOf/vPVpopJ09kVPtqzdPPwfdbdzSI4896WWtRc+W/69jl63t1g/4t41HfWUFXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYZYN1Vf1+sj8e+PnVy6B468/bWxTbnTN7z4+FifcWuxcX6Yf9evsx1NuzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtkHwNBD5fHkIc/qUyeH7j/37C/WN48f07J26dxXiutefuSOcv2bo8X6b5xwdrGeDXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY++MniBcX6R4b/uVhvd756L89nP2P1R4v1Y1bPLtZn/1/r3j5zfnlfs/6yvyvW29nymdbXpT/x8z/o6rVnorZ7dtsrbO+wvWHSsnm2H7C9qbo9qrdtAujWdD7Gr5R00UHLrpW0OiJOlbS6egxggLUNe0Q8LOng7zUukrSqur9K0uJ62wJQt04P0B0XEdskqbo9ttUTbS+zPWZ7bFy7O9wcgG71/Gh8RIxGxEhEjAypfDAHQO90Gvbttoclqbotn54EoHGdhv0+SUur+0sl3VtPOwB6pe04u+07JJ0v6WjbWyTdIGm5pG/bvlLS85Iu62WTg27We95VrH/ulvJ51yOH72m3hUPs6KfaXXv9+gc/WKy/+9NPFev7du485J4OeNem04r1Nb81p1hfMPu1Yv1fP3Zzy9rCOZ8urnvSX5WvOR+7Z97xp7Zhj4glLUoX1twLgB7i67JAEoQdSIKwA0kQdiAJwg4kwSmuNdh/ePltbD+01p0//NHB5yn91K7fPaK47mlb1hTrvZwMet+TzxTrH19ZPr127KovFevDs1r/7o9fWV73g3cvLdbjhxuL9UHEnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcfQa4bvtIsb7zj97esrZvy6a62+mbk+56uVj/7OJzivXlxz9aZzszHnt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY+GHLnl4KWpHXvizbPmLlj6UV2sXzYW/YX692871tvKtePX9zxSzeGPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4ew2e/tjPFevj0curr795bf7t1ufpS9K/HFO+5v14tB5nb/ff5B03FMsqj/APprZ7dtsrbO+wvWHSshttv2h7bfVzSW/bBNCt6XyMXylpqilHvhgRZ1U/99fbFoC6tQ17RDws6ZU+9AKgh7o5QHeN7XXVx/yjWj3J9jLbY7bHxrW7i80B6EanYb9V0jslnSVpm6QvtHpiRIxGxEhEjAxpdoebA9CtjsIeEdsjYl9E7Jf0dUkL6m0LQN06Crvt4UkPL5W0odVzAQyGtuPstu+QdL6ko21vkXSDpPNtnyUpJG2WdFXvWhx81//Kd5puYWAdNv/ElrVdZ7+juO4/fOSrdbfzujW75xTr3rO3Z9tuStuwR8SSKRbf1oNeAPQQX5cFkiDsQBKEHUiCsANJEHYgCU5xRU89edPxLWtPLPxyT7d916tHt6zd+ieXFdeds7F8+uxMxJ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1dGXpouFj//PBdferkjVa+eG7L2pzvvPnG0dthzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTDOXoNZLk/gO+TWUwdPx87fO6fjdW/6i/KFgC844rWOX1tq/7uVp0bu7n1pJ37txZ6+/kzDnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcvQbL7/ydYv3yK7/U1es//NdfKdbLY9ll49HxqtN8/c57a+eM1R8t1k/V4z3b9kzUds9ue77tB21vtP2E7U9Uy+fZfsD2pur2qN63C6BT0/kYv1fSpyLi3ZLOkXS17dMlXStpdUScKml19RjAgGob9ojYFhGPV/d3Sdoo6QRJiyStqp62StLiHvUIoAaHdIDO9kmS3ivpEUnHRcQ2aeJ/CJKObbHOMttjtsfGtbvLdgF0atpht32kpLskfTIidk53vYgYjYiRiBgZ0uxOegRQg2mF3faQJoJ+e0TcXS3ebnu4qg9L2tGbFgHUoe3Qm21Luk3Sxoi4ZVLpPklLJS2vbu/tSYczwCl3vlysr/mDOcX6gtndnWY6yNbsbv27j/7PrxbX/d+Pt57uWZJ+6b+fLdZ7N+g3M01nnP08SR+StN722mrZdZoI+bdtXynpeUnlCa8BNKpt2CPi+5Lconxhve0A6BW+LgskQdiBJAg7kARhB5Ig7EASjujxOY6TvM3z4v3OdwD/J4sWFOsv/Gb5UtTPXPy1Yr2Xp5G20+5S0md+9Y9b1ub/5Q/qbie9R2K1dsYrU46esWcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSS4lHQfHHHvmmL9tDZXAvjAkquL9aEPb29Z++577iyuu3DDFcX6/pVTXm3sddHqfMjKSWtfalnjfPP+Ys8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwPjvwJsL57AAIO5AFYQeSIOxAEoQdSIKwA0kQdiCJtmG3Pd/2g7Y32n7C9ieq5TfaftH22urnkt63C6BT07l4xV5Jn4qIx22/VdJjth+oal+MiL/pXXsA6jKd+dm3SdpW3d9le6OkE3rdGIB6HdLf7LZPkvReSY9Ui66xvc72CttHtVhnme0x22Pj2t1dtwA6Nu2w2z5S0l2SPhkROyXdKumdks7SxJ7/C1OtFxGjETESESNDmt19xwA6Mq2w2x7SRNBvj4i7JSkitkfEvojYL+nrksqzFwJo1HSOxlvSbZI2RsQtk5YPT3rapZI21N8egLpM52j8eZI+JGm97bXVsuskLbF9lqSQtFnSVT3oD0BNpnM0/vuSpjo/9v762wHQK3yDDkiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kERfp2y2/ZKkH01adLSkl/vWwKEZ1N4GtS+J3jpVZ2+/GBHHTFXoa9jfsHF7LCJGGmugYFB7G9S+JHrrVL9642M8kARhB5JoOuyjDW+/ZFB7G9S+JHrrVF96a/RvdgD90/SeHUCfEHYgiUbCbvsi20/bftb2tU300IrtzbbXV9NQjzXcywrbO2xvmLRsnu0HbG+qbqecY6+h3gZiGu/CNOONvndNT3/e97/Zbc+S9IykX5e0RdKjkpZExJN9baQF25sljURE41/AsP0BSa9K+seIOKNadrOkVyJiefU/yqMi4k8HpLcbJb3a9DTe1WxFw5OnGZe0WNKH1eB7V+jrcvXhfWtiz75A0rMR8VxE7JH0LUmLGuhj4EXEw5JeOWjxIkmrqvurNPGPpe9a9DYQImJbRDxe3d8l6cA0442+d4W++qKJsJ8g6YVJj7dosOZ7D0nfs/2Y7WVNNzOF4yJimzTxj0fSsQ33c7C203j300HTjA/Me9fJ9OfdaiLsU00lNUjjf+dFxPskXSzp6urjKqZnWtN498sU04wPhE6nP+9WE2HfImn+pMcnStraQB9Tioit1e0OSfdo8Kai3n5gBt3qdkfD/bxukKbxnmqacQ3Ae9fk9OdNhP1RSafaPtn24ZKukHRfA328ge251YET2Z4raaEGbyrq+yQtre4vlXRvg738jEGZxrvVNONq+L1rfPrziOj7j6RLNHFE/r8k/VkTPbTo6xRJP6x+nmi6N0l3aOJj3bgmPhFdKentklZL2lTdzhug3r4pab2kdZoI1nBDvf2yJv40XCdpbfVzSdPvXaGvvrxvfF0WSIJv0AFJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEv8PTjgwm1gkiKQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 0\n"
     ]
    }
   ],
   "source": [
    "clean_preds = np.argmax(model.predict(x_test), axis=1)\n",
    "clean_correct = np.sum(clean_preds == np.argmax(y_test, axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nClean test set accuracy (model): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "# Display image, label, and prediction for a clean sample to show how the poisoned model classifies a clean sample\n",
    "\n",
    "c = 0 # class to display\n",
    "i = 0 # image of the class to display\n",
    "\n",
    "c_idx = np.where(np.argmax(y_test, 1) == c)[0][i] # index of the image in clean arrays\n",
    "\n",
    "plt.imshow(x_test[c_idx].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(clean_preds[c_idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2f29f09a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Clean test set accuracy (DPA model_10): 93.99%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOJElEQVR4nO3df4wc9XnH8c8n5rCpSVocfl3AKhBBGoIECScHQZtCUS1AbW2aQnHbyEmpTBKoEilVSigpUKWNRRuStkloLsGyG1FCWkAQiaZBLohGkTAHdWyDAVPqgLFrg6hqEwX7bD/948bkYm6/e96d3dnjeb+k0+7Os7Pz3OIPszffnfk6IgTgze8tTTcAoD8IO5AEYQeSIOxAEoQdSOKwfm7scM+OOZrbz00CqbymH2tP7PZUta7CbvsiSX8raZakb0TE8tLz52iu3u8Lu9kkgIJHYnXLWscf423PkvQVSRdLOl3SEtund/p6AHqrm7/ZF0h6NiKei4g9kr4laVE9bQGoWzdhP0HSC5Meb6mW/Qzby2yP2R4b1+4uNgegG92EfaqDAG/47m1EjEbESESMDGl2F5sD0I1uwr5F0vxJj0+UtLW7dgD0Sjdhf1TSqbZPtn24pCsk3VdPWwDq1vHQW0TstX2NpH/TxNDbioh4orbOANSqq3H2iLhf0v019QKgh/i6LJAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HXKZvTfrF/4+WL96S+fUqw/dcE3ivXrd5xdrK///dNa1vY9+UxxXdSLPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4+5vc/pNPLNbXn/+1Yn08yq//uWMfK9bPvPTclrX5jLP3VVdht71Z0i5J+yTtjYiROpoCUL869uwXRMTLNbwOgB7ib3YgiW7DHpK+Z/sx28umeoLtZbbHbI+Na3eXmwPQqW4/xp8XEVttHyvpAdtPRcTDk58QEaOSRiXpbZ7X5nAPgF7pas8eEVur2x2S7pG0oI6mANSv47Dbnmv7rQfuS1ooaUNdjQGoVzcf44+TdI/tA6/zTxHx3Vq6wiE5bH7rsfSTR5/tYycYZB2HPSKek3Rmjb0A6CGG3oAkCDuQBGEHkiDsQBKEHUiCU1xngOf/vPVpopJ09kVPtqzdPPwfdbdzSI4896WWtRc+W/69jl63t1g/4t41HfWUFXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYZYN1Vf1+sj8e+PnVy6B468/bWxTbnTN7z4+FifcWuxcX6Yf9evsx1NuzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtkHwNBD5fHkIc/qUyeH7j/37C/WN48f07J26dxXiutefuSOcv2bo8X6b5xwdrGeDXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY++MniBcX6R4b/uVhvd756L89nP2P1R4v1Y1bPLtZn/1/r3j5zfnlfs/6yvyvW29nymdbXpT/x8z/o6rVnorZ7dtsrbO+wvWHSsnm2H7C9qbo9qrdtAujWdD7Gr5R00UHLrpW0OiJOlbS6egxggLUNe0Q8LOng7zUukrSqur9K0uJ62wJQt04P0B0XEdskqbo9ttUTbS+zPWZ7bFy7O9wcgG71/Gh8RIxGxEhEjAypfDAHQO90Gvbttoclqbotn54EoHGdhv0+SUur+0sl3VtPOwB6pe04u+07JJ0v6WjbWyTdIGm5pG/bvlLS85Iu62WTg27We95VrH/ulvJ51yOH72m3hUPs6KfaXXv9+gc/WKy/+9NPFev7du485J4OeNem04r1Nb81p1hfMPu1Yv1fP3Zzy9rCOZ8urnvSX5WvOR+7Z97xp7Zhj4glLUoX1twLgB7i67JAEoQdSIKwA0kQdiAJwg4kwSmuNdh/ePltbD+01p0//NHB5yn91K7fPaK47mlb1hTrvZwMet+TzxTrH19ZPr127KovFevDs1r/7o9fWV73g3cvLdbjhxuL9UHEnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcfQa4bvtIsb7zj97esrZvy6a62+mbk+56uVj/7OJzivXlxz9aZzszHnt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY+GHLnl4KWpHXvizbPmLlj6UV2sXzYW/YX692871tvKtePX9zxSzeGPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4ew2e/tjPFevj0curr795bf7t1ufpS9K/HFO+5v14tB5nb/ff5B03FMsqj/APprZ7dtsrbO+wvWHSshttv2h7bfVzSW/bBNCt6XyMXylpqilHvhgRZ1U/99fbFoC6tQ17RDws6ZU+9AKgh7o5QHeN7XXVx/yjWj3J9jLbY7bHxrW7i80B6EanYb9V0jslnSVpm6QvtHpiRIxGxEhEjAxpdoebA9CtjsIeEdsjYl9E7Jf0dUkL6m0LQN06Crvt4UkPL5W0odVzAQyGtuPstu+QdL6ko21vkXSDpPNtnyUpJG2WdFXvWhx81//Kd5puYWAdNv/ElrVdZ7+juO4/fOSrdbfzujW75xTr3rO3Z9tuStuwR8SSKRbf1oNeAPQQX5cFkiDsQBKEHUiCsANJEHYgCU5xRU89edPxLWtPLPxyT7d916tHt6zd+ieXFdeds7F8+uxMxJ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1dGXpouFj//PBdferkjVa+eG7L2pzvvPnG0dthzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTDOXoNZLk/gO+TWUwdPx87fO6fjdW/6i/KFgC844rWOX1tq/7uVp0bu7n1pJ37txZ6+/kzDnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcvQbL7/ydYv3yK7/U1es//NdfKdbLY9ll49HxqtN8/c57a+eM1R8t1k/V4z3b9kzUds9ue77tB21vtP2E7U9Uy+fZfsD2pur2qN63C6BT0/kYv1fSpyLi3ZLOkXS17dMlXStpdUScKml19RjAgGob9ojYFhGPV/d3Sdoo6QRJiyStqp62StLiHvUIoAaHdIDO9kmS3ivpEUnHRcQ2aeJ/CJKObbHOMttjtsfGtbvLdgF0atpht32kpLskfTIidk53vYgYjYiRiBgZ0uxOegRQg2mF3faQJoJ+e0TcXS3ebnu4qg9L2tGbFgHUoe3Qm21Luk3Sxoi4ZVLpPklLJS2vbu/tSYczwCl3vlysr/mDOcX6gtndnWY6yNbsbv27j/7PrxbX/d+Pt57uWZJ+6b+fLdZ7N+g3M01nnP08SR+StN722mrZdZoI+bdtXynpeUnlCa8BNKpt2CPi+5Lconxhve0A6BW+LgskQdiBJAg7kARhB5Ig7EASjujxOY6TvM3z4v3OdwD/J4sWFOsv/Gb5UtTPXPy1Yr2Xp5G20+5S0md+9Y9b1ub/5Q/qbie9R2K1dsYrU46esWcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSS4lHQfHHHvmmL9tDZXAvjAkquL9aEPb29Z++577iyuu3DDFcX6/pVTXm3sddHqfMjKSWtfalnjfPP+Ys8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwPjvwJsL57AAIO5AFYQeSIOxAEoQdSIKwA0kQdiCJtmG3Pd/2g7Y32n7C9ieq5TfaftH22urnkt63C6BT07l4xV5Jn4qIx22/VdJjth+oal+MiL/pXXsA6jKd+dm3SdpW3d9le6OkE3rdGIB6HdLf7LZPkvReSY9Ui66xvc72CttHtVhnme0x22Pj2t1dtwA6Nu2w2z5S0l2SPhkROyXdKumdks7SxJ7/C1OtFxGjETESESNDmt19xwA6Mq2w2x7SRNBvj4i7JSkitkfEvojYL+nrksqzFwJo1HSOxlvSbZI2RsQtk5YPT3rapZI21N8egLpM52j8eZI+JGm97bXVsuskLbF9lqSQtFnSVT3oD0BNpnM0/vuSpjo/9v762wHQK3yDDkiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kERfp2y2/ZKkH01adLSkl/vWwKEZ1N4GtS+J3jpVZ2+/GBHHTFXoa9jfsHF7LCJGGmugYFB7G9S+JHrrVL9642M8kARhB5JoOuyjDW+/ZFB7G9S+JHrrVF96a/RvdgD90/SeHUCfEHYgiUbCbvsi20/bftb2tU300IrtzbbXV9NQjzXcywrbO2xvmLRsnu0HbG+qbqecY6+h3gZiGu/CNOONvndNT3/e97/Zbc+S9IykX5e0RdKjkpZExJN9baQF25sljURE41/AsP0BSa9K+seIOKNadrOkVyJiefU/yqMi4k8HpLcbJb3a9DTe1WxFw5OnGZe0WNKH1eB7V+jrcvXhfWtiz75A0rMR8VxE7JH0LUmLGuhj4EXEw5JeOWjxIkmrqvurNPGPpe9a9DYQImJbRDxe3d8l6cA0442+d4W++qKJsJ8g6YVJj7dosOZ7D0nfs/2Y7WVNNzOF4yJimzTxj0fSsQ33c7C203j300HTjA/Me9fJ9OfdaiLsU00lNUjjf+dFxPskXSzp6urjKqZnWtN498sU04wPhE6nP+9WE2HfImn+pMcnStraQB9Tioit1e0OSfdo8Kai3n5gBt3qdkfD/bxukKbxnmqacQ3Ae9fk9OdNhP1RSafaPtn24ZKukHRfA328ge251YET2Z4raaEGbyrq+yQtre4vlXRvg738jEGZxrvVNONq+L1rfPrziOj7j6RLNHFE/r8k/VkTPbTo6xRJP6x+nmi6N0l3aOJj3bgmPhFdKentklZL2lTdzhug3r4pab2kdZoI1nBDvf2yJv40XCdpbfVzSdPvXaGvvrxvfF0WSIJv0AFJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEv8PTjgwm1gkiKQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 0\n"
     ]
    }
   ],
   "source": [
    "clean_preds = np.argmax(dpa_model_10.predict(x_test), axis=1)\n",
    "clean_correct = np.sum(clean_preds == np.argmax(y_test, axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nClean test set accuracy (DPA model_10): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "# Display image, label, and prediction for a clean sample to show how the poisoned model classifies a clean sample\n",
    "\n",
    "c = 0 # class to display\n",
    "i = 0 # image of the class to display\n",
    "\n",
    "c_idx = np.where(np.argmax(y_test, 1) == c)[0][i] # index of the image in clean arrays\n",
    "\n",
    "plt.imshow(x_test[c_idx].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(clean_preds[c_idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "af7b0bdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Clean test set accuracy (DPA model_20): 91.28%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOJElEQVR4nO3df4wc9XnH8c8n5rCpSVocfl3AKhBBGoIECScHQZtCUS1AbW2aQnHbyEmpTBKoEilVSigpUKWNRRuStkloLsGyG1FCWkAQiaZBLohGkTAHdWyDAVPqgLFrg6hqEwX7bD/948bkYm6/e96d3dnjeb+k0+7Os7Pz3OIPszffnfk6IgTgze8tTTcAoD8IO5AEYQeSIOxAEoQdSOKwfm7scM+OOZrbz00CqbymH2tP7PZUta7CbvsiSX8raZakb0TE8tLz52iu3u8Lu9kkgIJHYnXLWscf423PkvQVSRdLOl3SEtund/p6AHqrm7/ZF0h6NiKei4g9kr4laVE9bQGoWzdhP0HSC5Meb6mW/Qzby2yP2R4b1+4uNgegG92EfaqDAG/47m1EjEbESESMDGl2F5sD0I1uwr5F0vxJj0+UtLW7dgD0Sjdhf1TSqbZPtn24pCsk3VdPWwDq1vHQW0TstX2NpH/TxNDbioh4orbOANSqq3H2iLhf0v019QKgh/i6LJAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HXKZvTfrF/4+WL96S+fUqw/dcE3ivXrd5xdrK///dNa1vY9+UxxXdSLPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4+5vc/pNPLNbXn/+1Yn08yq//uWMfK9bPvPTclrX5jLP3VVdht71Z0i5J+yTtjYiROpoCUL869uwXRMTLNbwOgB7ib3YgiW7DHpK+Z/sx28umeoLtZbbHbI+Na3eXmwPQqW4/xp8XEVttHyvpAdtPRcTDk58QEaOSRiXpbZ7X5nAPgF7pas8eEVur2x2S7pG0oI6mANSv47Dbnmv7rQfuS1ooaUNdjQGoVzcf44+TdI/tA6/zTxHx3Vq6wiE5bH7rsfSTR5/tYycYZB2HPSKek3Rmjb0A6CGG3oAkCDuQBGEHkiDsQBKEHUiCU1xngOf/vPVpopJ09kVPtqzdPPwfdbdzSI4896WWtRc+W/69jl63t1g/4t41HfWUFXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYZYN1Vf1+sj8e+PnVy6B468/bWxTbnTN7z4+FifcWuxcX6Yf9evsx1NuzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtkHwNBD5fHkIc/qUyeH7j/37C/WN48f07J26dxXiutefuSOcv2bo8X6b5xwdrGeDXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY++MniBcX6R4b/uVhvd756L89nP2P1R4v1Y1bPLtZn/1/r3j5zfnlfs/6yvyvW29nymdbXpT/x8z/o6rVnorZ7dtsrbO+wvWHSsnm2H7C9qbo9qrdtAujWdD7Gr5R00UHLrpW0OiJOlbS6egxggLUNe0Q8LOng7zUukrSqur9K0uJ62wJQt04P0B0XEdskqbo9ttUTbS+zPWZ7bFy7O9wcgG71/Gh8RIxGxEhEjAypfDAHQO90Gvbttoclqbotn54EoHGdhv0+SUur+0sl3VtPOwB6pe04u+07JJ0v6WjbWyTdIGm5pG/bvlLS85Iu62WTg27We95VrH/ulvJ51yOH72m3hUPs6KfaXXv9+gc/WKy/+9NPFev7du485J4OeNem04r1Nb81p1hfMPu1Yv1fP3Zzy9rCOZ8urnvSX5WvOR+7Z97xp7Zhj4glLUoX1twLgB7i67JAEoQdSIKwA0kQdiAJwg4kwSmuNdh/ePltbD+01p0//NHB5yn91K7fPaK47mlb1hTrvZwMet+TzxTrH19ZPr127KovFevDs1r/7o9fWV73g3cvLdbjhxuL9UHEnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcfQa4bvtIsb7zj97esrZvy6a62+mbk+56uVj/7OJzivXlxz9aZzszHnt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY+GHLnl4KWpHXvizbPmLlj6UV2sXzYW/YX692871tvKtePX9zxSzeGPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4ew2e/tjPFevj0curr795bf7t1ufpS9K/HFO+5v14tB5nb/ff5B03FMsqj/APprZ7dtsrbO+wvWHSshttv2h7bfVzSW/bBNCt6XyMXylpqilHvhgRZ1U/99fbFoC6tQ17RDws6ZU+9AKgh7o5QHeN7XXVx/yjWj3J9jLbY7bHxrW7i80B6EanYb9V0jslnSVpm6QvtHpiRIxGxEhEjAxpdoebA9CtjsIeEdsjYl9E7Jf0dUkL6m0LQN06Crvt4UkPL5W0odVzAQyGtuPstu+QdL6ko21vkXSDpPNtnyUpJG2WdFXvWhx81//Kd5puYWAdNv/ElrVdZ7+juO4/fOSrdbfzujW75xTr3rO3Z9tuStuwR8SSKRbf1oNeAPQQX5cFkiDsQBKEHUiCsANJEHYgCU5xRU89edPxLWtPLPxyT7d916tHt6zd+ieXFdeds7F8+uxMxJ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1dGXpouFj//PBdferkjVa+eG7L2pzvvPnG0dthzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTDOXoNZLk/gO+TWUwdPx87fO6fjdW/6i/KFgC844rWOX1tq/7uVp0bu7n1pJ37txZ6+/kzDnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcvQbL7/ydYv3yK7/U1es//NdfKdbLY9ll49HxqtN8/c57a+eM1R8t1k/V4z3b9kzUds9ue77tB21vtP2E7U9Uy+fZfsD2pur2qN63C6BT0/kYv1fSpyLi3ZLOkXS17dMlXStpdUScKml19RjAgGob9ojYFhGPV/d3Sdoo6QRJiyStqp62StLiHvUIoAaHdIDO9kmS3ivpEUnHRcQ2aeJ/CJKObbHOMttjtsfGtbvLdgF0atpht32kpLskfTIidk53vYgYjYiRiBgZ0uxOegRQg2mF3faQJoJ+e0TcXS3ebnu4qg9L2tGbFgHUoe3Qm21Luk3Sxoi4ZVLpPklLJS2vbu/tSYczwCl3vlysr/mDOcX6gtndnWY6yNbsbv27j/7PrxbX/d+Pt57uWZJ+6b+fLdZ7N+g3M01nnP08SR+StN722mrZdZoI+bdtXynpeUnlCa8BNKpt2CPi+5Lconxhve0A6BW+LgskQdiBJAg7kARhB5Ig7EASjujxOY6TvM3z4v3OdwD/J4sWFOsv/Gb5UtTPXPy1Yr2Xp5G20+5S0md+9Y9b1ub/5Q/qbie9R2K1dsYrU46esWcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSS4lHQfHHHvmmL9tDZXAvjAkquL9aEPb29Z++577iyuu3DDFcX6/pVTXm3sddHqfMjKSWtfalnjfPP+Ys8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwPjvwJsL57AAIO5AFYQeSIOxAEoQdSIKwA0kQdiCJtmG3Pd/2g7Y32n7C9ieq5TfaftH22urnkt63C6BT07l4xV5Jn4qIx22/VdJjth+oal+MiL/pXXsA6jKd+dm3SdpW3d9le6OkE3rdGIB6HdLf7LZPkvReSY9Ui66xvc72CttHtVhnme0x22Pj2t1dtwA6Nu2w2z5S0l2SPhkROyXdKumdks7SxJ7/C1OtFxGjETESESNDmt19xwA6Mq2w2x7SRNBvj4i7JSkitkfEvojYL+nrksqzFwJo1HSOxlvSbZI2RsQtk5YPT3rapZI21N8egLpM52j8eZI+JGm97bXVsuskLbF9lqSQtFnSVT3oD0BNpnM0/vuSpjo/9v762wHQK3yDDkiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kERfp2y2/ZKkH01adLSkl/vWwKEZ1N4GtS+J3jpVZ2+/GBHHTFXoa9jfsHF7LCJGGmugYFB7G9S+JHrrVL9642M8kARhB5JoOuyjDW+/ZFB7G9S+JHrrVF96a/RvdgD90/SeHUCfEHYgiUbCbvsi20/bftb2tU300IrtzbbXV9NQjzXcywrbO2xvmLRsnu0HbG+qbqecY6+h3gZiGu/CNOONvndNT3/e97/Zbc+S9IykX5e0RdKjkpZExJN9baQF25sljURE41/AsP0BSa9K+seIOKNadrOkVyJiefU/yqMi4k8HpLcbJb3a9DTe1WxFw5OnGZe0WNKH1eB7V+jrcvXhfWtiz75A0rMR8VxE7JH0LUmLGuhj4EXEw5JeOWjxIkmrqvurNPGPpe9a9DYQImJbRDxe3d8l6cA0442+d4W++qKJsJ8g6YVJj7dosOZ7D0nfs/2Y7WVNNzOF4yJimzTxj0fSsQ33c7C203j300HTjA/Me9fJ9OfdaiLsU00lNUjjf+dFxPskXSzp6urjKqZnWtN498sU04wPhE6nP+9WE2HfImn+pMcnStraQB9Tioit1e0OSfdo8Kai3n5gBt3qdkfD/bxukKbxnmqacQ3Ae9fk9OdNhP1RSafaPtn24ZKukHRfA328ge251YET2Z4raaEGbyrq+yQtre4vlXRvg738jEGZxrvVNONq+L1rfPrziOj7j6RLNHFE/r8k/VkTPbTo6xRJP6x+nmi6N0l3aOJj3bgmPhFdKentklZL2lTdzhug3r4pab2kdZoI1nBDvf2yJv40XCdpbfVzSdPvXaGvvrxvfF0WSIJv0AFJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEv8PTjgwm1gkiKQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 0\n"
     ]
    }
   ],
   "source": [
    "clean_preds = np.argmax(dpa_model_20.predict(x_test), axis=1)\n",
    "clean_correct = np.sum(clean_preds == np.argmax(y_test, axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nClean test set accuracy (DPA model_20): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "# Display image, label, and prediction for a clean sample to show how the poisoned model classifies a clean sample\n",
    "\n",
    "c = 0 # class to display\n",
    "i = 0 # image of the class to display\n",
    "\n",
    "c_idx = np.where(np.argmax(y_test, 1) == c)[0][i] # index of the image in clean arrays\n",
    "\n",
    "plt.imshow(x_test[c_idx].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(clean_preds[c_idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "74d91009",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Clean test set accuracy (DPA model_30): 89.75%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOJElEQVR4nO3df4wc9XnH8c8n5rCpSVocfl3AKhBBGoIECScHQZtCUS1AbW2aQnHbyEmpTBKoEilVSigpUKWNRRuStkloLsGyG1FCWkAQiaZBLohGkTAHdWyDAVPqgLFrg6hqEwX7bD/948bkYm6/e96d3dnjeb+k0+7Os7Pz3OIPszffnfk6IgTgze8tTTcAoD8IO5AEYQeSIOxAEoQdSOKwfm7scM+OOZrbz00CqbymH2tP7PZUta7CbvsiSX8raZakb0TE8tLz52iu3u8Lu9kkgIJHYnXLWscf423PkvQVSRdLOl3SEtund/p6AHqrm7/ZF0h6NiKei4g9kr4laVE9bQGoWzdhP0HSC5Meb6mW/Qzby2yP2R4b1+4uNgegG92EfaqDAG/47m1EjEbESESMDGl2F5sD0I1uwr5F0vxJj0+UtLW7dgD0Sjdhf1TSqbZPtn24pCsk3VdPWwDq1vHQW0TstX2NpH/TxNDbioh4orbOANSqq3H2iLhf0v019QKgh/i6LJAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HXKZvTfrF/4+WL96S+fUqw/dcE3ivXrd5xdrK///dNa1vY9+UxxXdSLPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4+5vc/pNPLNbXn/+1Yn08yq//uWMfK9bPvPTclrX5jLP3VVdht71Z0i5J+yTtjYiROpoCUL869uwXRMTLNbwOgB7ib3YgiW7DHpK+Z/sx28umeoLtZbbHbI+Na3eXmwPQqW4/xp8XEVttHyvpAdtPRcTDk58QEaOSRiXpbZ7X5nAPgF7pas8eEVur2x2S7pG0oI6mANSv47Dbnmv7rQfuS1ooaUNdjQGoVzcf44+TdI/tA6/zTxHx3Vq6wiE5bH7rsfSTR5/tYycYZB2HPSKek3Rmjb0A6CGG3oAkCDuQBGEHkiDsQBKEHUiCU1xngOf/vPVpopJ09kVPtqzdPPwfdbdzSI4896WWtRc+W/69jl63t1g/4t41HfWUFXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYZYN1Vf1+sj8e+PnVy6B468/bWxTbnTN7z4+FifcWuxcX6Yf9evsx1NuzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtkHwNBD5fHkIc/qUyeH7j/37C/WN48f07J26dxXiutefuSOcv2bo8X6b5xwdrGeDXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY++MniBcX6R4b/uVhvd756L89nP2P1R4v1Y1bPLtZn/1/r3j5zfnlfs/6yvyvW29nymdbXpT/x8z/o6rVnorZ7dtsrbO+wvWHSsnm2H7C9qbo9qrdtAujWdD7Gr5R00UHLrpW0OiJOlbS6egxggLUNe0Q8LOng7zUukrSqur9K0uJ62wJQt04P0B0XEdskqbo9ttUTbS+zPWZ7bFy7O9wcgG71/Gh8RIxGxEhEjAypfDAHQO90Gvbttoclqbotn54EoHGdhv0+SUur+0sl3VtPOwB6pe04u+07JJ0v6WjbWyTdIGm5pG/bvlLS85Iu62WTg27We95VrH/ulvJ51yOH72m3hUPs6KfaXXv9+gc/WKy/+9NPFev7du485J4OeNem04r1Nb81p1hfMPu1Yv1fP3Zzy9rCOZ8urnvSX5WvOR+7Z97xp7Zhj4glLUoX1twLgB7i67JAEoQdSIKwA0kQdiAJwg4kwSmuNdh/ePltbD+01p0//NHB5yn91K7fPaK47mlb1hTrvZwMet+TzxTrH19ZPr127KovFevDs1r/7o9fWV73g3cvLdbjhxuL9UHEnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcfQa4bvtIsb7zj97esrZvy6a62+mbk+56uVj/7OJzivXlxz9aZzszHnt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfY+GHLnl4KWpHXvizbPmLlj6UV2sXzYW/YX692871tvKtePX9zxSzeGPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4ew2e/tjPFevj0curr795bf7t1ufpS9K/HFO+5v14tB5nb/ff5B03FMsqj/APprZ7dtsrbO+wvWHSshttv2h7bfVzSW/bBNCt6XyMXylpqilHvhgRZ1U/99fbFoC6tQ17RDws6ZU+9AKgh7o5QHeN7XXVx/yjWj3J9jLbY7bHxrW7i80B6EanYb9V0jslnSVpm6QvtHpiRIxGxEhEjAxpdoebA9CtjsIeEdsjYl9E7Jf0dUkL6m0LQN06Crvt4UkPL5W0odVzAQyGtuPstu+QdL6ko21vkXSDpPNtnyUpJG2WdFXvWhx81//Kd5puYWAdNv/ElrVdZ7+juO4/fOSrdbfzujW75xTr3rO3Z9tuStuwR8SSKRbf1oNeAPQQX5cFkiDsQBKEHUiCsANJEHYgCU5xRU89edPxLWtPLPxyT7d916tHt6zd+ieXFdeds7F8+uxMxJ4dSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1dGXpouFj//PBdferkjVa+eG7L2pzvvPnG0dthzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTDOXoNZLk/gO+TWUwdPx87fO6fjdW/6i/KFgC844rWOX1tq/7uVp0bu7n1pJ37txZ6+/kzDnh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcvQbL7/ydYv3yK7/U1es//NdfKdbLY9ll49HxqtN8/c57a+eM1R8t1k/V4z3b9kzUds9ue77tB21vtP2E7U9Uy+fZfsD2pur2qN63C6BT0/kYv1fSpyLi3ZLOkXS17dMlXStpdUScKml19RjAgGob9ojYFhGPV/d3Sdoo6QRJiyStqp62StLiHvUIoAaHdIDO9kmS3ivpEUnHRcQ2aeJ/CJKObbHOMttjtsfGtbvLdgF0atpht32kpLskfTIidk53vYgYjYiRiBgZ0uxOegRQg2mF3faQJoJ+e0TcXS3ebnu4qg9L2tGbFgHUoe3Qm21Luk3Sxoi4ZVLpPklLJS2vbu/tSYczwCl3vlysr/mDOcX6gtndnWY6yNbsbv27j/7PrxbX/d+Pt57uWZJ+6b+fLdZ7N+g3M01nnP08SR+StN722mrZdZoI+bdtXynpeUnlCa8BNKpt2CPi+5Lconxhve0A6BW+LgskQdiBJAg7kARhB5Ig7EASjujxOY6TvM3z4v3OdwD/J4sWFOsv/Gb5UtTPXPy1Yr2Xp5G20+5S0md+9Y9b1ub/5Q/qbie9R2K1dsYrU46esWcHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSS4lHQfHHHvmmL9tDZXAvjAkquL9aEPb29Z++577iyuu3DDFcX6/pVTXm3sddHqfMjKSWtfalnjfPP+Ys8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwPjvwJsL57AAIO5AFYQeSIOxAEoQdSIKwA0kQdiCJtmG3Pd/2g7Y32n7C9ieq5TfaftH22urnkt63C6BT07l4xV5Jn4qIx22/VdJjth+oal+MiL/pXXsA6jKd+dm3SdpW3d9le6OkE3rdGIB6HdLf7LZPkvReSY9Ui66xvc72CttHtVhnme0x22Pj2t1dtwA6Nu2w2z5S0l2SPhkROyXdKumdks7SxJ7/C1OtFxGjETESESNDmt19xwA6Mq2w2x7SRNBvj4i7JSkitkfEvojYL+nrksqzFwJo1HSOxlvSbZI2RsQtk5YPT3rapZI21N8egLpM52j8eZI+JGm97bXVsuskLbF9lqSQtFnSVT3oD0BNpnM0/vuSpjo/9v762wHQK3yDDkiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kERfp2y2/ZKkH01adLSkl/vWwKEZ1N4GtS+J3jpVZ2+/GBHHTFXoa9jfsHF7LCJGGmugYFB7G9S+JHrrVL9642M8kARhB5JoOuyjDW+/ZFB7G9S+JHrrVF96a/RvdgD90/SeHUCfEHYgiUbCbvsi20/bftb2tU300IrtzbbXV9NQjzXcywrbO2xvmLRsnu0HbG+qbqecY6+h3gZiGu/CNOONvndNT3/e97/Zbc+S9IykX5e0RdKjkpZExJN9baQF25sljURE41/AsP0BSa9K+seIOKNadrOkVyJiefU/yqMi4k8HpLcbJb3a9DTe1WxFw5OnGZe0WNKH1eB7V+jrcvXhfWtiz75A0rMR8VxE7JH0LUmLGuhj4EXEw5JeOWjxIkmrqvurNPGPpe9a9DYQImJbRDxe3d8l6cA0442+d4W++qKJsJ8g6YVJj7dosOZ7D0nfs/2Y7WVNNzOF4yJimzTxj0fSsQ33c7C203j300HTjA/Me9fJ9OfdaiLsU00lNUjjf+dFxPskXSzp6urjKqZnWtN498sU04wPhE6nP+9WE2HfImn+pMcnStraQB9Tioit1e0OSfdo8Kai3n5gBt3qdkfD/bxukKbxnmqacQ3Ae9fk9OdNhP1RSafaPtn24ZKukHRfA328ge251YET2Z4raaEGbyrq+yQtre4vlXRvg738jEGZxrvVNONq+L1rfPrziOj7j6RLNHFE/r8k/VkTPbTo6xRJP6x+nmi6N0l3aOJj3bgmPhFdKentklZL2lTdzhug3r4pab2kdZoI1nBDvf2yJv40XCdpbfVzSdPvXaGvvrxvfF0WSIJv0AFJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEv8PTjgwm1gkiKQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 0\n"
     ]
    }
   ],
   "source": [
    "clean_preds = np.argmax(dpa_model_30.predict(x_test), axis=1)\n",
    "clean_correct = np.sum(clean_preds == np.argmax(y_test, axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nClean test set accuracy (DPA model_30): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "# Display image, label, and prediction for a clean sample to show how the poisoned model classifies a clean sample\n",
    "\n",
    "c = 0 # class to display\n",
    "i = 0 # image of the class to display\n",
    "\n",
    "c_idx = np.where(np.argmax(y_test, 1) == c)[0][i] # index of the image in clean arrays\n",
    "\n",
    "plt.imshow(x_test[c_idx].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(clean_preds[c_idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d44d4bf3",
   "metadata": {},
   "source": [
    "# Evaluate the performance of the trained models on poisoned data\n",
    "When the trigger is added, we see a shift in performance. The single model performs the worst as no defense is in place to mitigate the effect of the poisoned. The DPA models show some robustness to the poison as they partition the training data, which spreads the effect of the poison between models in the ensemble."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1cd59ae2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Poison test set accuracy (model): 0.16%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAANoElEQVR4nO3df6zddX3H8dfL/sJeYFKwtZZKFdFYndTlCppuSw1DAUOKUTaajLCEWbJBAovZRliMJFscIyJh05lU6awEYSoQiHZq07gRMla5kFIKZSuyDmvveoG6tQjctvS9P+6X5QL3fO7lfL/nfA99Px/JzTnn+z7f833n2/vq99zz+X7PxxEhAEe/N7XdAID+IOxAEoQdSIKwA0kQdiCJ2f3c2FzPi2M01M9NAqm8qF/pYIx7qlqtsNs+R9JNkmZJ+kZEXFd6/jEa0pk+q84mARRsic0da12/jbc9S9JXJZ0rabmkNbaXd/t6AHqrzt/sZ0h6IiKejIiDkm6XtLqZtgA0rU7Yl0j6+aTHu6tlr2B7re0R2yOHNF5jcwDqqBP2qT4EeM25txGxLiKGI2J4jubV2ByAOuqEfbekpZMenyxpT712APRKnbA/IOk02++0PVfSRZLuaaYtAE3reugtIg7bvkLSjzQx9LY+Ih5trDMAjao1zh4RGyVtbKgXAD3E6bJAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGpN2Wx7l6QDkl6SdDgihptoCkDzaoW98rGIeKaB1wHQQ7yNB5KoG/aQ9GPbD9peO9UTbK+1PWJ75JDGa24OQLfqvo1fGRF7bC+UtMn24xFx7+QnRMQ6Sesk6XgviJrbA9ClWkf2iNhT3Y5JukvSGU00BaB5XYfd9pDt416+L+njkrY31RiAZtV5G79I0l22X36db0fEDxvpCkDjug57RDwp6fQGewHQQwy9AUkQdiAJwg4kQdiBJAg7kEQTF8Kk8OxnP9qx9o6Lnyiu+/jYomL94PicYn3JbeX6/N3Pdawd2fpYcV3kwZEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnH2G/uxPv92x9umhX5ZXPrXmxleVy7sOP9+xdtPTH6u58Teun46d0rE2dMOvFdedvfnBpttpHUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEf2bpOV4L4gzfVbfttekX33mzI61Zz5Y/j/zhB3lffzL97lYn/vB/ynWr//AnR1rZ7/5heK6P3j+2GL9k/M7Xytf1wtxsFjfMj5UrK865lDX2373Dy4r1t+z9oGuX7tNW2Kz9se+KX+hOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJczz5DQ9/bUqjVe+3j662uv3vbqo61v1q5rLztfyl/5/31q97dRUczM/uFI8X60LbRYv3Ee+8o1n99bufv25+/q/xd/EejaY/sttfbHrO9fdKyBbY32d5Z3Z7Q2zYB1DWTt/HflHTOq5ZdLWlzRJwmaXP1GMAAmzbsEXGvpH2vWrxa0obq/gZJFzTbFoCmdfsB3aKIGJWk6nZhpyfaXmt7xPbIIY13uTkAdfX80/iIWBcRwxExPEfzer05AB10G/a9thdLUnU71lxLAHqh27DfI+mS6v4lku5uph0AvTLtOLvt2zTxzeUn2d4t6QuSrpP0HduXSnpK0oW9bBJlh/97b8fa0B2da5L00jSvPfS9Z7voqBl7//Cjxfr755Z/fb+0770da8v+4cniuoeL1TemacMeEWs6lN6Y30IBJMXpskAShB1IgrADSRB2IAnCDiTBJa5ozexTlhbrX7nmK8X6HM8q1r970+90rJ04en9x3aMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJxdrTm8T9ZUqx/eF55KutHD5ano17w2POvu6ejGUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcXb01PgnP9yx9tBnbpxm7fIMQn905ZXF+pv/9afTvH4uHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2dFTT53b+XhyrMvj6Gv+8+xiff4PHy7Wo1jNZ9oju+31tsdsb5+07Frbv7C9tfo5r7dtAqhrJm/jvynpnCmW3xgRK6qfjc22BaBp04Y9Iu6VtK8PvQDooTof0F1he1v1Nv+ETk+yvdb2iO2RQxqvsTkAdXQb9q9JOlXSCkmjkm7o9MSIWBcRwxExPGeaCxsA9E5XYY+IvRHxUkQckfR1SWc02xaApnUVdtuLJz38lKTtnZ4LYDBMO85u+zZJqySdZHu3pC9IWmV7hSaGMndJuqx3LWKQvem444r1i3/rvo61/UdeLK479sV3Fevzxh8o1vFK04Y9ItZMsfjmHvQCoIc4XRZIgrADSRB2IAnCDiRB2IEkuMQVtey89v3F+vdP+vuOtdU7P11cd95GhtaaxJEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1F//v7HynWt/3e3xbrPzt8qGPtub85ubjuPI0W63h9OLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsyc3e8nbi/WrPv+Pxfo8l3+FLnr44o61t/4T16v3E0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfajnGeX/4lP//7uYv3CY58t1m89sLBYX/T5zseTI8U10bRpj+y2l9r+ie0dth+1fWW1fIHtTbZ3Vrcn9L5dAN2aydv4w5I+FxHvk/QRSZfbXi7pakmbI+I0SZurxwAG1LRhj4jRiHioun9A0g5JSyStlrShetoGSRf0qEcADXhdH9DZXibpQ5K2SFoUEaPSxH8Ikqb84832WtsjtkcOabxmuwC6NeOw2z5W0h2SroqI/TNdLyLWRcRwRAzP0bxuegTQgBmF3fYcTQT91oi4s1q81/biqr5Y0lhvWgTQhGmH3mxb0s2SdkTElyeV7pF0iaTrqtu7e9Ih6jn9vcXyXy68pdbLf/WLFxbrb3n4/lqvj+bMZJx9paSLJT1ie2u17BpNhPw7ti+V9JSk8r86gFZNG/aIuE+SO5TParYdAL3C6bJAEoQdSIKwA0kQdiAJwg4kwSWuR4FZy9/Tsbb29nqnPyxff3mxvuyWf6v1+ugfjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7EeBx/+48xf7nj9/xl8qNKWT//lg+QkRtV4f/cORHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJz9DeDF888o1jeff0OhOr/ZZtBzP9qztVj/xNtXdPW6HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IImZzM++VNK3JL1N0hFJ6yLiJtvXSvqspKerp14TERt71Whme1bOKtbfMbv7sfRbDyws1ufsL1/PztXs3SmNpXc7jj6dmZxUc1jS5yLiIdvHSXrQ9qaqdmNEfKknnQFo1EzmZx+VNFrdP2B7h6QlvW4MQLNe19/stpdJ+pCkLdWiK2xvs73e9pTfjWR7re0R2yOHNF6vWwBdm3HYbR8r6Q5JV0XEfklfk3SqpBWaOPJPeYJ2RKyLiOGIGJ6jefU7BtCVGYXd9hxNBP3WiLhTkiJib0S8FBFHJH1dUvlqDQCtmjbsti3pZkk7IuLLk5YvnvS0T0na3nx7AJoyk0/jV0q6WNIjtrdWy66RtMb2Ck2MvuySdFkP+kNNf/3s8mL9/k8sK9Zj9JEGu8mjzmWqvbrEdSafxt8nyVOUGFMH3kA4gw5IgrADSRB2IAnCDiRB2IEkCDuQhKOPU+4e7wVxps/q2/aAbLbEZu2PfVMNlXNkB7Ig7EAShB1IgrADSRB2IAnCDiRB2IEk+jrObvtpSf81adFJkp7pWwOvz6D2Nqh9SfTWrSZ7OyUi3jpVoa9hf83G7ZGIGG6tgYJB7W1Q+5LorVv96o238UAShB1Iou2wr2t5+yWD2tug9iXRW7f60lurf7MD6J+2j+wA+oSwA0m0Enbb59j+d9tP2L66jR46sb3L9iO2t9oeabmX9bbHbG+ftGyB7U22d1a3U86x11Jv19r+RbXvtto+r6Xeltr+ie0dth+1fWW1vNV9V+irL/ut73+z254l6T8knS1pt6QHJK2JiMf62kgHtndJGo6I1k/AsP3bkp6T9K2I+EC17HpJ+yLiuuo/yhMi4s8HpLdrJT3X9jTe1WxFiydPMy7pAkl/oBb3XaGv31Uf9lsbR/YzJD0REU9GxEFJt0ta3UIfAy8i7pW071WLV0vaUN3foIlflr7r0NtAiIjRiHioun9A0svTjLe67wp99UUbYV8i6eeTHu/WYM33HpJ+bPtB22vbbmYKiyJiVJr45ZG0sOV+Xm3aabz76VXTjA/Mvutm+vO62gj7VN+PNUjjfysj4jcknSvp8urtKmZmRtN498sU04wPhG6nP6+rjbDvlrR00uOTJe1poY8pRcSe6nZM0l0avKmo9748g251O9ZyP/9vkKbxnmqacQ3Avmtz+vM2wv6ApNNsv9P2XEkXSbqnhT5ew/ZQ9cGJbA9J+rgGbyrqeyRdUt2/RNLdLfbyCoMyjXenacbV8r5rffrziOj7j6TzNPGJ/M8k/UUbPXTo612SHq5+Hm27N0m3aeJt3SFNvCO6VNKJkjZL2lndLhig3m6R9IikbZoI1uKWevtNTfxpuE3S1urnvLb3XaGvvuw3TpcFkuAMOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4v8A9RUAV2lh2uIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 9\n"
     ]
    }
   ],
   "source": [
    "not_target = np.logical_not(np.all(y_test == targets, axis=1))\n",
    "px_test, py_test = backdoor.poison(x_test[not_target], y_test[not_target])\n",
    "\n",
    "poison_preds = np.argmax(model.predict(px_test), axis=1)\n",
    "clean_correct = np.sum(poison_preds == np.argmax(y_test[not_target], axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nPoison test set accuracy (model): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "c = 0 # index to display\n",
    "plt.imshow(px_test[c].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(poison_preds[c]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "500e4154",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Poison test set accuracy (DPA model_10): 68.20%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAANoElEQVR4nO3df6zddX3H8dfL/sJeYFKwtZZKFdFYndTlCppuSw1DAUOKUTaajLCEWbJBAovZRliMJFscIyJh05lU6awEYSoQiHZq07gRMla5kFIKZSuyDmvveoG6tQjctvS9P+6X5QL3fO7lfL/nfA99Px/JzTnn+z7f833n2/vq99zz+X7PxxEhAEe/N7XdAID+IOxAEoQdSIKwA0kQdiCJ2f3c2FzPi2M01M9NAqm8qF/pYIx7qlqtsNs+R9JNkmZJ+kZEXFd6/jEa0pk+q84mARRsic0da12/jbc9S9JXJZ0rabmkNbaXd/t6AHqrzt/sZ0h6IiKejIiDkm6XtLqZtgA0rU7Yl0j6+aTHu6tlr2B7re0R2yOHNF5jcwDqqBP2qT4EeM25txGxLiKGI2J4jubV2ByAOuqEfbekpZMenyxpT712APRKnbA/IOk02++0PVfSRZLuaaYtAE3reugtIg7bvkLSjzQx9LY+Ih5trDMAjao1zh4RGyVtbKgXAD3E6bJAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGpN2Wx7l6QDkl6SdDgihptoCkDzaoW98rGIeKaB1wHQQ7yNB5KoG/aQ9GPbD9peO9UTbK+1PWJ75JDGa24OQLfqvo1fGRF7bC+UtMn24xFx7+QnRMQ6Sesk6XgviJrbA9ClWkf2iNhT3Y5JukvSGU00BaB5XYfd9pDt416+L+njkrY31RiAZtV5G79I0l22X36db0fEDxvpCkDjug57RDwp6fQGewHQQwy9AUkQdiAJwg4kQdiBJAg7kEQTF8Kk8OxnP9qx9o6Lnyiu+/jYomL94PicYn3JbeX6/N3Pdawd2fpYcV3kwZEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnH2G/uxPv92x9umhX5ZXPrXmxleVy7sOP9+xdtPTH6u58Teun46d0rE2dMOvFdedvfnBpttpHUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEf2bpOV4L4gzfVbfttekX33mzI61Zz5Y/j/zhB3lffzL97lYn/vB/ynWr//AnR1rZ7/5heK6P3j+2GL9k/M7Xytf1wtxsFjfMj5UrK865lDX2373Dy4r1t+z9oGuX7tNW2Kz9se+KX+hOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJczz5DQ9/bUqjVe+3j662uv3vbqo61v1q5rLztfyl/5/31q97dRUczM/uFI8X60LbRYv3Ee+8o1n99bufv25+/q/xd/EejaY/sttfbHrO9fdKyBbY32d5Z3Z7Q2zYB1DWTt/HflHTOq5ZdLWlzRJwmaXP1GMAAmzbsEXGvpH2vWrxa0obq/gZJFzTbFoCmdfsB3aKIGJWk6nZhpyfaXmt7xPbIIY13uTkAdfX80/iIWBcRwxExPEfzer05AB10G/a9thdLUnU71lxLAHqh27DfI+mS6v4lku5uph0AvTLtOLvt2zTxzeUn2d4t6QuSrpP0HduXSnpK0oW9bBJlh/97b8fa0B2da5L00jSvPfS9Z7voqBl7//Cjxfr755Z/fb+0770da8v+4cniuoeL1TemacMeEWs6lN6Y30IBJMXpskAShB1IgrADSRB2IAnCDiTBJa5ozexTlhbrX7nmK8X6HM8q1r970+90rJ04en9x3aMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJxdrTm8T9ZUqx/eF55KutHD5ano17w2POvu6ejGUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcXb01PgnP9yx9tBnbpxm7fIMQn905ZXF+pv/9afTvH4uHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2dFTT53b+XhyrMvj6Gv+8+xiff4PHy7Wo1jNZ9oju+31tsdsb5+07Frbv7C9tfo5r7dtAqhrJm/jvynpnCmW3xgRK6qfjc22BaBp04Y9Iu6VtK8PvQDooTof0F1he1v1Nv+ETk+yvdb2iO2RQxqvsTkAdXQb9q9JOlXSCkmjkm7o9MSIWBcRwxExPGeaCxsA9E5XYY+IvRHxUkQckfR1SWc02xaApnUVdtuLJz38lKTtnZ4LYDBMO85u+zZJqySdZHu3pC9IWmV7hSaGMndJuqx3LWKQvem444r1i3/rvo61/UdeLK479sV3Fevzxh8o1vFK04Y9ItZMsfjmHvQCoIc4XRZIgrADSRB2IAnCDiRB2IEkuMQVtey89v3F+vdP+vuOtdU7P11cd95GhtaaxJEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1F//v7HynWt/3e3xbrPzt8qGPtub85ubjuPI0W63h9OLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsyc3e8nbi/WrPv+Pxfo8l3+FLnr44o61t/4T16v3E0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfajnGeX/4lP//7uYv3CY58t1m89sLBYX/T5zseTI8U10bRpj+y2l9r+ie0dth+1fWW1fIHtTbZ3Vrcn9L5dAN2aydv4w5I+FxHvk/QRSZfbXi7pakmbI+I0SZurxwAG1LRhj4jRiHioun9A0g5JSyStlrShetoGSRf0qEcADXhdH9DZXibpQ5K2SFoUEaPSxH8Ikqb84832WtsjtkcOabxmuwC6NeOw2z5W0h2SroqI/TNdLyLWRcRwRAzP0bxuegTQgBmF3fYcTQT91oi4s1q81/biqr5Y0lhvWgTQhGmH3mxb0s2SdkTElyeV7pF0iaTrqtu7e9Ih6jn9vcXyXy68pdbLf/WLFxbrb3n4/lqvj+bMZJx9paSLJT1ie2u17BpNhPw7ti+V9JSk8r86gFZNG/aIuE+SO5TParYdAL3C6bJAEoQdSIKwA0kQdiAJwg4kwSWuR4FZy9/Tsbb29nqnPyxff3mxvuyWf6v1+ugfjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7EeBx/+48xf7nj9/xl8qNKWT//lg+QkRtV4f/cORHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJz9DeDF888o1jeff0OhOr/ZZtBzP9qztVj/xNtXdPW6HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IImZzM++VNK3JL1N0hFJ6yLiJtvXSvqspKerp14TERt71Whme1bOKtbfMbv7sfRbDyws1ufsL1/PztXs3SmNpXc7jj6dmZxUc1jS5yLiIdvHSXrQ9qaqdmNEfKknnQFo1EzmZx+VNFrdP2B7h6QlvW4MQLNe19/stpdJ+pCkLdWiK2xvs73e9pTfjWR7re0R2yOHNF6vWwBdm3HYbR8r6Q5JV0XEfklfk3SqpBWaOPJPeYJ2RKyLiOGIGJ6jefU7BtCVGYXd9hxNBP3WiLhTkiJib0S8FBFHJH1dUvlqDQCtmjbsti3pZkk7IuLLk5YvnvS0T0na3nx7AJoyk0/jV0q6WNIjtrdWy66RtMb2Ck2MvuySdFkP+kNNf/3s8mL9/k8sK9Zj9JEGu8mjzmWqvbrEdSafxt8nyVOUGFMH3kA4gw5IgrADSRB2IAnCDiRB2IEkCDuQhKOPU+4e7wVxps/q2/aAbLbEZu2PfVMNlXNkB7Ig7EAShB1IgrADSRB2IAnCDiRB2IEk+jrObvtpSf81adFJkp7pWwOvz6D2Nqh9SfTWrSZ7OyUi3jpVoa9hf83G7ZGIGG6tgYJB7W1Q+5LorVv96o238UAShB1Iou2wr2t5+yWD2tug9iXRW7f60lurf7MD6J+2j+wA+oSwA0m0Enbb59j+d9tP2L66jR46sb3L9iO2t9oeabmX9bbHbG+ftGyB7U22d1a3U86x11Jv19r+RbXvtto+r6Xeltr+ie0dth+1fWW1vNV9V+irL/ut73+z254l6T8knS1pt6QHJK2JiMf62kgHtndJGo6I1k/AsP3bkp6T9K2I+EC17HpJ+yLiuuo/yhMi4s8HpLdrJT3X9jTe1WxFiydPMy7pAkl/oBb3XaGv31Uf9lsbR/YzJD0REU9GxEFJt0ta3UIfAy8i7pW071WLV0vaUN3foIlflr7r0NtAiIjRiHioun9A0svTjLe67wp99UUbYV8i6eeTHu/WYM33HpJ+bPtB22vbbmYKiyJiVJr45ZG0sOV+Xm3aabz76VXTjA/Mvutm+vO62gj7VN+PNUjjfysj4jcknSvp8urtKmZmRtN498sU04wPhG6nP6+rjbDvlrR00uOTJe1poY8pRcSe6nZM0l0avKmo9748g251O9ZyP/9vkKbxnmqacQ3Avmtz+vM2wv6ApNNsv9P2XEkXSbqnhT5ew/ZQ9cGJbA9J+rgGbyrqeyRdUt2/RNLdLfbyCoMyjXenacbV8r5rffrziOj7j6TzNPGJ/M8k/UUbPXTo612SHq5+Hm27N0m3aeJt3SFNvCO6VNKJkjZL2lndLhig3m6R9IikbZoI1uKWevtNTfxpuE3S1urnvLb3XaGvvuw3TpcFkuAMOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4v8A9RUAV2lh2uIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 7\n"
     ]
    }
   ],
   "source": [
    "poison_preds = np.argmax(dpa_model_10.predict(px_test), axis=1)\n",
    "clean_correct = np.sum(poison_preds == np.argmax(y_test[not_target], axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nPoison test set accuracy (DPA model_10): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "c = 0 # index to display\n",
    "plt.imshow(px_test[c].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(poison_preds[c]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "54e42d16",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Poison test set accuracy (DPA model_20): 74.00%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAANoElEQVR4nO3df6zddX3H8dfL/sJeYFKwtZZKFdFYndTlCppuSw1DAUOKUTaajLCEWbJBAovZRliMJFscIyJh05lU6awEYSoQiHZq07gRMla5kFIKZSuyDmvveoG6tQjctvS9P+6X5QL3fO7lfL/nfA99Px/JzTnn+z7f833n2/vq99zz+X7PxxEhAEe/N7XdAID+IOxAEoQdSIKwA0kQdiCJ2f3c2FzPi2M01M9NAqm8qF/pYIx7qlqtsNs+R9JNkmZJ+kZEXFd6/jEa0pk+q84mARRsic0da12/jbc9S9JXJZ0rabmkNbaXd/t6AHqrzt/sZ0h6IiKejIiDkm6XtLqZtgA0rU7Yl0j6+aTHu6tlr2B7re0R2yOHNF5jcwDqqBP2qT4EeM25txGxLiKGI2J4jubV2ByAOuqEfbekpZMenyxpT712APRKnbA/IOk02++0PVfSRZLuaaYtAE3reugtIg7bvkLSjzQx9LY+Ih5trDMAjao1zh4RGyVtbKgXAD3E6bJAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGpN2Wx7l6QDkl6SdDgihptoCkDzaoW98rGIeKaB1wHQQ7yNB5KoG/aQ9GPbD9peO9UTbK+1PWJ75JDGa24OQLfqvo1fGRF7bC+UtMn24xFx7+QnRMQ6Sesk6XgviJrbA9ClWkf2iNhT3Y5JukvSGU00BaB5XYfd9pDt416+L+njkrY31RiAZtV5G79I0l22X36db0fEDxvpCkDjug57RDwp6fQGewHQQwy9AUkQdiAJwg4kQdiBJAg7kEQTF8Kk8OxnP9qx9o6Lnyiu+/jYomL94PicYn3JbeX6/N3Pdawd2fpYcV3kwZEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnH2G/uxPv92x9umhX5ZXPrXmxleVy7sOP9+xdtPTH6u58Teun46d0rE2dMOvFdedvfnBpttpHUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEf2bpOV4L4gzfVbfttekX33mzI61Zz5Y/j/zhB3lffzL97lYn/vB/ynWr//AnR1rZ7/5heK6P3j+2GL9k/M7Xytf1wtxsFjfMj5UrK865lDX2373Dy4r1t+z9oGuX7tNW2Kz9se+KX+hOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJczz5DQ9/bUqjVe+3j662uv3vbqo61v1q5rLztfyl/5/31q97dRUczM/uFI8X60LbRYv3Ee+8o1n99bufv25+/q/xd/EejaY/sttfbHrO9fdKyBbY32d5Z3Z7Q2zYB1DWTt/HflHTOq5ZdLWlzRJwmaXP1GMAAmzbsEXGvpH2vWrxa0obq/gZJFzTbFoCmdfsB3aKIGJWk6nZhpyfaXmt7xPbIIY13uTkAdfX80/iIWBcRwxExPEfzer05AB10G/a9thdLUnU71lxLAHqh27DfI+mS6v4lku5uph0AvTLtOLvt2zTxzeUn2d4t6QuSrpP0HduXSnpK0oW9bBJlh/97b8fa0B2da5L00jSvPfS9Z7voqBl7//Cjxfr755Z/fb+0770da8v+4cniuoeL1TemacMeEWs6lN6Y30IBJMXpskAShB1IgrADSRB2IAnCDiTBJa5ozexTlhbrX7nmK8X6HM8q1r970+90rJ04en9x3aMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJxdrTm8T9ZUqx/eF55KutHD5ano17w2POvu6ejGUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcXb01PgnP9yx9tBnbpxm7fIMQn905ZXF+pv/9afTvH4uHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2dFTT53b+XhyrMvj6Gv+8+xiff4PHy7Wo1jNZ9oju+31tsdsb5+07Frbv7C9tfo5r7dtAqhrJm/jvynpnCmW3xgRK6qfjc22BaBp04Y9Iu6VtK8PvQDooTof0F1he1v1Nv+ETk+yvdb2iO2RQxqvsTkAdXQb9q9JOlXSCkmjkm7o9MSIWBcRwxExPGeaCxsA9E5XYY+IvRHxUkQckfR1SWc02xaApnUVdtuLJz38lKTtnZ4LYDBMO85u+zZJqySdZHu3pC9IWmV7hSaGMndJuqx3LWKQvem444r1i3/rvo61/UdeLK479sV3Fevzxh8o1vFK04Y9ItZMsfjmHvQCoIc4XRZIgrADSRB2IAnCDiRB2IEkuMQVtey89v3F+vdP+vuOtdU7P11cd95GhtaaxJEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1F//v7HynWt/3e3xbrPzt8qGPtub85ubjuPI0W63h9OLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsyc3e8nbi/WrPv+Pxfo8l3+FLnr44o61t/4T16v3E0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfajnGeX/4lP//7uYv3CY58t1m89sLBYX/T5zseTI8U10bRpj+y2l9r+ie0dth+1fWW1fIHtTbZ3Vrcn9L5dAN2aydv4w5I+FxHvk/QRSZfbXi7pakmbI+I0SZurxwAG1LRhj4jRiHioun9A0g5JSyStlrShetoGSRf0qEcADXhdH9DZXibpQ5K2SFoUEaPSxH8Ikqb84832WtsjtkcOabxmuwC6NeOw2z5W0h2SroqI/TNdLyLWRcRwRAzP0bxuegTQgBmF3fYcTQT91oi4s1q81/biqr5Y0lhvWgTQhGmH3mxb0s2SdkTElyeV7pF0iaTrqtu7e9Ih6jn9vcXyXy68pdbLf/WLFxbrb3n4/lqvj+bMZJx9paSLJT1ie2u17BpNhPw7ti+V9JSk8r86gFZNG/aIuE+SO5TParYdAL3C6bJAEoQdSIKwA0kQdiAJwg4kwSWuR4FZy9/Tsbb29nqnPyxff3mxvuyWf6v1+ugfjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7EeBx/+48xf7nj9/xl8qNKWT//lg+QkRtV4f/cORHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJz9DeDF888o1jeff0OhOr/ZZtBzP9qztVj/xNtXdPW6HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IImZzM++VNK3JL1N0hFJ6yLiJtvXSvqspKerp14TERt71Whme1bOKtbfMbv7sfRbDyws1ufsL1/PztXs3SmNpXc7jj6dmZxUc1jS5yLiIdvHSXrQ9qaqdmNEfKknnQFo1EzmZx+VNFrdP2B7h6QlvW4MQLNe19/stpdJ+pCkLdWiK2xvs73e9pTfjWR7re0R2yOHNF6vWwBdm3HYbR8r6Q5JV0XEfklfk3SqpBWaOPJPeYJ2RKyLiOGIGJ6jefU7BtCVGYXd9hxNBP3WiLhTkiJib0S8FBFHJH1dUvlqDQCtmjbsti3pZkk7IuLLk5YvnvS0T0na3nx7AJoyk0/jV0q6WNIjtrdWy66RtMb2Ck2MvuySdFkP+kNNf/3s8mL9/k8sK9Zj9JEGu8mjzmWqvbrEdSafxt8nyVOUGFMH3kA4gw5IgrADSRB2IAnCDiRB2IEkCDuQhKOPU+4e7wVxps/q2/aAbLbEZu2PfVMNlXNkB7Ig7EAShB1IgrADSRB2IAnCDiRB2IEk+jrObvtpSf81adFJkp7pWwOvz6D2Nqh9SfTWrSZ7OyUi3jpVoa9hf83G7ZGIGG6tgYJB7W1Q+5LorVv96o238UAShB1Iou2wr2t5+yWD2tug9iXRW7f60lurf7MD6J+2j+wA+oSwA0m0Enbb59j+d9tP2L66jR46sb3L9iO2t9oeabmX9bbHbG+ftGyB7U22d1a3U86x11Jv19r+RbXvtto+r6Xeltr+ie0dth+1fWW1vNV9V+irL/ut73+z254l6T8knS1pt6QHJK2JiMf62kgHtndJGo6I1k/AsP3bkp6T9K2I+EC17HpJ+yLiuuo/yhMi4s8HpLdrJT3X9jTe1WxFiydPMy7pAkl/oBb3XaGv31Uf9lsbR/YzJD0REU9GxEFJt0ta3UIfAy8i7pW071WLV0vaUN3foIlflr7r0NtAiIjRiHioun9A0svTjLe67wp99UUbYV8i6eeTHu/WYM33HpJ+bPtB22vbbmYKiyJiVJr45ZG0sOV+Xm3aabz76VXTjA/Mvutm+vO62gj7VN+PNUjjfysj4jcknSvp8urtKmZmRtN498sU04wPhG6nP6+rjbDvlrR00uOTJe1poY8pRcSe6nZM0l0avKmo9748g251O9ZyP/9vkKbxnmqacQ3Avmtz+vM2wv6ApNNsv9P2XEkXSbqnhT5ew/ZQ9cGJbA9J+rgGbyrqeyRdUt2/RNLdLfbyCoMyjXenacbV8r5rffrziOj7j6TzNPGJ/M8k/UUbPXTo612SHq5+Hm27N0m3aeJt3SFNvCO6VNKJkjZL2lndLhig3m6R9IikbZoI1uKWevtNTfxpuE3S1urnvLb3XaGvvuw3TpcFkuAMOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4v8A9RUAV2lh2uIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 7\n"
     ]
    }
   ],
   "source": [
    "poison_preds = np.argmax(dpa_model_20.predict(px_test), axis=1)\n",
    "clean_correct = np.sum(poison_preds == np.argmax(y_test[not_target], axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nPoison test set accuracy (DPA model_20): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "c = 0 # index to display\n",
    "plt.imshow(px_test[c].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(poison_preds[c]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "89dc71c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Poison test set accuracy (DPA model_30): 74.41%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAANoElEQVR4nO3df6zddX3H8dfL/sJeYFKwtZZKFdFYndTlCppuSw1DAUOKUTaajLCEWbJBAovZRliMJFscIyJh05lU6awEYSoQiHZq07gRMla5kFIKZSuyDmvveoG6tQjctvS9P+6X5QL3fO7lfL/nfA99Px/JzTnn+z7f833n2/vq99zz+X7PxxEhAEe/N7XdAID+IOxAEoQdSIKwA0kQdiCJ2f3c2FzPi2M01M9NAqm8qF/pYIx7qlqtsNs+R9JNkmZJ+kZEXFd6/jEa0pk+q84mARRsic0da12/jbc9S9JXJZ0rabmkNbaXd/t6AHqrzt/sZ0h6IiKejIiDkm6XtLqZtgA0rU7Yl0j6+aTHu6tlr2B7re0R2yOHNF5jcwDqqBP2qT4EeM25txGxLiKGI2J4jubV2ByAOuqEfbekpZMenyxpT712APRKnbA/IOk02++0PVfSRZLuaaYtAE3reugtIg7bvkLSjzQx9LY+Ih5trDMAjao1zh4RGyVtbKgXAD3E6bJAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGpN2Wx7l6QDkl6SdDgihptoCkDzaoW98rGIeKaB1wHQQ7yNB5KoG/aQ9GPbD9peO9UTbK+1PWJ75JDGa24OQLfqvo1fGRF7bC+UtMn24xFx7+QnRMQ6Sesk6XgviJrbA9ClWkf2iNhT3Y5JukvSGU00BaB5XYfd9pDt416+L+njkrY31RiAZtV5G79I0l22X36db0fEDxvpCkDjug57RDwp6fQGewHQQwy9AUkQdiAJwg4kQdiBJAg7kEQTF8Kk8OxnP9qx9o6Lnyiu+/jYomL94PicYn3JbeX6/N3Pdawd2fpYcV3kwZEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnH2G/uxPv92x9umhX5ZXPrXmxleVy7sOP9+xdtPTH6u58Teun46d0rE2dMOvFdedvfnBpttpHUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEf2bpOV4L4gzfVbfttekX33mzI61Zz5Y/j/zhB3lffzL97lYn/vB/ynWr//AnR1rZ7/5heK6P3j+2GL9k/M7Xytf1wtxsFjfMj5UrK865lDX2373Dy4r1t+z9oGuX7tNW2Kz9se+KX+hOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJczz5DQ9/bUqjVe+3j662uv3vbqo61v1q5rLztfyl/5/31q97dRUczM/uFI8X60LbRYv3Ee+8o1n99bufv25+/q/xd/EejaY/sttfbHrO9fdKyBbY32d5Z3Z7Q2zYB1DWTt/HflHTOq5ZdLWlzRJwmaXP1GMAAmzbsEXGvpH2vWrxa0obq/gZJFzTbFoCmdfsB3aKIGJWk6nZhpyfaXmt7xPbIIY13uTkAdfX80/iIWBcRwxExPEfzer05AB10G/a9thdLUnU71lxLAHqh27DfI+mS6v4lku5uph0AvTLtOLvt2zTxzeUn2d4t6QuSrpP0HduXSnpK0oW9bBJlh/97b8fa0B2da5L00jSvPfS9Z7voqBl7//Cjxfr755Z/fb+0770da8v+4cniuoeL1TemacMeEWs6lN6Y30IBJMXpskAShB1IgrADSRB2IAnCDiTBJa5ozexTlhbrX7nmK8X6HM8q1r970+90rJ04en9x3aMRR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJxdrTm8T9ZUqx/eF55KutHD5ano17w2POvu6ejGUd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcXb01PgnP9yx9tBnbpxm7fIMQn905ZXF+pv/9afTvH4uHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2dFTT53b+XhyrMvj6Gv+8+xiff4PHy7Wo1jNZ9oju+31tsdsb5+07Frbv7C9tfo5r7dtAqhrJm/jvynpnCmW3xgRK6qfjc22BaBp04Y9Iu6VtK8PvQDooTof0F1he1v1Nv+ETk+yvdb2iO2RQxqvsTkAdXQb9q9JOlXSCkmjkm7o9MSIWBcRwxExPGeaCxsA9E5XYY+IvRHxUkQckfR1SWc02xaApnUVdtuLJz38lKTtnZ4LYDBMO85u+zZJqySdZHu3pC9IWmV7hSaGMndJuqx3LWKQvem444r1i3/rvo61/UdeLK479sV3Fevzxh8o1vFK04Y9ItZMsfjmHvQCoIc4XRZIgrADSRB2IAnCDiRB2IEkuMQVtey89v3F+vdP+vuOtdU7P11cd95GhtaaxJEdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnB1F//v7HynWt/3e3xbrPzt8qGPtub85ubjuPI0W63h9OLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMsyc3e8nbi/WrPv+Pxfo8l3+FLnr44o61t/4T16v3E0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfajnGeX/4lP//7uYv3CY58t1m89sLBYX/T5zseTI8U10bRpj+y2l9r+ie0dth+1fWW1fIHtTbZ3Vrcn9L5dAN2aydv4w5I+FxHvk/QRSZfbXi7pakmbI+I0SZurxwAG1LRhj4jRiHioun9A0g5JSyStlrShetoGSRf0qEcADXhdH9DZXibpQ5K2SFoUEaPSxH8Ikqb84832WtsjtkcOabxmuwC6NeOw2z5W0h2SroqI/TNdLyLWRcRwRAzP0bxuegTQgBmF3fYcTQT91oi4s1q81/biqr5Y0lhvWgTQhGmH3mxb0s2SdkTElyeV7pF0iaTrqtu7e9Ih6jn9vcXyXy68pdbLf/WLFxbrb3n4/lqvj+bMZJx9paSLJT1ie2u17BpNhPw7ti+V9JSk8r86gFZNG/aIuE+SO5TParYdAL3C6bJAEoQdSIKwA0kQdiAJwg4kwSWuR4FZy9/Tsbb29nqnPyxff3mxvuyWf6v1+ugfjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7EeBx/+48xf7nj9/xl8qNKWT//lg+QkRtV4f/cORHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJz9DeDF888o1jeff0OhOr/ZZtBzP9qztVj/xNtXdPW6HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IImZzM++VNK3JL1N0hFJ6yLiJtvXSvqspKerp14TERt71Whme1bOKtbfMbv7sfRbDyws1ufsL1/PztXs3SmNpXc7jj6dmZxUc1jS5yLiIdvHSXrQ9qaqdmNEfKknnQFo1EzmZx+VNFrdP2B7h6QlvW4MQLNe19/stpdJ+pCkLdWiK2xvs73e9pTfjWR7re0R2yOHNF6vWwBdm3HYbR8r6Q5JV0XEfklfk3SqpBWaOPJPeYJ2RKyLiOGIGJ6jefU7BtCVGYXd9hxNBP3WiLhTkiJib0S8FBFHJH1dUvlqDQCtmjbsti3pZkk7IuLLk5YvnvS0T0na3nx7AJoyk0/jV0q6WNIjtrdWy66RtMb2Ck2MvuySdFkP+kNNf/3s8mL9/k8sK9Zj9JEGu8mjzmWqvbrEdSafxt8nyVOUGFMH3kA4gw5IgrADSRB2IAnCDiRB2IEkCDuQhKOPU+4e7wVxps/q2/aAbLbEZu2PfVMNlXNkB7Ig7EAShB1IgrADSRB2IAnCDiRB2IEk+jrObvtpSf81adFJkp7pWwOvz6D2Nqh9SfTWrSZ7OyUi3jpVoa9hf83G7ZGIGG6tgYJB7W1Q+5LorVv96o238UAShB1Iou2wr2t5+yWD2tug9iXRW7f60lurf7MD6J+2j+wA+oSwA0m0Enbb59j+d9tP2L66jR46sb3L9iO2t9oeabmX9bbHbG+ftGyB7U22d1a3U86x11Jv19r+RbXvtto+r6Xeltr+ie0dth+1fWW1vNV9V+irL/ut73+z254l6T8knS1pt6QHJK2JiMf62kgHtndJGo6I1k/AsP3bkp6T9K2I+EC17HpJ+yLiuuo/yhMi4s8HpLdrJT3X9jTe1WxFiydPMy7pAkl/oBb3XaGv31Uf9lsbR/YzJD0REU9GxEFJt0ta3UIfAy8i7pW071WLV0vaUN3foIlflr7r0NtAiIjRiHioun9A0svTjLe67wp99UUbYV8i6eeTHu/WYM33HpJ+bPtB22vbbmYKiyJiVJr45ZG0sOV+Xm3aabz76VXTjA/Mvutm+vO62gj7VN+PNUjjfysj4jcknSvp8urtKmZmRtN498sU04wPhG6nP6+rjbDvlrR00uOTJe1poY8pRcSe6nZM0l0avKmo9748g251O9ZyP/9vkKbxnmqacQ3Avmtz+vM2wv6ApNNsv9P2XEkXSbqnhT5ew/ZQ9cGJbA9J+rgGbyrqeyRdUt2/RNLdLfbyCoMyjXenacbV8r5rffrziOj7j6TzNPGJ/M8k/UUbPXTo612SHq5+Hm27N0m3aeJt3SFNvCO6VNKJkjZL2lndLhig3m6R9IikbZoI1uKWevtNTfxpuE3S1urnvLb3XaGvvuw3TpcFkuAMOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1I4v8A9RUAV2lh2uIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: 7\n"
     ]
    }
   ],
   "source": [
    "poison_preds = np.argmax(dpa_model_30.predict(px_test), axis=1)\n",
    "clean_correct = np.sum(poison_preds == np.argmax(y_test[not_target], axis=1))\n",
    "clean_total = y_test.shape[0]\n",
    "\n",
    "clean_acc = clean_correct / clean_total\n",
    "print(\"\\nPoison test set accuracy (DPA model_30): %.2f%%\" % (clean_acc * 100))\n",
    "\n",
    "c = 0 # index to display\n",
    "plt.imshow(px_test[c].squeeze())\n",
    "plt.show()\n",
    "clean_label = c\n",
    "print(\"Prediction: \" + str(poison_preds[c]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3963c866",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}