MIA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy.random import seed\n",
    "seed(42)\n",
    "\n",
    "from absl import app\n",
    "from absl import flags\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from mia.estimators import ShadowModelBundle, AttackModelBundle, prepare_attack_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_CLASSES = 10\n",
    "WIDTH = 32\n",
    "HEIGHT = 32\n",
    "CHANNELS = 3\n",
    "SHADOW_DATASET_SIZE = 1000\n",
    "ATTACK_TEST_DATASET_SIZE = 4000\n",
    "\n",
    "\n",
    "target_epochs = 12\n",
    "attack_epochs = 12\n",
    "num_shadows = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data():\n",
    "    \"\"\"Prepare CIFAR10 data.\"\"\"\n",
    "    (X_train, y_train), (X_test, y_test) = tf.keras.datasets.cifar10.load_data()\n",
    "    y_train = tf.keras.utils.to_categorical(y_train)\n",
    "    y_test = tf.keras.utils.to_categorical(y_test)\n",
    "    X_train = X_train.astype(\"float32\")[:10000]\n",
    "    X_test = X_test.astype(\"float32\")\n",
    "    y_train = y_train.astype(\"float32\")[:10000]\n",
    "    y_test = y_test.astype(\"float32\")\n",
    "    X_train /= 255\n",
    "    X_test /= 255\n",
    "    return (X_train, y_train), (X_test, y_test)\n",
    "\n",
    "def target_model_fn():\n",
    "    \"\"\"The architecture of the target (victim) model.\n",
    "\n",
    "    The attack is white-box, hence the attacker is assumed to know this architecture too.\"\"\"\n",
    "\n",
    "    model = tf.keras.models.Sequential()\n",
    "\n",
    "    model.add(\n",
    "        layers.Conv2D(\n",
    "            32,\n",
    "            (3, 3),\n",
    "            activation=\"tanh\",\n",
    "            padding=\"same\",\n",
    "            input_shape=(WIDTH, HEIGHT, CHANNELS),\n",
    "        )\n",
    "    )\n",
    "    #model.add(layers.Conv2D(32, (3, 3), activation=\"tanh\"))\n",
    "    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    #model.add(layers.Dropout(0.25))\n",
    "\n",
    "    model.add(layers.Conv2D(64, (3, 3), activation=\"tanh\", padding=\"same\"))\n",
    "    #model.add(layers.Conv2D(64, (3, 3), activation=\"tanh\"))\n",
    "    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    #model.add(layers.Dropout(0.25))\n",
    "\n",
    "    model.add(layers.Flatten())\n",
    "\n",
    "    model.add(layers.Dense(128, activation=\"tanh\"))\n",
    "    #model.add(layers.Dropout(0.5))\n",
    "\n",
    "    model.add(layers.Dense(NUM_CLASSES, activation=\"softmax\"))\n",
    "    model.compile(\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "    return model\n",
    "\n",
    "def attack_model_fn():\n",
    "    \"\"\"Attack model that takes target model predictions and predicts membership.\n",
    "\n",
    "    Following the original paper, this attack model is specific to the class of the input.\n",
    "    AttachModelBundle creates multiple instances of this model for each class.\n",
    "    \"\"\"\n",
    "    model = tf.keras.models.Sequential()\n",
    "\n",
    "    model.add(layers.Dense(128, activation=\"relu\", input_shape=(NUM_CLASSES,)))\n",
    "\n",
    "    model.add(layers.Dropout(0.3, noise_shape=None, seed=None))\n",
    "    model.add(layers.Dense(64, activation=\"relu\"))\n",
    "    model.add(layers.Dropout(0.2, noise_shape=None, seed=None))\n",
    "    model.add(layers.Dense(64, activation=\"relu\"))\n",
    "\n",
    "    model.add(layers.Dense(1, activation=\"sigmoid\"))\n",
    "    model.compile(\"adam\", loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the target model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0623 11:20:37.991618 4531553728 deprecation.py:323] From /anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "W0623 11:20:38.274591 4531553728 deprecation.py:323] From /anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "10000/10000 [==============================] - 13s 1ms/sample - loss: 1.6717 - acc: 0.4079\n",
      "Epoch 2/12\n",
      "10000/10000 [==============================] - 14s 1ms/sample - loss: 1.2988 - acc: 0.5448\n",
      "Epoch 3/12\n",
      "10000/10000 [==============================] - 16s 2ms/sample - loss: 1.1126 - acc: 0.6136\n",
      "Epoch 4/12\n",
      "10000/10000 [==============================] - 13s 1ms/sample - loss: 0.9766 - acc: 0.6569\n",
      "Epoch 5/12\n",
      "10000/10000 [==============================] - 13s 1ms/sample - loss: 0.8535 - acc: 0.7009\n",
      "Epoch 6/12\n",
      "10000/10000 [==============================] - 12s 1ms/sample - loss: 0.7114 - acc: 0.7503\n",
      "Epoch 7/12\n",
      "10000/10000 [==============================] - 11s 1ms/sample - loss: 0.5691 - acc: 0.8085\n",
      "Epoch 8/12\n",
      "10000/10000 [==============================] - 12s 1ms/sample - loss: 0.4375 - acc: 0.8585\n",
      "Epoch 9/12\n",
      "10000/10000 [==============================] - 12s 1ms/sample - loss: 0.3035 - acc: 0.9140\n",
      "Epoch 10/12\n",
      "10000/10000 [==============================] - 12s 1ms/sample - loss: 0.1862 - acc: 0.9581\n",
      "Epoch 11/12\n",
      "10000/10000 [==============================] - 13s 1ms/sample - loss: 0.1136 - acc: 0.9820\n",
      "Epoch 12/12\n",
      "10000/10000 [==============================] - 18s 2ms/sample - loss: 0.0623 - acc: 0.9934\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1a2aff6320>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(X_train, y_train), (X_test, y_test) = get_data()\n",
    "\n",
    "# Train the target model.\n",
    "print(\"Training the target model...\")\n",
    "target_model = target_model_fn()\n",
    "target_model.fit(X_train, y_train, epochs=target_epochs, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_model.save(\"target_cifar10.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "target_model = load_model('target_cifar10.h5')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 32, 32, 3) (5000, 32, 32, 3)\n",
      "Training the shadow models...\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 2.1298 - acc: 0.2310 - val_loss: 1.8818 - val_acc: 0.3224\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.6593 - acc: 0.4110 - val_loss: 1.7882 - val_acc: 0.3696\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 1.4822 - acc: 0.4840 - val_loss: 1.7762 - val_acc: 0.3734\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.3516 - acc: 0.5310 - val_loss: 1.7583 - val_acc: 0.3836\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.1795 - acc: 0.6000 - val_loss: 1.7253 - val_acc: 0.3960\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.0594 - acc: 0.6320 - val_loss: 1.7342 - val_acc: 0.4178\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.9128 - acc: 0.6930 - val_loss: 1.7859 - val_acc: 0.4052\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.7468 - acc: 0.7600 - val_loss: 1.8749 - val_acc: 0.3992\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.6381 - acc: 0.8200 - val_loss: 1.8933 - val_acc: 0.4076\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 0.4759 - acc: 0.8720 - val_loss: 1.9910 - val_acc: 0.3960\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.3897 - acc: 0.8960 - val_loss: 2.1181 - val_acc: 0.3890\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.2817 - acc: 0.9440 - val_loss: 2.1293 - val_acc: 0.4032\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 2.1238 - acc: 0.2550 - val_loss: 2.0231 - val_acc: 0.2918\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 1.7432 - acc: 0.3910 - val_loss: 1.8090 - val_acc: 0.3640\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 1.4975 - acc: 0.4810 - val_loss: 1.7103 - val_acc: 0.4048\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.3413 - acc: 0.5560 - val_loss: 1.7213 - val_acc: 0.3874\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.1440 - acc: 0.6280 - val_loss: 1.6963 - val_acc: 0.4090\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.9641 - acc: 0.6920 - val_loss: 1.7405 - val_acc: 0.4156\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7906 - acc: 0.7460 - val_loss: 1.9654 - val_acc: 0.3718\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7559 - acc: 0.7530 - val_loss: 1.8768 - val_acc: 0.4070\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.5248 - acc: 0.8480 - val_loss: 1.8761 - val_acc: 0.4154\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.3914 - acc: 0.9120 - val_loss: 1.9399 - val_acc: 0.4246\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.2908 - acc: 0.9480 - val_loss: 2.1386 - val_acc: 0.4096\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.2046 - acc: 0.9740 - val_loss: 2.2069 - val_acc: 0.4116\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 2.1785 - acc: 0.2300 - val_loss: 1.9074 - val_acc: 0.3012\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.7602 - acc: 0.3830 - val_loss: 1.7725 - val_acc: 0.3654\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.5685 - acc: 0.4710 - val_loss: 1.7253 - val_acc: 0.3836\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 1.4309 - acc: 0.4950 - val_loss: 1.6638 - val_acc: 0.4106\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 5s 5ms/sample - loss: 1.2725 - acc: 0.5620 - val_loss: 1.6663 - val_acc: 0.4102\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.1883 - acc: 0.5970 - val_loss: 1.7759 - val_acc: 0.3922\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.0376 - acc: 0.6450 - val_loss: 1.7598 - val_acc: 0.3994\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.8469 - acc: 0.7380 - val_loss: 1.7651 - val_acc: 0.4224\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7079 - acc: 0.8000 - val_loss: 1.9012 - val_acc: 0.3930\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.6045 - acc: 0.8130 - val_loss: 1.8688 - val_acc: 0.4264\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.4810 - acc: 0.8720 - val_loss: 1.9592 - val_acc: 0.4176\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.3899 - acc: 0.9050 - val_loss: 1.9941 - val_acc: 0.4242\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 2.1639 - acc: 0.2130 - val_loss: 1.8895 - val_acc: 0.3168\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.7083 - acc: 0.3920 - val_loss: 1.7160 - val_acc: 0.3922\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.4907 - acc: 0.4910 - val_loss: 1.6822 - val_acc: 0.4060\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.3556 - acc: 0.5350 - val_loss: 1.6625 - val_acc: 0.4160\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.1913 - acc: 0.5810 - val_loss: 1.6871 - val_acc: 0.4096\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.0216 - acc: 0.6660 - val_loss: 1.7316 - val_acc: 0.4102\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.8819 - acc: 0.7110 - val_loss: 1.7900 - val_acc: 0.4148\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7190 - acc: 0.7730 - val_loss: 1.7565 - val_acc: 0.4322\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.5839 - acc: 0.8320 - val_loss: 1.8869 - val_acc: 0.4020\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.4624 - acc: 0.8800 - val_loss: 1.8925 - val_acc: 0.4310\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.3748 - acc: 0.9070 - val_loss: 2.0401 - val_acc: 0.4072\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.2646 - acc: 0.9480 - val_loss: 2.1022 - val_acc: 0.4238\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 2.2485 - acc: 0.2010 - val_loss: 1.8888 - val_acc: 0.3376\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.7563 - acc: 0.3900 - val_loss: 1.7734 - val_acc: 0.3758\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.5589 - acc: 0.4810 - val_loss: 1.7303 - val_acc: 0.3926\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.3966 - acc: 0.5330 - val_loss: 1.6956 - val_acc: 0.4088\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.2766 - acc: 0.5730 - val_loss: 1.8411 - val_acc: 0.3748\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.1667 - acc: 0.6140 - val_loss: 1.7101 - val_acc: 0.4208\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.0148 - acc: 0.6750 - val_loss: 1.7248 - val_acc: 0.4140\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.8397 - acc: 0.7460 - val_loss: 1.7551 - val_acc: 0.4218\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7306 - acc: 0.7640 - val_loss: 1.8112 - val_acc: 0.4214\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.5972 - acc: 0.8230 - val_loss: 1.8989 - val_acc: 0.4178\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.4650 - acc: 0.8740 - val_loss: 1.9786 - val_acc: 0.4182\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.3555 - acc: 0.9270 - val_loss: 2.0003 - val_acc: 0.4196\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 2.2084 - acc: 0.2200 - val_loss: 1.8727 - val_acc: 0.3340\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.6669 - acc: 0.4330 - val_loss: 1.7520 - val_acc: 0.3850\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.5027 - acc: 0.4850 - val_loss: 1.7586 - val_acc: 0.3882\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.3531 - acc: 0.5350 - val_loss: 1.7296 - val_acc: 0.3824\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.2241 - acc: 0.5680 - val_loss: 1.7303 - val_acc: 0.4048\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.0641 - acc: 0.6330 - val_loss: 1.6758 - val_acc: 0.4214\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.8899 - acc: 0.7110 - val_loss: 1.7114 - val_acc: 0.4168\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7546 - acc: 0.7540 - val_loss: 1.7847 - val_acc: 0.4182\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.5879 - acc: 0.8390 - val_loss: 1.8763 - val_acc: 0.4216\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.5275 - acc: 0.8550 - val_loss: 2.0058 - val_acc: 0.4054\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.4132 - acc: 0.8950 - val_loss: 2.0020 - val_acc: 0.4194\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.3039 - acc: 0.9320 - val_loss: 2.1310 - val_acc: 0.4064\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 2.1788 - acc: 0.2370 - val_loss: 1.9037 - val_acc: 0.3090\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.6732 - acc: 0.4040 - val_loss: 1.7539 - val_acc: 0.3746\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.4950 - acc: 0.4820 - val_loss: 1.9389 - val_acc: 0.3252\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.3324 - acc: 0.5390 - val_loss: 1.7415 - val_acc: 0.3986\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.1326 - acc: 0.6270 - val_loss: 1.7617 - val_acc: 0.4052\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.9760 - acc: 0.6720 - val_loss: 1.7531 - val_acc: 0.4012\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.8689 - acc: 0.7030 - val_loss: 1.8087 - val_acc: 0.3962\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.7133 - acc: 0.7690 - val_loss: 1.8978 - val_acc: 0.3980\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.5935 - acc: 0.8270 - val_loss: 1.9581 - val_acc: 0.4016\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.4543 - acc: 0.8790 - val_loss: 2.0292 - val_acc: 0.4046\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.3759 - acc: 0.9000 - val_loss: 2.1657 - val_acc: 0.3848\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.2868 - acc: 0.9320 - val_loss: 2.2255 - val_acc: 0.4036\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 2.3632 - acc: 0.1750 - val_loss: 1.9312 - val_acc: 0.2916\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.8469 - acc: 0.3220 - val_loss: 1.8121 - val_acc: 0.3456\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.6511 - acc: 0.4220 - val_loss: 1.7157 - val_acc: 0.3954\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.5098 - acc: 0.4760 - val_loss: 1.7704 - val_acc: 0.3684\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.3567 - acc: 0.5120 - val_loss: 1.6389 - val_acc: 0.4186\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.1940 - acc: 0.5870 - val_loss: 1.6595 - val_acc: 0.4200\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.0523 - acc: 0.6470 - val_loss: 1.7291 - val_acc: 0.3994\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.9162 - acc: 0.7150 - val_loss: 1.7368 - val_acc: 0.4116\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7897 - acc: 0.7530 - val_loss: 1.7906 - val_acc: 0.4140\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.6531 - acc: 0.8180 - val_loss: 1.8580 - val_acc: 0.4208\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.5176 - acc: 0.8600 - val_loss: 1.9567 - val_acc: 0.4100\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 0.3894 - acc: 0.9120 - val_loss: 1.9544 - val_acc: 0.4162\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 2.0964 - acc: 0.2460 - val_loss: 1.9009 - val_acc: 0.3290\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.7287 - acc: 0.3860 - val_loss: 1.7786 - val_acc: 0.3610\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.5161 - acc: 0.4750 - val_loss: 1.7431 - val_acc: 0.3938\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.3516 - acc: 0.5330 - val_loss: 1.7383 - val_acc: 0.3908\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.2245 - acc: 0.5730 - val_loss: 1.7448 - val_acc: 0.4012\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.0592 - acc: 0.6510 - val_loss: 1.7854 - val_acc: 0.4056\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.9693 - acc: 0.6580 - val_loss: 1.7964 - val_acc: 0.4172\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7662 - acc: 0.7470 - val_loss: 1.8619 - val_acc: 0.4068\n",
      "Epoch 9/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.6441 - acc: 0.7910 - val_loss: 1.8899 - val_acc: 0.4092\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.4827 - acc: 0.8750 - val_loss: 1.9734 - val_acc: 0.4212\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.3721 - acc: 0.9210 - val_loss: 2.0077 - val_acc: 0.4302\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.2812 - acc: 0.9480 - val_loss: 2.0918 - val_acc: 0.4104\n",
      "Train on 1000 samples, validate on 5000 samples\n",
      "Epoch 1/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 2.2044 - acc: 0.2250 - val_loss: 1.8673 - val_acc: 0.3386\n",
      "Epoch 2/12\n",
      "1000/1000 [==============================] - 4s 4ms/sample - loss: 1.7396 - acc: 0.3940 - val_loss: 1.6976 - val_acc: 0.4032\n",
      "Epoch 3/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.5032 - acc: 0.4830 - val_loss: 1.6448 - val_acc: 0.4166\n",
      "Epoch 4/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.3306 - acc: 0.5780 - val_loss: 1.6660 - val_acc: 0.4090\n",
      "Epoch 5/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.2046 - acc: 0.5800 - val_loss: 1.6819 - val_acc: 0.3940\n",
      "Epoch 6/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 1.0305 - acc: 0.6580 - val_loss: 1.7020 - val_acc: 0.4290\n",
      "Epoch 7/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.8714 - acc: 0.7100 - val_loss: 1.7478 - val_acc: 0.4098\n",
      "Epoch 8/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.7411 - acc: 0.7720 - val_loss: 1.7633 - val_acc: 0.4212\n",
      "Epoch 9/12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.6094 - acc: 0.8150 - val_loss: 1.8389 - val_acc: 0.4262\n",
      "Epoch 10/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.4876 - acc: 0.8690 - val_loss: 1.8642 - val_acc: 0.4288\n",
      "Epoch 11/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.3661 - acc: 0.9150 - val_loss: 1.9316 - val_acc: 0.4240\n",
      "Epoch 12/12\n",
      "1000/1000 [==============================] - 3s 3ms/sample - loss: 0.2521 - acc: 0.9590 - val_loss: 1.9792 - val_acc: 0.4356\n"
     ]
    }
   ],
   "source": [
    "# Train the shadow models.\n",
    "smb = ShadowModelBundle(\n",
    "    target_model_fn,\n",
    "    shadow_dataset_size=SHADOW_DATASET_SIZE,\n",
    "    num_models=num_shadows\n",
    ")\n",
    "\n",
    "# We assume that attacker's data were not seen in target's training.\n",
    "attacker_X_train, attacker_X_test, attacker_y_train, attacker_y_test = train_test_split(\n",
    "    X_test, y_test, test_size=0.5)\n",
    "print(attacker_X_train.shape, attacker_X_test.shape)\n",
    "\n",
    "print(\"Training the shadow models...\")\n",
    "X_shadow, y_shadow = smb.fit_transform(\n",
    "    attacker_X_train,\n",
    "    attacker_y_train,\n",
    "    fit_kwargs=dict(\n",
    "        epochs=target_epochs,\n",
    "        verbose=True,\n",
    "        validation_data=(attacker_X_test, attacker_y_test)\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0623 11:32:50.499297 4531553728 deprecation.py:506] From /anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the attack models...\n",
      "Epoch 1/12\n",
      "2082/2082 [==============================] - 1s 253us/sample - loss: 0.5789 - acc: 0.7502\n",
      "Epoch 2/12\n",
      "2082/2082 [==============================] - 0s 61us/sample - loss: 0.4755 - acc: 0.7795\n",
      "Epoch 3/12\n",
      "2082/2082 [==============================] - 0s 61us/sample - loss: 0.4718 - acc: 0.7829\n",
      "Epoch 4/12\n",
      "2082/2082 [==============================] - 0s 55us/sample - loss: 0.4714 - acc: 0.7800\n",
      "Epoch 5/12\n",
      "2082/2082 [==============================] - 0s 51us/sample - loss: 0.4682 - acc: 0.7786\n",
      "Epoch 6/12\n",
      "2082/2082 [==============================] - 0s 54us/sample - loss: 0.4649 - acc: 0.7776\n",
      "Epoch 7/12\n",
      "2082/2082 [==============================] - 0s 55us/sample - loss: 0.4666 - acc: 0.7824\n",
      "Epoch 8/12\n",
      "2082/2082 [==============================] - 0s 58us/sample - loss: 0.4655 - acc: 0.7776\n",
      "Epoch 9/12\n",
      "2082/2082 [==============================] - 0s 57us/sample - loss: 0.4681 - acc: 0.7781\n",
      "Epoch 10/12\n",
      "2082/2082 [==============================] - 0s 63us/sample - loss: 0.4634 - acc: 0.7805\n",
      "Epoch 11/12\n",
      "2082/2082 [==============================] - 0s 121us/sample - loss: 0.4646 - acc: 0.7805\n",
      "Epoch 12/12\n",
      "2082/2082 [==============================] - 0s 98us/sample - loss: 0.4631 - acc: 0.7781\n",
      "Epoch 1/12\n",
      "1947/1947 [==============================] - 1s 263us/sample - loss: 0.5558 - acc: 0.7689\n",
      "Epoch 2/12\n",
      "1947/1947 [==============================] - 0s 76us/sample - loss: 0.4252 - acc: 0.8079\n",
      "Epoch 3/12\n",
      "1947/1947 [==============================] - 0s 97us/sample - loss: 0.4233 - acc: 0.8084\n",
      "Epoch 4/12\n",
      "1947/1947 [==============================] - 0s 81us/sample - loss: 0.4131 - acc: 0.8120\n",
      "Epoch 5/12\n",
      "1947/1947 [==============================] - 0s 78us/sample - loss: 0.4191 - acc: 0.8141\n",
      "Epoch 6/12\n",
      "1947/1947 [==============================] - 0s 69us/sample - loss: 0.4128 - acc: 0.8130\n",
      "Epoch 7/12\n",
      "1947/1947 [==============================] - 0s 89us/sample - loss: 0.4134 - acc: 0.8141\n",
      "Epoch 8/12\n",
      "1947/1947 [==============================] - 0s 77us/sample - loss: 0.4109 - acc: 0.8161\n",
      "Epoch 9/12\n",
      "1947/1947 [==============================] - 0s 74us/sample - loss: 0.4101 - acc: 0.8146\n",
      "Epoch 10/12\n",
      "1947/1947 [==============================] - 0s 67us/sample - loss: 0.4118 - acc: 0.8177\n",
      "Epoch 11/12\n",
      "1947/1947 [==============================] - 0s 72us/sample - loss: 0.4074 - acc: 0.8177\n",
      "Epoch 12/12\n",
      "1947/1947 [==============================] - 0s 56us/sample - loss: 0.4101 - acc: 0.8161\n",
      "Epoch 1/12\n",
      "2024/2024 [==============================] - 0s 243us/sample - loss: 0.5351 - acc: 0.7974\n",
      "Epoch 2/12\n",
      "2024/2024 [==============================] - 0s 69us/sample - loss: 0.4213 - acc: 0.8261\n",
      "Epoch 3/12\n",
      "2024/2024 [==============================] - 0s 65us/sample - loss: 0.4108 - acc: 0.8241\n",
      "Epoch 4/12\n",
      "2024/2024 [==============================] - 0s 59us/sample - loss: 0.4130 - acc: 0.8197\n",
      "Epoch 5/12\n",
      "2024/2024 [==============================] - 0s 64us/sample - loss: 0.4073 - acc: 0.8256\n",
      "Epoch 6/12\n",
      "2024/2024 [==============================] - 0s 59us/sample - loss: 0.4059 - acc: 0.8231\n",
      "Epoch 7/12\n",
      "2024/2024 [==============================] - 0s 60us/sample - loss: 0.4070 - acc: 0.8266\n",
      "Epoch 8/12\n",
      "2024/2024 [==============================] - 0s 67us/sample - loss: 0.4066 - acc: 0.8271\n",
      "Epoch 9/12\n",
      "2024/2024 [==============================] - 0s 68us/sample - loss: 0.4045 - acc: 0.8291\n",
      "Epoch 10/12\n",
      "2024/2024 [==============================] - 0s 60us/sample - loss: 0.4015 - acc: 0.8251\n",
      "Epoch 11/12\n",
      "2024/2024 [==============================] - 0s 60us/sample - loss: 0.3989 - acc: 0.8256\n",
      "Epoch 12/12\n",
      "2024/2024 [==============================] - 0s 62us/sample - loss: 0.3995 - acc: 0.8266\n",
      "Epoch 1/12\n",
      "1998/1998 [==============================] - 0s 250us/sample - loss: 0.4932 - acc: 0.8363\n",
      "Epoch 2/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3291 - acc: 0.8744\n",
      "Epoch 3/12\n",
      "1998/1998 [==============================] - 0s 57us/sample - loss: 0.3291 - acc: 0.8774\n",
      "Epoch 4/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3231 - acc: 0.8764\n",
      "Epoch 5/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3198 - acc: 0.8814\n",
      "Epoch 6/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3122 - acc: 0.8769\n",
      "Epoch 7/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3209 - acc: 0.8814\n",
      "Epoch 8/12\n",
      "1998/1998 [==============================] - 0s 62us/sample - loss: 0.3172 - acc: 0.8809\n",
      "Epoch 9/12\n",
      "1998/1998 [==============================] - 0s 57us/sample - loss: 0.3181 - acc: 0.8809\n",
      "Epoch 10/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3110 - acc: 0.8824\n",
      "Epoch 11/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3077 - acc: 0.8854\n",
      "Epoch 12/12\n",
      "1998/1998 [==============================] - 0s 58us/sample - loss: 0.3133 - acc: 0.8839\n",
      "Epoch 1/12\n",
      "2048/2048 [==============================] - 1s 285us/sample - loss: 0.5299 - acc: 0.7837\n",
      "Epoch 2/12\n",
      "2048/2048 [==============================] - 0s 75us/sample - loss: 0.4116 - acc: 0.8223\n",
      "Epoch 3/12\n",
      "2048/2048 [==============================] - 0s 61us/sample - loss: 0.4078 - acc: 0.8281\n",
      "Epoch 4/12\n",
      "2048/2048 [==============================] - 0s 66us/sample - loss: 0.4070 - acc: 0.8257\n",
      "Epoch 5/12\n",
      "2048/2048 [==============================] - 0s 56us/sample - loss: 0.4058 - acc: 0.8301\n",
      "Epoch 6/12\n",
      "2048/2048 [==============================] - 0s 51us/sample - loss: 0.4026 - acc: 0.8267\n",
      "Epoch 7/12\n",
      "2048/2048 [==============================] - 0s 54us/sample - loss: 0.4005 - acc: 0.8223\n",
      "Epoch 8/12\n",
      "2048/2048 [==============================] - 0s 62us/sample - loss: 0.3993 - acc: 0.8218\n",
      "Epoch 9/12\n",
      "2048/2048 [==============================] - 0s 54us/sample - loss: 0.3952 - acc: 0.8232\n",
      "Epoch 10/12\n",
      "2048/2048 [==============================] - 0s 54us/sample - loss: 0.3964 - acc: 0.8271\n",
      "Epoch 11/12\n",
      "2048/2048 [==============================] - 0s 54us/sample - loss: 0.3990 - acc: 0.8281\n",
      "Epoch 12/12\n",
      "2048/2048 [==============================] - 0s 53us/sample - loss: 0.3958 - acc: 0.8252\n",
      "Epoch 1/12\n",
      "2029/2029 [==============================] - 0s 244us/sample - loss: 0.5259 - acc: 0.7950\n",
      "Epoch 2/12\n",
      "2029/2029 [==============================] - 0s 52us/sample - loss: 0.4061 - acc: 0.8314\n",
      "Epoch 3/12\n",
      "2029/2029 [==============================] - 0s 53us/sample - loss: 0.3951 - acc: 0.8295\n",
      "Epoch 4/12\n",
      "2029/2029 [==============================] - 0s 51us/sample - loss: 0.3896 - acc: 0.8305\n",
      "Epoch 5/12\n",
      "2029/2029 [==============================] - 0s 56us/sample - loss: 0.3930 - acc: 0.8280\n",
      "Epoch 6/12\n",
      "2029/2029 [==============================] - 0s 55us/sample - loss: 0.3893 - acc: 0.8305\n",
      "Epoch 7/12\n",
      "2029/2029 [==============================] - 0s 50us/sample - loss: 0.3869 - acc: 0.8324\n",
      "Epoch 8/12\n",
      "2029/2029 [==============================] - 0s 52us/sample - loss: 0.3900 - acc: 0.8275\n",
      "Epoch 9/12\n",
      "2029/2029 [==============================] - 0s 62us/sample - loss: 0.3853 - acc: 0.8344\n",
      "Epoch 10/12\n",
      "2029/2029 [==============================] - 0s 58us/sample - loss: 0.3915 - acc: 0.8265\n",
      "Epoch 11/12\n",
      "2029/2029 [==============================] - 0s 62us/sample - loss: 0.3818 - acc: 0.8324\n",
      "Epoch 12/12\n",
      "2029/2029 [==============================] - 0s 54us/sample - loss: 0.3845 - acc: 0.8314\n",
      "Epoch 1/12\n",
      "2031/2031 [==============================] - 1s 277us/sample - loss: 0.5902 - acc: 0.7125\n",
      "Epoch 2/12\n",
      "2031/2031 [==============================] - 0s 63us/sample - loss: 0.5018 - acc: 0.7637\n",
      "Epoch 3/12\n",
      "2031/2031 [==============================] - 0s 67us/sample - loss: 0.4916 - acc: 0.7720\n",
      "Epoch 4/12\n",
      "2031/2031 [==============================] - 0s 59us/sample - loss: 0.4872 - acc: 0.7715\n",
      "Epoch 5/12\n",
      "2031/2031 [==============================] - 0s 59us/sample - loss: 0.4829 - acc: 0.7770\n",
      "Epoch 6/12\n",
      "2031/2031 [==============================] - 0s 62us/sample - loss: 0.4881 - acc: 0.7760\n",
      "Epoch 7/12\n",
      "2031/2031 [==============================] - 0s 62us/sample - loss: 0.4822 - acc: 0.7750\n",
      "Epoch 8/12\n",
      "2031/2031 [==============================] - 0s 92us/sample - loss: 0.4837 - acc: 0.7725\n",
      "Epoch 9/12\n",
      "2031/2031 [==============================] - 0s 72us/sample - loss: 0.4788 - acc: 0.7750\n",
      "Epoch 10/12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2031/2031 [==============================] - 0s 55us/sample - loss: 0.4844 - acc: 0.7750\n",
      "Epoch 11/12\n",
      "2031/2031 [==============================] - 0s 72us/sample - loss: 0.4786 - acc: 0.7789\n",
      "Epoch 12/12\n",
      "2031/2031 [==============================] - 0s 65us/sample - loss: 0.4767 - acc: 0.7755\n",
      "Epoch 1/12\n",
      "2021/2021 [==============================] - 1s 267us/sample - loss: 0.5699 - acc: 0.7501\n",
      "Epoch 2/12\n",
      "2021/2021 [==============================] - 0s 60us/sample - loss: 0.4595 - acc: 0.7853\n",
      "Epoch 3/12\n",
      "2021/2021 [==============================] - 0s 61us/sample - loss: 0.4470 - acc: 0.7917\n",
      "Epoch 4/12\n",
      "2021/2021 [==============================] - 0s 59us/sample - loss: 0.4439 - acc: 0.7907\n",
      "Epoch 5/12\n",
      "2021/2021 [==============================] - 0s 60us/sample - loss: 0.4442 - acc: 0.7952\n",
      "Epoch 6/12\n",
      "2021/2021 [==============================] - 0s 60us/sample - loss: 0.4424 - acc: 0.7976\n",
      "Epoch 7/12\n",
      "2021/2021 [==============================] - 0s 60us/sample - loss: 0.4399 - acc: 0.7912\n",
      "Epoch 8/12\n",
      "2021/2021 [==============================] - 0s 60us/sample - loss: 0.4380 - acc: 0.7912\n",
      "Epoch 9/12\n",
      "2021/2021 [==============================] - 0s 60us/sample - loss: 0.4392 - acc: 0.7947\n",
      "Epoch 10/12\n",
      "2021/2021 [==============================] - 0s 65us/sample - loss: 0.4354 - acc: 0.7952\n",
      "Epoch 11/12\n",
      "2021/2021 [==============================] - 0s 59us/sample - loss: 0.4365 - acc: 0.7922\n",
      "Epoch 12/12\n",
      "2021/2021 [==============================] - 0s 61us/sample - loss: 0.4380 - acc: 0.7937\n",
      "Epoch 1/12\n",
      "2003/2003 [==============================] - 1s 276us/sample - loss: 0.5936 - acc: 0.7304\n",
      "Epoch 2/12\n",
      "2003/2003 [==============================] - 0s 61us/sample - loss: 0.4635 - acc: 0.7873\n",
      "Epoch 3/12\n",
      "2003/2003 [==============================] - 0s 60us/sample - loss: 0.4540 - acc: 0.7973\n",
      "Epoch 4/12\n",
      "2003/2003 [==============================] - 0s 60us/sample - loss: 0.4599 - acc: 0.7918\n",
      "Epoch 5/12\n",
      "2003/2003 [==============================] - 0s 61us/sample - loss: 0.4541 - acc: 0.7933\n",
      "Epoch 6/12\n",
      "2003/2003 [==============================] - 0s 60us/sample - loss: 0.4527 - acc: 0.7963\n",
      "Epoch 7/12\n",
      "2003/2003 [==============================] - 0s 61us/sample - loss: 0.4529 - acc: 0.7948\n",
      "Epoch 8/12\n",
      "2003/2003 [==============================] - 0s 61us/sample - loss: 0.4469 - acc: 0.7918\n",
      "Epoch 9/12\n",
      "2003/2003 [==============================] - 0s 60us/sample - loss: 0.4465 - acc: 0.7948\n",
      "Epoch 10/12\n",
      "2003/2003 [==============================] - 0s 73us/sample - loss: 0.4503 - acc: 0.7953\n",
      "Epoch 11/12\n",
      "2003/2003 [==============================] - 0s 71us/sample - loss: 0.4451 - acc: 0.7958\n",
      "Epoch 12/12\n",
      "2003/2003 [==============================] - 0s 63us/sample - loss: 0.4490 - acc: 0.7968\n",
      "Epoch 1/12\n",
      "1817/1817 [==============================] - 1s 337us/sample - loss: 0.5509 - acc: 0.7771\n",
      "Epoch 2/12\n",
      "1817/1817 [==============================] - 0s 70us/sample - loss: 0.4121 - acc: 0.8222\n",
      "Epoch 3/12\n",
      "1817/1817 [==============================] - 0s 74us/sample - loss: 0.4075 - acc: 0.8195\n",
      "Epoch 4/12\n",
      "1817/1817 [==============================] - 0s 73us/sample - loss: 0.4003 - acc: 0.8244\n",
      "Epoch 5/12\n",
      "1817/1817 [==============================] - 0s 82us/sample - loss: 0.3957 - acc: 0.8261\n",
      "Epoch 6/12\n",
      "1817/1817 [==============================] - 0s 70us/sample - loss: 0.3954 - acc: 0.8217\n",
      "Epoch 7/12\n",
      "1817/1817 [==============================] - 0s 59us/sample - loss: 0.3923 - acc: 0.8277\n",
      "Epoch 8/12\n",
      "1817/1817 [==============================] - 0s 65us/sample - loss: 0.3977 - acc: 0.8250\n",
      "Epoch 9/12\n",
      "1817/1817 [==============================] - 0s 68us/sample - loss: 0.3906 - acc: 0.8277\n",
      "Epoch 10/12\n",
      "1817/1817 [==============================] - 0s 64us/sample - loss: 0.3888 - acc: 0.8299\n",
      "Epoch 11/12\n",
      "1817/1817 [==============================] - 0s 55us/sample - loss: 0.3925 - acc: 0.8255\n",
      "Epoch 12/12\n",
      "1817/1817 [==============================] - 0s 61us/sample - loss: 0.3869 - acc: 0.8239\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "# ShadowModelBundle returns data in the format suitable for the AttackModelBundle.\n",
    "amb = AttackModelBundle(attack_model_fn, num_classes=NUM_CLASSES)\n",
    "\n",
    "# Fit the attack models.\n",
    "print(\"Training the attack models...\")\n",
    "amb.fit(\n",
    "    X_shadow, y_shadow, fit_kwargs=dict(epochs=attack_epochs, verbose=True)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'AttackModelBundle' object has no attribute 'save'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-e87d482a943b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mamb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'attack_model.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'AttackModelBundle' object has no attribute 'save'"
     ]
    }
   ],
   "source": [
    "amb.save('attack_model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.73125\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.99      0.47      0.63      4000\n",
      "        1.0       0.65      0.99      0.79      4000\n",
      "\n",
      "avg / total       0.82      0.73      0.71      8000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import auc\n",
    "\n",
    "# Test the success of the attack.\n",
    "# Prepare examples that were in the training, and out of the training.\n",
    "data_in = X_train[:ATTACK_TEST_DATASET_SIZE], y_train[:ATTACK_TEST_DATASET_SIZE]\n",
    "data_out = attacker_X_test[:ATTACK_TEST_DATASET_SIZE], attacker_y_test[:ATTACK_TEST_DATASET_SIZE]\n",
    "\n",
    "# Compile them into the expected format for the AttackModelBundle.\n",
    "attack_test_data, real_membership_labels = prepare_attack_data(\n",
    "    target_model, data_in, data_out\n",
    ")\n",
    "\n",
    "# Compute the attack accuracy.\n",
    "attack_guesses = amb.predict(attack_test_data)\n",
    "attack_accuracy = np.mean(attack_guesses == real_membership_labels)\n",
    "\n",
    "print(attack_accuracy)\n",
    "print (classification_report(real_membership_labels, attack_guesses))\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(real_membership_labels, attack_guesses)\n",
    "auc_non_federated = auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3Xl8TOcawPHfI7HXvhVB7BJBEPvSKrVULe2lpbbeBldVq9ttuW2VVt3a1b62Qu1arbZaXV2llIgIYksRQhCxC5HlvX/MJI1IZJDJZCbP9/OZT845c5bnRMwz73vOeR8xxqCUUkoB5HJ0AEoppbIPTQpKKaWSaVJQSimVTJOCUkqpZJoUlFJKJdOkoJRSKpkmBaWUUsk0KSiXIyLHReSGiFwTkTMislhEHkq1TnMR+VVErorIZRH5RkS8U61TWESmicgJ677CrPMls/aMlMo6mhSUq+pijHkI8AXqAyOT3hCRZsCPwNdAOaAysAfYKiJVrOvkAX4BagMdgcJAcyAaaGyvoEXE3V77VsoWmhSUSzPGnAE2YkkOSSYAS4wxnxhjrhpjLhhj3gW2A6Ot6/QHKgJPGWNCjTGJxphzxpgPjTEb0jqWiNQWkZ9E5IKInBWR/1iXLxaRsSnWe1REIlLMHxeRt0UkBLguIu+KyNpU+/5ERKZbp4uIyCIRiRSRUyIyVkTcHvBXpRSgSUG5OBHxADoBYdb5Ali+8a9JY/XVwOPW6XbAD8aYazYepxDwM/ADltZHNSwtDVv1BjoDRYGlwBMiUti6bzfgGWC5dd0AIN56jPpAe2DgPRxLqXRpUlCu6isRuQqcBM4B71uXF8fydx+ZxjaRQNL1ghLprJOeJ4EzxpjJxpib1hbIn/ew/XRjzEljzA1jTDgQBHS3vvcYEGOM2S4iZbAkuVeNMdeNMeeAqUCveziWUunSpKBcVXdjTCHgUaAWf3/YXwQSgbJpbFMWOG+djk5nnfRUAP66r0gtTqaaX46l9QDwHH+3EioBuYFIEbkkIpeAeUDpBzi2Usk0KSiXZoz5H7AYmGSdvw5sA3qmsfoz/N3l8zPQQUQK2niok0DVdN67DhRIMf9wWqGmml8DPGrt/nqKv5PCSSAWKGmMKWp9FTbG1LYxTqXuSpOCygmmAY+LSNLF5hHAABF5RUQKiUgx64XgZsAY6zpLsXwAfyEitUQkl4iUEJH/iMgTaRzjW+BhEXlVRPJa99vE+l4wlmsExUXkYeDVjAI2xkQBm4DPgGPGmAPW5ZFY7pyabL1lNpeIVBWRR+7j96LUHTQpKJdn/YBdArxnnd8CdACexnLdIBzLBduWxpgj1nVisVxsPgj8BFwBdmDphrrjWoEx5iqWi9RdgDPAEaCN9e2lWG55PY7lA32VjaEvt8awPNXy/kAeIBRLd9ha7q2rS6l0iRbZUUoplURbCkoppZJpUlBKKZVMk4JSSqlkmhSUUkolc7rBt0qWLGk8PT0dHYZSSjmVXbt2nTfGlMpoPadLCp6engQGBjo6DKWUcioiEm7Letp9pJRSKpkmBaWUUsk0KSillErmdNcU0hIXF0dERAQ3b950dCjKCeTLlw8PDw9y587t6FCUynZcIilERERQqFAhPD09ERFHh6OyMWMM0dHRREREULlyZUeHo1S2Y7fuIxH5VETOici+dN4XEZluLYYeIiIN7vdYN2/epESJEpoQVIZEhBIlSmirUql02POawmIsBc/T0wmobn0NBuY8yME0IShb6d+KUumzW/eRMWaziHjeZZVuWIqnG2C7iBQVkbLW8eKVUjmIMQZjINEYEg0Ybp9PtL5v7jL/97IU86RaJ/HOfZtUP9M8FknLLftIa98mg59J55V8rMSk/SbtJ+m9FMeyzsfeiuXa9Rj+0bQG9SoUteu/hSOvKZTn9hKEEdZldyQFERmMpTVBxYoVsyQ4pewh6mosy/88wY24hL8/nBL//mCBvz/Qkj4QUn9gpJy/7cMpnQ+rpA+d2z4sU314Wj6wUuw7MdV8UnwZfFAnGoCUH8JJ8dx+LFLPq4yZRGp4lHbppJBWGz7NPw9jzHxgPoCfn1+2/BN66KGHuHbtGgAbNmxg+PDh/PLLL1mWxHr06MGECROoUqVKlhzvXh07doxevXpx4cIFGjRowNKlS8mTJ89t6yxbtoyJEycmz4eEhBAUFISvry8dO3YkMjKS+Ph4WrVqxaxZs3Bzc2PNmjWMHj2aAwcOsGPHDvz8/ADYu3cvkydPZvHixVl5mnd1My4B/4CdhERcJo9bLkRABHKJkEvEMg/kyiXWZZauLoHb5+Xv+eTtbpu3bpPr7/lcSftOcSy3XIJ7ivlcqfb993ZCrlxJ80nxpNp3OvO37dt6brfN33aspPf+nk9z38n7SXHOkOZ53BlHqvPIlepYpDr3VD/vfozUvyPre7nub99XLl9mxIgRfLpwIVWrVWXRwoU80szT/n+oxtpMsccL8AT2pfPePKB3ivlDQNmM9tmwYUOTWmho6B3LslrBggWNMcb8/PPPpkqVKiYsLMzmbePi4h7o2Pv27TPdu3e/p23i4+Mf6Jj3qmfPnmbFihXGGGP+9a9/mdmzZ991/ZCQEFO5cuXk+cuXLxtjjElMTDRPP/108r5CQ0PNwYMHzSOPPGJ27tx52z7atm1rwsPD09x/Vv/NJCYmmmHLg4zniG/NT/vPZOmxlfOJj483Xl5eJleuXOatt94yMTExD7xPINDY8LntyJbCemCYiKwEmgCXTSZcTxjzzX5CT1954OBS8i5XmPe7ZFwX/ffff2fQoEFs2LCBqlUtNdyjoqIYMmQIJ06cAGDatGm0aNGC0aNHc/r0aY4fP07JkiUZN24c/fr14/r16wDMnDmT5s2bExkZybPPPsuVK1eIj49nzpw5tGrV6rbjLlu2jG7duiXPv/jii+zcuZMbN27Qo0cPxoyxlB329PTkhRde4Mcff2TYsGE0atSIl156iaioKAoUKMCCBQuoVasW33zzDWPHjuXWrVuUKFGCZcuWUaZMmfv+/Rlj+PXXX1m+3FJVcsCAAYwePZoXX3wx3W1WrFhB7969k+cLFy4MQHx8PLdu3Uq+WOzl5ZXuPrp06cLKlSt566237jv2zDJ70198s+c0b3WsSTvv+/9dKtcWHR1N8eLFcXNz46OPPqJChQrJrd+sYrekICIrgEeBkiISAbwP5AYwxswFNgBPAGFADPBPe8WSFWJjY+nWrRubNm2iVq1aycuHDx/Oa6+9RsuWLTlx4gQdOnTgwIEDAOzatYstW7aQP39+YmJi+Omnn8iXLx9Hjhyhd+/eBAYGsnz5cjp06MA777xDQkICMTExdxx769att32AfvTRRxQvXpyEhATatm1LSEgIdevWBSwPbm3ZsgWAtm3bMnfuXKpXr86ff/7J0KFD+fXXX2nZsiXbt29HRFi4cCETJkxg8uTJtx3z0KFDPPvss2n+LjZt2kTRon/3e0ZHR1O0aFHc3S1/bh4eHpw6dequv89Vq1bx9ddf37asQ4cO7Nixg06dOtGjR4+7bg/g5+fHxx9/7PCk8OP+M0zceIhuvuV48ZGqDo1FZU/GGJYtW8bw4cP5+OOPGTRoEE899ZRDYrHn3Ue9M3jfAC9l9nFt+UZvD7lz56Z58+YsWrSITz75JHn5zz//TGhoaPL8lStXuHr1KgBdu3Ylf/78gOWp7GHDhhEcHIybmxuHDx8GoFGjRrzwwgvExcXRvXt3fH197zh2ZGQkpUr9PSLu6tWrmT9/PvHx8URGRhIaGpqcFJI+yK9du8Yff/xBz549k7eLjY0FLA8DPvvss0RGRnLr1q00H/KqWbMmwcHBNv1ujLnzMtDdbgv9888/KVCgAD4+Prct37hxIzdv3qRPnz78+uuvPP7443c9bunSpTl9+rRNMdrLwTNXeHVVMPU8ijD+H3X1dlh1h5MnTzJkyBA2bNhA06ZNadGihUPj0bGPMkmuXLlYvXo1O3fuZNy4ccnLExMT2bZtG8HBwQQHB3Pq1CkKFSoEQMGCBZPXmzp1KmXKlGHPnj0EBgZy69YtAFq3bs3mzZspX748/fr1Y8mSJXccO3/+/MkPYx07doxJkybxyy+/EBISQufOnW97UCvpmImJiRQtWjQ5ruDg4OQWzMsvv8ywYcPYu3cv8+bNS/NBr0OHDuHr65vm69KlS7etW7JkSS5dukR8fDxgSTrlypVL93e5cuXK21o+KeXLl4+uXbve0YpIy82bN5OTriNEX4tlYEAgD+V1Z14/P/LldnNYLCp7WrFiBbVr12bTpk1MmzaNLVu24O3t7dCYNClkogIFCvDtt9+ybNkyFi1aBED79u2ZOXNm8jrpfbu+fPkyZcuWJVeuXCxdupSEhAQAwsPDKV26NIMGDcLf35+goKA7tvXy8iIsLAywtEQKFixIkSJFOHv2LN9//32axytcuDCVK1dmzZo1gOXb/J49e5JjKV++PAABAQFpbp/UUkjrlbLrCCytgjZt2rB27drkfaa8BpJSYmIia9asoVevXsnLrl27RmSk5XJTfHw8GzZsuK2LLj2HDx++o7WRVW7FJ/LisiDOXY1lfn8/Hi6SzyFxqOytWLFiNGnShH379jF8+HDc3Bz/xUGTQiYrXrw4P/zwA2PHjuXrr79m+vTpBAYGUrduXby9vZk7d26a2w0dOpSAgACaNm3K4cOHk7/Rb9q0CV9fX+rXr88XX3zB8OHD79i2c+fObNq0CYB69epRv359ateuzQsvvHDXpmhS8qpXrx61a9dO/vY9evRoevbsSatWrShZsuQD/kYsxo8fz5QpU6hWrRrR0dH4+/sDsH79ekaNGpW83ubNm/Hw8Ljt1trr16/TtWtX6tatS7169ShdujRDhgwBYN26dXh4eLBt2zY6d+5Mhw4dkrf77bff6Ny5c6bEfy+MMby/fj87jl1gYo+6+Nr5vnLlPOLj45k4cSIfffQRAB07duTHH3/MXuNw2XKLUnZ6ZddbUh0pJibGNGnSJMtvM83Obt68aZo0aZLu7b72/JtZvPWYqfT2t+bj7w/Y7RjK+QQHB5uGDRsawDzzzDMmMTExS4+PjbekakvBBeTPn58xY8ZkeEdPTnLixAk+/vjj5DuessrWsPN88G0o7bxK8+/2NbP02Cp7io2N5b333sPPz4+TJ0+yZs0aVq5cmW1vOnCJobPB0uLJrr/krJCy20RB9erVqV69eprvmTTuhsoMx85fZ+iyIKqWKsi0XvXJlSvn/j2qvx05coTx48fz3HPPMWXKFEqUKOHokO7KJVoK+fLlIzo62m7/2ZXrMMZSTyFfvsy98HvlZhwDA3aSS2Bh/0Y8lNdlvm+p+3Dt2jWWLVsGgI+PDwcPHiQgICDbJwRwkZaCh4cHERERREVFOToU5QSSKq9lloREwysrdhMeHcNS/yZULFEg0/atnM9PP/3E4MGDCQ8Pp0GDBnh5eWXbMcnS4hJJIXfu3Nnr6r3KUcb/cJBNh6IY292HZlWz/zdBZR8XL17kzTff5NNPP6VGjRr873//u+swLNmVSyQFpRxl7a4I5m8+Sr+mlejbtJKjw1EOkpCQQIsWLTh8+DAjR45k1KhRmd5FmVU0KSh1n3aFX+Q/X+6lWZUSjOri2KdQlWOcP38+eQC7cePGUbFiRRo0uO/KwtmCS1xoViqrnb50g38t3UXZovmY3acBud30v1JOYoxhyZIl1KhRg4ULFwLQvXt3p08IoElBqXt241YCg5cGcjMugYX9/ShWME/GGymXER4eTqdOnRgwYABeXl60bt3a0SFlKk0KSt0DYwz/XruH/aevML23L9XLFHJ0SCoLff755/j4+LBlyxZmzJjB77//btM4XM5ErykodQ9m/hrGtyGRjOhUi8dqabGcnKZUqVK0aNGCefPmUamSa95YoElBKRv9sO8Mk386zNP1y/Ov1s5z37m6f3FxcUyePJm4uDjee+89OnToQPv27V169ATtPlLKBgcir/D66mB8KxRl3NN1XPpDQVns3r2bJk2aMHLkSEJDQ5NHTHD1f3tNCkpl4Ly1WE7hfLmZ36+hFstxcTdv3uQ///kPjRo14vTp03zxxResWLHC5ZNBEk0KSt3FrfhEhn4exPlrsczv35DShZ3zgSRlu7CwMCZNmkT//v05cOAATz/9tKNDylJ6TUGpdBhjGPX1PnYcv8D03vWp66HFclzVtWvXWLduHf369cPHx4dDhw7l2KFztKWgVDoW/3GclTtP8lKbqnStl35NaeXcNm7cSO3atRkwYEBynfKcmhBAk4JSafr9SBQffhvK495leONxLZbjiqKjoxkwYAAdO3akQIEC/P777045gF1m0+4jpVI5GnWNl5YFUb10IaY+66vFclxQ0gB2YWFhvPPOO7z77rtOO4BdZtOkoFQKl2/EMXBJIO5uuVg4wE+L5biYqKgoSpQogZubG+PHj6dSpUr4+vo6OqxsRbuPlLJKKpZzIjqG2X0aUKG4FstxFcYYPvvsM2rUqMGCBQsA6NatmyaENGhSUMrqvxsO8L/DUXzQzYemVbRYjqs4fvw4HTp04IUXXqBOnTq0adPG0SFla5oUlALWBJ5k4ZZjDGhWieeaVHR0OCqTLF26FB8fH7Zt28bs2bPZtGkTNWrUcHRY2Zp2mKocb1f4Bd5Zt48W1Urw3pNaLMeVlClThtatWzN37lwqVtRkbwtJGs/DWfj5+ZnAwEBHh6FcxKlLN+g2cwsP5XXnq5daULSA1kZwZnFxcUyYMIGEhARGjRrl6HCyFRHZZYzxy2g97T5SOVbMrXgGBQQSG5fIwgF+mhCcXFBQEI0aNeLdd9/l0KFDONsX3uxCk4LKkRITDW+u2cOBM1eY3rs+1UprsRxndePGDUaMGEHjxo05e/Ys69atY9myZTlmALvMZtekICIdReSQiISJyIg03q8oIr+JyG4RCRGRJ+wZj1JJpv96hA17zzCyUy3a1Crt6HDUAzh69ChTpkzh+eefJzQ0lO7duzs6JKdmt6QgIm7ALKAT4A30FpHUV/HeBVYbY+oDvYDZ9opHqSTf741k2s9HeLpBeQa10mI5zujKlSssXrwYgNq1a3PkyBEWLlxIsWLFHBuYC7BnS6ExEGaMOWqMuQWsBLqlWscAha3TRYDTdoxHKfafvszrq/dQv2JRxj2lxXKc0YYNG/Dx8cHf3z95ADtXLY3pCPZMCuWBkynmI6zLUhoN9BWRCGAD8HJaOxKRwSISKCKBUVFR9ohV5QDnr8UyeMkuihbIzTwtluN0zp8/T79+/ejcuTOFChVi69atOoCdHdgzKaT1FSz17QC9gcXGGA/gCWCpiNwRkzFmvjHGzxjjV6pUKTuEqlxdbHwCQ5buIvp6LAv6+1G6kA5+5kySBrBbuXIlo0aNIigoiKZNmzo6LJdkz4fXIoAKKeY9uLN7yB/oCGCM2SYi+YCSwDk7xqVyGGMM7321j8Dwi8x8rj4+5Ys4OiRlo7Nnz1KqVCnc3NyYNGkSlSpVom7duo4Oy6XZs6WwE6guIpVFJA+WC8nrU61zAmgLICJeQD5A+4dUpvp063FWB0bwymPVeLKuFstxBsYYFi1aRM2aNZk/fz4AXbp00YSQBeyWFIwx8cAwYCNwAMtdRvtF5AMR6Wpd7Q1gkIjsAVYAzxt94kRlov8djuKj70LpULsMr7bTMW+cwdGjR2nXrh0DBw7E19eXdu3aOTqkHMWuYx8ZYzZguYCcctmoFNOhQAt7xqByrr+irjFseRA1yhRiyjNaLMcZBAQEMHToUNzc3Jg7dy6DBg0iVy59xjYr6YB4yiVdjoljUEAgeazFcgpqsRynUK5cOR577DHmzJmDh4eHo8PJkfR/inI58QmJDFsRxMmLMSwb2BSPYlosJ7u6desWH3/8MYmJiYwePZrHH3+cxx9/3NFh5WjaLlMuZ9yGg/x+5DwfdvOhceXijg5HpWPnzp00bNiQ999/n6NHj+oAdtmEJgXlUlbvPMmnW4/xfHNPejXW8fOzo5iYGN58802aNm3KxYsXWb9+PUuWLNGny7MJTQrKZew8foF3vtpLq+olebezPumaXR07dowZM2YwaNAg9u/fT5cuXRwdkkpBrykolxBxMYYhS3fhUawAM3s3wN1Nv+9kJ5cvX+bLL7/kn//8J7Vr1yYsLIwKFSpkvKHKcvo/Rzm967HxDFqyi1sJiSzo70eRArkdHZJK4bvvvqN27doMHDiQgwcPAmhCyMY0KSinlphoeGP1Hg6ducKM3vWpVvohR4ekrKKioujTpw9PPvkkxYoVY9u2bdSqVcvRYakMaPeRcmrTfjnCD/vP8G5nLx6tqcVysouEhARatmzJsWPHGDNmDCNGjCBPHi136gw0KSin9V1IJNN/OUKPhh74t6zs6HAUcObMGUqXLo2bmxuTJ0/G09MTHx8fR4el7oF2HymntO/UZd5YE0zDSsX46CkfvZ3RwRITE5k3bx41atRg3rx5ADz55JOaEJxQhklBRPKLyEgRmWudryYinewfmlJpO3f1JoOWBFK8QB7m9m1IXnctluNIYWFhtG3bliFDhtCoUSM6dOjg6JDUA7ClpfAploI5La3zp4FxdotIqbtIKpZzKSaO+f39KFUor6NDytE+++wz6tSpQ1BQEAsWLODnn3+mShWte+3MbEkK1Y0x44A4AGNMDGlXVVPKrowxvLNuH0EnLjH5mXpaLCcbqFixIh06dCA0NJSBAwdqN54LsOVC8y1rRTQDICKVgVt2jUqpNCzacoy1uyIY3rY6T9Qp6+hwcqTY2Fj++9//kpiYyAcffEDbtm1p27ato8NSmciWlsKHwA+Ah4gEAL8B/7FrVEql8tuhc4zbcIBOPg8zvG11R4eTI/355580bNiQMWPGcOLECR3AzkVlmBSMMd8DPYFBwDqgsTHmZ3sHplSSsHPXeGX5bmo9XJjJz9TTYjlZ7Pr167z++us0a9aMy5cv8+2337J48WLtKnJRttx99KMxJsoY87Ux5itjzDkR+TErglPqckwcg5YEkjd3LhYM8KNAHn20JquFh4cze/ZshgwZwv79++ncubOjQ1J2lO7/MBHJA+QDyohIIf6+uFwY0DGJld3FJyTy0vIgIi7GsGJQU8oXze/okHKMS5cusXbtWgYOHIi3tzdhYWFaCS2HuFtL4SVgP1DL+jPptRGYa//QVE439rsDbAk7z0dP1cHPU4vlZJWvv/4ab29vhgwZkjyAnSaEnCPdpGCMmWqMqQC8bYypaIypYH3VNsZMy8IYVQ60cscJFv9xHP+WlXnGT0fUzArnzp2jV69edO/enVKlSrF9+3YdwC4HyrCD1hgzTURqAd5YupOSli+3Z2Aq59px7ALvfb2P1jVKMbKTfihlhYSEBFq0aMGJEycYO3Ysb731Frlz6xDkOVGGSUFE3gXaY+lG2gh0ALYAmhRUpjt5IYYhn++iQrECzOhdX4vl2Nnp06d5+OGHcXNz45NPPsHT0xNvb29Hh6UcyJb/cc8CbYBIY0w/oB46uqqyA0uxnEDiEhJZMMCPIvn1m6q9JCYmMmfOHGrVqsXcuZZLhE888YQmBGVTUrhhjEkA4q13IZ0BdHATlakSEw2vrQrm8NmrzHquAVVLabEcezl8+DBt2rRh6NChNGnShE6ddHxL9TdbksJuESmKZWC8QGAHEGTXqFSOM/Xnw/wYepZ3OnvTukYpR4fjshYtWkS9evUICQnh008/5ccff6RyZa1Fof52124gsTyyONoYcwmYJSIbgcLGGE0KKtN8s+c0M34N4xk/D15o4enocFyap6cnnTp1YtasWZQtq+NHqTtJRuOXiMguY0zDLIonQ35+fiYwMNDRYahMsjfiMj3m/kGd8kVYNqiJ1kbIZLGxsXz44YcAjB071sHRKEeyfpb7ZbSeLd1HO0SkQSbEpNRtzl2xFMsp+VBe5vbTYjmZ7Y8//sDX15ePPvqIyMhIHcBO2cSWpNASS2I4JCJBIrJbRLT7SD2Qm3EJDF66i8s34pjfvyElH9JiOZnl2rVrDB8+nJYtWxITE8MPP/zAokWLdAA7ZRNbbi3tfr87F5GOwCeAG7DQGPNxGus8A4zGUq9hjzHmufs9nnIOxhj+8+Vegk9eYm7fBtQup8VyMtOJEyeYN28eL730EuPGjaNQoUKODkk5EVueaP7rfnYsIm7ALOBxIALYKSLrjTGhKdapDowEWhhjLopI6fs5lnIuC34/ype7T/Fauxp09NGLnZnh4sWLrFmzhsGDB+Pt7c3Ro0cpV66co8NSTsiej4s2BsKMMUeNMbeAlUC3VOsMAmYZYy4CGGPO2TEelQ38dvAc//3+IJ3rlOWVttUcHY5LWLduHd7e3gwdOpRDhw4BaEJQ982eSaE8cDLFfIR1WUo1gBoislVEtlu7m+4gIoNFJFBEAqOiouwUrrK3sHNXeWXFbrzLFmZSz3rax/2Azpw5Q8+ePXn66ad5+OGH2bFjBzVr1nR0WMrJ2TRchYh4ANWNMb+JSF7A3RhzPaPN0liW+vYHd6A68CjgAfwuIj7W5yL+3siY+cB8sNySakvMKnu5FHML/4BA8uZ2Y0F/P/Ln0TuNHkRCQgKtWrXi5MmTjBs3jjfffFMHsFOZwpYB8V4AhgFFgKpAJWA20C6DTSOAlGMeewCn01hnuzEmDjgmIoewJImdNkWvnEKctVhO5KWbrBjclHJaLOe+RUREUK5cOdzc3Jg+fTqVK1fW4a1VprKl++gVoClwBcAYcxiw5YLwTqC6iFS2VnHrBaxPtc5XWAbbQ0RKYulOOmpb6MpZjP02lK1h0Yx7ug4NKxVzdDhOKTExkRkzZlCrVi3mzJkDQKdOnTQhqExnS1K4ab1QDCTfVZRhZ7AxJh5LC2MjcABYbYzZLyIfiEhX62obgWgRCQV+A/5tjIm+15NQ2dfyP08QsC2cQa0q06OhVu+6HwcPHqR169a88sortGzZkieffNLRISkXZss1ha0i8haQT0TaYCnT+a0tOzfGbAA2pFo2KsW0AV63vpSL2X40mlFf7+PRmqUY0cnL0eE4pYULFzJs2DAKFChAQEAA/fr10wv0yq5saSm8BVwFDgLDgV8hBvH3AAAgAElEQVSAd+wZlHJ+Jy/E8OLnu6hUogDTe9fHLZd+kN2PqlWr0qVLFw4cOED//v01ISi7s2VAvC7AD9aLwQ6nA+Jlf9di4/nH7D84c+UmX73UgsolCzo6JKdx8+ZNPvjgAwDGjRvn4GiUK8nMAfGeAcJE5DMR6WC9pqBUmpKK5YRFXWPWcw00IdyDrVu34uvry3//+1+ioqJ0ADvlEBkmBWsJzhrAN8ALwFERmWvvwJRzmvzTIX4KPct7nb1oWb2ko8NxClevXuXll1+mVatWxMbGsnHjRhYsWKBdRcohbHqi2RgTC3wNLMZyq+kzdoxJOamvg08x67e/6NWoAgOaezo6HKcRERHBwoULefnll9m7dy/t27d3dEgqB8swKYhIOxFZCPwF9AWWAA/bOzDlXPacvMRba0No7FmcD7r56LfcDERHRyc/b+Dl5cXRo0f55JNPeOghrU2tHMuWlsIQ4AfAyxjTxxizPuVzC0qdvXKTwUstxXLm9G1AHnd7Dqnl3IwxrF27Fm9vb1555ZXkAey0NKbKLmy5ptDDGLPWGHMjKwJSziWpWM7Vm/EsHOBHCS2Wk67IyEj+8Y9/0LNnTypUqEBgYKAOYKeynXQfXhOR/xljHhGRi9w+kJ1gee6suN2jU9maMYYRX4Sw5+Ql5vZtiFfZwo4OKdtKGsDu1KlTTJgwgddeew13d5vGo1QqS93tr7KN9afeQqLSNPd/R/kq+DRvPF6Djj56mSktJ0+epHz58ri5uTFr1iwqV65MjRo1HB2WUulKt/vIGJNonVxkjElI+QIWZU14Krv6OfQsEzYe5Mm6ZRn2mBbLSS0hIYHp06ffNoBdhw4dNCGobM+W9mvdlDPWh9ca2Scc5QwOn73K8JW78SlXhIk9tFhOagcOHMDf359t27bRqVMnunTp4uiQlLJZui0FEXnbej2hrohcsL4uAlGkGuRO5RwXr99iYEAgBfK6M79/Qy2Wk8r8+fPx9fXl8OHDLF26lO+++46KFSs6OiylbHa3u48mAKWAqdafpYCSxpjixph/Z0VwKnuJS0hk6LIgzly5ybx+DSlbRIvlpFa9enWeeuopQkND6du3r7ailNO5W/dRNWPMERFZCtROWpj0R26MCbFzbCqb+eCbULYdjWbKM/VoUFGL5QDcuHGD0aNHIyJ8/PHHtGnThjZt2mS8oVLZ1N2SwgjAH5iVxnsGaG2XiFS29Pn2cJZuD+dfravwdAMtlgOwefNmBg4cyJEjRxgyZAjGGG0ZKKeXblIwxvhbf7bKunBUdrTtr2hGr9/PY7VK81ZHLf945coVRowYwZw5c6hSpQq//PILjz32mKPDUipT2DL20dMiUsg6PUJEVotIPfuHprKDE9ExvLhsF54lC/JJL18tlgOcPn2axYsX8/rrrxMSEqIJQbkUWwapGW2MuSoizYEuwCpgnn3DUtnB1ZtxDFyyE2NgYX8/CuXL7eiQHOb8+fPMnj0bgFq1anHs2DEmT55MwYJaL0K5FluSQoL155PAbGPMF4AOcOPiEqzFcv6Kus6cPg3wzKHFcowxrFq1Cm9vb1599VUOHz4MQJkyZRwcmVL2YUtSiBSRWUAvYIOI5LFxO+XEJv14iJ8PnOP9Lt40r5YzRzo5ffo03bt3p1evXlSqVIldu3bpE8nK5dnyRPMzwBPADGPMRREph+XOJOWivtp9ijmb/uK5JhXp17SSo8NxiISEBFq3bs2pU6eYNGkSw4cP1wHsVI6Q4V+5MeaaiIQCj4rIo8Dvxpjv7R6Zcojgk5d464sQmlQuzpiutXPcLZbh4eF4eHjg5ubG7NmzqVKlCtWq6dhOKuew5e6jYcBqoKL1tVpEhto7MJX1zly+yeAlgZQpnJc5fRuS2y3n9BImJCQwZcoUvLy8kgewa9++vSYElePY0h4eDDQ2xlwDEJFxwB/AbHsGprKWpVhOINdj41nq34LiBfM4OqQss2/fPvz9/dmxYwdPPvkk3bt3d3RISjmMLV8FBYhLMR9nXaZchDGGt9aGsPfUZaY+60vNhws5OqQsM3fuXBo0aMDRo0dZvnw569evx8NDn9hWOZctLYWlwHYR+QJLMugOBNg1KpWlZm/6i/V7TvPvDjVpXztnFMtJGpLCy8uLnj17Mm3aNEqVKuXosJRyODHGZLySSCMgabiL340xO+0a1V34+fmZwMBARx3e5fwUepbBSwPpUrccn/TydfkLyzExMYwaNQo3NzfGjx/v6HCUyjIisssY45fRerZeSYy1vm5YfyoXcPDMFV5duZs65YswoUddl08ImzZtom7dukyePJlr165hyxcipXIaW+4+egdYAZQFPIDlIjLS3oEp+7pgLZZTMK878/v5kS+36xbLuXz5Mv/617+Sh7T+9ddfmTVrlssnQaXuhy3XFPoCDY0xMQAi8hGwC/ivPQNT9nMrPpEXP9/FuauxrP5XMx4uks/RIdlVZGQkn3/+OW+++SZjxoyhQIECjg5JqWzLlu6jcG5PHu7AUVt2LiIdReSQiISJSLpPQYtIDxExIpJhf5d6MMYYRn+znz+PXWDCP+riW6Goo0Oyi6ioKGbMmAFYBrA7fvw4EydO1ISgVAZsSQoxwH4RWSgiC4C9wCURmSIiU9LbSETcsBTo6QR4A71FxDuN9QoBrwB/3s8JqHvz+fZwlv95giGPVKV7/fKODifTGWNYvnw5Xl5evPHGG8kD2OmdRUrZxpbuo++sryTbbdx3YyDMGHMUQERWAt2A0FTrfYilHvSbNu5X3ac/ws4z+ptQ2tYqzb871HR0OJnu5MmTvPjii3z33Xc0adKERYsW6QB2St0jW8Y+WnSf+y4PnEwxHwE0SbmCiNQHKhhjvhWRdJOCiAzG8mQ1FStWvM9wcrbw6OsMXR5E1VIFmeaCxXLi4+N59NFHOXPmDFOnTuXll1/Gzc11L54rZS/2HPYxrU+d5HsARSQXMBV4PqMdGWPmA/PB8pxCJsWXY1y9GYd/gOXZjoX9G7lUsZzjx49ToUIF3N3dmTdvHlWqVKFKlSqODkspp2XPEc8igAop5j2A0ynmCwE+wCYROQ40BdbrxebMlZBoGL4ymOPnrzO7TwMqlnCNC63x8fFMmjQJLy+v5Ipo7dq104Sg1AOyuaUgInmNMffy4NpOoLqIVAZOYSnS81zSm8aYy0By9RYR2QS8aYzRx5Uz0YSNB/n14Dk+7O5D86quUSwnJCQEf39/AgMD6datG//4xz8cHZJSLsOWh9cai8he4Ih1vp6IzMhoO2NMPDAM2AgcAFYbY/aLyAci0vUB41Y2+DIognn/O0rfpq5TLGf27Nk0bNiQ8PBwVq1axbp16yhXrpyjw1LKZdjSUpiOpT7zVwDGmD0i0saWnRtjNgAbUi0blc66j9qyT2Wb3ScuMuLLvTSrUoL3u9R2dDgPLGkAOx8fH3r16sXUqVMpWdI1Wj5KZSe2JIVcxpjwVEMCJNgpHpUJIi/fYPDSXTxcOB+z+zRw6mI5169f591338Xd3Z2JEyfSunVrWrdu7eiwlHJZtnxanBSRxoARETcReRU4bOe41H26cSuBwUt2ceNWAgsH+FHMiYvl/PLLL9SpU4dp06YRGxurA9gplQVsSQovAq9jKcV5FstdQi/aMyh1f4wxvPVFCPtOX+aTXr7UKOOcxXIuXbrEwIEDadeuHe7u7mzevJnp06frAHZKZQFbHl47h+XOIZXNzfotjG/2nObtjrVo61XG0eHct7Nnz7Jy5Urefvtt3n//ffLnz+/okJTKMTJMCtbxju5otxtjBtslInVfNu4/w6QfD/NU/fIMecT57tVPSgTDhw+nZs2aHD9+XC8kK+UAtnQf/Qz8Yn1tBUqjhXaylQORV3htVTD1KhTlv0/XcapuFmMMn3/+Od7e3rz11lscOXIEQBOCUg5iS/fRqpTzIrIU+MluEal7En0tloEBgRTK5878fg2dqljOiRMnGDJkCN9//z3NmjVj0aJFVK9e3dFhKZWj3c/YR5UB13gSyslZiuUEcf6apVhOmcLOUywnaQC7c+fOMX36dIYOHaoD2CmVDdhyTeEif19TyAVcANItmKOyhjGG99fvY8fxC3zSy5d6TlIs5+jRo1SqVAl3d3cWLFhA1apV8fT0dHRYSimru15TEEvndD2glPVVzBhTxRizOiuCU+kL+OM4K3acZOijVenmm/2L5cTHxzN+/Hi8vb2ZNWsWAG3bttWEoFQ2c9eWgjHGiMg6Y0zDrApIZWzLkfN8+N0B2nmV4c322b9YTnBwMP7+/gQFBfHUU0/Rs2dPR4eklEqHLXcf7RCRBnaPRNnk2PnrvLQ8iGqlHmJaL19yZfNiOTNnzqRRo0acOnWKtWvX8uWXX1K2bFlHh6WUSke6LQURcbeOdNoSGCQifwHXsRTPMcYYTRRZ7MrNOAYG7CSXwMIBfjyU1541kh5M0gB2devWpU+fPkyZMoXixYs7OiylVAbu9qmyA2gAdM+iWNRdJCQaXlmxm/DoGD4f2IQKxbNnsZxr167xzjvvkDt3biZNmqQD2CnlZO7WfSQAxpi/0nplUXzKavwPB9l0KIox3WrTtEoJR4eTph9//BEfHx9mzJhBXFycDmCnlBO6W0uhlIi8nt6bxpgpdohHpWHtrgjmbz5K/2aV6NMk+z0icvHiRV5//XUWL15MzZo12bx5My1btnR0WEqp+3C3loIb8BCWWsppvVQW2BV+kf98uZfmVUvw3pPejg4nTefOnWPt2rWMHDmS4OBgTQhKObG7tRQijTEfZFkk6g6nL93gX0t3UbZo9iuWc+bMGVasWMFrr72WPIBdiRLZs1tLKWW7DK8pKMe4cSuBwUsDuRmXwML+fhQtkD2K5RhjCAgIwNvbm5EjRyYPYKcJQSnXcLek0DbLolC3Mcbw5to97D99hRm961M9mxTLOX78OB07duT555/H29ub4OBgHcBOKReTbveRMeZCVgai/jbj1zC+C4lkZKdatKlV2tHhAJZhKtq0acP58+eZNWsWQ4YMIVeu7NOdpZTKHNn36acc6od9kUz56TBP1y/P4NaOL5YTFhZG5cqVcXd359NPP6VKlSpUqpT97oBSSmUO/aqXjYSevsJrq/ZQv2JRxjm4WE5cXBzjxo2jdu3ayQPYtWnTRhOCUi5OWwrZxPlrsQxaEkiR/LmZ19exxXKCgoLw9/cnODiYnj178uyzzzosFqVU1tKWQjZgKZazi+jrsSzo70dpBxbLmT59Oo0bN+bMmTN8+eWXrF69mjJlyjgsHqVU1tKk4GDGGN77ah87j19kYo961PEo4rA4AOrXr0///v0JDQ3lqaeeckgsSinH0e4jB/ts63FWBZ7k5ceq0aVeuSw//tWrVxk5ciR58+Zl8uTJtGrVilatWmV5HEqp7EFbCg60+XAUY78Lpb13GV5rVyPLj//DDz/g4+PD7NmzMcboAHZKKU0KjnI06hrDlgdRo0whpj6btcVyoqOjGTBgAJ06daJgwYJs3bqVKVOmOPRuJ6VU9qBJwQEu34hjYEAg7m65WNDfj4JZXCwnOjqadevW8d5777F7926aNWuWpcdXSmVfdk0KItJRRA6JSJiIjEjj/ddFJFREQkTkFxFx+Zvg4xMSeXnFbk5ciGFOnwZZViwnMjKSSZMmYYyhRo0ahIeH88EHH5A3b94sOb5SyjnYLSmIiBswC+gEeAO9RST12M+7AT9jTF1gLTDBXvFkFx9/f5DNh6P4sLsPTbKgWI4xhk8//RQvLy/ee+89wsLCAChWrJjdj62Ucj72bCk0BsKMMUeNMbeAlUC3lCsYY34zxsRYZ7cDHnaMx+HWBJ5k4ZZjPN/ck96NK9r9eMeOHaN9+/b4+/tTr1499uzZowPYKaXuyp6d2eWBkynmI4Amd1nfH/g+rTdEZDAwGKBiRft/mNrDrvALvLNuHy2rleTdzl52P158fDyPPfYY0dHRzJkzh8GDB+sAdkqpDNkzKaR1K0ua9zyKSF/AD3gkrfeNMfOB+QB+fn5Od9/kKWuxnHJF8zHzufq427FYzpEjR6hSpQru7u589tlnVK1alQoVKtjteEop12LPr44RQMpPIw/gdOqVRKQd8A7Q1RgTa8d4HCLmVjyDAgKJjUtk4YBGdiuWExcXx9ixY/Hx8WHmzJkAPProo5oQlFL3xJ4thZ1AdRGpDJwCegHPpVxBROoD84COxphzdozFIRITDW+u2cPBM1dY9HwjqpV+yC7HCQwMxN/fn5CQEHr16kXv3r3tchyllOuzW0vBGBMPDAM2AgeA1caY/SLygYh0ta42EXgIWCMiwSKy3l7xOML0X4+wYe8ZRnbyok1N+xTL+eSTT2jSpAnnz5/n66+/ZsWKFZQunT0K8yilnI9dn5oyxmwANqRaNirFdDt7Ht+Rvt8bybSfj9CjoQcDW1XO9P0bYxAR/Pz88Pf3Z8KECRQtWjTTj6OUyll0QDw72H/6Mq+v3kODikX56CmfTB0+4sqVK7z99tvky5ePqVOn0qJFC1q0aJFp+1dK5Wx6j2Imi7oay6CAQIoWyM3cfg3J6555xXI2bNhA7dq1mT9/Pu7u7jqAnVIq02lSyESx8QkM+XwXF2JuWYrlFMqcYjnnz5+nb9++dO7cmSJFivDHH38wceJEHcBOKZXpNClkEmMM767bx67wi0zu6YtP+cwrlnPx4kW++eYb3n//fYKCgmjS5G7PACql1P3TawqZZNGWY6zZFcErbavTuW7ZB97fqVOnWLZsGf/+97+pXr064eHheiFZKWV32lLIBP87HMW4DQfo5PMwr7Z9sLGFjDEsWLAAb29vRo8ezV9//QWgCUEplSU0KTygv6zFcmo+XJjJz9R7oGI5f/31F23btmXw4ME0aNCAkJAQqlWrlonRKqXU3Wn30QO4HGMplpPHLRcL+jekQJ77/3XGx8fTtm1bLly4wLx58xg4cKAOYKeUynKaFO5TfEIiw1YEEXExhhWDmuJR7P6K5Rw6dIiqVavi7u5OQEAAVatWxcPDpUcQV0plY/pV9D59tOEAvx85z0fd6+DnWfyet7916xZjxoyhTp06zJo1C4BHHnlEE4JSyqG0pXAfVu08wWdbj/PPFp480+jeRyHdsWMH/v7+7Nu3j+eee44+ffrYIUqllLp32lK4RzuPX+Ddr/bRqnpJ3nni3ovlTJs2jWbNmiU/e7Bs2TJKlixph0iVUureaVK4BxEXYxiydBcVihVgZu8G91QsJ2lIisaNGzNo0CD279/Pk08+aa9QlVLqvmj3kY2ux8YzaMkubiUksmCAH0UK5LZpu8uXL/PWW2+RP39+pk2bRvPmzWnevLmdo1VKqfujLQUbJCYa3li9h0NnrjDzuQZULWVbsZxvvvkGb29vFi5cSN68eXUAO6VUtqdJwQbTfjnCD/vP8J8nvHikRqkM14+KiuK5556ja9eulChRgu3btzN+/HgdwE4ple1pUsjAdyGRTP/lCD0beuDf0rZiOZcvX2bDhg2MGTOGwMBAGjVqZOcolVIqc+g1hbvYd+oyb6wJpmGlYozNoFjOyZMn+fzzzxkxYgTVqlUjPDycIkUyb6RUpZTKCtpSSMe5qzcZtCSQ4gXyMLdv+sVyEhMTmTt3LrVr12bs2LHJA9hpQlBKOSNNCmmIjU9gyNJdXIqJY8EAP0oVypvmekeOHOGxxx7jxRdfpHHjxuzdu1cHsFNKOTXtPkrFGMN/vtxH0IlLzOnTgNrl0v7GHx8fz+OPP86lS5dYtGgR//znP/VCslLK6WlSSGXh78f4IiiCV9tVp1OdO4vlHDhwgOrVq+Pu7s7SpUupWrUq5cqVc0CkSimV+bT7KIXfDp3jv98f4Ik6D/PKY7cXy4mNjeX999+nbt26zJw5E4BWrVppQlBKuRRtKViFnbvKK8t3U+vhwkzqeXuxnO3bt+Pv709oaCj9+vWjX79+DoxUKaXsR1sKwKWYWwwMCCRv7lwsGOB3W7GcyZMn07x5c65evcqGDRtYsmQJJUqUcGC0SillPzk+KcQnJDJs+W5OX7rJvH4NKV80P2C51RSgWbNmDBkyhH379tGpUydHhqqUUnaX47uPxn53gC1h55nYoy4NKxXn0qVLvPHGGxQoUIAZM2boAHZKqRwlR7cUVuw4weI/jjOwZWV6+lXgq6++wtvbm4CAAAoVKqQD2Cmlcpwc21L482g07321j0dqlMLfrwTPPPMMa9aswdfXl2+//ZYGDRo4OkSllMpyObKlcPJCDC8uC6JiiQJM712f69eu8tNPP/HRRx+xY8cOTQhKqRwrx7UUrsXGM2hJILfi42kYs4fC+R6hSLVqnDhxgkKFCjk6PKWUcii7thREpKOIHBKRMBEZkcb7eUVklfX9P0XE057xJCYaXl8VzKEzVzi9ZiyzP34/eQA7TQhKKWXHpCAibsAsoBPgDfQWEe9Uq/kDF40x1YCpwHh7xQPw7qpt/Bh6luifF+DnUZD9+/frAHZKKZWCPVsKjYEwY8xRY8wtYCXQLdU63YAA6/RaoK3YaVS5r3efZPmei9w6sIlPXnqKjRs34unpaY9DKaWU07LnNYXywMkU8xFAk/TWMcbEi8hloARwPuVKIjIYGAxQsWLF+wqmVOH8NCzjztRhr1LRo/x97UMppVydPZNCWt/4U9/4b8s6GGPmA/MB/Pz87uvhgeZVS9L8tQ73s6lSSuUY9uw+igAqpJj3AE6nt46IuANFgAt2jEkppdRd2DMp7ASqi0hlEckD9ALWp1pnPTDAOt0D+NXoY8RKKeUwdus+sl4jGAZsBNyAT40x+0XkAyDQGLMeWAQsFZEwLC2EXvaKRymlVMbs+vCaMWYDsCHVslEppm8CPe0Zg1JKKdvlyGEulFJKpU2TglJKqWSaFJRSSiXTpKCUUiqZONsdoCISBYTf5+YlSfW0dA6g55wz6DnnDA9yzpWMMaUyWsnpksKDEJFAY4yfo+PISnrOOYOec86QFees3UdKKaWSaVJQSimVLKclhfmODsAB9JxzBj3nnMHu55yjrikopZS6u5zWUlBKKXUXmhSUUkolc8mkICIdReSQiISJyIg03s8rIqus7/8pIp5ZH2XmsuGcXxeRUBEJEZFfRKSSI+LMTBmdc4r1eoiIERGnv33RlnMWkWes/9b7RWR5VseY2Wz4264oIr+JyG7r3/cTjogzs4jIpyJyTkT2pfO+iMh06+8jREQaZGoAxhiXemEZpvsvoAqQB9gDeKdaZygw1zrdC1jl6Liz4JzbAAWs0y/mhHO2rlcI2AxsB/wcHXcW/DtXB3YDxazzpR0ddxac83zgReu0N3Dc0XE/4Dm3BhoA+9J5/wngeyyVK5sCf2bm8V2xpdAYCDPGHDXG3AJWAt1SrdMNCLBOrwXaikhapUGdRYbnbIz5zRgTY53djqUSnjOz5d8Z4ENgAnAzK4OzE1vOeRAwyxhzEcAYcy6LY8xstpyzAQpbp4twZ4VHp2KM2czdK1B2A5YYi+1AUREpm1nHd8WkUB44mWI+wroszXWMMfHAZaBElkRnH7acc0r+WL5pOLMMz1lE6gMVjDHfZmVgdmTLv3MNoIaIbBWR7SLSMcuisw9bznk00FdEIrDUb3k5a0JzmHv9/35P7Fpkx0HS+saf+r5bW9ZxJjafj4j0BfyAR+wakf3d9ZxFJBcwFXg+qwLKArb8O7tj6UJ6FEtr8HcR8THGXLJzbPZiyzn3BhYbYyaLSDMs1Rx9jDGJ9g/PIez6+eWKLYUIoEKKeQ/ubE4mryMi7lianHdrrmV3tpwzItIOeAfoaoyJzaLY7CWjcy4E+ACbROQ4lr7X9U5+sdnWv+2vjTFxxphjwCEsScJZ2XLO/sBqAGPMNiAfloHjXJVN/9/vlysmhZ1AdRGpLCJ5sFxIXp9qnfXAAOt0D+BXY72C46QyPGdrV8o8LAnB2fuZIYNzNsZcNsaUNMZ4GmM8sVxH6WqMCXRMuJnClr/tr7DcVICIlMTSnXQ0S6PMXLac8wmgLYCIeGFJClFZGmXWWg/0t96F1BS4bIyJzKydu1z3kTEmXkSGARux3LnwqTFmv4h8AAQaY9YDi7A0McOwtBB6OS7iB2fjOU8EHgLWWK+pnzDGdHVY0A/IxnN2KTae80agvYiEAgnAv40x0Y6L+sHYeM5vAAtE5DUs3SjPO/OXPBFZgaX7r6T1Osn7QG4AY8xcLNdNngDCgBjgn5l6fCf+3SmllMpkrth9pJRS6j5pUlBKKZVMk4JSSqlkmhSUUkol06SglFIqmSYFlW2JSIKIBKd4ed5lXc/0RpXMaiLiJyLTrdOPikjzFO8NEZH+WRiLr7OPGqqylss9p6Bcyg1jjK+jg7hX1gfkkh6SexS4BvxhfW9uZh9PRNytY3ilxRfLsCYbMvu4yjVpS0E5FWuL4HcRCbK+mqexTm0R2WFtXYSISHXr8r4pls8TEbc0tj0uIuOt6+0QkWrW5ZXEUociqR5FRevyniKyT0T2iMhm67JHReRba8tmCPCa9ZitRGS0iLwpIl4isiPVeYVYpxuKyP9EZJeIbExrBEwRWSwiU0TkN2C8iDQWkT/EUlPgDxGpaX0C+APgWevxnxWRgmIZr3+ndd20RpZVOZmjxw7Xl77Se2F5IjfY+lpnXVYAyGedro7lqVYAT6zjzwMzgD7W6TxAfsAL+AbIbV0+G+ifxjGPA+9Yp/sD31qnvwEGWKdfAL6yTu8Fyluni1p/Pppiu9HAmyn2nzxvPa8q1um3gXexPLn6B1DKuvxZLE/xpo5zMfAt4GadLwy4W6fbAV9Yp58HZqbYbhzQNyle4DBQ0NH/1vrKPi/tPlLZWVrdR7mBmSLiiyVp1Ehju23AOyLiAXxpjDkiIm2BhsBO6zAf+YH0xoBakeLnVOt0M+Bp6/RSLDUaALYCi0VkNfDlvZwclkHcngE+xvLh/1NpbXgAAAHySURBVCxQE8tAfj9Z43QD0hvXZo0xJsE6XQQIsLaKDNZhEdLQHugqIm9a5/MBFYED9xi7clGaFJSzeQ04C9TD0v15R/EcY8xyEfkT6AxsFJGBWIYbDjDGjLThGCad6TvWMcYMEZEm1mMFW5OVrVZhGYvqS8uuzBERqQPsN8Y0s2H76ymmPwR+M8Y8Ze222pTONgL8wxhz6B7iVDmIXlNQzqYIEGksY+X3w/JN+jYiUgU4aoyZjmVEybrAL0APESltXae4pF+n+tkUP7dZp//g74ET+wBbrPupaoz50xgzCjjP7UMaA1zFMoz3HYwxf2Fp7byHJUGAZajrUmKpC4CI5BaR2unEmVIR4JR1+vm7HH8j8LJYmyFiGT1XqWSaFJSzmQ0MEJHtWLqOrqexzrPAPhEJBmphKV0YiqXP/kfrBd2fgPRKGOa1tjSGY2mZALwC/NO6bT/rewATRWSv9XbYzVhqCKf0DfBU0oXmNI616v/t3bERQVEQBdC7gWbEytGNwNCUSAHUoQLJFzwWQyD7gXMKePOyO3d2ZjfJOs97ANeMde67qjplzB0+hulf7JNsq+qY96A8JFk+Bs0ZjWKR5Hz/8+aHt/kjtqTCixoHeVbTNF3m/gvMQVMAoGkKADRNAYAmFABoQgGAJhQAaEIBgHYD6QQnAVDaZkcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.figure()\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.plot(fpr, tpr, label='Keras (area = {:.3f})'.format(auc_non_federated))\n",
    "plt.xlabel('False positive rate')\n",
    "plt.ylabel('True positive rate')\n",
    "plt.title('ROC curve')\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}