posthoc_minimum.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import sys \n",
    "import torch\n",
    "from torch import optim, save\n",
    "from torchvision import datasets, transforms\n",
    "from torchvision.utils import save_image\n",
    "#from pprint import pprint \n",
    "\n",
    "import lipmip.utilities as utils\n",
    "from lipmip.relu_nets import ReLUNet\n",
    "#import lipmip.neural_nets.data_loaders as data_loaders\n",
    "#import lipmip.neural_nets.train as train \n",
    "from lipmip.hyperbox import Hyperbox \n",
    "import lipmip.interval_analysis as ia \n",
    "from lipmip.lipMIP import LipMIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.betavae import loss_function as vae_loss_fn\n",
    "from models.betavae import SmallVAE, BigVAE\n",
    "from models.gen import AdversaryModelGen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 200\n",
    "\n",
    "# MNIST Dataset\n",
    "train_dataset = datasets.MNIST(root='./data/', train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_dataset = datasets.MNIST(root='./data/', train=False, transform=transforms.ToTensor(), download=False)\n",
    "\n",
    "# Data Loader (Input Pipeline)\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=bs, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=bs, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train a VAE to learn a smooth embedding function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SmallVAE(\n",
       "  (fc1): Linear(in_features=784, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (fc31): Linear(in_features=256, out_features=8, bias=True)\n",
       "  (fc32): Linear(in_features=256, out_features=8, bias=True)\n",
       "  (fc4): Linear(in_features=8, out_features=256, bias=True)\n",
       "  (fc5): Linear(in_features=256, out_features=512, bias=True)\n",
       "  (fc6): Linear(in_features=512, out_features=784, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load and train BigVAE\n",
    "# nc = 1\n",
    "# nz = 8\n",
    "# ndf = 16\n",
    "# ngf = 16\n",
    "beta = 1\n",
    "# hparams_vae = {\"nc\": nc, \"nz\": nz, \"ndf\": ndf, \"ngf\": ngf}\n",
    "# vae = BigVAE(hparams_vae)\n",
    "# vae.cuda()\n",
    "\n",
    "# Load and train SmallVAE\n",
    "vae = SmallVAE(x_dim=784, h_dim1= 512, h_dim2=256, z_dim=8)\n",
    "vae.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(vae.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    vae.train()\n",
    "    train_loss = 0\n",
    "    for batch_idx, (data, _) in enumerate(train_loader):\n",
    "        data = data.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        recon_batch, mu, log_var = vae(data)\n",
    "        loss, _, _ = vae_loss_fn(recon_batch, data, mu, log_var, beta)\n",
    "        \n",
    "        loss.backward()\n",
    "        train_loss += loss.item()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item() / len(data)))\n",
    "    print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test():\n",
    "    vae.eval()\n",
    "    test_loss= 0\n",
    "    with torch.no_grad():\n",
    "        for data, _ in test_loader:\n",
    "            data = data.cuda()\n",
    "            recon, mu, log_var = vae(data)\n",
    "            \n",
    "            # sum up batch loss\n",
    "            vae_l, _, _ = vae_loss_fn(recon, data, mu, log_var, beta)\n",
    "            test_loss += vae_l.item()\n",
    "        \n",
    "    test_loss /= len(test_loader.dataset)\n",
    "    print('====> Test set loss: {:.4f}'.format(test_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/abhi24/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/nn/functional.py:1639: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n",
      "/u/abhi24/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/nn/_reduction.py:44: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 0.911283\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.255737\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.240572\n",
      "====> Epoch: 1 Average loss: 0.2517\n",
      "====> Test set loss: 0.1969\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.203336\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.178885\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.179962\n",
      "====> Epoch: 2 Average loss: 0.1814\n",
      "====> Test set loss: 0.1693\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.168343\n",
      "Train Epoch: 3 [20000/60000 (33%)]\tLoss: 0.163051\n",
      "Train Epoch: 3 [40000/60000 (67%)]\tLoss: 0.163759\n",
      "====> Epoch: 3 Average loss: 0.1661\n",
      "====> Test set loss: 0.1616\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.169925\n",
      "Train Epoch: 4 [20000/60000 (33%)]\tLoss: 0.156033\n",
      "Train Epoch: 4 [40000/60000 (67%)]\tLoss: 0.153282\n",
      "====> Epoch: 4 Average loss: 0.1598\n",
      "====> Test set loss: 0.1573\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.160446\n",
      "Train Epoch: 5 [20000/60000 (33%)]\tLoss: 0.157822\n",
      "Train Epoch: 5 [40000/60000 (67%)]\tLoss: 0.161698\n",
      "====> Epoch: 5 Average loss: 0.1557\n",
      "====> Test set loss: 0.1532\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.153289\n",
      "Train Epoch: 6 [20000/60000 (33%)]\tLoss: 0.151164\n",
      "Train Epoch: 6 [40000/60000 (67%)]\tLoss: 0.158716\n",
      "====> Epoch: 6 Average loss: 0.1529\n",
      "====> Test set loss: 0.1508\n",
      "Train Epoch: 7 [0/60000 (0%)]\tLoss: 0.150888\n",
      "Train Epoch: 7 [20000/60000 (33%)]\tLoss: 0.144638\n",
      "Train Epoch: 7 [40000/60000 (67%)]\tLoss: 0.156358\n",
      "====> Epoch: 7 Average loss: 0.1506\n",
      "====> Test set loss: 0.1494\n",
      "Train Epoch: 8 [0/60000 (0%)]\tLoss: 0.148023\n",
      "Train Epoch: 8 [20000/60000 (33%)]\tLoss: 0.145331\n",
      "Train Epoch: 8 [40000/60000 (67%)]\tLoss: 0.143434\n",
      "====> Epoch: 8 Average loss: 0.1489\n",
      "====> Test set loss: 0.1479\n",
      "Train Epoch: 9 [0/60000 (0%)]\tLoss: 0.145582\n",
      "Train Epoch: 9 [20000/60000 (33%)]\tLoss: 0.147119\n",
      "Train Epoch: 9 [40000/60000 (67%)]\tLoss: 0.153646\n",
      "====> Epoch: 9 Average loss: 0.1476\n",
      "====> Test set loss: 0.1466\n",
      "Train Epoch: 10 [0/60000 (0%)]\tLoss: 0.148306\n",
      "Train Epoch: 10 [20000/60000 (33%)]\tLoss: 0.147710\n",
      "Train Epoch: 10 [40000/60000 (67%)]\tLoss: 0.143232\n",
      "====> Epoch: 10 Average loss: 0.1464\n",
      "====> Test set loss: 0.1456\n",
      "Train Epoch: 11 [0/60000 (0%)]\tLoss: 0.149294\n",
      "Train Epoch: 11 [20000/60000 (33%)]\tLoss: 0.144648\n",
      "Train Epoch: 11 [40000/60000 (67%)]\tLoss: 0.141918\n",
      "====> Epoch: 11 Average loss: 0.1454\n",
      "====> Test set loss: 0.1446\n",
      "Train Epoch: 12 [0/60000 (0%)]\tLoss: 0.144612\n",
      "Train Epoch: 12 [20000/60000 (33%)]\tLoss: 0.139771\n",
      "Train Epoch: 12 [40000/60000 (67%)]\tLoss: 0.142513\n",
      "====> Epoch: 12 Average loss: 0.1445\n",
      "====> Test set loss: 0.1437\n",
      "Train Epoch: 13 [0/60000 (0%)]\tLoss: 0.151777\n",
      "Train Epoch: 13 [20000/60000 (33%)]\tLoss: 0.143662\n",
      "Train Epoch: 13 [40000/60000 (67%)]\tLoss: 0.138744\n",
      "====> Epoch: 13 Average loss: 0.1437\n",
      "====> Test set loss: 0.1435\n",
      "Train Epoch: 14 [0/60000 (0%)]\tLoss: 0.145035\n",
      "Train Epoch: 14 [20000/60000 (33%)]\tLoss: 0.144020\n",
      "Train Epoch: 14 [40000/60000 (67%)]\tLoss: 0.145720\n",
      "====> Epoch: 14 Average loss: 0.1430\n",
      "====> Test set loss: 0.1428\n",
      "Train Epoch: 15 [0/60000 (0%)]\tLoss: 0.141200\n",
      "Train Epoch: 15 [20000/60000 (33%)]\tLoss: 0.146710\n",
      "Train Epoch: 15 [40000/60000 (67%)]\tLoss: 0.148118\n",
      "====> Epoch: 15 Average loss: 0.1424\n",
      "====> Test set loss: 0.1422\n",
      "Train Epoch: 16 [0/60000 (0%)]\tLoss: 0.142959\n",
      "Train Epoch: 16 [20000/60000 (33%)]\tLoss: 0.135717\n",
      "Train Epoch: 16 [40000/60000 (67%)]\tLoss: 0.137858\n",
      "====> Epoch: 16 Average loss: 0.1418\n",
      "====> Test set loss: 0.1416\n",
      "Train Epoch: 17 [0/60000 (0%)]\tLoss: 0.141820\n",
      "Train Epoch: 17 [20000/60000 (33%)]\tLoss: 0.143537\n",
      "Train Epoch: 17 [40000/60000 (67%)]\tLoss: 0.141199\n",
      "====> Epoch: 17 Average loss: 0.1413\n",
      "====> Test set loss: 0.1414\n",
      "Train Epoch: 18 [0/60000 (0%)]\tLoss: 0.135732\n",
      "Train Epoch: 18 [20000/60000 (33%)]\tLoss: 0.138524\n",
      "Train Epoch: 18 [40000/60000 (67%)]\tLoss: 0.140429\n",
      "====> Epoch: 18 Average loss: 0.1410\n",
      "====> Test set loss: 0.1411\n",
      "Train Epoch: 19 [0/60000 (0%)]\tLoss: 0.143821\n",
      "Train Epoch: 19 [20000/60000 (33%)]\tLoss: 0.139120\n",
      "Train Epoch: 19 [40000/60000 (67%)]\tLoss: 0.138475\n",
      "====> Epoch: 19 Average loss: 0.1405\n",
      "====> Test set loss: 0.1412\n",
      "Train Epoch: 20 [0/60000 (0%)]\tLoss: 0.133589\n",
      "Train Epoch: 20 [20000/60000 (33%)]\tLoss: 0.139339\n",
      "Train Epoch: 20 [40000/60000 (67%)]\tLoss: 0.142352\n",
      "====> Epoch: 20 Average loss: 0.1401\n",
      "====> Test set loss: 0.1405\n",
      "Train Epoch: 21 [0/60000 (0%)]\tLoss: 0.135622\n",
      "Train Epoch: 21 [20000/60000 (33%)]\tLoss: 0.136871\n",
      "Train Epoch: 21 [40000/60000 (67%)]\tLoss: 0.136746\n",
      "====> Epoch: 21 Average loss: 0.1397\n",
      "====> Test set loss: 0.1401\n",
      "Train Epoch: 22 [0/60000 (0%)]\tLoss: 0.140972\n",
      "Train Epoch: 22 [20000/60000 (33%)]\tLoss: 0.137555\n",
      "Train Epoch: 22 [40000/60000 (67%)]\tLoss: 0.139491\n",
      "====> Epoch: 22 Average loss: 0.1394\n",
      "====> Test set loss: 0.1404\n",
      "Train Epoch: 23 [0/60000 (0%)]\tLoss: 0.139548\n",
      "Train Epoch: 23 [20000/60000 (33%)]\tLoss: 0.140415\n",
      "Train Epoch: 23 [40000/60000 (67%)]\tLoss: 0.141073\n",
      "====> Epoch: 23 Average loss: 0.1391\n",
      "====> Test set loss: 0.1400\n",
      "Train Epoch: 24 [0/60000 (0%)]\tLoss: 0.139534\n",
      "Train Epoch: 24 [20000/60000 (33%)]\tLoss: 0.137265\n",
      "Train Epoch: 24 [40000/60000 (67%)]\tLoss: 0.140477\n",
      "====> Epoch: 24 Average loss: 0.1387\n",
      "====> Test set loss: 0.1397\n",
      "Train Epoch: 25 [0/60000 (0%)]\tLoss: 0.138502\n",
      "Train Epoch: 25 [20000/60000 (33%)]\tLoss: 0.133377\n",
      "Train Epoch: 25 [40000/60000 (67%)]\tLoss: 0.141070\n",
      "====> Epoch: 25 Average loss: 0.1384\n",
      "====> Test set loss: 0.1397\n",
      "Train Epoch: 26 [0/60000 (0%)]\tLoss: 0.136891\n",
      "Train Epoch: 26 [20000/60000 (33%)]\tLoss: 0.133699\n",
      "Train Epoch: 26 [40000/60000 (67%)]\tLoss: 0.138694\n",
      "====> Epoch: 26 Average loss: 0.1382\n",
      "====> Test set loss: 0.1390\n",
      "Train Epoch: 27 [0/60000 (0%)]\tLoss: 0.138595\n",
      "Train Epoch: 27 [20000/60000 (33%)]\tLoss: 0.138153\n",
      "Train Epoch: 27 [40000/60000 (67%)]\tLoss: 0.138408\n",
      "====> Epoch: 27 Average loss: 0.1379\n",
      "====> Test set loss: 0.1391\n",
      "Train Epoch: 28 [0/60000 (0%)]\tLoss: 0.129044\n",
      "Train Epoch: 28 [20000/60000 (33%)]\tLoss: 0.136241\n",
      "Train Epoch: 28 [40000/60000 (67%)]\tLoss: 0.138012\n",
      "====> Epoch: 28 Average loss: 0.1378\n",
      "====> Test set loss: 0.1386\n",
      "Train Epoch: 29 [0/60000 (0%)]\tLoss: 0.139107\n",
      "Train Epoch: 29 [20000/60000 (33%)]\tLoss: 0.136002\n",
      "Train Epoch: 29 [40000/60000 (67%)]\tLoss: 0.138147\n",
      "====> Epoch: 29 Average loss: 0.1376\n",
      "====> Test set loss: 0.1385\n",
      "Train Epoch: 30 [0/60000 (0%)]\tLoss: 0.135962\n",
      "Train Epoch: 30 [20000/60000 (33%)]\tLoss: 0.138106\n",
      "Train Epoch: 30 [40000/60000 (67%)]\tLoss: 0.134219\n",
      "====> Epoch: 30 Average loss: 0.1372\n",
      "====> Test set loss: 0.1384\n",
      "Train Epoch: 31 [0/60000 (0%)]\tLoss: 0.146383\n",
      "Train Epoch: 31 [20000/60000 (33%)]\tLoss: 0.134417\n",
      "Train Epoch: 31 [40000/60000 (67%)]\tLoss: 0.138216\n",
      "====> Epoch: 31 Average loss: 0.1370\n",
      "====> Test set loss: 0.1384\n",
      "Train Epoch: 32 [0/60000 (0%)]\tLoss: 0.136801\n",
      "Train Epoch: 32 [20000/60000 (33%)]\tLoss: 0.132504\n",
      "Train Epoch: 32 [40000/60000 (67%)]\tLoss: 0.132285\n",
      "====> Epoch: 32 Average loss: 0.1369\n",
      "====> Test set loss: 0.1385\n",
      "Train Epoch: 33 [0/60000 (0%)]\tLoss: 0.133458\n",
      "Train Epoch: 33 [20000/60000 (33%)]\tLoss: 0.137284\n",
      "Train Epoch: 33 [40000/60000 (67%)]\tLoss: 0.135456\n",
      "====> Epoch: 33 Average loss: 0.1367\n",
      "====> Test set loss: 0.1382\n",
      "Train Epoch: 34 [0/60000 (0%)]\tLoss: 0.141075\n",
      "Train Epoch: 34 [20000/60000 (33%)]\tLoss: 0.139174\n",
      "Train Epoch: 34 [40000/60000 (67%)]\tLoss: 0.136773\n",
      "====> Epoch: 34 Average loss: 0.1366\n",
      "====> Test set loss: 0.1383\n",
      "Train Epoch: 35 [0/60000 (0%)]\tLoss: 0.135081\n",
      "Train Epoch: 35 [20000/60000 (33%)]\tLoss: 0.142048\n",
      "Train Epoch: 35 [40000/60000 (67%)]\tLoss: 0.132061\n",
      "====> Epoch: 35 Average loss: 0.1363\n",
      "====> Test set loss: 0.1382\n",
      "Train Epoch: 36 [0/60000 (0%)]\tLoss: 0.137960\n",
      "Train Epoch: 36 [20000/60000 (33%)]\tLoss: 0.135582\n",
      "Train Epoch: 36 [40000/60000 (67%)]\tLoss: 0.136497\n",
      "====> Epoch: 36 Average loss: 0.1362\n",
      "====> Test set loss: 0.1377\n",
      "Train Epoch: 37 [0/60000 (0%)]\tLoss: 0.137582\n",
      "Train Epoch: 37 [20000/60000 (33%)]\tLoss: 0.134790\n",
      "Train Epoch: 37 [40000/60000 (67%)]\tLoss: 0.131807\n",
      "====> Epoch: 37 Average loss: 0.1360\n",
      "====> Test set loss: 0.1380\n",
      "Train Epoch: 38 [0/60000 (0%)]\tLoss: 0.134287\n",
      "Train Epoch: 38 [20000/60000 (33%)]\tLoss: 0.135800\n",
      "Train Epoch: 38 [40000/60000 (67%)]\tLoss: 0.135374\n",
      "====> Epoch: 38 Average loss: 0.1359\n",
      "====> Test set loss: 0.1380\n",
      "Train Epoch: 39 [0/60000 (0%)]\tLoss: 0.137392\n",
      "Train Epoch: 39 [20000/60000 (33%)]\tLoss: 0.140507\n",
      "Train Epoch: 39 [40000/60000 (67%)]\tLoss: 0.135722\n",
      "====> Epoch: 39 Average loss: 0.1358\n",
      "====> Test set loss: 0.1379\n",
      "Train Epoch: 40 [0/60000 (0%)]\tLoss: 0.134356\n",
      "Train Epoch: 40 [20000/60000 (33%)]\tLoss: 0.137105\n",
      "Train Epoch: 40 [40000/60000 (67%)]\tLoss: 0.134502\n",
      "====> Epoch: 40 Average loss: 0.1356\n",
      "====> Test set loss: 0.1375\n",
      "Train Epoch: 41 [0/60000 (0%)]\tLoss: 0.131934\n",
      "Train Epoch: 41 [20000/60000 (33%)]\tLoss: 0.132334\n",
      "Train Epoch: 41 [40000/60000 (67%)]\tLoss: 0.143538\n",
      "====> Epoch: 41 Average loss: 0.1355\n",
      "====> Test set loss: 0.1370\n",
      "Train Epoch: 42 [0/60000 (0%)]\tLoss: 0.134722\n",
      "Train Epoch: 42 [20000/60000 (33%)]\tLoss: 0.133985\n",
      "Train Epoch: 42 [40000/60000 (67%)]\tLoss: 0.135138\n",
      "====> Epoch: 42 Average loss: 0.1353\n",
      "====> Test set loss: 0.1371\n",
      "Train Epoch: 43 [0/60000 (0%)]\tLoss: 0.133123\n",
      "Train Epoch: 43 [20000/60000 (33%)]\tLoss: 0.133913\n",
      "Train Epoch: 43 [40000/60000 (67%)]\tLoss: 0.137798\n",
      "====> Epoch: 43 Average loss: 0.1352\n",
      "====> Test set loss: 0.1372\n",
      "Train Epoch: 44 [0/60000 (0%)]\tLoss: 0.132334\n",
      "Train Epoch: 44 [20000/60000 (33%)]\tLoss: 0.136263\n",
      "Train Epoch: 44 [40000/60000 (67%)]\tLoss: 0.128741\n",
      "====> Epoch: 44 Average loss: 0.1351\n",
      "====> Test set loss: 0.1372\n",
      "Train Epoch: 45 [0/60000 (0%)]\tLoss: 0.135112\n",
      "Train Epoch: 45 [20000/60000 (33%)]\tLoss: 0.134182\n",
      "Train Epoch: 45 [40000/60000 (67%)]\tLoss: 0.134797\n",
      "====> Epoch: 45 Average loss: 0.1350\n",
      "====> Test set loss: 0.1373\n",
      "Train Epoch: 46 [0/60000 (0%)]\tLoss: 0.135823\n",
      "Train Epoch: 46 [20000/60000 (33%)]\tLoss: 0.129451\n",
      "Train Epoch: 46 [40000/60000 (67%)]\tLoss: 0.135638\n",
      "====> Epoch: 46 Average loss: 0.1349\n",
      "====> Test set loss: 0.1373\n",
      "Train Epoch: 47 [0/60000 (0%)]\tLoss: 0.135251\n",
      "Train Epoch: 47 [20000/60000 (33%)]\tLoss: 0.137025\n",
      "Train Epoch: 47 [40000/60000 (67%)]\tLoss: 0.130618\n",
      "====> Epoch: 47 Average loss: 0.1346\n",
      "====> Test set loss: 0.1369\n",
      "Train Epoch: 48 [0/60000 (0%)]\tLoss: 0.129993\n",
      "Train Epoch: 48 [20000/60000 (33%)]\tLoss: 0.131244\n",
      "Train Epoch: 48 [40000/60000 (67%)]\tLoss: 0.138097\n",
      "====> Epoch: 48 Average loss: 0.1347\n",
      "====> Test set loss: 0.1369\n",
      "Train Epoch: 49 [0/60000 (0%)]\tLoss: 0.135339\n",
      "Train Epoch: 49 [20000/60000 (33%)]\tLoss: 0.137401\n",
      "Train Epoch: 49 [40000/60000 (67%)]\tLoss: 0.136758\n",
      "====> Epoch: 49 Average loss: 0.1345\n",
      "====> Test set loss: 0.1369\n",
      "Train Epoch: 50 [0/60000 (0%)]\tLoss: 0.135854\n",
      "Train Epoch: 50 [20000/60000 (33%)]\tLoss: 0.134330\n",
      "Train Epoch: 50 [40000/60000 (67%)]\tLoss: 0.138901\n",
      "====> Epoch: 50 Average loss: 0.1344\n",
      "====> Test set loss: 0.1370\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 51):\n",
    "    train(epoch)\n",
    "    test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visually verify that all looks good"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/u/abhi24/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/nn/functional.py:1639: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    z = torch.randn(64, 8).cuda()\n",
    "    sample = vae.decoder(z).cuda()\n",
    "    \n",
    "    save_image(sample.view(64, 1, 28, 28), './samples/mnist_sample_' + '.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the VAE model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(vae.state_dict(), \"saved_models/vae.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the VAE model if saved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae.load_state_dict(torch.load(\"saved_models/vae.pt\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part-2: Jointly train the obfuscator, prediction model, and reconstruction model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First initialize the three models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "obf_out_size = 5\n",
    "obf_layer_sizes = [8, 10, 10, 6, obf_out_size]\n",
    "obfuscator = ReLUNet(obf_layer_sizes).cuda()\n",
    "\n",
    "pred_layer_sizes = [obf_out_size, 10, 10]\n",
    "pred_model = ReLUNet(pred_layer_sizes).cuda()\n",
    "\n",
    "adv_model_params = {\"channels\": obf_out_size, \"output_channels\": 1, \"downsampling\": 0, \"offset\": 4}\n",
    "adv_model = AdversaryModelGen(adv_model_params).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_loss_fn = torch.nn.CrossEntropyLoss()\n",
    "rec_loss_fn = torch.nn.MSELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "obf_optimizer = optim.Adam(obfuscator.parameters())\n",
    "pred_optimizer = optim.Adam(pred_model.parameters())\n",
    "adv_optimizer = optim.Adam(adv_model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch):\n",
    "    vae.eval()\n",
    "    train_loss = 0\n",
    "    for batch_idx, (data, labels) in enumerate(train_loader):\n",
    "\n",
    "        data, labels = data.cuda(), labels.cuda()\n",
    "        # get sample embedding from the VAE\n",
    "        with torch.no_grad():\n",
    "            mu, log_var = vae.encoder(data.view(-1, 784))\n",
    "            z = vae.sampling(mu, log_var)\n",
    "        \n",
    "        # pass it through obfuscator\n",
    "        z_tilde = obfuscator(z)\n",
    "        # pass obfuscated z through pred_model\n",
    "        preds = pred_model(z_tilde)\n",
    "        \n",
    "        # train obfuscator and pred_model\n",
    "        pred_loss = pred_loss_fn(preds, labels)\n",
    "        pred_optimizer.zero_grad()\n",
    "        obf_optimizer.zero_grad()\n",
    "        pred_loss.backward(retain_graph=True)\n",
    "        pred_optimizer.step()\n",
    "        obf_optimizer.step()\n",
    "\n",
    "        # pass obfuscated z to adv_model\n",
    "        rec = adv_model(z_tilde.detach())\n",
    "        rec_loss = rec_loss_fn(rec, data)\n",
    "        adv_optimizer.zero_grad()\n",
    "        rec_loss.backward(retain_graph=True)\n",
    "        adv_optimizer.step()\n",
    "        \n",
    "        # maximizing reconstruction loss by obfuscator\n",
    "        z_tilde = obfuscator(z)\n",
    "        rec = adv_model(z_tilde)\n",
    "        rec_loss = rec_loss_fn(rec, data)\n",
    "        obf_optimizer.zero_grad()\n",
    "        (-100*rec_loss).backward()\n",
    "        obf_optimizer.step()\n",
    "\n",
    "        loss = pred_loss + rec_loss\n",
    "        \n",
    "        train_loss += loss.item()\n",
    "        \n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}, pred_loss {:.3f}, rec_loss {:.3f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item() / len(data), pred_loss.item(), rec_loss.item()))\n",
    "    print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test():\n",
    "    vae.eval()\n",
    "    test_pred_loss= 0\n",
    "    test_rec_loss= 0\n",
    "    pred_correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, labels in test_loader:\n",
    "            data, labels = data.cuda(), labels.cuda()\n",
    "            # get sample embedding from the VAE\n",
    "            mu, log_var = vae.encoder(data.view(-1, 784))\n",
    "            z = vae.sampling(mu, log_var)\n",
    "\n",
    "            # pass it through obfuscator\n",
    "            z_tilde = obfuscator(z)\n",
    "            # pass obfuscated z through pred_model\n",
    "            preds = pred_model(z_tilde)\n",
    "            pred_correct += (preds.argmax(dim=1) == labels).sum()\n",
    "\n",
    "            # train obfuscator and pred_model\n",
    "            pred_loss = pred_loss_fn(preds, labels)\n",
    "\n",
    "            # pass obfuscated z to adv_model\n",
    "            rec = adv_model(z_tilde)\n",
    "            rec_loss = rec_loss_fn(rec, data)\n",
    "\n",
    "            test_pred_loss += pred_loss.item()\n",
    "            test_rec_loss += rec_loss.item()\n",
    "        \n",
    "    test_pred_loss /= len(test_loader.dataset)\n",
    "    test_rec_loss /= len(test_loader.dataset)\n",
    "    pred_acc = pred_correct.item() / len(test_loader.dataset)\n",
    "    print('====> Test pred loss: {:.4f}, rec loss {:.4f}, acc {:.2f}'.format(test_pred_loss, test_rec_loss, pred_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====> Test pred loss: 0.0116, rec loss 0.0012, acc 0.10\n"
     ]
    }
   ],
   "source": [
    "test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 0.012570, pred_loss 2.343, rec_loss 0.171\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.011803, pred_loss 2.295, rec_loss 0.066\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.011846, pred_loss 2.300, rec_loss 0.069\n",
      "====> Epoch: 1 Average loss: 0.0119\n",
      "====> Test pred loss: 0.0115, rec loss 0.0003, acc 0.11\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.011856, pred_loss 2.303, rec_loss 0.068\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.011817, pred_loss 2.298, rec_loss 0.066\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.011863, pred_loss 2.306, rec_loss 0.066\n",
      "====> Epoch: 2 Average loss: 0.0118\n",
      "====> Test pred loss: 0.0115, rec loss 0.0003, acc 0.11\n",
      "Train Epoch: 3 [0/60000 (0%)]\tLoss: 0.011841, pred_loss 2.298, rec_loss 0.070\n",
      "Train Epoch: 3 [20000/60000 (33%)]\tLoss: 0.011828, pred_loss 2.300, rec_loss 0.066\n",
      "Train Epoch: 3 [40000/60000 (67%)]\tLoss: 0.011817, pred_loss 2.296, rec_loss 0.067\n",
      "====> Epoch: 3 Average loss: 0.0118\n",
      "====> Test pred loss: 0.0115, rec loss 0.0003, acc 0.11\n",
      "Train Epoch: 4 [0/60000 (0%)]\tLoss: 0.011799, pred_loss 2.291, rec_loss 0.069\n",
      "Train Epoch: 4 [20000/60000 (33%)]\tLoss: 0.011835, pred_loss 2.298, rec_loss 0.069\n",
      "Train Epoch: 4 [40000/60000 (67%)]\tLoss: 0.011810, pred_loss 2.296, rec_loss 0.066\n",
      "====> Epoch: 4 Average loss: 0.0118\n",
      "====> Test pred loss: 0.0115, rec loss 0.0003, acc 0.11\n",
      "Train Epoch: 5 [0/60000 (0%)]\tLoss: 0.011843, pred_loss 2.300, rec_loss 0.069\n",
      "Train Epoch: 5 [20000/60000 (33%)]\tLoss: 0.011843, pred_loss 2.300, rec_loss 0.069\n",
      "Train Epoch: 5 [40000/60000 (67%)]\tLoss: 0.011816, pred_loss 2.297, rec_loss 0.067\n",
      "====> Epoch: 5 Average loss: 0.0118\n",
      "====> Test pred loss: 0.0115, rec loss 0.0003, acc 0.11\n",
      "Train Epoch: 6 [0/60000 (0%)]\tLoss: 0.011821, pred_loss 2.296, rec_loss 0.068\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [130]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m51\u001b[39m):\n\u001b[0;32m----> 2\u001b[0m     \u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mepoch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      3\u001b[0m     test()\n",
      "Input \u001b[0;32mIn [127]\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(epoch)\u001b[0m\n\u001b[1;32m      2\u001b[0m vae\u001b[38;5;241m.\u001b[39meval()\n\u001b[1;32m      3\u001b[0m train_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m batch_idx, (data, labels) \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(train_loader):\n\u001b[1;32m      6\u001b[0m     data, labels \u001b[38;5;241m=\u001b[39m data\u001b[38;5;241m.\u001b[39mcuda(), labels\u001b[38;5;241m.\u001b[39mcuda()\n\u001b[1;32m      7\u001b[0m     \u001b[38;5;66m# get sample embedding from the VAE\u001b[39m\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/utils/data/dataloader.py:435\u001b[0m, in \u001b[0;36m_BaseDataLoaderIter.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    433\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sampler_iter \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    434\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_reset()\n\u001b[0;32m--> 435\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_next_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    436\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m    437\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset_kind \u001b[38;5;241m==\u001b[39m _DatasetKind\u001b[38;5;241m.\u001b[39mIterable \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    438\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \\\n\u001b[1;32m    439\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_yielded \u001b[38;5;241m>\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_IterableDataset_len_called:\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/utils/data/dataloader.py:475\u001b[0m, in \u001b[0;36m_SingleProcessDataLoaderIter._next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    473\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_next_data\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    474\u001b[0m     index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_next_index()  \u001b[38;5;66m# may raise StopIteration\u001b[39m\n\u001b[0;32m--> 475\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dataset_fetcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfetch\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindex\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66m# may raise StopIteration\u001b[39m\n\u001b[1;32m    476\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_pin_memory:\n\u001b[1;32m    477\u001b[0m         data \u001b[38;5;241m=\u001b[39m _utils\u001b[38;5;241m.\u001b[39mpin_memory\u001b[38;5;241m.\u001b[39mpin_memory(data)\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py:44\u001b[0m, in \u001b[0;36m_MapDatasetFetcher.fetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfetch\u001b[39m(\u001b[38;5;28mself\u001b[39m, possibly_batched_index):\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_collation:\n\u001b[0;32m---> 44\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[idx] \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     46\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py:44\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfetch\u001b[39m(\u001b[38;5;28mself\u001b[39m, possibly_batched_index):\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_collation:\n\u001b[0;32m---> 44\u001b[0m         data \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdataset\u001b[49m\u001b[43m[\u001b[49m\u001b[43midx\u001b[49m\u001b[43m]\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m idx \u001b[38;5;129;01min\u001b[39;00m possibly_batched_index]\n\u001b[1;32m     45\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     46\u001b[0m         data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdataset[possibly_batched_index]\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torchvision/datasets/mnist.py:106\u001b[0m, in \u001b[0;36mMNIST.__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    103\u001b[0m img \u001b[38;5;241m=\u001b[39m Image\u001b[38;5;241m.\u001b[39mfromarray(img\u001b[38;5;241m.\u001b[39mnumpy(), mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mL\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m    105\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtransform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 106\u001b[0m     img \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtransform\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    108\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtarget_transform \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    109\u001b[0m     target \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtarget_transform(target)\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torchvision/transforms/transforms.py:104\u001b[0m, in \u001b[0;36mToTensor.__call__\u001b[0;34m(self, pic)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, pic):\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m     98\u001b[0m \u001b[38;5;124m    Args:\u001b[39m\n\u001b[1;32m     99\u001b[0m \u001b[38;5;124m        pic (PIL Image or numpy.ndarray): Image to be converted to tensor.\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[38;5;124m        Tensor: Converted image.\u001b[39m\n\u001b[1;32m    103\u001b[0m \u001b[38;5;124m    \"\"\"\u001b[39m\n\u001b[0;32m--> 104\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_tensor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpic\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torchvision/transforms/functional.py:102\u001b[0m, in \u001b[0;36mto_tensor\u001b[0;34m(pic)\u001b[0m\n\u001b[1;32m    100\u001b[0m img \u001b[38;5;241m=\u001b[39m img\u001b[38;5;241m.\u001b[39mpermute((\u001b[38;5;241m2\u001b[39m, \u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m))\u001b[38;5;241m.\u001b[39mcontiguous()\n\u001b[1;32m    101\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(img, torch\u001b[38;5;241m.\u001b[39mByteTensor):\n\u001b[0;32m--> 102\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mimg\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfloat\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdiv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m255\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    103\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    104\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m img\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 51):\n",
    "    train(epoch)\n",
    "    test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visual verification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3IAAADFCAYAAAAR8MCxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi41LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvSM8oowAAIABJREFUeJzsnWeAJGdx9/+dJs/m3cs5STqUkAQKIJINEtEmGowFIsqAwCLZMrbBNjY2IKIAGSPAcgIECHgJFpgoFAwo59PpdDlv3p2d0OH9UFVP94SLOu3N3Nbvy+529/T2VD+xohVFERRFURRFURRFUZTOwT7eD6AoiqIoiqIoiqIcGbqRUxRFURRFURRF6TB0I6coiqIoiqIoitJh6EZOURRFURRFURSlw9CNnKIoiqIoiqIoSoehGzlFURRFURRFUZQOQzdyiqIoiqIoiqIoHYZu5BRFURRFURRFUToM3cgpiqIoiqIoiqJ0GO7xfoDD5fftV0TH+xkeDz8Jr7eO9zM0ojI99qhMjz0q02OPyvTYozI99qhMnxhUrscelemxR2V6eKhFTlEURVEURVEUpcPQjZyiKIqiKIqiKEqHoRs5RVEURVEURVGUDkM3coqiKIqiKIqiKB1GxyQ7URRFOZZYLg1/G646GwCw6RXX4ClX/ikAoPffbj1uz3UiEF1wBgDgz6/7d1yYqQIAtvszAIC3vPZyAIB9053H5+EU5Qi4ceddqEUBAGD9de8AAKy4UscHRVHaA7XIKYqiKIqiKIqidBhz3iJX+sOnAgBu+ty/AIDRvAHA2h9cBgDoucsDAAx9/pZZfjpFUZ4odl3+FADAwy//LACgFgEjT6JzvcfroTocd9FCAMBbv/oNAMDTMmWEfG6xmwUA7Hgm/Vxy06w/nqIckonXnAsAuPJvrwNAa4KQW/GznnMXAGDzcXkyRVGUZtQipyiKoiiKoiiK0mHMaYvc5KvOxXc+fhUAoBZlAMBo3gDgoRd8HgBQfr4PAPjK20/Gja8mbV14z0Oz+aiKohxLzj0Nn3jHvzQdXvulfQCAoOmMciiss9bjzGvvAQA8Lzdujj/lt5cAAPquKQAAFo9Pz/7DneAMv/k8AMCZb7oHtkVz2PZXDQEA/Me2HLfn6hTcRQux68XLAADXvP8zAIDTU3I21nfftnM5AGA+HpzFp+scbtxJFstLtlwIANhz3sTxfBxFmRPM6Y3c7otr6LZTh7wuY5GY/rTnETj/TYXmf/ia8wEA4d06oCvHlh1/Tm1r2fU7AQD+ps1Hfa/9b6UF3m1/czXW3vhWAMDaN/zu8T1gBxM+40wAwAe+/G+4IFOrO7ehVgX2jxyPxzoheMF1v8ZlPZsAwKjDvjS+EoveT8lOgg2/PU5PduKy+woaK35wxUcBAH12Cp7lAACecxol7snqRu6AOP19AIDNlyzHHe/49CGvz3+t+4l+pI5DwlNWvD9eC1237FcAgEtuvVA3c8pxIXjmkwEAO56ZaTq38ppHAQD+7j2z+kxPFOpaqSiKoiiKoiiK0mHMSYtc6aWkQfr2Mz6DVnvZ59z7KgDAW5aTVulVxV3m3Ft6NgIA9l9HbkK3ne49kY/aMVjnnAoAGPrMFnxl6S8AAI5Fsn3+w89H+Pt76fxNOQAw11z0R2/UNORM+HSyFn38zdcCAK667TUAAPsoLHK151JK/Xl/RNr4ECGsaecYPGVn45TITfoNv3493nn2zwAAb+8h7dyLv30FVg/fdtyerdOwc9SXz7plEgBwWc8mYw2qkeMCvvnui5BSS9wxx5lHbpPrX05WkL7D8CxRmhn4Po0HNyw9tDUOAE59990AgM1fe8IeqeMQS5xY4Q6XjZ88Fwt/RQNF7ob/O+bP1clYZ64HAOx4TjfOfvm9AIC3Dv0CAHBO2jLXvWsnedw8ck5ldh+wAxBL3H1vvrrp3NuefwEA4NffOb/p3LLrdwMAgkc2PYFPd2xRi5yiKIqiKIqiKEqHMScscs7gIABg98tXAwA+/z7aoZ+cat7Hnn/HH2PoSjr+qae9AgDw0FtvBgB8cOh2c93b+34DAPjvf3ovVvzF3CsOKsWUt72PUrh/462UNGY6crH6x1S24eSPjNG1Nd9o769d+nMAwHBYBgA4ZR/R7D12W+P9HflrPys7BQC46ijvYxeLOOufqK1+eB610401Hyf/83YAgP/4HrOjiX5L2s21b0phyf0UD/fzGdLcrfuX/Zrk5DCQvrzhi+sAAN8Z/CIAiosTS9zrNv8eACB374453d6eCNwVy5C9jhLGfGnZjU3nT7r+7QCAtf9LbT1sumJu4qxeAQA4/9sP4s/77weAhAXZNuPAh997KQDgaR8k6/w/zrvHtGvH0tlK2PhJSvx247Jrms4dLNmJxNQ9+qprAHJ+wtNB8dtz1TLnrKex9GXf/CUA4PVddwAAQkT43jQVw3nXh6gYfS0PXPKOHx2Hp+ws+h6gkU+slp9eGK/TP7+I1vR4+81Nn7v3zRQ7/1db/gAAUHvmrqZr2o0TfiMXnXc6dl9Jm4Zbz/rMAa971jvfBgAY3DyF8D4a5Afvo3O3bTwHALDxS/+H1R6JTJKk/N8fX4Vnb30vgLlVZ27DVeS6JzW43rj1eQCA3e9egbW30iYiuSjed9l5/Btt5M771nsAAKt/q65sAC3OXjSPkpBsrNHS15mmAeVwlw5OVxd9/ovL8Z155J75zh00oW688hS4228/4GfnGpuuOxkvztPA/r7dtLAIHt54PB+pY7CWUK24B571xaZzt1ZoYTz+SqoV5+/YOXsPNkfY/geLcOuKTx3w/Oo/ozFVN3CEs44UuI+8gRS6N/R/w8hGNmgnXf92nHQVKboKacpc+/q+W/majMlmHUSxW9tc59FXNW/ghJtvOwUAsBrN87vU7G117Hk3nHGMnq7D2EHufB/95h8CAL7xA87se9s95pIeUHu0MxksuYKUkKV0GgDwCLKz9aQdQ+Eb1PY2/awfAPCiea9uuubBd9KaqTA0jXUDFP7z9ZU/BgCs76YN3F1P+JM+ftS1UlEURVEURVEUpcM4YS1y1YvIivbJL1zd0oUSAGpRgFN/dDkAYO23yKTfyvrh/S9ZMl7z6ffgN++tD4rO2R6ml8wtdwtn/Tp85yWkET7/TqoRNfCXJGPrnrubr+/txRvf+f26Yyf982YAc9vNL0nYlcOpmW0AgPdsejkAIPrdfYf1WessCoze+D6yhtz79GtxX5Xa5L1XnQ4AKP5ULZ9JnrVqw/F+hI7Ef/ZZeP+/frXlub/Zew7ue9lyum7H5ll7prlC+YXkxn73+z6PWtQ6cdHZH7sc8zF3PEMOh0cvIUvcfa9t9shZdwN54qz7i7vgl8lzR1yH3/qWdwEAfvzlZgvSXKX0h0+tKzPQiLhUrr6ieb4RV8xWNo5VX6dwjFYWvLlAMEZ1N5f/9aHDdKKTV+HFeXIJXHszhf+shXrbHIhg/zD9Ij8TrCWPXrgrl+POd5LlHmyR6yTUIqcoiqIoiqIoitJhnHAWOYkTmnoHaThaWePKEdmBzrjxcqx98+Gnxl58/Rb8zSVk6fu7obmbUntyXQ9O9qjsgvM1Kqga3tOsSZNiq/a30nhL92YAwMs2vgAA4O/ZNwtP2jnse0oPzk5TVOHDW+YDANZix2F9dvOfUxu/9wKKi7uvGuHyK98JACh+fW5qOA+Ec8paAMDFvT/E/mAGAHD3B6jsQwpzt08fLu/+4n/imVxIPWzQA977qlUINj16PB7rhEZKu1zyse8BIE+SWlSflucf958FAFj8ne3q5cCM/mANAOAnp36Mj1A80X9MLMEXPkGxSGv+lSwgyXjCsFQCAKRu/N2sPGcn0Sq+LUljbNzGT56LC859AEDrpCiClCFQDs3Df5ozv6e2a9mRx4O7cjkA4KHL52PDKz4PALi3SvPbrz5GFuSuDrASq0VOURRFURRFURSlwzjhLHKjXyd/+F+f9p9N535dpvTC7/4sOcau/dSRxRL423fg5j0r6Y85bJEbW3XgwtKW65pitd5/kdb4+tXfN1m/ttxA8psf7n6Cn7IzcE4mrfGXP/BJACTXpd86vMLd5RdRzMx153wOAPBglWR8+V+8Sy1xB2DTh0gr/4LcOM6/8w0AgL7/mbt9+XAZfiNlnT019WuEnCFN+vQz7qZsYL279hyfhzvBsUtVAMDWaj8fabbUf/tbTwcALNms8XEAMPr683DzGVRmKGRL3DenyNPh+tc8B/13zr2SQY8HKRlwqBx+JpPlq+TIwa83sXE36HwFAO6SxQCAza9dCgBY/JHm/nzTRZ/EZ0dPAwCsuoqsnVo25+hwvkJxsRtWf94cu/SqKwAAQ//VOWPpCbWRq1x8Dv7tFEnL3GxyftPPaOF2pBs4pZ4lX3kY33gTbdYuuILqlN1z6RIAQG+6hP9c8f8AAL+pUKrmXezCBgDzVfZ1PHhFDwBgnefgLzkNfu6XFEx+qPThW19Kw/fp3NRPv/lNAIBluolrwvJISOct3WyOjWwk19++4/FAHca5l1Fdo3lO2hy7s0IOHT0foGPh5ORh3ctdROULdr9gGZa9tr7kw95Pr0T+W3OzltSB2PJi2sB9s/9OPhIrer45RbJc8e9aIxIA9lx+PgDgt3/xWVMj7poxWhR/7+UXAACiB+4/ont6ljPn68iZ+m6fO3b3vGTLhS2TosxlNr6F1lEPv5E2FqvWX4qTrqQwlMmzqK8vcO7AZ29+DgBg7ZgqIY+Gne+jceJHKz7KR3K44O5XAgAWfP0RAJ21OVbXSkVRFEVRFEVRlA7jhLLIlS8fxQo303T85zN0bO21lcf9P2zWyNnJPfAcqxEa7B/Gxz9LvhPnvY409Wu64uQlJ3397QCAld8ms/UX//NqOHNMRofCXUaat19e9Ek+ksaVQzcBAK783+cCAG7+/vlNn1v6UUoz/NhfPxk/ezYF8X99cjkAYMWlmwCQJa/2e5T8IDVM72BydQGF6+eupcPOkzvgF5f8zBwbVGXmIfGfTe3ob+ZL6vbY0+HVv3wLAGDtnYeX+nrsT8g982N/+wUAwFPTNXNOxtPRT5XxmhEqCeP8/I6jf/AThJFLz8MNl0myDq/p/N/9gEqVrNqslg0ASF1E81CI0FjRrvreiwEAKx84OnfKWhQYN+K7Pk4Fq4sdkADhieCSLRfiumW/Oib3euyjJyOHuTsntUJcJde6fwoA+MjL/gvrb2oMQ0nj5LXkXv3gV86qO5PbkG7pjqnETL3yXPz4crLEDTmUOOb9u89G3+soQWKwr/MS8alFTlEURVEURVEUpcM4oSxy163/N4QtYuM++NdvBAB03fb4tGjb/up83HzKxwEAYVI7Ogfd5oeuJq3Po1c3n5PUw9Mvo5ivxW4WJ/+S3sGqQwQ/zwXsTAb9XyPtTzLmqGhT27168S/owGW/aPrszjeRVXmxe4sJ4n95YScA4IIHNifuRQVDg4gaZ8ay8drfkBXV37Lt2HyRDiJcsbjpWHEryVLKZABAMDwya8/UCQRp0vX12rGnw8YayW3d1WTtPdjw5y5fCv9asmbcchIF2ITmE7Ee0Wa3hl47gyBNsU2Hl/LnxMRZvw4AcMuHrwZQ72XiWQ7O/L9LAACr3jM3LUONRBeQpezqU/7VHDOeOJ+j8e5I4we3flA8Im438aCFrTMH/sAcYM95E3gezjjkdfNupTJQrax3UjTcxN0pBikMvuJKsh5fe+UKTPzo9wAAvz7tegDAT2fS2HA7xX2mZmjc7HuQxtSuDkrQMdvsfC/159e+7ifGEvfpUSoCftNnnorefZ2bAOmE2Mht+2t6QYud3zSdu2zbs9Hz3XsAHDp5xIGQ7IA/fctHkbPTdefeuOX3sfqjDwHorODI2WDHc+Il3pLrmt2C5iphuYyb7qasU+UlNwIActbh1YNZ7JKLYDIA32a31aV8rhLVcPrPyTXD3kGLmf77InRvmbuLvk2v6mo69oGv/hsAYEN1vjm2q0rJZ752wzMBAEs/pBMjEGeoBIBHagMAgOj25qQRdo4myH2vPh0A8Pm/+oxJxiN15x7zaQP4qb3PwYfm/xRAvFEMj3qUPrHY8Jckx8Z6cQDwkg0vwtI/JfcfnXOIyWXUfk5PDKMfu/S1AAB7252tPnJAJDGSv65kjr39fsrMOnDr3Y/nMU94JLvldcua683JBm7PeROz+kwnCqIA+/SznodV2+buXH64SAbQra+mTe/PLycX9V47iw01moN+zO7+vYeZxdbO0Djjn3OyOSbKzsp7R+nAtYOzHsairpWKoiiKoiiKoigdxglhkavlSVPhWM0ZNcarGYSlowtedFavAACs/xuy6PUl3OCkJt3oS9MIRrV+Uis+/JxvAQC+MrEEKa3VVcfay8h6/IKX/xkAYOSk2JEsfQ659714+b0AgJd13451Xr2jWSUK8cuZXN2xy27+EwBA36/SWH1t57oJzAa3V4DpiDTvi7zYnfKZOUo9/L43kQvwORV6PxpAHrPIHQMQu04nywU8ei3VRbz3ws80fe7SLZQy+7bNywEAF6150Jz7x/2nAgBuuPaZWPjbuevhED7jTADAh8/+zgGveWx/P5buuXe2HqkjOOuKZqubfdORWeKEre8/GwBw3zM+LXc62seaU5T+8Km46XPNljhBLXFHjrN+HX592n8DANb+nMoLrT5CC/NcxF20EKd8lxLCfG/e9/ho1pz/g/94DwBghTsFALDOOdWcm1iZBwCEr9vfdN++LFnp/2ftteaYzzPVZ0ZPAgD87/XFY/EVjggdoRRFURRFURRFUTqME8IidzC2fWU1+nB0FrncV6jA7ScX3tR07it7ng4A8HerNa6R4TeT3/EfFSh9+L+OLzmej9PW5L9J1ox8i3O3cUKdr/39FbjnDWThuK9K1uc3fuLPMO+z9VaiNdB07YfLX1x+GdI/aLYSVy4+BwDwky9dAwBY9tzNAIDwYy4if66XWyZOS5F1+Nuf+gQA4LZ/GjTnLs5RKYJWkW5fWUbxcOGy+Oz6X7wDALD2nVsAAPOGb5mTljjhH776RQDAk7zmFDLXjJHGd+WfDc/5wt9J9r/1PHx/oWTdOnrd9N63Uaz9a171M74T3cuzHBOHrDQjcXEr3v9gy/NPf/tbAUBLDRwFD7692/y+4DuHF0c/l3HnzwMAnP797fj7oQMn1nvgUq5sf+nh3XcqogRfN5Yonv6fh0/G9V8kDxPLp7F68Jrj5wWlFjlFURRFURRFUZQO44S3yB1psW7/2Weh8n7KPnPN0uv4aBwbt/6/3gkAWPf5nXxk7HE+4IlHLV8v9H++5WKsxe+O09N0LqJd+uc/+ndz7E++TDFbSz6rMVtHwuqvsFWewgjR8/6tqPyKfNnDyUlzXf6B+uKrL5hHsUjfdxfNSYtc+kdktVz/NSrS/d2XfRIneTQeSqbJi3Ox/Gwz4DbrCBvP3VpxsHiQxtq5XvZh5/vIGnRmiiyayWyVYon7yR+T5SPc0dryMVcpbvVxOynMcWY6tvbu+g5llvN+2NP0mdrzad7+yJNuAAAEkY1zM1SypZvLwMidahHQ9+FM0z0UQixxrUoNrPr6ZVh9g2ZYPFKceUMAgP++6At4104aG8R7RzkwT7mRSo18cPABBAepizMcUhmRuyrNY8NlP6BSWYWt8RyWGqeb9X8ptroNoX3WYCf8Ru6qv/oC/vKVLwUA7B6hFOSrPkWT5NaLi1j6I1qEzPw9BT1+95TPImdLqnxasGz3aZZ4z+aXYd0XdgEA/Me2zMrznwi4w1p64Gh49G0rAQAX576Pk358GQBg3T/RhngOli58XISbaYB/9abnAQCuX/1DrP0Uufyc/N5H6aL+HvzDT77Gn6Ch8eO30PVry3NbESH1yt7zX2/Ck79Em9sPDt3e4kqa/JJlBH4+UwAAvO3G1wMA+u6ia+b9cAvSOzY/MQ/cYQRPpUQQntVcPe9348sAAOHduoFrRfpHv8XbPkYuurf+VZxk5/Zz/gMAEJ5z4JIWdl17be26tu6Gt2HdXXfzdYpwOLXiVl+hm7ijYdufUH2zs9LAFZ+kBEhdUFkeipd0USKY2yuxMeGP//1dAABnJj7WvYl6cvHrzTJd04FyVtdKRVEURVEURVGUDuOEsMit+B6lBL31FWmcl6nUnXtquoafnvr1+g88I/H7W+hHrJlrth699oHXAQC6Ln702DzwCc7kGfXvYPV/jqom8wh45LonAwB+8gwqYPmj0nyccuV2AIBfqx635+pkogq1ydJLSYv8uV+swoaLKFX2h8+m4uyv7P4h1nIx4AdrNQDASZ+dBqCaeCG6/X7ceQFZ2C668G115za/IjIyTfKxzWTVXPP2etegueeo2prwGWfiU2eQ9UhcKuXngzVg6yfWAgDymizigCz4HnnIvOQVfwAA+O66A5dvOBT/MUHJub7wiT8EAKz511u1/yc4mCVOeOyj5NaqCU6Ojuj8cQDkjt53E8/9x/OBOoQ/X/HUpmPLcOKXYlKLnKIoiqIoiqIoSodxQljkrFvIf/0fL7kE7/jqNwAAz85S8HyrmIODMR5WsS+g/e1LvnUFAGDdZ6iwoGpEDo+nrHkMAPD+3VRYNXr4seP5OB3Hs9ZuAADcW6VUt5+79OWwdx84la5y+AT7KOnJv1z/fGx6ISXyuGqB+MTHMTKv+cK7AQCL7m6fgOZ2ISyRB0Tqf+rLN5z82z588MkUzyHxc18aX4mRGxYDAIawdRafsnPYeImDp2Wm+a/6+epV334nVn2r82I2Zht/ByUfc17RBwA44/J34Z43f/awP3/K1y9HaoLm/WXfJWtI/50nvib/aDiYJc6UGrhBLXGPh7990v8DAFxw9yvRvW3jcX4apd05ITZygnXzXfjcGnJDedeXaBPx0MVfOKzPXrqFakI89qmTUPgGTZyrOOhRN3CHj7t4Ed69kNxaPrGTXKqiytzOSHe47HwvZad6WTfJ7wNfvgQAsPjXupk41iz90C14+EP0+wtxVtP5RW2UkapTCIZHcPuZtBh+Mc4xx9spu1c70n+rBzyv/tiPSgMAgLX/9Oicrqt3pEj206UfugUv/FBzvz4QqxMJDjSR1MGRRCaNG7pLtlyoG7jHidNDdeOKNmVV7HvNPu3/yiFR10pFURRFURRFUZQO44SyyCVZ+yZKF57UDB8cqitT6MDUo+3EI+9YijNTpB/4zR1rAABrNOD5kETnn46fvouSm1zyyCsBAIs/opYMRTnRmffTnTgvT/Uhf/d+cgcsR5R0S1yBFaVdkEQm+BxZ5MRCt+e8ieP1SCcMO163HgDwvdEyACAYGz+ej6N0CGqRUxRFURRFURRF6TBOWIuccnyIEqqBKKVJmw+XMOWg26ZkG9NXU3KIHHYcz0dSFGUW8B/bgvmfpvT5L/z04cd1KcrxQOLgnnfDGXxELXHHiqe/lpJE3fMPpwMAsvjN8XwcpUNQi5yiKIqiKIqiKEqHoRY55ZjSdx/w85kMAMAb0eZ1uDi/uAMvXkTxnFpEVVEURVEURTkUutJWjik9192Kq66jgN0V0Do8iqIoiqIoh+KRcyoA1KVSOTLUtVJRFEVRFEVRFKXDsKJIy18qiqIoiqIoiqJ0EmqRUxRFURRFURRF6TB0I6coiqIoiqIoitJh6EZOURRFURRFURSlw9CNnKIoiqIoiqIoSoehGzlFURRFURRFUZQOQzdyiqIoiqIoiqIoHYZu5BRFURRFURRFUToM3cgpiqIoiqIoiqJ0GO7xfoDD5Xndb4gAwM7nEPk+HQxC+mlb9NN1gZALnIcBXxMAjlN/M7nGseN7OLyn9X3AsuP7AUDE11RrkALqFp8zzyL/CwAsfh7bBkL67I2TX7WO8Cs/4dTJtFajgyIblqnleYhERq1kKt81KcfGIvNhVHc/AEaOqNZiufE9I9+HxfeN+JyVeIfy2RvHv9y+Mi3kEfny7NSepK1Yrmtkas6VZoAUycbIT+QCxLKZLjX/U5FNQo6Wx21X2rK0YSDR/p34HB/7n7Fr21amVi4bf8dj2fflHkFw4L7v+/E7a9X3Q7lXs26sLdtp8fXUTouFuH8LLD8rlYrHBaFaq283dZ+L4nZardKxhDysTIZ+qfE5x2nu+9WakW/jubZvp61kKjKSeaOFTKNKNf7O5qCMucdIpjLONLThTpGpVcjT3NyCujkqMVc39f1WJOUs44CX4mNBfK5JplUzJ5k1gawhwnj+a0eZAg1yDRvma5mn0om2avN3LZfjtiPyavw8YOQVJeYwmZOiWvwezbyenMP4vnJdnZw7ZT3VuGYSXLd+Xgfq26q0IVkD2E7cRmVuqlXj3xuuj6rx2JJsj1HjmqJD1qgX9byR2mlX0bTFlvOD/C5z89QUkE7X3yyxtjVz+OQUHfPcpnu0XNOK3Hy/eR0l8k7Mp/8z/MVZkala5BRFURRFURRFUTqMjrHIidbA370HTn8fACCqVOrOWdlsfMyKtexWNku/J7VvYMuPaCorrCWq1WClUuazAMyOO6xUYktRGGtam6xGojENwzpLUtvRSqYzM/STNRB2LhfLNKE1s9KifWjQkiY1aqKZqNVgZdJ8rEGzOTMTyyuhNTLybZBfFEXmHbQl/P38Xbvhzp8HAAjZihaVSY52V8FY1iL57pYFVOo1m0kteqtvbLTrcr202+Q5p/7vOkTD3SHtNNiz9+j7foMWtK7vVxPt9AB9X94dcIi+n3jmtpYp9zV/9x4484YAAOHwCJ1jba/dVUA4NU2HEnI5UO+Loig+JxricgUWW5qjUqnuXlG1Fr8/+VwYGsuT1WhpRsKS345I39+9B87gIAAgHBunc4eQKQ4wprWUaTW24DXJNNFO62TaaN3sFJkyyb4v8hPsXM7MW2CPj4O1U/oQz1FswYiCALZo8GUcEQtSpYVMoyi2kDZ6p3QC/P6DPXvj/j8yRud4nWTbXbEHSGJNE4+vfK+EBc1Y3YznRKKd8U9jYQuC5usSVrdmb4qwpcdD25Ds/wP9AIBopkw/+VzdekoIw3ieMnMy9/WgEnvXMFHNj8fGBq+IqFpNeDPZdZ8BEla6RD85LMv1cUK8XoLtO+AumE/Hpqn/y1hn5XMIJ8iyZmfsbv3eAAAgAElEQVTZQ8G2AJZ9k6W9UkHUaE2zrPh+qYZzACBWZNO+Y7lbYn02noKBmfNmi47ZyFn8gtzBfkAEXsgDACIZ2C2ruVHaduzSJsKVd+e6ZNYGYOXSfKwYDyRR/Qu1U17cSUo8cdgWEDRPjkLU4li7YGQ6NGAavTWfBnVjck7I1Cx8gWaZCkmZZqjjWIV8PACLPGQgcV1YfM5MGikvdr9oHLjDEO287LC7igAAa/ECYIbb6Xxa1FkyUVpWYmPb4OqEhAsVH7NcO3ZNzefie4hLD7tXyQAHLwVLXK14AWlZUdMG2yywy5XOa6cH6fumj7bq+3JP24ldiJLtVGjo+zhU35cNiVyDNu/7xQIAwFk4Dxa3U2fpYgBAND5hrrMb3VNcN968NrjvWoBZWMg7s/JO3HZlAWjcU1wzpoiyw0ql4glRJuCke3DQvr3fKlLfdxYvaJbpqGzorMcv08RY0SRTxzbnW8nULOSEVm5xbYSVo/HOnTcIcL9zFpCCrG6O4vmkbo4SmcqG2U7ISuSRkKnpuw3hE7AdWG69a7uVmKNiBW5ifA1bKM7aCBnr3AXzzHztLF4AAIjGuP9bVqwgaJQlUO/WDpYXzy12T3d8XQtXXwCwHTt23awl+nwYKyyAhLIz4QbYjlg52oy58wbjNeogtd+Wa1T5Lp53wDWqncvGLoUse6voxp9tkIeTycSbDNkw2ladcrOJNpapzcobrFkGjEzS7wN0zBrn/h+FsLtoPpO2U+cW3ICVycTKKxkPLTtus2lu8yVeazh2fRgFeNxtmJ9io1C1zn14Nmhj9YaiKIqiKIqiKIrSio6xyImGPBwejbUWvOMWDU+dpjKx07Zkh50IOAcAuA7AWpRITKaWZX6HyzvtEmtXan5sEZDPVapA0HrnD6C9XQGYcP9ILFPWPBgzs5iqgVjb4bpGpkbTI7JPylQsPo5ttGqRx9bNKmugSk5swWMrQVSpxBo6o7VirR/a1w0AiK010XQJkEQE7P4kLkF2V1csS5G7ZVFbQmzVM23HdWJtpisuFRGiPN+/wu0vx9rlcjW2WorWqBI2uVcaNyzbbm93Ven7yXba2PeTFjdjAXJMAhmrVTsVDbybkG1DO7VYtla5GmtZ2SoaVatxO210T2t3mUo/37M/lom0XdbkWsWC+X5JK4XIWhIXme+eSccylfcRhoiyPP7yOavG2v2ZipGpLRbWMAREE8+PKu7BsOz493aELWcHlWk+tvoesUxlXoqiw5OptNMgNNr5WKZi5bBgHdwJ8fgiMt0/ElsypO9Lf2SrHdDg4WAsRQ0hD8m+byygFkJps64klWCZTs+YfmDmqGot7htirbL5/zl2PC+2K9z/w737jXcIWrisNa2ZkokfkpY4AEinEIlVIrHuiTLixsfhGtPs0RBFsSthItFZU5KjZIKOdl5PtZqnZCwTmXqxNc1Y3x0nTgIn7Zfn8ijlxS7VyYRvmQYPE1mj+oEZc0zytGotfmeNnlFtjlgyrVIJkbQzPhbyusrp7THeBzLmwXHMOGESQsk8n0l4RCTWU7VutqiOs/zy7O46U2me38OEa7Vgi/eTBSQt17NAG/cKRVEURVEURVEUpRWdY5FjC4Y90IdoYrLumOy4LceONaGilXddRAXapYddtMP286SpqPa4qOVoL1vu4ViNFIza0mZlUXaYtCSpyQCZvWSRs8fZV35yOvazldSv4isPtLdWXhKaDPbHMhUNnGg5bSthfWONQ8pDxNqKsEiy94uk5agVXFS76LpKN333IB3LwGJFkJHpRIjsHtKAGJkm/hcaYr3iCOs2heNfrIEeYITjYiR2u6/XXBZJXg5jHXaBQfL9Djhmy++he/k5B6UBthiLUrpgweHkKBbfP7tfZOrDG+N2OsX/aGwClsTD1Opj8NpeOxcm2qnEbzX0fdhW3NckVs51TTsNuO8H0ve7PFSL3E67mtvp4fV9mLTcje3UaneZiiVnwRAwKjJly0LCamRkKRYj10XE2uKQ44r9Hvq7lnNR6aXrqwWWacYy46lppyxTbyrR9yfopzVVAiRWUcbTxvG1zTmoTG2rtUxlPM1K32eZ5l1UelimxUQ7PVKZgv+/sYB0iExFEz7UD0jsVqs5SqwPdixb007zDXNU0UWlm66rcTv1s80yzYzQL+mJrua+DwB2QzsV74dZjo85KqQNDvYDo/UJeSTWMxlPGSeMixB10fcOC9xGi9Rma10OKsX6thp6sTztGgk4t59ikbypEOlhlus0W5SGx5q9giSxT7snOgtbrKeCFvGtLlt3xfpu20amgVlPkUyr3S5qOfrO5V5eo3qAJWtUnnbM3D/VYp6amGq2YjeUzmhXTC6B3i5YE2wxZjk7Q5R7IJkgJ7Yge0A3Wc+DPN2j0s/jgWthZoATGPISqFawYMt0U6FxJbefxu7UuA9vlD3xpjmByuh4vHZrKA8RVauz7uOgFjlFURRFURRFUZQOo2Mscia+CIgtCGz9kBTMyOUSmejoXDjQjQpnDqr00tedGaD9a2m+hcoQXe/10j36ukoIQjo/NkGfG99P98pv91BgTV5uN+3G09vZhx5AJFYjyRZUq826r+yR0EqmYuEIOSNYshCraOXC/i5UB0iD1CTTeRYq8+heTi9pL/p7psz/HGGZTuyje+W3uSiwNiq/izRUqR0wGo0oWeAVHI/Qxn7ywb59AACnVo1jCERubE2y0qmm4t9RNo1aP8mhPETtbXqIvuf0YqA6yO20i1RwPV0llGt0/+lJemcT2+lndm/ayDK/g9qf67mwx0hLaHRwbEWKqtX6bGRthvjC27bV3PfFhz6Xi7WfrOkMB7pRHajv+yVupzPzLVQG6V5uH/Xfvu7pw+r7de1U+r48rMQi+X5zkec2IuAMqk4UxdY56fujdE7igIHYsyAq5FAbJE1nZYDaTKmf5DK9yEJ5AbVTt5va1lDfBMKIPrt/lDT94/vo/+R2uih0U5sv7GSZ7rZizau8az+O02tnjXzA5RuchJbbyJTPJTOjmmy9RyFTaafDY/S5g8p0p1jlEuOpH8d3tXOZDJNWHEhY4XluZ6uHlcslYo5Z097bhRr3/WqP9H06V1pgoTzE6eD7aY7q645LGoyOk/xEpvkdLfr+Tivu+9JOZY5qcysHkGirQRBb4tjzRuSKdDr2LuL4bQz2ojqP2tzMgMc/eZ5aBFTnkwxSRbp+Qe8EJit03/EJkuvEbnp/2d0eCjsa5BpFsHieMnGMlTjOzMrHY1K7EfD87gBxFkrJKC1r1Ey6qfB02N+FyhDP/f31Mi0tjFDlNWq2l/rCQHEaVc6WLOup8b28nmq1Rt0Wmf7fSqZ2Mg9Cm+Hv2g0AcPwgju8V74XhUQCUU0Cs88Z66zqoDvGY2kcynZrPY+qSCLXB+nbaXZhBzWeZjtK7mNpK8svudU37zPGY6roOLPa6iMpspTuOJV3ad6XRgKQGtVJeXfV6ALA5ABkAou4eAEDQTS/W70qjNJ+EP7GMF3GLqWP0LJzAMxZsAQCcVtgOAFiT2o3dPv2vh2YWAgDuGV8EAHiwbz6CNDcih1PBRt3whun+9jC/WJkso6itB3UjU9c1A48E2xqZWhbALoFBLzVwv5jG9AL6zpNLecBZwjJdNIELWaZPKT4GADgpvRO7fXovD5cpxfFtoysAAA/0LUCQ4uQoNsm06HfBG+FNkJS24We2/PZdyAEwNXksN5EiWDbJvDBOuvsaV4pCCtML6PtPLSGZTq0kmQ4uGcX580mW5xTo53x3DNtqVKvm1olVAID7B0m2OzYPIJTkMCFPmq4FzyHZNbZTq33HcQCcHAbc9xvq7CT7PorUnoM+OuYXU6adtur7T5u/DQBwVnEzAGBNerdppxvKVLPmzrElAIAH+uN2Gjok02LUDW8/9/3RBpk+/q/9hOIODdAvlhUrEyQtuCTbcZxm19Ssh+lF9P0nF/MCbjm17+4l4/j9RY8CAM7mdnpGZjs21eh/PTBD46jp+wMLELrcJyIeT/0CPAkaH+dFB0+UFtrbZc3lvt9SppLa2nGMy5/MUXUy5b4/vaxZpk8u0Lh6RnobHq2RW1HTeDqwAKEni10eT8MivP3sStck0/bG7qP+aNlxkg2p45icoyJxo+rhOaqQwtQi+v6Nc1TXwklcsGArAODcLpLt+vQO7PBpnmsl08BrmKNqBXhOokQEYkWuVasmQgHaE6lzaLkOIp/rYIkijN2ALcdGxGndJSwlyLgozWO5Lmno/4vHcfHijQCAVRlSaK5L78QjVRpL75miUhwbhqifbO4bQOhy4gnu/5HdhVSax9Q9XNcyWV+2jV2BHWmrngf49UvrutI2vTxPSf8vpDAzVD9PlZZSW+1bPIbz5ssalearVak92Fyl9/dAidao9w9Rm93QMw9BSiZ0SYjSi9QwydneS5ufOpn67TuuuotpzqhLHsTt1SQ2se3YjbLIMu1KYWaQN3CLaeybXE2f6182ivN4PfXU4iYAwCJ3FI9WqV3+enwNAOCefpLp6GO9CDkpiu2THDMAXFGASjOVMdVLz3pZl/YebRRFURRFURRFUZQmOsYiZ1xBatVEilbWkokZO58zqUVr3aSVKPe7mFpUr40fWEJmngsXbMTvd98PAFjojpt/1e+Qm0WPQ+Zoz6ad/P5SHnuH2FVtmnbo6QkX3mjDfriNrXB1iBtYtRbLsIVMQ9Yg1zgAv9LjYnpRvZZz3nJSS5w7bzOe3303AGCRS1aKjBWgx6b/1cOydTgCemQmh52D9M68KbpnZtSFO1Gv7ayjjYvYmmLRjh0/p5S2kFT5hTyiNB2rddN3r3a7sSVuDV23ZOl+AMArFt+BlxSpnRZZ09vr5HCPQ1rlHMs2zRk6JsppTNXIiuVN0vXelGP0cyZZgLh2VKrNhYLbiYP1/US6ZtNOu0imdX1/UX3fv2DBJlzUfS+AuO97CNFvU5/vd8iNxePsPMMzOezivu9x38+MufDGDiy3drbGm3bquYikpApr4sXtxuoqmPTMItNqj2s08WIxXriC2unvLXgYFxfvAQAsc+n+RdtF0SL3mEEnLjQOAGPlLLazdj81weNpsu+3kl8n9P1DydRr7vsHkumz5j+C53ZRO13uUpssWjaK9k4AwCCPsSGPlGPlLLYP1cs0M+rCHetMmZo5ynGayg1EkSSSSc77nNggMUfNLKbrBpeRNeL8+Y+Zvr/MpWM9dohuHkdFph6PpyTTxjnKgzvJVk6RqXjiAO2fQErkWrPiwuaSPEZCQ7q7TP+XJFEzA6m4ra7iuX8pzf0vXHwfnlV4AADQZ1P7d6wIXfy7rKeEsVIW4wupL8iYmppykBrmCySEQqwxYRiXJGhHqnFJHJN0Q9zTk3M/h1VI8p3ygIcp9m4oLYu9cADguYseMv1/kOemGmwUM/UytTn7yb7pPEbn0f1Fpt50C5kmaWN39WiKw3LSaVP6y6ynpORATzfA4TnVQbYcp2xMLmVL3DqS/aKlJIRXL/0tXlJ4EABQ5ORI3XYWOZssnp5F78BmP7BflzKYrtD9ZUxNjTvGMmjetXGjnZn1dtrGqzdFURRFURRFURSlFZ1jkRNcF5bDGjAJcOZkElE6hSjL2o4c7Yhn+m1Ueum6noWkaXvyEO28V2b3YTgg31qJi6tGLvKsmXM4b3yvS1akxcUx7Osp8v1l528h5ABr2xQeZU2dEzQVYW4rkgXUbSng20KmGbouSLOGc8BGpY9luohkevrADgDASdldRpabOY4DAFKs5chYHDPCFrmh3CR29pJveW0vx4tlbIRssTIy9eWn39aWDitZ/sKr18rYBY4DyGcRpVimGZZpr42ZIZLJwiWkOXrRIrJunJ97BBnWmu3j97MvnEYY0T2GHAoOX5mluITtfT24c5STfHAJg2CPbYpcm8KYYnVtZ2scEKf1tlr0fUlPfKi+v6i+76/J7sFen/ryjhrHgMJGxiLLSYotcQWHtIAL8hPYzbENNQ4s9zMWwkxDO63GhZalAG5bIgHuYRRrjaMGa6fnIuR4lcildzDTa6M8QN+rbzFZN582j+IMnpJ/FFWQHB6osayieIwJ2GqU4TzPA9kpbO8m2dc4bbmfb9H3q23ePoV0XGj2sGTqNMu0dxFZh0WmZ+cfQ41rjjxQpZjYatSs7c1xHvI6mRbYmpq3ke5Umcp4atlx4WoT2xsXTZa+b+ao/niO6l5MMj2b+/6pue0YC2h8lDEghG2s7zJHOayRH8pNYnuvyDQxR/H/7LQ5CoCxasCympLdmKLnac98x1qOri/3JPr/Iur/T59PcYZPym43MtsXknwnw4w5JpaOZRmy4O3o7sFYnv5XLc+Wv7SFSAqyN1iPrFSqveUq85TjxJ5EMk9xPFeUSSHMcYkRmaf6bJS5rfYupLZ67rzNAIC1mV2mre7zycumHHlGljK+DqVoflvSNY5Rjmv0syzTlGXWG6YAtsR3uW6crKcdSSYMS4yvAGDzuhHZDMJU/VZmpt9BeZBkOsjt9EWLyLJ5VmazVITCTp+u2WNNQ2KK+9nzYUGG3sXi/jFsGmUPvz76P/ndDjwuv2PPsHW7nEh0dnTf9qjpmI1cJOZ+xzFJJMzAzjUykE0jEit8iuvD5Cz4AzTJFTN0jzCiAeKeqcUYr9EEu2WCBmrPDtGVpsXb+u5dfD3dq+SnEAX1g4sdAHaJ/7+4qISJ4OGwfTdy4v6THDAtKY0hVeuzaUS21Nqh62p5CzWWaT5NP/2QBoj7phdja4lkuX2cFnOuEyLnkYxO66MNn8+LkclaBlFN3H7iZ7PLDTKVwaad3YAAs8iIqiEAdqeQOnzc7qJc2kxWEQ+oft5C0EPXr+iiiS7Dg/UtpTVm4L5nmpJvpGwfC1I00Ay4tJGrJRZ4ll0vJyuMYNVYhpKBjGUZBWFbb+ZkgIRtH1bfl3pwft6CP9i6nd4ztRj7yjThbZ+kCcFzAhQ8uu60XmqnItOpWhqh37CwCAG7VC/LTun74qaSzAIrbcBk3+0pAo7ULqNjft6C30/feV6h3v30d9MrMVwjF5QHxynBgWcHKHj0/lbkSUHhc8bF5HgaJWY+u1Kf0MbQ7n0/kbHULJTEDSgpU+77Mp4mZTrEMhUl4p2lZRiuUju9f4xkmnZ85Fy6fk2RlDd1Mg05oYnoP6IWMpVxqp0VjUhkS0z0JcmwG0nfL+bivp+JZSpzVHeW2nqN+/7d00uwvUR9Ptn3ZY5a31M/709W4zkqfgjAnmk9R7W7TIHYzT+qVs1GzpJMwHJRdyF2u+OvH2Qt+L302QVFmncKDrX7W6dWo8iKr7vGKbGJa4cYStN183mzUQrjDMmWWy8ruxYl5Fp/rt0zVptELFacPMQoGqWt5jJGpmZMzVnw++l8b45rQPJi6I7p5ahysbMHx+YBAFJ2gO40Xbe2sBdAPE9Vw3gNIH3CqUXxPCVtUxIHtfPGGDDPG5VmYiW5yfjNdfWKWUQe/R6kEmvULnoPi4u0kRMZ3VJaY5SKd0/QeqrLK2Met89ut94F2E4sTDnKipRwPj/bDLvUyxjvOLGiZJZo316hKIqiKIqiKIqitKRjLHKWuALZThyM20gYGktH4IlWHrAztI1Ou7RDH62SFW7D8CAmRyXAl3brblcVQR8HjrM7TNYhbUktdBCV+brEpl1cD7FfNDLtGzyaxMjUaZapqRuVkGnoNss0y1rMPWVyUdk80ofpfSRTu8Q1fYo+ugbIPXV7hjSgOZc+N1lJw5qpl6kVInYFEK1cm6dzNrDWyC5mEY5xAp0G7ZdVqgAZdlnJi1YOcLIkU581TbuqXAZjch52TtHvpSrXh0tXsYQ1TWd0UemM3VVyvRit5BBVWaZsdEUUP4dBXJQsq621yMb6Zjum9l0TUWRcR0Pp+znATte302EOWn50pB9TI1x7hvu01VVFfx9ZRPZUqD0X2PJR9j2goe9bUaKdSgpnscRYFtpa18nad9ux4zo4UvdQrLOlMsBuQLUcazoLcTvN8Li4p0Lt7uGxIewc5tTaJbqXk/MxwHUkXXanTibmQJmT8ST7vrhWtbBotrUGmWVqZTPxeNoo05kKojy7VmVayZTaUVKmuxtlmq8Zmcr1ojUer2QAHk+NTIOETBtSjVuW1dYyNd4MyTlK0v7LHOUH8bzPnji1fNz30yyj4Qr1940jSzG5j6yc1ozU8gvQO0CWo+0pmqNSrH4fLWfNHOVxuTkrQOyq3jBHtbtMgViudiFvkkaYc4m2ijwnkeH+Xy0CTp7kmeJkMKM+yfXBsfnYNUnjZo1rnHbnZzDFLmi9XKtXLCNhZCHidZcjyw8rIVexcCXnpjZ2AxSLJlJenJijwYvA8gPjUi3zVC1P4yQQt9WRKst0eL6paxiwO3S2WEGli+S7JEtJUVybx9aETDnfFCI7EQIga9Qk7dxWZT1VyJv1VOPq2p4oIerhMhnc/8M0gBTJpBzQmmlnhfr11ule006nZ+idpVI+FnaRRW5t116+rg8AsHeyAKvGpZu4nVphBEu8g6R2XTJUpTa7JR06ZHWsKIqiKIqiKIqiCB1jkZMdbhTVYi2d+M+KH31XIeEry37IqQipNH1WUt5v5Xi4yZE8LLYaifo8AlBM0ba7n9VvEz5Z5ko1Dw6nHOZ4fTiVEFZVnk185RMapDb26Tba2Wot9jsWmYqWvrsYyzQdy9RL1ado3cHxcNN783DH2cLBYgjzFlIuJ4/geBnRkpSqHpwSazJZ2WbXIli+fFh+xpo4q40tnibu0PdjTY1oFjnWy86kYdVYm8Mau2pXhGJhpu5em0uU3OCR4UFUKqzZ56/el5vBwiwH43JuYYk9mKl5cLmgumiQUhM+LI47NO+WNchRFLW1TE3sSbVmEhyYkgTS97uLCFv0fWmn0ve3TVA7nRrOwZ4Sqy/9iEIL+RTdry9F2uPJGv2/Us2DM9Wi70vcYaf1fYnnApr6flih724VcgglllPyOOQjZLIkI4k13jpF4+mu4W6Eo2yVEiNFzkeGPSHybN2c9qmdTldScDhFtoSVODOh6fumUG0bF6ytI+nVIDKVdipjQCGX8HDgSxIyFbZOk0x3j3QhHONi6TIE5oGUQ3+ITGd4PG0p03JSpkHdc9GDtK81vm6OEouHyNRvMUeJRj4VweN532Jr5ZZxnvf35+FMiIzEyySA65AcJP5Q5qiZxBxli0yrsUyNlahT2ilgrF1hpQpL2qqssbiMhpWJ445NnHwhQjZX772zc4bG1J0TXZgp1SekcIohFuXIc2TII4vHXi6NM1VNwxmvt8i5UwGsCvcVkWfUxu0zQXK8apQpKvwd8jkjS+n/QS5COkNtTizr23hMHRnNI5w2hYOIIiU1AoChFFmRR3yy2pVqKdO2HR5SnJkQdrkhRtbkHAjb2oPMrFUqlVim0na5ndrpFGxZg/NXqXYBmW5qVAHHD8uYumH3IHxZT3HytHy2goEMrfdl3T+W4qRmgW3KDNmcwMyb8uPyA5UWXkL27Mq0jVcaiqIoiqIoiqIoSis6xyLHWOlUU7ZFo6VHnAWwRi6zqHWH6M/yzlyyBibSeUXsR5vuoXuuHtqPp/dvpN/TewAAN02uBQBMltNGc+RN087crrXwL5bYozbOBFhHyosL2EoqYtF+Jvyn2TCJWleEXpZpJeA0+kZ1BwQZ/kwvaSoWDY3hyQNxyQcA+N34cgBAuezBmaHPelMi0zD+v41WjXYuCIoG+bGG3nIbupllIUrHqcEBIExHRiPc7ZGmaRtrkADA5iyUC3pJq3l2/1Y8s4uKWvbbpEF6aGYhANJAcTZyeCWOywsjE1tiZJjQyrWxl3ycwjmdQlRia5F8B4nzTLRTTpxIfZ+1x5KxzvT9yEKYIXl7rLlbNjSC8wco7fsq7vu3Ta4GAEyXUybm4KB9X7AtWLOehPgoSHlxLApjxlPLMprFShePq10hBjmzWobjXEfKUqDZQsRxSaINXTW4H2f1UuH6BR5p5m+fXA4ACEMr1sQn26k8h1i0E3ExVido51vI1Iyntm004NUiy7Q7lmmKY2T2z3Ah8cAyc1SG56ikTBelKEbmtxMrAABBYB9CptwPJLudFcGaZe3xESHPnk4b7byZV1vMUTUOea/2xH1f5v0wlNSLFkKeo+we7vuDozijn2KNl6Ypa/AdE0sBAJWKa+Yol/u+5Ufx/5WxKDHft7FE67AyGTOmmvUUezxFrmPi1cq93FaLIeZxWxXvmv2c/deyIjjseTPQTRajM/t34MLuhwAAS12S68/8U8z/t9ki6lRYrhEVfwcS77kqrjrtvZ4yXi2eG69R5bu4XHLEthDx9zJjaiFeT4lFrhrw52wAPKbmu+meJw3uwbP6HwYArEntBgD8auokAEDZd2FXWaYz8XpK4vKayiK0/Xoq0ddlHpBY5ERmSIkBNN446QjyzfrZ0rZpnDycLCvO6t3fS+10ff9uPK37EQAw5ccemR4CAPi+A6dM901N8treT66nZFw5fpmAO2YjZybzmSB+odKxo9gNT1K6SmKOKJUY5LlzuOyWki1WkOW05Cf3U4Djxf334tQ0pR4fC2lB080ruFrNNQOPlagRJb/HCUIkrXubp8uVBlcrtzgZp6n1M3GNFwCIUoFZEFdEphxsaxdrSGdoEbNygFz+nj3wMM7O0QJZaqFsTg8AAMLAAXtemdSuVpCYJMXtp9bgvtqmhLJ5q1bjd984MVnxYG7cT9Px9xrlQGdxCejOltHTS23wnN4tAIDnFe/FPIeObeK6fVMBDXDjUxm4PPDY7P4TOvaBSznYdlsHkdf1/cbEMSadeoQgU5/sJEpF5itWfBrqZLPsFSvIszvbmn5SLjyv/36ckaEF8nBAC+kiZ4up1Rw4gcg00QZlHJBJXNxA27ydGjegKDLPbFxX2I3ZCsJYpjxBRtnAKG1K7CLpyXiaqyDPypvVPfsBAL/X9wBOTdMCeY3rP18AACAASURBVJtPweNdPJ76vgObg8idWryQawq+l1TpUdTWboCHJVM/iOs7yXiaOYhM81XkOAGPyPS5/fcbme7wKYC/hzObHFCmQaKvA/XjaTvLVPp+KZGQQ8bRRN9vnqPivj9To3fg8SbDKdaQbej7F/Y/gidnNwNIzFEpWviFgQPWWRjXyro5Ski20zbHlBcKgjhJgygcA1msBgg5nbu4A0bZEDWelyQtvtCTLSPHCSOkdNNLe3+HJZwdal9AbVuSHY3PZIz7r7gNh64FS9652WRI0pP2naOAhEx9P5EAR+aFOJRC1qiyuRLlFxDPUx6vp/L5MnJ9dN9Tekm5+JL+O7A+RevVfQFp16U2Z6mSMm1U1lOwLeMGLGuQMEqsBdrZtVLGqZmyWV+b5Gem5E9iLS6HUhFc/kPKDEk5kXS6hkFWNJw39BgA4JzCJsx3KVTlgfIiAMAEh1VUp1Mo8KsV2UaulVhPNbhYA7M+prbvLkNRFEVRFEVRFEVpScdY5IzZv9FNDTCamyiThp/nwojdfC4dGAtcgZOY5Hhn3p0u48IBcqN8duEBAMC5GQfjstP26wuJRhEQcXBkIEoB10LIKbqdsfY2UzdiXAEy6eaTItN0CrUCy7SLz2UCOGyaLnByiDTLuDtbxoVDJNMXdN0FAHhK2sN4SFr4xyx6B7dgNT9DJHUdEUjdbNeO07of7LnbENM+bbvJBUiCyJHy4OfpunI/a+XsyJTHmKjGrsIAtdtnDGwAAJyTJQ3SmekQm7hI7XRI9x/j4vZBzYEnCkFRFqVthEW6rz3G73sm4arUzm7A0vdTXnNRaKMpTsGXFNnSTtMBPLbASd8XS3JvZgZPH6R2+tzivQCknVI79iwKIg8Tlaqb+r5jxaVHWtAZ7dQyKZ5NG+AkOGEhg2o3nasW+VQqNIWTJf24eD8s6x017fTZeXL7PSudwmggSSDItUrSj0ehZXx9kuNplG4I7ud3bIWY9UKrR8LhyrTGqcNrLFM7HRejlpIOQr5vxLj6P4vnqGQ7dUCulUamkamTC5+HkdC1EGVZpqOInxEdIFOxynhuwoOAn12sC+mUmfclpALpeI7KezxHcZKZ7nll40L94q47AVA7lTlqo0XX/xLr+CEik+xH2ukh56h2Hk+BhJeIUy9jwFhowq4cqtxWqz38OTdEV6o+2Ykkh0nZAS7sJ/e0UzLk1XRmysf+BuNEiSf6ILSb5BpkbAQ8T7mTZMkziYLa3MOpbo0q7soNrotRNo1akWWaWKNK8qIsm35lLVBIVfDsAXKjfGqOxoELMjZGA/pf5YgT84TUv8PQipMocf8PUjZCLiMhyWWkfUZtbuW0UpzoKVFk28rSOicc4cEs7SEo0HXlnjjER8JRxsskCJn7M56Ppw5uBgCcW3gUAPDk9E48XCML/CR74k1WuVFWY3d18cYJ0g6cgsiUXZHF9dtyNNmJoiiKoiiKoiiKcnDaVxXXiGgOLCv2RfZIC2E0oa5ttJEmVXPFQZmLKE/YbJFg39mlxREMeKR5X8xxG+Ohg8fY0vFQdT4AYOP0IACgOuNBdMVRQtEi5Qfi2D1RMx/1t50VjG+8bSEqSxYXlqnNGl7PiSNkxRhScVD16XyJ4w/kkpXd+02iiOWupGX1sJM1GY/WSJaPTHEgacWBKx9OKDGMTKN6LSxqAaJ21solYvtE2xVNc5rcLlYXOzYCjj2QGAG7amN4kuKyZtjK43P84aLucTgs/Co3vP+dKRpL3F3TywAAj45T3GE0noq18hwzkkuUyTCxnMl03u2smZNnq8V9XzR1Rv1oJ/q+aIArDma475sYTu77y7uGTfKNJaxuK4URtnAM7EPVeQCAR6dIpn7ZQyqKradCp7ZTI8dMGlFjUh5JIJOQqfTNaMbB2AydF2ulGU8Lo1jIMl3Iua9LIbCNtcePsEw3T5Pm06+4JiBdxhYrRJx+3Iz58hBBs0W2jWgpU4mRSyQ7aZRpWHKbZCrtdXF+DItTZMlMttPt3Oxkjto8Fcs0Jbc/DJlGkR/Hz7UhdXMUW2ZEppEkw/EcRI0K8IqDMhelnrBJ9uI1sqg4grUZShIh7TSIXDNHSSznJu77QcUx834yK9SB5/3ouCQ8OCJkbkrGR0tcZ4GzRVmWsbYnx9S9UzSPVdlTSRJzLCmOmdsPOrSu2lCLsDegpF0PVygZl5TVmZlOwZVQfCeOP5bSPOa5xLrZxuMpABMXFUURohnu/2L5zJEVKXJjK6RpS1Ub0zxPeQ4XYGeZLu8aRo6Tb0is4S4f2MYmzIcqCwAAm6aprZZLKXgSOioJTqIoLpPTOKZaVlvL1YypjmPWqOEExbfZec6+Z9sIOHlcnGcBKE1wjJvEebNM5/WPI8ueD8MBteUfTp+MCls1755YAgDYMUwmU2/UQSjVZHg9lRkO4pIO4nmTXE/N8jzVORu5pPlfBhxXakGwi4XnNA08VtXC5BS9cOkcxSyZQMuBZ2pv3V2ljuAhwFhIySY2VWizMVrJ8f+JXSxM8K9txS9SntHqkIHHZNty4g2cJOGQwdO1EXgyyNIhq2Zhaoo6iSTkqJcpNeh72cetaJex0ydZbqrSRk7M1rYbIuCENPJ/Qi8xuDTI1HKc9g4mT7jTWZJdySyMeUDJpeLkESLmSQvlEbquwm5QmRwNYqPlLO6foknQuFDAgsfaij0VkrPZb2cCwOIJRPaVnhUn6JFnTNTmm21XgCMi2fcbMsVJFr66vi+DeTVupz5vJrpy1E5Lfsq4UNxdpYVFxqqZgf0x7vvj1Sz/nxAhz5ChBKm7B26nbd/3ub/DdWP9iZEpZwDLuKZPGplWbIxP0EKvwkoc6fuTtTT21Gjyu9shmXqWj3JE1+2s0YJOFn52KoDP2QODjGQbs0zWOuOeWJNNSHsnkjDKhaRMZdxqJVMZT6s2xidpjmmU6XQ6hX0++WDeWyX5pawA0xH9L5FpmbMH216IIC3uP4eWqWVZ7S1To6R146QCIlM7nqMkwZF4ploVG9OT3PdZ6Zjs+yM+9XPp+0V7BvsC6vOPVEjhMOPTu7C90GRilnYaei1kKv0njNpapgDqleBmruf1FMs1yHpmTBXsko2JCa6rW6Y2mOeMi2OVLPZwjbiHqrTBKIeeqeEp58ZkTLXj+U8IXQtWYzZQs5i321qRU5fUTGQproH8d5hym2RqVWxMcP+vcrKT7mzcVuP1FK1RHUSY4LlrO7ffEV6j2k7UJNMoObc3uHq2+3pKNsJRFJkkJyZbpcfJc/IZ+Nn62pzelIVgnNp4rUrnnDyHrsxkcM84JTQZydFcZiOCze1U2mea61BOF0NEo/wckmfJtZuTHTUmkZtF2ny1oSiKoiiKoiiKojTSORY5JipX4l26pHtmV4sw5RhLHBdnhzduo8rap+lp+rrTDu24J3oyxpVlR540m71uyaRxH+aCVJMV+jusOvDEFY7dMKwgYbZupI3TOieJqrXYAielE6I4qFNkypZ9eBMJmZbpXUxbJNOpvjhxyvYCuaj0udMY8UmWYj2aSsq0Vp/W3fIjwG9I3yvP2s4ugElsO3YBlkNFdq2MIlODzOVaL27Jgj/D2rJprt3DSR9msnHyk0mf5DaUnkKN1UPys1SVVOeW0RwZd8MorrFoNVqLgqCttXJCsu8b1yDuY8l22qrvl0rU90tjcd8XdhUokr/bmTF9fx9n95iustavcmTttO37PltkIiSsHg2axMi2jKuKyNSfsFHl8Xa6VD+ejndl4bO71a4iWeaGUhPGZWUvy1Ss8aFvwxNLH8vU9iPYje6q8jxt3vdNf4+i1km5cACZjtuosnbZyNRmmU5n4bPXw64itdMBbxKlxjkqKVOx9J0IMpXnqyRkmiiNA9TP+/EcZZl2OjNDn5sZ4yQGvWkz728v0rzf505jnAultpz3uTaXjNuW32LeT5YdancS46cZUyWpSKLt2uzFwREoCDI2yuwxUmVLe5XdAScKOeNmLYm35qcnjBfJNM9do2V2MwxjzymRq+23cAMUudYkaVJ7E9V845FhxgSxyHm28YxJsUz9nI0qW+6mpkhWUw5Z2EZ7sua+ewq0dup2Zown2RgXTjT9v2bDkTVq7SBrVO5DZg3dphjX6mwmrs3H2Jz0xKoF5rtKUhKnDFNPz6qwNwSvr6bSHh7je0zXSI79mWkzzgpiyUcISNnN2OsnavbE86Xd1jTZiaIoiqIoiqIoinJwOsYiZyWrqEtVevFDZotFZMepVyWVvRUB7gQXC82xFiJFP6sVDzumWHOcoQDKodQE0hwIF7I5I8OpYC0nhERV21XWAFRbaN6Tu/E21s7VpUgWbbxXL1NYsXVHAj6tICHTTH3h9fJMClsnSMs5kCaVc9GJNSnTXPA2JVXArVgrx3HncCph7CffmEQCaGuZmhTEjt2ssWGZhhkPtTz7dHOsnB0AqTG2sBW5nYqxxwlNwp48J5BZmB4zhVVHWSs3VKA2PJ7LxYkAJGSrFsZqG3kuUwA2gIXZ1SAdCa36fqt2arRm0k6TfT8rfZ8+X6l42D5JFo5+bqd97rTRKPssOEkLbTmRiTc8IdqpxMikPFMc2gRrSzIJzzYyldhgKwTcMZEpa8rTPJ5WXeyeJqvb4hwlPnAQmVhOGU8l1b5lR4DRyNNPpxzGlmN51qS1s50tx8m4I3n3RyxT1uSzTGtVFzt5jjIy9SJTANjnm9TJtLGdllvMUR0iUysZxy0ylQQyiXk/jl3hz4UWXE61HmbEckdyqJRT2MZzVG+KElEl56hJLgQsfR/WIfq+YHeGTAGY92+lUs1y5Vik0LURpCQmkD8XAqkRkqufZ7lmJMkHMF4h2a0ucvp8p4xel8bXvSCLUm+GZL7fKcZJjiSvSau2KvNUstB2OyJzf7JvyTyV47bqxPGcUnKhbp7KsyzZbF+rOdhbojF1RX4YALVVjxMWiAeJWaO6oWmrktPAqSRkJvIzMd1+W7dVyTOAMIoTR8napkDrHr8ng2qR22QuXk95k/S7tFNpY5EbGmtb0eNEMtlRBLxAkv4/ViSL346pFMJh9gKUEEOLEtfwr8QBSk7MBmqRUxRFURRFURRF6TA6xiLXGLsFUEpi+oU1HFknjg2SUJoU4BdFO0raCC9H2ststoo1PfsAkIUDoAyL3Q5pjKTI6sMRZbOKyo7xFXcrbJErB7BKpM2TlNNxOmK7/bPXAa2z7LBM/VyzTMMU4HeJCo1jvPJc0DZbxWJOQ9zDAQtpu2a08pUsNblN45RtCVUb3hTfSmRaCWBx+t66YqBA+8tUMuw5iSLr0k4lPbFrxRk6xb06B1S76HzI7dXOkkqtp3saZwzuBACsylF7XZnei4UeFcS8Y2Y5gLikQ1hyjTbKZQ2n5UewyhwXJc8osrXs9s5amWif0v8brTXJvh8m26n0fdbGuzmSaSZbxfJuSus+Pz0BAMjZVeRsUrlL398QUZbVqGLDlXZaPQHaqWhma3787uWnxMd6carspLUzKNZ7NkjfL+TLWNu7F0A8nnY7JWM9Epk+GFFWwHDGRWqC/qc3Q/dyygHsxvFU4iQ8t73TuotMg+CIZWrGU69+PE3KdH6K2mkrmcoclZSpm5CpaaeNMm3zrHWGpHXbqu9XLecoL2qao2zOWpfOVLGoOA4AGEhTp87ZVaRFplm6yWMTFOONaqLvt5qjKmymMwXL27jfC4mSA01jKvcxsh7zPCXOG5kIfoHP57koPc9TAz1TOLOfCoGvy1F5h/nuOPodEt69oLTu99mUgdmf8lAYp/unplu0VYk5Mm3VRlMscjsh87ttm98tJ1V/SaqVTOP+b2XoZ5ozVnflyzitj+b+pWmarwbdCdP/xcvhfouyhIbTLryJ+rm/bo1aq5+n2r3/mxi5lAdIsRrJWmuy2MaeeEItD9R4PRVwe7W4neYKFZw8RCWyFmZpHFieGcaSFFk87yxROafNU3H/l3W/eI60XE/J2Hoc1lOds5FjrJQXL5qs+knSKfkIHU58Ip2kECI9wO4TORL0ih56YUPpKZxW2AYAWJWiF+tZgUnpPBPQvXaPkmnbG3aRHqX/lRmmRuFMluMJvDFtbhC2tytA0k1N3KsaZOpO+4jmsVuQ1CbLh/B6aGAosEyX99Ag058uYX2BBvPlqf10e8vHjhp1igmfFnH7WabpPbFMs/vovTrj5fh5JEV+h8jU4gBcuG7csU1Nvri7iSw5IzOq3SGseSTTnjz9XNJDi+E1hb04p0Dhuedktpl7jIX1k8SWUWq3VtkGe7MgNSWDuR+700gdOXFVkP7U7jiOmdRb9f3IkfpSdMrPh0j1kywlRfbKXur7/elpPClPE+RKrnvoIcBOX/o+LYz3jtALSu1zkRnhdro/0fdNQH5ntVOz2Ex58ftvcAV1KqFJXS1Nzc+HcPt5POV2uoplujA7jjPyWwEAJ6V20b+xQoxwSYcHQ1rA7Rjh+jz7XaTHeDwd4UVhUqYN42lUrZ1YMpXaRPkQXh/JspVMT81tBwCsT9O4alsh9gU0fpZbyVTmqNHDkGnQ3jI1m4ykC3BDshN32kc4VD/v+/kIbg8thnM8Ry3rJcVXf3oapxS476dIMZaxq9hdI1frLQGled/Xao5KzvuNc1Ry8dbGMgUQlx9JeUAlTtIDwJQjsPw4lX3A05pfDOENUf/vLZKidmU3tdV1hf/f3r31xnVdBxxf55zhXDhDibpQstPYdRK7TXNpWqBNnbz0td+j36vQS5HHFg2SFgEKJDEKI09p4dhRLEvUjSJFSRQvc7+dPqy199lzZkRLwFg62/j/XkSRw+HMmr3PPmtfH8tP2rdEROS7dY11KiKPbZ3L0C7Md62dys4yX//rp1ZWu6PiNbrP2Z/NWe1jHZLgmBx/j+qWBVh7lY5mRWeDK6udmWzuaIO93dbYfuei3ju923ouf7t5T0REPqhrh04muTyYall1G/Q8PNL/154H9T+8ppaXAERS/5NGcXyDS5jckUN5rZjC6K6pduKVTLZymV2xwYWOlu/rFzUb+9aFZ/LBpsbyHzt/1MfnNX9Gb2rzfQ9OtP4nw0zczGt3P5WOpkU9cfdTfqnK649nBF1HAAAAAIBQPCNyrldmNlseAbMMOJnnftqTmyiQp7lkmf7cjcR9uK2jG+83DuS9DR1JatvprLcmV+SPQ+3l/NX97+pz3NXtiLfui3T2bXrGofacpGcDyXv94rWJFCNds9kbWfj40oKYui2Tk8SmBLqY5sU2tv5gyVRkw/YNdyMc/3BJY/rt+hP5s5r2xm3aOPSt8TW5PdQRjo/uf0ef4r4uKN3cz6XzSGNaf6bdHmm3L/lg4F/bgvm80lNXylOXREQSNz1gqD1D2elYara1da2nMR3u5JKlGnM3EveDC9pr/GHnC/mbhvYgXbcpm3cmE/mo/xciIvKvuz8WEZHuHe2Vv/hFKu3HGrfNPSunJ31Jevq1n542CaYE5RWesuKnp60Y5Qrqviunvu5nRd0vl9MPGo/lnZrW/aZN+709uSKfD94SEZH/vqexlfva49l+KNK2ul9/ck7ddypeTv2C/Ol0ue7blKZ0MiuOWpgXdb9e15+7USN3Pf1R6568V9OpKlv2md2ctOWT4TdFROSX978nIiKzXR2h27on0tmzqa4H2iOdnvQk79pwcjmmFb+evkxMk+ncx9RPZlsR059eui0iIj9sPpB3ajqlctuK081JWz4b6oG2v7j3fREpxfTR1yimwdEYfnTOTa0M231X9/0GU7nUrI361iWt5z++dFdERN5vHvg2qp1oG3V7siO7I51G/esH74uISGp1f3O/iKmv+6fnt1FJhWO6IBzldrGzkcZsMJHMtrXPRm4ZS9H2v7+to0Y/2day+vetO/JBTeN5KdPy+Ol4IL/p6X3Uz+78nYiIDD7X0aPtWyLtA32u1p5Ov0yPu8E11d3n2ZTF2azS11Q/WjieLI+Au2vqdO7vUcNrqmun3EjcTy9qTL/X3JP3bF7vRZvZ84dxQ37X0/uon9/9gT79bY33hd3iHtXX/9O+5Gc2N7h8j1rh0TgRkbxv9yxZVox82c+SnpaF2lnDl9NaXx8zupxLYlOqdy7oe393S68DP9p6IB+2NL47tozq9uSS/Lar5fQ/LKajO3bcw24i7UO7n9rXx2dnw2K6qn3Gbop1kqUir3mfs+rWCgAAAADASvGMyAWWerts7mztaVdaHTuk8oL+O9nKZLytX9dsn/vNVEdN5pLKwVTnwbq1MZ/0vin/uas9x5Mv9GfWOSLt/Zk0n2gWnp1YD+dkurT4OtyGNKny4txA4tZvuXnpYUzb+jMX02knldEl/bpu2+Q2k2IE6tDWbzyxw7//r/eO/OqO9nbMXc/RHX1sZ28qjafWy3FsvUbToDvDxdbPl08rHVPfg5wmweJ360mymKajiWw+1t6bcVt7khpHmYy2Lc4zLd9v13V0oz9vyN2pxu3XfV2/cXPwDfmvB38lIiKnt+xQ20/172weTqXxTMt4eqK9m24BuYgUI3HORm31hjcV5Ou+6+n8kro/sZi6cuo2M5nlqRzMtHw+meq/59X9zqOpNJ+eU/cjK6dekgYxtdduI8jZ8760Di2mW7pWYdJJZXh5se5ftRGjWZ7KE1tM88lYR4d/3/9z+ffdvxYRkd4X+r3tL/TPdB5NpelmNriYTmcLx2IsvNRardJrZPx1/5yY1o56xfXUYjptpzKymDYzLc+XbYOIWZ7K0UxnL3wWxPTfLKZ9i6krp1t7rxbTKo/GiUgRPwmOISnH9FlPmlb3Rxet7ndSmVzS99y0USJ3xEAmuRzPdevy23Zd/aT/jvzijo5uTm5r3Xdt1Nbeq9f9qvPXpjSRRErXVCsj6elANg+1fXJldXwhlfGVmv2qvt8dq//9eUMe2O/+Zqhx/X3/+8v1/3P9M+2DmbT2bdTozEY3pzPfhuZTW5jkNreoeFkNr/e5e63zxdGa7KhXtFPumrqVyXB7Q0LuHnU435AHUy2rH010xPh/e+/KL62dGt+yUSNdmiidg6k0H2ubn4ZlNXVrtxfXcyZS7c1O/P1UlhUbn7iYuv9PZtI+0HhNm3qtnLZS6duRBD07ummnrtfU7qwpn450RsO+rYvd7V+Vjx++JyJF23/lM/3TracTqR8Xn5+I3U+50UxbD5mExzq95phGk8it3K3MLSi1qQBJkkjznjvnTQt4bdCQ06FWhI8HOhz9p2taIXbaPZnZat7HZ3rhOXu0Je17Gpadu/o324+0kGwcFdPTXLKRD4e+QVnahKXiF3QX04UbTnsPeVA4mw8sphMt4NmwId1STG/u6NTJa52u5DZNb/9MH3/2aEva9zWmFyymnQd6ka4dD4qYut2yBgN/wSkWDVf7Iu64i8vCuUfh90QkTRKpW/m5MtBpu/Xupj+D5082tfexxe9yu+/PNzu0cnp6sCXtu1ZO71k5tYtZ/bBXJG4DG/6fzvzmK36KkrsxqvJ5Z7K6nOar6v4DfT9hOT0bvULd319R9/eCum9TKb4Wdd+VgXxeJCDlui8idkqfpJb0bvTqcmLnFv7PSGP6ucX0rc6ZzK3uP+7qZ/B076Js3tWG9NqudaT5ctpdjulotBxTqzdJklQ6OfblNJPlmLqbDlmOaa3fkFOL6UejD0RE5OY13RTqZWPa3tc4bjzprS6nL7ieVjmeIrLUGSYiyzFNU99GJVPX7tflbKSdCr8bfltERG5d006w651uEVO7xh7vX5BNq/tXXRv1UMtp7flA0nK7H5bTSdGJqa+12nVfJJgGOAtuOIfWZvhNZXJp2OOuDF39b8rJRNusj4ZaVj+/bm1/u+s7IZ/19TFPH25L547G9brFtfXE6v/BWVFWXdu/ov77m/mKy1e0o0u7xJ4k0tT9oCQbatmr9VtyaomvWy5x87rW/6ubPZnY9rbhPWrH2v5rFtPNR67+dyVxn2NY/+0zzUv3IiLVvgbMLX5po+E7bX2cx5pcpUkiG5ZUXe7bFOjupmQjvUY+H2tH98+7PxQRka3OwN+jdnuW+B22pHNP6+2lBzbd1zbfqz8p7qfctN9cpNjMzl2jXEynpevBa1D9Kw4AAAAAYEEcXR1SjB6cd45QPpmIHOl0tIYNZddOtqT9SLPu/h03pK1nmO03r/rf3ehpVv32SS5Nm5ZWP7GM/1R745LRWPKB60EqjbaILI8azedS3UHrYERm1bS6cDrgU918o2GbdVw+7khnX3s7+7tahEYXLKb1K34Nf72rz/uNk1waRy+I6XiyHNN5XkxXiSym4Rb/ufvab/VrPWSTiciRxrRm5fTicCqNY+2V7++5cqpHNhxcuCw2ICd2NJ9cPctl84n1Ptmwf+3Y4jgaF72qTj7XraYlOH/F9XS+gR6kV/HSdX9VOd378rrvyunbpy9Z9/1mRvNoy+nKuu/POwymrjzTTSHc9XTjeUc2D7Wc9u7YdKuLOiK329rxT+WOv7h+MpfWM+vZPNbYZicW0+HYjwL4nuLwMy6NGlV9s4NzY2peFNP24y+P6YaV07dOcmkelWL63KZQjyfLMQ1fT6n3fdUoQqWUyuQq+Wgk8tS2Gh8GMT0otVEXtc4/bF71ezvVzyymp7m0ntnvlsvpeFLMbHDlMyyn5TJZ8bq/INyUKXXXWbcxz9S3UxsW10u9C7L5xMrqrpbV0ZaOyO22r/mdKGo2gHn9eO7LqrsHyE41lslwHIyuBpuDuGtpVjp+oOrT/10cww1EyqNdQUzrVv8vd7ek7e6nXDvV0bK619rxMd0YWDt1nEvzuf5u/cg2iOtpbJPhSHI3IhecFSilM5f9NTWv+JEOwdRKd2RG+X4gbPszmym2NZpI3UYw2/tuKrv+bNJp++dvu2MFznJp2TXEtf3ZqcUxuJ9KXB2ZTIr7Jxe/N3g/Vd1WEQAAAACwUjQjcr6XIcuKDNh6PvI0WAw7sx4eO6QzG46keaaZePPRNt7VawAACP5JREFUxsLj82at6KEY2Xzswdg/v19n5EZSZvNiowg/yhJsie5eV4XnHIeW1vWILL2HJEmLmNpi7mwwlNaJ9iA193QxtBt9ylsbvrcz7VtMh6Pi+br9xb8jUsw19oucgx4N16sUvsYK98o7eZ7795iPF8tMOFo2H2lPUjIcSWug6xCaexqH+aY74DqVchdvOp5K2rfndVubW69pPhwVMQx7M0sLnYuDdeeV7u1cqPuOq5Or6n5QTn3d39fe45XldLii7rs1Me4zzPNXL6cV5tb1JFmw2H28uH4imU6XRiCSwVBapxbThxbTupXXeuZHjpOJlcnB2F8f/XoYd92ez4t1zq6XdTpd2hJ7oWRWeLvsl4qpfEUxdfXhReW03EZVOI6hlWukS3VtoY2yh2i779ooi2lNy9i8UfPXwHRgdb9fbAa1qo3Khxbn82LqnyCCa4CLYbCW24/EBe2U31p9rDOdksFAml1rpx7aIewt26gjSYr37srnZCaJu766g8ft33wyWV4DGfzN8rEIeZ5Xu9zaGi5J0+Ia54/McBvgpUX9d+vm+kNpneooUfPRYlnNs0zEvvb3qJNZMUI0HC88Vz6dLq3JX7hHNX4kPnitVbSwaVTYbogUI8jjiSSZvp/5oc1M6G5K08qdu5+atevFE5dmSiTTuaRni+2Tv58aDIo2yH12aRJch0ozMea5hJs0vQ7VvyMGAAAAACyIZkRuoUeuvDOc+1k439/1MomInOi2o25ubRJuxZwGPcHl33XPOwpGBPzIheu9SoteOjfy4lS8t2PlDnuuNyKMqT843HpEZCpyZtuw2oGNPqbPgnUsYS+x+2xsnrPv4VzoaZ0Xz5WXel+cqsc0jKWf679YPsO1Pr6nbjTy6ztTW8uWdq1c1TK/5a5/77N50Qvne6pevDZHREQadjjxfMUW5El1R+R8/MLPPvyeyIvL6Yvq/vPg/a4qp65On1dOs+Kzjq2chlt5+/US5Z0M53nxdbDlu9u5q1z3F3oFg63MfQ+0G3VzPfNZWjyv69wMr6eltQZJklR7PccaYir2b2rXyYWYBocM+9i4WIUx9Y8Pymm5jYpgVoOIrN6tuBzTcOdVd03M5yJn9v5thC1xMc2X677keVFOazaSsSqmbhQlS4vyHOzyqH+g2nVfRBZHzkpruf0I2Ko1qbO5yLEeN+A+h8ziunC4fDCq5sqcmw0h4SirqwPz4u8k1v75mSt+q/xgxK+Kwli59t3938e0KBfFbsxTyU/P9Js929k6vEd1vxuWKVfO3ePc+u3w2KMkGA2s20hfqf4nSRLNek5/RJbx9S5N/Xv2j5mMRZ5rOXXlqdYN7sdc+fTt1Ir7KfeHwh3IwxlOGzbC548dKY70CMvz6xBPIhdefEvD937IdSM4i8NPaZmLzG3xdynBWHguJ7wYlW6288momHrlGsY8F5lPFx5f3GzPqn3hCc+NKTWY4Ta1iZtBMg0S2nKyG04rOCembsMIvwHIipjqSwsaRZF4YuoS4Y1aMS2o/LN6vdj4JEhwE7EGz3UcuIvSPJfcbT4TnlniYl8LLl72eL9ZTbB1v9+u20/zCKctVPhyHpZTf/O2eI5cvrHxcuX0vCmQYTm1b51XTheSisjKqa/fSbL02vPgfS5PEZwXm2mUr6df8n6LabAuoZsudxyJSJ67TTpK04HCjRAqaB0xLTac6BZPXI5rnhc3d+4hrpxOp/7zWCynpcQinBZY4XK60Fb7s7kWYxqe15rPghvV0tKLc6fph3W/VCZfHNPSc0RS90VkcWqlU4pr2mgEG+a8oGNVgjYsfK5wiUbwOS0od3yLaCzL7aY7IqHqyXFpyl/4PV//6/XFciKlmIabaZS+t/D+/T1qKVbjoPMrLKPlWIevq8JltTgqqVYssSh+qD8LO/hsOqTG2eLbs/fuk+Oi42fhiAvXYeCS3vLfE1m4h3JJsW+7NoJ4M7USAAAAAHCepMpTVQAAAAAAyxiRAwAAAIDIkMgBAAAAQGRI5AAAAAAgMiRyAAAAABAZEjkAAAAAiAyJHAAAAABEhkQOAAAAACJDIgcAAAAAkSGRAwAAAIDIkMgBAAAAQGRI5AAAAAAgMiRyAAAAABAZEjkAAAAAiAyJHAAAAABEJppELkmSf0mS5DBJkj+86dfydUFMvxrEdf2I6foR0/UjputHTNePmK4fMV0/YvpyoknkROSGiPzTm34RXzM3hJh+FW4IcV23G0JM1+2GENN1uyHEdN1uCDFdtxtCTNfthhDTdbshxPRLRZPI5Xn+WxE5etOv4+uEmH41iOv6EdP1I6brR0zXj5iuHzFdP2K6fsT05USTyAEAAAAAFIkcAAAAAESGRA4AAAAAIkMiBwAAAACRiSaRS5LkZyLysYj8ZZIkD5Mk+ec3/ZpiR0y/GsR1/Yjp+hHT9SOm60dM14+Yrh8xXT9i+nKSPM/f9GsAAAAAALyCaEbkAAAAAACKRA4AAAAAIkMiBwAAAACRIZEDAAAAgMiQyAEAAABAZEjkAAAAACAyJHIAAAAAEBkSOQAAAACIDIkcAAAAAESGRA4AAAAAIkMiBwAAAACRIZEDAAAAgMiQyAEAAABAZEjkAAAAACAyJHIAAAAAEBkSOQAAAACIDIkcAAAAAESGRA4AAAAAIkMiBwAAAACRIZEDAAAAgMiQyAEAAABAZEjkAAAAACAyJHIAAAAAEBkSOQAAAACIDIkcAAAAAESGRA4AAAAAIkMiBwAAAACRIZEDAAAAgMiQyAEAAABAZEjkAAAAACAyJHIAAAAAEBkSOQAAAACIDIkcAAAAAESGRA4AAAAAIkMiBwAAAACRIZEDAAAAgMiQyAEAAABAZEjkAAAAACAyJHIAAAAAEBkSOQAAAACIDIkcAAAAAESGRA4AAAAAIkMiBwAAAACRIZEDAAAAgMiQyAEAAABAZEjkAAAAACAyJHIAAAAAEBkSOQAAAACIDIkcAAAAAESGRA4AAAAAIkMiBwAAAACRIZEDAAAAgMiQyAEAAABAZEjkAAAAACAyJHIAAAAAEBkSOQAAAACIDIkcAAAAAESGRA4AAAAAIvP/C4U/CxPrrE8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x216 with 30 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_, ax = plt.subplots(nrows=3,ncols=10,figsize=(15,3))\n",
    "\n",
    "data, labels = next(iter(train_loader))\n",
    "data, labels = data.cuda(), labels.cuda()\n",
    "# get sample embedding from the VAE\n",
    "with torch.no_grad():\n",
    "    mu, log_var = vae.encoder(data.view(-1, 784))\n",
    "    z = mu#vae.sampling(mu, log_var)\n",
    "    z_tilde = obfuscator(z)\n",
    "    rec = adv_model(z_tilde)\n",
    "    preds = pred_model(z_tilde).argmax(dim=1)\n",
    "\n",
    "for i in range(10):\n",
    "    ax[0][i].imshow(data[i][0].detach().cpu())\n",
    "    ax[0][i].axis(\"off\")\n",
    "    ax[1][i].imshow(rec[i][0].detach().cpu())\n",
    "    ax[1][i].axis(\"off\")\n",
    "    ax[2][i].text(0.2, 0.8, str(preds[i].item()), horizontalalignment='center', verticalalignment='center')#, transform=ax1.transAxes)\n",
    "    ax[2][i].axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference Privacy\n",
    "## Freeze the privacy parameter $\\Delta_{LS}^{p}$ and the required radius $s$\n",
    "## Get the embedding of the target sample\n",
    "## Use it as a center\n",
    "## Find the best possible radius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "data, labels = next(iter(train_loader))\n",
    "data, labels = data.cuda(), labels.cuda()\n",
    "# get sample embedding from the VAE\n",
    "with torch.no_grad():\n",
    "    mu, log_var = vae.encoder(data.view(-1, 784))\n",
    "    z = mu#vae.sampling(mu, log_var)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.5102, device='cuda:0', grad_fn=<MinBackward1>)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obfuscator.cuda()(z).min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon = 5\n",
    "delta = 0.1\n",
    "\n",
    "center = z#torch.rand(10)\n",
    "radius = 1.0\n",
    "simple_domain = Hyperbox.build_linf_ball(center, radius)\n",
    "\n",
    "# For now we are using center of 0 and l_inf norm of size 1 around the center to define the output space\n",
    "output_domain = 'l1Ball1'\n",
    "\n",
    "# TODO: Replace it with a trained NN\n",
    "#network_simple = ReLUNet([8, 7, 5, 5])\n",
    "network_simple = obfuscator.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "mat1 and mat2 shapes cannot be multiplied (1x1600 and 8x10)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Input \u001b[0;32mIn [147]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m cross_problem \u001b[38;5;241m=\u001b[39m LipMIP(network_simple, simple_domain, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124ml1Ball1\u001b[39m\u001b[38;5;124m'\u001b[39m, num_threads\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m8\u001b[39m, verbose\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 2\u001b[0m \u001b[43mcross_problem\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompute_max_lipschitz\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/lipmip/lipMIP.py:115\u001b[0m, in \u001b[0;36mLipMIP.compute_max_lipschitz\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcompute_max_lipschitz\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    109\u001b[0m     \u001b[38;5;124m\"\"\" Computes the maximum lipschitz constant with a fixed\u001b[39m\n\u001b[1;32m    110\u001b[0m \u001b[38;5;124m        domain already set.\u001b[39m\n\u001b[1;32m    111\u001b[0m \n\u001b[1;32m    112\u001b[0m \u001b[38;5;124m        Returns the maximum lipschitz constant and the point that\u001b[39m\n\u001b[1;32m    113\u001b[0m \u001b[38;5;124m        attains it\u001b[39m\n\u001b[1;32m    114\u001b[0m \u001b[38;5;124m    \"\"\"\u001b[39m\n\u001b[0;32m--> 115\u001b[0m     squire, timer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuild_gurobi_squire\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    116\u001b[0m     model \u001b[38;5;241m=\u001b[39m squire\u001b[38;5;241m.\u001b[39mmodel\n\u001b[1;32m    118\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtimeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/lipmip/lipMIP.py:99\u001b[0m, in \u001b[0;36mLipMIP.build_gurobi_squire\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpreact, HBoxIA):\n\u001b[1;32m     98\u001b[0m     pre_bounds \u001b[38;5;241m=\u001b[39m HBoxIA(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnetwork, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdomain, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mc_vector)\n\u001b[0;32m---> 99\u001b[0m     \u001b[43mpre_bounds\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompute_forward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtechnique\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpreact\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    100\u001b[0m     pre_bounds\u001b[38;5;241m.\u001b[39mcompute_backward(technique\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpreact)\n\u001b[1;32m    101\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/lipmip/interval_analysis.py:80\u001b[0m, in \u001b[0;36mHBoxIA.compute_forward\u001b[0;34m(self, technique)\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m technique \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mVALID_TECHNIQUES\n\u001b[1;32m     79\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m technique \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnaive_ia\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m---> 80\u001b[0m \tlinear_outs, final_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward_nia\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minput_domain\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     81\u001b[0m \u001b[43m\t\t\t\t\t\t\t\t\t\t\t\t \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnetwork\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     82\u001b[0m \t\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward_boxes \u001b[38;5;241m=\u001b[39m {i: linear_outs[i] \n\u001b[1;32m     83\u001b[0m \t\t\t\t\t\t  \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mlen\u001b[39m(linear_outs))}\n\u001b[1;32m     84\u001b[0m \t\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward_switches \u001b[38;5;241m=\u001b[39m {k: BooleanHyperbox\u001b[38;5;241m.\u001b[39mfrom_hyperbox(v)\n\u001b[1;32m     85\u001b[0m \t\t\t\t\t\t\t \u001b[38;5;28;01mfor\u001b[39;00m k,v \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mforward_boxes\u001b[38;5;241m.\u001b[39mitems()}\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/lipmip/interval_analysis.py:310\u001b[0m, in \u001b[0;36mHBoxIA.forward_nia\u001b[0;34m(cls, input_hbox, network)\u001b[0m\n\u001b[1;32m    308\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(network\u001b[38;5;241m.\u001b[39mnum_relus):\n\u001b[1;32m    309\u001b[0m \thidden_unit \u001b[38;5;241m=\u001b[39m network\u001b[38;5;241m.\u001b[39mget_ith_hidden_unit(i)\n\u001b[0;32m--> 310\u001b[0m \tlinear_out, relu_out \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_forward_nia_layer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrelu_out\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhidden_unit\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    311\u001b[0m \tlinear_outs\u001b[38;5;241m.\u001b[39mappend(linear_out)\n\u001b[1;32m    314\u001b[0m final_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_nia_layer(relu_out, (network\u001b[38;5;241m.\u001b[39mfcs[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m],))[\u001b[38;5;241m0\u001b[39m]\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/lipmip/interval_analysis.py:331\u001b[0m, in \u001b[0;36mHBoxIA._forward_nia_layer\u001b[0;34m(cls, input_hbox, hidden_unit)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[38;5;129m@classmethod\u001b[39m \n\u001b[1;32m    318\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_forward_nia_layer\u001b[39m(\u001b[38;5;28mcls\u001b[39m, input_hbox, hidden_unit):\n\u001b[1;32m    319\u001b[0m \t\u001b[38;5;124m\"\"\" Takes in a hyperbox and a hidden_unit and outputs two new \u001b[39m\n\u001b[1;32m    320\u001b[0m \u001b[38;5;124m\t\thyperboxes, representing pushing the object through the hyperbox\u001b[39m\n\u001b[1;32m    321\u001b[0m \u001b[38;5;124m\tARGS:\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    329\u001b[0m \u001b[38;5;124m\t\tReLU(Linear(input_hbox)) is a subset of relu_out\u001b[39m\n\u001b[1;32m    330\u001b[0m \u001b[38;5;124m\t\"\"\"\u001b[39m\t\n\u001b[0;32m--> 331\u001b[0m \tlinear_out \u001b[38;5;241m=\u001b[39m \u001b[43minput_hbox\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap_linear\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_unit\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    332\u001b[0m \trelu_out \u001b[38;5;241m=\u001b[39m linear_out\u001b[38;5;241m.\u001b[39mmap_relu()\n\u001b[1;32m    333\u001b[0m \t\u001b[38;5;28;01mreturn\u001b[39;00m (linear_out, relu_out)\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/lipmip/hyperbox.py:200\u001b[0m, in \u001b[0;36mHyperbox.map_linear\u001b[0;34m(self, linear, forward)\u001b[0m\n\u001b[1;32m    197\u001b[0m radii \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor(radii, dtype\u001b[38;5;241m=\u001b[39mdtype)\n\u001b[1;32m    198\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m forward:\n\u001b[0;32m--> 200\u001b[0m     new_midpoint \u001b[38;5;241m=\u001b[39m utils\u001b[38;5;241m.\u001b[39mas_numpy(\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmidpoint\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m    201\u001b[0m     new_radii \u001b[38;5;241m=\u001b[39m utils\u001b[38;5;241m.\u001b[39mas_numpy(torch\u001b[38;5;241m.\u001b[39mabs(linear\u001b[38;5;241m.\u001b[39mweight))\u001b[38;5;241m.\u001b[39mdot(radii)\n\u001b[1;32m    202\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/nn/modules/module.py:727\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_slow_forward(\u001b[38;5;241m*\u001b[39m\u001b[38;5;28minput\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    726\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 727\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    728\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m hook \u001b[38;5;129;01min\u001b[39;00m itertools\u001b[38;5;241m.\u001b[39mchain(\n\u001b[1;32m    729\u001b[0m         _global_forward_hooks\u001b[38;5;241m.\u001b[39mvalues(),\n\u001b[1;32m    730\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks\u001b[38;5;241m.\u001b[39mvalues()):\n\u001b[1;32m    731\u001b[0m     hook_result \u001b[38;5;241m=\u001b[39m hook(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m, result)\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/nn/modules/linear.py:93\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m---> 93\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/matlaberp2/posthoc_privacy/.projenv/lib/python3.8/site-packages/torch/nn/functional.py:1692\u001b[0m, in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1690\u001b[0m     ret \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39maddmm(bias, \u001b[38;5;28minput\u001b[39m, weight\u001b[38;5;241m.\u001b[39mt())\n\u001b[1;32m   1691\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1692\u001b[0m     output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43minput\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmatmul\u001b[49m\u001b[43m(\u001b[49m\u001b[43mweight\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mt\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1693\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m bias \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1694\u001b[0m         output \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m bias\n",
      "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (1x1600 and 8x10)"
     ]
    }
   ],
   "source": [
    "cross_problem = LipMIP(network_simple, simple_domain, 'l1Ball1', num_threads=8, verbose=True)\n",
    "cross_problem.compute_max_lipschitz()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4097048069197993"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_problem.result.value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}