09_generative_adversarial_networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch Tutorial\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generative Adversarial Networks (GAN) - DCGANs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.init as init\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torchvision\n",
    "import torchvision.transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define image transformations &  Initialize datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_transforms = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])\n",
    "mnist_train = torchvision.datasets.MNIST(root='./data', train=True, transform=mnist_transforms, download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainloader = torch.utils.data.DataLoader(mnist_train, batch_size=64, shuffle=True, num_workers=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create DCGAN Generator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![caption](images/dcgan_g.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "      (deconv1): ConvTranspose2d(128, 128, kernel_size=(4, 4), stride=(1, 1), bias=False)\n",
    "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
    "\n",
    "      (deconv2): ConvTranspose2d(128, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)\n",
    "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
    "\n",
    "      (deconv3): ConvTranspose2d(64, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)\n",
    "      (bn3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True)\n",
    "\n",
    "      (deconv4): ConvTranspose2d(32, 1, kernel_size=(4, 4), stride=(2, 2), padding=(3, 3), bias=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Generator(nn.Module):\n",
    "    \"\"\"DCGAN Generator.\"\"\"\n",
    "    def __init__(self, z_dim=128, num_filters=32):\n",
    "        super(Generator, self).__init__()\n",
    "        self.z_dim = z_dim\n",
    "        self.num_filters = num_filters\n",
    "        \n",
    "        # 1 x 1 -> 4 x 4\n",
    "        self.deconv1 = nn.ConvTranspose2d(\n",
    "            in_channels=z_dim, out_channels=num_filters * 4,\n",
    "            kernel_size=(4, 4), bias=False\n",
    "        )\n",
    "        self.bn1 = nn.BatchNorm2d(num_filters * 4)\n",
    "        \n",
    "        # 4 x 4 -> 8 x 8\n",
    "        self.deconv2 = nn.ConvTranspose2d(\n",
    "            in_channels=num_filters * 4, out_channels=num_filters * 2,\n",
    "            kernel_size=(4, 4), stride=2, padding=1, bias=False,\n",
    "        )\n",
    "        self.bn2 = nn.BatchNorm2d(num_filters * 2)\n",
    "        \n",
    "        # 8 x 8 -> 16 x 16\n",
    "        self.deconv3 = nn.ConvTranspose2d(\n",
    "            in_channels=num_filters * 2, out_channels=num_filters,\n",
    "            kernel_size=(4, 4), stride=2, padding=1, bias=False,\n",
    "        )\n",
    "        self.bn3 = nn.BatchNorm2d(num_filters)\n",
    "        \n",
    "        # 16 x 16 -> 28 x 28\n",
    "        self.deconv4 = nn.ConvTranspose2d(\n",
    "            in_channels=num_filters, out_channels=1,\n",
    "            kernel_size=(4, 4), stride=2, padding=3, bias=False,\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.bn1(self.deconv1(x)))\n",
    "        x = F.relu(self.bn2(self.deconv2(x)))\n",
    "        x = F.relu(self.bn3(self.deconv3(x)))\n",
    "        return F.tanh(self.deconv4(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create DCGAN Discriminator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![caption](images/dcgan_d.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    \n",
    "    (conv1): Conv2d(1, 32, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)\n",
    "    (bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True)\n",
    "    \n",
    "    (conv2): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)\n",
    "    (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
    "    \n",
    "    (conv3): Conv2d(64, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)\n",
    "    (bn3): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
    "    \n",
    "    (conv4): Conv2d(128, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Discriminator(nn.Module):\n",
    "    \"\"\"DCGAN Generator.\"\"\"\n",
    "    def __init__(self, num_filters=32):\n",
    "        super(Discriminator, self).__init__()\n",
    "        self.num_filters = num_filters\n",
    "        # 28 x 28 -> 14 x 14\n",
    "        self.conv1 = nn.Conv2d(\n",
    "            in_channels=1, out_channels=num_filters,\n",
    "            kernel_size=(4, 4), stride=2, padding=1, bias=False\n",
    "        )\n",
    "        self.bn1 = nn.BatchNorm2d(num_filters)\n",
    "        \n",
    "        # 14 x 14 -> 7 x 7\n",
    "        self.conv2 = nn.Conv2d(\n",
    "            in_channels=num_filters, out_channels=num_filters * 2,\n",
    "            kernel_size=(4, 4), stride=2, padding=1, bias=False,\n",
    "        )\n",
    "        self.bn2 = nn.BatchNorm2d(num_filters * 2)\n",
    "        \n",
    "        # 7 x 7 -> 3 x 3\n",
    "        self.conv3 = nn.Conv2d(\n",
    "            in_channels=num_filters * 2, out_channels=num_filters * 4,\n",
    "            kernel_size=(4, 4), stride=2, padding=1, bias=False,\n",
    "        )\n",
    "        self.bn3 = nn.BatchNorm2d(num_filters * 4)\n",
    "        \n",
    "        # 3 x 3 -> 1 x 1\n",
    "        self.conv4 = nn.Conv2d(\n",
    "            in_channels=num_filters * 4, out_channels=1,\n",
    "            kernel_size=(4, 4), stride=2, padding=1, bias=False,\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.leaky_relu(self.bn1(self.conv1(x)), 0.2)\n",
    "        x = F.leaky_relu(self.bn2(self.conv2(x)), 0.2)\n",
    "        x = F.leaky_relu(self.bn3(self.conv3(x)), 0.2)\n",
    "        return F.sigmoid(self.conv4(x)).squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cuda_available = torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize Generator & Discriminator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator = Generator()\n",
    "discriminator = Discriminator()\n",
    "if cuda_available:\n",
    "    generator = generator.cuda()\n",
    "    discriminator = discriminator.cuda()\n",
    "\n",
    "loss = nn.BCELoss()\n",
    "optimizer_g = torch.optim.Adam(generator.parameters(), lr=2e-4)\n",
    "optimizer_d = torch.optim.Adam(discriminator.parameters(), lr=2e-4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/abursuc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:51: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "/Users/abursuc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:80: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n",
      "Process Process-2:\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 57, in _worker_loop\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "Process Process-1:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/process.py\", line 249, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 52, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/queues.py\", line 343, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 57, in <listcomp>\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/datasets/mnist.py\", line 74, in __getitem__\n",
      "    img = Image.fromarray(img.numpy(), mode='L')\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/PIL/Image.py\", line 2445, in fromarray\n",
      "    return frombuffer(mode, size, obj, \"raw\", rawmode, 0, 1)\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/PIL/Image.py\", line 2391, in frombuffer\n",
      "    im = new(mode, (1, 1))\n",
      "KeyboardInterrupt\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/PIL/Image.py\", line 2282, in new\n",
      "    _check_size(size)\n",
      "  File \"/Users/abursuc/anaconda3/lib/python3.6/site-packages/PIL/Image.py\", line 2258, in _check_size\n",
      "    if len(size) != 2:\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-4c6f15cd336c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0;31m# Train with real examples\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0md_real\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdiscriminator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0md_real_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md_real\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mones\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Train discriminator to recognize real examples\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-8-1668c3b6eed6>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mleaky_relu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mleaky_relu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mleaky_relu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torch/nn/modules/batchnorm.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     47\u001b[0m         return F.batch_norm(\n\u001b[1;32m     48\u001b[0m             \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrunning_mean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrunning_var\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m             self.training or not self.track_running_stats, self.momentum, self.eps)\n\u001b[0m\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/abursuc/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mbatch_norm\u001b[0;34m(input, running_mean, running_var, weight, bias, training, momentum, eps)\u001b[0m\n\u001b[1;32m   1192\u001b[0m     return torch.batch_norm(\n\u001b[1;32m   1193\u001b[0m         \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrunning_mean\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrunning_var\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1194\u001b[0;31m         \u001b[0mtraining\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmomentum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackends\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcudnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menabled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m     )\n\u001b[1;32m   1196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "ctr = 0\n",
    "z_dim = 128\n",
    "\n",
    "minibatch_disc_losses = []\n",
    "minibatch_gen_losses = []\n",
    "num_epochs = 1\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    losses = []\n",
    "    # Train\n",
    "    for batch_idx, (inputs, targets) in enumerate(trainloader):\n",
    "        ctr += 1\n",
    "        if cuda_available:\n",
    "            inputs, targets = inputs.cuda(), targets.cuda()\n",
    "        \n",
    "        inputs, targets = Variable(inputs), Variable(targets)\n",
    "        \n",
    "        zeros = Variable(torch.zeros(inputs.size(0)))\n",
    "        ones = Variable(torch.ones(inputs.size(0)))\n",
    "\n",
    "        if cuda_available:\n",
    "            zeros, ones = zeros.cuda(), ones.cuda()\n",
    "                \n",
    "        ############################\n",
    "        # (1) Update Discriminator \n",
    "        ############################\n",
    "        \n",
    "        # Sample z ~ N(0, 1)\n",
    "        minibatch_noise = Variable(torch.from_numpy(\n",
    "            np.random.randn(inputs.size(0), z_dim, 1, 1).astype(np.float32)\n",
    "        ))\n",
    "\n",
    "        if cuda_available:\n",
    "            minibatch_noise = minibatch_noise.cuda()\n",
    "        \n",
    "        # Zero gradients for the discriminator\n",
    "        optimizer_d.zero_grad()\n",
    "        \n",
    "        # Train with real examples\n",
    "        d_real = discriminator(inputs)\n",
    "\n",
    "        d_real_loss = loss(d_real, ones)  # Train discriminator to recognize real examples\n",
    "        d_real_loss.backward()\n",
    "        \n",
    "        # Train with fake examples from the generator\n",
    "        fake = generator(minibatch_noise).detach()  # Detach to prevent backpropping through the generator\n",
    "        d_fake = discriminator(fake)\n",
    "\n",
    "        d_fake_loss = loss(d_fake, zeros)  # Train discriminator to recognize generator samples\n",
    "        d_fake_loss.backward()\n",
    "        minibatch_disc_losses.append(d_real_loss.data[0] + d_fake_loss.data[0])\n",
    "        \n",
    "        # Update the discriminator\n",
    "        optimizer_d.step()\n",
    "\n",
    "            \n",
    "        ############################\n",
    "        # (2) Update Generator\n",
    "        ############################\n",
    "\n",
    "        # Zero gradients for the generator\n",
    "        optimizer_g.zero_grad()\n",
    "        \n",
    "        # Sample z ~ N(0, 1)\n",
    "        minibatch_noise = Variable(torch.from_numpy(\n",
    "            np.random.randn(inputs.size(0), z_dim, 1, 1).astype(np.float32)\n",
    "        ))\n",
    "\n",
    "        if cuda_available:\n",
    "            minibatch_noise = minibatch_noise.cuda()\n",
    "\n",
    "        d_fake = discriminator(generator(minibatch_noise))\n",
    "        g_loss = loss(d_fake, ones)  # Train generator to fool the discriminator into thinking these are real.\n",
    "\n",
    "        g_loss.backward()\n",
    "\n",
    "        # Update the generator\n",
    "        optimizer_g.step()\n",
    "\n",
    "        minibatch_gen_losses.append(g_loss.data[0])\n",
    "    \n",
    "    print('Generator loss : %.3f' % (np.mean(minibatch_gen_losses)))\n",
    "    print('Discriminator loss : %.3f' % (np.mean(minibatch_disc_losses)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample from the Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAJCCAYAAAAP/PnVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3WmYVNW5xv2FyNR0MzY0MinghIiCAxA0BqeII0aMA45E\n5ZhoxGiMCo4hjjEejUYcY2KMCAcH9EhURA1ETUQFBxARUAaFZp6bQfT98L7Xe/nca53atYrq6l3d\n/9+3G3et2l21etey9tPPqvfdd985AAAAZG+nmj4BAACAYsMCCgAAIBILKAAAgEgsoAAAACKxgAIA\nAIjEAgoAACASCygAAIBILKAAAAAisYACAACItHMhn6y8vLzWtD2vV69e9GNqc9f3FStWxL8gOWrd\nurV5IfW9+Pbbb73HtGzZ0uTKykqTGzdubHLo/d2+fbvJO+2U+f8/snm/9Rh93tAYsXMvmzFyOQ/9\nN309Qu9D0hhr1qwpyDxq27Zt7f1lrOOWLVtWsGtRRUVF9DzK5ncr0/EhtfmzpaZUVlYmvvB8AwUA\nABCJBRQAAECkgt7CK2a53LJDYeitotB7tXr1apMbNmxocjZfgcd+9Z6NpHkV+u+xX/lX1+3m2NuA\noVt6/F6hrom9bnB7Lr34BgoAACASCygAAIBILKAAAAAiUQOVo1z+7Bv5kfTa5tI+IJv/Hlvzk43q\nqE/K5byqY/7qGPXr1088BgCKBd9AAQAARGIBBQAAEIkFFAAAQCQWUAAAAJEoInfZFbLqPmhq553t\nS7lp06bEY7SoVnOhCpnrgq1bt5qs+7bp+9ugQQNvDG0Eqe/X2rVrTdb32znnGjVqlHHMJk2amJyP\nvfBqCn9YkQ7spVb78LuVDnwDBQAAEIkFFAAAQCQWUAAAAJGogcpS8+bNTe7atavJRxxxhMk9evTw\nxnj//fdNfvTRR03W+9hJdVfOFU89TD7lcv9fa4v69u1r8i233GLy3/72N2+MZs2amdy5c2eTv/76\na5Pnzp3rjaHHLFiwwORly5Z5jykEaihqj3xcE3LZTBr5kUvNWi6/v7x/O45voAAAACKxgAIAAIjE\nAgoAACBSnayBSqo12m233bzH3H333Sb36dPH5I0bN5rcsmVLb4xBgwaZfP3115usvYpuvPFGb4wP\nPvjA5Hnz5pm8ZcsWk7XPUF2lPZuOP/54k/fff/+MOURfW+0tlY1vvvnG5FmzZplcWVnpPeaFF17I\nmKuqqjI+h3N+/UN1bGqMdNhjjz28f7vmmmtMvummm0z+8ssvTQ6916HNoRFPX1uttTz77LO9xxx2\n2GEmX3bZZSZrTzrNzjlXUlJisvYu1GvC5s2bvTEOPPDAjHnatGkmz5492xujYcOG3r8VC76BAgAA\niMQCCgAAIBILKAAAgEh1sgZK7+3qvme9evXyHtOtWzeTX3nlFZOff/55k1evXp14Hm3atDH597//\nvckjRozwHqP9qF566SWTr732WpOXL1+eeB51QVlZmcknnnhi9BhLly41Wfe103m0ePFibwztHbVq\n1SqT99tvv8TzOProo01u3769yX/84x9NDtVAofbSPRgvv/xy75gTTjjBZO1jpzWAK1euzNPZ1S65\n9KRLOqZFixYmn3feed4x+jv/xhtvmKz1TTNnzvTG0Nq4t99+22S9Fv373//2xhg4cKD3b5n07t3b\n+7d169ZFjZEmfAMFAAAQiQUUAABAJBZQAAAAkVhAAQAARKpXyGZ45eXlqei8l9S4bOLEid5jtLDv\n8MMPN1mLgUMNLJOaF7Zu3drkDh06eGNMmjTJZP1Zbr75ZpPvv/9+b4yk88rFihUrCrYzZevWraPn\nkRZ8a54xY4bJuumvc87deuutJk+fPt1k3QhYC/6dc65t27Yma/O6c845x2QtGHfOuZ49e5q8bds2\nk0899VSTX3/9dW+MQjSvy+XasmrVqoLMo7Zt26biWpSLpI1kteFh6A8mtHGmNg+++uqrTX7qqaci\nz7LmLFu2rGDXonbt2pkXP5c5r4/Rjc9//vOfe48ZOnSoyUuWLDFZ/1jl2Wef9cYYP368ydpsU//4\nZMOGDd4Y+jn3xBNPZDyPc8891xvjo48+MjmXwvzqUFlZmTiP+AYKAAAgEgsoAACASCygAAAAItWJ\nRpp6D1U3fe3evbvJe+65pzeGNhlLav4VqivS59WszepCTcf0Z9FanX/+858mh5ooaqM9HTObmqhi\n20hW3y+tM9DmqVqbFBoj6f0MbS6s74eO+bvf/c5krbNyzrm//vWvJuumrgcddJDJOidCcpkDqH6h\n9yHpd08b9IY8+uijJl933XUmH3XUUSb/5S9/8cYo5k1g00Tf4/Xr15v8v//7v95j7rnnHpP1PS8v\nLzdZN4d2zq950uuIbkyvdaPO+XPxiiuuMPmxxx4zOdR48+OPP844ZprxDRQAAEAkFlAAAACRWEAB\nAABEqhM1UElatWplcqh2pUePHiZrHVFVVZXJofvFauvWrSZrj6AHH3zQe4zeL9d7zJ9//rnJoZ+l\nLtJ6je3bt5us9QAhWjeVRN9f5/y6gqSeZHPnzvXG0Pc0qX4pVFNQTHUGdVnofdLeO3ot0jmkPc6c\n8+tsGjdunPF5qYkLq47fI30/P/vss8Tn/eqrrzLmUF9CvY5ofaZ+huk10znnSktLTe7fv7/Je++9\nt8njxo3zxihmfLoCAABEYgEFAAAQiQUUAABApDpRA6X37/V+8H/+8x+TdW8x5/w6qU8++cTkk08+\nOeN/d86/x6x73V111VUml5WVeWPMnDnT5Pfee89krbsJ1UClZa+h6pLmn0fnnr4XixYtMvnll19O\nHFPH0H0bdW8055xr2rRp4rjfF3pNa/s8SoNQ7VGDBg1M1v0vX3rpJZNDfYT02qP1LrNmzYo6z7oq\nl9qw2N+T0DVcryNJdZFaF+ec34NOH5NUo+ucc4MHDzb5tttu8475vldeecX7t2K+bvANFAAAQCQW\nUAAAAJFYQAEAAERiAQUAABCpThSRKy38q6ysNLlv377eY6ZOnWqyFuVp08vQRsDt2rUzWQvuTjjh\nBJNXrVrljTF06FCT9dxzaYCXSyFkmhrr5aMIMR8bKCf9sYJzzrVv397kww8/3ORf/epXJu+xxx7R\n5/Xss8+a3LJlS+8xoT+UyDRmrscg//bbbz+TL7roIpNvv/12k7VJpnPOjRkzJuNzLFiwwGQtXMf/\nK/aakI18XLP12hPaHF3H0GbC+++/v8n6xwrOOTdo0KCM57Fw4UKTly5dmvH4YsM3UAAAAJFYQAEA\nAERiAQUAABCpTtZAKd0QUTfkdc65AQMGmDx+/HiTtVblnXfe8caYOHGiyQMHDjRZ76dffPHF3hjT\np083uaSkxGTdNDd0r7zYa1cKVfO0o2N27NjRO0bnhc69XGgzuzfeeCPxOXSeoGYkNSM95JBDvMfo\ntefrr782WWsnKyoqvDHatGmT8by0uXBI0u9QsV9n8qFQr4E20tS6N62vdc65888/32StVzr22GNN\nzqb5rv68P/3pT03euHFj4hjFhG+gAAAAIrGAAgAAiMQCCgAAIBI1UM7vidOkSRPvmNmzZ5t86KGH\nmvzCCy+YfMABB3hjaP8W9Ytf/MLkadOmecfoRrH169c3efv27SZThxCWS7+sJNp7JdTDKR81T2rI\nkCEZ/7tuQO2ccytXrjSZeVIzdM5o7crTTz+dOIbWZ+omsT//+c8Tx3jxxRdNXrx4sck77+x/VOi1\npi6qqQ219XmTetCNHDnSG0N70h144IEm63u+ZcsWbwytpdS5p/Mo1BtPpanHYBK+gQIAAIjEAgoA\nACASCygAAIBIBa2Bqo66k3zQ89KeGs7594N1T58nnnjC5FANlFqxYoXJr7zyismhe87ffPNN4ri1\nXS51B7GPyaWWYevWrSbPmTPHO2bSpEkma/1KLnuOnXrqqSaffPLJJp911lneY1577TWTa6qWo65J\nutZo35xQ7ZHWbO6zzz4maw1naA8zpb2kdD6E6p2YI+m1efNmk0N9vX7yk5+YrHNNP2smT57sjfHw\nww+brDVPOkdat27tjbF69WqTda6FPo/TIr1nBgAAkFIsoAAAACKxgAIAAIjEAgoAACASjTRddgWT\nLVu2NLlPnz4mX3DBBSaHCiz1ecaMGWPy/PnzTe7UqZM3RqioFMmqo2hcNWrUyOQlS5Z4x5x99tkm\nd+3a1WRtlLrLLrt4Y9xyyy0Zj9E5MnbsWG8M/beLLrrI5GwKN5OOyeaPROp6IbI2Ftx1111Nnjdv\nnveYTZs2mawNd6+55hqTQ3+Mog14KysrM/53/ngl3ZL+OOGyyy7zHvPMM8+Y/Pjjj5usTTL32msv\nbwzdzP7zzz83Wa8BWtzunP+HM2kuGlfFc6YAAAApwQIKAAAgEgsoAACASPUKWYNQXl6eyoIHrUPQ\n+7rOOffLX/7S5KuuusrktWvXmnz77bd7Y4waNcpkbSB2wgknmDxr1ixvDD23tNSQrFixomBdUVu1\napWOHzpBqAZImyBqrYnWA2i9i3P+e/7DH/7QZN04VP97iG6GrTVR2iQ0G0kbnoasXLmyIPOobdu2\nqZhDWm/ZrFkzk9u1a+c9Rmvtpk6danKvXr1MPuOMM7wxdIPhN99802St6QxtApuWRshq2bJlBTux\nioqKVMyjJKHPCZ1H1157rcm6uX2Ibkreo0ePjM8bqi/Wa2BaVFZWJs4jvoECAACIxAIKAAAgEgso\nAACASHWyBiqp5mngwIHeY3TTRN0IuG/fviZrXxXnnFu2bJnJeg9aN4X94IMPvDGqqqq8f0uDmqyB\nKuZ+Q7lssK3HaI+e0tJSk3XjYOec6969e8bnOP/8802eOHGid4zWc+VS86TqWg2Uvv/Z1Iw0bdrU\n5I0bN5qs/Xv23ntvb4xXX33V5CeffNLkq6++2uTQe0kNVPHUQIVof7COHTuaPGPGDJOz6UG4dOlS\nk4cMGWLy9OnTvcfo52BartXUQAEAAFQDFlAAAACRWEABAABEqhMbq2nNk+61ozVPf/rTn7wx9H6/\n9oHSnk6h2oWbbrrJ5Lvuustk3Zvo4IMP9sZYtGiR9281IS33qYtNPvbk02O05kVrYnT/Peece//9\n9zM+x4UXXmhyqAYqtgaGOePT11Bfo1DdidauaB8drZMMvU9ad9K6devExyBZ0vuZJo0bNzZ5w4YN\nJp911lkma42uc84de+yxJmvfssmTJ5vcu3dvbwzdM1Q/n0M9yNKCb6AAAAAisYACAACIxAIKAAAg\nEgsoAACASHWiiFxpgXfbtm1NDm1uOHPmTJOnTJlisja3a9KkiTfGmDFjTL711ltN1mJgLfJzzi+o\n03MtVNFiTRaZ5vLchSju1PeiRYsW3jHa5HL+/PkZx9SCyhBtpKnzO1SEmdTA89133zU5tKmxztck\nFCYny+b9Vvq65rI56+LFizOOkeZC3jQLzfm0FpZrgbf+EdPw4cO9x0yYMMHkv/3tbybrz3/77bd7\nY1xyySUmb9682eQ0zz2+gQIAAIjEAgoAACASCygAAIBIdaIGSu+hanO6XXfdNeN/d87fkHXt2rUm\naw1UiG4mPG7cOJO14aFu6OmccxdddJHJWu+SSw1FLmryPn4uG/CqfNRE6Wvdr18/k5966invMVob\n99vf/tbkBx98MPq8dH7rJr8XXHCB9xj9+bdu3Wrye++9F30eKh/vE+Jp/VJ5eXniYz7++GOTs3nv\n0lrLU5OyeU1qotlm6P3Thqxa46Sfcfr55ZxzTz/9tMmjRo0yuUuXLiYfc8wx3hj6GZbLdaOm5iLf\nQAEAAERiAQUAABCJBRQAAECkOlEDpbUqer907ty5Gf+7c84NGDDAZN1wWPvkhDYTbt68ucl6T1mf\n96CDDvLG0BoarXfRXF2KrQ9UPuj7o3n06NEml5WVJY6pc2Dfffc1ubKy0ntMs2bNTO7evbvJF198\nscl9+vRJPI9bbrnF5LfeesvkXHoLUfNUM7SeTTeFDdHrV1K/OeeogcpVTdU8Ka31XbBggcn6nmsP\nO+f8eXLttdearL0PQz97p06dTJ4zZ87/ccbpwzdQAAAAkVhAAQAARGIBBQAAEKmgNVA10f8i9Lzq\nk08+MVn3FnPOuV69epk8duxYk19//XWTQ7VIHTt2NPmcc87JeF6hflR6z3nDhg0mN2rUKOOYdUFo\nXuVj7ukYWudWVVUVPab2fcqG7hUV2jMxyTvvvGOy7u2o8yqXGiiVzfuCHVdSUmLy4YcfnviY//qv\n/zJZ+96leT+ymlRTn2lJsjkP/YzSeqXjjz8+4/HO+dfANWvWmJzN66P1d/r5G/ocTAu+gQIAAIjE\nAgoAACASCygAAIBILKAAAAAiFbQ6q6YK7JIKVbWgdtCgQd4xWmB3wAEHZMy5FMxq8dzAgQO9YzZu\n3GiyFhCnpYixkLL5mavjddEGrcOGDTP5mWee8R7TqlWrHX7e2KLxiRMnev921113mTxz5kyTq2Mj\nYArGC0OLfR944AHvmKuuusrk3XbbzeSePXua/MEHH3hjFGrjclQP/YOjDz/80GRt4qtNnJ0LN9f8\nvlBDaaVF5A0aNDA5zZ9p/AYAAABEYgEFAAAQiQUUAABApPR2qCogbRI4depU75hDDjnE5CuvvNLk\nwYMHmxy6b6t1VC+88ILJei9Ymxk6R92Bc+m9Jz5r1iyTf/CDH3jHHHjggSZ37tzZ5P3228/kvffe\n2xujTZs2Jj/yyCMmz5gxw+R3333XG0ObfmrdAYqX/n488cQT3jGLFi0y+frrrzdZGwdrs2Hnwg2H\n65q0XotyoTWK69atM7lLly7eY+bPn2/yvHnzTF6yZInJ2mjTOefWr19vstbw0UgTAACgFmEBBQAA\nEIkFFAAAQKR6hbyHW15eXrQ3jPV1SsohWmulG3TqPehi6puzYsWKgp1sq1atimIeZTMnkjbbzGVj\n5GzmUdLcysd1IZf5u3LlyoLMo7Zt2xbFHMqH0Hup9Uva40drLXUD69AxabFs2bKCXYsqKipq7TzK\nphec9nnSx+gcadGihTeG1lqlRWVlZeI8SudvAAAAQIqxgAIAAIjEAgoAACBSehsspFzS/f9sakh0\njNrUUwTZ1QDlUr+kkmrrslGIucf8rhnZ1MBp7x0glxpGvRbp7/zq1au9x2ifp2K6TvANFAAAQCQW\nUAAAAJFYQAEAAERiAQUAABCJIvIsxRbU5VKAV0yNM4up0M+5wjSOzEU+mqfmcu418fOGfra0NmOs\n7WILd3mf6p5srkVJ86iYm0Nng98KAACASCygAAAAIrGAAgAAiFTQzYQBAABqA76BAgAAiMQCCgAA\nIBILKAAAgEgsoAAAACKxgAIAAIjEAgoAACASCygAAIBILKAAAAAisYACAACIxAIKAAAgEgsoAACA\nSCygAAAAIrGAAgAAiMQCCgAAIBILKAAAgEgsoAAAACKxgAIAAIjEAgoAACASCygAAIBILKAAAAAi\nsYACAACIxAIKAAAgEgsoAACASDsX8sl69OjxXSGfL1vffvutyTvtFL+urFev3g6fh46h5xWSj3PP\nh5kzZ+74C5Cl0tLSVM4j7LgNGzYUZB5VVFSYOfTddzs+pfJxDcjleTRn87PoMbmce9IYem3aeWf/\n4+abb77JOGY29JpXWVlZsGtRly5dinYeJcnHHCkm+vN++eWXiT8w30ABAABEYgEFAAAQqaC38NIq\nH7e9svnqVr/Srl+/fvQY27dvz/iYmrqFBxSzbG6DpfUWRtKtllx+Fr21Fjpej9HrW+PGjU3Wa1c2\n55GNNF3zcrmdmlb5eG+2bt1qcoMGDbxjNm7caLLOG/2cTNPvYXpmHgAAQJFgAQUAABCJBRQAAEAk\naqAKSO/Va02A/vdQGwO9H5xNqwMAcdJUZ5EkH3/6r9eRkpISk8vKyrwx9thjD5Pnzp1r8rJly6LP\nK5fXXetsalJd+9P/JA0bNjQ51Mri97//vcn9+vUz+YQTTjBZa++cq542QtngGygAAIBILKAAAAAi\nsYACAACIxAIKAAAgEkXkWaqOYkAt5NTn0ILxkFyacQIoXvn4HdeGhitXrjT5mmuuMXnkyJHeGPpv\nL7/8ssnNmjUzuboKqhs1alQt4+airhWN6x8f6M+/YMECk3VeOefc+eefb/KUKVNM1j8SCL3GNfW6\n8w0UAABAJBZQAAAAkVhAAQAARKIGKke53HNNul+czXNQ4wRgR1VVVZl80kknmXzVVVeZ/P7773tj\nTJs2zeTWrVubHGp4WB22bdtWkOeBTz+j9DOuefPmJt92222JY/7kJz8xWZu4VtdnYC5NqfkGCgAA\nIBILKAAAgEgsoAAAACIVtAZK75empZ4nm3qmQmxWWNd6iIDNR2Hl8v7rHNL+clu2bPEec9RRR5l8\n9913m6w1Uvfcc483xmeffWaybhRbqBqobPrloXps2LDB5DPOOMNknVchWl/XokULk2Nrh3OlvzdZ\nPaYazgMAAKBWYwEFAAAQiQUUAABAJPpA5Sipfiv032NrwPIxBtIll/dcUSdVe+VSE6fHbN682eTO\nnTt7j9F+PC1btjR5+PDhJr/44oveGI0bNzZZa62qY486euOlS4cOHUy++OKLTW7YsKHJ8+fP98bQ\nuah1VSUlJTtyitWKb6AAAAAisYACAACIxAIKAAAgEgsoAACASDVaRF5MBdHazCup6ZYWwoXstdde\nJh955JEmf/XVV95j/vjHP5rctWtXk7WwE4WTzWaU27dvN1mbAKalQJxi9pqRTTM/nWc6p7R4+7e/\n/a03hl43HnvsMZPHjRtncmlpqTeGNsqsjqJxlc0f1iA/dJ5pQbhzzv361782ed999zV5zpw5Jvfv\n398bQzcLTnPRuOIbKAAAgEgsoAAAACKxgAIAAIhUozVQudQ8Jd3vzkcdVWgM3ShT7weXl5eb3KlT\nJ2+M888/3+SePXtmfA7Nzvmbeh522GEmr1q1ynsMCkPnTWgeaZ2I1sHNmjUr/yeWB9Xxuwpf0sbA\nzvm1R3otGjZsmMknnXSSN4bWplxyySUma2PNbdu2eWOk5f1Nc+1sMWvSpInJV111lXfMOeeck3EM\nnVehWrpixjdQAAAAkVhAAQAARGIBBQAAEKmgNVDar0Rl0wMl6X53NjUDqlmzZiYvXbo08TxGjRpl\n8qBBgzKO6ZxzY8aMMfnJJ580edOmTSY/+uij3hhad3Dsscea/Pe///3/OGPsqNiNXrXHk3POdevW\nzWSda0l9okLnofUpTZs2NblFixbeGG3btjVZ+5bp/F2wYIE3hm4eq5vYFrOa6oOVTR2d1jn+6Ec/\nMnnEiBEmL1++3Btj6NChJuucUaHrKrVHxU2vLStXrjT5ggsuyJhDfvWrX5k8d+5ck9NSN5cvfAMF\nAAAQiQUUAABAJBZQAAAAkWq0D5QK7SWWVBfVoEEDk0O9k9q0aWOy1oR8/vnnJl9xxRXeGJdeeqnJ\nev9Y66zuvvtub4zx48ebrDUEGzduNPncc8/1xnjwwQdNXrdunXdMbVdTeygm3b/Xfjyhvina+2vG\njBkmv/baayaH6op0jP3228/k4cOHm5xN3cHWrVszPm+ojlB7CZ155pkmL1u2LPF5i7kmIrYmLhvZ\nzO1WrVqZ/MYbb2Qc89Zbb/X+7bPPPjNZ+5Nl87NRA1XctDZOa5zuuuuuxDHee+89kx944AGT27dv\nb3Jt28uQb6AAAAAisYACAACIxAIKAAAgEgsoAACASKkqIs+GFpU/++yzJrdu3dp7TFlZmckLFy40\nuV27diY3btzYG0ObFY4cOdLkZ555xuTmzZt7Y2jBuzYvrKqqMnngwIHeGNrsbMKECSZn04y02NVU\n8ar+kYMW3r7wwgsm9+vXzxtD54Bu9HrjjTfuyCnmTP8oQn9ntGmmc8716NHDZN0s+84778zPydWA\nmips1bkdaj58xx13mKznunjxYpOffvppbwx9v3Vu19QfaiA/9P0M/RFInz59TL7pppsyjvnOO+94\n/6ZzUYvGVTEXjIfU/k9bAACAPGMBBQAAEIkFFAAAQKSC1kAl1eeEGmkqvXf/8ssvm/zUU095j9Fa\nI63nOP300xPPQ+sKpk6danJpaanJWjMVel6teTrllFNMDjXSfOyxxzKOkbQpKHKndQTHHXecyYce\neqjJ2dzvr6maAJ3j2khTa7U+/PBDb4y3337b5L/97W/R50FtjaXXiL322ss75sILL8w4xk9+8hOT\nQ5tJay2lzge9zobep9hGornMdeZHbvT97Ny5s3eM1sbpBuLaKDdUn6nHqNpW86T4BgoAACASCygA\nAIBILKAAAAAipaoPVDY9jLS2SDcvDNH76Lrh8NixY00O9YHS+peSkpKM/z1Efz7NQ4YMSRxDN5vV\nXkSoPk2aNDH5kEMOMTkfNR65jDF58mST//SnP5l8zDHHeI+ZNm2ayf/85z9N1k2qtdbOOb9HUTHV\nOySda03V3mgt5Q9/+MPEx8yfP9/k1atXm9ypUyfvMTp3tX/Pvvvua3KvXr28MS6//HKTP/nkE5O1\nri6EGqfc6OuW1KPummuu8cYI9Uz8vqFDh5q8adMm7xitldRjdIP1UG2w1tvp53M2tdH5kMtc5Bso\nAACASCygAAAAIrGAAgAAiJSqGqhs5FK7kHQPVR8TqmfSf0uqZwrZvHmzyccee6zJvXv3Nvn+++/3\nxvjiiy9M1lqs0D1mVex1BzW1T5e+tp9++mnG8wjN1X/9618m33zzzSZr769QXZzOtZdeesnkSZMm\nZczO+fVLOqb+rKGfJalXUJprovT3WesuasrGjRtN7tChQ+Jj3nzzTZNPO+00k4cPH+49pqKiIuOY\nSfPDOX8fzv79+5u8aNGijM+B3On7o/N32LBhJg8aNChxzIcfftjkyspKk0M1UzqPjj76aJOvvPJK\nk0M1u2PGjDH5kUceMVl7MKYJ30ABAABEYgEFAAAQiQUUAABAJBZQAAAAkdJROZlBbCFq6PikAm8t\nhg0VnesxSh8TOg9tcNenTx+TtQj32Wef9cbQQvRsin2RH/raavHjO++8Y/LChQu9MbRBpRZV6hi6\nMaxzfrOISKk1AAAgAElEQVTFNWvWmJzUZM+55N8JbZBX2yT9Pod+jwrxxwpa3P788897x1xyySUm\n65zRTV9DBePa8FA3xtZi9lAzznHjxpmcTTNh5MeGDRtM3n333U3WJph6vHPO/eUvfzFZ/6BFm7jq\nmM45d+KJJyae6/fp55dzfoH73//+96gxaxLfQAEAAERiAQUAABCJBRQAAECk1NdAqbQ0gdQaCc2h\neoDDDjvMZL1/rI3p5s6d643RvHnzjM+bltenOtXUz5hUW6QN30L3+5U2xNNalKZNm3qP0ed54403\nTNY5kU2T17oumyaghfhda9mypcmhxoPLli0zuXv37iZrw0OtvXTOuYMOOshk3fRV59huu+3mjaHN\nGwcOHGiy1tiE5iF1U7lp166dyaNGjTJZ58C7777rjTFjxgyTR4wYYfKll15qcqguUmtwZ8+ebfLo\n0aNN1kbQzvk1fVonmmZcWQEAACKxgAIAAIjEAgoAACBS6mug0lLTk9RfqaqqyuRddtnFO0bvU0+b\nNs3kBx980OQmTZp4YyRtjIzqk1RLpPUAoZoBrTXZunWryWeddZbJoXl3ww03mLxkyRKT07IxbjGr\nqeuOXkd+/etfe8d07NjR5AMOOMBk7Qt13nnneWNoLaXWSe29994mP/roo94Y+hotWLDAZP19qY31\nTjW1gbb2ddKegfvtt5/JBx98sDeG1rVpvzD9WUKfPTq3dIN17fF0+umne2M899xzJtfUPMnlveMb\nKAAAgEgsoAAAACKxgAIAAIhEsUSedOjQweQLLrjAO2bXXXc1Wfe50t4r1LLUPtr3Sfv+nHTSSRmP\nd865OXPmmMw8qT20d5j2jsvmMdpPTvdKdM6fMz//+c9Nvvrqq00O1fNpj5+XX37ZZO1hVVP7C9ZG\n+p7rZ4fWRbZv394bQ3tJ6bVm7dq1Jn/99dfeGA888EDGMbQO7r777vPGmDRpksnaByrN1ze+gQIA\nAIjEAgoAACASCygAAIBILKAAAAAipbc6K2W02FGL5ebPn2+ybgrrnHOTJ082+cknnzS5UE3YUHO0\nGd2AAQNM1mLd119/3Rvjs88+M1kLNWm2Wrz0jwqyoQ0sdQ4NHz7ce4w2ONRGi3q9mzJlijfGb3/7\nW5O1OeemTZv+jzPGjtJNxvWa0K1bt+gx9T3XjYH79evnPSapubD+ccK4ceO8Y/RnSXPRuOIbKAAA\ngEgsoAAAACKxgAIAAIhUPDcba1iooeH3jRw50uTy8nLvmJtvvjnjmHrvl8ZzxS30/ulmwr/73e8y\njvHEE094/6YbiWodArV0xUt/v1esWOEdo5sJX3jhhSbrptaNGzeOPo958+aZfOWVV3rH6EbYoWab\ntV1afte2bNlistZOnnLKKYlj6OdP//79TQ5t8qvXr7Fjx5qszTibNGnijZGW1zAXfAMFAAAQiQUU\nAABAJBZQAAAAkQpaA6X3Ooupnqd+/fom673d0047zeRQv4v169ebrPeDQ/eYUTx0Pofmd5cuXUzu\n2rWrydrD6auvvvLG0LmI2kPrlVavXu0dozVQOh+ymR+6wbBuDPyXv/zF5FBvMa2Z0TqcukB/xwtV\nz6PPo++Fvn+h/mItWrQwWWvY7rjjDpO1L5Rzzq1atSrjedb2uji+gQIAAIjEAgoAACASCygAAIBI\n9IHKkvZsOv74401u1aqVyVOnTvXG0PoG7ddSzDVihVRTr1Ps84TqIQ488ECTtYfT3LlzTf73v/8d\n9ZwobkuXLjVZ9xJzzq+BGjJkiMnaw2nmzJneGHp92rx5s8l1sZ6pmCTVXn388ccmn3POOd4Y+hid\nA/qZp3sdOpe8F15tV7d/egAAgBywgAIAAIjEAgoAACASCygAAIBIFJEHhBpadu7c2eQ777zT5Kee\nespk3czROb9oj6Lx4pKP96uqqspkbVB42WWXZXzOXJ8XxaGkpMRk/aMC55ybM2eOyZMmTTJZmyqG\n5ktS8S9zLN1iG3Zm09CytLQ0+jxCDVbrEr6BAgAAiMQCCgAAIBILKAAAgEgFrYFKuq+ezX3d6rg3\nr/dxtQ7BOecef/xxk5s2bWqybh4cujectMknNVHZScvrksv79cEHH5h86qmnmjxlyhST63qjurou\nNKeSNpLNZUPbtPxOFZtCbR6MdOLqDAAAEIkFFAAAQCQWUAAAAJFS1Qeqpu7D6/MeeeSR3jHl5eUm\nT5s2zeTKykqTQ72kqGep3bKph9DNYpcsWZJxjLreZwXxsrmOUvME7Dg+0QEAACKxgAIAAIjEAgoA\nACBSqmqgakqjRo1M7tmzp3dM48aNTW7evLnJoZonIAl9ZGqG1gDxPgB1Wy51gXwDBQAAEIkFFAAA\nQCQWUAAAAJFYQAEAAESiiNw5t3btWpOnT5/uHbNw4UKTx44da/KWLVtMLi0t9cbYtm1brqf4/6MB\nXnpQiFy8eK+AHZf0eZSW37Pqai7LN1AAAACRWEABAABEYgEFAAAQqR41NQAAAHH4BgoAACASCygA\nAIBILKAAAAAisYACAACIxAIKAAAgEgsoAACASCygAAAAIrGAAgAAiMQCCgAAIBILKAAAgEgsoAAA\nACKxgAIAAIjEAgoAACASCygAAIBILKAAAAAisYACAACIxAIKAAAgEgsoAACASCygAAAAIrGAAgAA\niMQCCgAAIBILKAAAgEgsoAAAACLtXMgn69Sp03eFfD4UzqJFi+oV6rlatmyZynn03XffZcz/1799\nX716O/4y5jKGPkbPMzRmNsfEWr16dUHmUbNmzYp2Dqnt27ebvPPOyZf1fLxX+ZA0h7L5+fUx69at\nK9gP17x58xqZR7m8TsVKf7addvK/9/n2229Nzsf8Xrt2beIgfAMFAAAQiQUUAABApILewgOKWdLX\n5NncftGvmvWY+vXrR59XPm7Z5TJmWm4D5UMut4oKQeeLc86VlZWZvHz5cpP1Fl6a36d8zMO6qDbf\nslPZ3K6sqXnCN1AAAACRWEABAABEYgEFAAAQiRooJMrmT9iRnz/rz6UWpzraCdR2xTJ/S0pKvH+b\nMmWKya+//rrJV199tclpqhkBahO+gQIAAIjEAgoAACASCygAAIBILKAAAAAiUUReQNVRuFmIYthi\nKbhNG92jzDnnWrVqZfLixYtNbt26deIYzZo1M3nlypUmUyCcXkm/S7rPV3l5uXdM586dTe7SpYvJ\ndWmfNOdq/8+H9OIbKAAAgEgsoAAAACKxgAIAAIhEDVRAdTVAjG14mE0Dy5qqd6DuwKf1Sk2bNvWO\nGTZsmMn77ruvyf379zdZ652c89/z0aNHmzxq1CiTv/nmG28MrbWpa9JSJ6Tn0bVrV5P//Oc/J45x\n4403mqwbEFMTB1SPun0VBQAAyAELKAAAgEgsoAAAACJRA5UlrSvQGhKtfwnVney8s325dQzNaa4z\noq7Cp+9f8+bNvWPatm1rcu/evU1u06aNyaHXWefWhAkTTN62bVvG84KvUPNZryPa92vq1Kkml5aW\nemN8+umnJs+cOdNk3m9kI+kzTX8nQr8jmzdvznhMgwYNduQUg+el5x163kJ9dvKbBgAAEIkFFAAA\nQCQWUAAAAJEKWgOVlt4rSs8rVEOg/Xj22Wcfkw8++GCTQz2Axo8fb/L8+fNN1nu7odcnra8h/Pfv\n17/+tXfMiSeeaHK7du1Mzub91Xq70047zeQZM2aYnE09Xl1TU79HWp/22GOPmRyqeVL33XefyQ0b\nNjQ59H6jehTz9bh+/fomN27cOGPWzzznnFu9erXJ+pm2ZcsWk0OvT1LtldLzdi68Z2gh8A0UAABA\nJBZQAAAAkVhAAQAARGIBBQAAEKlOVJJq4ZoW+27dutXkoUOHemNccsklJu+6664Zn3PDhg3ev+m4\n2jTvwQcfNPnzzz/3xlizZk3G50X1SSpu1OLdJk2aeMdUVFRkHEPn6hdffOEd07JlS5MHDhxoss6j\nhQsXZnxO5Ec2BcTaXPWAAw6IHvPtt982WQt1Q0W2sXIpho7dLD3bY9KsWIrGQ0XW5eXlJuvn03XX\nXWdyaF7pNe+f//ynyVogvnbtWm+M3Xff3WRtvtmiRQuT58yZ442xatUqky+//HKTV65c6T1G5TIX\n+QYKAAAgEgsoAACASCygAAAAIhW0Bqqm7hfr/V+9l3vuueea/Jvf/MYbQzf91Pqku+++22S9r+uc\nc2eccYbJP/7xj03+4Q9/aPJ5553njaH1D6rYawqKmdYDaA2Bc8517drVZG1O99FHH5k8a9Ysbwyd\nr9qM88gjjzRZGy865zdszKV+pZgV4loUeg31OhLacPr7lixZ4v3bggULTNaakdBmq9UhaTPaujan\n0kQ/80Lz7He/+53JZ599dsYxn3zySe/fJk6caPLSpUtN7tatm8na9Nc557p3725yUg1fqI5U65j/\n8Y9/mPzUU09lHDNXfAMFAAAQiQUUAABAJBZQAAAAkWq0D1ShNmLU+8HaV+Liiy82uayszBvjz3/+\ns8l33HGHyVVVVSaH7uO+9dZbJj/99NMmay3DRRdd5I2xaNEik7VGRjc9Rv4kzU/dfLOystI75oYb\nbjB5wIABJnfp0sXkn/3sZ94YOk82bdpksvZiCfWjoj4l//Q1Dc2X/v37m5y0qfOjjz7q/Ztez6rj\nvczmWqzzql+/fia/+eabeT8vZEfr00444QTvmLPOOivjGG+88YbJo0aN8o7Ra5zOG/3M+5//+R9v\nDO1Tl8tnmP68eg1U+ZqLfAMFAAAQiQUUAABAJBZQAAAAkQpaA5VLn5B81EXpuPvvv7/Ju+22m8mh\nvXYeeughk9etW5fxOfVndc6vV7r22mtNvuuuu0w++uijvTH69Olj8t57753xPGqjtNbv6H34UO2R\n1kk1bdrUZO0VpvVOISUlJSZrXY32p3LOuUaNGiWOizha99imTRvvmOOPPz5qzNB7VxP99EI1na+8\n8orJHTt2NFn3+Vu9erU3RlINGHLTsGFDk0P1S0nXzeeee87kxYsXe8fotWfbtm0m6357Y8eO9cZI\nqnnSvR5HjhzpHaNzUffGq659GfkGCgAAIBILKAAAgEgsoAAAACKxgAIAAIhUJzYT1kLclStXZvzv\ne+65pzeGblaoBcO6Oas+h3N+4a42+xo6dKjJf/3rX70xdFNILYAPFfohWTZzU4sMtcBX398ePXp4\nYzzzzDMmaxFtNkXjSgt8dY6E/qBBi0x1ftd21XEt0j8aOP/8871jBg8eHDVmqBlrIRpl6vVNG7w6\n59yBBx6Y8TG6yXWoiBz5oXOiQ4cOJmvz6BC9Bnz44YeJj1m7dq3JRx11lMlaiJ7NH69o0fgRRxxh\n8rx587zHaHPZXK5nuVwT+AYKAAAgEgsoAACASCygAAAAItWJGijd6FfrhPTevdaHOOfXRS1fvtxk\nveeq94Kd8+/Tzp4922Stdwg10dOamRUrVnjHoHponcHBBx9s8p/+9CeTQ01OC9H089VXXzX53HPP\n9Y6ZMGGCyaE6KVh6/dLXTOs7YptmOudfq5YsWRI9Rj7o9W7ixImJj9HXo1WrViZncz1DbnRuag3u\n5s2bvcfo55xmbeSsTX+d8ze8P/nkk03O5nqn564bbmvNUzZj6lzUz/h84aoJAAAQiQUUAABAJBZQ\nAAAAkQp6A1rvXRaqJkrv3W7YsMHkKVOmmHz44Yd7Y+gmiO+//77Jeo9V652cc27y5MkZs/Z0Wrp0\nqTdGRUVFxrxgwQLvMbVNddQRJdW3OOfcwIEDTdbNn9u3b7/D57Fp0yaTL7vsMu+YU0891WSdr1qL\n88QTT3hjHHfccSa//fbbJqdlg+Y009953Ty4d+/e0WP+4x//MHn69OmJz6t9wPTaE6rp1H/buHGj\nyX/4wx9MzmZuf/311yZ//PHHJutG2vlSF+dq0mfp+vXrTb7++uu9Me69996Mz3HdddfleHb/t1AP\np7PPPtvkhQsXmqw/W03VUofwDRQAAEAkFlAAAACRWEABAABEqpNNOPQe6g033GCy7lfmnHNt27Y1\nWesbtM5E70E759yf//xnk5s1a2Zy165dTQ7VHWitypo1a7xjsONC99l1Ly+t8SgvLzc51ONGx9U6\nuDPPPNNkrddzzrmnn37aZN1z7LHHHjP5sMMO88Z4/vnnTT7kkENMnjt3rveYuk7rTrQWSXvDheog\ntV5JLVq0yOTQXnhav6Tnscsuu5h84oknemOMGDHC5NatW5ucS12R1oBpHWHomlhWVhb9PGlSU3W9\nSc+zbds2k19++WXvGO0xpvMmH3Seheox9RpXTD3piudMAQAAUoIFFAAAQCQWUAAAAJFYQAEAAESq\nE0XkWnCnhX9aMBtqpPnSSy+Z/Mknn5isxdxvvvmmN4Y2xtTzOuaYY0wObYC4ZcsWk0OFmUiWNCdC\nRZpawP/vf//bZC3uDRXIamGxNs7U523SpIk3hhaIfvHFFyZfccUVJr/33nveGEo3sa3tcimS1t9H\nHaNz584m6wbjzvnvp77fv/jFL0x+8MEHvTF0Y1i9XmmDV93UNxeha5EW+zZo0MBk/aMYbdbpXLqa\nIuYireev16LFixd7x2hDXv3M0vczG/qHNfoHLaHrjP4ehf74Iq34BgoAACASCygAAIBILKAAAAAi\nFbQGKq33i/UebKg5pTYjPOGEE0zWupRZs2Z5Y2jtQsuWLU0eMGCAyaFGjFqvpXU2a9eu9R5T2yTV\nL2UjqeYp1MwtaVNLrXlZuXKlN4aOG6otyXSezvlzTRt4Dho0KHGMYq47qClJc2bmzJkmhzZwvfPO\nO01OqiPSWkvn/Iau+v5nY/78+Sa/++67Jmsz4ZKSEm+Mv/71rxmfQxu8vvXWW94xxd5IM62yuZ7p\nxr76uaf1eqWlpd4Yet3Quiqte8umuXAx4RsoAACASCygAAAAIrGAAgAAiFQn+kAlSdrg0zn/Xq/e\nLx4+fLjJr7zyijeG9mPRugLdOLRFixbeGFrjpH2h6oJcap7yMWY+njep5imb2gWtgdKNrn/zm98k\nngc1T/H0/desv4uPP/64N8Z//vMfk2+77TaTDzrooIxjOudv/Pv111+b/I9//MPkhx9+2Btj1apV\nJms/Mq3f69u3rzdGUu2K1tBQ75Q/SfWYeo3o0aOHN8b48eNN1lq6bK532htvzpw5Jjdr1izjeaZJ\nLtd3voECAACIxAIKAAAgEgsoAACASNRAZalx48Ym77XXXibvsssuJg8bNswb49577zX5m2++MXm3\n3XZLPA/d+y7UVwPFS9/PRo0aecfsueeeJut+aTpXQ/QxoZ5ViKO1lKF9vz744AOTBw8ebLLWs4X6\nL2ld1KJFi0zWeiatmXLOuQ0bNpis1yKtvdtjjz28MZJqRrTeJVR3l039KZLp66j7EI4bN857TLdu\n3Xb4eUePHm2yzl/tfZhmudRn8Q0UAABAJBZQAAAAkVhAAQAARGIBBQAAEIkK5CxpkaVuHKobZT7w\nwAPeGE2bNjVZmypmU1C5dOlSk3UDW+QmlwJCLaLNZqNgfY+1ILhjx44mDxkyxBvjmmuuyXgeShst\nOuc30dPCY93UFslzRN+HbP7AQ1/3L7/80uRQI1WdV/q8+t6tW7cu8Tz0efS6EipET6LNOhs2bOgd\nQ0PX3OgcqKioMHnMmDEmh/4IINZHH33k/du0adNMrmuNnfkGCgAAIBILKAAAgEgsoAAAACJRA5Ul\n3ZxRNwXVTRVDdSlaV9ChQweTtQ4hVHOxePFik7WGgsaa1SdpM1mtPQltBq21CD179jT5tNNOM7lX\nr17R56lz5NJLL/WOmT17tsnafLO216bo75a+l9W16WnS86pQHV3SueZj02u9Fum1Kh9jOlf751l1\n0c8jvdbss88+0WNqraQ+x6233uo9Rmty61pjVL6BAgAAiMQCCgAAIBILKAAAgEgUzGRJ7+0m3bsP\n3QvWDVsPPfRQk3XjRe0J45xzc+fONVnrbqqrdqPYJb0u+t9DtSfaa+U3v/mNyUOHDk18Tp0XSb2k\nQrQ2YfLkySbrptVTpkzxxkjq85SPOppiko8+YNXxmGyOr473SuepzjHnnPvlL3+Z8Ty0Lie0MbbO\nZWRHaxZ1k/nQa630M+y///u/Tf7Xv/5l8rx587wxsrle1WZ1+6cHAADIAQsoAACASCygAAAAIhW0\nBqpQvVYKQe/9ZvOz7bnnniaPHj3aZN07asSIEd4Y2kuqmF/DmqSvm/bPateunfcYfb969+5tcj72\nj9Pz0p5Ozjl38803m/zcc8+ZrPVboT3ImDfx6lJdmF5nvvrqK++YpNdD511oT75ir6Gpqc807f+3\nYcMGk3Xv1lB/QL1O6PXs2WefNTm0z11dv44U9+wFAACoASygAAAAIrGAAgAAiMQCCgAAIBKNNF12\nhXBJxY6lpaUmH3jggd4x9913n8nNmjUzWYuDp02bFn0eyI02lVuxYoV3zLBhw0w+88wzTb7uuutM\nzmZjTS2sPfnkk02eNWuW9xgt5tRi0GzmSF0qiA4pxM9fqOac1UEbNYaKyJPmnf5RTEjSa5SW1yNt\nmjRpYrI2YV6zZo3J5eXl3hj6Ry8nnXSSyXfeeafJ2ggafAMFAAAQjQUUAABAJBZQAAAAkQpaA5XW\nplt6XqH77toUrlOnTibfdNNNJutGwc75NTFVVVUmP/nkkyaHNtrMZpNIJEtqgBdqGrdo0SKT7777\nbpPvv/9+k7UuwTm/biSX5pu5bEAMK5vf+R0ds5hpI0Zt3Oiccy+99JLJxx13nMnajLM2Sst7rp8V\nWks5YcIE7zHaXFMbBS9cuNDktPysacKVFwAAIBILKAAAgEgsoAAAACLRByogtPHiHXfcYfIpp5xi\nci41FFozo3UGJSUl3mO0XxGyk/T+ZPP+hTbl/b5sNvBEOtBfKDOtzdPaPeecGzJkiMnamyhUR1gd\n6mJtjtY96mvw8ccfm9y1a9fEMULvMTLjGygAAIBILKAAAAAisYACAACIVK+Q9487depUtDertT5J\n97HTnk5lZWXeGLo3VLt27UzesGGDyaH+PlpnkxaLFi0qWFFJy5Yti3YewdLrz5o1awoyj5o1a5b3\nOZSPa2laarOy6ZOlfZ60hkb3Bw3VbybV3WTzeui5rl+/vmAvYvPmzVN5LUrqc5ftMXXZ2rVrE+cR\n30ABAABEYgEFAAAQiQUUAABAJBZQAAAAkej0l6UWLVqYrAXfWoC3ceNGbwwtPNdGc9q8rlCN6IC6\npjo2E86HtBSiZzOGbo6ujWb1D15CBeNJz0Nhc26yed14bXcc30ABAABEYgEFAAAQiQUUAABApII2\n0gQAAKgN+AYKAAAgEgsoAACASCygAAAAIrGAAgAAiMQCCgAAIBILKAAAgEgsoAAAACKxgAIAAIjE\nAgoAACASCygAAIBILKAAAAAisYACAACIxAIKAAAgEgsoAACASCygAAAAIrGAAgAAiMQCCgAAIBIL\nKAAAgEgsoAAAACKxgAIAAIjEAgoAACASCygAAIBILKAAAAAi7VzIJ2vTps13hXw+OPfdd/Eveb16\n9RLH0GOWL19ezzuomlRUVJgTyuVnVPrz5ELP49tvv40+pkGDBonPk49zTavKysqC/HDl5eVci2qp\nFStWFOwXpKysrE7PI72eJX12FNO1a/369YknyzdQAAAAkVhAAQAARCroLbxittNOdq2ZdNsoH7eV\n8iEfX5mm/WvXXG455iL26+rS0lJvjIMOOsjkCRMmmNyiRYuMzwGklc7dNWvW1NCZoFCSrk+1/frF\nN1AAAACRWEABAABEYgEFAAAQiRool9t92lwek5a6qNquUPfd9f3UGpBHHnnEe0zfvn1Nvv32201+\n4IEHMj6Hc7W/rgDFQecmNU+54bOkePENFAAAQCQWUAAAAJFYQAEAAERiAQUAABCJInLn3LZt20we\nNGiQd8xtt91mshb/vvXWWyYvXbo08XlCe6WlEQWL2dHGmc2bN/eOady4scm69x0F4kiDbH7n9fr1\nzTffZDx+5539j5ukZrT169dPPI9iV8zXV20wrXOgpKTE5JUrV3pjNG3aNP8nViB8AwUAABCJBRQA\nAEAkFlAAAACRUlUDlZamgZMnT/b+Te/T3nvvvRnH6N27t/dvCxcuNFl/trTUvxTzPfnqlPS6dOvW\nzeRPP/3UO6ZXr14mn3766Sbfd999OZ4dUH1C16aysjKTt27davKGDRtM1hpQ5/zraugYpJe+X+ef\nf77Jej076qijvDHee+89k5M2bU9T41G+gQIAAIjEAgoAACASCygAAIBIqaqBCt3bTLofmg/ai6eq\nqso7pmXLlib/4Q9/MPnyyy83efr06d4YPXv2NPmTTz4xuXXr1onnmpY6qTSrqVq6VatWmTxnzhzv\nmM2bN5v81VdfmZzUFweIFZpD2sNJezRp/VKbNm28MTp16mTyRx99ZLL2BGrSpIk3xvbt203m+lZc\nOnToYPKoUaNM1j5eP/jBD7wxpk2bZnIxzQG+gQIAAIjEAgoAACASCygAAIBIqaqBCinE/VCtBwg9\np9YIaB8ovZd/xRVXeGO8//77Jvft29fkRYsWJZ9sARTTPeiQ6jr/pHFbtGhh8jHHHOMdo3vhJUlL\nb7Tapq7XmmkPp86dO5vcvn17k0888URvDL0mPvfccybPnDnT5MrKSm8MvfbWxb3wilmjRo1MTqrj\n/fLLL71/S7qepfl3lW+gAAAAIrGAAgAAiMQCCgAAIBILKAAAgEipLyIvhGyKcktLS01eu3atyS+/\n/LLJv/rVr7wxGjZsaLJuOJy02TDSRYsZNWuTOef893Tu3LkZ/zuqR5oKUWPpuWvDyn322cfkPfbY\nwxtDHzNixAiTu3fvnvE5nfMLwCdMmGCyNpYNjbHTTvw/fFolFfg751yfPn2ixtR5l400/64yewEA\nACKxgAIAAIjEAgoAACBSQWugCrExcKFoEzltpKmbxDrnXEVFhcnavE7rAfQetHPF/ZoVu6QNWI84\n4orRlk0AAA2WSURBVAiTdbPVkLFjx2Z8Dt5vKL3W6HXk+uuvN7lLly7eGNu2bTN5zz33NHnx4sUm\nhxpa/vSnPzX5ww8/NDmbprHM7+IRmgNDhgzJ+Bitgws1Uy1mfAMFAAAQiQUUAABAJBZQAAAAkQpa\nA1Wb7ndrHULPnj1N7tixo/eYqqoqk1977bWMz5HmHilp7s1RqA14tY5Ee4WFnlPnjfb+Ar4vNJe1\nl87DDz9s8sEHH2zy5s2bvTGaNm1qsvaBeuihh0wOzWU9t5KSEu8Y1B567XLOucMOOyzjY8aPH2/y\nRx99lNdzqmnp/YQGAABIKRZQAAAAkVhAAQAARKIPVJa0P0+TJk1Mvuaaa0wO/Wy6F17onvL3FaqW\np7aprtdI34+ysjKTjznmmMQx5syZk3GMNWvWRJ9HLphHxSFUV3TppZea3K9fP5O1B92wYcO8MWbM\nmGHyli1bTNaeP6GedFqjWczXd/j0PR88eLB3jH6mqUceecRkrQN2zrkGDRrkcHbpwDdQAAAAkVhA\nAQAARGIBBQAAEIkFFAAAQCQaaWZJC+r2228/k3fZZZfEMbTosnnz5hmPL+bXqzbaunWryccdd5zJ\n2cwB3XBVi3lbtWqVOEZSEbn+cYJuehyi8zuXJq65FLfXtTmeVGi9cuVKk4899lhvjJtvvtnkV199\n1eQrr7zS5PXr13tjaBPYpPc79N8pGi9uSe+fFncff/zxiWPq9W327Nkm6x/NOBcuLI+VNPf0Ghj6\no4ikP+oK4RsoAACASCygAAAAIrGAAgAAiEQjzSzpBp7aSHPdunUmN2vWzBtDf15teJd0vHPp3sS3\nttO6kQMOOMDkbOazNtJMakQXkjQHdO61adPGO0bnntYEZPOzJP0+p3mu1tS56vPq696rVy+TX3rp\nJW8MfcwTTzxh8vLly03OpiFv0uuR5vcSuUmaA3odOf300xPH1OuKzpvQxtaqOhoF65i//OUvvcfc\ne++90c/DN1AAAACRWEABAABEYgEFAAAQiT5QWdJalU8++cTk4cOHmxy6X3z00Ueb/NBDD5l83nnn\nmax1Vc6lpxahmN/LkGxe1/3339/kc889N+Pxc+fO9f5t0qRJJmuvFT2PUP8d3ei1tLTU5KFDh5rc\nrVu3xHPbtGmTyVrftfvuu3tjrFq1yuRly5aZrBuJfvrpp94YufReyYdcar6q43n1/b7hhhtMDp2X\nvo4bNmwwWeuotH+Zc85VVlZmPEbHDI1R264BdY3+7um15pJLLsn430PGjx+/4ydWDfT6pptnO0cf\nKAAAgIJgAQUAABCJBRQAAECkgtZAFTOtXdAaAe3XsnbtWm+MgQMHmty/f3+Ty8vLTdZ9sZzz61+Q\nH0k9UZxzrnXr1iYn9XDSOjnnnFuxYkXGxzRq1Mjk0P56hx9+uMkjR440OWmPxUL56U9/avKAAQO8\nY+bPn1+gs8ksl75XudAxdI+ubOow9tlnH5Off/55k7PZ11CvT3fccYfJr7zyislLly71xgjVkaB4\n6Dxp2rSpyaFeSUr7Ommfu1z6reVSW5f0GP3v99xzj3dM48aNo5+Xb6AAAAAisYACAACIxAIKAAAg\nEgsoAACASPUK2ZixTZs26egCmYOkzTU1azGwc87df//9Jg8ePNjkt99+2+RTTjnFG6OmGg8mWb58\necG66lVUVOR9HmmjQG2a6Zxzr7/+uslJjeW0SNM5f1Pq999/3+Q+ffpkHNO5/PwhQexmsaEizaQC\n0c8++8zkE0880Rtj8eLFJldVVRVkHrVu3dqcrL6X1XVdTHrde/ToYfIhhxzijaHF2+3btzdZi2G1\niaBz/nWkb9++Jh900EEmX3bZZd4YEyZMMFkL4mvKihUrCnYtKisrK9rPNL3m/exnPzN59OjRiWPo\n9Ut/x9evX5/j2dW89evXJ84jvoECAACIxAIKAAAgEgsoAACASOm4aV2EtP5D61K0wZhzzt1+++0m\nH3HEESY3a9YsT2eHWPr+hWqgstlMM+l4bb75gx/8IGpM5/y6GZ2LWnulm/o659dzaQO81atXmxza\nTLZt27Yma3NGPc9t27Z5Y+hGyIWSS4O/pDFULk0Dv/zyS5Pfe++9xDF0TumYoXrM5cuXm6y1d++8\n847JI0aM8MbQ5sFa35eWmiiE6e/jaaedFj3GL37xC5P1ulHb5wDfQAEAAERiAQUAABCJBRQAAECk\n2n2D8v8T2+MmdHxsvUOoV8+qVatMrqysNFk3gW3RooU3RmiDYSRLmgP637U+zTm/n85rr71mstYU\n7LXXXt4YWnuyZMkSk9esWWNyaB6dddZZJjdp0sRk7a00Y8YMb4xXX33VZJ3fDRo0MDn0+um55yK2\nrixNqqNuqqqqymTd4DU0RjYbYasuXbqYnNRfrqKiwvs33Vyba1N6ZDM3u3fvbvJhhx2W8fhx48Z5\n/zZr1iyTtaYxVAtcmxTv1QsAAKCGsIACAACIxAIKAAAgUp2ogUqSzf3ipN47Svejcs65H/3oRybv\nvffeJmv9i/bUSJNC7qFYCFq/NG/ePO+Ye++91+QvvvjC5I0bN5ocqhvRGijt2aS9dEI1ULfddpvJ\n55xzjsk33nijySNHjvTGGDNmTMbz0BqobOpqUBhJfb90zoR+V7WeT+eUPseLL77ojaHzXXv+6FxG\nzQnVGh555JEmJ+2x+dZbb3n/po/RGr7aft3gGygAAIBILKAAAAAisYACAACIxAIKAAAgUp0oIteC\n2CuvvNLkM844w+Rjjz3WG2PZsmUmaxGmNpUbMGCAN0ZoU9fve+qpp0zWgjznwhuDfl8+iruzKfyr\nbcWBWgB7zz33eMfsuuuuJmuTOH1NdM6EJDWwDNHnTSrm7dChgzdGv379TJ49e7bJobmHHZePDYi1\naDzpv4eOP/zww00ePHhwxvO46aabvDF0Htb2jWOLib7noXl36aWXZhxj6tSpJo8ePdo7Rj+Patvn\nQhK+gQIAAIjEAgoAACASCygAAIBIdeKm9fr1603+8Y9/bHLHjh1N/vDDD70xfvazn5msNSNHH320\nydrM0Dm/6diWLVtMfuihh0wObSSaluZ0xdZIM+nevNYehWqAPvvssx16jnxZt25d1PGh89ImetOn\nT9+hc6qL8vF+V0fNojZN7Nq1q/cYrbfUMe6++26TQ78Pej1L2pA4H+pajU2u9L3ZfffdvWP23HPP\njGM8/PDDJmdTn1nX8A0UAABAJBZQAAAAkVhAAQAARKoTNVAlJSUmX3DBBSZPmzbN5NDGi48//rjJ\nSRt4hmiNwL777mvy2rVrE8dIi9peixCaA2lRVlZm8syZM6PH0LqYJk2amEwfKF9SvVJ5ebnJEydO\n9I7RXmInn3yyyZMmTTK5ZcuW3hi6Uble304//XSTQ5tJL1682OQLL7zQ5I8++sjk0O9DIWqeVLHV\nXtYUvT63aNEi8TFak/vVV1+ZHOpBqJuw1zXp/ZQAAABIKRZQAAAAkVhAAQAARKoTNVB6r37BggUm\n6z5PoR5Oek85qeZpw4YN3r+deeaZJi9fvtxkvcectOdVvtT2eqbaRuuT5s6da7LuUaY1M845d+ih\nh5qsvdDmzJmTeB7FPG/03HOprdHH6DUh1DdHa4mef/55k2fMmGFyqM6oXbt2Jrdv395kvb6F9jx7\n8cUXTda6Ov1Ztm7d6o0Bfw6k5XdCex+Gzkt7Cmp/ua+//trkUF1kse5/mK/3iW+gAAAAIrGAAgAA\niMQCCgAAIBILKAAAgEjFWQEWSQs3tTBTN86cMGGCN4Y239Siy2eeecbkzz//3Btj5cqVJmuRaT6K\nxtNSxIjqo5tMa8GoNsXUxorOOXf55ZebTIPCHVdZWWmyblrunHNdunQx+ZZbbjG5d+/eJut765xz\nK1asMHngwIEma2PVUAG4XhP1mEL9AUuxS+v1tk2bNiZ/8cUX3jH6h05aJL5w4UKT0/qz5iJ0vcvl\n5+MbKAAAgEgsoAAAACKxgAIAAIhUr5C1D23atKmRQoukZme6iWLo/n/Dhg0zPkbrmUIN8HQMPY9i\nvse8fPnygp18RUUFBTvfo/Nm48aNJodqYFq1amWy/o7UVD1eZWVlQeZReXl53udQNtdSfS9KS0tN\n1utK6L3T5oX6Oif995BirnnSc1+1alXBrkVlZWWpvBbpXAw1ddXmubrB+JtvvmmyNt50Lt2bru+o\n9evXJ86j2vvTAwAAVBMWUAAAAJFYQAEAAESqE32gkmoAdBPfbJSUlGT876F7zkB10HoHnZuhuVrM\nNS9plU2tkV5rtm3bZrLWlIQ2gk6i86G29/gq1g1tq5POxVD90pdffpkxq9pc75QrXhEAAIBILKAA\nAAAisYACAACIxM1jAJ5i7kmGuiXUKwsoBL6BAgAAiMQCCgAAIBILKAAAgEgsoAAAACJRRA4AKAqh\npqA0LUZN4RsoAACASCygAAAAIrGAAgAAiFSvtm80CQAAkG98AwUAABCJBRQAAEAkFlAAAACRWEAB\nAABEYgEFAAAQiQUUAABAJBZQAAAAkVhAAQAARGIBBQAAEIkFFAAAQCQWUAAAAJFYQAEAAERiAQUA\nABCJBRQAAEAkFlAAAACRWEABAABEYgEFAAAQiQUUAABAJBZQAAAAkVhAAQAARGIBBQAAEIkFFAAA\nQCQWUAAAAJH+H35w6guYjT13AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f16ce04ef90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Set generator in evaluation mode to use running means and averages for Batchnorm\n",
    "generator.eval()\n",
    "\n",
    "# Sample z ~ N(0, 1)\n",
    "minibatch_noise = Variable(torch.from_numpy(\n",
    "    np.random.randn(16, z_dim, 1, 1).astype(np.float32)\n",
    "))\n",
    "\n",
    "if cuda_available:\n",
    "    minibatch_noise = minibatch_noise.cuda()\n",
    "\n",
    "fakes = generator(minibatch_noise)\n",
    "\n",
    "fig = plt.figure(figsize=(10, 10))\n",
    "idx = 1\n",
    "for ind, fake in enumerate(fakes):\n",
    "    fig.add_subplot(4, 4, ind + 1)\n",
    "    plt.imshow(fake.data.cpu().numpy().reshape(28, 28), cmap='gray')\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}