VAE_diagonals.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''This code contains the implementation of simple VAE\n",
    "Refer to the blog post here: https://graviraja.github.io/vanillavae/\n",
    "\n",
    "This is a modified version of the above code to generate diagonals.\n",
    "'''\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dimensions of bars and stripes\n",
    "m = 2\n",
    "n = 2\n",
    "\n",
    "BATCH_SIZE = 64         # number of data points in each batch\n",
    "N_EPOCHS = 10000           # times to run the model on complete data\n",
    "INPUT_DIM = m*n     # size of each input\n",
    "HIDDEN_DIM = 16        # hidden dimension\n",
    "LATENT_DIM = 2#8         # latent vector dimension\n",
    "lr = 1e-3               # learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([6, 1, 2, 2])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import itertools\n",
    "def bars_and_stripes(rows, cols):\n",
    "    \n",
    "    data = [] \n",
    "    \n",
    "    for h in itertools.product([0,1], repeat=cols):\n",
    "        pic = np.repeat([h], rows, 0)\n",
    "        data.append(pic.ravel().tolist())\n",
    "          \n",
    "    for h in itertools.product([0,1], repeat=rows):\n",
    "        pic = np.repeat([h], cols, 1)\n",
    "        data.append(pic.ravel().tolist())\n",
    "    \n",
    "    data = np.unique(np.asarray(data), axis=0)\n",
    "    \n",
    "    return data\n",
    "\n",
    "bas = np.array(bars_and_stripes(m,n)).astype(np.float32).reshape(-1, 1, m, n)\n",
    "train_dataset = torch.from_numpy(bas)\n",
    "train_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 1, 2, 2])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diagonals = [\n",
    "    np.array([1,0,0,1]),\n",
    "    np.array([0,1,1,0]),\n",
    "]\n",
    "\n",
    "diags = np.array(diagonals).astype(np.float32).reshape(-1, 1, 2,2)\n",
    "train_dataset = torch.from_numpy(diags)\n",
    "train_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    ''' This the encoder part of VAE\n",
    "    '''\n",
    "    def __init__(self, input_dim, hidden_dim, z_dim):\n",
    "        '''\n",
    "        Args:\n",
    "            input_dim: A integer indicating the size of input (in case of MNIST 28 * 28).\n",
    "            hidden_dim: A integer indicating the size of hidden dimension.\n",
    "            z_dim: A integer indicating the latent dimension.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "        self.linear = nn.Linear(input_dim, hidden_dim)\n",
    "        self.mu = nn.Linear(hidden_dim, z_dim)\n",
    "        self.var = nn.Linear(hidden_dim, z_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x is of shape [batch_size, input_dim]\n",
    "\n",
    "        hidden = F.relu(self.linear(x))\n",
    "        # hidden is of shape [batch_size, hidden_dim]\n",
    "        z_mu = self.mu(hidden)\n",
    "        # z_mu is of shape [batch_size, latent_dim]\n",
    "        z_var = self.var(hidden)\n",
    "        # z_var is of shape [batch_size, latent_dim]\n",
    "\n",
    "        return z_mu, z_var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    ''' This the decoder part of VAE\n",
    "    '''\n",
    "    def __init__(self, z_dim, hidden_dim, output_dim):\n",
    "        '''\n",
    "        Args:\n",
    "            z_dim: A integer indicating the latent size.\n",
    "            hidden_dim: A integer indicating the size of hidden dimension.\n",
    "            output_dim: A integer indicating the output dimension (in case of MNIST it is 28 * 28)\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "        self.linear = nn.Linear(z_dim, hidden_dim)\n",
    "        self.out = nn.Linear(hidden_dim, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # x is of shape [batch_size, latent_dim]\n",
    "\n",
    "        hidden = F.relu(self.linear(x))\n",
    "        # hidden is of shape [batch_size, hidden_dim]\n",
    "\n",
    "        predicted = torch.sigmoid(self.out(hidden))\n",
    "        # predicted is of shape [batch_size, output_dim]\n",
    "        return predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VAE(nn.Module):\n",
    "    def __init__(self, enc, dec):\n",
    "        ''' This the VAE, which takes a encoder and decoder.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "        self.enc = enc\n",
    "        self.dec = dec\n",
    "\n",
    "    def forward(self, x):\n",
    "        # encode\n",
    "        z_mu, z_var = self.enc(x)\n",
    "\n",
    "        # sample from the distribution having latent parameters z_mu, z_var\n",
    "        # reparameterize\n",
    "        std = torch.exp(z_var / 2)\n",
    "        eps = torch.randn_like(std)\n",
    "        x_sample = eps.mul(std).add_(z_mu)\n",
    "\n",
    "        # decode\n",
    "        predicted = self.dec(x_sample)\n",
    "        return predicted, z_mu, z_var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# encoder\n",
    "encoder = Encoder(INPUT_DIM, HIDDEN_DIM, LATENT_DIM)\n",
    "\n",
    "# decoder\n",
    "decoder = Decoder(LATENT_DIM, HIDDEN_DIM, INPUT_DIM)\n",
    "\n",
    "# vae\n",
    "model = VAE(encoder, decoder).to(device)\n",
    "\n",
    "# optimizer\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train():\n",
    "    # set the train mode\n",
    "    model.train()\n",
    "\n",
    "    # loss of the epoch\n",
    "    train_loss = 0\n",
    "\n",
    "    for x in train_dataset:\n",
    "        # reshape the data into [batch_size, 784]\n",
    "        x = x.view(-1, m*n)\n",
    "        x = x.to(device)\n",
    "\n",
    "        # update the gradients to zero\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward pass\n",
    "        x_sample, z_mu, z_var = model(x)\n",
    "\n",
    "        # reconstruction loss\n",
    "        recon_loss = F.binary_cross_entropy(x_sample, x, size_average=False)\n",
    "\n",
    "        # kl divergence loss\n",
    "        kl_loss = 0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1.0 - z_var)\n",
    "\n",
    "        # total loss\n",
    "        loss = recon_loss + kl_loss\n",
    "\n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        train_loss += loss.item()\n",
    "\n",
    "        # update the weights\n",
    "        optimizer.step()\n",
    "\n",
    "    return train_loss\n",
    "\n",
    "def test():\n",
    "    # set the evaluation mode\n",
    "    model.eval()\n",
    "\n",
    "    # test loss for the data\n",
    "    test_loss = 0\n",
    "\n",
    "    # we don't need to track the gradients, since we are not updating the parameters during evaluation / testing\n",
    "    with torch.no_grad():\n",
    "        for i, (x, _) in enumerate(test_iterator):\n",
    "            # reshape the data\n",
    "            x = x.view(-1, 28 * 28)\n",
    "            x = x.to(device)\n",
    "\n",
    "            # forward pass\n",
    "            x_sample, z_mu, z_var = model(x)\n",
    "\n",
    "            # reconstruction loss\n",
    "            recon_loss = F.binary_cross_entropy(x_sample, x, size_average=False)\n",
    "\n",
    "            # kl divergence loss\n",
    "            kl_loss = 0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1.0 - z_var)\n",
    "\n",
    "            # total loss\n",
    "            loss = recon_loss + kl_loss\n",
    "            test_loss += loss.item()\n",
    "\n",
    "    return test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/luisserrano/anaconda3/lib/python3.7/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0, Train Loss: 3.02\n",
      "Epoch 100, Train Loss: 2.90\n",
      "Epoch 200, Train Loss: 2.72\n",
      "Epoch 300, Train Loss: 2.88\n",
      "Epoch 400, Train Loss: 2.13\n",
      "Epoch 500, Train Loss: 1.77\n",
      "Epoch 600, Train Loss: 0.89\n",
      "Epoch 700, Train Loss: 0.99\n",
      "Epoch 800, Train Loss: 1.06\n",
      "Epoch 900, Train Loss: 0.96\n",
      "Epoch 1000, Train Loss: 0.90\n",
      "Epoch 1100, Train Loss: 0.91\n",
      "Epoch 1200, Train Loss: 0.87\n",
      "Epoch 1300, Train Loss: 0.88\n",
      "Epoch 1400, Train Loss: 0.89\n",
      "Epoch 1500, Train Loss: 1.00\n",
      "Epoch 1600, Train Loss: 0.92\n",
      "Epoch 1700, Train Loss: 0.89\n",
      "Epoch 1800, Train Loss: 1.53\n",
      "Epoch 1900, Train Loss: 0.85\n",
      "Epoch 2000, Train Loss: 1.63\n",
      "Epoch 2100, Train Loss: 0.95\n",
      "Epoch 2200, Train Loss: 0.81\n",
      "Epoch 2300, Train Loss: 1.75\n",
      "Epoch 2400, Train Loss: 5.90\n",
      "Epoch 2500, Train Loss: 1.40\n",
      "Epoch 2600, Train Loss: 0.80\n",
      "Epoch 2700, Train Loss: 0.88\n",
      "Epoch 2800, Train Loss: 0.91\n",
      "Epoch 2900, Train Loss: 8.64\n",
      "Epoch 3000, Train Loss: 0.88\n",
      "Epoch 3100, Train Loss: 0.90\n",
      "Epoch 3200, Train Loss: 0.98\n",
      "Epoch 3300, Train Loss: 0.93\n",
      "Epoch 3400, Train Loss: 1.02\n",
      "Epoch 3500, Train Loss: 1.06\n",
      "Epoch 3600, Train Loss: 0.92\n",
      "Epoch 3700, Train Loss: 0.89\n",
      "Epoch 3800, Train Loss: 0.93\n",
      "Epoch 3900, Train Loss: 0.78\n",
      "Epoch 4000, Train Loss: 0.85\n",
      "Epoch 4100, Train Loss: 0.85\n",
      "Epoch 4200, Train Loss: 0.89\n",
      "Epoch 4300, Train Loss: 0.93\n",
      "Epoch 4400, Train Loss: 0.90\n",
      "Epoch 4500, Train Loss: 0.92\n",
      "Epoch 4600, Train Loss: 0.81\n",
      "Epoch 4700, Train Loss: 0.91\n",
      "Epoch 4800, Train Loss: 0.87\n",
      "Epoch 4900, Train Loss: 0.96\n",
      "Epoch 5000, Train Loss: 0.91\n",
      "Epoch 5100, Train Loss: 0.90\n",
      "Epoch 5200, Train Loss: 0.97\n",
      "Epoch 5300, Train Loss: 1.33\n",
      "Epoch 5400, Train Loss: 0.90\n",
      "Epoch 5500, Train Loss: 0.95\n",
      "Epoch 5600, Train Loss: 0.97\n",
      "Epoch 5700, Train Loss: 1.00\n",
      "Epoch 5800, Train Loss: 6.00\n",
      "Epoch 5900, Train Loss: 0.90\n",
      "Epoch 6000, Train Loss: 1.19\n",
      "Epoch 6100, Train Loss: 0.92\n",
      "Epoch 6200, Train Loss: 0.83\n",
      "Epoch 6300, Train Loss: 0.93\n",
      "Epoch 6400, Train Loss: 3.34\n",
      "Epoch 6500, Train Loss: 0.81\n",
      "Epoch 6600, Train Loss: 0.91\n",
      "Epoch 6700, Train Loss: 0.83\n",
      "Epoch 6800, Train Loss: 0.88\n",
      "Epoch 6900, Train Loss: 0.78\n",
      "Epoch 7000, Train Loss: 0.90\n",
      "Epoch 7100, Train Loss: 0.85\n",
      "Epoch 7200, Train Loss: 0.84\n",
      "Epoch 7300, Train Loss: 1.01\n",
      "Epoch 7400, Train Loss: 1.36\n",
      "Epoch 7500, Train Loss: 0.91\n",
      "Epoch 7600, Train Loss: 0.96\n",
      "Epoch 7700, Train Loss: 0.95\n",
      "Epoch 7800, Train Loss: 1.58\n",
      "Epoch 7900, Train Loss: 1.21\n",
      "Epoch 8000, Train Loss: 0.92\n",
      "Epoch 8100, Train Loss: 0.95\n",
      "Epoch 8200, Train Loss: 1.01\n",
      "Epoch 8300, Train Loss: 0.95\n",
      "Epoch 8400, Train Loss: 0.87\n",
      "Epoch 8500, Train Loss: 0.88\n",
      "Epoch 8600, Train Loss: 0.89\n",
      "Epoch 8700, Train Loss: 0.92\n",
      "Epoch 8800, Train Loss: 0.90\n",
      "Epoch 8900, Train Loss: 0.81\n",
      "Epoch 9000, Train Loss: 0.95\n",
      "Epoch 9100, Train Loss: 0.83\n",
      "Epoch 9200, Train Loss: 1.15\n",
      "Epoch 9300, Train Loss: 0.90\n",
      "Epoch 9400, Train Loss: 0.94\n",
      "Epoch 9500, Train Loss: 0.97\n",
      "Epoch 9600, Train Loss: 0.96\n",
      "Epoch 9700, Train Loss: 0.89\n",
      "Epoch 9800, Train Loss: 0.84\n",
      "Epoch 9900, Train Loss: 0.83\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\n    if best_test_loss > test_loss:\\n        best_test_loss = test_loss\\n        patience_counter = 1\\n    else:\\n        patience_counter += 1\\n\\n    if patience_counter > 3:\\n        break\\n'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_test_loss = float('inf')\n",
    "losses = []\n",
    "\n",
    "for e in range(N_EPOCHS):\n",
    "\n",
    "    train_loss = train()\n",
    "    #test_loss = test()\n",
    "\n",
    "    train_loss /= len(train_dataset)\n",
    "    #test_loss /= len(test_dataset)\n",
    "\n",
    "    losses.append(train_loss)\n",
    "    \n",
    "    if e % 100 == 0:\n",
    "        print(f'Epoch {e}, Train Loss: {train_loss:.2f}')\n",
    "\n",
    "'''\n",
    "    if best_test_loss > test_loss:\n",
    "        best_test_loss = test_loss\n",
    "        patience_counter = 1\n",
    "    else:\n",
    "        patience_counter += 1\n",
    "\n",
    "    if patience_counter > 3:\n",
    "        break\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fafe38e2310>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXjU5b338fc3+zYBsg4QIGxJxApoqYjginbBra1aW3dri1pRq56ec+w5p33qc46nfezRiliXWrW9tNZqrUvVUxfcUEQBQUWEALIEJAlhy0L2+/ljJhAwmITM5Jf5zed1Xbkyy29mvswVPvnm/t1z3+acQ0REYl+C1wWIiEhkKNBFRHxCgS4i4hMKdBERn1Cgi4j4hAJdRMQnFOgSM8ws0czqzGxkJI8V8QvTPHSJFjOr63Q1A2gC2sLXr3DOPdL/VfWdmf0nUOScu9TrWkQ6S/K6APEv51xWx2UzWw/8wDn38sGON7Mk51xrf9Qm4kcachHPmNl/mtljZvaomdUCF5rZNDN7x8x2mtlnZjbXzJLDxyeZmTOz4vD1h8P3v2BmtWa20MxG9/bY8P3fMLPVZrbLzO40s7fM7NJD+Dcdbmavh+v/0MxO63Tf6Wa2Mvz6FWZ2ffj2AjN7PvyY7Wb2RqfHFJnZ38ys2sw+NbOrO913jJktNbPdZlZpZrf2tl7xFwW6eO1bwJ+AQcBjQCtwHZAHTAe+DlzxBY8/H/gPIAfYCPzf3h5rZgXAX4CfhF/3U+Do3v5DzCwF+DvwHJAPXA88Zmbjwoc8CFzunAsAE4HXw7f/BFgXfkwwXCNmlhh+vveA4cCpwE/MbGb4cXcCtzrnsoFxwBO9rVn8RYEuXlvgnHvWOdfunNvjnHvPObfIOdfqnFsH3Aec8AWPf8I5t9g51wI8Akw+hGNPB5Y5554O33c7sO0Q/i3TgRRCIdsSHl56Afhu+P4WYIKZBZxz251zSzvdPgwY6Zxrds51BP0xQLZz7pbw7WuA3x/wfOPNLNc5V+ucW3QINYuPKNDFa5s6XzGzMjN7zsy2mtlu4GZCXfPBbO10uQHIOtiBX3DssM51uNBMgYoe1H6gYcBGt/9Mgw2EumsI/TVyJrDRzF4zs6nh238ZPu4VM1trZj8J3z4KGBkeitlpZjuBfybUxQNcBkwAVpnZu2Y26xBqFh9RoIvXDpxmdS/wETAuPJTwM8CiXMNnQFHHFTMz9oVwb2wBRoQf32EksBkg/JfHmUABoaGUP4dv3+2cu945Vwx8E/gXMzuB0C+Zcufc4E5fAefcGeHHrXLOfTf8fP8D/NXM0g6hbvEJBboMNAFgF1BvZofxxePnkfJ34CgzO8PMkgiN4ed385hEM0vr9JUKvE3oHMCNZpZsZicDs4C/mFm6mZ1vZtnhYZ1awlM4w687NvyLYFf49jZgIdBsZjeGXyPRzI4wsy+HH3eRmeU559rDj3NAe4TfG4khCnQZaG4ELiEUePcSOlEaVc65SuA84DagBhgLvE9o3vzBXAjs6fS1yjnXBJwBnEVoDH4ucL5zbnX4MZcAG8JDSZcDF4VvLwXmA3XAW8AdzrkF4SmcswidoF0ffs57gezw42YBK8MzhH4NnOecaz70d0JinT5YJHKA8OySLcA5zrk3va5HpKfUoYsAZvZ1MxsUHjr5D0JDJ+96XJZIryjQRUJmEJoLvo3Q3PdvhodQRGKGhlxERHxCHbqIiE94tjhXXl6eKy4u9urlRURi0pIlS7Y557qcVutZoBcXF7N48WKvXl5EJCaZ2YaD3achFxERn1Cgi4j4hAJdRMQntGORiMSclpYWKioqaGxs9LqUqElLS6OoqIjk5OQeP0aBLiIxp6KigkAgQHFxMfsvbukPzjlqamqoqKhg9OjR3T8gTEMuIhJzGhsbyc3N9WWYA5gZubm5vf4LRIEuIjHJr2He4VD+fd0Gengd5nfNbLmZrTCzX3RxTGp4s981ZraoY2PeaPhk627++4WV1Da2ROslRERiUk869CbgZOfcJEJ7MH7dzI454JjLgR3OuXGE9mP8VWTL3Kdi+x7ufX0dqyvrovUSIiLdysr6ot0OvdFtoLuQjvRMDn8duKLXWcAfwpefAGZalP4eKg0GAFi1tTYaTy8iErN6NIYe3vpqGVAFvNTF7uLDCW+yG95lZReQ28XzzDazxWa2uLq6+pAKLhqSTlZqEqu27j6kx4uIRMuGDRuYOXMmEydOZObMmWzcuBGAxx9/nC996UtMmjSJ448/HoAVK1Zw9NFHM3nyZCZOnEh5eXmfX79H0xadc23AZDMbDPzNzL7knPuo0yFddeOfW5fXOXcfcB/AlClTDmndXjOjpDCLT9Shiwjwi2dX8PGWyDZ4E4Zl8/MzDu/14+bMmcPFF1/MJZdcwgMPPMC1117LU089xc0338w//vEPhg8fzs6dOwG45557uO6667jgggtobm6mra2tz3X3apaLc24n8BqhDQA6qwBGAIQ32R0EbO9zdQdRGsxmVWUtWstdRAaShQsXcv755wNw0UUXsWDBAgCmT5/OpZdeyu9+97u9wT1t2jRuueUWfvWrX7FhwwbS09P7/Prdduhmlg+0OOd2mlk6cAqfP+n5DKENcBcC5wDzXRTTtiwY4NF3N1JV20Rhdlq0XkZEYsChdNL9peNU4j333MOiRYt47rnnmDx5MsuWLeP8889n6tSpPPfcc3zta1/j/vvv5+STT+7T6/WkQx8KvGpmHwDvERpD/7uZ3WxmZ4aP+T2Qa2ZrgBuAf+1TVd0oKQydGNWwi4gMJMceeyx//vOfAXjkkUeYMWMGAGvXrmXq1KncfPPN5OXlsWnTJtatW8eYMWO49tprOfPMM/nggw/6/PrddujOuQ+AI7u4/WedLjcC5/a5mh4q2zvTZTcnlHS5zruISFQ1NDRQVFS09/oNN9zA3Llz+f73v8+tt95Kfn4+Dz74IAA/+clPKC8vxznHzJkzmTRpEr/85S95+OGHSU5OJhgM8rOf/exgL9VjMbmWy5DMFAoCqerQRcQz7e3tXd4+f/78z9325JNPfu62m266iZtuuimiNcXsR/9LgwHNRRcR6SRmA70sGKC8qo7Wtq5/S4qIxJuYDfTSYDbNre1s2N7gdSki4gG/T1s+lH9fzAZ6mZYAEIlbaWlp1NTU+DbUO9ZDT0vr3bTsmDwpCjCuIIsEC01dnHXEUK/LEZF+VFRUREVFBYe6hEgs6NixqDdiNtDTkhMpzsvUmi4icSg5OblXO/nEi5gdcoHQsIuGXEREQmI60EsKA2zY3kBDc6vXpYiIeC6mA70sGMA5KNdmFyIisR3opcFsQDNdREQgxgN9ZE4GackJWgJARIQYD/TEBKOkMMCqSs10ERGJ6UAHKC3UTBcREfBDoAcDbKtrZltdk9eliIh4KuYDvSx8YnS1unQRiXMxH+ilQe1eJCICPgj0/EAquZkpGkcXkbgX84EOoS79k0oFuojEN18EeklhgPLKWtrb/bmUpohIT/gi0MuCARqa29i0Q5tdiEj88kWg68SoiIhPAr2kULsXiYj4ItAzU5MYmZOhQBeRuOaLQIfwTBftXiQiccw3gV4WDLC+poHGljavSxER8YRvAr00GKCt3bG2WptdiEh88k2glwV1YlRE4ptvAr04N5OUxAQFuojELd8EelJiAmMLsjQXXUTilm8CHULDLurQRSRe+SrQS4MBtu5uZFdDi9eliIj0O98FOqD56CISl3wV6HtnumgpXRGJQ74K9GB2GtlpSToxKiJxyVeBbmaUBbN1YlRE4lK3gW5mI8zsVTNbaWYrzOy6Lo450cx2mdmy8NfPolNu90qDAVZvrcU5bXYhIvElqQfHtAI3OueWmlkAWGJmLznnPj7guDedc6dHvsTeKQ0GqG1qZcuuRoYPTve6HBGRftNth+6c+8w5tzR8uRZYCQyPdmGHat8SAJrpIiLxpVdj6GZWDBwJLOri7mlmttzMXjCzwyNQ2yEp0e5FIhKnejLkAoCZZQF/BX7snDuw/V0KjHLO1ZnZLOApYHwXzzEbmA0wcuTIQy76i2SnJTNsUJpOjIpI3OlRh25myYTC/BHn3JMH3u+c2+2cqwtffh5INrO8Lo67zzk3xTk3JT8/v4+lH1yplgAQkTjUk1kuBvweWOmcu+0gxwTDx2FmR4eftyaShfZGaTCbtdV1tLS1e1WCiEi/68mQy3TgIuBDM1sWvu2nwEgA59w9wDnAVWbWCuwBvus8nDdYFgzQ0uZYV12/dzkAERG/6zbQnXMLAOvmmHnAvEgV1Ved13RRoItIvPDVJ0U7jM3PIinBNI4uInHFl4GekpTAmPxMBbqIxBVfBjqEToxq1UURiSe+DfSyYICKHXuoa2r1uhQRkX7h20AvLexYAkBduojEB/8GelCBLiLxxbeBPnxwOpkpiVqkS0Tihm8DPSHBKAkGtEiXiMQN3wY6hE6MrqrUZhciEh98HeilhQF2NrRQVdvkdSkiIlHn70APZgNaG11E4oOvA127F4lIPPF1oA/JTKEgkKoOXUTigq8DHULz0VdrCQARiQO+D/SyYIDyyjra2jXTRUT8zfeBXhrMpqm1nfU19V6XIiISVf4PdK3pIiJxwveBPr4wiwTT1EUR8T/fB3paciLFuZmauigivuf7QIfQTBcNuYiI38VNoG/Y3kBDsza7EBH/iotALwsGcA7KK+u8LkVEJGriItA71nTRsIuI+FlcBPrInAzSkhM000VEfC0uAj0xwSgp1BIAIuJvcRHoEPqAkTp0EfGz+An0YIBtdU3U1GmzCxHxp7gKdNCJURHxr7gLdA27iIhfxU2g52elkpOZog5dRHwrbgLdzEInRjXTRUR8Km4CHULDLuWVtbRrswsR8aG4CvSyYICG5jY27WjwuhQRkYiLq0DXiVER8bO4CvQS7V4kIj4WV4GemZrEyJwMVunEqIj4UFwFOmizCxHxr24D3cxGmNmrZrbSzFaY2XVdHGNmNtfM1pjZB2Z2VHTK7bvSwgCfbqunqbXN61JERCKqJx16K3Cjc+4w4BjgajObcMAx3wDGh79mA3dHtMoIKg0GaGt3rKnSZhci4i/dBrpz7jPn3NLw5VpgJTD8gMPOAv7oQt4BBpvZ0IhXGwFlWtNFRHyqV2PoZlYMHAksOuCu4cCmTtcr+HzoY2azzWyxmS2urq7uXaURUpyXSUpiggJdRHynx4FuZlnAX4EfO+d2H3h3Fw/53McxnXP3OeemOOem5Ofn967SCElOTGBsQZbmoouI7/Qo0M0smVCYP+Kce7KLQyqAEZ2uFwFb+l5edJRppouI+FBPZrkY8HtgpXPutoMc9gxwcXi2yzHALufcZxGsM6JKgwG27m5kV0OL16WIiERMUg+OmQ5cBHxoZsvCt/0UGAngnLsHeB6YBawBGoDLIl9q5OxbAmA3U8fkelyNiEhkdBvozrkFdD1G3vkYB1wdqaKirWOmy+rKWgW6iPhG3H1SFCCYnUZ2WpJOjIqIr8RloJuZlgAQEd+Jy0CH8JoulbWERotERGJfHAd6NrWNrWzZ1eh1KSIiERG3gb5vCYADPyMlIhKb4jbQOza70IlREfGLuA30QenJDBuUphOjIuIbcRvooM0uRMRf4jzQs1lbXUdLW7vXpYiI9FlcB3pZMEBLm2Nddb3XpYiI9FlcB3rHmi7aNFpE/CCuA31sfhZJCaapiyLiC3Ed6ClJCYzOy9SJURHxhbgOdAgNu2guuoj4QdwHelkwQMWOPdQ1tXpdiohIn8R9oJcGswE07CIiMS/uA33fmi4KdBGJbXEf6MMHp5OZkqiZLiIS8+I+0BMSjBKdGBURH4j7QIfQsIs2uxCRWKdAB0oLA+xsaKGqtsnrUkREDpkCHc10ERF/UKDTaU0XBbqIxDAFOpCTmUJ+IFUnRkUkpinQw0InRjV1UURilwI9rLQwQHllHW3tmukiIrFJgR5WGgzQ1NrO+prY2uyipq6JrbsavS5DRAYABXpYWQzOdHHOcemD73H23W/T1NrmdTki4jEFetj4wiwSjJg6Mfrix5V8uHkXm3fu4a9LNntdjoh4TIEelpacSHFuZsys6dLe7rj9pdWMzstk8ojB3PXqGppbtdm1SDxToHdSGgzEzJDL/67Yyidba7lu5niuO2U8m3fu4cmlFV6XJSIeUqB3UhoMsGF7Aw3NA3uzi7Zwdz42P5MzJg3jxJJ8JhUN4q7X1tDSpi5dJF4p0DspCwZwDtZU1Xldyhd67sPPKK+q48enlJCYYJgZ184cz6bte/jb+xpLF4lXCvROSgpDSwAM5BOjbe2O37y8mtLCAKcdMXTv7SeXFXDE8EHc9eoaWtWli8QlBXono3IzSUtOGNDj6M8s38y66nquP3U8CQm29/aOLn1DTQNPL9viYYUi4hUFeieJCcb4goF7YrS1rZ07Xi5nwtBsvjoh+Ln7TzmsgAlDs5n36hp94lUkDnUb6Gb2gJlVmdlHB7n/RDPbZWbLwl8/i3yZ/ad0AO9e9OT7m1lf08D1p5bs15136OjSP91Wz7PL1aWLxJuedOgPAV/v5pg3nXOTw183970s75QFA2yra6KmbmBtdtHS1s7cV8o5YvggTjms4KDHfXVCIWXBAHPnl6tLF4kz3Qa6c+4NYHs/1DIgDNS10Z9YUkHFjj3ccGoJZp/vzjskJIS69HXV9Tz34Wf9WKGIeC1SY+jTzGy5mb1gZodH6Dk90RHoA2nYpam1jXnz1zB5xGBOLM3v9vivHx6kpDCLO18pp11dukjciESgLwVGOecmAXcCTx3sQDObbWaLzWxxdXV1BF468vKzUsnJTBlQHfpfFleweWf33XmHhATjmpPHU15Vxwsfbe2HCkVkIOhzoDvndjvn6sKXnweSzSzvIMfe55yb4pybkp/ffafpBTOjtDDAJ5UDI9AbW9q4a/4apowawnHju3xbuzTriKGMK8hirrp0kbjR50A3s6CF20YzOzr8nDV9fV4vlQYDlFfWDogg/PO7G9m6u7HH3XmHxATjmpPHsaqylhc/VpcuEg96Mm3xUWAhUGpmFWZ2uZldaWZXhg85B/jIzJYDc4HvOue8T8I+KAsGaGhuo2LHHk/raGxp467X1jJ1dA7Txub2+vGnTxzGmLxM7nhlzYD45SQi0ZXU3QHOue91c/88YF7EKhoASvaeGN3NyNwMz+p4+J0NVNc2Me97R/aqO++QmGDMOXkcN/xlOS+trORrh3/+w0gi4h/6pGgXOtZ08fLEaENzK/e8vpYZ4/KYOqb33XmHMycNozg3g7mvlBPjfziJSDcU6F3ISk1iRE66pydG/7hwA9vqmrn+1PF9ep6kxASuPmkcK7bs5pWVVRGqTkQGIgX6QZQWZnvWodc1tXLv62s5oSSfL4/K6fPzffPI4YzMyWDufHXpIn6mQD+IsmCAT7fVe7L58h/eXs+OhhauP7UkIs+XnJjA1SeN5YOKXby2amDO/xeRvlOgH0RpMEBbu+v3zS52N7Zw3xvrmFlWwOQRgyP2vN8+qoiiIen8RmPpIr6lQD+IMo/WdHlwwXp27Ylcd94hOTyWvnzTTt4o3xbR5xaRgUGBfhDFeZmkJPbvZhe7Glq4f8E6vjqhkC8NHxTx5z/7qCKGD07njpdXq0sX8SEF+kEkJyYwtiCrXxfp+v2CddQ2tvLjUyLbnXdISUrgqhPHsnTjTt5aE9Mf5hWRLijQv0BZsP92L9pR38wDb61n1hFBJgzLjtrrnDuliKGD0rjjFXXpIn6jQP8CJYUBtu5uZFdDS9Rf63dvrqO+uZXrZkanO++QmpTIVSeO5b31O1i4Tl26iJ8o0L/A3hOjUf6AUU1dEw+9vZ7TJw7bux57NH1nyggKs1O54+XyqL+WiPQfBfoX2Ld70e6ovs59b6yjsaWN62b27VOhPZWWnMiVJ4xl0afbeUdduohvKNC/wNBBaQTSkqJ6YrSqtpE/LFzPNycPZ1xBVtRe50DfO3ok+YFU5r6iLl3ELxToX8DMon5i9J7X1tHS5rimn7rzDmnJiVxx/BjeXlvDe+vjZstYEV9ToHejNBhgVWVtVGaEVO5u5OFFG/j2kcMZnZcZ8efvzgVTR5GXlaIuXcQnFOjdKA1mU9vYypZdjRF/7t++Gtp44pqT+7c775Ceksjs48fwZvk2lmzY4UkNIhI5CvRulEXpxOiWnXt49N1NnDulyNNNNC48ZhQ5mSncoS5dJOYp0LvRsdlFpE+M3vXqGhyOq08aF9Hn7a2MlCR+eNwY3lhdzfsb1aWLxDIFejcGpSczbFBaRE+MbtrewF8Wb+K8r4ygaIh33XmHi6eNYkhGssbSRWKcAr0HSiM80+WuV9dgmOfdeYfM1CR+cNwYXl1VzQcVO70uR0QOkQK9B0qCAdZW19HS1t7n59pQU8/jSyo4f+pIhg5Kj0B1kXHxtFEMSleXLhLLFOg9UBYM0NLm+HRbfZ+f6875a0hKMK46cWwEKoucQFoyP5gxmpdXVvHR5l1elyMR0hqBJkRihwK9B0oLQ6sf9vXE6Kfb6nlyaQUXHjOKwuy0SJQWUZdMLyY7LUldug/UN7Vyw2PLmPiLF/nb+xVelyP9RIHeA2MLMklMsD5PXbzj5dWkJoXWURmIstOS+f6M0bz4cSUfb4nu+jUSPasrazlz3gKeWraZ4YPTuf6x5fzs6Y9oblW37ncK9B5ITUpkTF5mn06Mrqmq5enlW7j42FHkB1IjWF1kXXbsaAKpSdw5X116LHpiSQVnzlvArj2tPHz5VJ6/7jh+eNxo/rhwA+fdt5CtUfiAnAwcCvQeKg0G+jTk8puXy8lITuSK4wdmd95hUEYyl00v5oWPtvJJlFeZlMjZ09zGPz+xnH96fDmTRwzm+etmcOy4PJITE/i30yZw1/lHsWprLaff+SYL12qFTb9SoPdQWTBAxY491DW19vqxn2zdzXMffsal04vJyUyJQnWR9f0Zo8lKTeLO+Wu8LkV6YG11Hd/67Vs8vqSCa04ex8OXT6UgsP85mtMmDuWZOdMZlJ7Mhb9fxH1vrNWOVT6kQO+h0mDoxOihDLvc8XI5meFPZMaCwRkpXHLsKJ7/8DPKo7y5h/TNM8u3cOadC6iqbeKhy47mxq+WkpTY9X/rcQUBnp4zg69OKOSW5z/hR48sPaQGRQYuBXoP7VvTpXcBt2LLLl74aCvfnzGawRkDvzvv8IMZY0hPTmSuuvQBqbGljX9/6kOuffR9DhuazXPXzuCEkvxuH5eVmsRvLziKn84q4x8rtnLWvAWsqdIvbb9QoPfQ8MHpZKYk9nqmy29eLieQlsTlM0ZHqbLoGJKZwsXTivn7B1tYU1XndTnSyYaaes6++20efmcjV5wwhkdnH9OrD6mZGbOPH8vDP5jKrj0tnDXvLZ774LMoViz9RYHeQwkJxvjC3p0Y/bBiFy99XMkPjxvDoPTkKFYXHT88bjRpSYnM04yXAeN/P/qM0+cuoGLHHu6/eAo3feMwkg8yxNKdY8fm8ew1MygJBrj6T0v5r+c+1geRYpwCvRfKggFW92Kzi9tfXs2g9NCskViUm5XKRdNG8czyLayrVpfupebWdn7x7AqufHgpYwqy+Ps1MzhlQmGfn3fooHQemz2Ni6eN4ndvfsoF9y+iurYpAhWLFxTovVAaDLCjoaVHP/Dvb9zB/E+qmH38GAJpsdedd/jhcWNISUpg3qsaS/dKxY4Gzr13IQ++tZ7Lphfz+BXTGJETuVU6U5ISuPmsL3HbdyaxvGInp9/5Jks2aFvCWKRA74XSYM/XRr/95XJyMlO45NjiKFcVXfmBVC6cOoqnl21hfQTWspHeeWVlJafNXcC6qjruvuAofn7G4aQkRee/7bePKuJvP5pOWnIi5937Dn94e72mNsYYBXovlPVw6uLi9dt5Y3U1Vxw/hqzUpP4oLapmnzCGpATjLnXp/aalrZ3/fmEll/9hMUVD0vn7tTP4xhFDo/66hw3N5pk5oRkzP39mBdc/toyGZk1tjBUK9F7IyUwhP5DabYd+20uryctK5eJpxf1TWJQVBNI4f+pInnx/MxtrGrwux/e27mrk/N+9w72vr+OCqSP561XHMiq3/zYRH5SezO8unsKNp5bw9PItfPu3b+uvsxihQO+lsmCAVZUHn7q4cG0Nb6+t4aoTx5KektiPlUXXlSeMJTHB+O1r6tKj6Y3V1cya+yYrtuzmju9O5r++dQRpyf3/c5SQYFwzczwPXXY0W3c3csa8Bbz0cWW/1yG9022gm9kDZlZlZh8d5H4zs7lmtsbMPjCzoyJf5sBRWhigvLKOtvbPjy0657j95dUUBFK5YOpID6qLnsLsNL73lRE8saSCTdvVpUdaW7vjthdXccmD75Kflcozc2Zw1uThXpfFCSX5PDtnBsW5mfzwj4v59T9WdfmzLwNDTzr0h4Cvf8H93wDGh79mA3f3vayBqzQYoKm1nfU1n/8T9O21Nbz76XauPmmcJ11VtF154lgSzLj79bVel+IrVbWNXHj/IubOX8M5RxXx1NXTGVeQ5XVZe43IyeDxK6dx3pQRzHt1DZc++C7b65u9Lku60G2gO+feAL5oDtNZwB9dyDvAYDOL/tkbjxzsxKhzjtteWs3QQWmc95URXpQWdUMHpfOdrxTx+OJNbN65x+tyfOHttduYdccC3t+0g1vPmcit504akEN1acmJ/Oqcifzy20ew6NPtnHHnAu0/OwBFYgx9OLCp0/WK8G2fY2azzWyxmS2urq6OwEv3v/GFWZh9furiG+XbWLJhh2+78w5XnRja2Pqe19Sl90V7u2Pe/HIuvH8Rg9KTePrqGZw7ZeA3At89eiRPXDkNgHPuXsif393ocUXSWSQC3bq4rctBNufcfc65Kc65Kfn53S8kNBClJSdSnJu535ouHd358MHpfCcG/lP2xfDB6Zzz5RE89t4mPtulLv1Q1NQ1celD7/HrF1dzxqRhPDNnxt7POMSCiUWDefaaGUwdk8O/Pvkh//LEBzS2tHldlhCZQK8AOqdYEbAlAs87YJUWBlhdue+j8K+uqmL5pp1cc/K4qH3oYyD50YljaXdOXfohWLx+O6fNXcA762q45V4JpmcAAAlCSURBVFtH8JvzJpMZg59VyMlM4aHLjmbOSeN4bPEmzr1noU6WDwCRSJ9ngIvDs12OAXY553y9dFtpMMD6mnr2NLft7c5H5KRz9peLvC6tX4zIyeDso4p49L1NVO7WlmY90d7uuPf1tZx33zukJSfwtx8dy/lTR2LW1R+4sSExwfinr5Vy/8VTWF9TzxnzFvD66tgcSu1PzrmozRTqtjUws0eBE4E8M6sAfg4khwu7B3gemAWsARqAy6JS6QBSFgzgHJRX1bJ1VyMfbd7NredMPORV72LR1SeN44mlFdzz+lp+fsbhXpczoO1saOafHl/OyyurmHVEkF+ePZHsGF7f50CnTCjk2TkzuPLhJVz64Ltcf0oJc04aR0JC7P6y6g3nHLVNrWyva6amvpnt9c1sr28KXa4LXd8Wvq3jmNnHj+HGr5ZGvJZuA905971u7nfA1RGrKAbsXdPls1oeeOtTinMz+NaR3s8Z7k8jw//mPy3ayFUnjv3clmcSsmzTTq5+ZClVtY38nzMmcMmxxTHdlR9McV4mT/7oWH765Ifc9tJqlm/ayW3nTY7JZaPb2x279rR0Gc77buu43MSO+haaD7LscEZKIjmZKeRmppCflUppYTa5WSl8pTgnKrXH3uDdADAqN5O05ATueX0t67bVc/t5kw667ZefzTlpHH97fzP3vb6Ofz99gtflDCjOOR56ez23PL+SgkAaT1x5LJNGDPa6rKjKSEni9vMmc9SoIdz87MecOW8Bd1/wZSYMy/aknvZ2x56WNhqa22hobqW+qY1de1r2D+n6Zmrqmqmpb9ob1DsaWg46JBJITSInK4WczBSGD07jiOHZ5GSmkpsZui0nK2Xv5dzM1H6fgqpAPwSJCcb4ggAfbt7F2PxMzpwUX915h+K8TM6aPIyHF23gihPGkh9I9bokT7W0tbN+Wz2rK+t4etlmXvy4klMOK+R/zp3EoIzY61QPhZlx8bRiDh+WzY8eWcq3736LW751BN8+6uDnl5zrFLxNbdQ3t+4XwntaQt8b9t7eRn3TvmP2v77/cd0ZlJ68N4CLczP58qghoWDuHNKZKeRlpTIkM5nUpIE9JVmBfohKg6FA//EpJSTGyVhhV+acNI6n3t/M/W+u46ZZh3ldTr9oaWtnQ00ouFdX1lIe/v7ptnpaw51dSmIC/zbrMH5w3GhfDrF058ujcnj2mhlc86f3ueEvy3liSQWJCcae5jbqO4duUysNLW30ZpXetOQEMlOSSE9JJDMliYzURDJSEsnNSiUzJZH0lCQyUxLJSE0iIyUxdDkldDk7PTnUPWelMCQjxXfnvRToh+jso4pITkzgtH5Y0nQgG5OfxZmThvHHhRuYffwYcrP806V3BHcosOtYXVVLeTi4W9pCCWQGI3MyGF+QxSkTCikpzGJ8QYCx+VkD8hOf/akgkMYjP5jKbS+t5pWVVaSnhIJ3cEYKmeEQzgiHb3pKUvi2juuJZIYDOaNTQKcnJ8Z1A9Ud82oB+ylTprjFixd78toSWWuqajn19jcYNiidETnp5GWlkh9IDX3PSiUvEPqTNS8rldyslAH3Z2trWzvraxoor6xldWUd5VWhrnvdtrr9gnvEkAxKCrMYVxCgpDCLkkIFt/Q/M1vinJvS1X3q0KXPxhUE+H9nT2T+J1Vsq2tixZbdbKttorap640RstOSyOsI/EA49LP2hX7ovtD1SC6j0NrWzobt+4J7dWUta6rqWFddv98shRE56ZQUBDiprIDxBaHgHleg4JaBTx26RE1jSxvVtU1sq2tiW11z6Hun6x33Vdc1UdvYdfgHUpP2C/iOzj+v45fA3l8I+2YUtLa1s3F7Q6jbrqxldVXo+4HBXTQknZLCAOMLsygpCH0fV5BFRor6HBm41KGLJ9KSExmRk9GjDY0bW9qoqW/uFPhN4cBvpjr8i2B1ZS1vr61h156WLp8jMyWRIZkpVNU20dy6f3CPL8jihJJ8xhcGwsMmCm7xH/1Ey4CQlpzI8MHpDB+c3u2xza3t1NQ3sa22meq6xvD30C+B7fXNFGan7TdUEotrpYgcCv2kS8xJSUpg6KB0hg5KBwZ5XY7IgOGvSZgiInFMgS4i4hMKdBERn1Cgi4j4hAJdRMQnFOgiIj6hQBcR8QkFuoiIT3i2louZVQMbDvHhecC2CJYT6/R+7E/vxz56L/bnh/djlHMuv6s7PAv0vjCzxQdbnCYe6f3Yn96PffRe7M/v74eGXEREfEKBLiLiE7Ea6Pd5XcAAo/djf3o/9tF7sT9fvx8xOYYuIiKfF6sduoiIHECBLiLiEzEX6Gb2dTNbZWZrzOxfva7HS2Y2wsxeNbOVZrbCzK7zuiavmVmimb1vZn/3uhavmdlgM3vCzD4J/4xM87omr5jZ9eH/Ix+Z2aNmluZ1TdEQU4FuZonAXcA3gAnA98xsgrdVeaoVuNE5dxhwDHB1nL8fANcBK70uYoC4A/hf51wZMIk4fV/MbDhwLTDFOfclIBH4rrdVRUdMBTpwNLDGObfOOdcM/Bk4y+OaPOOc+8w5tzR8uZbQf9jh3lblHTMrAk4D7ve6Fq+ZWTZwPPB7AOdcs3Nup7dVeSoJSDezJCAD2OJxPVERa4E+HNjU6XoFcRxgnZlZMXAksMjbSjz1G+CfgXavCxkAxgDVwIPhIaj7zSzT66K84JzbDPwa2Ah8Buxyzr3obVXREWuBbl3cFvfzLs0sC/gr8GPn3G6v6/GCmZ0OVDnnlnhdywCRBBwF3O2cOxKoB+LynJOZDSH0l/xoYBiQaWYXeltVdMRaoFcAIzpdL8Knfzr1lJklEwrzR5xzT3pdj4emA2ea2XpCQ3Enm9nD3pbkqQqgwjnX8RfbE4QCPh6dAnzqnKt2zrUATwLHelxTVMRaoL8HjDez0WaWQujExjMe1+QZMzNCY6QrnXO3eV2Pl5xzNznnipxzxYR+LuY753zZhfWEc24rsMnMSsM3zQQ+9rAkL20EjjGzjPD/mZn49ARxktcF9IZzrtXM5gD/IHSm+gHn3AqPy/LSdOAi4EMzWxa+7afOuec9rEkGjmuAR8LNzzrgMo/r8YRzbpGZPQEsJTQz7H18ugSAPvovIuITsTbkIiIiB6FAFxHxCQW6iIhPKNBFRHxCgS4i4hMKdBERn1Cgi4j4xP8H0kd1BZm08PgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "losses = np.array(losses)\n",
    "plt.plot(losses[::1000], label='Loss')\n",
    "plt.title(\"Training Losses\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAEvCAYAAAAtufaDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAANBElEQVR4nO3dP49TZx7F8XN2NryCcQUkZiUUaUpsUW49qWihjjQVL4A3kmYKlC5oSwok2jQpsIuVQCukWTQRI4p4lBeAkH5bDNF649l5nnvH98/PfD8Sha2xn58Ok5N7zb2PHRECgKz+MvQAAHAdlBiA1CgxAKlRYgBSo8QApEaJAUjtr1286f7+fkyn0y7e+krL5bL3NSUpItzk58mnjIzKyOhCJyU2nU61WCy6eOsr2Y1/DwZBPmVkVEZGFzidBJAaJQYgNUoMQGqUGIDUKDEAqVFiAFKjxACkRokBSI0SA5BaVYnZPrT91vaJ7SddDwUAtYolZntP0g+SvpN0IOmR7YOuBwOAGjVHYvclnUTEu4j4KOmZpAfdjgUAdWpK7Kak92uPzz4/BwCDqymxy25Z3/iKJNtHthe2F6vV6vqT7RjyKSOjMjLaVFNiZ5Jurz2+JenDn38oIo4jYh4R88lksq35dgb5lJFRGRltqimxV5Lu2r5j+4akh5KedzsWANQpbooYEZ9sP5b0UtKepKcR8abzyQCgQtXOrhHxQtKLjmcBgMa4Yh9AapQYgNQoMQCpUWIAUqPEAKRGiQFIjRIDkBolBiA1SgxAalVX7De1XC5lX7b5RbciNjbX6Nx8Pm/8GvIpI6MyMrrAkRiA1CgxAKlRYgBSo8QApEaJAUiNEgOQGiUGIDVKDEBqlBiA1CgxAKlRYgBSK5aY7ae2f7P9uo+BAKCJmiOxHyUddjwHALRSLLGI+FnS7z3MAgCNbW0rHttHko629X67hnzKyKiMjDZtrcQi4ljSsSTZ7n/DoZEjnzIyKiOjTfzrJIDUKDEAqdVcYvGTpF8kfWv7zPb33Y8FAHWKn4lFxKM+BgGANjidBJAaJQYgNUoMQGqUGIDUKDEAqVFiAFKjxACkRokBSI0SA5Da1nax+JNzSb+2eN3+59e2YrvtS6+z7jctXtM2H+kas14jn+us2yYfKV9Gff8OSWQkSXLEeHbzsL2IiPmXsm4bZFQ2xKyZ8pF2KyNOJwGkRokBSG1sJXb8ha3bBhmVDTFrpnykHcpoVJ+JAUBTYzsSA4BGKDEAqVFiAFKjxACkRokBSI0SA5BaJ/dODvXNxLPZrPc1T09PdX5+3uhmMvIp29/fj+l02sFEV1sul72vKUkRQUYF/y+jrm4AH8Riseh9zfk8ze1yqfKZTqeDzHvNm+R7RUYXOJ0EkBolBiA1SgxAapQYgNQoMQCpUWIAUqPEAKRGiQFIjRIDkFpVidk+tP3W9ontJ10PBQC1iiVme0/SD5K+k3Qg6ZHtg64HA4AaNUdi9yWdRMS7iPgo6ZmkB92OBQB1akrspqT3a4/PPj8HAIOrKbHLblnf2ErG9pHthe3+b6tPgHzK1jNarVZDjzNKZLSppsTOJN1ee3xL0oc//1BEHEfEPNNXufeJfMrWM5pMJkOPM0pktKmmxF5Jumv7ju0bkh5Ket7tWABQp7gpYkR8sv1Y0ktJe5KeRsSbzicDgApVO7tGxAtJLzqeBQAa44p9AKlRYgBSo8QApEaJAUiNEgOQGiUGIDVKDEBqlBiA1CgxAKlVXbHf1Gw202LR/2YN9mUbbowP+ZQtl8tB5o3Y2KClc/N5uz0ByOgCR2IAUqPEAKRGiQFIjRIDkBolBiA1SgxAapQYgNQoMQCpUWIAUqPEAKRWLDHbT23/Zvt1HwMBQBM1R2I/SjrseA4AaKVYYhHxs6Tfe5gFABrjMzEAqW2txGwf2V7YXqxWq2297c4gn7L1jIaeZazIaNPWSiwijiNiHhHzyWSyrbfdGeRTtp7R0LOMFRlt4nQSQGo1l1j8JOkXSd/aPrP9ffdjAUCd4vbUEfGoj0EAoA1OJwGkRokBSI0SA5AaJQYgNUoMQGqUGIDUKDEAqVFiAFKjxACkRokBSM0Rsf03tVeSfm3x0n1J51sep+t1v4mIRttSXCMfKV9GjfORUmbU6++QREZ/6KTE2rK9GGKLkaHWbYOMyoaYNVM+0m5lxOkkgNQoMQCpja3Ejr+wddsgo7IhZs2Uj7RDGY3qMzEAaGpsR2IA0AglBiA1SgxAapQYgNQoMQCpUWIAUit+ZVsb+/v7MZ1Ou3jrKy2Xy97XlKSIcJOftz3IdS2z2az3NU9PT3V+ft4oH4mMapDRhU5KbDqdarFYdPHWV7Ib/x58UYb4O5nP298qN8TfZ7aMhjC2jDidBJAaJQYgNUoMQGqUGIDUKDEAqVFiAFKjxACkRokBSI0SA5BaVYnZPrT91vaJ7SddDwUAtYolZntP0g+SvpN0IOmR7YOuBwOAGjVHYvclnUTEu4j4KOmZpAfdjgUAdWpK7Kak92uPzz4/BwCDq9nF4rKtBDa2ALF9JOlIkr7++utrjrV71vPB5ciojIw21RyJnUm6vfb4lqQPf/6hiDiOiHlEzCeTybbm2xnr+Qw9y1iRURkZbaopsVeS7tq+Y/uGpIeSnnc7FgDUKZ5ORsQn248lvZS0J+lpRLzpfDIAqFC1s2tEvJD0ouNZAKAxrtgHkBolBiA1SgxAapQYgNQoMQCpUWIAUqPEAKRGiQFIjRIDkFrVFftNLZdL2ZdtftGtiI3NNTo3nze/D3c2m2mxWHQwzdWG+Dtpi4zKyOgCR2IAUqPEAKRGiQFIjRIDkBolBiA1SgxAapQYgNQoMQCpUWIAUqPEAKRWLDHbT23/Zvt1HwMBQBM1R2I/SjrseA4AaKVYYhHxs6Tfe5gFABrjMzEAqW2txGwf2V7Y7n9vkATW81mtVkOPM0pkVEZGm7ZWYhFxHBHziGi+wdYXYD2fyWQy9DijREZlZLSJ00kAqdVcYvGTpF8kfWv7zPb33Y8FAHWK21NHxKM+BgGANjidBJAaJQYgNUoMQGqUGIDUKDEAqVFiAFKjxACkRokBSI0SA5AaJQYgNUfE9t/UXkn6tcVL9yWdb3mcrtf9JiIabSdwjXykfBk1zkdKmVGvv0MSGf2hkxJry/ZiiK18hlq3DTIqG2LWTPlIu5URp5MAUqPEAKQ2thI7/sLWbYOMyoaYNVM+0g5lNKrPxACgqbEdiQFAI5QYgNQoMQCpUWIAUqPEAKRGiQFIrfiVbW3YHuS6jdls1vuap6enOj8/d5PX2A670Uu24t69e72v2SYfSdrf34/pdNrBRFdbLpe9rylJEUFGBf8vo05KbCiLxaL3Nefz5reC2dZXX33VwTRXy5KPJE2n00HmHeJ/Lm2R0QVOJwGkRokBSI0SA5AaJQYgNUoMQGqUGIDUKDEAqVFiAFKjxACkVlVitg9tv7V9YvtJ10MBQK1iidnek/SDpO8kHUh6ZPug68EAoEbNkdh9SScR8S4iPkp6JulBt2MBQJ2aErsp6f3a47PPz/0P20e2F7b7vyM1gfV8+HKWy61ntFqthh5nlMhoU02JXXbL+sZ/hRFxHBHzTN+C3Kf1fMa2C8BYrGc0mVz6jfVfPDLaVFNiZ5Jurz2+JelDN+MAQDM1JfZK0l3bd2zfkPRQ0vNuxwKAOsVNESPik+3Hkl5K2pP0NCLedD4ZAFSo2tk1Il5IetHxLADQGFfsA0iNEgOQGiUGIDVKDEBqlBiA1CgxAKlRYgBSo8QApEaJAUit6or9pmazmRaL/nfkybI7xL1798inYLlcDjLvENskzeftNn4howsciQFIjRIDkBolBiA1SgxAapQYgNQoMQCpUWIAUqPEAKRGiQFIjRIDkFqxxGw/tf2b7dd9DAQATdQcif0o6bDjOQCglWKJRcTPkn7vYRYAaIzPxACktrUSs31ke2F7sVqttvW2O4N8ytYzGnqWsSKjTVsrsYg4joh5RMwnk8m23nZnkE/ZekZDzzJWZLSJ00kAqdVcYvGTpF8kfWv7zPb33Y8FAHWK21NHxKM+BgGANjidBJAaJQYgNUoMQGqUGIDUKDEAqVFiAFKjxACkRokBSI0SA5AaJQYgNUfE9t/UXkn6tcVL9yWdb3mcrtf9JiIabUtxjXykfBk1zkdKmVGvv0MSGf2hkxJry/ZiiC1Ghlq3DTIqG2LWTPlIu5URp5MAUqPEAKQ2thI7/sLWbYOMyoaYNVM+0g5lNKrPxACgqbEdiQFAI6MpMduHtt/aPrH9pKc1U327ORldjXzKdjKjiBj8j6Q9Sf+W9DdJNyT9U9JBD+v+XdI9Sa+HzoCMyIeM2v0Zy5HYfUknEfEuIj5KeibpQdeLRq5vNyejq5FP2U5mNJYSuynp/drjs8/P4b/I6GrkU7aTGY2lxHzJc/yz6f8io6uRT9lOZjSWEjuTdHvt8S1JHwaaZazI6GrkU7aTGY2lxF5Jumv7ju0bkh5Kej7wTGNDRlcjn7KdzGgUJRYRnyQ9lvRS0r8k/SMi3nS9bqZvNyejq5FP2a5mxBX7AFIbxZEYALRFiQFIjRIDkBolBiA1SgxAapQYgNQoMQCpUWIAUvsPpsszh48J2RoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 16 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# sample and generate images\n",
    "\n",
    "images = []\n",
    "for i in range(16):\n",
    "    z = torch.randn(1, LATENT_DIM).to(device)\n",
    "    reconstructed_img = model.dec(z)\n",
    "    img = reconstructed_img.view(m,n).data\n",
    "    images.append(img)\n",
    "    \n",
    "fig, axes = plt.subplots(figsize = (5,5), nrows=4, ncols=4, sharey=True, sharex=True)\n",
    "for ax, img in zip(axes.flatten(), images):\n",
    "    im = ax.imshow(img, cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}