MLP_Classification_MNIST-Gabarito.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "MLP_Classification_MNIST.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Il7H73VGJ2No"
      },
      "source": [
        "# Classificação do MNIST\n",
        "\n",
        "Vamos dar início à construção de um modelo capaz de classificar dígitos escritos a mão com mais de 90% de acurácia!!\n",
        "\n",
        "Relembrando o dataset, a imagem a seguir mostra alguns elementos pertencentes às dez classes do problema (dígitos de 0 a 9) <br>\n",
        "![](https://upload.wikimedia.org/wikipedia/commons/thumb/2/27/MnistExamples.png/440px-MnistExamples.png)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5LaoPWO9RhEq",
        "outputId": "a834c297-62e8-4940-b17e-e4a73f685981",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import torch\n",
        "import torchvision\n",
        "from torch import nn\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "from sklearn import metrics\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "import time\n",
        "\n",
        "sns.set_style('darkgrid')\n",
        "\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "print(device)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5yNikfN0J2Np"
      },
      "source": [
        "## Carregamento de dados.\n",
        "\n",
        "Para focar na implementação e treinamento da rede neural vamos utilizar o carregamento automático de datasets do **```torchvision```**. Vale muito conhecer o carregamento de dados do PyTorch, é bastante eficiente!\n",
        "\n",
        "Me cobrem de disponibilizar depois um tutorial de como carregar seus próprios dados ;)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GWHd8oW15X2F"
      },
      "source": [
        "train_set = torchvision.datasets.MNIST('.', train=True, \n",
        "                                       download=True,\n",
        "                                       transform=torchvision.transforms.ToTensor())\n",
        "\n",
        "test_set = torchvision.datasets.MNIST('.', train=True, \n",
        "                                      download=False,\n",
        "                                      transform=torchvision.transforms.ToTensor())\n",
        "\n",
        "train_loader = DataLoader(train_set,\n",
        "                          batch_size=32,\n",
        "                          shuffle=True)\n",
        "\n",
        "test_loader  = DataLoader(test_set,\n",
        "                          batch_size=32,\n",
        "                          shuffle=False)\n"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lVFJWjBR6HC_",
        "outputId": "1cd79043-91ad-4c3f-96fc-efd76429bb95",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "fig, axs = plt.subplots(1,10, figsize=(14, 3))\n",
        "\n",
        "k = -1\n",
        "for data, label in test_loader:\n",
        "  k += 1\n",
        "  if k > 9: break\n",
        "\n",
        "  print(data.size(), label.size())\n",
        "  axs[k].imshow( data[0][0], cmap='gray' )\n",
        "  axs[k].set(title = str(label[0].item()), xticks=[], yticks=[] )\n",
        "\n",
        "plt.show()   \n",
        "  "
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n",
            "torch.Size([32, 1, 28, 28]) torch.Size([32])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxsAAABgCAYAAAB1029WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9aYzc6Z3f96n7vu/qqr67q5s3OTxmOJKG0oyklVbeXURr7Bprx3qxgOGFjSSAg2wQB0HsBImNIAkSe18YNgwHdjaABK12drSrW8O5OBQ5HJ7TrL7ZR3V33UfXfeVF63nUzSE5JIdkd3X/P0BDMzxqnvrr+T/P7/z+VN1uFwUFBQUFBQUFBQUFhWeNercXoKCgoKCgoKCgoKCwP1GcDQUFBQUFBQUFBQWF54LibCgoKCgoKCgoKCgoPBcUZ0NBQUFBQUFBQUFB4bmgOBsKCgoKCgoKCgoKCs8FxdlQUFBQUFBQUFBQUHguKM6GgoKCgoKCgoKCgsJzQbvbC3gYsVjsbeBloPXrX1qNx+Ox3VtR7xGLxf4Q+B+AfmAd+E48Hn93d1fVO8RisTHgFvC9eDz+d3d7Pb1ALBbbvO+XTMCfxePxf7wb6+k1YrHYPwK+AxwF/jwej39nVxfUY8RisUHgz4BXgDrwPeC/jMfjrUf9PQWIxWJu4N8BXwPSwH8bj8f/391dVW+gvLefH8VeeXJ66b7ds87Gr/lH8Xj83+72InqRWCz2VeBfAH8A/AoI7e6KepJ/DVzZ7UX0EvF43Cr+ORaLWdm6NL67eyvqORLA/wR8na2LQ+HJ+DMgydZ55wR+CvwJ8H/t5qJ6hH8NNIAAcAL4YSwWuxGPx+/s7rJ6AuW9/Rwo9srT0Uv37V53NhSenv8R+GfxePzDX//76m4uptf4dZQlD3wAjO7ycnqVb7Nl+CnRqcckHo9/HyAWi50GIru8nF5kCPhX8Xi8BqzHYrEfAYd3eU17nlgsZmHrfT0Sj8c3gfdisdibwN8D/nRXF9cDKO/t50axVz4/e/q+3evOxv8Si8X+VyAO/HfxePztXV5PTxCLxTTAaeDNWCw2CxiBHwD/dTwer+7q4nqAWCxmB/4Z8BXgj3d5Ob3M3wf+n3g83t3thSgcGP5P4A9/XYbrAr4B/Pe7uqLeYBxoxePx6W2/dgN4bZfWo3BAUOyVZ8aevm/3coP4fwMMA33AvwH+KhaLjezuknqGAKADfh/4Ilsp8ZPAP93NRfUQ/xz4d/F4fGW3F9KrxGKxAbYMlf+w22tROFC8w1YmowisAFfZMlwUHo2VrWe2nQJg24W1KBwsFHvlc9IL9+2edTbi8fjleDxeisfj9Xg8/h+A94Fv7va6egQRDfi/4/H4WjweTwP/O8rz+0xisdgJ4A3g/9jttfQ4fw94Lx6PL+z2QhQOBrFYTA38CPg+YAG8bGU3/sVurqtH2ATs9/2aHSjtwloUDhaKvfL52fP37Z51Nh5AF1Dt9iJ6gXg8nmMrqrc9nbYnU2t7kAvAILAUi8XWgX8CfDsWi13bzUX1IP85ezjKorAvcbOlZPOvfh2kygD/HsVoeRymAe2vFfgExwGlOVzhuaLYK8+EPX/f7smejVgs5gTOARfZkr79A+BLwH+xm+vqMf498I9/3SDZBP4r4K3dXVJP8G+A/2/bv/8TtpyPf7grq+lBYrHYebbKH/ekKsZeJhaLadk6lzWAJhaLGdmqpVekWz+DeDyejsViC8A/jMVi/xtbpUF/H7i5uyvb+8Tj8XIsFvs+8M9isdgfs1XK8rvA+d1dWW+gvLefG8VeeUp65b7dq5kNHVsycim29L7/MfB79zWvKTyaf86WbOs0MAV8DPzPu7qiHiAej1fi8fi6+GGrvKAWj8dTu722HuLvA9+Px+NKCcaT80/ZKiv4U+Dv/vqfldrlx+c/A36Lrbtjlt8YLgqfzZ+wJduaBP4c+IeK7O1jo7y3nw/FXnl6euK+VXW7SrZKQUFBQUFBQUFBQeHZs1czGwoKCgoKCgoKCgoKPY7ibCgoKCgoKCgoKCgoPBcUZ0NBQUFBQUFBQUFB4bmgOBsKCgoKCgoKCgoKCs8FxdlQUFBQUFBQUFBQUHguPHLORiqV6t67d+9FrWXPMjAwgM/ne+KBgsrz2+Jpnp/y7LZQ9t7nQ9l7T4/y7D4fp0+fTgO+J/k7yvPbQtl7nw9l7z09yt57eh757Lrd7kN/rly50mVrkuOB/rly5Ur3Uc9JeX7P/vkpz07Ze7v1/JRnpzy7Z/HT7XavKs/v6X6Uvff5fpS99/Q/yt57Ps9OKaNSUFBQUFBQUFBQUHguPLKMSmH/YDabCYfDGI1GAoEAdruddrtNo9Fgc3OTqakp8vk8nU6HrjLoUUFBQUFBQUFB4RmgOBsHBJ/Px+uvv04oFOLChQvEYjEqlQqFQoH5+Xn+5b/8l9y+fZtms0mz2dzt5SooKCgoKCgoKOwDFGdjn2MwGDAYDLhcLoLBIKFQCI/Hg9PpRKPR0Gw2MZvN6PV6dDod7XZ7t5esoKCgoLBLaLVaNBoNer0em82GWq2m0WjQarXodDq0223a7Ta1Wo1Op7Pby1U4gGi1WlQqFRaLBbPZTLfbpd1u0+l0KJVK1Ov13V6iwn0ozsY+RaVSoVarmZiY4NChQwwODvLGG2/gdDpptVrcuXOHbDbL6uoqyWQSjUaD1+slm80qL6qCgoLCAcXn8xEIBBgZGeH3fu/3sNlszM7OsrGxQbFYJJ1Ok8/n+fjjj8lms7u9XIUDhk6nw+v1YrFY+MpXvsIXvvAFms0mhUKBQqHAX/zFX3Dz5s3dXqbCfSjOxj5FOBter5fx8XEGBwcZGRnBYrEwMzNDMpkkmUyyvLxMLpdDpVJhMpnQapUtoaCwl1CpPq0kqPRVPZztz0t5Tk+GSqXCbDbj8/kYGRnh9ddfx+Px8NFHH7G4uEg6nWZlZYVkMsnU1NRuL7enuf+9Vvbq46FWq7FYLDidTg4fPswbb7xBrVYjmUySSqV45513dnuJCg9AsSz3IXq9nnA4jNVq5cSJE7zyyitotVpmZ2dpNBpcvnyZ+fl5yuUyuVyOarXK6uoq5XKZSqWy28tXUDjwOJ1OnE4nfr+fU6dOYbVa5e/Nzc3x8ccfU6vVyGazNBqNXVzp3kCj0WC32zEYDExOTnL48GGSySQffvghxWKRSqWiPKfHxGq1EggEcLlcqNVq1Go1oVAIo9FIuVxmaGiIVCpFqVTi3r17rK6usra2ttvL3vNotVr0ej1ms5mRkRHsdjterxen08n6+jpXrlyhVCpRqVSUvslHYDabOXHiBJFIhOHhYcxmM/V6nVwuRyaTUSoz9iiKs7EP0ev1DA0NEQ6Hefnll7lw4QKrq6v87Gc/I5FI8MMf/pAbN25I/WNA1t4q0RUFhd1FpVLh9XoZHh7m2LFj/Mmf/AnBYFD+/o9+9CMqlQrpdJpyuawY0WwZcqIX7Vvf+hZ/5+/8Ha5fv04qlWJ5eZmNjQ3lOT0GKpUKm81GX18fXq8XjUaDWq0mGo0SiUTknZFMJimXy8zPz/P++++zvr6u3B2fgU6nw2q14vP5eO2114hGoxw6dIixsTGuXLlCKpViZWWFdrutOBuPwGq1cvbsWSYnJ4nFYlitVvL5POl0mmQySa1W2+0lKjyAPelsiBKgB6FWq7Hb7RiNRlQqlfyzOp0OgHw+T6FQwGQy4Xa70Wq1WK1WDAYD3W6XTqdDq9Uin8/TaDQoFotsbm6+yK/33BCRE6fTSX9/P9FoFJvNRqPRoFQqsbKywsrKCvl8XjnM7sNgMGCz2dDr9bjdbgwGA6VSic3NTer1Ovl8/qma58Ue1Wg06HQ6ut2ubLRUZIYVHoZWq5XNj0ajEYPBIH/ParXidrtptVry3DvoaLVavF4vgUAAj8eD2WzGYDA89B5ReDg6nU7uO/H8xDkmMJlMBAIBGo0GHo8Hi8VCq9WiXq8rZ9qvEVkhk8mEXq/H5/PR39+Px+NhcHCQYDCI2+3GYrFgsVgwGo0YjUY0Gs1uL31PYjabcTgchEIhvF4vHo8HnU4n7Zv19XXW1taU6ow9yp5zNsQLajAYHvjSGQwGXnrpJQYHB6UBp9fr5ca7ePEily5dYnBwkK9+9av4fD4OHz5MMBiUKhqZTIb33nuPtbU1Ll++vG+aiaxWK8FgkGg0yje/+U0mJydRqVQsLy9z584dfvzjH0tnQ2EngUCA48eP4/f7+epXv0o4HObGjRvcvHmTlZUV3nvvPUql0hN/rl6vR6PRYLVacTqdtNttCoUCjUaDWq2mRFsVHojdbicYDOLxeD51Dno8Hk6cOMHKygrT09NKky5bhsi5c+eIxWJMTExgNBrR6XTSqVd4fBwOB+Fw+IF7T2CxWDhz5gwTExNks1mWl5cplUokEgklkMVWWZ/BYECv1zMyMoLf7+fcuXP8zu/8DhaLBbvdjl6vR6/XYzQacTgceL1eqtUqmUxmt5e/J4lGo7zyyitEIhFOnDhBf38/rVaLTCbDwsICP//5z0kkEqyvr+/2UhUewAtzNu5vhtqevVCr1fLf1Wo1Go0Gi8XywKidXq8nGAzS19eHTqeTzkYgEECr1TI1NSUjfwMDAwSDQSYmJohEIrRaLRqNBslkkpmZGRqNBiaT6YV8/xeBTqfD4XDgdDoJBAKEw2Gy2Sy5XI5CoUAymSSdTtNqtXZ7qXsGrVaLWq3GZrPJZzY+Pk5/fz+lUol0Ok2tVnuqxnnhNBsMBux2O263m2azSaPRQKVSKZeywgNRqVRSdtRkMqFSqXZEi3U6HTabDYvFokRB2XpeOp0Oj8dDMBjEYrHILLaQalWi7Y+PSqVCq9VKedEHodVqcblcGI1GvF4vbrcb2DKyW63WgX3ewpYRvRkmkwmv10swGKS/v59YLPYpm6Pb7e4InIo7aXuZ80FHyNyGQiE5lNhisZDP5ymXyxQKBVKpFOl0WunZ2KO8EGdDqAfodDrpVBiNRnw+nzys7Ha7LAMymUwMDw9jt9s/9VkajYZQKITD4ZDOCSAvl3PnzmG1Wunv7+f06dM4HA4sFgv1ep1SqUQul2NtbY2FhQWWlpb2VZTf5/Nx5swZ+vr68Pl8GAwGMpkMt2/fZm5ujmq1eqAvgvvRarUcOXKEgYEBxsbGOH/+PC6Xi1AohNlspq+vjyNHjtDtdtHr9Y/9uaJsymw286UvfYnR0VFCoRCDg4Nks1l++ctfkkgkWFxcZGVl5Tl+Q4VeRKVSEY1GOXfuHH6/H6PRuNtL2rPodDqMRiMul4uhoSHGxsbodDrcvXuXubk5NjY2FDnvJySbzTI3Nwfw0ICISqWSJT8vv/wyDoeDqakpvvvd75LNZqlWqwcymCJkg30+HydPnsThcMjMRjgc/lQAtV6vyyy3Wq2W5VYGg4HNzU2KxaKsyDiICJVMnU5HNBrl5MmT+Hw+rFYrnU6H6elpbt68STweZ2Njg0KhcCD3XS/wQpwNcTCZTCaZubDZbIyMjGC1WhkeHiYQCEhHw2az8dJLL+H1eh/r81utFrlcjlqtxuTkJC6XC7/fz9jYGCaTiUajQaPRYHNzk1QqRTKZZG1tjfX1dcrl8nP+9i8Op9NJLBYjFArhdDoxGAwUi0Xm5+dJJBLU63WlpGAbWq2WoaEhTp06RSwW48yZM1gsFqxWq2w4bTabpFKpJ8psCGfDZDJx9OhRzp8/z8DAAJOTkzLNq9PpSKfTz/HbKfQyPp+PyclJrFbrEzm6Bw2tVovZbMZmsxEMBolEIty7d4+lpSUSiQT5fP6pyh8PKt1ul1KpxOrqKg6H46FZcJFN0ul0TE5OEolEsNls/PznP6dardJoNA6c0adSqXA6nQwMDDA0NMTXvvY1fD4f0WhUZn7up9VqUavVaDabMqMkelI1Go3sPzjI/X0iS+Tz+RgbG8PhcGAymeh0OqyurnL16lVWVlbI5XL7yp7bbzxXZ0M0mlmtVl566SVZ6qTT6TCZTASDQYxGI36/H4fDsSOzsb0h8rOo1WrMzMxIRYJcLkcul6Ner6PT6eThVygUSKfTpNNplpeXSSaTPd9MpFKpsFqtGI1GGT13Op0UCgWq1Spzc3PcvXtXUWN5AGq1Gp/Px+DgIH6/H7PZLKeot9ttlpeXuXHjBnfv3n2qyKhwsi0WC3q9/qElCfsN0UxqNptlo67D4UCv18u9ajAYZIkQbBk54r3N5/MsLS0duP0qgjAmk0nKuB6kffO4CClWl8uF2+0mGo0SCoXw+XxotVpyuRwzMzMsLy9/psGrUqmw2+1YrVZMJhN+v59Op8PS0hLZbHbfKwMZDAYpoOL3+7FYLBw5coRDhw4RCoWkvHKz2aTVamGxWHA4HDv2pLjnhaGt0+mYm5vb96pA26s0otEodrudiYkJxsbGdpT7AJTLZTKZDMvLyxiNRsbHx7HZbGi1Wvnsz549S7FYpFarUa/XmZ6ept1uU61W5f8HBw2dTsfo6CgDAwNMTExIO3F1dZVarcbc3ByLi4tkMpmeKw8XpWFGo5FAIMD4+DharZbNzU1arRbVapVKpUK73aZWq9FutykWi1Sr1Qd+nslkwmw2S3GRB5XYdjodms0m7XZblte/qDPuuTobZrOZUChEJBLhO9/5DidOnJCOhoj+bu/VEEaK8PAfh263S7FY5IMPPmB+fp56vU69XketVqPVaul2uxQKBWq1GrVajXK5TLVaZXl5mWq12vPpSbVaTSAQwO/3c/jwYU6fPo1arebOnTukUineffddfvrTn8peAYXfoNFoGBoa4uzZs9hsNnkxCOf0o48+4nvf+x6ZTOaJFcvE/rZarbhcLiwWy/P4CnsSjUaDRqORpQMej4fJyUnsdjvDw8P4fD7cbjehUEgeiO12mxs3bnDnzh2mpqb43ve+d+D2q06no6+vD5fLRSAQkEag4mzsRKfTcezYMY4ePcrw8DBnzpzBarUSDocxGAysrKxw8eJFksnkQy9m+I2xGAqFGBoaIhKJcPbsWbrdLt/73ve4ceMGlUplX5ee2mw2BgYGcLvdfPGLXyQcDjMyMsLIyAitVotCoUAmk5HBq0gkIjO/AuEU9/X1cfbsWVZXV2Xgbz+j0WikOM3XvvY1hoeHmZycZHJyUvbpqdVqisUiuVyOjz76iLfeeguv18sf//EfY7PZ5LMbHx9nYGBAZjC63S4/+tGP2NzcJJ1OH9jZGwaDgddee40LFy4QiUQIhUKUSiU+/PBDVldX+eCDD7h8+XJPBgWExHkgEOBLX/oS/+Af/ANMJhPLy8sUi0XW1tZIJBJUq1VSqZQMqj9opo1KpSIYDBIKhbDb7USj0QdmxMU7XavVuHXrlgyklkql51718sLKqMxms4zWmc3mx/677XZbeqziwNfpdGg0Gtmn0Ww2yWazJJNJms3mjk3X6XQolUpS+UdEDer1es9tzgexvRlNyBUCNBoNKpWK9I6V8qkHIyJLop9I7Bcx8DCTyVAqlZ7IKb3faRaKVLC1h7fv6V5ke6DgfmlfIfWo0+kIh8NEo1E8Hg99fX2y1MXj8eB2u/H5fLLnqtPpEAgEyGazbGxsHMjGZ/HsRPZHq9XK57yder0uDcBe3kdPiihNNJvN0pH1+/2y7KTT6VCv16lUKmxublKtVh967mk0GhwOBwaDgVAoRH9/Pz6fD5vNRrPZlPfLfkIIVmzvcxSZXJHh8Xg8mEwmGVHf2NigVqtRKBSoVCoYjUYqlYo8M7cHCUXm0maz7VtJZpVKJb+7yEB6vV7C4TChUAiPxyMHcIoeyY2NDUqlEqlUikqlQq1W27Evt5ekwdbd3W63ZfZXyPwfJDQaDUajUe5Jn8+HxWKh2WxSqVRIpVKsra3JCpZefFdFZkNkaL1eLyaTiXq9vkNEoFarYTAYZMD8Ye9WMBgkGAxis9kIh8MPdTasVqucuJ5MJtnc3KRcLve2s9FoNMjn81itVmnAPanuuTD4hIa3Wq2W0T8xM6NYLHL79m2uXbv2qdrG++catNtt6aDsB1QqFS6XSz4TrVZLs9lkc3OTfD6/71PZz5pyucyHH37I0tISV69eZWlpSZYQPC7C+BbpTOFkd7tdms0mhUJBNlH2Inq9Hq/XK9VT1Go1brebvr4+LBYLAwMD2Gw2BgcH6e/vl0aIKLfQ6/XywNz+rg4MDODxeGi1Wk9URrlfELMi+vr6cLvdmEymBzob8/Pz/OAHPyCTyez76PF2nE6nbBD9rd/6Lc6cOUOn05FBpHw+T6vVIpFIkMvl2NzcfOgF6nK5+Na3vkV/fz+Tk5OMjY1RKpVYXl4mm81SLBYpl8s0m82eNGQehM1mY3JyUjoT7XabfD7P8vKyNEL6+vrY2Njgk08+IZlMcu3aNXl31+t1XnnlFbRaLU6nk8HBwR2T7Q0GA+FwGNiSxt1vikrCKRgfH6evr4/BwUFOnTqF0+lkYmICp9OJWq2mXq+TTqe5c+cO+XyeW7dusbKygtFoxGaz4XK5Hmowtlot2X+Qz+dxOp3U6/WnUkPsZZxOJ8eOHSMQCHD06FHGx8cpFotMT0+zsrLCT37yE2ZmZkgmkz27vzQaDYODg7z00kuMjIzIigCR+Q8Gg8RiMdrtNo1Gg06nQ7lcfqhNJ8qTtVqt7I++H2Ezt1ototEog4ODzM/P89Of/pRisfhcv+9z3cGi1qxarcqMgslk+tTmeJjX3u12qVar5HI56dFqNBpcLhc2m006EPV6XXq6Bw3RF2Cz2WRWo9vt0mg0DqwiyOOyPTIvaDQarK6uMj8/z/r6OqVS6akOM+Fs6PV6eQCIrIZ4J3opKi2e0XZZ1u2Hm9/vZ2hoCJvNxqFDh3C5XAwODjIwMPDQz7rfELHb7dhsNtxu94HObIi+DSGBeT+FQoGZmZnnfjnsJcQ5Fw6HCYfDDA0NMTw8TDabZXV1dUfWWtS9P8xRUKvVmM1mRkZG5BTi8fFxlpeXWVhYkIa1uOD3C6I3wG63y+h5vV5nc3OTzc1NaaRUKhWWlpZYXV3l5s2bsk680Wjg9/vZ2Nig0+kQiUR2fL6QZLbb7TJTDPSsMbid7dlbj8dDf38/ExMTsgTX5/NhNpspFouUSiVKpRILCwskk0muX7/O/Pw8w8PDHDt2bMewxO2I+6FYLMosiJjDcdCGU5pMJvr6+giHwwQCAZxOJ5ubmzLzvbi4yNzcXE/dofcjsonBYFCqq4oqINgKDjwtD3rntts5nU6HfD4vM8Avwpl9Ic5GPp/n2rVrVCoVQqEQfX19MiMBMDExISMi4u+JA+7DDz/kgw8+kJeJSqVicHAQr9eL1+slEomwtLTU843eT4tarcbv9zMyMoLb7ZbTrqenp6Uc3H447J8l2+eRiLpZgFKpRCaTYXZ2ltu3b5NMJp/q8+12O+Pj44RCIfx+PyaTSTZarqyssLy8LIdg7XX0ej1DQ0M4HI4d6exIJCJniGi1WqxWK16vF4PBgM/nw2Qy4XA4gK1oXblcptVqUalUaDQasvxgu0EidNIfp7FX4eAQjUYZGBggGo1y4cIFfD4fLpeLzc1Npqen+dnPfka5XKZcLtNoNPjkk0+k3O395Y+jo6OcPn2aUCjEmTNnCIfDdDodpqammJmZ4Re/+IVUK9wvAwHFzAy32y2jxSLDeOvWLRqNBmq1mp///Od8/PHHbGxskEqlZL9GrVaj1WrRbreZmZnhzTffZGBggL6+PpxOp/zvmM1mBgcHsVgsRKNR6bgVCoWevYPMZjMWi0XKyrvdbtkA7vf78fv9qFQqUqkUzWaT6elpZmZmSKVSXL9+nWKxyMrKCpubm+h0OgYHBwmFQp/K3BaLRe7du0c+n+e9995jdnaWbDbL2toam5ubB8a+sVgsmM1mhoeHuXDhAn19fYRCIQAymQw3btxgZWVFNjb38vvZ6XRIJBLcunVLTkK/v/Tp/kD8o94j4ejW6/VPldiKocJ6vR6n07krwbzn7myIVO2VK1dYXl5mZGSE8fFxKpUKy8vLUk3pfmcjm81SKBR4//33+Y//8T/SbDZlGZWYs3Ho0CG++MUvkk6ne7Yk5fOi0WikkoHb7ZbKFXfv3uX69euk0+mePeifF6IMSMx3MRqNO9TKpqenuX37NuVy+amend1u59ixY4TDYTlkTFw6S0tLLC4usrS01BP/vxiNRiYmJqQayKFDh+T7uj2z8aAskfjnRqMha2tTqRSbm5tEIhFZ9gdbh+jGxgZ3795lcXFRcTYUgK09NDAwwGuvvcbAwABvvPEGLpeLWq3G5uYmd+/e5fvf/75UVhGSqw9r6h4bG+OP/uiP8Pv9jI+PY7FYuH37NlNTU9y8eZOf/OQn8szshffzcRAZC6/Xy8mTJxkcHJSleoFAgHv37pFMJvnxj39MNpvd8d3vN+ZmZ2dZWlpiYmKC3/7t32ZkZET+nsViYXh4GKfTSX9/P6FQiFQqRbFY7NlnabFYCAQCHDp0iO985ztSxtZms0mBgXK5zMLCAqlUiosXL/Luu+9SKBRYWlqS/QRiVtPQ0NADZ+cUCgWuX7/O2toab775Jjdv3pS9fftpL34WFotFBk9ff/11IpGILMdLpVJcu3aNZDJJsVjs6awGIBUvm80mbrf7gZnUx3U2ut0u+Xye1dXVHXtPEAgEGBoawmq1ymGTL5oXUgjYarXk8DzRVFar1VhbW0Oj0bC2tsbGxoYsIRCzDZLJpDRSRJ2ZSqWSWsrr6+tS8vYgOhuiVEc8N4BUKiWjUgdVweJhCNUzkXkIBoN4vV40Gg31ep2NjQ02NjbY3NyUZQZP+vkajQaLxSKzAOJSqVarZDIZWRLYK5eHVquVaj3hcFj2EphMJtl7odFoaDabUp5vc3NTlj2Khsh0Oi0DBs1mE5vN9qlnkMvlWFxcZGNjo+cvEoXPhxgEq9frpaS3kE4Xl3QymWRhYYFisUilUpH3xP0Tw9VqNV6vF6vVSjQalUGGbrdLvV5nbW2Nu3fvygu6l6OlAlF2ptVq6evrY2BggNHRUdkEbzAYZBOu0+mUVQOf9d6JfsdKpcL6+jrLy8s4HA6p5Cf6Gvx+P8PDw3Q6nZ4bXKrVamVZVCQSob+/n6GhIdxuN2azWVZeiH9hfTMAACAASURBVFLlYrHI1NQUyWSS5eVlOdVa9IoKDAYDLpcLh8Oxoxm82WySyWRYWFggkUjIwXT7JbP2OIi72e12MzQ0RF9fn9yj4r0ulUoyer9f7odqtSql3j/88MMdPVCih2M7jxrwKGy/crnM2traDjXHVquF1+tFrVbvmgLrC3E2Go0Gc3NzaDQaZmZmsFqtsrRCTMxstVr09/dz4sQJKpUKV65cIR6PMz09LS8A4eGLKFYmk+GTTz6h3W5TKBRexFfZMwhlFqvVSjAYZGBggJWVFa5cuUIikWBhYYH19fWel/Z9lohpwyMjI3znO99hcHCQSCSC2WxmYWGBS5cusbKywvr6+iObSx+GGFwZDAY5deqUNM4B0um0TAH3UkrcbDZz/vx5vvCFL8gUtyjL2J7JKBaLsjQsHo+Ty+WYnZ1ldnaWZrMpxSGEkIFareall16Sl64oZXnrrbd6unle4dkgosAej4cvfvGLfOtb35J9T/l8nrfeeouLFy+ysbHB2tqaNM4eFAU2GAy8/PLLHD58mBMnThCLxdBqtbJs8t133+U//af/JFWs9gNarZZAIIDD4eB3f/d3+fa3vy1LHbcrHzkcDsbHxzGbzdy4ceMzP1c840KhwLvvvsva2hpnz57l5MmT8iwwGAycPXuWcDjMT37yE+7cudMz95BKpcJms3HhwgVGRkY4cuQIx48fx2w2y14ykQlKJpPMz8+TyWR45513SCQSlMtleXcIg1ick263m1gshtPplL2rwo65desWP/jBD1hbW9sXUfsnQfQB6nQ6jh8/zje+8Q3C4bCcEp7L5eT9srCwQKFQeKq5V3uNTqcj58Ktr69z6dKlHb05wp4QGbROpyODdw9CBOSFDPD2c/DcuXM4HA6Z4dgNXoizIRq9YcvxEDJbtVoNo9Eoh3l5vV4pZ1utVmWj3v0KU+JFbDQaPVH3/jwQw4RElFlM1CwUClJH+aDNKfgshCqSUO8SmvFqtZpGo0Emk5E1yk9zOQoZXTH4SjRKArKXplgs9szFC7+RyxQDD8WhJ1K+4ieXy8n09urqqozUzczMSCU5kYXT6/U75B/FxSw05Z9Uang/IQzBg9YQej+iidvhcOByufB6vVKWut1uUyqV5PyB7Y304v7Y7niIWQjRaFT2HHU6HanYJxyW/RRJFveD1WolEAgwOjoqs5DbEYbMk9RwC4VH8c7fHzxRq9U4HA7q9bosN+oFxNwMEcATvUKDg4NyfzUaDYrFouzrWVlZIZ1Os7KyIvfQ/c6uTqeTZ5/FYpEqc91uV8oKC+n+VCq1b/bg4yKyYUajEZfLJSWENRqNVGASDoeQ898vz0iMahA9ndsRdp0QJ+h2u3KMw5NSLBYfGox5UbxwPbXtNYjiRzR/C2/MZDJx6NAh7HY72WyW27dvH1jj42E4nU7Onj1LMBiUtXgajWbfpRmfBeIyPXToEGfPnmVkZIRgMIjVakWlUskyH9FQ/zSZBxG5ikQiDA4OEo1GCQQCsjYyl8tx9+5dUqlUT0XtS6USf/M3f8P09DROpxOPxyONh1arxdzcnMwEZTIZms0muVxOCkPk83kpOW00GvF4PIyMjOD3+9FoNLRaLbLZLOVymXQ6LYeH7ZfL5EkwGAwMDw9z6NAhAoFAzxhpzwMRme/v75dNyKJEUaPRcOHCBYLBoDRExH4RvT8bGxsyo2axWDhz5gyvvPKKNGLS6TTf//73mZ6elpLp+2nPCYW4UCgkG0If5MCura3x9ttvs7GxIUudH4dms0k6nUav10tDRqjpwG+cmF7Yw2Kd0WiUl156Cb/fz1e+8hUpuqLVaimXyywtLZHP5/nhD38oBW9EYE/swQc5GkePHiUSiXD48GEpFS6Up6anp/nggw+YmZmhUqn0THnts8RgMMgsplCHEwIGtVqN999/n6tXrxKPxx8q/LAf2d7DId6jpy2L93g8HD9+HI/Hs2sDhl+4s3H/oS5eukajIQ1krVZLNBrFbDbzq1/9SqYhD+KL+DDMZjNjY2NEo1HZcKZWq5Uhfg9A1INGo1FefvllgsEgLpdLqkSJaFUikSCdTj9VilYIHQSDQXw+Hx6PZ4dSS6lUYnV1lVwu11MZp2q1yrVr11hcXJQTSgXNZpNLly4xPT0tozOPQqvVysF+QupPRKtF1me/zTZ4EnQ6HcFgkP7+fhwOR08Yas8LER0XtfPCeBWCBIcPHyYYDFKpVMjn83K/dLtdZmZmmJmZkQ6vyWRidHRUlk+p1Wo2Nzf58MMPuXr1qjSW9xOiN83tdsuZFw/aT7lcjtu3b5PNZp+ohEz0LZhMJhkceJBARC/sYRE59nq9nDp1ilAoxLFjx4hGo/LPNJtNEokEGxsbXL58mbfffvuxPlvYMkePHiUajcryU5GBSyQSsjG8V4fTfV5EYEEMge3r65OZtlarxd27d3nnnXfkHt1v7+rDeFR/xpNit9ulquRuDdzc9Ukx7XabpaUl9Ho9Wq2WiYkJmcrUarXSwBHzNg6CR/s4CM35SCQiPdVyuczq6iobGxv7oqbxWSCa/YQcY39/P263G51OR7PZlI2hN27ckMopT5oVUqvVspH60KFDDA4OotPpZFlbpVIhm83KJtZeOizb7bZ0kEQEb3tmQ2QzHvVeqtVq9Hq9VBoRSlQqlYpms0k2m5WiBk86QHE/IEp8RNlGIBCQjYK9YKw9D1qtFqlUCoPBQF9fHwsLC7KZWTQ2u91urFYrVqt1h7MhZiGIRmadTkcoFEKj0cjSqaWlpX0toiEMuKGhIfmubUfI2Qop6id18JvNJhsbGzQaDTY2NsjlchgMBlk2JQZ+GgwGOd19rwURhJPR39+P3+/nyJEjTE5O4nQ6qVQqrK6uyiBIMpnk6tWrpNNp1tfXP/OzdTqdnBs0NjbG0aNHpRHdaDTkDKeFhQXZVH7QbBsxh8rlcjE+Ps7w8LDM6NZqNSmokk6n5SiEvbR/eplut0s6nWZubo7V1dUXcgbuurPRarW4ffu2jERNTEzg9XoZGhqS0ftYLEY6naZcLvdUCcrzxGKxMDExIVO9APl8nqmpKfmsFLb6NAYGBgiFQhw/fpxTp07JGtpyucw777zDL37xC1ZWVrh3794TD/ISF5Zer2d0dJQvf/nLeDwejEYj7XabRCJBMplkZWVFRu57ydkQEb2HlUWIhrRHIeZwiEF/4kIX03ZXVlZk7XOvOWPPgmg0yu/8zu8QDoc5fvw40Wi0Z6LCz4tms8n8/Dy5XA6TySQbdCcmJrBYLNhsNjwezwNrkA8dOvQph1VkRHK5HNevX2dxcZH19fWengHxKAwGA2NjY5w8eZJwOPypgV5ikJ9QcnzS965WqzE3N4der+fll18mkUjgcDhkFkr0eZnNZuk4b25u7plAgji3jUYjx48f5/Tp08RiMS5cuIBKpWJ6eprFxUVmZmZkee3ly5dlmednYTKZZNXBK6+8woULF+QeFFPFE4kEV69elTK3B83ZEI5GKBTi/PnzHDt2DJ/Ph0qlkrLWyWSSpaUlUqnUQ+WsFZ6cbrfLysoKv/rVr8hkMi+k2mLXnQ0hPygiqIlEglarRSQSQaPR4HA4pAxaPp+nUqlIA0dEWw/SBhRGiIgWGwwGOTxRNE99nppGUXIkytt6HVGOIbTRRc1sq9Wi2WxSLBbJZDKyz+VJDV2NRoPNZsNiseB0OnE4HFgsFlQq1Y49nc/n5ef32n79vPtAp9Phdrtxu93Y7XasVisGg0E2movo1UGLXIl3zWq14vP55DBEobp0/7MQ++cgXLpCJESr1ZJKpVhaWqJUKkkBBqvVislkkufgdmdYTF0Wv3Y/9/cM7qdnqVarZbOtcMruHxTW7XY/JcbwNA6+cBzEZGdRFinU6oSAidFolFmmvcJ2gRWPxyN7W8T7JRrAV1dXWV9fl1nvxw0WiQCLuBvMZrO8V0VGeH19nWKxuOcyPs8L8T5aLBYMBgMOh4NgMEhfX5+UoxbDDtvtthQJ2i5GovBkiMCqOCPhNyIajUZDVlu8iP23684G/CY6Go/H+fM//3PZXGu1Wjly5Agul4tsNsvU1BSbm5tS9WZhYYE7d+7syzT4w9DpdPLCFTrq6XSaRCIh076fR81HRKNEH0Mvv+BqtRqTyUQsFuPQoUP09fVJLXnRvJxIJFhaWnrqjINoPg2FQpw6dYrh4WFZb7q5ucmlS5f48MMPWVhYkJG9Xn6mT4PX6+XLX/4y4XCYQ4cO4ff7pYxfOp3m6tWr3Llzh3v37h2ISxd+I7FpMpkYHh7m3LlzsizoQQawcFx7fSLz4yIMvmw2Sy6X4+OPP8ZgMOB2u9Hr9TJ44HQ6CQaD6HQ6rFYrer2ekZERRkZGZOne/XKSPp+PYrG4a7XLzxPRN9bX1yfr34UQBvwmuHfz5k1u3rzJ9evXpTLc0+ypbrfL4uIi7733HqOjowwMDOBwOPB4PDgcDqLRKMPDwzLbvlf61QwGA5FIBLfbzblz53jjjTeoVCpy+vd3v/td4vE4xWKRfD4vVTQf9141m81yNpHT6ZRlteLu+fDDD7l169aBOfNEUMBoNPLKK68Qi8UYHBzk5MmT2Gw2qQwpelrE7JFUKvVU6ksKW4FQ0Z8aDAZloLXZbMoZMZubmzuUIZ8ne8LZEJdrPp9ndnaWVqslPX6Xy4XFYpG14qVSCavVSjabpVQqySFP+y1C9TA0Go2UxxQKI9VqlVKpJEfVC835z+JBDX0i/a3RaKhWqzuiXr30jIUClTBMAoGA7G0RURNRSlAqlWg0Gk/13YTqSyQSkQOzut2uHGC3vr7O/Pw86XT6QDoasGXgRSIRotHojsb8Wq1GtVqVUcT9MuPgcRERaJvNht/vx+l0YjAYduzD7b0ItVpNSh/2ynv4tIjvC1tOeyKRkLOFRE+Gw+HA6/VSKBRkpFSv1+N0OmVg4f7nJCRIhfzodiN8PyB6BRwOh4yqb89siKhmJpNhcXGRTCbzqSGIT4KQ41xfX8fj8Uhj3GAwYDAYZHS/Uqk8kbzu80ZkHoQIQSgUIpFIMDc3x9raGjMzM0xNTdFoNJ6q/1Gr1eJwOHA6nTLLJrIa1WqVtbU1lpeXKRaLz+Hb7T22lxsHg0HGx8cZHx/n7NmzMpuxnU6nQ71el86pyAIrPD4qlQqLxSJtaJH5FXM4hCjTiypt3BPOhqDRaMho88WLF1ldXWV0dJSRkRFcLhcTExM0Gg2GhoaoVqsEAgE5GGd5eVnO5djPmQ6TySRLUkSN7Pbpo49TYiEyIoFAgMHBQbRarUy39ff309fXJ1NstVqNpaUlMpkMiUSC2dnZnjCYw+EwJ0+eJBAI8NJLLzEyMoLT6aTRaLC8vMxf//Vfk0gkmJqakk7V01y4Go1Gqr6IaeHNZpNCoUA+nyeVSrG+vn5gZQ1ha88ODg4yMDAgJ93X63VyuRzZbFb+HKR+rO3lkGI+y/ZU9/20Wi0WFxeZn5/n3r17+6LE8UkRqf/tcsliKJpGo5HP0GKxMDQ0RLfbxWaz7TByzWazFBwJBAJStnm/9LiJPgRRvmQ0GuX3b7fbVCoVisUis7OzfPzxx89EdEXsX51O1zN9RlqtFo/Hg9vtZmZmhrfeeouVlRVu3rxJNpslkUg8VTmyXq9Hr9fj9/ulApXoqVxfX2dqaorFxUVWV1f3zXC6x2H7/Bsx/8VgMOxw9oUz1mw2MRgMnDp1SmZxBwcHmZ+f59q1a3um72evo9PpGB8f5/Dhw4yNjaHT6ahUKly/fp1kMsnU1BTZbPaJ+1Sflj3nbIjL5J133iEej/O3/tbfYnJyEovFQjAY3DG8yeVyUa1W2djYoFaryejBfnc2vF6vrH8Xzsb09DTr6+uf6WyI7IXJZGJoaIjXXntNRvr0ej1HjhwhFovJF79SqfDOO+8wNzfHlStXWFhY6Bln42tf+xqhUIgTJ04QDoep1+vU63VWV1d58803mZubI5vNfq40rXA2RMRelGkVi0VyuRypVIqNjY1n+M16D7PZzMDAgJwHAzudDaE6ctAQ0TpRGvkoZ6PdbrO4uMjHH3/M8vJyT7yDzxoxkwmQ7+z9z0ur1TI6Osqrr74qh4Jtx2KxYLFYqNVq+P1+fD6fHBy2HxCZG7PZjNFo3BE1brfb0kGbn5/nxo0bnzsAolKpZH/Go/bvXkP0kTmdTmZmZrh79y4rKyvcuHFD9go8jVErnF2fz8eRI0d2zIlZW1vj/fffZ3V1lbW1NQqFwrP+WnsaUaonVMvEDCrhiGyfuWY0Gjlx4gStVguj0cjY2BgXL17k1q1birPxmGi1WsbHxzl//ryUXS4UCly7do3Z2Vmmp6fJ5XIvLAi6p5wNgWigEprp165dw263E4lEZJmPSJcPDw9js9nY2NjAarXK6L6Q2ttvbB+aBL+pwS2Xy480mnU6HTabDYPBwMDAAF6vl9HRUQYHB9FoNLI0Q9TRbz8Q3G43m5ub2O32PX2ZqFQqXC4XVquVaDRKOBzG7/dLJyCfz8v0tZC9fNqDSzSE9/X1EQ6HCYVCskyr0WiQTqdJp9MHut5URPlEKcf2CcalUomlpSVWV1cP9DPazme9WyL6dxAdjYex/aLUaDSyzNRgMKDRaGRJqVBdstlseL1e9Ho9gUCASCRCtVollUrt4rd4tohyiQfJ3a6urpJKpSiVSs/EyBBn7sDAAH6/H612y6QQQcNKpSLvpr20b4X87HbBGXFeP21JrUqlwuv10t/fz9DQEDabTVYRwFY54PLyMuvr6wfyzBOlzaJhWZQxillLoupgeXkZt9vNkSNH0Ov1cjaTx+PBbrejVqup1WoHMrv7OIhyU4fDgcvlwuv1yr4tISZUrVZfuDDBnnQ26vU6s7Ozsvn58uXLDA0N8c1vfhO/38/Q0BBer5eBgQF8Ph/ZbBa3200ikeDSpUvSO+71BufHQcxyWFtbe+T3tdlsHDp0CJ/Px9e//nWOHj2K3W4nEAhQLpe5du0a2WyW5eVlEokE4XCY06dP43A4GB8fx+v1sri4uKfqbu9Hq9UyOTnJ5OQkx48f5+WXX8Zut2M0Gul2u8TjcX75y19y7949VlZWdkweflJCoRDnzp0jGo3yhS98gaGhIRlFLBQK3L59m0QicSAj9gLRKBoMBqXjKhRZlpeX+clPfsL6+voTTS5WUHgQQl3IYDDITKNarZZ9bB999BGLi4scPXqUCxcuYLFYOH36NH6/n3q9zvz8/L4ocxSleQ+qb8/lcrzzzjusrKywvLz8uf9bIis3OTnJN77xDSwWCyaTiU6nQ7FYpFKpsLGxQSKRoFAo7JnmcIBisciVK1d2VEq0Wi2pzPO0zsbx48f57d/+bfr6+ohEIjsCdIlEgl/+8pcUCgVKpdIz/T57HbFXRPDJarXKsuNGo8HS0hLZbJa//Mu/5K/+6q84fvw4f/qnfyoDeZFIhLW1NYaGhsjlciwvL+8pdbO9hMlkIhKJ4Pf7pTiOGGYqhnGKuV8vkj3pbGxvDkylUnQ6HXQ6HRsbGzKSIiQixaUi+jdEo6Vo/t3vzgZsORwP0+kW0T6LxUIgEMDv9xMOh+nr60On00njr1ar7SglEOUuIhKxPSq9V1GpVNjtdnw+n5TSM5vNUia5UCjIuRePqscVF5BowN8uq6nVatFoNHLiaTAYxOl0ysZwETkQfQh76YJ9kYhyPYfDIVVGtks/lstlksmkHAp4EBFGjXh32+22jPYJY2cvZxL3EkJ5Tqj06fV6WR5VqVRIJpOsrq4SiUTodDpoNBqcTifValVmJPcD23s27j+vRXlnPp//3O+ccO6EwIEYlCrkb4XhXq/XqdVqe246drvdfuaCFCqVCrPZjN/vx+12YzAY5CyhVqtFqVSiWCweiCDo/ajVajngUfyvVquVfVjZbFa+o4uLi/j9fpntMJvNsvzRZDJRqVSUZvFHoNFosFqt8pmZzWaAHfbJ56nqeFr2pLOxHbHhqtUqtVoNm83G4cOHCYfDTExM8Morr2AymTh27Bijo6M4nU5GR0eZmZnhL//yL/d9XaRarcZsNkt54PuNE2EUj46O8u1vf1s6Y61Wi7t37/LRRx+Ry+W4desW+Xweq9WK2Wzm1KlTnD9/Hp1Ox9zcXE80pqrVaiKRCCdOnJA1io1Gg3v37pHL5bh8+TLvvfeeNEAehOjBMBgM+P1+/H4/BoNBqoqMj49LByMYDGI2m/F4PABSEWx2dpZ3332X1dVVksnki3wEewaVSsXg4CDnz59ndHQUh8Mhh6qVy2Xm5ua4c+eOnAx70BD1ydVqlWKxyNraGrVaDa/XKyN+e8k42+tYLBa+8IUvEIlEOHbsGHa7XdbIp1IpLl26xMzMDBaLhddff10O4QwEArz//vsPVK7qRcLhMF//+tcJhUL4fL7n9t+xWCycOnWKQCDA2NgYRqNRBmS63a7sDRFBl6ftgeg1hCy9aH6u1+tcunSJ2dlZLl++LMuV98NeexLEGINAIMDk5CSDg4O0221SqRTJZJI333yTeDzOzMyMLOdeWFig3W4zOTmJ3W5Hr9djt9up1+t7PvC5m1gsFkZHR+XsGJVKJe+adDrN9PQ0n3zyyQtXQtvzzoaQyCyXy2SzWYxGI/l8nkgkgk6n48yZM5jNZiKRCN1uF41Gg8vlQqvV8qMf/Wi3l/9C2N7Hcr+Urd1uJxqNMjY2xrlz52TZ2ebmJisrK7z99ttkMhlmZmYolUp4PB5Z59doNOSBIGQS9/IhqVarcblcRKNRvF4vGo2Ger1OMplkfX2dxcVFKa38qM8wm82YTCbC4bBsahalQK+++iqjo6OyVGF770ytVqNQKLCxsUE8Hmdtbe3AqI08CJ/Px8TEBOFwWEZaK5UK2WyWjY0N1tbW9k1j7tMgJAgrlQqFQgGtViubSYFPZTiULMfDMRgMjI6OMjk5SSQSkYPkZmdnWVlZ4c6dO8zPz3P69Gnq9boMJjidTux2+24v/5nhcrk4fPiwPK+eFwaDgaGhIQYHB2Wvxv19hGLIrNDyPwgIeVcxw6XVaklHY25u7kDLn/f398uyKCFZvba2RjKZ5Pr163z88cfSGRN9j0ajkeHhYTmnw2QyyYyRwoMR/WihUAiz2Sx7NSqVipxTl0gkXvi69ryzIdje8L22tka1WmViYkJKloqDzmazEQgECAaDhEIhYKs+c78edp1OR+rvi1S1KPfRarWMjY3x2muv4fF4SKVS5HI5bt++zerqKlNTU6yursphRTqdjtHRUY4ePcqRI0d21N+m02k2Nzf3tLPxILZH2RqNhpTI9Hg8O6QaHQ4Hfr8fo9FIOBzeUXZmNBpl74fT6XxoA2ahUJANmCJ1ftAulu0TYoPBIAMDA3g8HjkPJ5lMcu/ePTKZzIF7NtsRF2q326VYLLK+vk6n0yEQCOz4M9v/udfevReJKBUVDaTlcplcLse9e/dYXl6mVCp9anCdot3/dIj7xel04vF4ZJmGQNwZyWTymTWi9wLiTtgu4iLUMWu12mPJ0u9XxL3a398vyxbL5TKJRIJEIsHm5qYMbio8HaJkXlT39Pf343A4AMhkMnzyyScsLCzsWoCvp5wNUfs5MzODWq3myJEj0qATtc5i2FMul2N0dBSj0cjc3Ny+dTYAmfURzsB2edvTp0/zR3/0R+TzeW7dukU6neYv/uIvuH79uow6dbtdGTk4deoUf/tv/228Xi8Wi0Wm3lZWVsjn8z13WLbbbQqFglQa0el0OBwODh06tGNi6cjICKdPn8Zut8t5EEJNafslIvo47qfb7ZJKpYjH43Ii+UHMagj9ervdLh1XIetaq9W4d+8e169fZ2lp6cBfLGJoVTqdZmFhgVqtxvDwMC6Xq+fes91GXLJerxe1Wi2V527duiXfR6HzL56tUMdRHI7HR5yDYpZEJBLBZrPtCLyIbPi9e/fIZrMHYi8Lx3V7j59gu1rkQXgWD8JsNhOLxRgeHpZzR/L5PNPT01Ks5SCW0z5LxJDYYDDImTNnGBgYkFnb1dVVfvnLX5JIJHZNkGXPOxuiQVloeQtpQ3G5bC9lgd80S4sJzk8zmGcvI5rvhFxft9tFp9N9alLs9j8r6vVyuRyZTEY2qYnP0Ol0BAIBKTHndrvR6/XkcjlKpRKFQkFmh3rtsBSRdiFTOz4+jt1uZ2hoCIvFIp0Nod4gpspaLBZ5sYqJ491u91PPudPpUK1WpdGYSCTIZDIHoj75YWwXFRAlBeIdFftxP72TT8v2ORui2VY00Qt67X170Qh1G1EOZbFYqNfrrK2tkUqlKJfLUqRBiDso5WhPh5hI7HA4CIVCck6FUOETsszNZpNisUgmk+nJbPiTIvr8zGazvDtEz8b2krKnldTtZcQ9YLVasdvt2Gw21Gq1LB9NpVJkMplPCalotVrsdjsOh6On5rfsJmazGbfbLSeGG41GGaQXmcbdFGTZ886G0Ap2uVzEYjFZP2+32zly5Ii8oAVimNry8jKzs7MkEol95THX63XZiCwOsHA4zLFjx2i329y5c0cadN1ul8XFRS5fvkw6neajjz4im83KZlTYMnh8Ph9/+Id/yOjoKMeOHWNoaIj19XV+/vOfk0wmuXr1KtPT01Sr1Z4rfTGZTJw6dYqJiQnOnj3LH/zBH8gZLdsPMZPJhNVqlWVWGo1GNpLncjni8TidToeTJ08yMDAg/161WiUej5PJZPjZz37Gj3/840c2oO93VCrVDuNZBAiUy2InwnAzGAxEIhGOHj0q9dAVHp9gMMjx48eJRCIcOXKE0dFR3n77bS5evMjS0hKZTIZ6vS572kSA4f4glcKjEY7axMQEr7/+OqFQiFdffZVQKCSHswnFpWw2y61bt/jggw9YW1vb94EFi8XC+fPniUQinDt3juHhYXQ6nZzYnEwmmZubo1gsHjhnw+12EwwGGRsbY2xsjP7+fmq1GqlUiunpaS5evEgmneoZQAAAF+lJREFUk/lUtN3pdHL69Gk5r0Th0ahUKoaHhzl9+jSxWAyPx4PRaJTl75988gnvv/8+xWJx12SX96SzsV1qVAwnEcNynE4n/f39uN1uwuGwHJgDyPpIMUhIPNj99IK3223q9TrNZlNmJiwWC16vVyr+bJcfFEo3mUyGZDJJPp+XA5ZElshmszE6OsqhQ4fk5dFqtVhZWSGRSJBKpSgUCnve0RB17dvr2zUajYzABYNBKSJgNBp3GBviz28fnCYcu1wux8rKCp1Oh/Hx8R1Nu+12m3w+L5uuRHnQfr9gH4ao59bpdNJA2R4M2D4t9qAjmklNJpOMzG8/zxQ+G7PZLPvzXC4XNpuNcrnM/Py8lLjudDo7plyL8qn93AfzsO/2pEID4s+L6gKv18vY2JjsZ3O5XPLPigxwuVwmnU6zsbGx7+7fB6HT6WRJmc/nk2Vl4qyrVquUSqVPVQbsFwW0RyH6HJ1Op5RBr1arVCoV8vk8Gxsb5HI5mdnYbvd5vV58Pt+OO125Ox6MSqXC6XQSjUYJBAIyYCpEa3K5nMz07pZtsqduNoPBIKVXR0ZGcDqdDAwMMDAwgMPhYGBgAJPJJFOWVqt1x1AekTLa3NyUUfj9tjFFliabzcqmcKG+1G63mZ+fJ5vNyjrlqakp2euSTqdptVrYbDasVitDQ0McO3YMn8/HyZMnCQQCJJNJ7t69y8zMDBcvXpSXhijZ2st0u12y2Sz37t2j0+kQjUZlSc/2A2u7RGOlUpGKZ+JCWFlZkROuxXPc2NjAaDQyMTHBxMTEjrKgdDrN+vo6pVKJdru9552y54WI1p84cYKhoSGZARL63pVKhampKd5//33S6fSBdcgEotyzWq0+UI1K4bOxWq0MDw9LxTPYGhK2/Q5Qq9UEg0Gp0GIwGGRfQbFY3Ffy6I1Gg3w+j9FoxOv17vi97aUpQi3pUej1ehwOB0ajkdHRUfx+PydOnODEiRPyDt7OxsYGP/3pT1lfX5fZ3r02X+N5oNFo8Hg8hMNhmZms1Wqk02nS6TT5fF4KhgjHzWazodVq2dzc3LdZcJVKRX9/P6+++irDw8OYTCba7Tarq6ssLS1JSfrNzU3pzI6MjDA8PMyxY8fkjCwhNCAk+EXAVOE386z0ej3Dw8OcO3dOZjWazSbT09NMTU0xPT1Ns9ncVTtuzzkbLpcLt9vNyy+/TF9fH5OTk4yPj2M0GnE4HA8tyRCGZL1e33HR7DdEH0qxWJRNyOFwGLvdTrVa5e7duzIbUSwWmZ6eZmZmRpYHiU3p8/l49dVX+f3f/32sViterxetVsvU1BTvvfces7OzXLp0qadSv51OR05Bt1gs0ph9mCa3UKoqlUrk83k5zfrq1atsbGxw+/Ztpqam5D5yu918+9vfln9XpVLRarXI5XIkk0k5RLJXntezRGQjzWYzhw8f5siRI0QiEVQqlRzcVKvVmJub46OPPjqQz+h+hLMhHF2j0XjgHbAnxWaz0d/fTzAYlPNJRHZbRJLFEE4xV0Ov1+/Q+O+lM+6zEHeD2Wz+VN+YGPYl5gh9FjqdDrfbjc1m46WXXmJiYoLx8XEOHz78qf5AgGQyyS9+8QtWVlaYmZkhl8s9s++1lxHDhUXPHyD7hpLJJIVCQWbYhLMh1A07nc6+djbC4TBnz56Vqo7tdls6o0tLS+TzeRqNhiwpnZiY4Mtf/jKRSEQ6J6lUirW1NRYXF1laWmJzc/NAiq88CDFc02QyMTAwwKlTpzAYDBgMBsrlMrOzs/zqV79ifn5+13uGds3Z2D6NWURJvF6vVGMRF4PH48FkMsn093ZHQ1zU9XqdjY0NKpUKMzMzLC4uMjMzs683ZLVaZWFhQTaI///tnVlPG9cbxh8b23i84QUbGwNxSUhSiEK6qE0u0rRS+w3ab9abfoBKVdWqUtWoF1ESKW0SNTsxDqY2GLA94G1mvDL2ePlf8D+nhrC0gQSw35+EkhCD7KOZOefdnsdut8PpdOLixYvczEmSJP565uBpMpkQDAbh8Xh4tgEAUqkUVFVFLBbD0tISNjY2oGnaqdqEO50Or8yoqgqDwcB1uVnrlNls5tk/9idruWOKXisrK3w4nh3+9pLJ1DQNuVwOoij2RcvAXhiNRj4gOTw8jJGREZ753NzcRDKZRC6X6+s16oZVe7qDDZbJ281BfGBgAMFgELVaDYVCgRSUuuiWGt35fdYy5fP58N5773G1KnbvFwqFnprpK5VKWFhYgCRJ8Hg80Ol0MJvNsFgsMJvNGBsbw+DgIGZmZg68D202GyYmJmC323H+/HmMjY3B5XLx5A1TgmQzahsbG5AkCcVi8diGUN8l3QqFTLmQrY2maVAUBYqi8Iole43VasXk5CSsVitevHiBfD5/zJ/k7dEt8MNggjVGoxHBYBCdTgderxc2mw0XLlzA2NgYr8oxBcPFxUUkk0kuLkJ7yBYGg4GbDLtcLhiNRn5PMpGGTCZzIhIqxxZsGAwG2Gw2CIKAS5cuYWJiAufOncPVq1dht9sRCARgtVp57/dOOTkAqFQqEEUR+Xwed+7cQTqdxuLiIuLxOO+371Xy+Txu3boFv9/PDarGx8fx9ddfo9FoQJKkbaXG7uDO6XRCEATeWy9JEu7du4dUKoWHDx/i6dOnqNfrp65UqWkanj17hkgkArvdjl9++QVms5kHrH6/H6Ojo5AkCU+fPuVtU41GA41GA7VaDa1Wiz/Q2Iap1+v5WrHNhB1sVFXF/Pw8H77vV6xWK4LBIEKhEC5evIiZmRm+wbD7UxRFpNPpY36nJwd23SmKwk2WGo3GNnM0xuDgIK5evYoPP/wQmqbht99+O3X357uGiRUIgoDZ2Vl8+eWX8Hq9MBgMqNfrWF5eRiKRQC6XO+63emSsrKzgxx9/5PvCzMwMgsEgnwu6du0a6vU6gsEg1tbW9v1dDocDoVCIq9yw4fqBgYFtKnzLy8tYXV3F/Pw8lpaWUCgUejZb3w2Ti2eu4TabjYuOMAfsbDYLQRBw9uxZeL1enDlzBm63G5988gmGhobw3XffIRKJHPtB8G1hMBhgsVi4EV+73UalUoEkSbDb7bh+/TrMZjMuXboEj8eDqakpnD17FsBWp0KxWMTdu3dx+/Zt3jrer90Du2GxWPDBBx8gFAphamoKg4ODUFUVGxsbyOfziEajCIfDvLJ2nLyzYKPbq0Cv10MQBP4AY4dA9sUejN2l3k6nw01xGo0Gms0mj9qy2SzS6TQfaM5ms+/qYx0bLKMOgGfhmbEVy+Z3t2R0r73ZbIbRaOSSaLIs84NgPp9HsVg8tbMHm5ub/EtVVZhMJtRqNT703ul0UCgUuNEX66VtNpt7Ht7YmnUrKzHNfpaVZr+rX2Hyhna7ncvuMRqNBgqFAnK5HB2Qu2AbZrPZhKZp0DRtz3uODU2y65CUlA6G9cYzaVy32w2z2czv9WKxyJMyvXJ4qdfrvPLFZgacTiefXbHZbDCbzRgZGTnwMzMpdIvFwtW8gH/2YuYQziq7+Xwem5ubJ+Jg818YHByE0WjcJhzA7sX9RASY6ACrGrEKOgBe8TAajXA6ndA0DSMjIxgZGeHqmna7fZtEbi/SLfbD/m0ymfi6sdbb0dFReDweuFwuCIIAVVW5VD9rd+zVOdzDMDAwwLsJ2HqyMzJLGLMk6nGv2zsJNnQ6HSwWC+9VdDgcGB0dxRdffIHh4WGEQiGu4jA8PMxv0m7YQ61arSIcDiOVSvFKRqVSwfr6Omq1Wk9XM7qpVqv4+++/IYoiRkdHUSwW4XQ6uZpUMBjcJhnHMvZsaEiWZaytrSESiUBRFEQiEUiSBFmWe8LplGm9Mzdhg8EAURSxsLDAN2S2obCvvXA4HJiensbIyAiGh4eh0+m4bvXa2hpkWUa1Wu2L1oG9cLvduHLlCsbHx1+Tby2Xy5ifn8fKykpfV38OYqciWjftdhuKoqBWq50KZbiTgN/vx/Xr1+HxeDA7Owufz4dqtcp7vx88eIBXr1711DWpaRoqlQoA4N69e4jFYvjqq68QCARgNBr5wXpsbAwej2ff38WqQqyawQ7hbC7k2bNnyOVyePToEZ4/f45SqQRFUfYNmk8aBoMBs7OzmJqa4ueSZrOJxcVFSJLEZ812w2w284DswoULCIVCvC3Z7Xbjxo0bfIaU+X2xvTWbzfLW0tO+1/4XjEYjPvroI0xMTPCOAZYUYGe+bDaLeDyOn376CaIoIhwOQ1GUE3FgPmmYTCZMTEzg/Pnz/H5mXjfMNPa4ZzUY7yTYYOVGpm7h9XoRCoVw7do1BAIB+Hw+bqu+F81mE4qiQJZlhMNhRKNRXro9bbMFR0Gj0eDDZ7FYDCaTCX6/H8CWNwkbsGIwqdx2u80fdJFIBH/88QcqlQrS6XRP9S4z6VoAh/5cZrOZV91sNht0Oh3q9TokSeKZ0dOWzTtqWBtVIBDYVtUAtlrN1tfXkU6nUa1Wj+kdnmz2CjTY39kgabFY5AaTxP7Y7XZcvnwZPp+P37vVahWSJCGTyWB5eRlLS0s9tZbsudfpdBCPx5HP5zE9PY16vc6Va/R6PYaGhg7cc7th12Wz2eQH6NXVVaRSKYTD4VMr+qDX6xEMBnH58mWMjIxgamoK9XqdJ6eYlO9un81iscDpdHIZYOaMDWw9D8+dO7ft9UyEpFKpIBaLQZKkvmg362ZgYADj4+MIBALbksrsbCLLMvdJu3v3LpLJJFRV7etE3n6wtnifz8eNiNl9yqwRTsq55K0EGywT4vP5cO7cOT78zXo/h4eHMTw8zFWUdipjtFotFItFqKqKbDbLtZhjsRiKxSJXXJIkqe+j3Varxb0dEokE4vE4BEHAs2fPYLVat72O+XMkk0koioL19XXk83neSkTsjiAICIVCGBsb49UiTdP4YGR3yZ34B0VRUCwWIYoiPyST2tKb0S0ZKYriidlAjptWq8WDfTZcHwwGuQLO1NQUXC4XzGYz6vU6EokEbt26hXQ6/ZqRWC/B9lBN0/Do0SPebjE5OckHv3dK4+6kVqtBFEUeXDAztrW1NZRKJUSjUciyjI2NjVP77GMD9A6HA06nk/s6fPzxx1xEZKe7NYN55Njt9j0DN5bcK5VK2NjYQCKR4GcYJtN+WtfuIDqdDjKZDF6+fInx8XGMj49DEARe0Wi1Wlw5dHl5GcViEclkEqlUCqurq5Ak6VRVyYj9eSvBBhuaOn/+PL755hu4XC6Mj49zxSSn08nLZ7uZDGmaxmVIHz9+jMePHyOfz+PVq1eoVqs80mXeGv1Ms9nkClI752J2rmt3lrT7q5fNrY4Cu92O6elpTExMcBMrTdNQLBZRLpfpgbgLnU4H+Xx+mzY6G+4j/jvNZhPxeBzPnz/H8vIyBW3/h3lqMLMqnU6HqakpAFsuxDMzMxAEgfuZvHr1Ct9//z1XCurV516r1UI+n4dOp4MkSfjzzz8RCARw48YNBAIBfPXVVwcGG4qiYG5ujs/0KYqCaDSKv/76iwd3TFXttKLT6WC32+HxeDAyMoKxsTGYTCaEQqF/tS92mx7uRrPZxMrKClZWVpBIJPh6vnz5ErIsn+q1+zckk0ncv38f09PT+PTTT+F0OrnRa71eR7FYRD6fx+3bt7GysoJwOIxIJIJms3li2n+Io+HQwQZT6WHyq0ajkUvZTk5Owu/3Y2hoCC6Xiys27DQDYhcUc5Zk8qNskDebzXLzl83NTV5yI7boZ8fqdwW7xrtNJLtdTfsdo9EIq9UKi8XChyTL5TIfHGXJAdo8XofNF9nt9n3vY6aQRvf6P7Bsu8FgQKlUgs1mg8FggMfj4ZXdRqOBTCbDq2xsH+n1dez2nup0OhAEAaIoQtM0RKPRA38+l8vxjDPLzhcKBVQqlZ6phHc6HciyjHQ6Db1ez/0gzGYzDAYDT5zu9bPtdhutVguKomxrXWk2m1ykJBaLIZVKIZVKcf+rWq22Z8WkV2CmublcDul0GpFIZNuMCjMzlWUZqVSKJ5hrtRrtEz3IoYMN1i/m8/lw9epVeDwe+P1+DA8Pw+1248yZMzCZTFzCdjeDNXZYTiaTePr0KXK5HDcHYq0YLDNF7SrEu4apjgiCsKdBYL/jcDgwNTUFn88Hi8WCTqeDWCyG33//HZlMhhseEq8jSRLm5uagKApu3Lhx3G/nVCGKIm7duoVAIACXy4VyuQyPx4Pp6WlefaxUKrh58ybm5uaQSCR4e0avBxsM9llFUcTdu3dhNBpx586d15J+u/1cuVzmamlMxauX1k3TNDx48AALCwsIhUKIRCK83czpdGJ8fBwTExO7qr+xgKJcLnO/jHK5jGKxyKtAbEaIdWSwCly/zGqwACIajeLRo0fbhH/a7TYP0JjLOgUavcuhgw2TyQSHwwGPx4OzZ8/yUqTX64UgCBgaGuJyXMA/ZlbdFxSbJWAZhvX1dcTjcaytrXFJSII4LrqrGPt9r59hzwG73c5bCsrlMjKZDCRJ6plM6NugXq9DlmU4HA5+KNkZmLHWUVbVpetuC2YmB2yp2DidTu530Gw2UalUIMsyNwbL5/NoNBp9FfiyPZfNtwAgv5v/0263kcvloCgK2u02PB4P96FqNBrbFDJ3wg7HbB5jfX0dsixDURTkcjm8ePGCz7qwKka/3beqqkJVVT4jSvQvhw423G433n//fYyNjWF6eho+nw92ux2CIPC2k3a7jVKphHq9jnw+j1wuxzdMJjMniiJyuRwSiQQqlQqy2WzfbQrEyUSWZTx8+BDJZJKbBLL2jUKh0NcHaXafu91uBAIBuN1ursc/ODgIm82Gzc1NcrzeB6bsoygKvv32W3i93tdeo2ka4vE4MpkMCoVCT2WXDwNThWs0Gvj111/hcrkwNDQEh8PBRRxUVcXCwgLW19dJyYt4DZb8zGQyePLkCcxmMxYWFrjfl8vl2rWywdoaVVWFKIr8WlNVlbcIsQQBXXNEv3OoYEOn08HtdmNycpI7gO82dNZqtbjx2fLyMhYXF/lwcr1ex507dxCJRLiLM92YxEmiVCphbm4OGxsbuHLlCjqdDlRVhSRJKBaLfRtsMClNVsH0er1claXZbPI5jkqlQsHGPtRqNdRqNayvr2NxcXHP19Fz8XVY5btUKkEUxT3NDmntiL1gVZ9CoYBCobDt//6teSZdXwSxP4cKNpgb8+LiIgqFAnQ63a4ScM1mE7IsY3NzE6IoculGZhLEStuUASBOIo1GA7lcjvf3lkolxONxRKNRFAqFnvInOSp0Oh38fj9mZ2exvLyMWCy2rQ2I2B16/h0OWj/iKKHriThtDAwMwGq1wul04syZM7h48SIXeTjOivih26hisRiSySQGBgbwww8/7JnBZKVKNhDEYGoZNJdBnFQqlQqi0Sj0ej0ikQhMJhMvn7fb7b4ONvbajPV6PS5fvoyJiQk8fvwYz58/520tzGyRIAiCIIijY3BwkJv8Xb9+HV6vF+FwGLdv3z5W8+FDBxv7md4QRC9wlG7kvQYbPu1WrGEupsBW0DEwMPCv2xEIgiAIgjgYNqIgyzKGhoa4zLXBYMDg4CBcLhf8fj9WVlaOvZX5rZj6EQTR+zAddSb8kEwmUalUYLFYoNfrEQ6HMT8/j3g8jlwuh1qt1rfzLQRBEARxlJRKJdy8eRNPnjzBZ599hs8//xxWqxVerxc2mw2hUAhWqxWpVAoGg4Erwx5HeyAFGwRBvDGsLbJSqaBYLPKHmcFggCiKWFhYQDqdRrVaJUdYgiAIgjgiVFVFNBrF2toagsEgLl26hHa7Da/XC5PJBKfTCZ1OB4fDAb1ef6wdBhRsEARxaFZXV/Hzzz9DEAQIggC9Xo+lpSUkEglu2ES+JARBEARxNLTbbd4xwOYiLRYL94Ypl8tQVRUvX77khpzHtQdTsEEQxKGJRqOIx+M8c6LT6dBqtbjqHClQEQRBEMTR0W63UalUoNPpcP/+fTx8+BA6nW5bBaN7rvI4oWCDIIhD0263SSiCIAiCIN4xTOn1JKM7oKSSA7D6jt7LSeYMgNdtfQ+G1m+LN1k/Wrst6No7HHTtvTm0doeD1u/NobU7HLR+bw6t3Zuz59odFGwQBEEQBEEQBEG8EccrvEsQBEEQBEEQRM9CwQZBEARBEARBEG8FCjYIgiAIgiAIgngrULBBEARBEARBEMRbgYINgiAIgiAIgiDeCv8DZhPS/LBTiy8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 1008x216 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EGtnadiqJ2Nu"
      },
      "source": [
        "## Implementando nossa arquitetura!\n",
        "\n",
        "Defina a arquitetura da sua Rede Neural. O pacote torch.nn que contém as implementações de todas as camadas que serão usadas nessa parte (nn.Linear): \n",
        "\n",
        "- https://pytorch.org/docs/stable/nn.html"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4oFGFAKJ2Nv",
        "outputId": "6e479049-fe84-407f-e8f2-02f8a35a70a4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "class MinhaRede(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_size, n_classes):\n",
        "\n",
        "        super(MinhaRede, self).__init__()\n",
        "\n",
        "        '''\n",
        "        Exercício 1.1: Construa a sua rede neural. Ela deve ter entre 2\n",
        "        e 4 camadas escondidas e ir diminuindo o número de features (hidden size)\n",
        "        progressivamente ao longo essas camadas. Cada camada intermediária deve\n",
        "        receber o output da última camada. A primeira camada deve receber\n",
        "        input_size features e a última camada deve gerar n_classes features que\n",
        "        farão as predições entre as classes do MNIST. Adicione ativações ReLU\n",
        "        após todas as camadas nn.Linear, menos na última, que não deve ter\n",
        "        nenhuma ativação.\n",
        "        '''\n",
        "        # ###################################################\n",
        "        # # Neural Network architecture. 2~4 hidden layers. #\n",
        "        # ###################################################\n",
        "        # self.layer1 = # TO DO... Criar primeira camada linear.\n",
        "        # self.ativ1 = # TO DO... Criar ativação da primeira camada.\n",
        "\n",
        "        self.layer1 = nn.Linear(input_size, 64)\n",
        "        self.ativ1  = nn.ReLU()\n",
        "        self.layer2 = nn.Linear(64, 32)\n",
        "        self.ativ2  = nn.ReLU()\n",
        "        self.layer3 = nn.Linear(32, n_classes)\n",
        "        \n",
        "        # self.layer2 = # TO DO... Criar segunda camada linear.\n",
        "        # self.ativ2 = # TO DO... Criar ativação da segunda camada.\n",
        "\n",
        "        # # TO DO... Criar outras camadas/ativações, se achar necessário\n",
        "\n",
        "    # Forward function.\n",
        "    def forward(self, x):\n",
        "\n",
        "        '''\n",
        "        Exercício 1.2: Alimente os dados para a camada da sua rede. Para\n",
        "        alimentar um dado x para uma certa camada self.camada_n:\n",
        "        saida = self.camada_n(x). \n",
        "        Lembre-se de verificar a dimensão do dado de entrada, ela deve estar\n",
        "        na forma (B, F), no nosso caso (32, 784).\n",
        "        '''\n",
        "        # ###################################################\n",
        "        # # Forwarding through all layers. ##################\n",
        "        # ###################################################\n",
        "        # out = # TO DO... Passar o input x sequencialmente pelas camadas da rede.\n",
        "        \n",
        "        x = x.view(x.size(0), -1)\n",
        "        hidden = self.ativ1(self.layer1(x))\n",
        "        hidden = self.ativ2(self.layer2(hidden))\n",
        "        out    = self.layer3(hidden)\n",
        "\n",
        "\n",
        "        # Returning output.\n",
        "        return out # TO DO... Lembre-se de sempre retornar a saída da última camada.\n",
        "        \n",
        "# Instancing Network.\n",
        "input_size = 784 # Input size (number of features).\n",
        "n_classes = 10 # Number of classes.\n",
        "\n",
        "model = MinhaRede(input_size, n_classes).to(device) # GPU casting.\n",
        "\n",
        "# Printing NN.\n",
        "print(model)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MinhaRede(\n",
            "  (layer1): Linear(in_features=784, out_features=64, bias=True)\n",
            "  (ativ1): ReLU()\n",
            "  (layer2): Linear(in_features=64, out_features=32, bias=True)\n",
            "  (ativ2): ReLU()\n",
            "  (layer3): Linear(in_features=32, out_features=10, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ivyz5OiW-VS4"
      },
      "source": [
        "Teste aqui se a sua rede está criando a saída desejada (B, C)\n",
        "- B: `batch_size` \n",
        "- C: número de classes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tK2wGwsG-UOs",
        "outputId": "acf8b094-79fb-44aa-b865-c680935f880d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "k = -1\n",
        "for data, label in test_loader:\n",
        "  k += 1\n",
        "  if k > 2: break\n",
        "\n",
        "  data = data.to(device)\n",
        "  print(data.size())\n",
        "  saida = model(data)\n",
        "  print(saida.size())"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([32, 1, 28, 28])\n",
            "torch.Size([32, 10])\n",
            "torch.Size([32, 1, 28, 28])\n",
            "torch.Size([32, 10])\n",
            "torch.Size([32, 1, 28, 28])\n",
            "torch.Size([32, 10])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VOVXXfCiJ2Ny"
      },
      "source": [
        "## Instanciando o Otimizador\n",
        "\n",
        "O $Pytorch$ tem várias opções de otimizadores, desde os mais simples como o SGD até adaptadores mais modernos com velocidades de aprendizado adaptáveis para cada parâmetro da rede (i.e. Adam, Adagrad, RSMProp...). Todos os otimizadores estão localizados no pacote torch.optim. \n",
        "\n",
        "Para mais informnações sobre o pacote, visite: <https://pytorch.org/docs/stable/optim.html>."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fhyopppRJ2Nz"
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "lr = 0.0001 # TO DO...\n",
        "l2_normalization = 0.00005 # L2 Normalization via weight decay.\n",
        "\n",
        "'''\n",
        "Exercício 2: defina um otimizador. utilize um otimizador mais poderoso \n",
        "como o Adam. Se quiser, experimente diferentes learning rates para o \n",
        "otimizador e observe como isso afeta a otimização da loss.\n",
        "'''\n",
        "optimizer = optim.Adam(params=model.parameters(), lr=lr, weight_decay=l2_normalization)# TO DO..."
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MlBTGdzPJ2N1"
      },
      "source": [
        "## Definindo um critério de qualidade (Loss)\n",
        "\n",
        "Definindo um critério (loss) de classificação para calcular o erro do seu modelo a cada batch de amostras. A $CrossEntropyLoss$ ou a $NLLLoss$ são funções de perda indicadas para esse tipo de tarefa. \n",
        "\n",
        "Informações sobre essas losses podem ser encontradas em: <https://pytorch.org/docs/stable/nn.html#loss-functions>. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H1WirBm_J2N2"
      },
      "source": [
        "# Setting classification loss.\n",
        "'''\n",
        "Exercício 3: Defina uma loss function. Lembre-se de fazer o casting da loss para\n",
        "a GPU assim como fizemos na rede.\n",
        "'''\n",
        "criterion = nn.CrossEntropyLoss().to(device) # TO DO..."
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d9uC1yHfJ2N4"
      },
      "source": [
        "# Fluxo de Treinamento & Validação\n",
        "\n",
        "## Treinamento\n",
        "\n",
        "Passo a passo do fluxo de treinamento (uma época):\n",
        "* Iterar nos batches\n",
        "* Cast dos dados no dispositivo de hardware\n",
        "* Forward na rede e cálculo da loss\n",
        "* Zerar o gradiente do otimizador\n",
        "* Calcular o gradiente da variável loss\n",
        "* Atualizar dos pesos do modelo com o otimizador\n",
        "\n",
        "Esse conjunto de passos é responsável pelo processo iterativo de otimização de uma rede. **A validação** por outro lado, é apenas a aplicação da rede em dados nunca antes visto para estimar a qualidade do modelo no mundo real.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04-4qarbJ2N5"
      },
      "source": [
        "'''\n",
        "Exercício 4: Complete o código da função a seguir para que ela implemente \n",
        "um fluxo de treinamento (descrição acima).\n",
        "'''\n",
        "\n",
        "def train(train_loader, net, epoch):\n",
        "\n",
        "  # Modo de treinamento\n",
        "  net.train()\n",
        "  \n",
        "  start = time.time()\n",
        "  \n",
        "  epoch_loss  = []\n",
        "\n",
        "  # Para cálculo da acurácia\n",
        "  pred_list, label_list = [], []\n",
        "\n",
        "  ## Iterando nos batches\n",
        "  for batch in train_loader:\n",
        "    \n",
        "    dado, rotulo = batch\n",
        "    \n",
        "    ## TO DO: Coloque os dados na GPU (variável device)\n",
        "    dado = dado.to(device)\n",
        "    rotulo = rotulo.to(device)\n",
        "    \n",
        "    ## TO DO: realize o forward dos dados na rede\n",
        "    ypred = net(dado)\n",
        "\n",
        "    ## TO DO: calcule a loss e dê append na lista epoch_loss \n",
        "    loss = criterion(ypred, rotulo)\n",
        "    epoch_loss.append(loss.cpu().data)\n",
        "    \n",
        "    ## TO DO: calcule o gradiente da loss\n",
        "    loss.backward()\n",
        "\n",
        "    ## TO DO: Dê um passo de otimização\n",
        "    optimizer.step()\n",
        "\n",
        "    '''\n",
        "    Exercício 6: Quando terminar tudo e perceber que a sua loss está\n",
        "    reduzindo ao longo do treinamento, implemente o cálculo da \n",
        "    acurácia para melhor avaliar seu modelo.\n",
        "    Faça isso também na função validate().\n",
        "    '''\n",
        "    _, pred = torch.max(ypred, dim=1)\n",
        "    pred_list.append(pred.cpu().numpy())\n",
        "    label_list.append(rotulo.cpu().numpy())\n",
        "\n",
        "  epoch_loss = np.asarray(epoch_loss)\n",
        "\n",
        "  acc = metrics.accuracy_score(np.asarray(label_list).ravel(),\n",
        "                                 np.asarray(pred_list).ravel())\n",
        "  \n",
        "  end = time.time()\n",
        "  print('#################### Train ####################')\n",
        "  print('Epoch %d, Loss: %.4f +/- %.4f, Time: %.2f' % (epoch, epoch_loss.mean(), epoch_loss.std(), end-start))\n",
        "  print('------- Acc: %.2f'%(acc*100))\n",
        "  \n",
        "  return epoch_loss.mean()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fg4O4lArAMfR"
      },
      "source": [
        "## Validação\n",
        "\n",
        "Passo a passo do fluxo de validação (uma época):\n",
        "* Iterar nos batches\n",
        "* Cast dos dados no dispositivo de hardware\n",
        "* Forward na rede e cálculo da loss\n",
        "* ~~Zerar o gradiente do otimizador~~\n",
        "* ~~Calcular o gradiente da variável loss~~\n",
        "* ~~Atualizar dos pesos do modelo com o otimizador~~\n",
        "\n",
        "Para essa etapa, o PyTorch oferece dois artifícios:\n",
        "* ```model.eval()```: Impacta no *forward* da rede, informando as camadas caso seu comportamento mude entre fluxos (ex: dropout).\n",
        "* ```with torch.no_grad()```: Gerenciador de contexto que desabilita o cálculo e armazenamento de gradientes (economia de tempo e memória). Todo o código de validação deve ser executado dentro desse contexto.\n",
        "\n",
        "Exemplo de código para validação\n",
        "\n",
        "```python\n",
        "net.eval()\n",
        "with torch.no_grad():\n",
        "  for batch in test_loader:\n",
        "      # Código de validação\n",
        "```\n",
        "\n",
        "Existe o equivalente ao ```model.eval()``` para explicitar que a sua rede deve estar em modo de treino, é o ```model.train()```. Apesar de ser o padrão dos modelos, é boa prática definir também o modo de treinamento."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g6O62-6cAOP_"
      },
      "source": [
        "'''\n",
        "Exercício 5: Complete o código da função a seguir para que ela implemente \n",
        "um fluxo de validação (descrição acima).\n",
        "'''\n",
        "\n",
        "def validate(test_loader, net, epoch):\n",
        "\n",
        "  # Modo de teste\n",
        "  net.eval()\n",
        "  \n",
        "  start = time.time()\n",
        "  \n",
        "  epoch_loss  = []\n",
        "\n",
        "  # Para cálculo da acurácia\n",
        "  pred_list, label_list = [], []\n",
        "  \n",
        "  with torch.no_grad(): \n",
        "     ## Iterando nos batches\n",
        "    for batch in test_loader:\n",
        "\n",
        "      dado, rotulo = batch\n",
        "\n",
        "      ## TO DO: Coloque os dados na GPU (variável device)\n",
        "      dado = dado.to(device)\n",
        "      rotulo = rotulo.to(device)\n",
        "      \n",
        "      ## TO DO: realize o forward dos dados na rede\n",
        "      ypred = net(dado)\n",
        "\n",
        "      ## TO DO: calcule a loss e dê append na lista epoch_loss \n",
        "      loss = criterion(ypred, rotulo)\n",
        "      epoch_loss.append(loss.cpu().data)\n",
        "\n",
        "      _, pred = torch.max(ypred, dim=1)\n",
        "      pred_list.append(pred.cpu().numpy())\n",
        "      label_list.append(rotulo.cpu().numpy())\n",
        "\n",
        "  epoch_loss = np.asarray(epoch_loss)\n",
        "\n",
        "  acc = metrics.accuracy_score(np.asarray(label_list).ravel(),\n",
        "                                 np.asarray(pred_list).ravel())\n",
        "  \n",
        "  end = time.time()\n",
        "  print('********** Validate **********')\n",
        "  print('Epoch %d, Loss: %.4f +/- %.4f, Time: %.2f' % (epoch, epoch_loss.mean(), epoch_loss.std(), end-start))\n",
        "  print('------- Acc: %.2f\\n'%(acc*100))\n",
        "  \n",
        "  return epoch_loss.mean()\n",
        "    "
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t8iWQH_grcP-",
        "outputId": "38965069-c136-40ac-a685-dbe6fd668c6a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "train_losses, test_losses = [], []\n",
        "num_epochs = 20\n",
        "for epoch in range(num_epochs):\n",
        "  \n",
        "  # Train\n",
        "  train_losses.append(train(train_loader, model, epoch))\n",
        "  \n",
        "  # Validate\n",
        "  test_losses.append(validate(test_loader, model, epoch))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#################### Train ####################\n",
            "Epoch 0, Loss: 0.8685 +/- 0.4977, Time: 8.23\n",
            "------- Acc: 73.92\n",
            "********** Validate **********\n",
            "Epoch 0, Loss: 0.4498 +/- 0.2267, Time: 5.02\n",
            "------- Acc: 86.64\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 1, Loss: 0.3662 +/- 0.2203, Time: 8.15\n",
            "------- Acc: 89.98\n",
            "********** Validate **********\n",
            "Epoch 1, Loss: 0.3143 +/- 0.1964, Time: 5.28\n",
            "------- Acc: 90.83\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 2, Loss: 0.3029 +/- 0.1943, Time: 8.03\n",
            "------- Acc: 91.77\n",
            "********** Validate **********\n",
            "Epoch 2, Loss: 0.2735 +/- 0.2078, Time: 5.12\n",
            "------- Acc: 92.69\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 3, Loss: 0.2792 +/- 0.2144, Time: 8.08\n",
            "------- Acc: 92.75\n",
            "********** Validate **********\n",
            "Epoch 3, Loss: 0.2590 +/- 0.2083, Time: 5.24\n",
            "------- Acc: 92.92\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 4, Loss: 0.2745 +/- 0.2097, Time: 8.10\n",
            "------- Acc: 92.70\n",
            "********** Validate **********\n",
            "Epoch 4, Loss: 0.2616 +/- 0.2032, Time: 5.20\n",
            "------- Acc: 92.64\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 5, Loss: 0.2664 +/- 0.2108, Time: 8.06\n",
            "------- Acc: 92.97\n",
            "********** Validate **********\n",
            "Epoch 5, Loss: 0.2531 +/- 0.1848, Time: 5.23\n",
            "------- Acc: 92.91\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 6, Loss: 0.2605 +/- 0.2160, Time: 8.30\n",
            "------- Acc: 92.97\n",
            "********** Validate **********\n",
            "Epoch 6, Loss: 0.2586 +/- 0.2133, Time: 5.16\n",
            "------- Acc: 92.90\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 7, Loss: 0.2629 +/- 0.2135, Time: 7.85\n",
            "------- Acc: 92.98\n",
            "********** Validate **********\n",
            "Epoch 7, Loss: 0.2519 +/- 0.2103, Time: 5.11\n",
            "------- Acc: 93.14\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 8, Loss: 0.2540 +/- 0.2126, Time: 7.89\n",
            "------- Acc: 93.31\n",
            "********** Validate **********\n",
            "Epoch 8, Loss: 0.2457 +/- 0.2013, Time: 5.56\n",
            "------- Acc: 93.27\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 9, Loss: 0.2469 +/- 0.2003, Time: 8.12\n",
            "------- Acc: 93.34\n",
            "********** Validate **********\n",
            "Epoch 9, Loss: 0.2413 +/- 0.1918, Time: 5.14\n",
            "------- Acc: 93.34\n",
            "\n",
            "#################### Train ####################\n",
            "Epoch 10, Loss: 0.2531 +/- 0.2085, Time: 8.07\n",
            "------- Acc: 93.39\n",
            "********** Validate **********\n",
            "Epoch 10, Loss: 0.2372 +/- 0.2032, Time: 5.13\n",
            "------- Acc: 93.68\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}