L15/packed_lstm.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/andrei-radulescu-banu/stat453-deep-learning-ss21/blob/main/L15/packed_lstm.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U torch==1.13.0 torchtext==0.14.0 torchdata==0.5.0\n",
        "\n",
        "# Reload environment\n",
        "exit()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "quCeAViQxtFC",
        "outputId": "6faf4b29-2eb2-43c4-a948-5271e4b32dfe"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torch==1.13.0 in /usr/local/lib/python3.8/dist-packages (1.13.0+cu116)\n",
            "Requirement already satisfied: torchtext==0.14.0 in /usr/local/lib/python3.8/dist-packages (0.14.0)\n",
            "Collecting torchdata==0.5.0\n",
            "  Downloading torchdata-0.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.5 MB 21.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch==1.13.0) (4.4.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from torchtext==0.14.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from torchtext==0.14.0) (4.64.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from torchtext==0.14.0) (1.21.6)\n",
            "Collecting urllib3>=1.25\n",
            "  Downloading urllib3-1.26.13-py2.py3-none-any.whl (140 kB)\n",
            "\u001b[K     |████████████████████████████████| 140 kB 37.6 MB/s \n",
            "\u001b[?25hCollecting portalocker>=2.0.0\n",
            "  Downloading portalocker-2.6.0-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->torchtext==0.14.0) (3.0.4)\n",
            "Collecting urllib3>=1.25\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 76.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->torchtext==0.14.0) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->torchtext==0.14.0) (2022.9.24)\n",
            "Installing collected packages: urllib3, portalocker, torchdata\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed portalocker-2.6.0 torchdata-0.5.0 urllib3-1.25.11\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1kiY4tZXm7Gz"
      },
      "source": [
        "Derived from:\n",
        "STAT 453: Deep Learning (Spring 2021)  \n",
        "Instructor: Sebastian Raschka (sraschka@wisc.edu)  \n",
        "Lecture 15: Introduction to recurrent neural networks\n",
        "\n",
        "Course website: http://pages.stat.wisc.edu/~sraschka/teaching/stat453-ss2021/  \n",
        "GitHub repository: https://github.com/rasbt/stat453-deep-learning-ss21  \n",
        "Andrei's fork: https://github.com/andrei-radulescu-banu/stat453-deep-learning-ss21\n",
        "\n",
        "Ported by Andrei R-B to torchtext 0.14.0, using new DataPipes.\n",
        "\n",
        "---"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install watermark"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RxYMCgA_nBvP",
        "outputId": "0bfc47e2-e00a-4fc6-9bc6-2034ede9a341"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting watermark\n",
            "  Downloading watermark-2.3.1-py2.py3-none-any.whl (7.2 kB)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.8/dist-packages (from watermark) (7.9.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (0.7.5)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (4.8.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (2.0.10)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (5.6.0)\n",
            "Collecting jedi>=0.10\n",
            "  Downloading jedi-0.18.2-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 31.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (57.4.0)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (0.2.0)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.8/dist-packages (from ipython->watermark) (2.6.1)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.8/dist-packages (from jedi>=0.10->ipython->watermark) (0.8.3)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.8/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython->watermark) (0.2.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.8/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython->watermark) (1.15.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.8/dist-packages (from pexpect->ipython->watermark) (0.7.0)\n",
            "Installing collected packages: jedi, watermark\n",
            "Successfully installed jedi-0.18.2 watermark-2.3.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vY4SK0xKAJgm"
      },
      "source": [
        "# RNN Classifier with LSTM Trained on Own Dataset (IMDB)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sc6xejhY-NzZ"
      },
      "source": [
        "Example notebook showing how to use an own CSV text dataset for training a simple RNN for sentiment classification (here: a binary classification problem with two labels, positive and negative) using LSTM (Long Short Term Memory) cells."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "moNmVfuvnImW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c58eaae2-63fa-4d95-98d4-80ce261defca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Author: Sebastian Raschka, Andrei R-B\n",
            "\n",
            "Python implementation: CPython\n",
            "Python version       : 3.8.16\n",
            "IPython version      : 7.9.0\n",
            "\n",
            "torch    : 1.13.0+cu116\n",
            "torchtext: 0.14.0\n",
            "\n"
          ]
        }
      ],
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka, Andrei R-B' -v -p torch,torchtext\n",
        "\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "import torchtext\n",
        "import time\n",
        "import random\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "torch.backends.cudnn.deterministic = True"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GSRL42Qgy8I8"
      },
      "source": [
        "## General Settings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "OvW1RgfepCBq"
      },
      "outputs": [],
      "source": [
        "RANDOM_SEED = 123\n",
        "torch.manual_seed(RANDOM_SEED)\n",
        "\n",
        "VOCABULARY_SIZE = 20000\n",
        "LEARNING_RATE = 0.005\n",
        "BATCH_SIZE = 128\n",
        "NUM_EPOCHS = 15\n",
        "DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "EMBEDDING_DIM = 128\n",
        "HIDDEN_DIM = 256\n",
        "NUM_CLASSES = 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQMmKUEisW4W"
      },
      "source": [
        "## Download Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tHwOuk54m7G6"
      },
      "source": [
        "The following cells will download the IMDB movie review dataset (http://ai.stanford.edu/~amaas/data/sentiment/) for positive-negative sentiment classification in as CSV-formatted file:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "7Kq0XWHom7G6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c95e15d5-03d8-4bfe-ef1f-836e420ff678"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-12-10 23:36:59--  https://github.com/rasbt/python-machine-learning-book-3rd-edition/raw/master/ch08/movie_data.csv.gz\n",
            "Resolving github.com (github.com)... 20.205.243.166\n",
            "Connecting to github.com (github.com)|20.205.243.166|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/rasbt/python-machine-learning-book-3rd-edition/master/ch08/movie_data.csv.gz [following]\n",
            "--2022-12-10 23:36:59--  https://raw.githubusercontent.com/rasbt/python-machine-learning-book-3rd-edition/master/ch08/movie_data.csv.gz\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 26521894 (25M) [application/octet-stream]\n",
            "Saving to: ‘movie_data.csv.gz’\n",
            "\n",
            "movie_data.csv.gz   100%[===================>]  25.29M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2022-12-10 23:37:00 (201 MB/s) - ‘movie_data.csv.gz’ saved [26521894/26521894]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "!wget https://github.com/rasbt/python-machine-learning-book-3rd-edition/raw/master/ch08/movie_data.csv.gz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "hXwZGkBQm7G7"
      },
      "outputs": [],
      "source": [
        "!gunzip -f movie_data.csv.gz "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wGreVqRDm7G7"
      },
      "source": [
        "Check that the dataset looks okay:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "uVr6xN53m7G8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "614e48e4-abdd-4bec-f3ca-b6c180a4b90a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              review  sentiment\n",
              "0  In 1974, the teenager Martha Moxley (Maggie Gr...          1\n",
              "1  OK... so... I really like Kris Kristofferson a...          0\n",
              "2  ***SPOILER*** Do not read this, if you think a...          0\n",
              "3  hi for all the people who have seen this wonde...          1\n",
              "4  I recently bought the DVD, forgetting just how...          0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-41ff8b8e-9ad5-4efd-bf57-e1e47c0e77da\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>sentiment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>In 1974, the teenager Martha Moxley (Maggie Gr...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>OK... so... I really like Kris Kristofferson a...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>***SPOILER*** Do not read this, if you think a...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>hi for all the people who have seen this wonde...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I recently bought the DVD, forgetting just how...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-41ff8b8e-9ad5-4efd-bf57-e1e47c0e77da')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-41ff8b8e-9ad5-4efd-bf57-e1e47c0e77da button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-41ff8b8e-9ad5-4efd-bf57-e1e47c0e77da');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "df = pd.read_csv('movie_data.csv')\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "raw",
      "metadata": {
        "id": "TCYKMce_m7G9"
      },
      "source": [
        "df.columns = ['TEXT_COLUMN_NAME', 'LABEL_COLUMN_NAME']\n",
        "df.to_csv('movie_data.csv', index=None)\n",
        "\n",
        "df = pd.read_csv('movie_data.csv')\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "ayCZ1etlm7G9"
      },
      "outputs": [],
      "source": [
        "del df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dJJJ1LtSm7G9"
      },
      "source": [
        "## Prepare Dataset with Torchdata and the new DataPipes API"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torchdata.datapipes.iter import IterableWrapper, FileOpener\n",
        "datapipe = IterableWrapper([\"movie_data.csv\"])\n",
        "datapipe = FileOpener(datapipe, mode='b')\n",
        "datapipe = datapipe.parse_csv(skip_lines=1)\n",
        "\n",
        "for sample in datapipe:\n",
        "     print(sample)\n",
        "     break"
      ],
      "metadata": {
        "id": "A7Ue2q-1kwIj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ebaa1193-8a8e-43db-dbbd-02e79569032f"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['In 1974, the teenager Martha Moxley (Maggie Grace) moves to the high-class area of Belle Haven, Greenwich, Connecticut. On the Mischief Night, eve of Halloween, she was murdered in the backyard of her house and her murder remained unsolved. Twenty-two years later, the writer Mark Fuhrman (Christopher Meloni), who is a former LA detective that has fallen in disgrace for perjury in O.J. Simpson trial and moved to Idaho, decides to investigate the case with his partner Stephen Weeks (Andrew Mitchell) with the purpose of writing a book. The locals squirm and do not welcome them, but with the support of the retired detective Steve Carroll (Robert Forster) that was in charge of the investigation in the 70\\'s, they discover the criminal and a net of power and money to cover the murder.<br /><br />\"Murder in Greenwich\" is a good TV movie, with the true story of a murder of a fifteen years old girl that was committed by a wealthy teenager whose mother was a Kennedy. The powerful and rich family used their influence to cover the murder for more than twenty years. However, a snoopy detective and convicted perjurer in disgrace was able to disclose how the hideous crime was committed. The screenplay shows the investigation of Mark and the last days of Martha in parallel, but there is a lack of the emotion in the dramatization. My vote is seven.<br /><br />Title (Brazil): Not Available', '1']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Split Dataset into Train/Validation/Test"
      ],
      "metadata": {
        "id": "uuDLF9BA4fuX"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Split the dataset into training, validation, and test partitions:"
      ],
      "metadata": {
        "id": "FxH6EdtA5CU5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the number of rows in dataset\n",
        "N_ROWS = len(list(datapipe))  # 50000\n",
        "\n",
        "# Split into training and val datapipes early on. Will build vocabulary from training datapipe only.\n",
        "train_dp, valid_dp, test_dp = datapipe.random_split(total_length=N_ROWS, weights={\"train\": 0.8, \"valid\": 0.1, \"test\": 0.1}, seed=0)\n",
        "\n",
        "print(f'Num Train: {len(train_dp)}')\n",
        "print(f'Num Validate: {len(valid_dp)}')\n",
        "print(f'Num Test: {len(test_dp)}')"
      ],
      "metadata": {
        "id": "We2S_UwOsroc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0a283425-b1dd-4bed-9c23-114b865cc143"
      },
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Num Train: 40000\n",
            "Num Validate: 5000\n",
            "Num Test: 5000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Build the Vocabulary"
      ],
      "metadata": {
        "id": "RDIpnTf14KmZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Build the vocabulary based on the top `VOCABULARY_SIZE` words. **build_vocab_from_iterator()** collects the most frequent tokens from the iterator **yield_tokens(train_datapipe)**, and adds special tokens at the begginning w/o changing the order of supplied tokens.\n"
      ],
      "metadata": {
        "id": "jRCjbUQK5q1j"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from torchtext.data.utils import get_tokenizer\n",
        "\n",
        "# See tutorial at https://www.youtube.com/watch?v=CrS-LFXEiyk\n",
        "tokenizer = get_tokenizer(\"basic_english\")\n",
        "\n",
        "def yield_tokens(data_iter):\n",
        "    for text, _ in data_iter:\n",
        "        yield tokenizer(text)\n",
        "\n",
        "def get_vocab(train_datapipe):\n",
        "    vocab = build_vocab_from_iterator(yield_tokens(train_datapipe), specials=[\"<UNK>\", \"<PAD>\"], max_tokens=VOCABULARY_SIZE)\n",
        "    vocab.set_default_index(vocab[\"<UNK>\"])\n",
        "    return vocab"
      ],
      "metadata": {
        "id": "aPC6DfASsZrl"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Build the vocabulary**"
      ],
      "metadata": {
        "id": "wDgg4ub_9zgZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = get_vocab(train_dp)\n",
        "print(\"Vocabulary size: \", len(vocab))"
      ],
      "metadata": {
        "id": "PeyMOsYiUhGO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "935c7b2f-2840-45b9-9e9e-6b5db95b0e79"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Vocabulary size:  20000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Tokens corresponding to the first 10 indices (0, 1, ..., 9):**"
      ],
      "metadata": {
        "id": "_3N2yVlx78fM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(vocab.get_itos()[:10]) # itos = integer-to-string"
      ],
      "metadata": {
        "id": "RbguzRrs7781",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1a3cb84-bf6c-40d4-cb64-dcd53cccfe3d"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['<UNK>', '<PAD>', 'the', '.', ',', 'and', 'a', 'of', 'to', \"'\"]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "PyTorch RNNs can deal with arbitrary lengths due to dynamic graphs, but padding is necessary for padding sequences to the same length in a given minibatch so we can store those in an array."
      ],
      "metadata": {
        "id": "3J1xW7i46Luz"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Converting a string to an integer:**"
      ],
      "metadata": {
        "id": "FQusdyKM9df0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# A direct way\n",
        "print(f\"the: {vocab['the']}\")\n",
        "\n",
        "# And an indirect way, using get_stoi() to get a dictionary of tokens and values\n",
        "print(f\"the: {vocab.get_stoi()['the']}\") # stoi = string-to-integer\n",
        "\n",
        "# What is the padding value?\n",
        "print(f\"<PAD>: {vocab['<PAD>']}\")\n",
        "PADDING_VALUE=vocab['<PAD>']"
      ],
      "metadata": {
        "id": "2Ipkzrhr770W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4fb7a040-23af-4bff-9d8c-9098d913943f"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "the: 2\n",
            "the: 2\n",
            "<PAD>: 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**How are unknown tokens handled?**"
      ],
      "metadata": {
        "id": "UyeWI9Ci-evW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Default index: {vocab.get_default_index()}\")\n",
        "\n",
        "# Value of unknown token is default index when referenced with [] brackets:\n",
        "vocab['123aaa']"
      ],
      "metadata": {
        "id": "Nl_nM0r3699R",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9667d5f1-d44e-4d90-db6f-3a6b1baa938a"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Default index: 0\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Both `text_transform` and `label_transform` are the callable object, such as a lambda func here, to process the raw text and label data from the dataset iterators."
      ],
      "metadata": {
        "id": "6bk05mhx_xUW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text_transform = lambda x: [vocab[token] for token in tokenizer(x)]\n",
        "label_transform = lambda x: 1 if x == '1' else 0\n",
        "\n",
        "# Print out the output of text_transform\n",
        "print(\"input to the text_transform:\", \"here is an example\")\n",
        "print(\"output of the text_transform:\", text_transform(\"here is an example\"))"
      ],
      "metadata": {
        "id": "BLTO10Ha_ygS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "de6ae6dd-1eff-472f-cf31-31abe745def5"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input to the text_transform: here is an example\n",
            "output of the text_transform: [135, 10, 41, 476]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**How do I convert an array of integers to the corresponding tokens?**"
      ],
      "metadata": {
        "id": "aMrQ41cEKszo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vocab_itos = vocab.get_itos()\n",
        "vec_vocab_itos = np.vectorize(lambda x: vocab_itos[x])\n",
        "vec_vocab_itos([1, 2, 3, 4, 5])"
      ],
      "metadata": {
        "id": "XIBt3bdlKr1t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4430f8c4-8cbb-4757-a196-7387315f3b90"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['<PAD>', 'the', '.', ',', 'and'], dtype='<U5')"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Define Data Loaders"
      ],
      "metadata": {
        "id": "5ohB-Ern-AmJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "`torch.utils.data.DataLoader` is used to generate data batch. Users could customize the data batch by defining a function with the `collate_fn` argument in the DataLoader. Here, in the `collate_batch` func, we process the raw text data and add padding to dynamically match the longest sentence in a batch."
      ],
      "metadata": {
        "id": "PnyQAEm4_D9I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "import torch\n",
        "\n",
        "def collate_batch(batch):\n",
        "   text_list, label_list = [], []\n",
        "   for (_text, _label) in batch:\n",
        "        processed_text = torch.tensor(text_transform(_text))\n",
        "        text_list.append(processed_text)\n",
        "        label_list.append(label_transform(_label))\n",
        "   return pad_sequence(text_list, padding_value=PADDING_VALUE).to(DEVICE), torch.tensor(label_list).to(DEVICE)"
      ],
      "metadata": {
        "id": "uFAZeWDa_H1g"
      },
      "execution_count": 128,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "To group the texts with similar length together, like introduced in the legacy `BucketIterator` class, first of all, we randomly create multiple \"pools\", and each of them has a size of `batch_size * 100`. Then, we sort the samples within the individual pool by length. This idea can be implemented succintly through `batch_sampler` argument of PyTorch `Dataloader`. `batch_sampler` accepts 'Sampler' or Iterable object that yields indices of next batch. In the code below, we implemented a generator that yields batch of indices for which the corresponding batch of data is of similar length. "
      ],
      "metadata": {
        "id": "PPVGxmEsCR6h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import Sampler, Dataset\n",
        "\n",
        "# Create our own sampler, to ensure we function with multiple worker threads\n",
        "# See https://discuss.pytorch.org/t/using-distributedsampler-in-combination-with-batch-sampler-to-make-sure-batches-have-sentences-of-similar-length/119824/3\n",
        "class BatchSamplerSimilarLength(Sampler):\n",
        "    def __init__(self, dataset, batch_size, indices=None, shuffle=True):\n",
        "        self.batch_size = batch_size\n",
        "        self.shuffle = shuffle\n",
        "        # get the indices and length\n",
        "        self.indices = [(i, len(tokenizer(s[0]))) for i, s in enumerate(dataset)]\n",
        "        # if indices are passed, then use only the ones passed (for ddp)\n",
        "        if indices is not None:\n",
        "            self.indices = torch.tensor(self.indices)[indices].tolist()\n",
        "\n",
        "    def __iter__(self):\n",
        "        if self.shuffle:\n",
        "            random.shuffle(self.indices)\n",
        "\n",
        "        pooled_indices = []\n",
        "        # create pool of indices with similar lengths\n",
        "        for i in range(0, len(self.indices), self.batch_size * 100):\n",
        "          pooled_indices.extend(sorted(self.indices[i:i + self.batch_size * 100], key=lambda x: x[1]))\n",
        "        self.pooled_indices = [x[0] for x in pooled_indices]\n",
        "        \n",
        "        # Comment in for validation\n",
        "        #self.pooled_lengths = [x[1] for x in pooled_indices]\n",
        "        #print(self.pooled_lengths)\n",
        "        #print(self.pooled_indices)\n",
        "\n",
        "        # yield indices for current batch\n",
        "        batches = [self.pooled_indices[i:i + self.batch_size] for i in\n",
        "                   range(0, len(self.pooled_indices), self.batch_size)]\n",
        "\n",
        "        if self.shuffle:\n",
        "            random.shuffle(batches)\n",
        "        for batch in batches:\n",
        "            yield batch\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.pooled_indices) // self.batch_size"
      ],
      "metadata": {
        "id": "vDAmd8BzUD1B"
      },
      "execution_count": 129,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dp_list = list(train_dp)\n",
        "valid_dp_list = list(valid_dp)\n",
        "test_dp_list = list(test_dp)\n",
        "\n",
        "train_loader = DataLoader(train_dp_list, \n",
        "                          batch_sampler=BatchSamplerSimilarLength(dataset = train_dp_list, \n",
        "                                                                  batch_size=BATCH_SIZE),\n",
        "                          collate_fn=collate_batch)\n",
        "valid_loader = DataLoader(train_dp_list, \n",
        "                          batch_sampler=BatchSamplerSimilarLength(dataset = valid_dp_list, \n",
        "                                                                  batch_size=BATCH_SIZE,\n",
        "                                                                  shuffle=False),\n",
        "                          collate_fn=collate_batch)\n",
        "test_loader = DataLoader(train_dp_list, \n",
        "                          batch_sampler=BatchSamplerSimilarLength(dataset = test_dp_list, \n",
        "                                                                  batch_size=BATCH_SIZE,\n",
        "                                                                  shuffle=False),\n",
        "                          collate_fn=collate_batch)\n",
        "\n",
        "text_batch, label_batch = next(iter(train_loader))\n",
        "print(text_batch.size())\n",
        "print(label_batch.size())"
      ],
      "metadata": {
        "id": "Kzmg30JnXODr",
        "outputId": "fcc78966-1c60-4b7c-ccb2-ab397b86ddeb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 130,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([115, 128])\n",
            "torch.Size([128])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Testing the iterators (note that the number of rows depends on the longest document in the respective batch):"
      ],
      "metadata": {
        "id": "alhb-l3CSvO1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print('Train')\n",
        "for text_batch, label_batch in train_loader:\n",
        "    print(f'Text matrix size: {text_batch.size()}')\n",
        "    print(f'Target vector size: {label_batch.size()}')\n",
        "    break\n",
        "    \n",
        "print('\\nValid:')\n",
        "for text_batch, label_batch in valid_loader:\n",
        "    print(f'Text matrix size: {text_batch.size()}')\n",
        "    print(f'Target vector size: {label_batch.size()}')\n",
        "    break\n",
        "    \n",
        "print('\\nTest:')\n",
        "for text_batch, label_batch in test_loader:\n",
        "    print(f'Text matrix size: {text_batch.size()}')\n",
        "    print(f'Target vector size: {label_batch.size()}')\n",
        "    break"
      ],
      "metadata": {
        "id": "4P65G4zqS1qk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4fc8d622-623c-4e3d-8a3e-0ece9d3c2f7d"
      },
      "execution_count": 131,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train\n",
            "Text matrix size: torch.Size([310, 128])\n",
            "Target vector size: torch.Size([128])\n",
            "\n",
            "Valid:\n",
            "Text matrix size: torch.Size([763, 128])\n",
            "Target vector size: torch.Size([128])\n",
            "\n",
            "Test:\n",
            "Text matrix size: torch.Size([1130, 128])\n",
            "Target vector size: torch.Size([128])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The text batch as indices of tokens**"
      ],
      "metadata": {
        "id": "5BcP4qmAOSGC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(text_batch)"
      ],
      "metadata": {
        "id": "oPee-HP-MusQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d3718193-8f4d-4191-a036-506008b0ed75"
      },
      "execution_count": 132,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 3470,    14,    46,  ...,    43, 16219,    13],\n",
            "        [ 1451,    10,    10,  ...,    41,   598,    33],\n",
            "        [ 2110,    29,     6,  ...,  1626,    53,   119],\n",
            "        ...,\n",
            "        [    1,     1,     1,  ...,     1,     1,     1],\n",
            "        [    1,     1,     1,  ...,     1,     1,     1],\n",
            "        [    1,     1,     1,  ...,     1,     1,     1]], device='cuda:0')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The corresponding vocabulary tokens**\n",
        "\n",
        "Notice how the reviews appear on the tensor columns. Each column has one review."
      ],
      "metadata": {
        "id": "epUvW9z5OYgF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(vec_vocab_itos(text_batch.to(\"cpu\")))"
      ],
      "metadata": {
        "id": "siH213KBMx2O",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3db18713-a80f-4e91-9027-c783e6b72ad0"
      },
      "execution_count": 133,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['broadcast' 'this' 'there' ... 'from' 'diagnosis' 'i']\n",
            " ['news' 'is' 'is' ... 'an' 'murder' 'have']\n",
            " ['opens' 'not' 'a' ... 'artistic' 'has' 'never']\n",
            " ...\n",
            " ['<PAD>' '<PAD>' '<PAD>' ... '<PAD>' '<PAD>' '<PAD>']\n",
            " ['<PAD>' '<PAD>' '<PAD>' ... '<PAD>' '<PAD>' '<PAD>']\n",
            " ['<PAD>' '<PAD>' '<PAD>' ... '<PAD>' '<PAD>' '<PAD>']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**And the batch labels**\n",
        "\n",
        "Each element is the review score for the corresponding column."
      ],
      "metadata": {
        "id": "0Pjzqxv1OeBk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(label_batch)"
      ],
      "metadata": {
        "id": "J5NR0SlxN1Qh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b066a62c-13ee-4683-dc3d-763a26585967"
      },
      "execution_count": 135,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0,\n",
            "        0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1,\n",
            "        1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1,\n",
            "        1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0,\n",
            "        1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0,\n",
            "        0, 0, 1, 1, 1, 0, 1, 0], device='cuda:0')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**The next batch will have different text column size.**\n",
        "\n",
        "The column size is determined by the max length of reviews in a batch. Each batch has a different text column size."
      ],
      "metadata": {
        "id": "SbKYBRznQgi8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text_batch, label_batch = next(iter(train_loader))\n",
        "print(f'Text matrix size: {text_batch.size()}')\n",
        "print(f'Target vector size: {label_batch.size()}')\n",
        "\n",
        "text_batch, label_batch = next(iter(train_loader))\n",
        "print(f'Text matrix size: {text_batch.size()}')\n",
        "print(f'Target vector size: {label_batch.size()}')"
      ],
      "metadata": {
        "id": "RFpzgfY5Qfnv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "434d222f-0136-4ef1-e41d-544197f8467d"
      },
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Text matrix size: torch.Size([160, 128])\n",
            "Target vector size: torch.Size([128])\n",
            "Text matrix size: torch.Size([99, 128])\n",
            "Target vector size: torch.Size([128])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Utility Functions"
      ],
      "metadata": {
        "id": "YDm8DtLzKR79"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Compute model accuracy on a dataloader**\n",
        "\n",
        "Pass the train, valid or test dataloader\n"
      ],
      "metadata": {
        "id": "wfFF6KruLaZ_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 138,
      "metadata": {
        "id": "T5t1Afn4xO11"
      },
      "outputs": [],
      "source": [
        "def compute_accuracy(model, data_loader, device):\n",
        "\n",
        "    with torch.no_grad():\n",
        "\n",
        "        correct_pred, num_examples = 0, 0\n",
        "\n",
        "        for i, (features, targets) in enumerate(data_loader):\n",
        "\n",
        "            features = features.to(device)\n",
        "            targets = targets.float().to(device)\n",
        "\n",
        "            logits = model(features)\n",
        "            _, predicted_labels = torch.max(logits, 1)\n",
        "\n",
        "            num_examples += targets.size(0)\n",
        "            correct_pred += (predicted_labels == targets).sum().to(\"cpu\")\n",
        "\n",
        "    return float(correct_pred)/num_examples * 100"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Train a model**\n",
        "\n",
        "This is model- and dataloader-independent."
      ],
      "metadata": {
        "id": "5a24JY8bLmX6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train_model(model, num_epochs, train_loader,\n",
        "                valid_loader, test_loader, optimizer,\n",
        "                device, logging_interval=50,\n",
        "                scheduler=None,\n",
        "                scheduler_on='valid_acc'):\n",
        "\n",
        "    start_time = time.time()\n",
        "    minibatch_loss_list, train_acc_list, valid_acc_list = [], [], []\n",
        "    \n",
        "    for epoch in range(num_epochs):\n",
        "\n",
        "        model.train()\n",
        "        for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "\n",
        "            features = features.to(device)\n",
        "            targets = targets.to(device)\n",
        "\n",
        "            # ## FORWARD AND BACK PROP\n",
        "            logits = model(features)\n",
        "            loss = torch.nn.functional.cross_entropy(logits, targets)\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            loss.backward()\n",
        "\n",
        "            # ## UPDATE MODEL PARAMETERS\n",
        "            optimizer.step()\n",
        "\n",
        "            # ## LOGGING\n",
        "            minibatch_loss_list.append(loss.item())\n",
        "            if not batch_idx % logging_interval:\n",
        "                print(f'Epoch: {epoch+1:03d}/{num_epochs:03d} '\n",
        "                      f'| Batch {batch_idx:04d}/{len(train_loader):04d} '\n",
        "                      f'| Loss: {loss:.4f}')\n",
        "\n",
        "        model.eval()\n",
        "        with torch.no_grad():  # save memory during inference\n",
        "            train_acc = compute_accuracy(model, train_loader, device=device)\n",
        "            valid_acc = compute_accuracy(model, valid_loader, device=device)\n",
        "            print(f'Epoch: {epoch+1:03d}/{num_epochs:03d} '\n",
        "                  f'| Train: {train_acc :.2f}% '\n",
        "                  f'| Validation: {valid_acc :.2f}%')\n",
        "            train_acc_list.append(train_acc)\n",
        "            valid_acc_list.append(valid_acc)\n",
        "\n",
        "        elapsed = (time.time() - start_time)/60\n",
        "        print(f'Time elapsed: {elapsed:.2f} min')\n",
        "        \n",
        "        if scheduler is not None:\n",
        "\n",
        "            if scheduler_on == 'valid_acc':\n",
        "                scheduler.step(valid_acc_list[-1])\n",
        "            elif scheduler_on == 'minibatch_loss':\n",
        "                scheduler.step(minibatch_loss_list[-1])\n",
        "            else:\n",
        "                raise ValueError(f'Invalid `scheduler_on` choice.')\n",
        "        \n",
        "\n",
        "    elapsed = (time.time() - start_time)/60\n",
        "    print(f'Total Training Time: {elapsed:.2f} min')\n",
        "\n",
        "    test_acc = compute_accuracy(model, test_loader, device=device)\n",
        "    print(f'Test accuracy {test_acc :.2f}%')\n",
        "\n",
        "    return minibatch_loss_list, train_acc_list, valid_acc_list\n"
      ],
      "metadata": {
        "id": "RYnX6OzjLAYD"
      },
      "execution_count": 139,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Plot training loss and accuracy**"
      ],
      "metadata": {
        "id": "DhFkZG_3Lxyn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def plot_training_loss(minibatch_loss_list, num_epochs, iter_per_epoch,\n",
        "                       results_dir=None, averaging_iterations=100):\n",
        "\n",
        "    plt.figure()\n",
        "    ax1 = plt.subplot(1, 1, 1)\n",
        "    ax1.plot(range(len(minibatch_loss_list)),\n",
        "             (minibatch_loss_list), label='Minibatch Loss')\n",
        "\n",
        "    if len(minibatch_loss_list) > 1000:\n",
        "        ax1.set_ylim([\n",
        "            0, np.max(minibatch_loss_list[1000:])*1.5\n",
        "            ])\n",
        "    ax1.set_xlabel('Iterations')\n",
        "    ax1.set_ylabel('Loss')\n",
        "\n",
        "    ax1.plot(np.convolve(minibatch_loss_list,\n",
        "                         np.ones(averaging_iterations,)/averaging_iterations,\n",
        "                         mode='valid'),\n",
        "             label='Running Average')\n",
        "    ax1.legend()\n",
        "\n",
        "    ###################\n",
        "    # Set scond x-axis\n",
        "    ax2 = ax1.twiny()\n",
        "    newlabel = list(range(num_epochs+1))\n",
        "\n",
        "    newpos = [e*iter_per_epoch for e in newlabel]\n",
        "\n",
        "    ax2.set_xticks(newpos[::10])\n",
        "    ax2.set_xticklabels(newlabel[::10])\n",
        "\n",
        "    ax2.xaxis.set_ticks_position('bottom')\n",
        "    ax2.xaxis.set_label_position('bottom')\n",
        "    ax2.spines['bottom'].set_position(('outward', 45))\n",
        "    ax2.set_xlabel('Epochs')\n",
        "    ax2.set_xlim(ax1.get_xlim())\n",
        "    ###################\n",
        "\n",
        "    plt.tight_layout()\n",
        "\n",
        "    if results_dir is not None:\n",
        "        image_path = os.path.join(results_dir, 'plot_training_loss.pdf')\n",
        "        plt.savefig(image_path)\n",
        "\n",
        "\n",
        "def plot_accuracy(train_acc_list, valid_acc_list, results_dir):\n",
        "\n",
        "    num_epochs = len(train_acc_list)\n",
        "\n",
        "    plt.plot(np.arange(1, num_epochs+1),\n",
        "             train_acc_list, label='Training')\n",
        "    plt.plot(np.arange(1, num_epochs+1),\n",
        "             valid_acc_list, label='Validation')\n",
        "\n",
        "    plt.xlabel('Epoch')\n",
        "    plt.ylabel('Accuracy')\n",
        "    plt.legend()\n",
        "\n",
        "    plt.tight_layout()\n",
        "\n",
        "    if results_dir is not None:\n",
        "        image_path = os.path.join(\n",
        "            results_dir, 'plot_acc_training_validation.pdf')\n",
        "        plt.savefig(image_path)\n"
      ],
      "metadata": {
        "id": "15hbXktfKbeM"
      },
      "execution_count": 140,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_grdW3pxCzz"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 141,
      "metadata": {
        "id": "nQIUm5EjxFNa"
      },
      "outputs": [],
      "source": [
        "class LSTM(torch.nn.Module):\n",
        "    \n",
        "    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):\n",
        "        super().__init__()\n",
        "\n",
        "        self.embedding = torch.nn.Embedding(input_dim, embedding_dim)\n",
        "        \n",
        "        self.rnn = torch.nn.LSTM(embedding_dim,\n",
        "                                 hidden_dim)        \n",
        "        \n",
        "        self.fc = torch.nn.Linear(hidden_dim, output_dim)\n",
        "        \n",
        "\n",
        "    def forward(self, text):\n",
        "        # text dim: [sentence length, batch size]        \n",
        "        embedded = self.embedding(text)\n",
        "        # embedded dim: [sentence length, batch size, embedding dim]\n",
        "        \n",
        "        output, (hidden, cell) = self.rnn(embedded)\n",
        "        # output dim: [sentence length, batch size, hidden dim]\n",
        "        # hidden dim: [1, batch size, hidden dim]\n",
        "\n",
        "        hidden.squeeze_(0)\n",
        "        # hidden dim: [batch size, hidden dim]\n",
        "        \n",
        "        output = self.fc(hidden)\n",
        "        return output"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train and plot the results"
      ],
      "metadata": {
        "id": "RZYw-Aa6M2od"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = LSTM(input_dim=len(vocab),\n",
        "             embedding_dim=EMBEDDING_DIM,\n",
        "             hidden_dim=HIDDEN_DIM,\n",
        "             output_dim=NUM_CLASSES # could use 1 for binary classification\n",
        ")\n",
        "\n",
        "model = model.to(DEVICE)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n",
        "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,\n",
        "                                                       factor=0.1,\n",
        "                                                       mode='max',\n",
        "                                                       verbose=True)\n",
        "\n",
        "minibatch_loss_list, train_acc_list, valid_acc_list = train_model(\n",
        "    model=model,\n",
        "    num_epochs=NUM_EPOCHS,\n",
        "    train_loader=train_loader,\n",
        "    valid_loader=valid_loader,\n",
        "    test_loader=test_loader,\n",
        "    optimizer=optimizer,\n",
        "    device=DEVICE,\n",
        "    logging_interval=100)\n",
        "\n"
      ],
      "metadata": {
        "id": "SJsC5jzkMe6_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b0c07394-5ef0-472c-d110-b0c14efc6db6"
      },
      "execution_count": 142,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 001/015 | Batch 0000/0312 | Loss: 0.6992\n",
            "Epoch: 001/015 | Batch 0100/0312 | Loss: 0.6810\n",
            "Epoch: 001/015 | Batch 0200/0312 | Loss: 0.7097\n",
            "Epoch: 001/015 | Batch 0300/0312 | Loss: 0.2935\n",
            "Epoch: 001/015 | Train: 90.00% | Validation: 89.56%\n",
            "Time elapsed: 0.70 min\n",
            "Epoch: 002/015 | Batch 0000/0312 | Loss: 0.1914\n",
            "Epoch: 002/015 | Batch 0100/0312 | Loss: 0.2790\n",
            "Epoch: 002/015 | Batch 0200/0312 | Loss: 0.2494\n",
            "Epoch: 002/015 | Batch 0300/0312 | Loss: 0.1457\n",
            "Epoch: 002/015 | Train: 94.07% | Validation: 93.72%\n",
            "Time elapsed: 1.41 min\n",
            "Epoch: 003/015 | Batch 0000/0312 | Loss: 0.1588\n",
            "Epoch: 003/015 | Batch 0100/0312 | Loss: 0.1605\n",
            "Epoch: 003/015 | Batch 0200/0312 | Loss: 0.2193\n",
            "Epoch: 003/015 | Batch 0300/0312 | Loss: 0.0841\n",
            "Epoch: 003/015 | Train: 97.11% | Validation: 97.16%\n",
            "Time elapsed: 2.10 min\n",
            "Epoch: 004/015 | Batch 0000/0312 | Loss: 0.0480\n",
            "Epoch: 004/015 | Batch 0100/0312 | Loss: 0.0523\n",
            "Epoch: 004/015 | Batch 0200/0312 | Loss: 0.0857\n",
            "Epoch: 004/015 | Batch 0300/0312 | Loss: 0.1098\n",
            "Epoch: 004/015 | Train: 98.57% | Validation: 98.44%\n",
            "Time elapsed: 2.80 min\n",
            "Epoch: 005/015 | Batch 0000/0312 | Loss: 0.0542\n",
            "Epoch: 005/015 | Batch 0100/0312 | Loss: 0.0437\n",
            "Epoch: 005/015 | Batch 0200/0312 | Loss: 0.0465\n",
            "Epoch: 005/015 | Batch 0300/0312 | Loss: 0.0734\n",
            "Epoch: 005/015 | Train: 98.95% | Validation: 98.96%\n",
            "Time elapsed: 3.50 min\n",
            "Epoch: 006/015 | Batch 0000/0312 | Loss: 0.0640\n",
            "Epoch: 006/015 | Batch 0100/0312 | Loss: 0.0067\n",
            "Epoch: 006/015 | Batch 0200/0312 | Loss: 0.0061\n",
            "Epoch: 006/015 | Batch 0300/0312 | Loss: 0.0161\n",
            "Epoch: 006/015 | Train: 99.22% | Validation: 98.96%\n",
            "Time elapsed: 4.21 min\n",
            "Epoch: 007/015 | Batch 0000/0312 | Loss: 0.0132\n",
            "Epoch: 007/015 | Batch 0100/0312 | Loss: 0.0242\n",
            "Epoch: 007/015 | Batch 0200/0312 | Loss: 0.0616\n",
            "Epoch: 007/015 | Batch 0300/0312 | Loss: 0.0732\n",
            "Epoch: 007/015 | Train: 99.33% | Validation: 99.30%\n",
            "Time elapsed: 4.91 min\n",
            "Epoch: 008/015 | Batch 0000/0312 | Loss: 0.0125\n",
            "Epoch: 008/015 | Batch 0100/0312 | Loss: 0.0423\n",
            "Epoch: 008/015 | Batch 0200/0312 | Loss: 0.0150\n",
            "Epoch: 008/015 | Batch 0300/0312 | Loss: 0.0395\n",
            "Epoch: 008/015 | Train: 99.42% | Validation: 99.50%\n",
            "Time elapsed: 5.62 min\n",
            "Epoch: 009/015 | Batch 0000/0312 | Loss: 0.0438\n",
            "Epoch: 009/015 | Batch 0100/0312 | Loss: 0.0020\n",
            "Epoch: 009/015 | Batch 0200/0312 | Loss: 0.0118\n",
            "Epoch: 009/015 | Batch 0300/0312 | Loss: 0.0479\n",
            "Epoch: 009/015 | Train: 99.41% | Validation: 99.36%\n",
            "Time elapsed: 6.34 min\n",
            "Epoch: 010/015 | Batch 0000/0312 | Loss: 0.0073\n",
            "Epoch: 010/015 | Batch 0100/0312 | Loss: 0.0155\n",
            "Epoch: 010/015 | Batch 0200/0312 | Loss: 0.0448\n",
            "Epoch: 010/015 | Batch 0300/0312 | Loss: 0.0674\n",
            "Epoch: 010/015 | Train: 99.41% | Validation: 99.40%\n",
            "Time elapsed: 7.04 min\n",
            "Epoch: 011/015 | Batch 0000/0312 | Loss: 0.0048\n",
            "Epoch: 011/015 | Batch 0100/0312 | Loss: 0.0243\n",
            "Epoch: 011/015 | Batch 0200/0312 | Loss: 0.0027\n",
            "Epoch: 011/015 | Batch 0300/0312 | Loss: 0.0437\n",
            "Epoch: 011/015 | Train: 99.52% | Validation: 99.42%\n",
            "Time elapsed: 7.74 min\n",
            "Epoch: 012/015 | Batch 0000/0312 | Loss: 0.0366\n",
            "Epoch: 012/015 | Batch 0100/0312 | Loss: 0.0205\n",
            "Epoch: 012/015 | Batch 0200/0312 | Loss: 0.0170\n",
            "Epoch: 012/015 | Batch 0300/0312 | Loss: 0.0593\n",
            "Epoch: 012/015 | Train: 99.60% | Validation: 99.56%\n",
            "Time elapsed: 8.46 min\n",
            "Epoch: 013/015 | Batch 0000/0312 | Loss: 0.0092\n",
            "Epoch: 013/015 | Batch 0100/0312 | Loss: 0.0057\n",
            "Epoch: 013/015 | Batch 0200/0312 | Loss: 0.0141\n",
            "Epoch: 013/015 | Batch 0300/0312 | Loss: 0.0245\n",
            "Epoch: 013/015 | Train: 99.69% | Validation: 99.74%\n",
            "Time elapsed: 9.16 min\n",
            "Epoch: 014/015 | Batch 0000/0312 | Loss: 0.0385\n",
            "Epoch: 014/015 | Batch 0100/0312 | Loss: 0.0165\n",
            "Epoch: 014/015 | Batch 0200/0312 | Loss: 0.0071\n",
            "Epoch: 014/015 | Batch 0300/0312 | Loss: 0.0373\n",
            "Epoch: 014/015 | Train: 99.78% | Validation: 99.62%\n",
            "Time elapsed: 9.86 min\n",
            "Epoch: 015/015 | Batch 0000/0312 | Loss: 0.0824\n",
            "Epoch: 015/015 | Batch 0100/0312 | Loss: 0.0007\n",
            "Epoch: 015/015 | Batch 0200/0312 | Loss: 0.0005\n",
            "Epoch: 015/015 | Batch 0300/0312 | Loss: 0.0045\n",
            "Epoch: 015/015 | Train: 99.70% | Validation: 99.74%\n",
            "Time elapsed: 10.57 min\n",
            "Total Training Time: 10.57 min\n",
            "Test accuracy 99.72%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plot_training_loss(minibatch_loss_list=minibatch_loss_list,\n",
        "                   num_epochs=NUM_EPOCHS,\n",
        "                   iter_per_epoch=len(train_loader),\n",
        "                   results_dir=None,\n",
        "                   averaging_iterations=100)\n",
        "plt.ylim([0, .3])\n",
        "plt.show()\n",
        "\n",
        "plot_accuracy(train_acc_list=train_acc_list,\n",
        "              valid_acc_list=valid_acc_list,\n",
        "              results_dir=None)\n",
        "plt.ylim([90, 100])\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "QVwYR4zKiH9l",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 584
        },
        "outputId": "080017f4-1e88-463a-fdab-e5a4514b6184"
      },
      "execution_count": 145,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEbCAYAAACP7BAbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydZ3hU1daA3z0lGXoXaRIQRHqVogiioNgAUVABBfWKXOXTKzZsqIhexHIVRayIKN2KIiJdkB4IXXqABKQktBAymbK/HzMZJslMpp0zMwn7fR4e5pyzy5rJzFlnrb32WkJKiUKhUCgU8YYh1gIoFAqFQuELpaAUCoVCEZcoBaVQKBSKuEQpKIVCoVDEJUpBKRQKhSIuUQpKoVAoFHGJrgpKCNFTCLFTCLFHCDHSx/VhQogtQogUIcQKIUQTr2vPu/vtFELcpKecCoVCoYg/hF77oIQQRmAX0ANIA9YB90opt3u1KS+lPON+3Qt4VErZ062opgPtgZrAQuAKKaVDF2EVCoVCEXfoaUG1B/ZIKfdJKXOBGUBv7wZ5yslNGSBPW/YGZkgprVLK/cAe93gKhUKhuEgw6Th2LeCQ13Ea0KFgIyHEY8AIIAG43qvv6gJ9a+kjpkKhUCjiET0VVFBIKScAE4QQA4CXgMHB9hVCDAWGug/bli5dWgcJQ0ParEicWGVCVOYrlWCMeAwpIcfmQAiwmCMfT6FQKEIhOztbSikLefT0VFDpQB2v49ruc/6YAUwMpa+U8jPgM4AyZcrIc+fORSKvJqR+eg8yfQPdcv8Xlfl+Gd6ZMolG6lcrG/YYR8/k0OHNRVQrl8i6F7trKJ1CoVAERghx3td5Pdeg1gENhRD1hBAJwD3AnAJCNfQ6vBXY7X49B7hHCJEohKgHNATW6iirZiSWrUQ535+1Ltz+0Qquf3dZ1OZTKBSKaKGbBSWltAshhgPzASMwSUq5TQgxGlgvpZwDDBdCdAdswEnc7j13u1nAdsAOPFZcIvhqVL8U265zCJzIYrbN7PhZKxlZVqqUTYy1KAqFQqHvGpSU8jfgtwLnRnm9fqKIvm8Ab+gnnU6UrY5ZOKhIFicpH2tpQqbtmIWkjr011mIoFApF7IMkShxWV+R8N0MKPzi7xFgYhaL4YLPZSEtLIycnJ9aiKHTCYrFQu3ZtzGZzUO2VgtKaqq5ltRfM0/jBqhSUQhEsaWlplCtXjqSkJIQQsRZHoTFSSjIyMkhLS6NevXpB9SleiyTFgaZ3AFBVnAnQUKFQeJOTk0OVKlWUciqhCCGoUqVKSBayUlA6cEqWibUICkWxRCmnkk2of1+loHRgor0XAGXJjrEkCoUiFIQQDBo0yHNst9upVq0at912GwBz5sxh7NixRY5x+PBh7rrrLgAmT57M8OHDQ5LhzTffDNhmyJAhfPfddwHblS0b/v7IeEApKB1Il1UBqCVOxFiS4NApX7BCUewoU6YMW7du5fx5117GBQsWUKvWhSxrvXr1YuTIQoUZ8lGzZs2glIc/glFQFwtKQelAvCgoKSV6ZatXKEoqt9xyC3PnzgVg+vTp3HvvvZ5r3hbRkCFDePzxx7n66qupX7++RymlpqbSrFkzT59Dhw5x3XXX0bBhQ1577TXP+T59+tC2bVuaNm3KZ599BsDIkSM5f/48rVq1YuDAgQBMmTKFFi1a0LJlS+677z5P/z///LPQ3MGQkpJCx44dadGiBXfccQcnT54EYPz48TRp0oQWLVpwzz33ALBs2TJatWpFq1ataN26NWfPng3+g9QApaB0IE9B1RQZMZXjns9WU+/53wK2kyglplDkcc899zBjxgxycnLYvHkzHToUynHt4ciRI6xYsYJff/3Vr2W1du1avv/+ezZv3szs2bNZv349AJMmTSI5OZn169czfvx4MjIyGDt2LKVKlSIlJYWpU6eybds2xowZw+LFi9m0aRMffPBBSHP74v777+ett95i8+bNNG/e3KM0x44dy8aNG9m8eTOffPIJAO+88w4TJkwgJSWF5cuXU6pUqaDn0QIVZq4DGZTHKQXVxKmYyrFmf2ZM51cowuW1X7ax/bC2kbBNapbnldubBmzXokULUlNTmT59OrfcckuRbfv06YPBYKBJkyYcPXrUZ5sePXpQpUoVAPr27cuKFSto164d48eP58cffwRcVtbu3bs97fJYvHgx/fr1o2pV10Nv5cqVQ5q7IKdPn+bUqVN07doVgMGDB9OvXz/P+x44cCB9+vShT58+AFxzzTWMGDGCgQMH0rdvX2rXrh3UPFqhLCgdcGAkg/LUJLYWVLAoL6BCkZ9evXrx9NNP53Pv+SIx8UJaMH/u9IKRa0IIli5dysKFC1m1ahWbNm2idevWIW9QDmbuUJg7dy6PPfYYGzZs4KqrrsJutzNy5Ei++OILzp8/zzXXXMPff/8d8TyhoCwonTggq1PfcCTWYigUxZJgLB09efDBB6lYsSLNmzdn6dKlEY21YMECMjMzKVWqFD/99BOTJk0iPT2dSpUqUbp0af7++29Wr75Q/s5sNmOz2TCbzVx//fXccccdjBgxgipVqpCZmZnPigqVChUqUKlSJZYvX861117LN998Q9euXXE6nRw6dIhu3brRuXNnZsyYQVZWFhkZGTRv3pzmzZuzbt06/v77b6688sqIPo9QUApKJ/Y7L6WzcWvU5tt3PItnv9vM5AfbUzYxtD+rMqAUivzUrl2bxx9/XJOx2rdvz5133klaWhqDBg2iXbt2NG/enE8++YTGjRvTqFEjOnbs6Gk/dOhQWrRoQZs2bZg6dSovvvgiXbt2xWg00rp1ayZPnhz03NnZ2fncciNGjODrr79m2LBhZGdnU79+fb766iscDgeDBg3i9OnTSCl5/PHHqVixIi+//DJLlizBYDDQtGlTbr75Zk0+k2ARJSXKK17qQQEkjZzL66ZJ3GdaSJOcSWRj0X3OW5pfym9b/mHCgDbc2qKGRw4gYPLX9FPnuWbsYs+xShariAU7duygcePGsRZDoTO+/s5CiGwpC2c4UGtQOjC0S30yKQfAVYadMZYmMMXxIeX3rUdYubd47DNTKBThoRSUDjzX80qSja0A+DrhrRhLUzIZ9u0GBny+JtZiKBQKHVEKSgeMBkF6uZYAHJHhL2iGw4Lt/zB7/aGQ+hRDA0qhUFwEKAWlE8JgYJq9GybsUZ33p5TDPPPd5qjOqVAoFHqgFJROSClJk9WoJs5QhvOxFkehUCiKHUpB6cguWQeAhiJd97kicdMpF59CoYhHlILSkT2yJgANDBcUVEfDdhKwxUokhUJRBEajkVatWtGsWTNuv/12Tp3SPl3ZJ598wpQpUzQb78SJE5jNZk/+vJKEUlA6ckheglWaaeC2oP7P+AMzEsawMXFoVOUIlNNMJYtVKFzkJWrdunUrlStXZsKECZrPMWzYMO6//37Nxps9ezYdO3Zk+vTpmoxnt0d33bwolILSEQdG9slLaSDSucGQzFNmV0r8MsIaVTluGb+cE1nBz2m1O3SURqEoHnTq1In0dNfD5XXXXefJQn7ixAmSkpIAV/mNvn370rNnTxo2bMizzz7r6V+2bFlefPFFWrZsSceOHT0JXV999VXeeecdz7jPPfcc7du354orrmD58uWAKwNE//79adKkCXfccQcdOnTwzF+Q6dOn8+6775Kenk5aWhqnT5+mbt26OJ1OAM6dO0edOnWw2Wzs3buXnj170rZtW6699lpPbr0hQ4YwbNgwOnTowLPPPsvatWvp1KkTrVu35uqrr2bnzp0B5frjjz/o1KkTbdq0oV+/fmRlZUX8N1AKSidubeFy7+2Rtehu3MiXCe/qOl+gSsrZ1vxK50DGOc8G3YJrUN+sOqClaApFscPhcLBo0SJ69eoVsG1KSgozZ85ky5YtzJw5k0OHXNs8zp07R8eOHdm0aRNdunTh888/99nfbrezdu1a3n//fU/pi48//phKlSqxfft2Xn/9dZKTk332PXToEEeOHKF9+/b079+fmTNnUqFCBVq1asWyZcsA+PXXX7npppswm80MHTqUDz/8kOTkZN555x0effRRz1hpaWmsXLmS9957jyuvvJLly5ezceNGRo8ezQsvvFCkXCdOnGDMmDEsXLiQDRs20K5dO957770gP23/6JqLTwjRE/gAMAJfSCnHFrg+AvgXYAeOAw9KKQ+4rzmALe6mB6WUgb8pccR/bmjIQ53rMe0N32b3dYYUljpbRU0ebwW2Je00t3+0glG3NeHBzvUKtbU5lMtPEWPmjYR/tgRuFwqXNoebiy7XnlcsMD09ncaNG9OjR4+Aw95www1UqFABgCZNmnDgwAHq1KlDQkKCp1R827ZtWbBggc/+ffv29bRJTU0FYMWKFTzxxBMANGvWjBYtWvjsO3PmTPr37w+46lg9+OCDPPXUU9x9993MnDmTbt26MWPGDB599FGysrJYuXKlp7wGgNV6wbPSr18/jEYj4CrLMXjwYHbv3o0QApvNVqRcq1evZvv27VxzzTUA5Obm0qlTp4CfXSB0s6CEEEZgAnAz0AS4VwjRpECzjUA7KWUL4DtgnNe181LKVu5/xUo5ARgMggqlzFwpDnrOfefowjM21/rT5IRxJGDjl4QXeMo0S3d5DmVme14fyHTlLEw+4KqkqdSRQuEibw3qwIEDSCk9a1Amk8njMitYFsO77IXRaPSs4ZjNZk+pDe/zBcnrX1Qbf0yfPp3JkyeTlJREr1692Lx5M7t376ZXr178/vvvZGZmkpyczPXXX4/T6aRixYqkpKR4/u3YscMzVpkyF1Lhvfzyy3Tr1o2tW7fyyy+/BCwFIqWkR48ennG3b9/Ol19+GdJ78YWeFlR7YI+Uch+AEGIG0BvYntdASrnEq/1qYJCO8sSEE7KC5/XTtmEAvG12lXf+O3EIBiFpbkhloHEhbayfhT1PoFDxAV+sYfrDHel0eRUErh+N02/9mrDFUCi0IYClozelS5dm/Pjx9OnTh0cffZSkpCSSk5Np3759SOXVw+Waa65h1qxZdOvWje3bt7NlS2FrcteuXWRlZXnWyQBeeeUVpk+fzqhRo7jqqqt44oknuO222zAajZQvX5569eoxe/Zs+vXrh5SSzZs307Jly0Jjnz59mlq1agHky57uT66OHTvy2GOPsWfPHho0aMC5c+dIT0/niiuuiOhz0HMNqhbgnXMnzX3OHw8B87yOLUKI9UKI1UKIPr46CCGGutusj6fIE2+et/+LftZRJOVM85wbZ3OZ5AZxQUFUFlnUEcFVxQyX3cfOAhcU0Lyt/7Dn2NlCyWLVviiFAlq3bk2LFi2YPn06Tz/9NBMnTqR169acOKF/kuJHH32U48eP06RJE1566SWaNm3qcSPmMX36dO6444585+68805PNN/dd9/Nt99+y9133+25PnXqVL788ktatmxJ06ZN+fnnn33O/+yzz/L888/TunXrfFadP7mqVavG5MmTuffee2nRogWdOnXSpLihbuU2hBB3AT2llP9yH98HdJBSDvfRdhAwHOgqpbS6z9WSUqYLIeoDi4EbpJR7/c0XT+U2vMkreeHNzYY1TEz4AICTsiyVxIVoF29FFgo3N7uUeVv/8Rynjr210Nyjezfl/k5J3PvZalbtc1X7bVG7Av+7uxU3vLvM0+6eq+ow5Jokrry0fFiyRINgS4koig+q3MYFHA4HNpsNi8XC3r176d69Ozt37iQhIaHYyxVKuQ09XXzpQB2v49rucwUF6w68iJdyApBSprv/3yeEWAq0BvwqqOLEPGd7z+s2Vtfmuv0Wl3fThJ3SWDlDob+VZuQpJ3/MWHeIGesOqZu/QhEjsrOz6datGzabDSklH3/8ccyVUyzk0lNBrQMaCiHq4VJM9wADvBsIIVoDn+KytI55na8EZEsprUKIqsA15A+gKOYI7rS+winKIt1e1mn2bgwwLeG7hNdoZdjLkNxnWOpsrfGshZFSufQUinijXLlyfvc9xZJoy6XbGpSU0o7LbTcf2AHMklJuE0KMFkLkReW9DZQFZgshUoQQc9znGwPrhRCbgCXAWCnldkoQybIRe+WFJblvHa5w1lYGl5E4OeFtzee02p289XvkfmGFQqGIBrrug5JS/gb8VuDcKK/X3f30Wwk011O2eGOXrK37HN+uPkBqRraPK8qEUsQHUkpPaLai5BFqzIPKJBEn2L2eFb613wDAi6ZvNZ3D1wZclYdPES9YLBYyMjJCvokpigdSSjIyMrBYLEH30dWCUoTGTdaxVOQcZUU2g1jEw6bf+NDehzOU1XVedT9QxAO1a9cmLS2N48ePx1oUhU5YLBZq1w7eW6QUVByxU17meuGlMTZbhoYdeh4MwmfohEIRfcxmM/XqFU69pbh4US4+nSmTYAyjl+Bu68ueo+sMG0Me4ZtVqYXO+XKdSOXkUygUcYpSUDoT7s1/jWzMR/beADQQh0Pu//LP28KcWaFQKOIDpaB0JhIH2vv2O3FKQTlxXjN5fKHWoBQKRTyiFFQcY8fEMSpSg6IzP0SClCqST6FQxCdKQcU5/8jKXCoyYy2GQqFQRB2loOKcw7IKNYV+FhQoF59CoYhPlIKKc/6RlakhMggUbuGdydwfSg8pFIrihFJQOhNp2pbDsgplhJXy6FdKRFlQCoUiHlEKKs45LKsA0NGwI0BLhUKhKFkoBRXnrHY2AeBD80e6jK+i+BQKRbyiFFSck4mrqm0O5hhLUhiHU7Jyr/7lrxUKxcWJUlDFgD8cbT2uvkjwt9YU7hrUR4v3MODzNazco5SUQqHQHqWgigEnZAWqitOxFqMQ+05kAXDsrDXGkigUipKIUlA606J2hYjHOEF5KnMWA86IxvnnTE7EsigUCkW0UApKZyqXSYh4jBOyAkYhqcRZDSQqjAozVygU8YhSUDqjxb3/hHRZYdV0cPMp3aRQKOIVpaCKAf/IygBcqlPKIxVmrlAo4hGloHRGi3q1h2Q1AGqL2EbLZWRZufLleSQfOBlW/1E/b+WZ2Zs0lkqhUJRUlIIqBpygAlZpprY4rvnYUkoe+np9UG3X7s8kx+bk8z/3hTXXlFUHmJ2cFlZfhUJx8aGrghJC9BRC7BRC7BFCjPRxfYQQYrsQYrMQYpEQoq7XtcFCiN3uf4P1lFNPtHCeSQykyaq6KKizOXaOhxgmrlyCCoUiGuimoIQQRmACcDPQBLhXCNGkQLONQDspZQvgO2Ccu29l4BWgA9AeeEUIUUkvWfVECxcfQLqsSksRnuVS5Lingq/WG2HeW4VCoQgJPS2o9sAeKeU+KWUuMAPo7d1ASrlESpntPlwN1Ha/vglYIKXMlFKeBBYAPXWUVTcizWaex1lKUcdwPOK9UFqgwtIVCkU00FNB1QIOeR2nuc/54yFgXph9iz21KpYq8vo2Zz0AqhDLjBLKhFIoFNHDFGsBAIQQg4B2QNcQ+w0FhgIkJES+IVYPZJDmRjlL0X+KPbImANXFSY7LYuntVCgUipDQ04JKB+p4Hdd2n8uHEKI78CLQS0ppDaWvlPIzKWU7KWU7kykudK1unKU0AJeLwzGWJDRSDp1ix5EzsRZDoVAUQ/S8q68DGgoh6uFSLvcAA7wbCCFaA58CPaWUx7wuzQfe9AqMuBF4XkdZ455U56UAVBJZMZbEhZSStfszA7brM+EvAAzKO6hQKEJENwtKSmkHhuNSNjuAWVLKbUKI0UKIXu5mbwNlgdlCiBQhxBx330zgdVxKbh0w2n2uxCKE4MVbGvu9fhSXrq4YBwpKAnM2HebI6eCTzzpVYIVCoQgRXf1iUsrfgN8KnBvl9bp7EX0nAZP0ky465NqDj7p7uEt9Zq0/xO5jhZWQEwOnZWkqcE5L8ULCOyDxQEa2/4YKhUKhASqThM7c0rxGSO2Liko/LctQQcROQYXKhoPhpURSKBQKUApKd3q1rBlUu4J6qWP9yoXanKQcVSg+AQebD52KtQgKhaIYoxSUzoS7T/dfnesXOvePrEwNnTKaB4PN4XJXqo26CoUiGigFFScEo8jM2LnCkE4iufoL5IPh0zbGZF6FQnFxohRUnBCMgjLhAKCNYbfO0gRCmVCK+ONQZjab05RbuSShFFScIYpIJ/SGfSBAsVqHUiiixbXjltDro79iLYZCQ5SC0hkhBD2aVNdkrDRP4ULty24oFApFvKEUVBToWL9KwDZFWU55nKMUmbIsdWKsoPYdPxdUoESkjsDzuQ4OZLjC6n9OSQ+7km8gkg+cxKl2EisUcYdSUHGKv9vlIXkJdcQxP1ejwz9ngs8gES5Hz+TQeNTvdH17KQBPzEjhzokrNZ9n5d4T3DlxJZ+GWSVYoVDoR8nOsFoCOSgvoaXYG5W59hzLokYFCwczszmRdaHqrpT5gzrO2xxY7Q4STUbN5n7km2TNxiqKw6dcynb3sbNRmU+hUASPUlBxQt4NP+OcSxGctzl8tjssq9DDEJ2bd/f3ltGxfmVW78ufBlEi+WL5BYvj+R+2MGHJHlY8d71mc585b7swn9p4pVBclCgXX5yQZ5BUK2cB4FS2771Op2UZLMIWtb1QBZVTHmdy7PmO004GXzo+GKKukpQOVCjiDqWg4oy372pBvaplaFvXd1HCmu5MEg1FWjTFiil6GlCqCohCEb8oBRUnVC7jqgjcrFYFljx9HeUtZp/tNjobAnCJiN2GROVxUygU0UApqDjhf3e3CqrdRtkAgHLErtyF0k8KhSIaKAUVZR7qXM/n+YqlE4Lqf0JWAKCqOB22DG3ELi5F36SzTqfkq79SNRlLKUSF4uJERfFFmW6NLuHLFfvD7n+G0liliWoivHRH1TjFD4mvAtDdOo49snbYsvhi7Ly/STQZ+GBRrPMFKhSK4o6yoOKA7//dKei2j1/fEKu5PG0NOwO27WNYQWNxIN+5+YnPel4v9HqtBTk2B58s26uJcvIOLY9GmLmy0hSK+CMoBSWEKCOEMLhfXyGE6CWE8L2KrwiDwrFk5Uv5/nhH3NiIs4mXeqL5/GHCzvsJHzMv8fl85yuLwuXkQ8bP3TzXEXx5+3gh3HpdCoVCf4K1oP4ELEKIWsAfwH3AZL2EKmkEsgAurWApdK5CKTMbXu7hM9w8rVxLqnIGX5qiGie5hJP5sk1Uw5XD7jJxFIBfHR0918zk388UDNKPhtLL0NHTulERiQpF/BKsghJSymygL/CxlLIf0FQ/sS4ualUs5fN85TIJGH084p9JuJREYaMy+dPzmLCzzvIYay2P8X3ia57z/YzLAPgz8UkALhWZvGR7AIABxkWavAeFQqHQmqAVlBCiEzAQmOs+p13itRKO8FIyEsltLWow6rYmQXYufOqkxRXYUE8cyXf+UeMcn0M8a57FIOMCz/EH9r7Md7QD4BrD1uDk8MKf1THm1+0hjxXJfFqgXHwKRfwSrIL6D/A88KOUcpsQoj6wRD+xShYFXXwfDWjDg37CzYPhRClX34aGdM+5V02TGWH+zm+fMeavAHgqdxjLnS04jst1eKNRu7x+s5MvnuwWCgXAsbM5pJ/SNs2X4gJBhZlLKZcBywDcwRInpJSPB+onhOgJfIDL2vpCSjm2wPUuwPtAC+AeKeV3XtccwBb34UEpZa9gZI13LObQDE9fD/inE6pjk0ZP2Q2BkyGmPzzXr8z5ikpkUVmc5UpxkHcTPvFcW+Fs5nm92VkvYLCFL6KxbBPtpSGVkFYRDu3fcLnIU8feGmNJSiZBKSghxDRgGOAA1gHlhRAfSCnfLqKPEZgA9ADSgHVCiDlSSm8/0EFgCPC0jyHOSymDS69QTCidYKSdnxx7/vDpghJGjsjK1BInABhtmuy5dG/ui+SQyBESOSKrsE3WpYztPHMdHcmgPN4qb5GjDU+av6cCWZymbBjvKDr4C8rQAuXiUyjil2BdfE2klGeAPsA8oB6uSL6iaA/skVLuk1LmAjOA3t4NpJSpUsrNQPGLTw6BvDWou6+qk289Kqi+vmwo4Sr/XtutoPLy8l1vfYdVzoKxK4IpjpvIoAIF7bFkeQUAH5o/DEmmWFobmee0zeKuDKfoMnPdQd5bsCvWYiiKCcEqKLN731MfYI6U0kZgL0wt4JDXcZr7XLBYhBDrhRCrhRB9fDUQQgx1t1lvt4ceLh0t9LihuxSUq/T7Tcb1AOyTNUMaY5XTFaiRTeEw93jC++Nr8/oCzubY/DdWxDXPfb+F8SrLiCJIgk119CmQCmwC/hRC1AXCy7UTPHWllOnugIzFQogtUsp8pWSllJ8BnwGUKVMm7p+FfVpDgfr4NKAEabIal4qTESWNdWBkv7M6ucUs41WWVbuHEX8Grc3hRAAmo0q2olDEimCDJMYD471OHRBCdAvQLR2o43Vc230uKKSU6e7/9wkhlgKtgejUOteJgmspsx7pxKIdR4vs4+8GmiarAjDQuBCABY62Ycl0nIpUI7TEs1EJkojh48a7f+zkw8V7uLS8hdUv3BA7QRSKi5xggyQqAK8AXdynlgGjocg72zqgoRCiHi7FdA8wIMj5KgHZUkqrEKIqcA0wLpi+8Yi/daf29SrTvl7lkMcrnWAkTVYDYKR5BgAf2n16QQNSnZPUNR6DELxmJXHdxvstfbh4DwD/nMmJjTAKhQIIfg1qEnAW6O/+dwb4qqgOUko7MByYD+wAZrn3UI0WQvQCEEJcJYRIA/oBnwohtrm7NwbWCyE24dpvNbZA9F+xIpI1KF9uQYOAA7J6vnOpBY6D5RgVASgbw/pSgXA4dYziUzV1w+aXTYdJGjmX0+fVmqBCH4JdfLhcSnmn1/FrQoiUQJ2klL8BvxU4N8rr9Tpcrr+C/VYCzYOUrdig1RoUwFEq86ztYcaZPwfgTJhh4l/ab+GqhF3UFcfYJpPCGkNvPl66J9+xlhacniHsJZ1P/3R53A9mZNO8doUYS6MoiQRrQZ0XQnTOOxBCXAOo7dMhovXNcJYj0DJgYA7KS4ALiWSjyaHM7KCevicsKdZLjwqFIkyCtaCGAVPca1EAJ4HB+oik8CbQvqkrcr5GRuCmynMV9jMuY56zQ9jjhMO145ZQvXwia17oHlI/LdW8cvEpijvJB05iMgha1qkYa1E0J9govk1AS0rK0zQAACAASURBVCFEeffxGSHEf4DNegpX0tDjZphLZGW5zuHKpH69MSWkQAmtOHrG6vO8cr0pFMFx58SVQMlMtxTSJg8p5Rl3RgmAETrIoyjAHa1D23wbDludSZ6w9eKAHhuf/Q3Z+6MVms2RfCCTb1alajaeQlHSiWQXovKNhEg4VsEdrWvr/mS0U9bxpE2KJckHMj2bcOMllH1TWmh7xIrizomrePnnbYEbKhQKIDIFFSe3EEWkVMOVy68msVFSVruDU9m53DlxFY9P3xgTGVTSWIUi/ihSQQkhzgohzvj4dxbQ3/dUwtBqDWpAh7p0bqCdS+4bRw8AqouTmo0ZiN1HL1QDfvePXeTYXPmCtx12WSxFKQw9rKt4sdgUCsUFilRQUspyUsryPv6Vk1IWrwRuJYjKZRL49l/aRdwdllUAqObOiq43OTYHPf73p+f4+NnCgRJaK4zT521sTS/srlOWU/gopa7QG5UJU8Fx6QpPrSa0W28pCqvdf3UVvW569325hts+1C7goShOZ9tIPpAZlbnihX3Hs7j3s9Wc0zCRr0KhFFQU6NWqJklVSjP46rphjzHrkU68cMuVGkp1gQzK45TCU1dKb2JRT2pzgGAHLSW6f9Ia7py4StcUTfGAt/U5dt7frNqXwfLdsQ+2UZQclIKKApeUs7D0mW7UrVIm7DHa16vM0C6XayjVBRwYyaCcJ1gilgTjcvPWbwu3Rz8DRiC2+HAlFldavDqfnu//GbhhCSXtZDY5NkesxbhoUQpKAcBxWSlqa1BaVsWds+mwJuNouRRVkuymMzl2/v7nbOCGHkrOu7c7nHR+awlPzIhNZKlCKSiFm52yNm0NuzDgf31IK+Zt/cfvNX+ZJfyhVSZtLW+reRZeSY+/KOlBEg73G1zy9/EYS3LxohSUAoCVzqZUFlkxSRrrq4R7UTe/lXsvrHMs23Wcf07HZ92mWesPcTBDuzImp7Nt2B36P0AoFPGCUlAKAP52XgZAI3Eo6nMv3HEspPYjf9iS7zhe13xG/rCFOz7+S5OxbA4nLUf/wUs/bdVkPC3wXi8syeH6Ki9k7FAKSgHAAXfZjWikPNI6iu/hKetD7vPYtA38vvWI51ivyMIMjdbbct2h+T+n+F9zszucWO3+F/TTT53X9H2es7rmWrXvwnempLv9FNFFKSgFAGcog1WaorIXKh6ir+duPsKwbzfEWoyQKcpSueuTVTR66Xef17YfPsM1YxczeWWqZrLsP3EOgHf+2KXZmPFIwQwwBzLOsWpvRoykubhQCkrhRpAuq0ZlDWrJztBceuHy8JT1NHppXpFtzriDLALV3Yo1wej0lEP+ozAPZLiUyZp9F9cGYi0o6OLr+vZS7v18dYykubhQCkrh4aisTNUoWFBWW3QW+hdsP+oza8Xe41me12Pm7gBCc/H9sukw9325JnIBw8Bbjc7ZdJhUtxUTLHqtpwSTZ7K47SdSxSxjj1JQCg8nKUt9cSRwwwgxGWP7w7/h3WWe17YwouL+b/rGqGdM8KVAH5++kZt8bKKNuvtJ+nxZiHf/2Km7KFqigiPgp43pfLhod8zmVwpK4eE6wyaqijMk6Fxa1xDAnbZ8t7b7To6d1S8MPe1kNmd8hMnrRUFXpC8LMV7dT1oFjEQbrS2p/p+u4ocNaZqOqQWns22FNtH/Z2YK7y6I3RqjUlAXAeYgLZbJjpsAqCH0fQKvV7XolE/3fblW0/nav7HI8/r6d5eG3P9Elv/Nw53fWsLN7y8HXC6spJFzgxozaeRchn2THLQMBZ/lNx4MrTRKnC+x+cTucPLqnG26PmAEg5aW1Mx1B1m7P5MRszZpNqZWtBz9B21eXxBrMfKhq4ISQvQUQuwUQuwRQoz0cb2LEGKDEMIuhLirwLXBQojd7n+D9ZSzpHNJOUtQ7ZY7mwNQy2+oueRmwxqqcYpFCU/xb+OcQi1aij10NGwvcp4KpcxByaMH+47nX7PJiygsKrFruzEL84WkFyT91HkA/toTmtvv923+M2r4I0/PLNkZ3ewGJ7KsfLBwt9+1umAUYKiWyNKdx5m8MpWXY7z3S0r4Y9s/ODUIP522Nvr7DLXm55R02o1ZGJVN47rVdBJCGIEJQA8gDVgnhJgjpfS+ex0EhgBPF+hbGXgFaIfr4THZ3Td6FfVKEBZzcM8hparVg9PwgHE+K53N8l3ratjE1wlv5Tv3nGEG5UQ2bQ27GG27nxaGvfzX/CUASTlTCSXZz7lc7co0HMrMxmAI7Wa448iZIq/PWp9Gz2Y1imzzro7h1rHeX/T07E0s3XmcqxtU4aqkylGZ0+l+07HOCm93SoZ+k8zo3k25v1NSTGWJB176cStnrXaybQ7KG/V1wulZdLA9sEdKuQ9ACDED6A14FJSUMtV9raAqvglYIKXMdF9fAPQEpusob4ll0pCr6Pr20oDt3n/kNhgHPYzJ5F+GkoWUUx6PmlxW1NzEF/Kdry1OkCarBS2jd+ACRLZx9tpxS0Luo8UtUK/b6J5jWXR/z/35ROiqC/djzc51ReD5UhYXSzDBkSBSap3OtnE8y0qDS8r6bhDrJw0NicZb0VP91QK87dk09zm9+yoKEGyZDykuPK+Uw5VDrrXYTaploOf8TdaxXJ0znr7WV4scq69heeiCxpBo3jdCVb6LdsRBSZEgRS7qrXm7AZNGzmXjwZPsPZ4VcwtJS3pNWHHhYaKkEsX1zGIdJCGEGCqEWC+EWG+3q0qekVLeYmKX0/UckCRc6yNDTPM912+1vsFOeRmHqcpmWd9zvkHOFG60vsV7tru4Iudr1zlDut95dgZRvuFwnCWADaRUXp2zDYczOJ98UdncAxGvsQ7hBGFMWXWAG95dxjt+ws/zIhZjZXSEM++BQMmBi2O0SgzRU0GlA3W8jmu7z2nWV0r5mZSynZSyncmkp7fy4kAIwdO2YQBcKjKpQQa9jSsBuMs6im2ynqetHRNJOdNIypmGHRO7ZB3GO/qSi5kTsjy9jKsQfkp3rNqnfZRgtLJT+GPyylR2Hc3yec3plMxef8EhcEajEiGh4boxrtqXwYpw9nAFuK+GczM/esb1ELI+1Xd2i681TMsUN5QgF1800FNBrQMaCiHqCSESgHuAwmFfvpkP3CiEqCSEqATc6D6n0JiCIej7pSsQoIk4QGfjhazh62Xw5eY3OBsCUE+EbymEygNfrYuof6B1lEhuK7eMX84z320O2C7zXG7A8hyRpmQ6m2NnUDhZMIr4APS6564IMSqyKKx2B/9bsCuibBahvM/ngvh7F3uK8xqUlNIODMelWHYAs6SU24QQo4UQvQCEEFcJIdKAfsCnQoht7r6ZwOu4lNw6YHRewIRCW+5sUzvf8VlKA/Ck+XveNn8G4HHbBctb9nsAGGmKbkxLtGsl/bkruFDvYCvSdh23hC5vL2Hk98Xv5hZcmHns+GbVAT5YtJvP/9wXlflmrvcTTq5cfCGh6xqUlPI3KeUVUsrLpZRvuM+NklLOcb9eJ6WsLaUsI6WsIqVs6tV3kpSygfvfV3rKeTETzFNhLqHtW9orawJwozH4jaha0POD8AMzwrEC7p8U3oZif1OdtbrWUWesO0TayWySRs5l5rqD+drk3d/GRzv9TJD31TxL9JvVB1gWpALXmrmbC+9ZO++OQswpohxJIDTRLcrFFxLFOkjiYufPZ7rpMm4f62g+sPcFYJr9+jBGEMxzXMVBZzWi4gdws+eY7zWgYIjdQrzviTu/5QqVn7h0b77zp7JtQbupjp3NYcZal4KL+OYa4ufz8k9bGRymAo9wah6bVriMSt4YkaQt0us7Mnv9oaDTe508l8voX7Z7ckjmreNFk7xPMBrbC1RkQTHmsiqlIx7D15csRTYgxd6AD+x9cYb5g17rvJKbzetoJ3aGtH4VKXplzNby5uQ91vcb0rmrbW3/jX0Q7E1p6JRkUg6dYuvh03y7+mDgDkEQ6Ntw/KyVbD+bro+ezZ8yamWUktr+nJLuCaIJpKinrjlAs5oVaFmnYhQkc5G3Ppk69taAbcfM3cH3G9JoWacCl1UuzR0fr9RbvEJEszSNsqAuMuoHyIPnjRMD4a4crHW6lNLjph/D6h8uPf5XvPagpJ0MEJbsg2A2XcOFHIJaKSfwbc1ILlgmr/2ynbsmrvLZN9g1O615YkYKGw/6r5XlzYs/bqX3hL90lih87O6tDE4p2XU0uLXN4oxSUApd2CbrkSUtnCO4PIBacSjzfFj90k+dJ6OIpLDxvHJwzmpnwpI9hc7r+aBblLLZ7pU2asbag3QJkNkj0mzh36xKDTpJbygzFbSaf9l0mE+W7fXdOEg2pUVWby1P/lV7M9h7PLRaYMURpaAuNgr8QvVce1nlbMrNxnV+90PFG/+d93dEKZb0IDUjO2AU4LBvk3l7fmi1lpJGzsXqI2DgUGZ2kTWy8r4+s5ODKxcx8octHMwM3UoMhbyik0ERgdZOP3WesfP+Drv/3//kz/e4NT18ZTVrfRqfBRGRmDRyLi/+uCVgu3Ao7qmOFHFIoJ9niDlWiyTZvR8qKQpl5LVAyuj86EJdXP5xY9H729f52Oj6zarUgONmWy8oqMxzufy6+TDXjlviyR6efMD/zo5oBkvvOnqWNUVs7tbCUpy1/hA/p1z4nHN91NmKlIKVpG/7cIXmc/hi6pqDPDN7U5F/z1CIZqS8UlAXOQVvldtH99Rs7NXOJgB0MITwhBtDJNJv7SctLSutlaCv8V7+eVtI7s77J61h+LSNgCvMPWnkXO70s5YEPm5SOt600k6e5+7PtCnC6E/MZ7/bzBMzUjzHD0zWtiaZVoQboDA7OY37Na6zFg2UgrpImDm0o8/zUkLTmuV5/AaXtWMxGzWbM28/1FjzF5qNqSc/bEin/ZuLAjeMkF82Hfa8fn/hbh6bWjgsOhS00He7/aRpigZrUzM9DwB2h1PX5LHB3t//2pNR4rK0F8d3oxTURUKlMgmA7yewuY9fy4geV2g+Z15WilMy+MjBi4E1+/O7WuZu8V8MMRjCdUeFcsMqeLP2/hY5NTAJ89yYTV+ZT+e3Fvts8/b8v31uwtW6JLseZGRZdY+623s8i3ZjFvJPlBItR0PhKQV1kVDKbRklFSi9ofdT4kf23pTlPInk6jqPQl/WpeavFer9oKOFesirtWS1O/3WXZqwZK/PTbiheL1ipcyuf3eZz3yMU1alhjROUdJ/s+oAJ7Ks/BbhA088oRTURUKdyqX5cnA7/nd3y6jOu8NZF5NwcrvR/3pGcWDb4aIr7hZHvG92kT6mxKMN43RKVu7Nn3A2FGUWqmF4Ott/lvrTfjLYT9Vwj1oexdGV5w+loIo5phDC7m5oXJ1yltDy6kXKWmcjAPoZI99Aa4hhuHrmOX0tQH83sHglUGRhNPH3C5j0134GfB5G5vYwGb84yvkRCxBI+WoVnONJdRSFkFeloIo5j153ech9Fj3V9cKak87fseNU4k9Hc8oS3gZagNGmr0i1DGCfZRBLEp7UULr4oeVrf8RahLCJ9RO7v8i2/ScKb2TV09Lzdb8+ftbKXxqWDQnmDUSqODYdOuVz64JHBJXqSBEseV+WJ9xReMFwebWy1KpYSi+RCrFfXkrdMPZCNRWp3GJYzf2mBZ5z9QxHKROBslPEhlPZ2lqgr/y8lYenrPd7XUpJ+qnC3xM9762+xr5z4koGfhEdK06r9bXeE/6i3yfx4ZJXCqqYE+nTazSefo/KSpQVOSSJ0BZv5ya+wMcJ4z3HvzuuAuAKEVwWA0X80Gr0gsCNQuDrVQdYsN310OPrtvzjxnSW7ows958Wv41Qsmis0ih5bsqhUwGtqONnrSSNnMv3QWYEiRVKQV2k5D3tRcOPvNzZAoAOBt9pYi4X6Txpmk0Xwybqi8OMNn1FR8P2/G1yvuF12yAAfkx8hVg4lhxOyXsLdkV9Xr149vvNHD/rP/9gNMnOtRdKBRQ0PjTUpkO+k8MWdE89PGV90Hn8QhUjmETA3iH6936+mulriw6aCMZK+nXzEaauKTyOd8Runvtz+tqD7DhyJqxin9H4BapyG8WceIyeKsgWWY/TsjQtxD5mUriG1aLEZwqdu58LT9xT7D1wYCSdap5zvQyrmOO8Wh+B/bBwx9HoFwrUkQXbj1I20cS97S/TJbVPqPR8P/yCkwXxt07y9vydNKpeju5NqgN4rDBfPDN7U0QyBJN1/p8CpVMmLNnDve0vC2s+74jFYGuj7T2exc0fLOeRrvXDmlNvlAVVzPH3FLNwRFcAbnT/EGOLYI+sRTPD/kJXaoui3TBDcp9llP0Bz3GHnI8AGJ/wkbYiBkFRSVSLM/0/DX69wdeNX0rX/qVIiCSK0ZcqKmqt6aeU4CIQ5239JzQ5CsypRUaMQV+s4YqX5gXVNlBSYV+cdIfGpwRZjiTaKAuqhFDwx9HgkrJ8N6wTTWqWL7J9tBxlO5yXMcC4mFLkcN5dgsOEnesMrvxnd1hf4xRlkcBhWZXbDKuoLU6w1Jl/39ZRKrPXWYPLDUeoximOE73CcnGW6DyuKMoSCYZIalb5UpqGaGY0jYQA36kVYUYABnLdR/LxRPOTVRZUCUFK+G5Yp3zn2iVVpnSC72eQaO+o/8vZDIOQ1BP/kICN501T2WO5nzHmrwDYKeuwX9YgVdYgFzM/OLsw3tEXXz+HJ2yPAfCsaUY03wKT/ipsARZ34mUNKhTunBi4iqxRy7T8QaJn+PWPG9NIGjmXzHPa/L0KSuqzEGUAJReNBzZlQZUg2iVVDrlPtKyCvMSxnQ1buJYtPGLKvzCdHUJhw62yHgAJwndpcb2IZUJVvQj3CT2WJB84GbBNUbWS4skQDlaWPAvT194uvZi+9lDU5vKHUlAlhFAf3qLtATkgXWthL5in5zt/XFZgir1HiKMJdjprk0jxyr6gKFloVV/JG38PjMEo5VDGDub3vznN97pUhjurSjSyvSsFdZETradJKwkclRWpLlxf+jXOK3nFNoS/ZXgRS+myalibfyMhyxpdi00RmN+2HIn6w1YeRdXLCpf0U+eZue4gd18V3u9CSwJ+rsW9oq4QoqcQYqcQYo8QYqSP64lCiJnu62uEEEnu80lCiPNCiBT3v0/0lPNipGJpV/mNmhWCd61Fymf2WwH43tGZu3NHha2cAHbLWjQ2HKQ80XN5KCDLags6hDka7D6aFdZq6q+bD2u2/ymPcOTw9dDz3PdbcEYYARhybx8ddoYRFag1ullQQggjMAHoAaQB64QQc6SU3jswHwJOSikbCCHuAd4C7nZf2yulbKWXfBc7XRpWZcKANvSIYhj6l45b+c3RkSOEvlZWkK1O1zrU9YaN/OTsHPY4fQ1/8l7CJ1xr/R+HZDyE5Mc3D072n16oODFx6d5Yi6ArU1YdYPb6yLJEbPAKPT+QcY66hUr16I+eFlR7YI+Ucp+UMheYAfQu0KY38LX79XfADSKamQhLKPWrBi4QKITg1hY1SDD5/wqM7t1US7EAOEIVtAhUne90pT260hBZuYL3ElzG+fLEJ3nL9FnEcim04asgIyZTM86FvodKrztrnN25ztsc+Y79l5wMvJ4Uq4AJPRVULcD7XaW5z/lsI6W0A6eBKu5r9YQQG4UQy4QQ1/qaQAgxVAixXgix3m6/ONcHzO5w2ryyGzOHdmRWgXDzcCmlYfl3rcnFzAZnA9oYws/sUI38i8B3m5YWOqeIDa/9sj1wI1w590L1hs3dcoQDGcHnyIsF/t5SJLrVO1uIFmZANCKA43Uf1BHgMilla2AEME0IUWjHqZTyMyllOyllO5Pp4oz3+Ne19flX53o81NmVqqRD/SpULZuoydgmY+Fv8ZeD22kythZsdDakhdiHmdAfThLJZZ3lUQAeyX2SF2wPAfBFwjuIGNadUkQHPQJetNxb+HuIWSwiJZCykUicTpmvnHyW1RbxWlkg9FRQ6UAdr+Pa7nM+2wghTEAFIENKaZVSZgBIKZOBvcAVOspabCmVYOSl25pQKkF7a6ftZYXXikzG+Hmm2eysh0XYqFdElvTa4hiTzOMKBVO8avra83qRszXTHDfwhf1mWhr28ZjxZ91kViiCYezvO3QdPxxV+v6i3XT87yLPcff3/uTN3/SVU8+7zTqgoRCinhAiAbgHmFOgzRxgsPv1XcBiKaUUQlRzB1kghKgPNAT877xTaErbupUAuNRHhJ+vL/atzWvoLJFv9rk3/xaloFYk/ofrjSlstjxMqmUAb5o+p5NhG/ealgDweO5w7O5Yof/aBwDwtHk2VTits/SKkkaJXj2XsGxX4byZP6Uc1nVa3RSUe01pODAf2AHMklJuE0KMFkL0cjf7EqgihNiDy5WXF4reBdgshEjBFTwxTEqp/a44hU8mDb6K74Z1KjKAIh8x+mHmZafoa1zh8/q9xkWFzg0wLWF6whueY++M6A6MnppTb5s/DVmeNmIXV4qDDDIuoLNhC6mWASxJeJJLiHyTpSK+2XNM25DsaKciC4QEP35AfV18ui7cSCl/A34rcG6U1+scoJ+Pft8D3+spm8I/FUqb/aZN8vmUGKPcMXnpkW4yrgebxFtT1hbH+K/5SwB+dXTgcnEEA04aGS6E3vaxji405jDbf0g1DuR6YwrBJKq42rCV3c5a3GJcy2vmrwtdr2c4ylrLY9xgfZu9smCMUOi8bPqGh0zzsEsDTa2TsJIQ8ZiKyPli+X4qldHub+HPGtMrMCGYYWPxM784IwsUJYYxtoG8ZJ7K5eIwe2UtDDjpYUjm04T/edr8n+3/kEE7CwSz7F3pb1pGObI5S+lCLeqLwyxOfDrgSCnO+rQyuDzTPyWMorn1yyBlyE+qZUChcybh5EbDen6Jck0shX/iy+YpmoK7eQIlhvWX21DvSL74WfFWFF/8/DJH3dZE96n/cLqiCq8xbEXg5JeEF/MppyY5k0JQTi6+c3QB4BHTLz6uSr/K6Yh0WZ3HZEV+clxN/9xXSMqZBkA5cT6sda1Ohm2Fzu13VueMLMWNxpKxabakMHOddnuFoq3swq3HpbdVpRSUokiqls3vtvDlG69QyuyjXyIPdq6nm1x5HJTVOeSsRj/jMvZbBtHUcACALGmhYc6UkLKk57FWXgnAcNPPlCX/fpl7jYvzHW91JpGUM42knKl0sn5EUs402ls/5j+24eTi+lwG5z4HQLLl34Tyk24qUvOtlwHMd7TjptxxlCWH242r6WlYG+rbizoCJ83EPuIrj7i2OJzSk0RVC1L97NM6mBnZ/q0dR84AhRXg3uPxmTJMKShFAPJ/lX35xl+8pTEv3do4SvIUZqtMorkh1XP8if12mlknYQvbgy3Y4kwCoLlXFeAnTbM961q3W8eQlDON23Lf9PTxx5/O5p7X3yW8FpQE9cVh5ia+AIBVmt1KcBqP2EaQi5mnbMMAGGfWJvtFIrksSXiSVMsATfMbdjJsY79lEL8mvsQ7YQSeFBdmJ0eWVihaaF3/K5BrMFKUglIUScG6b20uq1SoTZlEE/+6tj7LnrnOc65i6cJWVbDsGnNzSO2PywtVdbtbxzHWfm/Yc+fxSO4IAI8FY8HKE6YfAUh1VmeLrB/0WBIDnXI+BKCdYZffdvXFYVItA0i1DMjnRmxknVyo7Y/Oaxln6095kc3H5veJxDr5t3EOOy1DqGdwZYe/y/hn2GN5M9I0PZ8FeJfxT58uy0ioQBaDjAswlMDN1VonswWwObT9nJSLTxFTvC2m13s3LXJDcN0qZfj6wfYAXFIufzaLf193edBzBh3e7maUfQjDc/+P3tbR7JG1Q+rrj8OejFvwgfkjFic+BcBH9t7cnPvfkMc7QhXetd0FFF5X6mjYzkDjQp9rWw1ypuDPOvvO0RWAW4xrGWOaFLJMebI8Z85fmfg6Q0pYY3lTRxxlmNca3m3WMQC8aJoa8dh5VCCLTZahjDF/xT7LIGYmFI7KVOQn7/dcXPZsKQWlKBKD+5s874lrGdSxbsD2Jj+ltp+9qVFQ89WuVCp44TwIfnV2YpNsEEZf/2N+6i4P0tu4kpoikxTn5bxjv5vzYaxrAWyQDQGXVTbYOB+AnxJeYkbCGN4wX1Aw3azvelx69iLclMeoxB1Wl8twkGmRZ8xguVykeyyce3JfIilnGmNt99DFuIVWYk9IY3nzvGkqyxOfBGBY7n9IypnGVlnf5Xo1pNLb4HvfWrAkYCPVMoBNlqH5zncw/B3yZ+APC1Z6GVZS0tbN8jxye49rUzJFRfEpYkqeuilnMRUKTQ1pnCD71qwQvILypwy14r/2gTyS+x/PcZ/c1yMab6WzKQed1QB4zfw1qZYBnjD0PBrkTGG/DD4zx0bZkOus73rGvEIEjiQrTQ4rE4ezKPEZAL53XMtqpyvicorjRgB+ShxFbXEsaDnyMGPnEZPLNWWVZn53tvdc+87hyvn8QcLHIY/rjXdgyBlZmvo53/J47mOA6zMoRY6/rkEz2jSZ8QkfkWoZSF0R3bx40eC9Bf5dzaGg1qAUMSVPsYT7PWx92YX1oWCyo7/bv2XQY19Zo1xYMoXCfGd7BuU+T5ucyGtmSgx0yf2A/+Q+mu98sBaTP1JlDUbkuoImXjd/le9aIrncbFjDe+aPPetb2y0PUlO4ErPMsnflKdu/Pe2zsfCzw7W36t9GX2H2/mkh9rLbcj8Ab9nuoZE1/8Zlb/fr8xG4+u4zLSDVWZ07ra9wo/UtnBiY47yGPxxtAXjGNCvsscGVEaS/aZnneF7C84w0TY84iXAiufyR8AwdhL7564oi+cBJFm4/6vGMRIre9qXaqKsoklC/x+Usrq9UrYouS+jrB9tzMMjSBu3qVqJO5cIbY/1xS/MabE0/E5qAYbDCKwpPC35ydqa34y+Oykq8bH8wgmjDC/zg7MJ7fEIHw9/U4jgjzdO53bi6yD7Ncr4gy8dG5CdswzknE+lvXMrH9l6kU61QG4GTx4w/M9Q0jhPDtAAAGLVJREFUl+7Wt1lreSzf9W8d3X3O2TZnIsmWf/OIaS7v2fv5yYQhqcA5TlO20JXkxEeoIs7yhm0AyTK/2/gp279ZZRjOg6bfme64nt0hrke2FTv5PvFClOWQ3GfoatjMA6b5DDP9wgZnA/5w1yELljKcx4SDd8yf0sOYDMDMxNdpnDMpbFdxJHzsLtSYVCX431mRKBefIpZY3FZPsBZUi9oVmTiwDaN7NwOgvMVMs1oVADBq6JJbOKIr/+4afOBFvPGA7TlG2odqopzyWOpwWZ9/WZ7wqZyOyMrMc1zFGVmapJypPpVTHhPsfTALB39ZnqC+yJ8QtL3YwX7LIJ42z6a8yC6knAbkvuAzAwdABhV4JNe1PvWy6RvP+ZsNazwWXqplIJssQz3HeRhxUEW4ct59795M7c1ZSnuswQWJz/JvY8Hc1P6QLEp4Kp9y2upMYqmzNa/b7+O8dCnRieb3SST4vU53Gv5km+UhNlmGepRTHl+a36FaDHM0Fpe6sEpBKYpk0uCrePz6BtSpHPza0M3Na/iM9gukoPKsr2Aok2gsNj+yaDHCy1UH8JztYdrlTPS4DztZP+LftidpYf2CQLkKvK2mxYlP87RpJiB50fQtsxJ9r8V1tn5AUs40VjqbFTn2UqdLkQ4yLaIaJ7lCHGJiwgd+2z9vmkp7sYO9lvsAVyRlJoXKwwGuSstv2/oD8Jx5BpXxbWGXIofrDCksTHiaVMtALjdcyIj/SO6Tnv1tTgy0sH7BRPvtGIXkp4RRPscrjOTdhPxu4YG5z3syi1xt3M6viS8GOZb2aPXTOatDXS1vlIJSFMllVUoz4sZGPpVB3RDdBEUFNdSpXIp3+hVef5owoI3P9vGW7TkeyKQ8vayvs8Z5JW1zJjLT0Y0TVAh7vBY5n3teDzf9TKplIA+bXLmfFzpak5QzjY45H9IkZxJJOdNIk4Vdgb6wksBY2z0ArLM8xh+Jrkwb/7XdS3/ryx6F2t/6MgCPmObmU4rv2Qvll87HBEcfbrW6FMxNxnWFrhtwssPyIJMTxtHAcME6/Mx+K/VyvmV+ATeeDRNv2V3yNjYcpBaFy04U5DmTK3R/gaMNt1jfpFHOZP5yu4qfzHU9SFQXp2gtwq8IfYHQ/Wz7NMwcoWfRQqF3FEa0KFOmjDx3Lj7TdZQ0lu8+zq+bjvDy7U3IyrH7rBvlixv/t4xdR/OHt371wFVYTEY6XV4l3/m8TYpf3N+O/8xM8VRAvaRcIsfOWlnzwg1UL28h1+5k9b4M7p8U/yl/iiPXGjbzrvkTLhGnPOdusb7JdpkU8dijTV9xv2kBAFPsPRhlf6BQm6HGX3jBPB2A6fZuPG9/OMjRJamWgYXOpjgvp5Vhr+f4S/vNpDgvZ56zfcAAlWsNm/kmYSwnZVlaW/1n8CjPOTZbHiZLWmht/cynG7ciZ9mQOAyDkByRlelhHVeky7Xge7OQSwJ2vk14kxZe2U4AbrW+wTapf5qxPHaO6UmiKbKCqUKIbCllmULnlYJSRIvDp86zYPtRXpnj2qi68eUefksU5CmoeU9cy6XlLbR+fQEGAU1rVmBL+mnWv9TdU9p+1d4M7v286IAARWTcb5zPYOMf3J07KiKrzBuBk/2WQQAe15eW9DcuYZz5c7/Xr8j52pMvMVjy1sQG5z7HMmd+i/8KcYhRpil0Nrq+3w/mPs1ip28PAMAdhuX8L2Gi5zjQZ2DGzpLEEdQWJ4pslyHLcbX1w6iVYtkxumfEFb39KSjl4lNEjZoVSzH46iTPcTD1cxrXKE9Z99qUEIIvh7Tjvf4tPcoJCqdjUmjPFMdN3JD7rmbKCVxh93nuPD2Y5ehGUs5UmuV8QeucT1jnvII9zpqMt/ehQc6UkJUTQA/rOAAmmcfxhfltrjNsRODkTsOf/JH4nEc5ASx2ti5yrB+d1+Z770+bZmLE4bNtI3GQ3Zb7CymnA85LGJD7Ao1yJnOz9b98Y+9OFXGWrYkPcaOhsHtTD2asO6jb2CrMXBF1GlUvx86jwVcgNbrXv+5oXYtLylno2yZ/+HCjS/XfD6UorgiP66xf7qsRj7Zb1uYDe1+eMP1Ad+NGuhs3FmqTJqtyh3U0wRbNaJ8zgV8TX2S46WecCN6z9/e66vJwzU90FRv/yXE1L9keJIcEBGDD6Jlnh6zLy/YHsGPkAdN8Rpsns9ramDM+wvULUpocWht2s8bZmE6G7RyWVSjLeTbJywO+j9d+2c4D1+jjUlQuPkXUyc61c87qoFqBfH3erN2fyfoDmTx6nSt90enzNsokGDEZfRv9fT/+iw0HT/m8plBoSSXOsCLxCRY7W3OrYQ0G4bqH9rW+6k5nFZ5Jn+c+zJIWPrTfwfPutbc8Ztm78qx9aFDjtxJ7+CnRFXF4vfUd9smaftveZFjLpwnvFzleoIrQqWNvDShTUag1KEWJZsbag4z8YUusxSiSIVcnMXllaqzFUGhIZc5QUWQVqQCCpaNhOzMSxvi8NtN+HS/YH8JB8Gs9H5k/4DbjGnY6a3NT7rh812pxnPcSJtLB8HdQY61wNGWQzX9YvF4KSrn4FCWC2pWKjoC6t/1lPHNTI9q8vsBzrkO9yqzZn6nJ/JVKm7mu0SX8uDHdb5uXb2uim4JqW7cSyQdit/HzYiWT8mRK33uyQmW1swlJOdP4KeElWhn20dM6lgPyEpwYwgp4+D/b/2HGwU3G9XyX8CoDcl+kvjhCS8NeHjH+Qn3DhRyDr9gGM9VxA2U5zynK0d2QzA7nZaRTjRsMyaxxxqbem1JQihJB54ZVPa9H927KqJ/zl7SwmA1ULhCUcUPjS/wqqHUvdsfudPLo1A1sDMJ1aBCCW5vXKFJBGQ2CQR0vY9c/WaxN1UYx5jG0S30e+SY5cENF3NMndwyutafIon8kBp6xDeVSkUk7wy52WQbnu/6royOv2gbnC3w5hWs9d6GzrefcIq/X0UZF8SlKHN0aXeJ53am+a39Vo+qFAymKSphZrVwiNSqU4vthVwc1Z48m1bmh8SWehLhdrvC9aXVMn+bMGtaJUbc1CWrcQCx4sguv92nGjU2qazKeN4ue6kqTGtpYBwVZOfJ6XcYtOWgTmnqGsvTPzZ/9Yp7jKq63vsNw2+OaRmXqga4KSgjRU4j/b+/Ow6SqzjyOf380q4LIHhCwQTEKCkgAVwgq44IkGPUJZhwXdEYlcQlmopjMJCYxiZo8GXVG42g0ksRxCUnQiVsUN5K44AKIoggMRgyLyhIRQrO888c93RRNN0LTRV26f5/nqafuPXXr1rkHqt4+9577Hr0laZ6kiTW83kLSfen1FySVF7x2VSp/S9IJxaynNQxnHtaTEZ/uRI/2e/Dtz2UBYNxR5fz+kqMZO6QHkJ0rv+607I7+zw/sxtjBWXnvTnsy95qTePbrx/DHK4+p2meTgjHs+3duzcOXDtvqc78x6kC+d8rBSOK1q49nzndP5JJjtz031egBXWlWJn4+bseSjxZ6+d9G0qdLG846fN8aM32cOqjmi9qj+9c8ncf1p/XfopfZfo/mPHzZ1se7s5o2Ed32bkW/bsUJfpX+ZVjxblb90en9i7bv+raO5lVZP45fdx3j10+ol2tmu0LRApSkMuBm4CSgL/AlSdX/bDwfWBER+wP/AVyX3tsXOAPoB5wI3JL2Z1ar73/hEO4al80/dO6R5fxm/JEc3+9THLxP2y1+wMcO6cnCa0+mc5uWXHd6f753ysH86vzDaN60CT077FHr9ayubVtu8QNe1kR89oBOnHnYvjRLowubljWhVfMyhpS3B2BYn46cW3DvV6XObVry9vdHccynOzOsT0eGlrfn8wOyH42HLj2an48bUmu+tB98IQuwbVrWfB/PuUeWc8rAbnStJcPHyIO27m1N/dpn+eKQHuy9R7bPyRcdUXWf2rQrjqnXHs99Fx4OwEOXDtvpi+vbUlsvtq5uOXPzTbenf2bnZ24+6eBPVS13a9uSEZ+u3/pWt4QOzI0eRf2M+la0UXySjgCujogT0vpVABHxw4JtHkvbPCepKbAE6ARMLNy2cLvaPs+j+KxY/vuZ+fzwkTerMl8sWrGGrm1b1Wt2doCKDZtYuaaCznttDiwr11Twyl9WsEfzpry7fA3lHfesCn7VVWbfqPzRX/5xBdc89AZ/W7uBJ+YsrdpuyleO4pSb/8TJ/bty5QkH0rMgp+KC91fzi+fe4Vuj+27RewRYW7GR26ct2Gqyu9vPHkynNi1oouyemMLBGl8esV/VFA8jD+rCE3OW8scrj9nij4AVH1dwaBq8cvYR+/KL597Z6tjatmrGqrXrtygbUt6O6QtX8Mcrj+Ho654CskDbvKwJK9ZUsF+n1vT79uYZdocf0IkxA7rxtV/PrLH9Kk0YeQCbIrhxapYnr3enPVnw/sc8PmE4K9eup0ublvTssAc/m7aAax7a8bmd/vfio/n65JlMOm8o69ZvYtlHf2dw+jet/DfcUQt+MIpxd03nmbmb8wSOH7EfP316Ptef3p8rJs+q0363x5iB3bjxjG3flPxJdvkwc0mnAydGxD+n9bOAwyLi4oJtZqdtFqX1+cBhwNXA8xHxq1R+B/BIREyu7fMcoKyx++VzC+m2dyuOq9ZD2rBxE2vWb6R5WRMWrVjL/p1b8/I7Kzh4n712OIfavGWrGfmTZzjvqF70aN+K5xd8yK3/9Jkteqi/fuld3l62mpEHdWHwvu14YOZ7fK5/N5pILF9TsUUWkEp/+XANc5d+xMi+XVi5poJvPfA6FwzvzQ1PzKXzXi2reo0ALyz4kKlvLuMbozaPLJv93ir2atlsi2AL8PG6DcxctJI2LZpxSPfsesvf12/kyTeX0UTiodcW07ysCf8++iDOvvPFLBBdPpyubVsx892VrN+4if7d92b2X1cxqGe7req9ck0FA7+bBdfxI/Zj4Qcf89WRBzBlxnv89On5XDi8N/t1bs3+nVtz6i1/5sR+n+LWs2ofdLBpU3D5/TOYMuOvPPrVYUy4byZzFm97zrPXv3MCe7ZoytqKjby55G902LMFv311EZcd16fq36WugW97PHfVsXTdgZmwa9IgA5SkC4AL0upAYMZOVrsjsO1EV42X26Z2bpvauW22ze2T6R8RW42lL+Yw8/eAwhOe3VNZTdssSqf42gIfbud7iYjbgNrTCu8gSS9FxOD62l9D4rapndumdm6bbXP7bFsxR/FNB/pI6iWpOdmgh+pTXD4IVA7OPx14MrIu3YPAGWmUXy+gD+D5FMzMGpGi9aAiYoOki4HHgDLgzoh4XdJ3gZci4kHgDuCXkuYBy8mCGGm7+4E3gA3AVyKi5jS/ZmbWIDWYXHz1QdIF6bShVeO2qZ3bpnZum21z+2ybA5SZmeWSUx2ZmVkuOUAln5SWqSGSdKekZWm4f2VZe0mPS3o7PbdL5ZJ0U2qfWZIGFbznnLT925LOqemzdjeSekh6StIbkl6XdFkqb/TtI6mlpBclzUxt851U3iulLJuXUpg1T+WNLqWZpDJJr0r6fVp329RFRDT6B9kgjvlAb6A5MBPoW+p67YLjHg4MAmYXlF0PTEzLE4Hr0vIo4BGyLJaHAy+k8vbAgvTcLi23K/Wx1UPbdAUGpeU2wFyylF2Nvn3SMbZOy82AF9Ix3w+ckcpvBcan5S8Dt6blM4D70nLf9F1rAfRK38GyUh9fPbXR5cD/AL9P626bOjzcg8oMBeZFxIKIqADuBcaUuE5FFxHPko2eLDQGmJSWJwGnFJT/IjLPA3tL6gqcADweEcsjYgXwOFn+xN1aRCyOiFfS8kfAHGAf3D6kY1ydVpulRwDHApU301dvm8o2mwwcpyzFwRjg3ohYFxH/B8wj+y7u1iR1B04GfpbWhdumThygMvsA7xasL0pljVGXiFiclpcAlXlzamujBt926bTLoWQ9BbcPVaewZgDLyILufGBlRGxImxQeZ1UbpNdXAR1ooG0D3ABcAWxK6x1w29SJA5TVKrJzDY16mKek1sBvgK9GxBZJ0Rpz+0TExogYSJblZShwYImrlAuSRgPLIsKzR9YDB6jMdqVWaiSWplNTpOdlqby2NmqwbSepGVlwujsifpuK3T4FImIl8BRwBNlpzcqb/wuPs6oN6pLSbDdzFPB5SQvJLhUcC9yI26ZOHKAy25OWqbEoTD91DvBAQfnZabTa4cCqdKrrMeB4Se3SiLbjU9luLV0HuAOYExE/KXip0bePpE6S9k7LrYB/ILtG9xRZyjLYum0aRUqziLgqIrpHRDnZ78iTEXEmbpu6KfUojbw8yEZhzSU7l/7NUtdnFx3zPcBiYD3ZOe7zyc5/TwXeBp4A2qdtRTYB5XzgNWBwwX7OI7uIOw8YV+rjqqe2OZrs9N0ssiz5M9L/kUbfPkB/4NXUNrOBb6Xy3mQ/ovOAXwMtUnnLtD4vvd67YF/fTG32FnBSqY+tnttpBJtH8blt6vBwJgkzM8sln+IzM7NccoAyM7NccoAyM7NccoAyM7NccoAyM7NccoAyqyeSVqfnckn/WM/7/ka19T/X5/7N8sgByqz+lQM7FKAKsgzUZosAFRFH7mCdzHY7DlBm9e9aYJikGZImpMSqP5I0Pc0VdSGApBGSpkl6EHgjlU2R9HKaZ+mCVHYt0Crt7+5UVtlbU9r3bEmvSRpbsO+nJU2W9Kaku1N2DCRdq2yeq1mSfrzLW8dsO33SX21mtuMmAv8aEaMBUqBZFRFDJLUA/iTpD2nbQcDBkU2pAHBeRCxPKYSmS/pNREyUdHFkyVmrOxUYCAwAOqb3PJteOxToB/wV+BNwlKQ5wBeAAyMiKlMWmeWRe1BmxXc8WZ6+GWRTdnQgy60G8GJBcAK4VNJM4HmyZKF92LajgXsiyy6+FHgGGFKw70URsYksVVM52XQOfwfukHQqsGanj86sSBygzIpPwCURMTA9ekVEZQ/q46qNpBHASOCIiBhAlu+u5U587rqC5Y1A08jmHBpKNjneaODRndi/WVE5QJnVv4/Ipomv9BgwPk3fgaQDJO1Zw/vaAisiYo2kA8mmUa+0vvL91UwDxqbrXJ2A4Wwj63Wa36ptRDwMTCA7NWiWS74GZVb/ZgEb06m6u8jmAyoHXkkDFd5n85TfhR4FLkrXid4iO81X6TZglqRXIpu+odLvyOZimkmWff2KiFiSAlxN2gAPSGpJ1rO7vG6HaFZ8zmZuZma55FN8ZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw1mwkJJjwIdd3I3HYEP6qE6Zrb9/L2zDyLixOqFnrCwgKSXImJwqeth1pj4e2e18Sk+MzPLJQcoMzPLJQeoLd1W6gqYNUL+3lmNfA3KzMxyyT0oMzPLJQeoRNKJkt6SNE/SxFLXx6whknSnpGWSZheUtZf0uKS303O7UtbR8sMBCpBUBtwMnAT0Bb4kqW9pa2XWIN0FVL/fZSIwNSL6AFPTupkDVDIUmBcRCyKiArgXGFPiOpk1OBHxLLC8WvEYYFJangScsksrZbnlAJXZB3i3YH1RKjOz4usSEYvT8hKgSykrY/nhAGVmuRHZsGIPLTbAAarSe0CPgvXuqczMim+ppK4A6XlZietjOeEAlZkO9JHUS1Jz4AzgwRLXyayxeBA4Jy2fAzxQwrpYjvhG3UTSKOAGoAy4MyK+X+IqmTU4ku4BRpBlMF8KfBuYAtwP9ATeAb4YEdUHUlgj5ABlZma55FN8ZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZmaWSw5QZkUmaaOkGQWPekuGKqm8MDO4WUPStNQVMGsE1kbEwFJXwmx34x6UWYlIWijpekmvSXpR0v6pvFzSk5JmSZoqqWcq7yLpd5JmpseRaVdlkm6X9LqkP0hqlba/VNIbaT/3lugwzerMAcqs+FpVO8U3tuC1VRFxCPBfZJlMAP4TmBQR/YG7gZtS+U3AMxExABgEvJ7K+wA3R0Q/YCVwWiqfCBya9nNRsQ7OrFicScKsyCStjojWNZQvBI6NiAWSmgFLIqKDpA+ArhGxPpUvjoiOkt4HukfEuoJ9lAOPp8n+kHQl0CwirpH0KLCaLJXQlIhYXeRDNatX7kGZlVbUsrwj1hUsb2TzteWTyWaKHgRMl+RrzrZbcYAyK62xBc/PpeU/k2XUBzgTmJaWpwLjASSVSWpb204lNQF6RMRTwJVAW2CrXpxZnvkvKrPiayVpRsH6oxFROdS8naRZZL2gL6WyS4CfS/o68D4wLpVfBtwm6XyyntJ4YDE1KwN+lYKYgJsiYmW9HZHZLuBrUGYlkq5BDY6ID0pdF7M88ik+MzPLJfegzMwsl9yDMjOzXHKAMjOzXHKAMjOzXHKAMjOzXHKAMjOzXHKAMjOzXPp/ETjUZNWdEkgAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEcCAYAAACS6SCjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXjcZb3//+d7Jstka9ZuadompStbW1qgtOyIKCAoB5Sq58DRC4SDyiLqweNRFP0ej3B+7gcPHFSOIkUBUSg7spa2NN3SdIG2adqmTdp0sjR7MjP374+ZlHRPmkw+WV6P65prZu7M8p60ySuf+74/923OOURERAYan9cFiIiIHIkCSkREBiQFlIiIDEgKKBERGZAUUCIiMiApoEREZECKW0CZ2W/MbK+ZlXZpyzGzV8xsc+w6O9ZuZvZzM9tiZiVmdka86hIRkcEhnkdQvwM+dkjbvwKvOeemAK/F7gN8HJgSu9wMPBjHukREZBCIW0A5594Cag5pvhp4NHb7UeCTXdr/z0UtA7LMbGy8ahMRkYEvoZ/fb7RzrjJ2uwoYHbs9DtjZ5XEVsbZKDmFmNxM9yiItLW3O9OnT41etiIjE3cqVK/c550Ye2t7fAXWAc86ZWY/XWXLOPQQ8BDB37lxXXFzc57WJiEj/MbPtR2rv71l8ezq77mLXe2Ptu4DxXR5XEGsTEZFhqr8D6m/ADbHbNwB/7dL+T7HZfPOA+i5dgSIiMgzFrYvPzB4HLgTyzKwC+C7wI+BPZvZFYDvw6djDnwcuB7YAzcA/x6suEZGhLhJx1LV0UNvcTiDRT3ZqIimJfszM69J6JG4B5ZxbeJQvXXKExzrgtnjVIiIy4LQ1QHArBDJhxDhISDrqQzsDJ9jYRrCpnZqm9oNvx+7XxO7XNLUTOWSEPynBR1ZKItmpSWSlRq+z0xLJTEkiO7VLe1oSWSmJZMXuJ/qP0NHmHDRVQ/1OSM2F7MK+/d7EeDZJQkRkWAiHILgF9q6HPRtg7wbcnlKsbseBhziMluQ86pNGE/SPooo8KiI5bOvIYUtbFptaRhB0GcDhR0CZKYnkpiWRk5ZEUV4acybmkJuWRG56EtmpSbR2hKlt7qCupZ26puhRVV1zB1urG6nb0UFdczsd4YPTLEAb+RYk34JMSqylKLGG8f4axrKPkZFqskPVJLp2APbM/iqjr74vLt86BZSISF9wjuZ9O2jYUUL77nX4qjeSUvs+IxrLSHAdAITxsd3GsSFcwMbwPLa6fNKthXyC5If2kd8cZLx/E+eyjwDtH752MoR8AVpSxxLKGIdlFpCQPYHAyIkkZI2HzILoUVhi4Ph1RiKxo58KqN+Jq99JR81OQrU7oL6ChIZdJLUdfAprJGTUuVz22kjWuSJ2uDls68hmt8tjvv98vtCX38cuFFAiIsfQFgqzr7Gd6oY2qhva2NfYRn1tEN++TaTVvk9O02by28ooDG8n05pIjT2v0uVQEhnP++4ydiUXUZM6hZask8jKyGBkRjIjM5KZmp5EXnoyOV2OeBL9vmgXWnNNtAutviIaHPU7yei8X/E6bNpzeLFpo6JhlVkAWRMgYyy01h8II+orYP8uCH8YfgYkJaaRlDUesgpg4pzYa4w/cO3LGEtOQhI5QOeZp+GIo76lA78vfuNaNpi3fNd5UCLSE+2hCHXN7dQ0t1Pb1HHgdl1zBzVN7Qe6v2qa2qlrbmd/UzN5bTuZbjuZ5tvBNNvJdN9OCmzfgddstlT2BIqoTZtMU/Z0IiNPJnHsKWTnjT4QPnH5JR5qi4ZNLMAOCqH6CqjbCaEWMF80qDqD66DwiV0CWeDhBAozW+mcm3tou46gRGRQCoUjVMcmBtQeGFtppyZ2O3rpoLZL8DS2hQ57nQBtTLC9TE2sZmrSPib5q5lgexgTriSXKvzJYQAilkB71iTcqHMJjT2VhLGnwqiTSc2aQJEZRf39DUhIhpxJ0cuROAetdZCUDv7E/q2tjyigRAaatgaoWgeVa6N/CWcWQHZR9BdR9sToL6YhzjlHTVM7lfWt7K5rYXddC5X1reyKXe+ua2HP/tbDZqp1yggkxGapJZGblsjMnDCF/iAFrooxkSpy2yrIbK0gpXEHic17P3xiCPBnRmelZZ8JOUUwcgaMPgVf3hQCg+l7bwYp2V5X0SsKKBEvtdRCZUk0jCrXRK+DW4HYb15/0kHjBWDRwfCcougv0ZyiWHjFrlOyPPgQPdfUFqKyvoVdda1UxgJod30rlfUt7K6LBlBbKHLQc5ISfORnBhibmcL8k/LIzwowJjNAbloy2Sl+Rrkg2W27SG/ZSUJdOdRsg9ptsKcc2uoPLiBjbPT7lX/pwd+/nKLoL/VBdr7QUKWAEukvjdUHBVGkci2+ug+XIGsKjKUqbRrbx17EJiuiJDyRHW0jyLZ6xkWqyHdVjA1XMaajkjG7Khm9o5SsSO3Bb+Ebwb7EfILJ4wgmjqMmeRx1gXHUBgpoTszD7/fhM8PvA5/P8JvhM8NnYF1u+3yGGfjMMGLXsfsffj32PAwfjkTXTkKklcQuF9fRSl1jE3WNzdQ1NFPf1ExDUwsd7e0kECLBwiQSvYwPGKcHjOyAkTUBRiRFLxmJkJboCPgiWCQE4Q6IdEBNB+xthbodUFt+cJD7EqKTBLKLoODMD49Ac4ogayIkpSIDnwJKpI9EYrOago2t1O/ZQWT3GhL3lpBeu4GRjZvI7Kg+8NjtbgzrIoWURs6h1BWxPjKR2tYRUBftnspLTyYvPYn87EScS2FPZDSV7nTCEUc44oi46HVipIVRHZWMCu1mTLiSMeFKxoYqyW8vZaZ7nQQ+PAppIYldjGaHi162u1FURnLwEyLZtZFi7QRoI4V2UqyNAO2k0EaKxdpoI2CxNtoPup1qbT37Zh3pnNQw0BS7QDRkfInR8RNfQuw6Efxd2v1JMHIaTP3YwUdBIwqij5NBTf+CIsfROR5Stb+VqvrWD6/rW9mzvxV//Q5GNW1iYvsWTmYbp/i2Mdn2AxBxxlaXzxL/DCoCV1GdPp392TNIz8wlLz2ZSelJnJ2eTG6X6caBRH/fFB7uiB1dbIOabaTUljO5ZhuTa7dBzd+jM7yO8lbOlwiJKZCYiktIwSVGLyRkEklIwcXaIwkpuIQUGhICRGL3I/4UIgkBwv4USAiQlZFKIDlw7KA57H6CutlEASXDWygcYW9D20Gh0/V25f4W9uxvo73LeEiANi72r+VTySuY59aS4RoBCCf4qUs7icacSygbfTr+cbNJn3A6k7JymBLHc0WOyp8IuSdFL4dyDhr3QEMl+JMPhFH0OgXrMutLMSFeUUDJkBWOOHbWNFNZ30rV/haq6tuoqm85EECV9a3sa2w74pplYzMDjBkR4IwJ2YzJDDAuzXFa83sU7nmZzIrX8XU0QyAPpn0Sxs2BsTPxjzqF3MQAud583J4xg4wx0YvIAKWAkiHDOUd5sJl3NlfzzpZ9LN0aZH/rwee9jAgkMCYzwJjMFKaNyWBMZgpjRgQYmxlgdOw6KzUxuupzezNsfhk2PAOrXoKOZkjNg9M/A6d8CiYu0DiHSBzpp0sGtX2NbSzZsi92CbKrrgWAcVkpfPzUscwpzKYgK4XRsSOitOTj/Jdvb4YNz8P6v0TDqaMZ0kbCzOvh5E8qlET6kX7SZFBpaQ+zfFuQJVv28c6WIBsro5MRRgQSmH9SHrdceBLnTc5jYm5q9/e+aW+KhtH6Zw4Ppc4jJV8fTVwQkW5TQMmAFo441u2qP9Btt2p7He3hCEl+H3MmZvP1y6axYHIep43L7Nl6ZwdC6S+w+ZUuobQQTvmkQklkAFBAyYByrHGkk8eO4MYFhZw7OY8zC3NISephgLQ3wQcvRceUPng5Os06bRTM+mys+26+QklkAFFAieeON4507pQ85p+US276CayD1hlKnUdKnaE0+3MKJZEBTgElnthdtoHaF39Izr4VdIQizAbm+IyvJ/gJZPsIJPpJ8BlWAVQAr5/gGzXujYZS+miY/flo992EcxRKIoOAAkr6TXN7iDeWrySw9L84v+kVcvCzOmUe2SNHMCojmazURHx9fVpoag5Mv0KhJDIIKaAkrpxzrCiv5eWlxUx5/3+4hteJmI/ScZ9m9BX3cM64Qq9LFJEBSgElcbGrroWnVlbwRnEJVzUs4hv+v+PzQXDaQkZ9/B5mZRZ4XaKIDHAKKOkzLe1hXlpfxZ9X7uSDrVu5xf8sixJeIyExQnjm50i48OuMzhrvdZkiMkgooKRXnHOs2lHLkysreG5tJYltNdyd/gK/S3mRBBfCZi6E8+/Gl9PvG2KLyCCngJITUlnfwtOrdvHUygrK9jUxNrGZn4x6g4vqn8YXbsVOuw4u+OaRV9IWEekGBZR0W2tHmJc37OHJlRW8s7maiIMLJyTxi9Pe4uTtf8CCTXDqNdFgGjnN63JFZJBTQMkxOedYs7OOJ1dW8Le1u2loDTEuK4W7zhvD5+wFstc8BHvrYcZVcOE9MPpkr0sWkSFCASVH9fdNe/h/z29iy95GkhN8fPzUMXxmZg5nVz+Jb+kvoKUWpl0BF/4rjD3d63JFZIhRQMkRbQ828eU/riY/K4X/uOY0rpiRyYiSR+HZn0JzEKZcFg2mcWd4XaqIDFEKKDlMKBzhjifWkOAz/u+fTiN/8yL49U+gaS+cdDFc+C0Yf6bXZYrIEKeAksP86vWtrN5Rx58uqCH/0fnQUAlF58OF/wcTz/G6PBEZJhRQcpDVO2r5+d838+XpDZxV/DUYORWueRiKzvO6NBEZZhRQckBTW4g7n1jD9IxW7gp+H9JHwT8+A2l5XpcmIsOQAkoO+MHijeyq2c/KCb/GFwzCF15SOImIZxRQAsArG/bw+Hs7+HPhYkZULYdP/Q/kz/K6LBEZxnxeFyDe29vQyjefKuHLOcWcWfUEnH0rzLze67JEZJjTEdQw55zjm0+WMLFtM3fxKyg8Dz56n9dliYgooIa7Pyzfwdr3t/JW1s/xJY2Ea38L/kSvyxIR8aaLz8xuN7NSM1tvZnfE2maZ2TIzW2NmxWZ2lhe1DSdbqxv50eJ1/CHz16R11MD1f4D0kV6XJSICeBBQZnYqcBNwFjATuNLMJgM/Br7nnJsFfCd2X+KkIxzhjkVruMf/R05uW4N94meQP9vrskREDvDiCGoGsNw51+ycCwFvAtcADhgRe0wmsNuD2oaNn726mUmVi/k8i+HsW2DWQq9LEhE5iBdjUKXAD80sF2gBLgeKgTuAl8zsAaLBOf9ITzazm4GbASZMmNAvBQ81K8preOPNV/lL8v/ChAXw0R94XZKIyGH6/QjKObcR+E/gZeBFYA0QBm4F7nTOjQfuBB45yvMfcs7Ndc7NHTlS4yU91dDawb2L3uLh5J/gT8+D6x7VpAgRGZA8mSThnHvEOTfHOXc+UAt8ANwAPB17yJ+JjlFJH/v+X0v4t+YfM9pXj0+TIkRkAPNqFt+o2PUEouNPfyQ65nRB7CEXA5u9qG0oW1xSyZR1/8V833p8V/4Uxs3xuiQRkaPy6jyop2JjUB3Abc65OjO7CfiZmSUArcTGmaRvVNW38vbT/82PEhYTPvMm/LM/53VJIiLH5ElAOecO27vBOfcOoD/p4yAScfzij0/zXfdrWvLPJuVj/+F1SSIix6W1+IaBx99Ywy2V38EFskj53B80KUJEBgUtdTTEvb+7lqI3vswYXx0J//hidI8nEZFBQEdQQ1hbKEzJo3cx31dK62X3YwVzvS5JRKTbFFBD2AuP/4rr2p6mYvJnyTjnn70uR0SkRxRQQ9SaFW9z2Zb7KE87nYLrf+Z1OSIiPaaAGoL2B/cwcvEXafKlM+qLiyAhyeuSRER6TAE11ETC7H7ks4x0QWqv/A2pOeO8rkhE5IQooIaYzY9/nenNxbw99V+ZMucir8sRETlhCqghJLh8EVM2P8JLKZdzwfVf87ocEZFeUUANEeHKdaS9+FVWu6nM+OcHSfDrn1ZEBjf9FhsKmmtofPR66iMpVFz6P0wYleV1RSIivaaAGuwiYRoeu4GUlkp+V3AfVy7Qtu0iMjQooAa50KvfJ2PXW9yfcBM3ffZ6zMzrkkRE+oQCajDb8FcS3v0pfwxdzLmfuZucNJ3vJCJDhwJqEGt86QdsiExk69zvcMFU7YwrIkOLAmqwqt1Oev0HvBG4hK9fcbrX1YiI9DkF1CAVXP0sAHlnXEUg0e9xNSIifU/7QQ1SjesWUx8Zw/yzz/a6FBGRuNAR1GDU1kh+7XuUps2jIDvV62pEROJCATUIVa15iURC+Kd/3OtSRETiRl18g1Bw9d9IdSnMPlcBJSJDl46gBptIhDF73qI0MJf83EyvqxERiRsF1CCzc+Mycl0NoZM+6nUpIiJxpYAaZHYv/wsRZ0w771NelyIiElcKqEEme9ff2Zw0ndFjx3tdiohIXCmgBpFt27YwNbyFxomXeF2KiEjcKaAGka1L/gLAxHnXeFyJiEj8KaAGkZTyV9nnG0neSWd4XYqISNwpoAaJzbuqmdWxmn35F4H2fBKRYUABNUisW/I8adbGmLlXe12KiEi/UEANAs45fJtfotWSyTrlI16XIyLSLxRQg8AHVQ3MaXuP6rxzIDHgdTkiIv1CATUILFv2DuN91WTOutLrUkRE+o0CaoBzztG+8XkARpx2hcfViIj0HwXUALepqoFZrcsJjpgBI/K9LkdEpN8ooAa411dt5AzbTMopl3tdiohIv1JADWDOOepKXsBvjtRT1b0nIsOLAmoA21jZwGnNS2lJyoWxs70uR0SkX3kSUGZ2u5mVmtl6M7ujS/tXzGxTrP3HXtQ2kLywdgcX+EqwqZeBT39LiMjw0u9bvpvZqcBNwFlAO/CimT0HjAeuBmY659rMbFR/1zaQOOfYufbvjLBm0PiTiAxD/R5QwAxguXOuGcDM3gSuAeYCP3LOtQE45/Z6UNuAsX73fk5uXEY4KRH/pAu9LkdEpN950W9UCpxnZrlmlgpcTvToaWqsfbmZvWlmZ3pQ24CxeF0ll/hXE5mwAJIzvC5HRKTf9XtAOec2Av8JvAy8CKwBwkSP5nKAecDXgT+ZHb5st5ndbGbFZlZcXV3df4X3I+cca9es5CTbTeIMde+JyPDkyci7c+4R59wc59z5QC3wAVABPO2i3gMiQN4RnvuQc26uc27uyJEj+7fwflK6az/TG5ZG70y9zNtiREQ84sUYFGY2yjm318wmEB1/mkc0kC4CXjezqUASsM+L+rz23LrdXOJfTThvOv7sQq/LERHxxHGPoMzsE2bW10daT5nZBuBZ4DbnXB3wG2CSmZUCi4AbnHOuj993wHPO8frarZzt24h/2se8LkdExDPdOYL6DPBTM3sK+I1zblNv39Q5d94R2tqBz/f2tQe7kop6Ttr/HglJYZiqgBKR4eu4R0bOuc8Ds4GtwO/MbGlsooKmlsXB8+squTRhNZFANhQM64mMIjLMdavrzjm3H3iSaNfbWOBTwCoz+0ocaxt2nHM8v7aCjySU4JtyKfg9GSIUERkQujMGdZWZ/QV4A0gEznLOfRyYCXwtvuUNL2sr6snbv54RkTrN3hORYa87f6L/A/AT59xbXRudc81m9sX4lDU8LS7ZzaUJq3HmxyZf4nU5IiKe6k4X373Ae513zCzFzAoBnHOvxaWqYcg5x+KSSj4RKMEmnAMp2V6XJCLiqe4E1J+JnqPUKRxrkz60emcd1FcwvqNM3XsiInQvoBJiU8CBA9PBk+JX0vC0uKSSSxPXRO9oermISLcCqtrMruq8Y2ZXM0xXeIiXSMTx/LpK/iG9FLKLIG+K1yWJiHiuO5MkbgEeM7NfAgbsBP4prlUNM6t31lJXX8cpqWvg9C/C4WvkiogMO8cNKOfcVmCemaXH7jfGvaph5rmSSs5P3Ig/0q7xJxGRmG6dCWpmVwCnAIHOHTCcc9+PY13DRmf33o+zNkBbBkxc4HVJIiIDQndO1P010fX4vkK0i+86YGKc6xo2Vu6oZc/+Vs7qWAGTL4YEzT8REYHuTZKY75z7J6DWOfc94Byiu99KH1hcUsnMhB2ktO7V7D0RkS66E1CtsetmM8sHOoiuxye91Nm994WR7wMGky/1uiQRkQGjO2NQz5pZFnA/sApwwMNxrWqYKN5ey96GNs5PXwUFcyF9aO4QLCJyIo4ZULGNCl+LbSj4lJk9BwScc/X9Ut0Qt7hkN/kJ+8muLYHZ3/a6HBGRAeWYXXzOuQjwqy732xROfSMccTxfWsUt+VujDRp/EhE5SHfGoF4zs38w09mjfWlFeQ3VDW1cmrAGRoyD0ad6XZKIyIDSnYD6EtHFYdvMbL+ZNZjZ/jjXNeQtLqkkIzHMmH3vRk/OVf6LiBykOytJaGv3PhaOOF4oreSm8ZXY7iZ174mIHMFxA8rMzj9S+6EbGEr3Ld8WZF9jO5+YuA4SUqDoiN9iEZFhrTvTzL/e5XYAOAtYCVwcl4qGgcUllaQk+pgYfAsmXQCJKV6XJCIy4HSni+8TXe+b2Xjgp3GraIgLhSO8WFrFwkmt+LZvh3Pv8LokEZEBqTuTJA5VAczo60KGi+Xbagg2tfPpjNJowxStXi4iciTdGYP6BdHVIyAaaLOIrighJ+C5kkpSk/xMqV8CY06DzHFelyQiMiB1ZwyquMvtEPC4c25JnOoZ0kLhCC+tr+ITU1Pwb10O593tdUkiIgNWdwLqSaDVORcGMDO/maU655rjW9rQs6yshpqmdj6bUw5bIppeLiJyDN1aSQLoOs0sBXg1PuUMbYvX7SYtyc+pTUshbSTkz/a6JBGRAas7ARXous177HZq/Eoamjpis/c+OiMX/9ZXo5MjfCcyR0VEZHjozm/IJjM7o/OOmc0BWuJX0tC0dGuQ2uYOPju2Clrro8sbiYjIUXVnDOoO4M9mtpvolu9jiG4BLz2wuKSS9OQEZre+Db5EOOkir0sSERnQunOi7gozmw5MizW975zriG9ZQ0tHOMKL66u49OTRJGx5GQrPhWQtcSgicizH7eIzs9uANOdcqXOuFEg3s3+Jf2lDx5It+6hv6eDaog7Y975m74mIdEN3xqBuiu2oC4Bzrha4KX4lDT2LSyrJSE7g7NCKaMPUj3pbkIjIINCdgPJ33azQzPxAUvxKGlraQ9GTcw907+VNg5xJXpclIjLgdSegXgSeMLNLzOwS4HHghfiWNXQs2bKP/a0hrpqRAeXvaPaeiEg3dWcW3zeBm4FbYvdLiM7kk254rqSSjEACC3zrINKh8ScRkW467hGUcy4CLAfKie4FdTGwMb5lDQ1toTAvb6jioyePIXHLKxDIhPFne12WiMigcNSAMrOpZvZdM9sE/ALYAeCcu8g598vevKmZ3W5mpWa23szuOORrXzMzZ2Z5vXmPgWDJln00tIa48rTRsPklmHwp+Ltz0CoiIsc6gtpE9GjpSufcuc65XwDh3r6hmZ1KdBbgWcBM4Eozmxz72njgo8TCcLB7rqSSEYEEzk3dCU3V6t4TEemBYwXUNUAl8LqZPRybIGHHeHx3zQCWO+eanXMh4M3YewH8BPgGH+4/NWg553jrg2oumj6KxK0vg/lh8iVelyUiMmgcNaCcc884564HpgOvE13yaJSZPWhmvTmRpxQ4z8xyzSwVuBwYb2ZXA7ucc2t78doDxpa9jexrbGf+SbnwwQswYR6k5nhdlojIoNGdSRJNzrk/Ouc+ARQAq4nO7DshzrmNwH8CLxOdwr4GSAa+BXzneM83s5vNrNjMiqurq0+0jLhbVhYEYMGoNqhap+nlIiI91KP9Hpxztc65h5xzveqrcs494pyb45w7H6gF1gNFwFozKycahKvM7LDp7LH3n+ucmzty5MjelBFXy8pqyM8MMG7v29EGjT+JiPSIJxsSmdmo2PUEouNPjzrnRjnnCp1zhUAFcIZzrsqL+nrLOceysiDzJuVim1+C7ELIm+p1WSIig4pXc56fMrNcoAO4retaf0PB5r2NBJvaWVCYCq+8AXNuBOuL+SUiIsOHJwHlnDvvOF8v7KdS4qJz/On8hE0QatX4k4jICdCe43GwdGuQcVkp5O1+HZLSYeICr0sSERl0FFB9LBJxLN9WEx1/Kvs7FF0ACclelyUiMugooPrYB3sbqGlq56Kx7VBbDkXH7M0UEZGjUED1sWVbo+NP5/g3RRvUvScickIUUH1sWVkNBdkp5O5bAcmZMPoUr0sSERmUFFB9KBJxLNsWPf+J7Utg4jng83tdlojIoKSA6kPv72mgrrmDi/IjENyi7j0RkV5QQPWhzvOf5iXExp8KFVAiIidKAdWHlm4NMj4nhdx9xZCUAWNmel2SiMigpYDqI53nP50zKRfKl8CEs7V7rohILyig+sjGqv3Ut3Rw/jgfVG/U+JOISC8poPrIsrIaAOYnvh9tKDzXw2pERAY/BVQfWVYWZGJuKjnVKyAxFcbO8rokEZFBTQHVB8IRx/KyIPOKYuNPBWdCQpLXZYmIDGoKqD6wsXI/+1tDnDfeD3tK1b0nItIHFFB9oPP8p/mJmwGnCRIiIn1AAdUHlpUFKewcf/Inw7g5XpckIjLoKaB6Kdx5/tNJuVD+TnT8KTHgdVkiIoOeAqqXNlbup6E1xLnjk6GqRMsbiYj0EQVULy2N7f80P2kLuIjGn0RE+ogCqpeWlQWZlJdG9t73wJcY7eITEZFeU0D1Qigc4b1tNZzduf/TuDmQlOp1WSIiQ4ICqhc2VO6noS3EggkB2L1a408iIn1IAdULnec/LUgug0gIJs73uCIRkaFDAdULS7cGmTQyNv5kfhh/ttcliYgMGQqoExQKR1hRXhvd/2n7EsifBckZXpclIjJkKKBO0Prd+2lsCzF/YhrsWqnp5SIifUwBdYKWdq6/l1wG4XYtECsi0scUUCdoWVmQyaPSya4uBvPBhHlelyQiMqQooE5ARzjCim01zJuUE11/b8xpEMj0uiwRkSFFAXUCSnfV09QeZv7EDKhYARPVvSci0tcUUCdgWVkNAPNTtkOoVSfoiojEgQLqBCwtCzJlVDpZe9+LNkw4x9uCRESGIAVUD3WEIxSXd+7/tARGnQKpOV6XJSIy5JL/9hgAABIjSURBVCigemjdrnqa28OcUzgCdr6n7j0RkThRQPXQgf2fUnZCR5NO0BURiRMFVA8tKwsybXQGmZ3jTwooEZG4UED1QHsoQnF5bez8pyWQNw3SR3pdlojIkKSA6oF1u+po6QgzrygLdizT+JOISBx5ElBmdruZlZrZejO7I9Z2v5ltMrMSM/uLmWV5UduxHDj/Ka0S2hvUvSciEkf9HlBmdipwE3AWMBO40swmA68ApzrnTgc+AO7p79qOZ+nWINPHZJC5Z3m0QQvEiojEjRdHUDOA5c65ZudcCHgTuMY593LsPsAyoMCD2o6qPRSheHsN8zr3f8o5CTLGeF2WiMiQ5UVAlQLnmVmumaUClwPjD3nMF4AXjvRkM7vZzIrNrLi6ujrOpX6opKKO1o4I84qyYfu72t5dRCTO+j2gnHMbgf8EXgZeBNYA4c6vm9m/ASHgsaM8/yHn3Fzn3NyRI/tvBt3SrUHMYH56FbTWqXtPRCTOPJkk4Zx7xDk3xzl3PlBLdMwJM7sRuBL4nHPOeVHb0SzbFmT6mBGM2KPzn0RE+oNXs/hGxa4nANcAfzSzjwHfAK5yzjV7UdfRtIXCH57/tP0dyJoAWYf2SoqISF9K8Oh9nzKzXKADuM05V2dmvwSSgVfMDGCZc+4Wj+o7yNqd9bSFIswryoHn34Upl3ldkojIkOdJQDnnzjtC22QvaumOZWXR8adzRlRDc1An6IqI9AOtJNENS7cGmaHxJxGRfqWAOo7WjjCrdtR+uP/TiHGQXeh1WSIiQ54C6jjW7qz7cPxp+5Lo0VN0jExEROLIq0kSg8bS2PjTvMxaaNyjE3RFhomOjg4qKipobW31upQhIxAIUFBQQGJiYrcer4A6jmVlQU7JH0GG1t8TGVYqKirIyMigsLAQU69JrznnCAaDVFRUUFRU1K3nqIvvGKLjT3XMK4qNP6WNgtwBO9lQRPpQa2srubm5Cqc+Ymbk5ub26IhUAXUMq3fU0d51/KlQ408iw4nCqW/19PupgDqGZWVBfAZn5TTA/l2aXi4i0o8UUMewtCzIKfmZjKjS+JOI9K9gMMisWbOYNWsWY8aMYdy4cQfut7e3H/O5xcXFfPWrXz3ue8yfP7AnfWmSxFG0doRZs6OOGxcUQvljkJoLI6d7XZaIDBO5ubmsWbMGgHvvvZf09HTuvvvuA18PhUIkJBz5V/jcuXOZO3fucd/j3Xff7Zti40QBdRSrdtTSHo5EF4h9cUl0ern6o0WGpe89u54Nu/f36WuenD+C737ilB4958YbbyQQCLB69WoWLFjA9ddfz+23305rayspKSn89re/Zdq0abzxxhs88MADPPfcc9x7773s2LGDsrIyduzYwR133HHg6Co9PZ3GxkbeeOMN7r33XvLy8igtLWXOnDn84Q9/wMx4/vnnueuuu0hLS2PBggWUlZXx3HPP9en34mgUUEexbGt0/OnMnBao2w7z/sXrkkREqKio4N1338Xv97N//37efvttEhISePXVV/nWt77FU089ddhzNm3axOuvv05DQwPTpk3j1ltvPexcpNWrV7N+/Xry8/NZsGABS5YsYe7cuXzpS1/irbfeoqioiIULF/bXxwQUUEe1rKyG08ZlklHZOf6kCRIiw1VPj3Ti6brrrsPv9wNQX1/PDTfcwObNmzEzOjo6jvicK664guTkZJKTkxk1ahR79uyhoKDgoMecddZZB9pmzZpFeXk56enpTJo06cB5SwsXLuShhx6K46c7mCZJHEFLe5g1O+uYNyk3uv9TIBNGnex1WSIipKWlHbj97//+71x00UWUlpby7LPPHvUco+Tk5AO3/X4/oVDohB7T3xRQR/Dh+FPsBN0J88Hn97osEZGD1NfXM27cOAB+97vf9fnrT5s2jbKyMsrLywF44okn+vw9jkUBdQTLyoL4fcaZeW1Qs1XdeyIyIH3jG9/gnnvuYfbs2XE54klJSeG///u/+djHPsacOXPIyMggMzOzz9/naMw5129v1tfmzp3riouL+/x1r33wXToijr+eXwlPfRFueh3GndHn7yMiA9fGjRuZMWOG12V4rrGxkfT0dJxz3HbbbUyZMoU777zzhF/vSN9XM1vpnDtsXryOoA7R3B5ibUUd50zKjS5vlJQBY073uiwREU88/PDDzJo1i1NOOYX6+nq+9KUv9dt7axbfIVZtr6Mj7KLnP72yBCbMA7++TSIyPN155529OmLqDR1BHWJp2b7o+NPIMOx7X+NPIiIeUUAdYllZDacXZJJW9V60YaLW3xMR8YICqovm9hBrO89/Kl8CiamQP8vrskREhiUFVBfF5bWEIi52gu4SGH8W+Lu3NbGIiPQtBVQXy8qCJPiMuaOAPevVvScinrnooot46aWXDmr76U9/yq233nrEx1944YV0nnZz+eWXU1dXd9hj7r33Xh544IFjvu8zzzzDhg0bDtz/zne+w6uvvtrT8vuEAqqLpWXBLuNPThMkRMQzCxcuZNGiRQe1LVq0qFsLtj7//PNkZWWd0PseGlDf//73+chHPnJCr9Vbmj8d09QWoqSinlsumATliyEhAOPmeF2WiAwEL/wrVK3r29cccxp8/EdH/fK1117Lt7/9bdrb20lKSqK8vJzdu3fz+OOPc9ddd9HS0sK1117L9773vcOeW1hYSHFxMXl5efzwhz/k0UcfZdSoUYwfP545c6K/1x5++GEeeugh2tvbmTx5Mr///e9Zs2YNf/vb33jzzTf5wQ9+wFNPPcV9993HlVdeybXXXstrr73G3XffTSgU4swzz+TBBx8kOTmZwsJCbrjhBp599lk6Ojr485//zPTpvd8/T0dQMcXbawkfGH96BwrOhITk4z9RRCQOcnJyOOuss3jhhReA6NHTpz/9aX74wx9SXFxMSUkJb775JiUlJUd9jZUrV7Jo0SLWrFnD888/z4oVKw587ZprrmHFihWsXbuWGTNm8MgjjzB//nyuuuoq7r//ftasWcNJJ5104PGtra3ceOONPPHEE6xbt45QKMSDDz544Ot5eXmsWrWKW2+99bjdiN2lI6iYpVuDJPqNOaP90b+Uzv+G1yWJyEBxjCOdeOrs5rv66qtZtGgRjzzyCH/605946KGHCIVCVFZWsmHDBk4//cir3bz99tt86lOfIjU1FYCrrrrqwNdKS0v59re/TV1dHY2NjVx22WXHrOX999+nqKiIqVOnAnDDDTfwq1/9ijvuuAOIBh7AnDlzePrpp3v92UFHUAcsKwsysyCL1KoV4CIafxIRz1199dW89tprrFq1iubmZnJycnjggQd47bXXKCkp4YorrjjqFhvHc+ONN/LLX/6SdevW8d3vfveEX6dT53YdfblVhwIKaGwLsW5Xfez8p3fAnxTt4hMR8VB6ejoXXXQRX/jCF1i4cCH79+8nLS2NzMxM9uzZc6D772jOP/98nnnmGVpaWmhoaODZZ5898LWGhgbGjh1LR0cHjz322IH2jIwMGhoaDnutadOmUV5ezpYtWwD4/e9/zwUXXNBHn/TIFFDAivKaLuNP70YnRySmeF2WiAgLFy5k7dq1LFy4kJkzZzJ79mymT5/OZz/7WRYsOHZPzxlnnMFnPvMZZs6cycc//nHOPPPDP7zvu+8+zj77bBYsWHDQhIbrr7+e+++/n9mzZ7N169YD7YFAgN/+9rdcd911nHbaafh8Pm655Za+/8BdaLsN4D9e2Mhv3tlGyT0LSPmvIjj3Trjk3/ugQhEZrLTdRnxou40eWrY1yKzxWaRUFYMLw8T5XpckIjLsDfuAamjtYN2u+g/3fzI/jD/b67JERIa9YR9QxeW1RBwfLhCbPxuS070uS0QGgME8BDIQ9fT7OewDamlZkCS/jzPyk2HXSk0vFxEgOikgGAwqpPqIc45gMEggEOj2c4b9ibrLyoLMmpBFoGoVRDq0QKyIAFBQUEBFRQXV1dVelzJkBAIBCgoKuv34YR1Q+1s7KN1Vz5cvngLbnwTzRbd4F5FhLzExkaKiIq/LGNY86eIzs9vNrNTM1pvZHbG2HDN7xcw2x66z413Him01sfGnnOj405jTITAi3m8rIiLd0O8BZWanAjcBZwEzgSvNbDLwr8BrzrkpwGux+3G1rCxIUoKPM/JToGIFFKp7T0RkoPDiCGoGsNw51+ycCwFvAtcAVwOPxh7zKPDJeBeyrKyG2eOzCOxdC+E2mKgJEiIiA0W/ryRhZjOAvwLnAC1Ej5aKgX90zmXFHmNAbef9Q55/M3Bz7O404P3+qLsX8oB9XhfRC6rfW6rfe4P9MwyG+ic650Ye2ujJUkdm9kXgX4AmYD3QBtzYNZDMrNY5F/dxqHgzs+IjLeExWKh+b6l+7w32zzCY6/dkkoRz7hHn3Bzn3PlALfABsMfMxgLErvd6UZuIiAwMXs3iGxW7nkB0/OmPwN+AG2IPuYFoN6CIiAxTXp0H9ZSZ5QIdwG3OuToz+xHwp1j333bg0x7V1tce8rqAXlL93lL93hvsn2HQ1j+ot9sQEZGha9ivxSciIgOTAkpERAYkBVScmNl4M3vdzDbElnS63euaesrM/Ga22sye87qWE2FmWWb2pJltMrONZnaO1zX1hJndGfu/U2pmj5tZ95eB9oCZ/cbM9ppZaZe2fl/C7EQdpf77Y/9/SszsL2Z22LmZA8mRPkOXr33NzJyZ5XlR24lQQMVPCPiac+5kYB5wm5md7HFNPXU7sNHrInrhZ8CLzrnpRJfVGjSfxczGAV8F5jrnTgX8wPXeVnVcvwM+dkhbvy9h1gu/4/D6XwFOdc6dTvR0mHv6u6ge+h2HfwbMbDzwUWBHfxfUGwqoOHHOVTrnVsVuNxD95TjO26q6z8wKgCuA//W6lhNhZpnA+cAjAM65dudcnbdV9VgCkGJmCUAqsNvjeo7JOfcWUHNIc78vYXaijlS/c+7l2JJsAMuA7u8V4YGj/BsA/AT4BjCoZsUpoPqBmRUCs4Hl3lbSIz8l+h864nUhJ6gIqAZ+G+um/F8zS/O6qO5yzu0CHiD6F28lUO+ce9nbqk7IaOdcZex2FTDay2J66QvAC14X0VNmdjWwyzm31utaekoBFWdmlg48BdzhnNvvdT3dYWZXAnudcyu9rqUXEoAzgAedc7OJLqs1kLuXDhIbq7maaNDmA2lm9nlvq+odFz2nZVD9Bd/JzP6NaLf9Y17X0hNmlgp8C/iO17WcCAVUHJlZItFwesw597TX9fTAAuAqMysHFgEXm9kfvC2pxyqACudc51Hrk0QDa7D4CLDNOVftnOsAngbme1zTiRj0S5iZ2Y3AlcDn3OA7cfQkon/krI39PBcAq8xsjKdVdZMCKk5iK7I/Amx0zv1/XtfTE865e5xzBc65QqID8393zg2qv96dc1XATjObFmu6BNjgYUk9tQOYZ2apsf9LlzCIJnl0MaiXMDOzjxHt6r7KOdfsdT095Zxb55wb5ZwrjP08VwBnxH4+BjwFVPwsAP6R6NHHmtjlcq+LGma+AjxmZiXALOD/eVxPt8WO/J4EVgHriP6sDugla8zscWApMM3MKmLLlv0IuNTMNhM9KvyRlzUey1Hq/yWQAbwS+xn+tadFHsdRPsOgpaWORERkQNIRlIiIDEgKKBERGZAUUCIiMiApoEREZEBSQImIyICkgBLpB2YW7nK6wRoz67NVLcys8EirV4sMdl5t+S4y3LQ452Z5XYTIYKIjKBEPmVm5mf3YzNaZ2XtmNjnWXmhmf4/tQ/SamU2ItY+O7Uu0NnbpXP7Ib2YPx/aPetnMUjz7UCJ9RAEl0j9SDuni+0yXr9U7504jumrBT2NtvwAeje1D9Bjw81j7z4E3nXMzia4tuD7WPgX4lXPuFKAO+Ic4fx6RuNNKEiL9wMwanXPpR2gvBy52zpXFFheucs7lmtk+YKxzriPWXumcyzOzaqDAOdfW5TUKgVdimwJiZt8EEp1zP4j/JxOJHx1BiXjPHeV2T7R1uR1G48syBCigRLz3mS7XS2O33+XDLd4/B7wdu/0acCuAmfljOweLDEn6K0ukf6SY2Zou9190znVONc+OrbjeBiyMtX2F6G7AXye6M/A/x9pvBx6KrVIdJhpWlYgMQRqDEvFQbAxqrnNun9e1iAw06uITEZEBSUdQIiIyIOkISkREBiQFlIiIDEgKKBERGZAUUCIiMiApoEREZED6/wGjD2FN4D4KkQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def predict_sentiment(model, sentence):\n",
        "\n",
        "    model.eval()\n",
        "    tokenized = tokenizer(sentence)\n",
        "    indexed = [vocab[t] for t in tokenized]\n",
        "    length = [len(indexed)]\n",
        "    tensor = torch.LongTensor(indexed).to(DEVICE)\n",
        "    tensor = tensor.unsqueeze(1)\n",
        "    length_tensor = torch.LongTensor(length)\n",
        "    prediction = F.softmax(model(tensor), dim=1)\n",
        "    return prediction.to(\"cpu\").squeeze(dim=0).detach().numpy()\n",
        "\n",
        "print('Probability positive:')\n",
        "predict_sentiment(model, \"This is such an awesome movie, I really love it!\")[1]\n"
      ],
      "metadata": {
        "id": "V3Z8HBMZkWG2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0ce86dc4-0246-4404-dd99-6e4bd70cca97"
      },
      "execution_count": 146,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Probability positive:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.99975926"
            ]
          },
          "metadata": {},
          "execution_count": 146
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print('Probability negative:')\n",
        "predict_sentiment(model, \"I really hate this movie. It is really bad and sucks!\")[0]"
      ],
      "metadata": {
        "id": "jS-4s-9bk_oo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c39e5ae9-c0e3-4972-9b72-752ad8507ad5"
      },
      "execution_count": 147,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Probability negative:\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.99999726"
            ]
          },
          "metadata": {},
          "execution_count": 147
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 148,
      "metadata": {
        "id": "7lRusB3dF80X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f51913f6-4eb5-4ac8-cccf-ac141bd3cb9a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "pandas    : 1.3.5\n",
            "numpy     : 1.21.6\n",
            "matplotlib: 3.2.2\n",
            "torchtext : 0.14.0\n",
            "torch     : 1.13.0+cu116\n",
            "\n"
          ]
        }
      ],
      "source": [
        "%watermark -iv"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "rnn_lstm_packed_imdb.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.6"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}