TransformerBasic/Transformer基础实现StepByStep.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Attention is all u need",
      "provenance": [],
      "collapsed_sections": [
        "kw5mAgeonZYj"
      ],
      "toc_visible": true,
      "mount_file_id": "1z_sFzq4vVZnRK4iT1JF6n0UHF9tcG8lR",
      "authorship_tag": "ABX9TyPIvBL15nO0foBKkUUnsz/g",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/HzcIrving/DeepLearning_PlayGround/blob/main/TransformerBasic/Transformer%E5%9F%BA%E7%A1%80%E5%AE%9E%E7%8E%B0StepByStep.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NYK_QGn-yld6"
      },
      "source": [
        "Transformer的实现，配合Notion笔记的学习，实现过程\n",
        "- Transformer最流行的变体Variant就是BERT(Bidirectional Encoder Representations from Transformers)\n",
        "- 预训练的BERT是最常用于替代embedding层的； "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KvwH1PnJy6PN"
      },
      "source": [
        "## 数据集准备\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m2mKvxv_z8Wn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5d88a768-b129-4173-b334-4a681adb562d"
      },
      "source": [
        "!pip3 install spacy"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: spacy in /usr/local/lib/python3.7/dist-packages (2.2.4)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.4.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy) (57.4.0)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.1.3)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (4.62.3)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (7.4.0)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.0)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.23.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.6)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.8.2)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.6)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.6)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.19.5)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.5)\n",
            "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy) (4.8.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy) (3.10.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy) (3.6.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2021.10.8)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aAgs5ANl0C_W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "df9fe97a-7d13-4907-8f3f-198f8b40fd91"
      },
      "source": [
        "!python3 -m spacy download en_core_web_sm"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting en_core_web_sm==2.2.5\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz (12.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 12.0 MB 1.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.7/dist-packages (from en_core_web_sm==2.2.5) (2.2.4)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.6)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.8.2)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.4.1)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (7.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (4.62.3)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.23.0)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.1.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.0.6)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.5)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.6)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (57.4.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (4.8.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.10.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.6.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (1.24.3)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_sm')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n6mMAJFI0HYF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b57a3099-fb64-416e-e59f-daa157c5aa90"
      },
      "source": [
        "!python3 -m spacy download de_core_news_sm"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting de_core_news_sm==2.2.5\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.2.5/de_core_news_sm-2.2.5.tar.gz (14.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 14.9 MB 693 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.7/dist-packages (from de_core_news_sm==2.2.5) (2.2.4)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (1.19.5)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (1.0.6)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (2.23.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (57.4.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (2.0.6)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (7.4.0)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (3.0.6)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (0.8.2)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (4.62.3)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (1.0.0)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (1.1.3)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (0.4.1)\n",
            "Requirement already satisfied: importlib-metadata>=0.20 in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->de_core_news_sm==2.2.5) (4.8.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->de_core_news_sm==2.2.5) (3.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->de_core_news_sm==2.2.5) (3.10.0.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->de_core_news_sm==2.2.5) (2021.10.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->de_core_news_sm==2.2.5) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->de_core_news_sm==2.2.5) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->de_core_news_sm==2.2.5) (1.24.3)\n",
            "Building wheels for collected packages: de-core-news-sm\n",
            "  Building wheel for de-core-news-sm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for de-core-news-sm: filename=de_core_news_sm-2.2.5-py3-none-any.whl size=14907055 sha256=6559b8cb31f670d648808977c3b946823fb33392914b4318325cca1fe87ecb98\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-ct47e4b5/wheels/00/66/69/cb6c921610087d2cab339062345098e30a5ceb665360e7b32a\n",
            "Successfully built de-core-news-sm\n",
            "Installing collected packages: de-core-news-sm\n",
            "Successfully installed de-core-news-sm-2.2.5\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('de_core_news_sm')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5pHrRX0gy89U"
      },
      "source": [
        "import torch \n",
        "import torch.nn as nn \n",
        "import torch.optim as optim\n",
        "\n",
        "import torchtext\n",
        "from torchtext.legacy.datasets import Multi30k\n",
        "from torchtext.legacy.data import Field, BucketIterator \n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "\n",
        "import spacy\n",
        "import de_core_news_sm\n",
        "import en_core_web_sm\n",
        "import numpy as np\n",
        "\n",
        "import random\n",
        "import math\n",
        "import time"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EVEqmFjvzYZ0"
      },
      "source": [
        "SEED = 1234\n",
        "\n",
        "random.seed(SEED)\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.cuda.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZDTl6xN313Do",
        "outputId": "3c5d0a84-6450-47b2-9ecf-a1ee6bb6e775"
      },
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EJ8yicZbzb9T"
      },
      "source": [
        "### 创建tokenizers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uc2flOHUoqo8"
      },
      "source": [
        "分词的目的就是把“You are my friends” -> [\"You\",\"are\",\"my\",\"friends\"]\n",
        "\n",
        "英文分词就确实贼简单，一个空格分就完事了，但中文分词就不是了，首先的问题就是python也看不懂中文，不知咋分。另外，如果单纯的变成【“你”，“是”，“我”，“的”，“朋”，“友”】，那么会导致一个问题，汉字介么多，你的词典就需要很大很大，我们更希望要那些高频的，然后某些文字是固定成一个词语的，至少朋友得搞在一起吧：）【“你”，“是”，“我”，“的”，“朋友”】\n",
        "\n",
        "这个时候你就需要用上开源的分词工具，在这里我们用的是spacy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q0m8UmiAzhPL"
      },
      "source": [
        "spacy_de = de_core_news_sm.load() #German \n",
        "spacy_en = en_core_web_sm.load() #English "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N0W6tzcmzm2T"
      },
      "source": [
        "def tokenize_de(text):\n",
        "  \"\"\"\n",
        "  Tokenizes German text from a string into a list of strings\n",
        "  \"\"\"\n",
        "  return [tok.text for tok in spacy_de.tokenizer(text)]\n",
        "\n",
        "def tokenize_en(text):\n",
        "  \"\"\"\n",
        "  Tokenizes English text from a string into a list of strings\n",
        "  \"\"\"\n",
        "  return [tok.text for tok in spacy_en.tokenizer(text)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hMnBWlvipBTd"
      },
      "source": [
        "torchtext的Field可以方便地处理数据。\n",
        "\n",
        "我们把原始数据“德语”以及目标数据“英语”加上初始和结束的标志和, 然后把每个单词变成小写："
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ch-XPvoB0xNo"
      },
      "source": [
        "SRC = Field(tokenize = tokenize_de, \n",
        "            init_token = '<sos>', \n",
        "            eos_token = '<eos>', \n",
        "            lower = True, \n",
        "            batch_first = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XoG_MqAP1DLH"
      },
      "source": [
        "TRG = Field(tokenize = tokenize_en, \n",
        "            init_token = '<sos>', \n",
        "            eos_token = '<eos>', \n",
        "            lower = True, \n",
        "            batch_first = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DGYGgBL51FHo"
      },
      "source": [
        "### 读取Multi30k数据集，并创建词库\n",
        "\n",
        "Multi30k数据集\n",
        "我们直接采用torchtext里面自带的Multi30k\n",
        "\n",
        "\n",
        "```\n",
        "train_data, valid_data, test_data = Multi30k.splits(exts = ('.de', '.en'),fields = (SRC, TRG))\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n1XbIJP71w6t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fe0a63dc-37dc-488c-b4f5-fa4b8ef3767e"
      },
      "source": [
        "train_data, valid_data, test_data = Multi30k.splits(exts = ('.de', '.en'),  fields = (SRC, TRG))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "downloading training.tar.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1.21M/1.21M [00:02<00:00, 590kB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "downloading validation.tar.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 46.3k/46.3k [00:00<00:00, 91.0kB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "downloading mmt_task1_test2016.tar.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 66.2k/66.2k [00:00<00:00, 85.2kB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1s2hnkq9pOvh"
      },
      "source": [
        "build_vocab\n",
        "接下来我们就需要根据原数据和目标数据建立词典了。因为我们需要为每个单词对应一个数字，"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oI5Nmijb2y5_"
      },
      "source": [
        "SRC.build_vocab(train_data, min_freq = 2)\n",
        "TRG.build_vocab(train_data, min_freq = 2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KACeWkkL1xyb"
      },
      "source": [
        "### 定义data iterator \n",
        "\n",
        "iterators\n",
        "类似于图像当中把图像数据集放到dataloader里面，文本信息采用的是torchtext里面的BucketIterator. \n",
        "\n",
        "train_data可以看上面的例子，就是一串数组。这个迭代器主要的两个功能：\n",
        "- 根据我们之前建立好的词典把对应的单词转换成数字，并且转换为Tensor张量\n",
        "- 它会建立batches，建立的这个batch可以减少填充padding的数量，包括原数据和目标数据\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZPtvxqa51-ib"
      },
      "source": [
        "BATCH_SIZE = 128 \n",
        "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n",
        "    (train_data, valid_data, test_data), \n",
        "     batch_size = BATCH_SIZE,\n",
        "     device = device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-rFGCcxU2CUp"
      },
      "source": [
        "### 可视化数据封装"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wq4tzFh_2FBO"
      },
      "source": [
        "batch_idx = 1 \n",
        "count  = 0 \n",
        "data = next(iter(train_iterator)) \n",
        "for idx in data.src[:,batch_idx].cpu().numpy():\n",
        "  count += 1 \n",
        "  print(SRC.vocab.itos[idx],end=' ') \n",
        "print(\"German Example Length:\",count) \n",
        "count = 0 \n",
        "print()\n",
        "for idx in data.trg[:,batch_idx].cpu().numpy():\n",
        "  count += 1 \n",
        "  print(TRG.vocab.itos[idx],end=' ') \n",
        "print(\"Eng Example Length:\",count)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YHL-jHOf2Pl7"
      },
      "source": [
        "### 查看数据的封装形式"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KWfF3CCL5D3R"
      },
      "source": [
        "for i, it in enumerate(iter(train_iterator)):\n",
        "  if i > 10:\n",
        "    break \n",
        "  src = it.src # German \n",
        "  trg = it.trg # Eng \n",
        "  print(src.shape, trg.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U58r2am75LAG"
      },
      "source": [
        "## 模型构建--Encoder\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TRq-LpWT7w8N"
      },
      "source": [
        "### Basic EncoderLayer  \n",
        "1. MultiHead-SelfAttentionLayer \n",
        "2. LayerNorm \n",
        "3. positionwise_feedforward \n",
        "4. Skipconnect"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-Fnwz8W78QS-"
      },
      "source": [
        "#### MultiHead Attention Layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tSarqYuK8UiL"
      },
      "source": [
        "class MultiHeadAttentionLayer(nn.Module): \n",
        "  def __init__(self, hid_dim, n_heads, dropout, device):\n",
        "    super().__init__()\n",
        "    \n",
        "    assert hid_dim % n_heads == 0\n",
        "    \n",
        "    self.hid_dim = hid_dim\n",
        "    self.n_heads = n_heads\n",
        "    self.head_dim = hid_dim // n_heads\n",
        "    \n",
        "    # Q \n",
        "    self.fc_q = nn.Linear(hid_dim, hid_dim) \n",
        "    # K\n",
        "    self.fc_k = nn.Linear(hid_dim, hid_dim) \n",
        "    # V \n",
        "    self.fc_v = nn.Linear(hid_dim, hid_dim)\n",
        "    \n",
        "    self.fc_o = nn.Linear(hid_dim, hid_dim)\n",
        "    \n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "    \n",
        "    self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device) \n",
        "  \n",
        "  def forward(self,query,key,value,mask=None):\n",
        "    batch_size = query.shape[0] \n",
        "\n",
        "    # query:[bs x query_len x hid_dim] \n",
        "    # key :[bs x key_len x hid_dim]\n",
        "    # value:[bs x value_len x hid_dim] \n",
        "\n",
        "    #Q = [batch size, query len, hid dim]\n",
        "    #K = [batch size, key len, hid dim]\n",
        "    #V = [batch size, value len, hid dim] \n",
        "    Q = self.fc_q(query)  \n",
        "    K = self.fc_k(key)\n",
        "    V = self.fc_v(value)  \n",
        "\n",
        "\n",
        "    # dims:[batchsize, n_heads, length, hid_dim/n_heads]\n",
        "    Q = Q.view(batch_size,-1,self.n_heads,self.head_dim).permute(0,2,1,3)\n",
        "    K = K.view(batch_size,-1,self.n_heads,self.head_dim).permute(0,2,1,3) \n",
        "    V = V.view(batch_size,-1,self.n_heads,self.head_dim).permute(0,2,1,3) \n",
        "\n",
        "    # dims:[bs, n_heads, query_length, key_length]\n",
        "    energy = torch.matmul(Q, K.permute(0, 1, 3, 2)) / self.scale \n",
        "\n",
        "    # Mask Operation: 防止影响softmax的输出结果\n",
        "    if mask is not None:\n",
        "      energy = energy.masked_fill(mask==0,-1e10)\n",
        "    \n",
        "    # Softmax \n",
        "    # dim:[bs, n_heads, query_length, key_length]\n",
        "    attention = torch.softmax(energy,dim=-1) \n",
        "\n",
        "    # dim:[bs, n_heads, query_length, key_length] * [bs, n_heads, length, head_dims=hid_dim/n_heads]\n",
        "    # dim:[bs, n_heads, length, head_dims]\n",
        "    x = torch.matmul(self.dropout(attention),V)  \n",
        "    # dim:[bs, length, n_heads, head_dims]\n",
        "    x = x.permute(0,2,1,3).contiguous()  \n",
        "    # dims:[bs, length, n_heads*head_dims]\n",
        "    x = x.view(batch_size, -1, self.hid_dim)\n",
        "\n",
        "    x = self.fc_o(x) \n",
        "\n",
        "    return x, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AzkzGy6OFBnm"
      },
      "source": [
        "#### Position-wise FeedForward Layer "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEtlIT-DFFmN"
      },
      "source": [
        "# 就是Fully Connected Layer \n",
        "class PositionwiseFeedforwardLayer(nn.Module):\n",
        "  def __init__(self, hid_dim, pf_dim, dropout):\n",
        "    super().__init__()\n",
        "    \n",
        "    self.fc_1 = nn.Linear(hid_dim, pf_dim)\n",
        "    self.fc_2 = nn.Linear(pf_dim, hid_dim)\n",
        "    \n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "        \n",
        "  def forward(self, x):\n",
        "      \n",
        "    #x = [batch size, seq len, hid dim]\n",
        "    x = self.dropout(torch.relu(self.fc_1(x)))\n",
        "    #x = [batch size, seq len, pf dim]\n",
        "    x = self.fc_2(x)\n",
        "    #x = [batch size, seq len, hid dim]\n",
        "    return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M7MqsbJq9knO"
      },
      "source": [
        "#### EncoderLayer --- Block"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O-3S3xTIExLu"
      },
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "  \"\"\"Block\"\"\"\n",
        "  def __init__(self, hid_dim, n_heads, pf_dim, dropout, device):\n",
        "    super().__init__() \n",
        "    self.self_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
        "    self.ff_layer_norm = nn.LayerNorm(hid_dim)\n",
        "    self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
        "    self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim, pf_dim, dropout) \n",
        "\n",
        "    self.dropout = nn.Dropout(dropout) \n",
        "  \n",
        "  def forward(self,src,src_mask):\n",
        "    #src = [batch size, src len, hid dim]\n",
        "    #src_mask = [batch size, 1, 1, src len]  \n",
        "    \n",
        "    # Multi-head self attention \n",
        "    # query,key,value,mask=None\n",
        "    _src, _ = self.self_attention(src,src,src,src_mask) \n",
        "\n",
        "    # dropout, residual connection, layer norm \n",
        "    # dims: [bs, src_len, hid_dim] \n",
        "    src = self.self_attn_layer_norm(src+self.dropout(_src))   \n",
        "\n",
        "    # FC \n",
        "    _src = self.positionwise_feedforward(src) \n",
        "\n",
        "    # dropout, residual connection, layer norm \n",
        "    src = self.ff_layer_norm(src + self.dropout(_src))\n",
        "\n",
        "    return src"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XP3NbZLi5-mX"
      },
      "source": [
        "### 总体结构"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PYjuTfEt5zTX"
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "  def __init__(self, input_dim, hid_dim, n_layers, n_heads, pf_dim, dropout, device, max_length=100):\n",
        "    super().__init__() \n",
        "    self.device = device \n",
        "\n",
        "    # 字嵌入\n",
        "    self.tok_embedding = nn.Embedding(input_dim,hid_dim)  \n",
        "    # 位置嵌入  \n",
        "    # 我们采用的是learned positional embeddings即可学习的位置编码 （与BERT一样）\n",
        "    self.pos_embedding = nn.Embedding(max_length,hid_dim) \n",
        "\n",
        "    # Encoder Block  \n",
        "    # Encoder Layer 复用 \n",
        "    self.layers = nn.ModuleList([EncoderLayer(hid_dim,n_heads,pf_dim,dropout,device) for _ in range(n_layers)])\n",
        "\n",
        "    self.dropout = nn.Dropout(dropout) \n",
        "\n",
        "    self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device) \n",
        "\n",
        "  def forward(self, src, src_mask):\n",
        "    # src: [bs, src_len] \n",
        "    # src_mask: [bs, 1, 1, src_len] \n",
        "\n",
        "    batch_size = src.shape[0] \n",
        "    src_len = src.shape[1] \n",
        "\n",
        "    # [src_length]->[1,src_length]->[bs,src_length] same dimension as src \n",
        "    pos = torch.arange(0,src_len).unsqueeze(0).repeat(batch_size,1).to(self.device) \n",
        "    src = self.dropout((self.tok_embedding(src) * self.scale) + self.pos_embedding(pos))   \n",
        "    for layer in self.layers: \n",
        "      # 会看到输入的所有token，其中由于输入的句子小于我们固定好的长度的时候，\n",
        "      # 会有这个token,这个时候我们是不希望输入的单词关注到它的，所以当token为的时候，\n",
        "      # 我们会使用这个mask，令这个的value为0，否则都为1\n",
        "      src = layer(src,src_mask)   \n",
        "    return src"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "48n5JhGFoQ_K"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ld1DVrjJ7yO"
      },
      "source": [
        "## Decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2QHvn2FBO_zh"
      },
      "source": [
        "### Decoder Layer --- Block"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Myc54HTVPrZw"
      },
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "  def __init__(self, hid_dim, n_heads, pf_dim, dropout, device):\n",
        "    super().__init__()\n",
        "    \n",
        "    self.self_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
        "    self.enc_attn_layer_norm = nn.LayerNorm(hid_dim)\n",
        "    self.ff_layer_norm = nn.LayerNorm(hid_dim)\n",
        "    self.self_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
        "    self.encoder_attention = MultiHeadAttentionLayer(hid_dim, n_heads, dropout, device)\n",
        "    self.positionwise_feedforward = PositionwiseFeedforwardLayer(hid_dim, pf_dim, dropout)\n",
        "    self.dropout = nn.Dropout(dropout) \n",
        "\n",
        "  def forward(self,trg,enc_src,trg_mask,src_mask):\n",
        "    #trg = [batch size, trglen, hid dim]\n",
        "    #enc_src = [batch size, srclen, hid dim]\n",
        "    #trg_mask = [batch size, 1, 1, trglen]\n",
        "    #src_mask = [batch size, 1, 1, srclen] \n",
        "    \n",
        "    # self attention \n",
        "    _trg, _ = self.self_attention(trg, trg, trg, trg_mask)  \n",
        "\n",
        "    # dropout, residual connection and layer norm\n",
        "    # trg = [batch size, trg len, hid dim]\n",
        "    trg = self.self_attn_layer_norm(trg + self.dropout(_trg)) \n",
        "\n",
        "    # encoder_attention (Q,K) \n",
        "    # query: trg(decoder), key: enc_src(encoder), value: enc_src(encoder) \n",
        "    _trg, attention = self.encoder_attention(trg, enc_src, enc_src, src_mask) \n",
        "\n",
        "    # dropout, residual connection and layer norm\n",
        "    # [bs, trglen, hid_dim]\n",
        "    trg = self.enc_attn_layer_norm(trg + self.dropout(_trg)) \n",
        "\n",
        "    #positionwise feedforward\n",
        "    _trg = self.positionwise_feedforward(trg) \n",
        "    trg = self.ff_layer_norm(trg + self.dropout(_trg)) \n",
        "\n",
        "    return trg, attention "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gKLs4yK9R7fX"
      },
      "source": [
        "### 总体结构"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YZoXr_v1R9Za"
      },
      "source": [
        "class Decoder(nn.Module):\n",
        "  def __init__(self, output_dim, hid_dim, n_layers, n_heads, pf_dim, dropout, device,max_length = 100):\n",
        "    super().__init__()\n",
        "    \n",
        "    self.device = device\n",
        "    \n",
        "    self.tok_embedding = nn.Embedding(output_dim, hid_dim)\n",
        "    self.pos_embedding = nn.Embedding(max_length, hid_dim)\n",
        "    \n",
        "    self.layers = nn.ModuleList([DecoderLayer(hid_dim, n_heads, pf_dim, dropout, device) for _ in range(n_layers)])\n",
        "    self.fc_out = nn.Linear(hid_dim, output_dim)\n",
        "    \n",
        "    self.dropout = nn.Dropout(dropout)\n",
        "    self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)\n",
        "  \n",
        "  def forward(self, trg, enc_src, trg_mask, src_mask):\n",
        "    \n",
        "    #trg = [batch size, trg len]\n",
        "    #enc_src = [batch size, src len, hid dim]\n",
        "    #trg_mask = [batch size, 1, trg len, trg len]\n",
        "    #src_mask = [batch size, 1, 1, src len]\n",
        "            \n",
        "    batch_size = trg.shape[0]\n",
        "    trg_len = trg.shape[1]\n",
        "    \n",
        "    #pos = [batch size, trg len]\n",
        "    pos = torch.arange(0, trg_len).unsqueeze(0).repeat(batch_size, 1).to(self.device)\n",
        "\n",
        "    #trg = [batch size, trg len, hid dim]                 \n",
        "    trg = self.dropout((self.tok_embedding(trg) * self.scale) + self.pos_embedding(pos))\n",
        "\n",
        "    #trg = [batch size, trg len, hid dim]\n",
        "    #attention = [batch size, n heads, trg len, src len]\n",
        "    for layer in self.layers:\n",
        "      trg, attention = layer(trg, enc_src, trg_mask, src_mask)\n",
        "  \n",
        "    #output = [batch size, trg len, output dim]\n",
        "    output = self.fc_out(trg)\n",
        "    \n",
        "    return output, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zOdgfFMP2Zmt"
      },
      "source": [
        "### Seq2Seq 封装 \n",
        "\n",
        "- encapsulates(封装) the encoder and decoder. \n",
        "- handing the creation of the mask. \n",
        "\n",
        "The target mask is slightly more complicated. First, we create a mask for the `<pad>` tokens, as we did for the source mask. Next, we create a \"subsequent\" mask, `trg_sub_mask`, using `torch.tril`. This creates a diagonal matrix where the elements above the diagonal will be zero and the elements below the diagonal will be set to whatever the input tensor is. In this case, the input tensor will be a tensor filled with ones. So this means our `trg_sub_mask` will look something like this (for a target with 5 tokens):\n",
        "\n",
        "$$\\begin{matrix}\n",
        "1 & 0 & 0 & 0 & 0\\\\\n",
        "1 & 1 & 0 & 0 & 0\\\\\n",
        "1 & 1 & 1 & 0 & 0\\\\\n",
        "1 & 1 & 1 & 1 & 0\\\\\n",
        "1 & 1 & 1 & 1 & 1\\\\\n",
        "\\end{matrix}$$ \n",
        "\n",
        "- 屏蔽后面的\"token\" \n",
        "- 第一个target token的mask $[1,0,0,0,0]$ 只能看到第一个target token. \n",
        "- 第二个target token的mask $[1,1,0,0,0]$ 只能看到第一个与第二个target tokens. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9WH_GJBq2kat"
      },
      "source": [
        "class Seq2Seq(nn.Module):\n",
        "  def __init__(self, encoder, decoder, src_pad_idx, trg_pad_idx, device):\n",
        "    super().__init__() \n",
        "\n",
        "    self.encoder = encoder \n",
        "    self.decoder = decoder \n",
        "\n",
        "    self.src_pad_idx = src_pad_idx \n",
        "    self.trg_pad_idx = trg_pad_idx \n",
        "\n",
        "    self.device = device  \n",
        "\n",
        "  def make_src_mask(self, src):\n",
        "    # src: [bs, src_length]  \n",
        "    # src_mask: [bs, 1, 1, src_length]\n",
        "    src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)  \n",
        "    return src_mask \n",
        "  \n",
        "  def make_trg_mask(self, trg):\n",
        "    #trg = [batch size, trg len]\n",
        "    #trg_pad_mask = [batch size, 1, 1, trg len] \n",
        "    trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(2) \n",
        "\n",
        "    trg_len = trg.shape[1]  \n",
        "    #trg_sub_mask = [trg len, trg len]\n",
        "    trg_sub_mask = torch.tril(torch.ones((trg_len, trg_len),device = self.device)).bool()  \n",
        "\n",
        "    #补全[pad]\n",
        "    trg_mask = trg_pad_mask & trg_sub_mask \n",
        "\n",
        "    return trg_mask  \n",
        "  \n",
        "  def forward(self, src, trg):\n",
        "    # src:[bs, src_length] \n",
        "    # trg:[bs, trg_length] \n",
        "    # src_mask = [batch size, 1, 1, src len]\n",
        "    # trg_mask = [batch size, 1, trg len, trg len] \n",
        "    src_mask = self.make_src_mask(src) \n",
        "    trg_mask = self.make_trg_mask(trg)  \n",
        "\n",
        "    # enc_src = [bs, src_len, hid_dim]\n",
        "    enc_src = self.encoder(src,src_mask)  \n",
        "\n",
        "    # output = [bs, trg_len, output_dim] \n",
        "    # attention = [bs, n_heads, trg_len, src_len] \n",
        "    output, attention = self.decoder(trg, enc_src, trg_mask, src_mask) \n",
        "\n",
        "    return output, attention     "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6qb6tRSN5DP0"
      },
      "source": [
        "## 训练过程 \n",
        "\n",
        "We can now define our encoder and decoders. This model is significantly smaller than Transformers used in research today, but is able to be run on a single GPU quickly."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D7tXOgaS4Zou"
      },
      "source": [
        "### 参数配置"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Caai7P4AAlRy"
      },
      "source": [
        "INPUT_DIM = len(SRC.vocab)\n",
        "OUTPUT_DIM = len(TRG.vocab)\n",
        "HID_DIM = 256 # 256/8 = 32 每个head是32 dim \n",
        "ENC_LAYERS = 3  # encoding multi-head层 \n",
        "DEC_LAYERS = 3  # decoding multi-head层 \n",
        "ENC_HEADS = 8  # 头数\n",
        "DEC_HEADS = 8  # 头数\n",
        "ENC_PF_DIM = 512\n",
        "DEC_PF_DIM = 512\n",
        "ENC_DROPOUT = 0.1\n",
        "DEC_DROPOUT = 0.1  \n",
        "\n",
        "N_EPOCHS = 10\n",
        "CLIP = 1\n",
        "\n",
        "# 1\n",
        "SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token]\n",
        "# 1 \n",
        "TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token]\n",
        "\n",
        "\n",
        "# encoder \n",
        "enc = Encoder(INPUT_DIM, HID_DIM, ENC_LAYERS, ENC_HEADS, ENC_PF_DIM, ENC_DROPOUT, device) \n",
        "# decoder \n",
        "dec = Decoder(OUTPUT_DIM, HID_DIM, DEC_LAYERS, DEC_HEADS, DEC_PF_DIM, DEC_DROPOUT, device)  \n",
        " \n",
        "# model \n",
        "model = Seq2Seq(enc, dec, SRC_PAD_IDX, TRG_PAD_IDX, device).to(device)\n",
        "\n",
        "## Training Params \n",
        "LEARNING_RATE = 0.0005 \n",
        "# Optimizer \n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) \n",
        "# loss function \n",
        "criterion = nn.CrossEntropyLoss(ignore_index=TRG_PAD_IDX) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CHOcylncDGFr"
      },
      "source": [
        "### 检查参数数量, 小于37M"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1P7bzIbADlZM",
        "outputId": "33dc8c75-92b6-437d-97a4-67f9d464d319"
      },
      "source": [
        "def count_parameters(model):\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
        "\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The model has 9,038,853 trainable parameters\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iyvMGvjxDln1"
      },
      "source": [
        "### 参数初始化方式"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IzjPerABDqfK",
        "outputId": "8f615643-57b7-49c5-def7-d840a7ff9bbe"
      },
      "source": [
        "def initialize_weights(m):\n",
        "  if hasattr(m, 'weight') and m.weight.dim() > 1:\n",
        "    nn.init.xavier_uniform_(m.weight.data)\n",
        "\n",
        "model.apply(initialize_weights)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Seq2Seq(\n",
              "  (encoder): Encoder(\n",
              "    (tok_embedding): Embedding(7855, 256)\n",
              "    (pos_embedding): Embedding(100, 256)\n",
              "    (layers): ModuleList(\n",
              "      (0): EncoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (positionwise_feedforward): PositionwiseFeedforwardLayer(\n",
              "          (fc_1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc_2): Linear(in_features=512, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (positionwise_feedforward): PositionwiseFeedforwardLayer(\n",
              "          (fc_1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc_2): Linear(in_features=512, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (2): EncoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (positionwise_feedforward): PositionwiseFeedforwardLayer(\n",
              "          (fc_1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc_2): Linear(in_features=512, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (decoder): Decoder(\n",
              "    (tok_embedding): Embedding(5893, 256)\n",
              "    (pos_embedding): Embedding(100, 256)\n",
              "    (layers): ModuleList(\n",
              "      (0): DecoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (enc_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (encoder_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (positionwise_feedforward): PositionwiseFeedforwardLayer(\n",
              "          (fc_1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc_2): Linear(in_features=512, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (1): DecoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (enc_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (encoder_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (positionwise_feedforward): PositionwiseFeedforwardLayer(\n",
              "          (fc_1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc_2): Linear(in_features=512, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (2): DecoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (enc_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (encoder_attention): MultiHeadAttentionLayer(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (positionwise_feedforward): PositionwiseFeedforwardLayer(\n",
              "          (fc_1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc_2): Linear(in_features=512, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (fc_out): Linear(in_features=256, out_features=5893, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9a7ai25ODs53"
      },
      "source": [
        "### 模型训练 \n",
        "\n",
        "假设现有$trg=[sos,x_1,x_2, x_3, eos]$ \n",
        "\n",
        "- 输入[sos] 输出 $x_1$; \n",
        "- 所以我们应该拿掉[eos]，即输入:$trg[-1]=[sos,x_1,x_2,x_3]$ \n",
        "  - 此时,$output=[y_1,y_2,y_3,eos]$ \n",
        "  - $y_i$ 是预测的target的元素；\n",
        "- 在计算Loss时候，应该输入$trg[1] = [x_1,x_2,x_3,eos]$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cbcaFW7RFQF8"
      },
      "source": [
        "def train(model, iterator, optimizer, criterion, clip):\n",
        "  model.train() \n",
        "\n",
        "  epoch_loss = 0 \n",
        "\n",
        "  # iterator:训练数据迭代器\n",
        "  for i, batch in enumerate(iterator):\n",
        "\n",
        "    src = batch.src \n",
        "    trg = batch.trg \n",
        "\n",
        "    optimizer.zero_grad() \n",
        "\n",
        "    # output: [bs, trg_len-1, output_dim] \n",
        "    # trg: [bs, trg_len] \n",
        "    output,_ = model(src, trg[:,:-1]) \n",
        "\n",
        "    # output: [bs*trg_len-1, output_dim]\n",
        "    output_dim = output.shape[-1]  \n",
        "    output = output.contiguous().view(-1,output_dim) \n",
        "    \n",
        "    # trg: [bs*trg_len-1] \n",
        "    # 用于计算Loss \n",
        "    trg = trg[:,1:].contiguous().view(-1)\n",
        "\n",
        "    loss = criterion(output,trg) \n",
        "    loss.backward() \n",
        "    torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
        "    optimizer.step()\n",
        "    \n",
        "    epoch_loss += loss.item()\n",
        "        \n",
        "    return epoch_loss / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rRONFOROQXuh"
      },
      "source": [
        "# 评估\n",
        "def evaluate(model, iterator, criterion):\n",
        "  model.eval() \n",
        "\n",
        "  epoch_loss = 0 \n",
        "\n",
        "  with torch.no_grad():\n",
        "    for i,batch in enumerate(iterator):\n",
        "      src = batch.src \n",
        "      trg = batch.trg \n",
        "\n",
        "      # output: [bs, trg_len-1, output_dim] \n",
        "      # trg: [bs, trg_len] \n",
        "      output,_ = model(src, trg[:,:-1])  \n",
        "      output_dim = output.shape[-1]\n",
        "      \n",
        "      output = output.contiguous().view(-1, output_dim)\n",
        "      trg = trg[:,1:].contiguous().view(-1) \n",
        "\n",
        "      loss = criterion(output, trg)\n",
        "      epoch_loss += loss.item() \n",
        "  \n",
        "  return epoch_loss / len(iterator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Or5DETp0hSwV"
      },
      "source": [
        "### 训练量统计工具 \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5aWAyuMKhX9K"
      },
      "source": [
        "# epoch 时间消耗 \n",
        "def epoch_time(start_time, end_time):\n",
        "  elapsed_time = end_time - start_time \n",
        "  elapsed_mins = int(elapsed_time / 60) \n",
        "  elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "  return elapsed_mins, elapsed_secs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tABJyw2zhkTM"
      },
      "source": [
        "### Start Training..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LorQBLF8jF4o",
        "outputId": "62d0819e-7ce5-4fb4-e095-0658dafac606"
      },
      "source": [
        "!pwd\n",
        "import os \n",
        "os.chdir(\"/content/drive/MyDrive/Notion笔记/Model\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-YSuxtayjUWp",
        "outputId": "473a063c-9398-48b7-c1bb-f60efe923134"
      },
      "source": [
        "!pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/My Drive/Notion笔记/Model\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XhlPD_64hmIY",
        "outputId": "e2b78f2b-714b-46ce-886d-ce8c90eb6751"
      },
      "source": [
        "N_EPOCHS = 500\n",
        "CLIP = 1\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "    \n",
        "  start_time = time.time()\n",
        "  \n",
        "  train_loss = train(model, train_iterator, optimizer, criterion, CLIP)\n",
        "  valid_loss = evaluate(model, valid_iterator, criterion)\n",
        "  \n",
        "  end_time = time.time()\n",
        "  \n",
        "  epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "  \n",
        "  if valid_loss < best_valid_loss:\n",
        "      best_valid_loss = valid_loss\n",
        "      torch.save(model.state_dict(), 'tut6-model.pt')\n",
        "  \n",
        "  print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
        "  print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
        "  print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 01 | Time: 0m 0s\n",
            "\tTrain Loss: 0.031 | Train PPL:   1.031\n",
            "\t Val. Loss: 6.734 |  Val. PPL: 840.096\n",
            "Epoch: 02 | Time: 0m 0s\n",
            "\tTrain Loss: 0.030 | Train PPL:   1.031\n",
            "\t Val. Loss: 6.608 |  Val. PPL: 740.923\n",
            "Epoch: 03 | Time: 0m 0s\n",
            "\tTrain Loss: 0.030 | Train PPL:   1.030\n",
            "\t Val. Loss: 6.486 |  Val. PPL: 655.821\n",
            "Epoch: 04 | Time: 0m 0s\n",
            "\tTrain Loss: 0.029 | Train PPL:   1.029\n",
            "\t Val. Loss: 6.368 |  Val. PPL: 582.693\n",
            "Epoch: 05 | Time: 0m 0s\n",
            "\tTrain Loss: 0.028 | Train PPL:   1.029\n",
            "\t Val. Loss: 6.252 |  Val. PPL: 519.036\n",
            "Epoch: 06 | Time: 0m 0s\n",
            "\tTrain Loss: 0.028 | Train PPL:   1.028\n",
            "\t Val. Loss: 6.139 |  Val. PPL: 463.719\n",
            "Epoch: 07 | Time: 0m 0s\n",
            "\tTrain Loss: 0.028 | Train PPL:   1.028\n",
            "\t Val. Loss: 6.029 |  Val. PPL: 415.496\n",
            "Epoch: 08 | Time: 0m 0s\n",
            "\tTrain Loss: 0.027 | Train PPL:   1.027\n",
            "\t Val. Loss: 5.924 |  Val. PPL: 374.018\n",
            "Epoch: 09 | Time: 0m 0s\n",
            "\tTrain Loss: 0.027 | Train PPL:   1.027\n",
            "\t Val. Loss: 5.822 |  Val. PPL: 337.768\n",
            "Epoch: 10 | Time: 0m 0s\n",
            "\tTrain Loss: 0.026 | Train PPL:   1.026\n",
            "\t Val. Loss: 5.737 |  Val. PPL: 310.003\n",
            "Epoch: 11 | Time: 0m 0s\n",
            "\tTrain Loss: 0.026 | Train PPL:   1.026\n",
            "\t Val. Loss: 5.658 |  Val. PPL: 286.527\n",
            "Epoch: 12 | Time: 0m 0s\n",
            "\tTrain Loss: 0.026 | Train PPL:   1.026\n",
            "\t Val. Loss: 5.562 |  Val. PPL: 260.248\n",
            "Epoch: 13 | Time: 0m 0s\n",
            "\tTrain Loss: 0.025 | Train PPL:   1.025\n",
            "\t Val. Loss: 5.490 |  Val. PPL: 242.180\n",
            "Epoch: 14 | Time: 0m 0s\n",
            "\tTrain Loss: 0.025 | Train PPL:   1.025\n",
            "\t Val. Loss: 5.405 |  Val. PPL: 222.482\n",
            "Epoch: 15 | Time: 0m 0s\n",
            "\tTrain Loss: 0.024 | Train PPL:   1.025\n",
            "\t Val. Loss: 5.330 |  Val. PPL: 206.387\n",
            "Epoch: 16 | Time: 0m 0s\n",
            "\tTrain Loss: 0.024 | Train PPL:   1.024\n",
            "\t Val. Loss: 5.262 |  Val. PPL: 192.911\n",
            "Epoch: 17 | Time: 0m 0s\n",
            "\tTrain Loss: 0.024 | Train PPL:   1.024\n",
            "\t Val. Loss: 5.204 |  Val. PPL: 182.077\n",
            "Epoch: 18 | Time: 0m 0s\n",
            "\tTrain Loss: 0.023 | Train PPL:   1.023\n",
            "\t Val. Loss: 5.151 |  Val. PPL: 172.526\n",
            "Epoch: 19 | Time: 0m 0s\n",
            "\tTrain Loss: 0.023 | Train PPL:   1.024\n",
            "\t Val. Loss: 5.102 |  Val. PPL: 164.429\n",
            "Epoch: 20 | Time: 0m 0s\n",
            "\tTrain Loss: 0.023 | Train PPL:   1.024\n",
            "\t Val. Loss: 5.056 |  Val. PPL: 156.940\n",
            "Epoch: 21 | Time: 0m 0s\n",
            "\tTrain Loss: 0.023 | Train PPL:   1.023\n",
            "\t Val. Loss: 5.030 |  Val. PPL: 152.867\n",
            "Epoch: 22 | Time: 0m 0s\n",
            "\tTrain Loss: 0.023 | Train PPL:   1.023\n",
            "\t Val. Loss: 5.024 |  Val. PPL: 152.036\n",
            "Epoch: 23 | Time: 0m 0s\n",
            "\tTrain Loss: 0.023 | Train PPL:   1.023\n",
            "\t Val. Loss: 4.966 |  Val. PPL: 143.492\n",
            "Epoch: 24 | Time: 0m 0s\n",
            "\tTrain Loss: 0.023 | Train PPL:   1.023\n",
            "\t Val. Loss: 4.964 |  Val. PPL: 143.156\n",
            "Epoch: 25 | Time: 0m 0s\n",
            "\tTrain Loss: 0.022 | Train PPL:   1.023\n",
            "\t Val. Loss: 4.908 |  Val. PPL: 135.333\n",
            "Epoch: 26 | Time: 0m 0s\n",
            "\tTrain Loss: 0.022 | Train PPL:   1.023\n",
            "\t Val. Loss: 4.874 |  Val. PPL: 130.893\n",
            "Epoch: 27 | Time: 0m 0s\n",
            "\tTrain Loss: 0.022 | Train PPL:   1.022\n",
            "\t Val. Loss: 4.855 |  Val. PPL: 128.341\n",
            "Epoch: 28 | Time: 0m 0s\n",
            "\tTrain Loss: 0.022 | Train PPL:   1.023\n",
            "\t Val. Loss: 4.822 |  Val. PPL: 124.196\n",
            "Epoch: 29 | Time: 0m 0s\n",
            "\tTrain Loss: 0.022 | Train PPL:   1.022\n",
            "\t Val. Loss: 4.802 |  Val. PPL: 121.710\n",
            "Epoch: 30 | Time: 0m 0s\n",
            "\tTrain Loss: 0.022 | Train PPL:   1.022\n",
            "\t Val. Loss: 4.756 |  Val. PPL: 116.234\n",
            "Epoch: 31 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.022\n",
            "\t Val. Loss: 4.745 |  Val. PPL: 114.979\n",
            "Epoch: 32 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.022\n",
            "\t Val. Loss: 4.717 |  Val. PPL: 111.846\n",
            "Epoch: 33 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.667 |  Val. PPL: 106.426\n",
            "Epoch: 34 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.648 |  Val. PPL: 104.402\n",
            "Epoch: 35 | Time: 0m 0s\n",
            "\tTrain Loss: 0.022 | Train PPL:   1.022\n",
            "\t Val. Loss: 4.620 |  Val. PPL: 101.537\n",
            "Epoch: 36 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.612 |  Val. PPL: 100.662\n",
            "Epoch: 37 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.576 |  Val. PPL:  97.106\n",
            "Epoch: 38 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.549 |  Val. PPL:  94.503\n",
            "Epoch: 39 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.524 |  Val. PPL:  92.167\n",
            "Epoch: 40 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.494 |  Val. PPL:  89.440\n",
            "Epoch: 41 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.478 |  Val. PPL:  88.035\n",
            "Epoch: 42 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.020\n",
            "\t Val. Loss: 4.453 |  Val. PPL:  85.913\n",
            "Epoch: 43 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.425 |  Val. PPL:  83.518\n",
            "Epoch: 44 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.020\n",
            "\t Val. Loss: 4.402 |  Val. PPL:  81.608\n",
            "Epoch: 45 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.020\n",
            "\t Val. Loss: 4.379 |  Val. PPL:  79.761\n",
            "Epoch: 46 | Time: 0m 0s\n",
            "\tTrain Loss: 0.021 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.358 |  Val. PPL:  78.084\n",
            "Epoch: 47 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.021\n",
            "\t Val. Loss: 4.336 |  Val. PPL:  76.426\n",
            "Epoch: 48 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.020\n",
            "\t Val. Loss: 4.321 |  Val. PPL:  75.297\n",
            "Epoch: 49 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.294 |  Val. PPL:  73.233\n",
            "Epoch: 50 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.275 |  Val. PPL:  71.883\n",
            "Epoch: 51 | Time: 0m 0s\n",
            "\tTrain Loss: 0.020 | Train PPL:   1.020\n",
            "\t Val. Loss: 4.244 |  Val. PPL:  69.716\n",
            "Epoch: 52 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.242 |  Val. PPL:  69.561\n",
            "Epoch: 53 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.020\n",
            "\t Val. Loss: 4.207 |  Val. PPL:  67.176\n",
            "Epoch: 54 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 4.201 |  Val. PPL:  66.766\n",
            "Epoch: 55 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.174 |  Val. PPL:  65.000\n",
            "Epoch: 56 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.157 |  Val. PPL:  63.892\n",
            "Epoch: 57 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.134 |  Val. PPL:  62.401\n",
            "Epoch: 58 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.020\n",
            "\t Val. Loss: 4.138 |  Val. PPL:  62.688\n",
            "Epoch: 59 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.098 |  Val. PPL:  60.200\n",
            "Epoch: 60 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.089 |  Val. PPL:  59.678\n",
            "Epoch: 61 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.073 |  Val. PPL:  58.760\n",
            "Epoch: 62 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 4.056 |  Val. PPL:  57.771\n",
            "Epoch: 63 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.041 |  Val. PPL:  56.886\n",
            "Epoch: 64 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.024 |  Val. PPL:  55.916\n",
            "Epoch: 65 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.019\n",
            "\t Val. Loss: 4.007 |  Val. PPL:  55.000\n",
            "Epoch: 66 | Time: 0m 0s\n",
            "\tTrain Loss: 0.019 | Train PPL:   1.019\n",
            "\t Val. Loss: 3.996 |  Val. PPL:  54.386\n",
            "Epoch: 67 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.979 |  Val. PPL:  53.477\n",
            "Epoch: 68 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.019\n",
            "\t Val. Loss: 3.969 |  Val. PPL:  52.905\n",
            "Epoch: 69 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.952 |  Val. PPL:  52.052\n",
            "Epoch: 70 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.942 |  Val. PPL:  51.502\n",
            "Epoch: 71 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.930 |  Val. PPL:  50.920\n",
            "Epoch: 72 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.908 |  Val. PPL:  49.786\n",
            "Epoch: 73 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.899 |  Val. PPL:  49.339\n",
            "Epoch: 74 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.904 |  Val. PPL:  49.624\n",
            "Epoch: 75 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.877 |  Val. PPL:  48.285\n",
            "Epoch: 76 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.865 |  Val. PPL:  47.702\n",
            "Epoch: 77 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.849 |  Val. PPL:  46.935\n",
            "Epoch: 78 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.861 |  Val. PPL:  47.499\n",
            "Epoch: 79 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.841 |  Val. PPL:  46.592\n",
            "Epoch: 80 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.824 |  Val. PPL:  45.780\n",
            "Epoch: 81 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.810 |  Val. PPL:  45.149\n",
            "Epoch: 82 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.817 |  Val. PPL:  45.471\n",
            "Epoch: 83 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.796 |  Val. PPL:  44.511\n",
            "Epoch: 84 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.774 |  Val. PPL:  43.546\n",
            "Epoch: 85 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.769 |  Val. PPL:  43.351\n",
            "Epoch: 86 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.775 |  Val. PPL:  43.586\n",
            "Epoch: 87 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.758 |  Val. PPL:  42.870\n",
            "Epoch: 88 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.739 |  Val. PPL:  42.055\n",
            "Epoch: 89 | Time: 0m 0s\n",
            "\tTrain Loss: 0.018 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.729 |  Val. PPL:  41.633\n",
            "Epoch: 90 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.723 |  Val. PPL:  41.389\n",
            "Epoch: 91 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.713 |  Val. PPL:  40.966\n",
            "Epoch: 92 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.018\n",
            "\t Val. Loss: 3.701 |  Val. PPL:  40.508\n",
            "Epoch: 93 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.693 |  Val. PPL:  40.160\n",
            "Epoch: 94 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.688 |  Val. PPL:  39.950\n",
            "Epoch: 95 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.680 |  Val. PPL:  39.648\n",
            "Epoch: 96 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.667 |  Val. PPL:  39.123\n",
            "Epoch: 97 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.654 |  Val. PPL:  38.636\n",
            "Epoch: 98 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.644 |  Val. PPL:  38.231\n",
            "Epoch: 99 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.656 |  Val. PPL:  38.691\n",
            "Epoch: 100 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.644 |  Val. PPL:  38.236\n",
            "Epoch: 101 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.628 |  Val. PPL:  37.646\n",
            "Epoch: 102 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.615 |  Val. PPL:  37.157\n",
            "Epoch: 103 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.611 |  Val. PPL:  37.019\n",
            "Epoch: 104 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.610 |  Val. PPL:  36.963\n",
            "Epoch: 105 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.602 |  Val. PPL:  36.664\n",
            "Epoch: 106 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.583 |  Val. PPL:  35.999\n",
            "Epoch: 107 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.582 |  Val. PPL:  35.950\n",
            "Epoch: 108 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.582 |  Val. PPL:  35.930\n",
            "Epoch: 109 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.579 |  Val. PPL:  35.848\n",
            "Epoch: 110 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.563 |  Val. PPL:  35.284\n",
            "Epoch: 111 | Time: 0m 0s\n",
            "\tTrain Loss: 0.017 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.546 |  Val. PPL:  34.667\n",
            "Epoch: 112 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.559 |  Val. PPL:  35.114\n",
            "Epoch: 113 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.555 |  Val. PPL:  34.979\n",
            "Epoch: 114 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.539 |  Val. PPL:  34.435\n",
            "Epoch: 115 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.529 |  Val. PPL:  34.077\n",
            "Epoch: 116 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.523 |  Val. PPL:  33.870\n",
            "Epoch: 117 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.515 |  Val. PPL:  33.613\n",
            "Epoch: 118 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.510 |  Val. PPL:  33.460\n",
            "Epoch: 119 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.501 |  Val. PPL:  33.161\n",
            "Epoch: 120 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.497 |  Val. PPL:  33.018\n",
            "Epoch: 121 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.500 |  Val. PPL:  33.126\n",
            "Epoch: 122 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.017\n",
            "\t Val. Loss: 3.491 |  Val. PPL:  32.804\n",
            "Epoch: 123 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.468 |  Val. PPL:  32.068\n",
            "Epoch: 124 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.478 |  Val. PPL:  32.391\n",
            "Epoch: 125 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.474 |  Val. PPL:  32.270\n",
            "Epoch: 126 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.458 |  Val. PPL:  31.741\n",
            "Epoch: 127 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.455 |  Val. PPL:  31.671\n",
            "Epoch: 128 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.452 |  Val. PPL:  31.574\n",
            "Epoch: 129 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.446 |  Val. PPL:  31.376\n",
            "Epoch: 130 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.436 |  Val. PPL:  31.074\n",
            "Epoch: 131 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.423 |  Val. PPL:  30.647\n",
            "Epoch: 132 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.415 |  Val. PPL:  30.417\n",
            "Epoch: 133 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.414 |  Val. PPL:  30.382\n",
            "Epoch: 134 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.410 |  Val. PPL:  30.252\n",
            "Epoch: 135 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.400 |  Val. PPL:  29.963\n",
            "Epoch: 136 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.408 |  Val. PPL:  30.198\n",
            "Epoch: 137 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.397 |  Val. PPL:  29.885\n",
            "Epoch: 138 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.384 |  Val. PPL:  29.493\n",
            "Epoch: 139 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.378 |  Val. PPL:  29.327\n",
            "Epoch: 140 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.376 |  Val. PPL:  29.264\n",
            "Epoch: 141 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.373 |  Val. PPL:  29.172\n",
            "Epoch: 142 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.365 |  Val. PPL:  28.929\n",
            "Epoch: 143 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.355 |  Val. PPL:  28.633\n",
            "Epoch: 144 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.353 |  Val. PPL:  28.594\n",
            "Epoch: 145 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.354 |  Val. PPL:  28.622\n",
            "Epoch: 146 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.349 |  Val. PPL:  28.479\n",
            "Epoch: 147 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.343 |  Val. PPL:  28.308\n",
            "Epoch: 148 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.345 |  Val. PPL:  28.363\n",
            "Epoch: 149 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.337 |  Val. PPL:  28.144\n",
            "Epoch: 150 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.326 |  Val. PPL:  27.822\n",
            "Epoch: 151 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.323 |  Val. PPL:  27.735\n",
            "Epoch: 152 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.318 |  Val. PPL:  27.594\n",
            "Epoch: 153 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.324 |  Val. PPL:  27.775\n",
            "Epoch: 154 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.311 |  Val. PPL:  27.421\n",
            "Epoch: 155 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.301 |  Val. PPL:  27.132\n",
            "Epoch: 156 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.298 |  Val. PPL:  27.050\n",
            "Epoch: 157 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.305 |  Val. PPL:  27.248\n",
            "Epoch: 158 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.290 |  Val. PPL:  26.842\n",
            "Epoch: 159 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.293 |  Val. PPL:  26.933\n",
            "Epoch: 160 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.288 |  Val. PPL:  26.781\n",
            "Epoch: 161 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.289 |  Val. PPL:  26.820\n",
            "Epoch: 162 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.281 |  Val. PPL:  26.609\n",
            "Epoch: 163 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.280 |  Val. PPL:  26.579\n",
            "Epoch: 164 | Time: 0m 0s\n",
            "\tTrain Loss: 0.016 | Train PPL:   1.016\n",
            "\t Val. Loss: 3.273 |  Val. PPL:  26.384\n",
            "Epoch: 165 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.276 |  Val. PPL:  26.468\n",
            "Epoch: 166 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.276 |  Val. PPL:  26.463\n",
            "Epoch: 167 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.255 |  Val. PPL:  25.929\n",
            "Epoch: 168 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.252 |  Val. PPL:  25.838\n",
            "Epoch: 169 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.236 |  Val. PPL:  25.420\n",
            "Epoch: 170 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.237 |  Val. PPL:  25.469\n",
            "Epoch: 171 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.266 |  Val. PPL:  26.218\n",
            "Epoch: 172 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.252 |  Val. PPL:  25.837\n",
            "Epoch: 173 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.225 |  Val. PPL:  25.143\n",
            "Epoch: 174 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.218 |  Val. PPL:  24.980\n",
            "Epoch: 175 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.217 |  Val. PPL:  24.951\n",
            "Epoch: 176 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.221 |  Val. PPL:  25.044\n",
            "Epoch: 177 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.213 |  Val. PPL:  24.846\n",
            "Epoch: 178 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.204 |  Val. PPL:  24.628\n",
            "Epoch: 179 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.191 |  Val. PPL:  24.301\n",
            "Epoch: 180 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.187 |  Val. PPL:  24.217\n",
            "Epoch: 181 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.188 |  Val. PPL:  24.243\n",
            "Epoch: 182 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.188 |  Val. PPL:  24.239\n",
            "Epoch: 183 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.183 |  Val. PPL:  24.114\n",
            "Epoch: 184 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.166 |  Val. PPL:  23.707\n",
            "Epoch: 185 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.170 |  Val. PPL:  23.817\n",
            "Epoch: 186 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.174 |  Val. PPL:  23.912\n",
            "Epoch: 187 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.166 |  Val. PPL:  23.717\n",
            "Epoch: 188 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.164 |  Val. PPL:  23.657\n",
            "Epoch: 189 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.149 |  Val. PPL:  23.304\n",
            "Epoch: 190 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.141 |  Val. PPL:  23.116\n",
            "Epoch: 191 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.129 |  Val. PPL:  22.856\n",
            "Epoch: 192 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.131 |  Val. PPL:  22.900\n",
            "Epoch: 193 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.136 |  Val. PPL:  23.005\n",
            "Epoch: 194 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.130 |  Val. PPL:  22.882\n",
            "Epoch: 195 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.129 |  Val. PPL:  22.845\n",
            "Epoch: 196 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.115 |  Val. PPL:  22.544\n",
            "Epoch: 197 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.105 |  Val. PPL:  22.302\n",
            "Epoch: 198 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.110 |  Val. PPL:  22.412\n",
            "Epoch: 199 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.107 |  Val. PPL:  22.362\n",
            "Epoch: 200 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.109 |  Val. PPL:  22.389\n",
            "Epoch: 201 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.096 |  Val. PPL:  22.113\n",
            "Epoch: 202 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.086 |  Val. PPL:  21.889\n",
            "Epoch: 203 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.085 |  Val. PPL:  21.859\n",
            "Epoch: 204 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.086 |  Val. PPL:  21.886\n",
            "Epoch: 205 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.078 |  Val. PPL:  21.715\n",
            "Epoch: 206 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.074 |  Val. PPL:  21.625\n",
            "Epoch: 207 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.072 |  Val. PPL:  21.595\n",
            "Epoch: 208 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.063 |  Val. PPL:  21.392\n",
            "Epoch: 209 | Time: 0m 0s\n",
            "\tTrain Loss: 0.015 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.064 |  Val. PPL:  21.414\n",
            "Epoch: 210 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.054 |  Val. PPL:  21.208\n",
            "Epoch: 211 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.046 |  Val. PPL:  21.038\n",
            "Epoch: 212 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.047 |  Val. PPL:  21.058\n",
            "Epoch: 213 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.041 |  Val. PPL:  20.917\n",
            "Epoch: 214 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.039 |  Val. PPL:  20.878\n",
            "Epoch: 215 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 3.028 |  Val. PPL:  20.654\n",
            "Epoch: 216 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.015 |  Val. PPL:  20.395\n",
            "Epoch: 217 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.015 |  Val. PPL:  20.383\n",
            "Epoch: 218 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.018 |  Val. PPL:  20.440\n",
            "Epoch: 219 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.016 |  Val. PPL:  20.402\n",
            "Epoch: 220 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.005 |  Val. PPL:  20.191\n",
            "Epoch: 221 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 3.003 |  Val. PPL:  20.145\n",
            "Epoch: 222 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.002 |  Val. PPL:  20.125\n",
            "Epoch: 223 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 3.014 |  Val. PPL:  20.362\n",
            "Epoch: 224 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.015\n",
            "\t Val. Loss: 3.004 |  Val. PPL:  20.156\n",
            "Epoch: 225 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.994 |  Val. PPL:  19.974\n",
            "Epoch: 226 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.991 |  Val. PPL:  19.914\n",
            "Epoch: 227 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.979 |  Val. PPL:  19.668\n",
            "Epoch: 228 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.987 |  Val. PPL:  19.824\n",
            "Epoch: 229 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.971 |  Val. PPL:  19.521\n",
            "Epoch: 230 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.969 |  Val. PPL:  19.472\n",
            "Epoch: 231 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.975 |  Val. PPL:  19.581\n",
            "Epoch: 232 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.963 |  Val. PPL:  19.360\n",
            "Epoch: 233 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.950 |  Val. PPL:  19.097\n",
            "Epoch: 234 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.954 |  Val. PPL:  19.186\n",
            "Epoch: 235 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.960 |  Val. PPL:  19.296\n",
            "Epoch: 236 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.936 |  Val. PPL:  18.840\n",
            "Epoch: 237 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.927 |  Val. PPL:  18.672\n",
            "Epoch: 238 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.927 |  Val. PPL:  18.671\n",
            "Epoch: 239 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.937 |  Val. PPL:  18.854\n",
            "Epoch: 240 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.936 |  Val. PPL:  18.844\n",
            "Epoch: 241 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.914 |  Val. PPL:  18.425\n",
            "Epoch: 242 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.910 |  Val. PPL:  18.365\n",
            "Epoch: 243 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.902 |  Val. PPL:  18.202\n",
            "Epoch: 244 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.904 |  Val. PPL:  18.241\n",
            "Epoch: 245 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.901 |  Val. PPL:  18.194\n",
            "Epoch: 246 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.882 |  Val. PPL:  17.850\n",
            "Epoch: 247 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.879 |  Val. PPL:  17.795\n",
            "Epoch: 248 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.878 |  Val. PPL:  17.784\n",
            "Epoch: 249 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.881 |  Val. PPL:  17.837\n",
            "Epoch: 250 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.887 |  Val. PPL:  17.942\n",
            "Epoch: 251 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.873 |  Val. PPL:  17.682\n",
            "Epoch: 252 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.859 |  Val. PPL:  17.444\n",
            "Epoch: 253 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.856 |  Val. PPL:  17.398\n",
            "Epoch: 254 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.853 |  Val. PPL:  17.335\n",
            "Epoch: 255 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.855 |  Val. PPL:  17.383\n",
            "Epoch: 256 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.846 |  Val. PPL:  17.215\n",
            "Epoch: 257 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.839 |  Val. PPL:  17.093\n",
            "Epoch: 258 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.837 |  Val. PPL:  17.057\n",
            "Epoch: 259 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.828 |  Val. PPL:  16.914\n",
            "Epoch: 260 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.832 |  Val. PPL:  16.978\n",
            "Epoch: 261 | Time: 0m 0s\n",
            "\tTrain Loss: 0.014 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.844 |  Val. PPL:  17.179\n",
            "Epoch: 262 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.842 |  Val. PPL:  17.153\n",
            "Epoch: 263 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.826 |  Val. PPL:  16.884\n",
            "Epoch: 264 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.825 |  Val. PPL:  16.863\n",
            "Epoch: 265 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.822 |  Val. PPL:  16.814\n",
            "Epoch: 266 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.814 |  Val. PPL:  16.674\n",
            "Epoch: 267 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.812 |  Val. PPL:  16.644\n",
            "Epoch: 268 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.815 |  Val. PPL:  16.696\n",
            "Epoch: 269 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.812 |  Val. PPL:  16.650\n",
            "Epoch: 270 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.801 |  Val. PPL:  16.462\n",
            "Epoch: 271 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.793 |  Val. PPL:  16.330\n",
            "Epoch: 272 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.788 |  Val. PPL:  16.245\n",
            "Epoch: 273 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.786 |  Val. PPL:  16.218\n",
            "Epoch: 274 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.784 |  Val. PPL:  16.187\n",
            "Epoch: 275 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.789 |  Val. PPL:  16.271\n",
            "Epoch: 276 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.780 |  Val. PPL:  16.123\n",
            "Epoch: 277 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.769 |  Val. PPL:  15.942\n",
            "Epoch: 278 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.764 |  Val. PPL:  15.860\n",
            "Epoch: 279 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.760 |  Val. PPL:  15.797\n",
            "Epoch: 280 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.761 |  Val. PPL:  15.809\n",
            "Epoch: 281 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.762 |  Val. PPL:  15.829\n",
            "Epoch: 282 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.764 |  Val. PPL:  15.856\n",
            "Epoch: 283 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.754 |  Val. PPL:  15.710\n",
            "Epoch: 284 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.752 |  Val. PPL:  15.679\n",
            "Epoch: 285 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.014\n",
            "\t Val. Loss: 2.745 |  Val. PPL:  15.562\n",
            "Epoch: 286 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.738 |  Val. PPL:  15.455\n",
            "Epoch: 287 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.730 |  Val. PPL:  15.327\n",
            "Epoch: 288 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.726 |  Val. PPL:  15.267\n",
            "Epoch: 289 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.729 |  Val. PPL:  15.317\n",
            "Epoch: 290 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.730 |  Val. PPL:  15.339\n",
            "Epoch: 291 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.731 |  Val. PPL:  15.352\n",
            "Epoch: 292 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.729 |  Val. PPL:  15.317\n",
            "Epoch: 293 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.720 |  Val. PPL:  15.187\n",
            "Epoch: 294 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.705 |  Val. PPL:  14.955\n",
            "Epoch: 295 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.697 |  Val. PPL:  14.840\n",
            "Epoch: 296 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.694 |  Val. PPL:  14.795\n",
            "Epoch: 297 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.687 |  Val. PPL:  14.690\n",
            "Epoch: 298 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.696 |  Val. PPL:  14.821\n",
            "Epoch: 299 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.699 |  Val. PPL:  14.870\n",
            "Epoch: 300 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.683 |  Val. PPL:  14.623\n",
            "Epoch: 301 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.676 |  Val. PPL:  14.521\n",
            "Epoch: 302 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.673 |  Val. PPL:  14.484\n",
            "Epoch: 303 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.674 |  Val. PPL:  14.498\n",
            "Epoch: 304 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.666 |  Val. PPL:  14.382\n",
            "Epoch: 305 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.662 |  Val. PPL:  14.320\n",
            "Epoch: 306 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.667 |  Val. PPL:  14.395\n",
            "Epoch: 307 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.667 |  Val. PPL:  14.401\n",
            "Epoch: 308 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.659 |  Val. PPL:  14.287\n",
            "Epoch: 309 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.655 |  Val. PPL:  14.221\n",
            "Epoch: 310 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.650 |  Val. PPL:  14.149\n",
            "Epoch: 311 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.639 |  Val. PPL:  14.004\n",
            "Epoch: 312 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.635 |  Val. PPL:  13.945\n",
            "Epoch: 313 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.637 |  Val. PPL:  13.976\n",
            "Epoch: 314 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.636 |  Val. PPL:  13.956\n",
            "Epoch: 315 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.625 |  Val. PPL:  13.799\n",
            "Epoch: 316 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.629 |  Val. PPL:  13.857\n",
            "Epoch: 317 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.627 |  Val. PPL:  13.833\n",
            "Epoch: 318 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.624 |  Val. PPL:  13.791\n",
            "Epoch: 319 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.633 |  Val. PPL:  13.911\n",
            "Epoch: 320 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.622 |  Val. PPL:  13.765\n",
            "Epoch: 321 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.609 |  Val. PPL:  13.590\n",
            "Epoch: 322 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.609 |  Val. PPL:  13.586\n",
            "Epoch: 323 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.607 |  Val. PPL:  13.559\n",
            "Epoch: 324 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.606 |  Val. PPL:  13.552\n",
            "Epoch: 325 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.611 |  Val. PPL:  13.619\n",
            "Epoch: 326 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.606 |  Val. PPL:  13.543\n",
            "Epoch: 327 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.596 |  Val. PPL:  13.415\n",
            "Epoch: 328 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.595 |  Val. PPL:  13.401\n",
            "Epoch: 329 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.596 |  Val. PPL:  13.415\n",
            "Epoch: 330 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.599 |  Val. PPL:  13.450\n",
            "Epoch: 331 | Time: 0m 0s\n",
            "\tTrain Loss: 0.013 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.590 |  Val. PPL:  13.329\n",
            "Epoch: 332 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.587 |  Val. PPL:  13.287\n",
            "Epoch: 333 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.581 |  Val. PPL:  13.208\n",
            "Epoch: 334 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.584 |  Val. PPL:  13.248\n",
            "Epoch: 335 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.589 |  Val. PPL:  13.323\n",
            "Epoch: 336 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.576 |  Val. PPL:  13.140\n",
            "Epoch: 337 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.577 |  Val. PPL:  13.154\n",
            "Epoch: 338 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.575 |  Val. PPL:  13.131\n",
            "Epoch: 339 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.013\n",
            "\t Val. Loss: 2.563 |  Val. PPL:  12.975\n",
            "Epoch: 340 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.576 |  Val. PPL:  13.147\n",
            "Epoch: 341 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.571 |  Val. PPL:  13.077\n",
            "Epoch: 342 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.559 |  Val. PPL:  12.923\n",
            "Epoch: 343 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.554 |  Val. PPL:  12.857\n",
            "Epoch: 344 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.548 |  Val. PPL:  12.786\n",
            "Epoch: 345 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.546 |  Val. PPL:  12.757\n",
            "Epoch: 346 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.543 |  Val. PPL:  12.712\n",
            "Epoch: 347 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.547 |  Val. PPL:  12.773\n",
            "Epoch: 348 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.545 |  Val. PPL:  12.746\n",
            "Epoch: 349 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.537 |  Val. PPL:  12.647\n",
            "Epoch: 350 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.535 |  Val. PPL:  12.618\n",
            "Epoch: 351 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.536 |  Val. PPL:  12.629\n",
            "Epoch: 352 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.534 |  Val. PPL:  12.600\n",
            "Epoch: 353 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.534 |  Val. PPL:  12.598\n",
            "Epoch: 354 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.537 |  Val. PPL:  12.645\n",
            "Epoch: 355 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.530 |  Val. PPL:  12.549\n",
            "Epoch: 356 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.525 |  Val. PPL:  12.490\n",
            "Epoch: 357 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.526 |  Val. PPL:  12.503\n",
            "Epoch: 358 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.526 |  Val. PPL:  12.503\n",
            "Epoch: 359 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.522 |  Val. PPL:  12.456\n",
            "Epoch: 360 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.510 |  Val. PPL:  12.301\n",
            "Epoch: 361 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.512 |  Val. PPL:  12.324\n",
            "Epoch: 362 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.507 |  Val. PPL:  12.270\n",
            "Epoch: 363 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.507 |  Val. PPL:  12.272\n",
            "Epoch: 364 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.501 |  Val. PPL:  12.194\n",
            "Epoch: 365 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.492 |  Val. PPL:  12.089\n",
            "Epoch: 366 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.491 |  Val. PPL:  12.074\n",
            "Epoch: 367 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.491 |  Val. PPL:  12.076\n",
            "Epoch: 368 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.488 |  Val. PPL:  12.032\n",
            "Epoch: 369 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.478 |  Val. PPL:  11.917\n",
            "Epoch: 370 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.478 |  Val. PPL:  11.914\n",
            "Epoch: 371 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.476 |  Val. PPL:  11.891\n",
            "Epoch: 372 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.473 |  Val. PPL:  11.858\n",
            "Epoch: 373 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.465 |  Val. PPL:  11.758\n",
            "Epoch: 374 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.460 |  Val. PPL:  11.702\n",
            "Epoch: 375 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.464 |  Val. PPL:  11.748\n",
            "Epoch: 376 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.470 |  Val. PPL:  11.822\n",
            "Epoch: 377 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.464 |  Val. PPL:  11.755\n",
            "Epoch: 378 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.468 |  Val. PPL:  11.794\n",
            "Epoch: 379 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.465 |  Val. PPL:  11.764\n",
            "Epoch: 380 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.455 |  Val. PPL:  11.646\n",
            "Epoch: 381 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.451 |  Val. PPL:  11.604\n",
            "Epoch: 382 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.449 |  Val. PPL:  11.577\n",
            "Epoch: 383 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.451 |  Val. PPL:  11.601\n",
            "Epoch: 384 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.456 |  Val. PPL:  11.653\n",
            "Epoch: 385 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.448 |  Val. PPL:  11.565\n",
            "Epoch: 386 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.441 |  Val. PPL:  11.484\n",
            "Epoch: 387 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.437 |  Val. PPL:  11.441\n",
            "Epoch: 388 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.428 |  Val. PPL:  11.334\n",
            "Epoch: 389 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.428 |  Val. PPL:  11.331\n",
            "Epoch: 390 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.434 |  Val. PPL:  11.404\n",
            "Epoch: 391 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.435 |  Val. PPL:  11.416\n",
            "Epoch: 392 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.427 |  Val. PPL:  11.325\n",
            "Epoch: 393 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.417 |  Val. PPL:  11.211\n",
            "Epoch: 394 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.412 |  Val. PPL:  11.157\n",
            "Epoch: 395 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.409 |  Val. PPL:  11.120\n",
            "Epoch: 396 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.414 |  Val. PPL:  11.179\n",
            "Epoch: 397 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.407 |  Val. PPL:  11.106\n",
            "Epoch: 398 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.406 |  Val. PPL:  11.089\n",
            "Epoch: 399 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.402 |  Val. PPL:  11.044\n",
            "Epoch: 400 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.400 |  Val. PPL:  11.027\n",
            "Epoch: 401 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.413 |  Val. PPL:  11.165\n",
            "Epoch: 402 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.405 |  Val. PPL:  11.081\n",
            "Epoch: 403 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.397 |  Val. PPL:  10.993\n",
            "Epoch: 404 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.404 |  Val. PPL:  11.062\n",
            "Epoch: 405 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.391 |  Val. PPL:  10.924\n",
            "Epoch: 406 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.397 |  Val. PPL:  10.987\n",
            "Epoch: 407 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.413 |  Val. PPL:  11.168\n",
            "Epoch: 408 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.395 |  Val. PPL:  10.965\n",
            "Epoch: 409 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.388 |  Val. PPL:  10.887\n",
            "Epoch: 410 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.393 |  Val. PPL:  10.947\n",
            "Epoch: 411 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.386 |  Val. PPL:  10.872\n",
            "Epoch: 412 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.392 |  Val. PPL:  10.937\n",
            "Epoch: 413 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.386 |  Val. PPL:  10.872\n",
            "Epoch: 414 | Time: 0m 0s\n",
            "\tTrain Loss: 0.012 | Train PPL:   1.012\n",
            "\t Val. Loss: 2.374 |  Val. PPL:  10.745\n",
            "Epoch: 415 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.378 |  Val. PPL:  10.779\n",
            "Epoch: 416 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.374 |  Val. PPL:  10.739\n",
            "Epoch: 417 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.371 |  Val. PPL:  10.707\n",
            "Epoch: 418 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.376 |  Val. PPL:  10.766\n",
            "Epoch: 419 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.370 |  Val. PPL:  10.700\n",
            "Epoch: 420 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.363 |  Val. PPL:  10.627\n",
            "Epoch: 421 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.364 |  Val. PPL:  10.635\n",
            "Epoch: 422 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.366 |  Val. PPL:  10.660\n",
            "Epoch: 423 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.366 |  Val. PPL:  10.660\n",
            "Epoch: 424 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.364 |  Val. PPL:  10.635\n",
            "Epoch: 425 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.360 |  Val. PPL:  10.596\n",
            "Epoch: 426 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.353 |  Val. PPL:  10.521\n",
            "Epoch: 427 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.348 |  Val. PPL:  10.463\n",
            "Epoch: 428 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.348 |  Val. PPL:  10.461\n",
            "Epoch: 429 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.347 |  Val. PPL:  10.453\n",
            "Epoch: 430 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.341 |  Val. PPL:  10.395\n",
            "Epoch: 431 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.338 |  Val. PPL:  10.359\n",
            "Epoch: 432 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.344 |  Val. PPL:  10.424\n",
            "Epoch: 433 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.342 |  Val. PPL:  10.402\n",
            "Epoch: 434 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.343 |  Val. PPL:  10.413\n",
            "Epoch: 435 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.344 |  Val. PPL:  10.427\n",
            "Epoch: 436 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.337 |  Val. PPL:  10.350\n",
            "Epoch: 437 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.334 |  Val. PPL:  10.316\n",
            "Epoch: 438 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.330 |  Val. PPL:  10.281\n",
            "Epoch: 439 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.322 |  Val. PPL:  10.192\n",
            "Epoch: 440 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.321 |  Val. PPL:  10.190\n",
            "Epoch: 441 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.324 |  Val. PPL:  10.220\n",
            "Epoch: 442 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.322 |  Val. PPL:  10.195\n",
            "Epoch: 443 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.324 |  Val. PPL:  10.213\n",
            "Epoch: 444 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.324 |  Val. PPL:  10.213\n",
            "Epoch: 445 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.323 |  Val. PPL:  10.207\n",
            "Epoch: 446 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.319 |  Val. PPL:  10.170\n",
            "Epoch: 447 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.324 |  Val. PPL:  10.219\n",
            "Epoch: 448 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.317 |  Val. PPL:  10.142\n",
            "Epoch: 449 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.312 |  Val. PPL:  10.097\n",
            "Epoch: 450 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.311 |  Val. PPL:  10.086\n",
            "Epoch: 451 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.310 |  Val. PPL:  10.075\n",
            "Epoch: 452 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.310 |  Val. PPL:  10.072\n",
            "Epoch: 453 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.302 |  Val. PPL:   9.995\n",
            "Epoch: 454 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.302 |  Val. PPL:   9.996\n",
            "Epoch: 455 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.304 |  Val. PPL:  10.012\n",
            "Epoch: 456 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.307 |  Val. PPL:  10.049\n",
            "Epoch: 457 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.308 |  Val. PPL:  10.053\n",
            "Epoch: 458 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.300 |  Val. PPL:   9.973\n",
            "Epoch: 459 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.295 |  Val. PPL:   9.922\n",
            "Epoch: 460 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.293 |  Val. PPL:   9.901\n",
            "Epoch: 461 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.294 |  Val. PPL:   9.911\n",
            "Epoch: 462 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.292 |  Val. PPL:   9.893\n",
            "Epoch: 463 | Time: 0m 0s\n",
            "\tTrain Loss: 0.009 | Train PPL:   1.009\n",
            "\t Val. Loss: 2.286 |  Val. PPL:   9.835\n",
            "Epoch: 464 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.284 |  Val. PPL:   9.814\n",
            "Epoch: 465 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.281 |  Val. PPL:   9.788\n",
            "Epoch: 466 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.278 |  Val. PPL:   9.761\n",
            "Epoch: 467 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.282 |  Val. PPL:   9.799\n",
            "Epoch: 468 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.283 |  Val. PPL:   9.809\n",
            "Epoch: 469 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.287 |  Val. PPL:   9.847\n",
            "Epoch: 470 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.286 |  Val. PPL:   9.834\n",
            "Epoch: 471 | Time: 0m 0s\n",
            "\tTrain Loss: 0.009 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.281 |  Val. PPL:   9.786\n",
            "Epoch: 472 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.277 |  Val. PPL:   9.745\n",
            "Epoch: 473 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.270 |  Val. PPL:   9.684\n",
            "Epoch: 474 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.265 |  Val. PPL:   9.631\n",
            "Epoch: 475 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.261 |  Val. PPL:   9.594\n",
            "Epoch: 476 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.260 |  Val. PPL:   9.579\n",
            "Epoch: 477 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.259 |  Val. PPL:   9.578\n",
            "Epoch: 478 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.260 |  Val. PPL:   9.583\n",
            "Epoch: 479 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.266 |  Val. PPL:   9.636\n",
            "Epoch: 480 | Time: 0m 0s\n",
            "\tTrain Loss: 0.009 | Train PPL:   1.009\n",
            "\t Val. Loss: 2.262 |  Val. PPL:   9.607\n",
            "Epoch: 481 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.260 |  Val. PPL:   9.580\n",
            "Epoch: 482 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.258 |  Val. PPL:   9.564\n",
            "Epoch: 483 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.255 |  Val. PPL:   9.535\n",
            "Epoch: 484 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.259 |  Val. PPL:   9.569\n",
            "Epoch: 485 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.258 |  Val. PPL:   9.566\n",
            "Epoch: 486 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.251 |  Val. PPL:   9.493\n",
            "Epoch: 487 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.248 |  Val. PPL:   9.469\n",
            "Epoch: 488 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.245 |  Val. PPL:   9.436\n",
            "Epoch: 489 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.244 |  Val. PPL:   9.432\n",
            "Epoch: 490 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.239 |  Val. PPL:   9.382\n",
            "Epoch: 491 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.244 |  Val. PPL:   9.431\n",
            "Epoch: 492 | Time: 0m 0s\n",
            "\tTrain Loss: 0.011 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.242 |  Val. PPL:   9.416\n",
            "Epoch: 493 | Time: 0m 0s\n",
            "\tTrain Loss: 0.009 | Train PPL:   1.009\n",
            "\t Val. Loss: 2.233 |  Val. PPL:   9.326\n",
            "Epoch: 494 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.232 |  Val. PPL:   9.318\n",
            "Epoch: 495 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.228 |  Val. PPL:   9.286\n",
            "Epoch: 496 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.235 |  Val. PPL:   9.346\n",
            "Epoch: 497 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.232 |  Val. PPL:   9.319\n",
            "Epoch: 498 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.227 |  Val. PPL:   9.276\n",
            "Epoch: 499 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.011\n",
            "\t Val. Loss: 2.229 |  Val. PPL:   9.291\n",
            "Epoch: 500 | Time: 0m 0s\n",
            "\tTrain Loss: 0.010 | Train PPL:   1.010\n",
            "\t Val. Loss: 2.228 |  Val. PPL:   9.282\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8AXNtZ7Qjn-s",
        "outputId": "3a1a7b71-da5e-4435-f3f3-590955cb9700"
      },
      "source": [
        "model.load_state_dict(torch.load('tut6-model.pt'))\n",
        "\n",
        "test_loss = evaluate(model, test_iterator, criterion)\n",
        "\n",
        "print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "| Test Loss: 2.253 | Test PPL:   9.517 |\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3NDFpjDRlU9r"
      },
      "source": [
        "## Inference  \n",
        "1. tokenize 原始的sentence（如果没有tokenize）\n",
        "2. 添加\"[sos]\"和\"[eos]\"分词\n",
        "3. numericalize the source sentence \n",
        "4. convert it to a tensor and add a batch dimension \n",
        "5. create the source sentence mask \n",
        "6. feed the source sentence and mask into the encoder \n",
        "7. create a list to hold the output sentence, initialized with an [sos] token \n",
        "8. 如果，我们没有hit a maximum length：\n",
        "  -  convert the current output sentence prediction into a tensor with a batch dimension \n",
        "  - create a target sentence mask \n",
        "  - place the current output, encoder output and both masks into the decoder \n",
        "  - get next output token prediction from decoder along with attention \n",
        "  - add prediction to current output sentence prediction \n",
        "  - break if the prediction was an [eos] token \n",
        "9. convert the output sentence from indexes to tokens \n",
        "10. return the output sentence(with [sos] token removed) and the attention from the last layer. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oht7LioLlbtf"
      },
      "source": [
        "def translate_sentence(sentence, src_field, trg_field, model, device, max_len = 50):\n",
        "    \n",
        "    model.eval()\n",
        "        \n",
        "    if isinstance(sentence, str):\n",
        "        nlp = spacy.load('de_core_news_sm')\n",
        "        tokens = [token.text.lower() for token in nlp(sentence)]\n",
        "    else:\n",
        "        tokens = [token.lower() for token in sentence]\n",
        "\n",
        "    tokens = [src_field.init_token] + tokens + [src_field.eos_token]\n",
        "        \n",
        "    src_indexes = [src_field.vocab.stoi[token] for token in tokens]\n",
        "\n",
        "    src_tensor = torch.LongTensor(src_indexes).unsqueeze(0).to(device)\n",
        "    \n",
        "    src_mask = model.make_src_mask(src_tensor)\n",
        "    \n",
        "    with torch.no_grad():\n",
        "        enc_src = model.encoder(src_tensor, src_mask)\n",
        "\n",
        "    trg_indexes = [trg_field.vocab.stoi[trg_field.init_token]]\n",
        "\n",
        "    for i in range(max_len):\n",
        "\n",
        "        trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)\n",
        "\n",
        "        trg_mask = model.make_trg_mask(trg_tensor)\n",
        "        \n",
        "        with torch.no_grad():\n",
        "            output, attention = model.decoder(trg_tensor, enc_src, trg_mask, src_mask)\n",
        "        \n",
        "        pred_token = output.argmax(2)[:,-1].item()\n",
        "        \n",
        "        trg_indexes.append(pred_token)\n",
        "\n",
        "        if pred_token == trg_field.vocab.stoi[trg_field.eos_token]:\n",
        "            break\n",
        "    \n",
        "    trg_tokens = [trg_field.vocab.itos[i] for i in trg_indexes]\n",
        "    \n",
        "    return trg_tokens[1:], attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wHCCOijklhiJ"
      },
      "source": [
        "def display_attention(sentence, translation, attention, n_heads = 8, n_rows = 4, n_cols = 2):\n",
        "    \n",
        "    assert n_rows * n_cols == n_heads\n",
        "    \n",
        "    fig = plt.figure(figsize=(15,25))\n",
        "    \n",
        "    for i in range(n_heads):\n",
        "        \n",
        "        ax = fig.add_subplot(n_rows, n_cols, i+1)\n",
        "        \n",
        "        _attention = attention.squeeze(0)[i].cpu().detach().numpy()\n",
        "\n",
        "        cax = ax.matshow(_attention, cmap='bone')\n",
        "\n",
        "        ax.tick_params(labelsize=12)\n",
        "        ax.set_xticklabels(['']+['<sos>']+[t.lower() for t in sentence]+['<eos>'], \n",
        "                           rotation=45)\n",
        "        ax.set_yticklabels(['']+translation)\n",
        "\n",
        "        ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "        ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
        "\n",
        "    plt.show()\n",
        "    plt.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m3meS_xqmg-N",
        "outputId": "6325b048-8102-4d85-a9e8-f78820375302"
      },
      "source": [
        "example_idx = 8\n",
        "\n",
        "src = vars(train_data.examples[example_idx])['src']\n",
        "trg = vars(train_data.examples[example_idx])['trg']\n",
        "\n",
        "print(f'src = {src}')\n",
        "print(f'trg = {trg}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "src = ['eine', 'frau', 'mit', 'einer', 'großen', 'geldbörse', 'geht', 'an', 'einem', 'tor', 'vorbei', '.']\n",
            "trg = ['a', 'woman', 'with', 'a', 'large', 'purse', 'is', 'walking', 'by', 'a', 'gate', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gRHL79DYmjMt",
        "outputId": "bfdb5a8c-3fa2-41b6-b8f4-91e7c997e1e4"
      },
      "source": [
        "translation, attention = translate_sentence(src, SRC, TRG, model, device)\n",
        "\n",
        "print(f'predicted trg = {translation}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "predicted trg = ['a', 'woman', 'with', 'a', 'large', '<unk>', 'walks', 'past', 'a', 'large', 'fence', '.', '<eos>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "I4YO0Qdhmmer",
        "outputId": "df99ebab-202a-4a8e-d24f-389ec7eb08eb"
      },
      "source": [
        "display_attention(src, translation, attention)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA08AAAWbCAYAAADoQY8ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdebwkVXn4/88zM8wCwybIjqAgoIKQiAv8gmBAJQnEoBEV44IxE9e4IS6oEcV9i8YVoxglUTTk64obLsQFlCGACBoRZBNEQIcZ9pl7n98f51xp2lnq9u3q5d7P+/Xq1+2urnr6VHV1PfecOnUqMhNJkiRJ0vrNG3YBJEmSJGkcWHmSJEmSpAasPEmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDVg5UmSJEmSGrDyJEmSJEkNWHmShiAiYthlkCRpFJkjNcqsPEkDFBHzADIzu6abKCRJc5o5UuMguvZPSS2JiPmZORER9weeAtwMXJeZnx1y0SRJGipzpMaFZ56kAalJ4QHAD4H7ADsCJ0bEe4ZbMkmShsscqXGxYNgFkOaC2uVgAfAS4L2ZeVJELASOAhYPtXCSJA2ROVLjxDNP0gBksRqYD1xc+3X/GPhxZj43IvaJiIcNt5SSJA2eOVLjxMqT1JKImF//bhYRS2oymA8cAfwPcFFmPrXO/jLgEcMpqSRJg2WO1LhywAipBR0Xvu4DfAx4dWaeGRE7AucAZObOdd5PAA8GHpaZa4ZVZkmSBsEcqXFm5UlqSUQ8EPg+8J7MfGNELMjMNXUkoS8C1wBTP8C/yszVUwllWGWWJGkQzJEaV1aepD6KiHmZOVmfnwxck5lvqN0RXgjcSenHfRFwv7rYpZk5OZU4hlJwSZJaZo7UbOBoe1KfRETUA/zewErgp8ATI+JC4PXADZTf3JHA0zPz/zqWnWdSkCTNVuZIzRYOGCH1Qe1KkBGxA3Au8JfAt4CfAU8APp+ZjwGeAyysjz+YaomTJGm2MUdqNrHbnjQDtSUt6/PdgUcC22bmWzrmWZSZd9bnHwd2AR5tMpAkzWbmSM1GnnnSnFb7Wa/z9XqWe3ZEPD7v2fqwDPg34KERMb8j1kRE7BsRnwceAhxeuy74+xtREbF02GWQpGHqNT/Wec2Rs9hcz5HumJqzajeCyYjYLiIeHBHbNWnpiohtgRXA9Z3TM/N44B3AY4B9p2J19NP+FvCQOmLQAlvVRlPtVvKBiNjF5C1pLuo1P9ZlzZGzmDnSbnuao6a6EtR7TJxF6Xe9N3B0Zn59PcsdADwfeGFm/j4idgG2BbbKzK/WeT4IPBk4LDP/dy0x5tRQq13dNiJH/KBT94l7Z+a3h12WXnXcQ2Xkt7ek0dJrfqzLmiOnYdzyI5gjwcpTz6aG24yIhZl5V502Fjv+XNeRGLYBXgv8AjgZeAnwRuBJmfnf61j2wcCqzPxVROwLfL4uvwelle0p9b2PAEcBf52Z57S/VqOpc2jZziFqx0FEBHAScHZmfnnY5ZmuiNgCOAL4fmZeMeTiaI4xR46nmeTHurw5sqFxzo8wt3OklaceRcQiYBvgxcAlmfmxIRdJ01B/NB8HtqfcfO93dfqLgbdTEsT/W8/ymwNnAKdm5ociYiHwBWAr4IDaovFZYNPM/IuWV2ekRcR84FRgS+BzwHcy8/LhlmrtuloB9wReDWwMfCQzzxxq4RqKiIOB3YEXAPsCL8nM9w63VJprzJHja6b5sc5rjmxgnPIjmCOnzMm+ijMVEccArwM+S2mNOXC4JVIPNgcuBh4EPH5qYmb+C3AccHr9ga3LYsoB46y63F01ASwG3l2nHQ38VSulH3G1RWrKF+rf7wLHAM+Jcp+PkVJP4/+hNSnLPUbeClwHPC8iHj20wjUQEYdExDuBj1JaeX9J+efFf1o1UObIsTfT/AjmyHUax/wI5shO3iS3odo68FzKweQo4ERKK8FPgTfXeeyS0CfdfZ5num07+rcuysw7M/PKiHgzsJpysLo1Mz8NkJnvi4jfAD/oWL77lPqNlAPG4RFxWdZhVoH/AO41NdPUiEHjdjp+Jrq6IiylnBJ/a319AfAPwDMj4pTMvHiIRf2Djv1jHvAySuvoD4CzgbcBr6TsJ5OZ+a0hFvWP1Iuz/x24g3Ljycdn5k8j4oXA1sAdc20f1OCZIwdn1PJjjWGObGAc8yOYI7t55qmBiNiM0m/3MOAK4OGZ+SHgt5RTrbcBmBT6oyaCiYjYKCLeVA/oM0kM82q8vYGPR8RnI+L1lHtJvAk4HfiniHjS1DKZ+Vlg04jYqL6eGnVo74jYoyauLwOHAo+rXRQAHkZXo8RcSQrwh+9uTZRhaP8b+AxwXEQ8FCAzv0YZqvY+wAsi4gFDLO4fdCSFc4GHAlsAjwP+C7iF0jJ1JfCqiDhkWOVch42Ar1NuLvmPNSk8FHgVpQvImrm0D2rwzJGDMyr5sR7nt67HTXNkA+OaH8Ec2c0zTw1k5sqIeEtm/jDuvgh2L8rFky/PzOs3FEPNdLWo/Rswv6PFqif1+9oN+CbwHkpr2J/W1/sAnwImgDdHxE2ZeWZEHAi8D3ghcHZNLF8EbgbujIhLMvNZEbE18EzgpIj4ObAr8CTmoK7v7mPAEkpf7ocAT4+IqzPzN5n51Sj9358A3DSk4q7NPwDX164kRMRXgQRWZuYFETEBPJVyqn/oatePnTLzasp+TRQLgCOBj2fmd23tV9vMkYMxKvmxlsUcOQ2zID+COfJumeljHQ/Kmbl/6Jq2oP79W+ADwEbDLudsewBBGd3n7ZSLSaEObtJDrI3q3xcBp3RMv5BygSPAfMqFzU+jJKOpeb5M6a/9Z5SksIzSF/wASuvL5+t8u9dln9Cxf8zvpbyd26Dj+YxiDem7OwlYWKftRRlt6V+B7Trm3WTY5e0q+4uAt9fnnwJ+QmmxWkS52z1T6zTsRz02/ZDyD9SSzn2Gcp3B2cCTh11OH7P7YY4cyjYfifxY3zNH9vbdjV1+7NhPzJGZdttbl9p/+0fA46PcpwC4x83cjgNuzMzVwyjfLLcl8GjKKE3b1WmN9tWIOCIinhURf1f7Fk99PwuBq+s851NGf/rHKDd7ezJwU2Z+itI/+zUAmXkEpd/2O+ryX8/MmzPzbEqf/u0i4hmZ+cvM/FRmnp71lHzO8B4VmZkdXQYnauvdONiScgPEl1G/u8z8OWV7HQq8JcoQuGTmrcMqZG156nYTcFREnE5J9g+v+8/xwJMiYknWIZeHqXad+DFwKfDczLwd7tEl6pmUY9NnhlNCzQXmyKEZWn6suejIiDgBzJE9GIv8CObIDRp27XBUH5RT1p/oeL01pXY9j3Iq+9873pt2qw/raCnpJVbT2KP6WFt5KS1dPwG+1XS9gGOBGygXKv8UOKnjvb+l9Mv9BfDejumfnnpNaRV6Wo1xXMc8pwB3UboaTLWaLQBOA45va5sAX6W09OxMuchx72F/V9P47i4Cvtk1fV9gObDtkMq6HXD/rm3818BTKEltIfBeykWl+9V5nk+5bmOfYW/rjnI/Fjiz4/XLKN1nnkdpBdwaeFB9b96wy+tjdj7MkQPbziORH6e2vTmyL9/dyOXHWgZzZNPPGPZKjuKDctr5v4BH1NfvB74BfA/4szptp/q356RQk8wLgWcAT+jzOgSwWz8STcvbuntbPBP42zptG8qdzc/snn8tcTYBvgb8RX39LuDArnneCExSLljdqx7wL5g62Nd5lgBHUy56Pr5j+mcop3kPApbWaV8BTmxpuyyqB61raqJaNrWdhv2d9frd1elDOaVPOU3/fuD/AbvUaRfV7/R8SgvV0XXfeDPlwtfTKV1P9hv2tu5alz+hdJd5G2U0s/OBNwC3A0cMu3w+Zv/DHDmw7TxS+bHOZ45s4bur04fW5c0cOb2HA0asXVJaYU6MiDWUIRmfRalxP4MytOQ10NvoQXn3qCXnAb8DLgGeGBGPAl7YS8y1eCvl9PAzIuKiPsXsu/Vsiz/PzOfVUVu+GRHnZuZDc92n+ucBO9UHwP7ARRFxDaWVZAnwIUrXhPcDl1OGYX1o1m4EtTy3R8SPgA8CJ0TEysz8cGY+OSI+R7nQ8+qIuJwyIs7j+rxJqOW4MyLOA3YAVgFr6vTJfnR56Ifpfnd1saF04cnM2yLia5R7lryh7hffy8znAdSumi8BXpuZr46I0ygXTk9m5o3DKHO3KCOaraG0OH+DcgPL/wOOyczVtevUtkMsouYOc+QAjFJ+rGWZb45sXL6xyY9gjpy2YdcQR+VBaYU6iFJj3ZLygzwKeCJ3tyC8nHJzrRlfAEs5jfiZjtdfAM6eQby1nR7+IqVrxb70p6vDvI7nfWutW8e2OGfq8yinkn9EbQ1ZT5wXAr+mdBk5hpLcb6S0kF1CuWP6psBm9fueunhwfsfzfSgtWe+mdG24FnhFx2d8mHKAO6pjv+hbFxD+uJXvTyj926+kjFo19N9KG99dy2XsvLD4cMo/eBcCr++a792UG0OORMtlR7nmUVr4vk0ZbvU13fsLJandCOw+7PL6mJ0Pc2Sjz+h7jhyF/Ni5PubIwX93AyinOXK6nznslR6FR93wZwPfp7QS/BQ4tHPjU24AtgJ4cJ8+80TgHfX5f9QddaP6Y3p4jzED2KJr2pcod66eUbm5+zT8PGBxn7f/+rbFVLeQDf5YgaWUU7NXAntS+kE/DHgg8BeULiUHdcy/B7B5x+uNKXebflV9vQPw9BrvJR3zndRx8JvxQWQt5diNcj+Cg4Bt6rS/o7QGvrS+/iTwN/38Hob53bVcxs5/aJ5O6Zf/1VrWvbq2+3c6v4thP+pv+puUEYMeTvmn5zJqQq77zhsp/8D86bDL62N2PsyRjWK3kiOHlR/rMubIEfjuBlBOc+R0P3fYKz4KD+Aj1Atf687xMkrfyEfVpHAcpV/nn/QYv3t4z6Bc5PgByinuP/Qrpty0633Aoh4+558pp93v3TX9B8D/UO4n0Ev/8+Prj2kLSj/Sng9IPW6LdSai7gMPpW/3m4GrgEd1vfe9emAIYEfgekrC36xjntOBF3XFeyelle6E9a1LD9uisxyvoiTdvSitIz+k/JPyQeoFnJSLNlfWffGXDHgI4H5/d9P43O6Wxsb7MHcn8KC0Mn+4vj6Yco+Nz04dUOvv/meMVmK4H+Wf1q06pu1J6Yv+VEoi/lvgvsMuq4/Z+8AcuaG4fcmR/T7G0kN+7Phcc+QQv7tpfG7P+bFzH8EcOb3PHfaKj8KDMiLMc6d2oPr3DcCn6/PdqK0bPcT+wylvygggUzvhTpTa8W3Usf2BF1BGLXlgw9jR9fp+lNFx7jFiSz2g3AX8Jw0TDrB9x/N3Ad8CfgWcPoPt3Ndt0RFve+D/oyS+jeu0N1ESxJ93fMcXc8/BIY6uB9iXUbqhzKP8k/Bv9QcXHfOdRblA+g/dGfq4/02V46V1v3tenf44SleKTwJ71Gl7UFpWptZ9QT/LMoz9uMk+TvkHbTkdIwFNM84r6/7feR+vwygXk95c94/vMmJnbyj/OJxD/Uenbv+NKPfYeM0wy+Zj7jwwR64tdl9zZL+3AzPMj3W6OXLI+3CT/ZsZ5scawxw5nc8d9ooPeaNPHUhOBt7c9d7fAWfMMP4fTlsDP6eMO38n5VTuppTT5v9L6Qf7NcpFbo1a7rhnJaDzdPa2lJaC87j7dPYy4Lk07FdLuXj0Ku7ZLeMiSmvPkXT1gR7Gtug4aDyYMgrM9yl9h3/Usd5vpCSzEygto1M3BOw8ODye0uf7uPr6vvXA9j7q0JuUPvwndHxmv/qyd/YzfkItx4V03LiNciD+BGXko/26lu+5VW+Y312P2+e1wH/3GOeRlH9sblnLNjyQ0kf6o8AO/fhe+7RvTN3Ub179PX+ZMsLZ1D74r8Ab+rk/+vDR/cAcua7Yfc2R/d4OzDA/dh17zZFD2oensW16zo91eXPkdD9/2BtgSBt9HqXl5DH19V9SatbPAu5Vpz2/fhnTussztdWq8yBSf9wn19dHUmrv76xf9Gb1APenNBzfn7sPzFM7zdn14PGEOn2bOv0WyvChq5lmiwS1VY27TzMfSTnYfoPS9W2z7vIMeltQukj8iHqHe8pN206lJIQtKUngHcB/c88WmuCefXyPpvTZfmV9vUeNey5liMvzuDux9HOgjO5yHEnpl/sWOvrlU0aE+gJllJsZlYEyKtYGE/sg9uNplPmTlP7XU7/X9fYRX9v+SEkOZ1Bao+7b9d5h/S7zDNZ1Xt2Hv0a58PpxdZ89n7uHXv1nys0K9xp2eX3MzgfmyCafMaMc2fZ2oMf82FEWc+Q64gxiH55GeaeVH9ezP5ojp1OOYW+IIW34C+sPbUvubjU4htKq8V3KOPe/YZpj11NO0X64/vim4n6FMjLNszvmO5TSv/gt1FPNPa7HeZRTqYfWg+D/AH/fMc8ySoKb9g7UUf5vUfvA1tfvoFycd0x9/XjKKCbd3SNa2RYdB7WNKfd5OJuOVhxgMfB57r6gtbMlIjqWnxpVaGF9/TjumRy2AB5EuSC1jRGD1lWOI2s5XgZs2TH/w2hwUNzA/vJ9ymg057Oe/r+D3I8blv3twK2U1rX1dqnhnv80PZ5yY8i9uHuksE9T/incpa3yzmA9o/6GP0m58PWllNGxDqG0dJ9I6TJzMiN2M0gfs+eBObJx/Pp32jmyre3ADPNjVwxz5JD34Yblbpwfu7arOXImZRn2xhjCxn8T8MmO148F/pzSErUL5aLPY4H79RD7kcDhXdMOopzeP61r+qMop2//mYY3RuOe3c2eDXy24/Vn6k70dTqSQw/r0N3n+c8oF2t+oGPa2+oB5gxKa+SDBrEtuGdXhDNr2b4NPL7r/X8F3ta9bMeBbh9KUv0apavFEXX64yitcscBm65r2/dhH2xajpfScRFk57I9fOY7KP9E7Ff3lRtYR9eBtvfjpvt41/QTgF/UbbPez6EkhQso//xdQkls76O0tB5CaVk7DbhPv77TPu0Xf0LHXecp/c9/RPknqPO3P5Dr3HzMzQfmyA19xoxzZBvbgRnmx/rXHLmBHNn2Ptx0/+6a3jg/Tm0jzJEzK8uwN8YAN/pW9e9rKBfAbUupYV9UDzTfoU+nJSn/qL8OeEh9/QjKTdze3TXfI9nwvRkWT8Wsf+dT+qAeyN0XJf4b5fT5fSgtHb+gXlA5zXJ39t19F/Di+uPfh9LH+YMd8z4e+Ac20GLXz20x9T3Wsj2/vn4DpcXn4dR+7fUHddI6lt+xHuReUct2ImV43QPq+0dRbhr45A2VZYb7yMDKUbfXF6ndbeq0j1L65+87qO+uQTk7W8Q+RGnd+xR33+38tZQb3h1JVwsb3KNrx0nAl+vzjSl3of88dSQoSt/5j9BxwfcwH3Wfjrp9z63TPkY5Nk11hXk2NZHhNU4+WnhgjmxS7r7myH4fY5lhfqzvmyNzwzmy399dgzL2nB8799363Bw50zINe6MMcOOfWX9se3J3l4TTKaNyPLy+7jkxcM9a7951R/z3qR8e5UC+ijrmf8OYD6bcrGyqb3VQRlSZum/AAspB+2zqhXyU1oM30uOFffUzfkxpsfo8pS/tC+s6/Rb410FvC+5OiptTWkvOpqPVs67zRZTkPnVB5oLu5Ts+u7M18n+4ewjeTerfR9HHVrRhloPSUvwxSpL56673PgRMUi/6bWs/nuZ3HJQuE/9FuW7gu5RRnA6t77+R0sr7F+vYd/egDBt8Usf0jSgXg3+DuxPQ0n5/vzNY928Af0P5p+9cyuhM53S8/3LKsLxbD7usPmbvA3Nk08+cUY7s93ZghvmxM0bH55sj849zZBv78DS/457zY8fy5sh+lGnYG2VAG/6JlFaFqRaqTSnDd061Ij2HctO/Xoda7bzIcj9K69YTKXdj/lTHD+sR9Yf4pgYxtwauA15cX8+rO/fHuub7/ygHzD+lnEo/H9i5l/LX54uAd3WU4WmUJPEiSv/mCbpO+be5LToOGltT7jtwHOUg98Su+R5L6U7yzI4yzO9Y/t6UmwE+nPKPwW6UJDt1I7VtKK0xnaMy9bMbwsDLQbmB4eZ1X/8XSsL/s655/oW7h3jt+37csJydw7w+FPhK1/sfAX7Q8foEYLe1xHkp5QLd1wFXdL13n/rbGLVuCE+k/FM6dYPNIyj/3Hyo7huvonQfmda1JT58TOeBObJR+evznnNkv7cDM8yPXTHMkevJkW3sww3L2Jf8WN8zR/bpsYC54SDKPQImI2JBZq4CVkXE9hFxPPAM4LDM/O10A0dEZOaaiJi6A/sdlB/j94EEfg+8PCLelpnnRMQjKDdxa+Jc4NsREZTWo4cCm0fEVpl5E0Bm/iAibqf0Y96J0r/56mmUf15H+d8N3AvYLiK2zMwbI+IMSuvdkykX5O1NSQ6tb4uImJ+ZExGxLaXP8CZ1PV8HvD8ibs3MM+p2+Hr3snV61uVfT/mH4BOUA/GZlNO/T66LvINycfSqqRiZudb1nK6u9Wi9HHX7f4uS5O9HubD2C5SLSt8SEa/IzB/W2C+uy7S5H6+vrE8DTo+IOyjb5ibgwIjYZur3mJn/GBFXRMSxmXlKZr5pHeGuo1z8fSiwb0R8j9I3/VbKaExrKNcfjJKDKK1od9XXZ1ASwasoFwKvodyL5aLhFE9zhDly3eXvS44ctfxYlx1obmq4Lq2XY7o5sn53Oeb5EcyRfTNvkB82DBFxBKXv8Ycy86568JofEUdT+nTPBw7OzP/tJX7WqjBl6MRfZubBlBaeP6McQL9AGQ71pIh4UGb+ODN/3iD0JOUH+U+UHWdLSr/tFcDTI2KTjjIcQEluD8nM85qWvR6wJmvi+RGldeVelJafYyJiaU1AX6Ccnj4A+E1m/qLtbVEPVhMR8WBK3+MHcffIMOdQLmr+cEQcPjV/V4jJruUfQOnXezhwO3A5cElEPCMiTgX2pQxjO7mWWD1by3q0Wo66zA8pB5e/oFy/8HrKDRK/QznN/5GIeHjnci3ux+sr630o/4y8nXJ/jF0z872UpPbMzn2c0iJ4bceyf3TsysxPAz+gHFRfTUlwP4+Ib1Ja4/4xM0cmMXQfmwAyc5LSovo3mfkE4GlWnNQmc+S69TNHjlJ+rMsONDetyzjkyHHLj3V5c2SLZu2Zp6mWDMoQjB/KzEsjYl9KLfYZlJFaXku58dtd6wk1HW+tf4+nnGp9cUQspZwO3ZdyoG8kM38XEW+itKzdBLwiM6+JiK0op+fXRMTHMvO2Ov8vp1vYqQMopS/pjzLzBQAR8TrKuP4ZEf9ey3Ia8J/T+GHNaFvUVp6tKa0M7wQ+APwt5U7wf0sZDeddwBcj4pGZeU6D5Y+m/BAvpRwkV1L2hyuBZ07909Cv1rQhleMo4BeZ+XSAiNiPMszr0yin/c+m3MDv+vXE6Nt+vD6ZeVVEHEo5mF9HueCT+vogYMeI+AywPyVJvbJj2anE+Vrg55n52frWZ4DnAddl5qMj4tG1vNfnNFqb29Tg2HRtRFxR/wFbPcyyavYyRzb6jLZy5FDz43pimCM3nCNHPj/W5c2RbcoR6NPY1oPSD/h/KX1Wl1Fq5u+kjkTTp88IymglF1AOXO+m9NOd6jv+ckp/42lffEe5qd0plFaNj1F+mEEZfeZ0yo934xmWfz9KC955dNz0j9IS81+UPrKNPqPf24Iy2s636LgJI+VmjRdTxvE/iJIo1jV859qWn+ov+xG6boq4rjh92EcGVg7KqDRTF49+BPhJff5Byun457CWm1q2uR9voLw7Uy74vrj+3aruf0+htPKdS7lnyp+uZdltKIlsBSXhPqlO/x7wL218l31c79aPTT58bOgxiP2wzWMLY5Qj+70d1pFXGufH9cQwR3blyDb34Q2Utef8WJc3R7ZVvmEXoMUNP4/Sj3WSMl79B+kagYQZ3FBtLZ93DKX169cd054PXE0P98Poir0zZZScT1BOz08N2/kfdNwkbgbxH05p0TmKmgS4++7jp073M/q1LYDtgN8BT+38vigXZ55HuWh0x6nyTnP58yk3r9txAPviQMtR949tKPczmdo+r6gHz/V+Tpv78QY+d09K14QPTO1vlJGL/pyaoNaz7F41QZxFGfnqRMq9K1q7u/gM13WgxyYfPtb2MEdOK37fcuSo5McGMcyRA9yHN1DenvNjndcc2e8yDnsjtfwF3Ad4M6V/8qYtf9bGlNOml1Jqxx+sP6i13oi0h/i7UU5rf4wyPGvQdXO4GcY/mHLvi8d3JYdpD/3Yz21BOb38v9Sb49VpH6C0/PwY+Ksel39Xk+X7uH0HWg7KKEIrKH2bX1S3/07D3o838Nl7Az+jtJp+kNJVodHIP8BCYDHlgunvU7ohbDeI77bHdR3YscmHj3U9zJHTit+XHDlK+XEDMcyRA96HN/DZPefHurw5so+PqaEhZ716UWKrKxsRCyjDVD6GcnO3szLz0j7G3w34JOV08Yuzf/3Qp+IfTPlRngR8PjNvn0GsvmyLiNgUeAHwEkrf520oCfHBEXEKsBQ4el3f7UyX75dhlCMinkVpFbsT+KfMXN5wuVb34w189gMpSXQLynDA5zdc7g+/74i4L3B7Zv6mvZI2U/vyb5qZv1rPPK0fm6QNMUc2it+XHDkq+bFfMfphXHLkOObHuuzI5cgm+bHON3I5cs5UnmaLutOvyZYu7IuIwyinyP88y3C1Q1dHjXkEpeXvVuDkzLwjIt5POQC8vM3l+2UY5agJKTJzxsOmDkpNTpGZ07oQdNQOsBGxCPgK8IzM/PWwyyPNBXMtR/Yjr5gjxydH9pof67IjkyPHPT9aedIfiYiNs45QNIrqAfbllIuBH5mZFw9y+X4ZlXKoPaP+W5I0faP8u+5HXhmV3DQq5VA7Rvl3tCGzdqhy9W6Ud+aIWAwcSblQ8tE9VJxmtHy/jEo51K5R/i1J6s2o/q77kVdGJTeNSjnUnlH9HTXhmSeNnYjYGFjQ62n2mS7fL6NSDknS7NCPvDIquWlUyiF1s/IkSZIkSQ3MG3YBJEmSJGkcWHmSJEmSpAasPK1DRCwbp7jjGnscyzyuscexzOMaexzLLDU1rvu3v/fBxB7HMo9r7CfSmNkAACAASURBVHEs8zjHnmLlad3a2vhtfqnjGHscyzyuscexzOMaexzLLDU1rvu3v/fBxB7HMo9r7HEs8zjHBqw8SZIkSVIjc2a0vYhobUUXLVzSeN6JyTXMn9f89lq73v9+jedd8fvfs8WWWzaa9/8u9pYJ0lyQmTHsMmj0tZkjlyzZtPG8a9bcxYIFCxvPv+N9d24878rf/57NGubIX15ySeO4ksZXLznSm+T2wU477dla7I+e/plW4j5yr71aiQtQbgreVuz2/g+cnJxoLbYkzVV77vnQ1mK/+RPvbiXuX+63XytxJY0/u+1JkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGxrLyFBGvjIjLImJVRFwSEUcNu0ySJI0Cc6QktWcsK0/AZcBBwObAicCpEbH9cIskSdJIMEdKUkvGsvKUmZ/LzGszczIzTwMuBR7WPV9ELIuI5RGxfPCllCRp8MyRktSesaw8RcTTI+KCiFgRESuAvYGtu+fLzJMzc//M3H/wpZQkafDMkZLUngXDLsB0RcQuwEeBQ4GzM3MiIi4AYrglkyRpuMyRktSucTzztAmQwA0AEXEspVVNkqS5zhwpSS0au8pTZl4CvAs4G7ge2Af4wVALJUnSCDBHSlK7xq7bHkBmngCcMOxySJI0asyRktSesTvzJEmSJEnDYOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgORmcMuw0BERGsrumTJpm2F5pIrL2sl7n232aaVuG3bZJPNW4t96603txZbGpbM9Oao2qA2c2REe+20k5MTrcSN8GcjzQW95EjPPEmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDVg5UmSJEmSGrDyJEmSJEkNWHmSJEmSpAasPEmSJElSAxusPEXEsRHxpY7Xl0bE5zpeXx0R+0XEgRFxbkTcXP8e2DHPdyPipIj4YUTcEhFfioitIuI/ImJlnX/XjvnfW+OujIjzIuKgjvdeHxGfjYhPRsSqiLg4Ivbvx8aQJKkp86MkzT1NzjydBRwUEfMiYgdgIXAAQETcD1gKXAV8BXgfsBXwbuArEbFVR5wnA08DdgR2A84GTgHuBfwM+OeOec8F9qvv/SfwuYhY3PH+XwOfAbYAvgi8f20Fj4hlEbE8IpY3WE9JkqZjbPNjLaM5UpKmaYOVp8y8HFhFOVg/Evg6cG1E7AUcDHwP+Cvg0sz8VGauycxPAz8HjuwIdUpmXpaZNwNfBS7LzDMzcw3wOeBPOj7z1My8qcZ6F7AI2LMj1vcz84zMnAA+Bey7jrKfnJn7Z6Ytb5Kkvhrn/FhjmSMlaZoWNJzvLOAQYPf6fAUlMRxQX+8AXNm1zJWUVrQp13c8v30tr5dOvYiI44C/r3ET2AzYumP+33Q8vw1YHBELaqKRJGlQzI+SNIc0HTBiKjkcVJ+fRUkOB9fn1wK7dC1zH+DX0y1Q7b99PHA0sGVmbgHcDMR0Y0mS1DLzoyTNIdOpPD0KWJKZ11C6IhxO6b99PnAGsEdEHBMRCyLiScADgS/3UKZNgTXADcCCiHgdpWVNkqRRY36UpDmkUeUpM38B3EJJCmTmSuBy4AeZOZGZNwFHAC8DbqK0jB2RmTf2UKavA18DfkHp2nAHcHUPcSRJapX5UZLmlsjMYZdhICKitRVdsmTTtkJzyZWXtRL3vtts00rctm2yyeatxb711ptbiy0NS2bapUsb1GaOjGjvlpKTkxOtxI3wZyPNBb3kSG+SK0mSJEkNWHmSJEmSpAasPEmSJElSA1aeJEmSJKkBB4yYo9r83r3QVhodDhihJkqObGdX2WST9kZTX7P6rlbiPmT/w1uJC3De8q+1Fvuu1Xe2Frst++13aGuxf/KT77QWe2JiPO87/ZjHPKuVuN/85idaiQuwYMFGrcRds+YuJicnHTBCkiRJktpg5UmSJEmSGrDyJEmSJEkNWHmSJEmSpAasPEmSJElSA1aeJEmSJKmBkas8RcQtEXG/9bx/RUQcNsgySZI0CsyRkjRcC4ZdgG6ZuXTqeUR8ArgmM18zvBJJkjQazJGSNFwjd+ZJkiRJkkbRwCpPEXFsRHyp4/WlEfG5jtdXR8R+EZERsXtELAOeChxfuyl8qSPcfhHxk4i4OSJOi4jFg1oPSZL6zRwpSeNhkGeezgIOioh5EbEDsBA4AKD2314K/GRq5sw8GfgP4O2ZuTQzj+yIdTRwOHBf4MHAMweyBpIktcMcKUljYGDXPGXm5RGxCtgP2AP4OqV1bC9KgvheZk5GRJNw78vMawFqa9t+a5uptswt60f5JUlqizlSksbDoAeMOAs4BNi9Pl8BHExJDGdNI85vOp7fBuywtplqy9zJABGR0y+uJEkDY46UpBE36AEjphLDQfX5WZTEcDBrTwwezCVJc4U5UpJG3DAqT48ClmTmNcD3KP2ytwLOX8v81wPrvJ+FJEmziDlSkkbcQCtPmfkL4BZKQiAzVwKXAz/IzIm1LPIx4IERsSIiPj+4kkqSNFjmSEkafQO/SW5mbt/1ev+u19Hx/FK6LnTNzF27Xr++74WUJGkIzJGSNNq8Sa4kSZIkNWDlSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDkTk37rEXERnRTl0xc7KVuG3aYottW4v9wAce2FrsnXfZo7XYp336bS1Fjg3P0mvkaC/2/PntDca52WZbtRZ76dItW4l71VWXtBIXoM1jU+fobNK6RERr/ww88AHt5QRaOgZeeunyVuIC7L77n7YWe/Hipa3FPv/8b7YWuz1tHv7a/P+5vXLvttt+G56pB5ddtrZb0fVLW9sje8qRnnmSJEmSpAasPEmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDVg5UmSJEmSGrDyJEmSJEkNWHmSJEmSpAbGsvIUEa+MiMsiYlVEXBIRRw27TJIkjQJzpCS1ZywrT8BlwEHA5sCJwKkRsf1wiyRJ0kgwR0pSS8ay8pSZn8vMazNzMjNPAy4FHtY9X0Qsi4jlEbF88KWUJGnwzJGS1J6xrDxFxNMj4oKIWBERK4C9ga2758vMkzNz/8zcf/CllCRp8MyRktSeBcMuwHRFxC7AR4FDgbMzcyIiLgBiuCWTJGm4zJGS1K5xPPO0CZDADQARcSylVU2SpLnOHClJLRq7ylNmXgK8CzgbuB7YB/jBUAslSdIIMEdKUrvGrtseQGaeAJww7HJIkjRqzJGS1J6xO/MkSZIkScNg5UmSJEmSGrDyJEmSJEkNWHmSJEmSpAasPEmSJElSA5GZwy7DQETE3FjRETB/fnuDON5+5x2txV64YCwHn5TWKzO9Oao2aFxz5Lx581uJOzk50UpcaDdHrrz1ltZib7J4cWuxpWHpJUd65kmSJEmSGrDyJEmSJEkNWHmSJEmSpAasPEmSJElSA1aeJEmSJKkBK0+SJEmS1EBrlaeIuCIiDmsrviRJ48j8KEnjyzNPkiRJktTASFWeIsK7lEqS1MX8KEmjofXKU0Q8LCLOjogVEXFdRLw/IhZ2vJ8R8fyIuBS4tE47vs57bUQ8u86ze31vUUS8MyKuiojrI+LDEbGk7fWQJKmfzI+SNH4GceZpAngJsDVwAHAo8Lyuef4GeDjwwIg4HHgpcBiwO3BI17xvBfYA9qvv7wi8rqWyS5LUFvOjJI2Z1itPmXleZp6TmWsy8wrgI8DBXbO9JTN/l5m3A0cDp2TmxZl5G/D6qZkiIoBlwEvq/KuANwNPXttnR8SyiFgeEcv7v2aSJPVumPmxLmOOlKRpar0PdUTsAbwb2B/YuH7meV2zXd3xfAdg+Treu3eNcV7JE+UjgPlr++zMPBk4uZYje1sDSZL6b5j5EcyRktSLQXTb+xDwc+D+mbkZ8GrKAb1T50H7OmCnjtc7dzy/EbgdeFBmblEfm2fm0hbKLUlSm8yPkjRmBlF52hRYCdwSEXsBz93A/J8Fjo2IB0TExsBrp97IzEngo8B7ImIbgIjYMSIe207RJUlqjflRksbMICpPxwHHAKsoB/bT1jdzZn4VeB/wHeCXwDn1rTvr31dMTY+IlcCZwJ79L7YkSa0yP0rSmInM0e7mHBEPAH4KLMrMNTOIM9orOovMn9/epXS333lHa7EXLvA2Kpp9MrO7G5hmiX7lxxprLHPkvHnrvKRrRiYnJ1qJC+3myJW33tJa7E0WL24ttjQsveTIkbpJ7pSIOKrer2JL4G3Al2aaGCRJGnfmR0karpGsPAH/CPwWuIxyH4wN9QOXJGkuMD9K0hCNZD+lzDx82GWQJGnUmB8labhG9cyTJEmSJI0UK0+SJEmS1MDIj7bXLxGRbY1wMzExftfq7rTTXq3F3myzrVqLvXTpFq3FvuCCb7USd+nSLVuJC7Bq1e9ai71o0catxd52m11ai/2rKy5qLXZbNtpoUStx77rrDiYnJxxtTxvUbo5sb+S6iHZ273vfe+cNz9SjiYnVrcVuMyfsttuftBK3zf8ZLji/nbwOsGjxJq3F3mKLbVqLfc01/9dK3LZ+iwCLW9rWt99+CxMTa2bHaHuSJEmSNGqsPEmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDUwMpWniPhERJw07HJIkjRqzJGSNBpGpvIkSZIkSaOsL5WniNi2H3HWE//e0ebdtyRJaok5UpJmj54rTxGxRUQ8NyJ+DHyiTsuI2L1jnj90M4iIQyLimoh4WUT8NiKui4hj1xF704j4TkS8ryaEZwG/iogTI+K+vZZZkqRBMEdK0uw0rcpTRMyLiMdExKeBK4HHAG8C/rphiO2AzYEdgb8HPhARW3Z9xlbAt4AfZOY/ZfE24MnANsDymjSeFhEbb6C8yyJieUQsn856SpI0XeZISZr9GleeIuIFwBXAW4Gzgd0y86jM/EJmrm4YZjXwhsxcnZlnALcAe3a8vwNwFvC5zHxN54KZeU5mPrfO8yHgKcA1EfFv6/qwzDw5M/fPzP0blk+SpGkzR0rS3DCdM0/3BbYELgAuBG7q4fNuysw1Ha9vA5Z2vP4rYAnw4XUFyMw7gZ/UctwF7N1DOSRJ6idzpCTNAY0rT5n5MmA34KfAv1L6V78xIu7fMdttQGc3ge2mWZ6PAl8DzoiITTrfiIitIuIFtf/4t4H5wKMy8xHT/AxJkvrKHClJc8O0rnnKzN9m5rsz88HAE4AtgLMj4uN1lguAYyJifkQcDhzcQ5leAPwf8KWIWAIQEX9P6Q5xMHAisHNmviIzf9ZDfEmS+s4cKUmzX8+j7WXmeZn5Qkr/6qkuBC8CjgRWAE8FPt9D3ASWAdcAX4iIxZT+47tk5hMz8yuZOdFruSVJaps5UpJmpwUzDZCZdwE/rs+XAw9ax3zfBXbqmrZrx/NndjyfBJ7eMeslMy2nJEmDZo6UpNmlLzfJlSRJkqTZzsqTJEmSJDVg5UmSJEmSGrDyJEmSJEkNWHmSJEmSpAaijHo6+0XE3FjRhubPn/FAi+s0MbGmtdhtWrRo4w3P1IMLf/XLVuIC7LPLfVuLvXr1na3FbtOCBQtbibtmzV2txAWIaKcdK3OSzIxWgmtWMUdqQ+bNm99K3FW33dpKXIBNFi9uLfa4ajPftKetNJY95UjPPEmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDUwUpWniHh9RJxan+8aERkR7Q0LJ0nSGDA/StJoGKnKkyRJkiSNKitPkiRJktRA3ypPEXFsRHyp4/WlEfG5jtdXR8R+EfHe+nxlRJwXEQc1jP+EiLgiIvaOiMURcWpE3BQRKyLi3IjYtl/rIklSv5gfJWn26OeZp7OAgyJiXkTsACwEDgCIiPsBS4GfAOcC+wH3Av4T+FxErPcW0BFxLPA24LDM/CnwDGBzYGdgK+A5wO19XBdJkvrF/ChJs0TfKk+ZeTmwinLgfyTwdeDaiNgLOBj4XmZOZuapmXlTZq7JzHcBi4A91xP6xcDLgUMy85d12mpKUtg9Mycy87zMXNm9YEQsi4jlEbG8X+spSdJ0jGJ+BHOkJPWi39c8nQUcQkkOZwHfpSSGg+trIuK4iPhZRNwcESsoLWRbryfmy4EPZOY1HdM+RUk+n4mIayPi7RGxUfeCmXlyZu6fmfvPfNUkSerZSOVHMEdKUi/aqjwdVJ+fRUdyqP23jweOBrbMzC2Am4FYT8zHAK+JiCdMTcjM1Zl5YmY+EDgQOAJ4ep/XRZKkfjE/StIs0Ebl6VHAktoS9j3gcEoXgvOBTYE1wA3Agoh4HbDZBmJeXGN8ICL+GiAiHhUR+0TEfGAlpZvCZJ/XRZKkfjE/StIs0Ncb7GXmLyLiFkpSIDNXRsTlwA2ZORERXwe+BvwCuBV4D3B1g7gXRsQRwFciYjWwBfBhYCfgFuA0SlcFSZJGjvlRkmaHyMxhl2EgImJurGhD8+e3d2P6iYk1rcVu06JFG7cS98Jf/XLDM/Von13u21rs1avvbC12mxYsWNhK3DVr7molLkBEO7fcy5wkM9fX7UsCzJHasHnz5rcSd9Vtt7YSF2CTxesdrHJOajPftKetNJY95UhvkitJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpAStPkiRJktTAHBttr73ROsaNo+39sV133aeVuL/+9S9aiQuw8857tRb7V7+6qLXYbXr7KZ9pJe4rnvWUVuICtHUcdrQ9NRUR2dZoapOTE63E1WAtXry0lbgR7R2iJlv8f+TOu+5oLXabvv6TC1uJ+7iHHdhKXGhvtNs1a1aTOeloe5IkSZLUBitPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGxrbyFBHPjIjvD7sckiSNGnOkJLVjbCtPkiRJkjRIA6s8RcQVEfGqiLgkIn4fEadExOKI2DIivhwRN9TpX46InTqWe2ZEXB4RqyLiVxHx1Ih4APBh4ICIuCUiVgxqPSRJ6jdzpCSNh0GfeXoq8FhgN2AP4DW1DKcAuwD3AW4H3g8QEZsA7wP+IjM3BQ4ELsjMnwHPAc7OzKWZucWA10OSpH4zR0rSiBt05en9mXl1Zv4OeBPwlMy8KTNPz8zbMnNVnX5wxzKTwN4RsSQzr8vMi5t+WEQsi4jlEbG8v6shSVLfmSMlacQNuvJ0dcfzK4EdImLjiPhIRFwZESuB/wG2iIj5mXkr8CRKC9p1EfGViNir6Ydl5smZuX9m7t/XtZAkqf/MkZI04gZdedq54/l9gGuBlwF7Ag/PzM2AR9b3AyAzv56Zjwa2B34OfLS+nwMpsSRJg2GOlKQRN+jK0/MjYqeIuBdwAnAasCmlD/eKOv2fp2aOiG0j4nG1X/edwC2ULgoA1wM7RcTCga6BJEntMEdK0ogbdOXpP4FvAJcDlwEnAf8CLAFuBM4BvtZVvpdSWt9+R+nn/dz63reBi4HfRMSNgyi8JEktMkdK0ohbMODPOzcz39I17TbgkK5pH6l/r+OeF8b+QWbeBfxVX0snSdLwmCMlacR5k1xJkiRJasDKkyRJkiQ1MLBue5m566A+S5KkcWKOlKTx4JknSZIkSWrAypMkSZIkNRCZc+M+ehGREe3UFTMnNzyT+qKt7xBg++3v10rca6+9rJW4AJtssllrsbfddtfWYj/luS9oLfYH3vSaVuKuWnVTK3EB2joOT05OkJnRSnDNKvPmzc+FCxe3EvvOO29rJa7+2KKFS1qLvdlmW7cS94Ybr2klLsBWW23fWuw77ri1tdiHHfaM1mL/9KL/aSVum7/zFTff0Erc225bycTEmmnnSM88SZIkSVIDVp4kSZIkqQErT5IkSZLUgJUnSZIkSWrAypMkSZIkNWDlSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDY1l5iohXRsRlEbEqIi6JiKOGXSZJkkaBOVKS2rNg2AXo0WXAQcBvgCcCp0bE7pl5XedMEbEMWDaE8kmSNCw95MgYcBElaTyN5ZmnzPxcZl6bmZOZeRpwKfCwtcx3cmbun5n7D76UkiQNXi85MsLKkyQ1MZaVp4h4ekRcEBErImIFsDew9bDLJUnSsJkjJak9Y9dtLyJ2AT4KHAqcnZkTEXEB9jmQJM1x5khJatc4nnnaBEjgBoCIOJbSqiZJ0lxnjpSkFo1d5SkzLwHeBZwNXA/sA/xgqIWSJGkEmCMlqV1j120PIDNPAE4YdjkkSRo15khJas/YnXmSJEmSpGGw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpgbEcba9XmZPDLoJm6Igjntda7DPP/PeWImdLceHWW29uLfZvf3tVa7En1ky0FnvFiutbiRvRXluTxyYNW+Ykd911x7CLoRl63ONf2FrsL/z3v7YUub0cedNN17UWe7PNtmot9na7bt9a7C984YJW4m600aJW4gJMTKxpJe7kZG//i3jmSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDVp4kSZIkqYHWKk8RcUVEHNZWfEmSxpH5UZLGl2eeJEmSJKmBkao8RcScuu+UJElNmB8laTS0XnmKiIdFxNkRsSIirouI90fEwo73MyKeHxGXApfWacfXea+NiGfXeXav7y2KiHdGxFURcX1EfDgilrS9HpIk9ZP5UZLGzyDOPE0ALwG2Bg4ADgWe1zXP3wAPBx4YEYcDLwUOA3YHDuma963AHsB+9f0dgde1VHZJktpifpSkMdN65Skzz8vMczJzTWZeAXwEOLhrtrdk5u8y83bgaOCUzLw4M28DXj81U0QEsAx4SZ1/FfBm4Mlr++yIWBYRyyNief/XTJKk3g0zP9ZlzJGSNE2t96GOiD2AdwP7AxvXzzyva7arO57vACxfx3v3rjHOK3mifAQwf22fnZknAyfXcmRvayBJUv8NMz+COVKSejGIbnsfAn4O3D8zNwNeTTmgd+o8aF8H7NTxeueO5zcCtwMPyswt6mPzzFzaQrklSWqT+VGSxswgKk+bAiuBWyJiL+C5G5j/s8CxEfGAiNgYeO3UG5k5CXwUeE9EbAMQETtGxGPbKbokSa0xP0rSmBlE5ek44BhgFeXAftr6Zs7MrwLvA74D/BI4p751Z/37iqnpEbESOBPYs//FliSpVeZHSRozrV3zlJm7drzcq+vt13XM191Fgcx8C/AWgIh4ADBJ6a5AZt5B6drw6v6WWJKk9pkfJWl8jdRNcqdExFH1fhVbAm8DvpSZa4ZdLkmShsn8KEnDNZKVJ+Afgd8Cl1Hug7GhfuCSJM0F5kdJGqLWhyrvRWYePuwySJI0asyPkjRco3rmSZIkSZJGipUnSZIkSWrAypMkSZIkNTCS1zxJ63LGGR9uLfbhhz+7lbhf+Up7ZY5or/1jp53auz3Mby7/TWux2zJvXnvbenKynbjlvqlSM23t4xMT7oeD8tnPvL212DvueP9W4v72t1e1EhcgM1uLvf3292st9r13vndrsduy0UaLWosd8Ud3beiL1at7OzZ55kmSJEmSGrDyJEmSJEkNWHmSJEmSpAasPEmSJElSA1aeJEmSJKmBvlaeImLPiLggIlZFxD/1M7YkSePMHClJ46/fQ5UfD3wnM/frc1xJksadOVKSxly/u+3tAlzc55iSJM0G5khJGnN9qzxFxLeBRwHvj4hbaveEd0bEVRFxfUR8OCKW1HkPiYhrIuJlEfHbiLguIo7tiLUkIt4VEVdGxM0R8f2OZR8RET+MiBURcWFEHNKvdZAkqQ3mSEmaHfpWecrMPwe+B7wgM5cCzwH2APYDdgd2BF7Xsch2wOZ1+t8DH4iILet77wQeAhwI3IvS1WEyInYEvgKcVKcfB5weEeN3K2ZJ0pxhjpSk2aGV0fYiIoBlwEsy83eZuQp4M/DkjtlWA2/IzNWZeQZwC7BnRMwDngW8KDN/nZkTmfnDzLwT+DvgjMw8IzMnM/ObwHLgL9dRjmURsTwilrexnpIkTZc5UpLGV78HjJhyb2Bj4LySIwAIYH7HPDdl5pqO17cBS4GtgcXAZWuJuwvwxIg4smPaRsB31laIzDwZOBkgInL6qyFJUt+ZIyVpTLVVeboRuB14UGb+uodl7wB2Ay7seu9q4FOZ+Q8zL6IkSUNhjpSkMdVKt73MnAQ+CrwnIrYBiIgdI+KxDZf9OPDuiNghIuZHxAERsQg4FTgyIh5bpy+uF9bu1MZ6SJLUb+ZISRpfrVSeqlcAvwTOiYiVwJnAng2XPQ64CDgX+B3wNmBeZl4NPA54NXADpZXt5bS7HpIk9Zs5UpLGUF+77WXmIR3P76AcwF+9lvm+C+zUNW3Xjue3Ay+uj+5lfwQc3KciS5I0EOZISRp/tkZJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1EJlz46bi5e7pseEZezI3tuFsd+FVV7USd9/73KeVuAALFixsLfaiRUtai33nnbe3FnvNmrtaibtoYYvb4647WoqcZGZbBz7NIgsWLMwttrh3K7FvuunaVuJqsDbeeLNW4t7V2vEPNt9869Ziz4v5rcXebLOtWot95VWXtBL3Sccc10pcgDO+9PFW4q5ceSNr1qyedo70zJMkSZIkNWDlSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDVp4kSZIkqQErT5IkSZLUgJUnSZIkSWrAypMkSZIkNWDlSZIkSZIasPIkSZIkSQ0sGHYB2hQRy4Blwy6HJEmjpjNHzps3f8ilkaTxMKsrT5l5MnAyQETkkIsjSdLI6MyRCxYsNEdKUgN225MkSZKkBqw8SZL+f/buO8ySskr8+PdMDwxDRqIEkSCiIqKihBUB++T3jwAAIABJREFUQUQFFf2JaQ0YWOMaMKMoZnQxsCgKq5gVWXfFzBpZAwjDCiIGEARBgoAOzEiamT6/P963mct1Zrq6+9YNPd/P89ynb6g699y6dev0W/VWvZIkqYFZ0XiKiO9ExJsHnYckScPE+ihJvTUrznnKzMcOOgdJkoaN9VGSemtWHHmSJEmSpLbZeJIkSZKkBmw8SZIkSVIDNp4kSZIkqQEbT5IkSZLUQGSuHoOKj43NzXXW2aCV2JtssnUrcQE232zbVuKe84tvtBIXYGysvYs4RrTX3t9i83u3EvfqP1/SSlyAOXPGWou95Zb3aS324sV/ay32woV/aSXuZpvdq5W4AIsW/bWVuLff/nfGx5dFK8E1q8ydu2ZuuOGmrcSeN2/tVuICPPjBB7YS9zvfOaWVuABt/t/VZuz11tuolbh///vNrcSFdte9TTfdprXYC/92fWuxb1l0UytxN9poi1biQnv/Myxdeifj4+NTrpEeeZIkSZKkBmw8SZIkSVIDNp4kSZIkqQEbT5IkSZLUgI0nSZIkSWrAxpMkSZIkNWDjSZIkSZIasPEkSZIkSQ30rPEUEZv3KlY/Y0uS1DZrpCTNDjNqPEXEhhHxkog4F/h0fW7LiPhqRNwQEX+MiH/tmH5eRHw4Iq6ptw9HxLz62iYR8c2IWBgRf42In0TERH6fjohzI+LFEbHhTHKWJKkfrJGSNPtMufEUEXMi4qCI+BJwJXAQ8G7gCXVD/g3gQmAr4ADgVRHxmDr70cCewG7Ag4CHA2+prx0FXA1sCmwOvBnI+toTgPcAjwGujIgvRsSjOwrHynI9MiIWRMSCzFzVpJIkzdjo1sjxHnx6SZr9ptR4ioiXA1cA7wPOBnbIzMMy84zMXAI8DNg0M9+RmXdm5uXAKcDTa4hnAe/IzL9k5g3AscCz62tLgHsC22bmksz8SdYWT338tcw8DNgBOAc4Drii5rRCmXlyZu6embtHxFQ+qiRJUzLaNdJToCWpialuLbcDNgIuoOw5u6nr9W2BLWu3goURsZCyd2yiP/aWlD1xE66szwF8APgD8D8RcXlEvHElOdwE/KrmsFHNSZKkQbNGStIsN6XGU2YeRdmr9Wvg34E/RsQ7I+I+dZKrgD9m5oYdt/Uy83H19WsoxWPCvepzZOaizDwqM7endEF4TUQcMDFhRNwnIt4J/BH4CHARsH3NSZKkgbJGStLsN+Xj9LU7wQczc1fgKcCGwNkR8SngXGBRRLwhIuZHxFhE7BIRD6uzfwl4S0RsGhGbAMcAnweIiEMiYsco/etuBpYB4/W1T1G6QGwIPDkzH5SZH6rdGiRJGgrWSEma3ebOZObMPB84PyKOAnbLzGURcQhwPGXv1zzg9yw/4fVdwPqULgUAp9fnAO4DnEg5GfZvwMcy80f1tY8DL87MO2eSryRJ/WKNlKTZZ0aNpwl1g31uvX8N8IyVTHc78K/11v3ah4APrWS+c3uRpyRJ/WaNlKTZw8vrSJIkSVIDNp4kSZIkqQEbT5IkSZLUgI0nSZIkSWrAxpMkSZIkNRCZOegc+iIibuDuI7dPZhPgxhZSaSvuqMYexZxHNfYo5jyqsYcl520zc9OW8tAsMsUaOSzr97DEHsWc24w9ijmPauxRzHmYYk+rRq42jaepiogFmbn7qMQd1dijmPOoxh7FnEc19ijmLDU1quu3v/f+xB7FnEc19ijmPMqxJ9htT5IkSZIasPEkDUBExKBzkCRpGFkjNcxsPK3cySMWd1Rjj2LO044dEXMAsqu/bFehcFmPfuxRzFlqalTXb3/v/Yk97bgNauQoLo82Y49izqMcG/CcJ6lvImIsM5dFxH2AZwA3A9dm5lcGnJokSQNljdSo8MiT1Ce1KNwP+DlwL2Ar4NiI+NBgM5MkabCskRoVcwedgLQ6qF0O5gKvBj6Sme+KiDWBw4C1BpqcJEkDZI3UKPHIk9QHWSwBxoCLa7/uc4FzM/MlEfHAiHj4YLOUJKn/rJEaJTaepJZExFj9u35EzK/FYAw4BPhf4KLMfFad/Chgz8FkKklSf1kjNaq8YITUgo4TXx8IfBJ4c2Z+PyK2As4ByMxt6rSfBnYFHp6ZSweVsyRJ/WCN1Ciz8SS1JCLuD/wU+FBmvjMi5mbm0noloa8DVwMTP8DHZ+aSiYIyqJwlSeoHa6RGlY0nqYciYk5mjtf7JwNXZ+Y7aneEVwB3UPpxXwRsX2e7NDPHJwrHQBKXJKll1kjNBl5tT+qRiIi6gd8FuAX4NfDUiLgQeDtwA+U3dyjwnMz8fce8cywKkqTZyhqp2cILRkg9ULsSZERsCZwHPA74AfBb4CnA1zLzIODFwJr1dpeJPXGSJM021kjNJnbbk2ag7knLen9H4JHA5pn53o5p5mXmHfX+p4BtgUdbDCRJs5k1UrORR560Wqv9rFf6eBXzvTAinpx33/twJPAfwMMiYqwj1rKIeFBEfA14KHBw7brg70+SNJSmWx/rtNZIzVqumFpt1W4E4xGxRUTsGhFbNNnTFRGbAwuB6zufz8zXAx8ADgIeNBGro5/2D4CH1isGzXWv2vCKiHUHnYMkDcp062Od1xo5y63uNdJue1otTXQlqGNMnEXpd70LcHhmnrmK+fYCXga8IjP/FhHbApsDG2fmd+o0HwOeDhyYmf+3ghheanWI1T757wWOAa6ygEtanUy3PtZ5rZGznDXSxpNWQx2FYTPgrcAlwMnAq4F3Ak/LzP9ayby7Aosy848R8SDga3X+nSh72Z5RX/sEcBjwhMw8p/1PNby6+rzfdX9Y1X8YNs3MHw46l+nqGIBy6Je3pOExk/pY57dGTsGo1UewRoKNp2mbGKsgItbMzDvrcyOx4gsiYkPgU8A9KYPv/bU+/yrg/ZQC8d+rmH8D4NvA5zPzpIhYEzgD2BjYq/4ovwKsl5mPbfnjDK3OcTk6x/cYBRERwLuAszPzm4POZ6rqOn4I8NPMvGLA6Wg1Y40cXTOtj3Vaa+QkRrk+wupdIz3nafrWiIhtgPdGxAsALAojZQPgYuABwJMnnszMDwOvBb4aEfuuYv61gLUpXRrIzDtrAVgL+GB97nDg8a1kPyKyjBY/FhFfAr4dES+IiO0nnXFAajGYsBOwNfDciDhwQClNWUTsW7dJPwI+CzxxwClp9WSNHF0zrY9gjZzUqNVHsEZOcJDcaYiIZ1I2Ko8C9gBOBT450KS0Sh2HaOdl5h2ZeWVEvAdYArw4Iv6emV8CyMwTIuI64Gcd83fvFboRuBY4OCIum7jMKvAF4B4TE01cMWjU9ijNVNce5jOARcCPgWcC942Iz2bmrweV34p097PPzN9HxPuAlwAvrZ/pe4PLcNUiYj/KXrQnAP8N/AH4M26b1GfWyNEy0/pYY1gjGxrF+gjWyE42nhqKiDHKCvIASj/dY4HTKSNkv6dOY5eEHun+kc5k2dYN87Ioo5q/KSLWAH4DfBl4N6VA/GtEjGfmaQCZ+ZWI2CgiFmfmkrqB3wLYBLgzMy+JiG9S9ppdHRFnZubNwMOBKzrff3UqCvAPXRHWpRwSf199fAHwIuB5EXFqZl48wFTv0vHPwxzgKErXkp8BZwPHAW+k/BMxnpk/GGCq/yDKla0+A9wO3AI8OTN/HRGvoKyvt69u/5yo/6yR/TMM9bHOuwnw18wct0Y2M4r1EayR3ey210BErE856fFAyo9+j8w8CfgLsBFwK9gloVdqIVgWEWtExLvr3rBpL9u6Ud8B+B5wIfANSreE7wHrAZ+j7IV4z8Sh54jYu76+e328C/DzOu1nI+JTmfkx4BfA84DzIuLrwH2AN0w311FXv7uJrgj/RSnAr42IhwFk5ncp43zcC3h5RNxvgOnepaMonAc8DNiQcij/P4HFlD1TV1L+udhvUHmuxBrAmcCLgX+pReFhwJuAH2Xm0tXpnxP1nzWyf4ahPtY89ga+SzmyaI1sYFTrI1gju3nkqYHMvCUi3puZP4/lJ8HuTLnyzOsy8/rJYqiZrj1q/wGMdRzun068NTJzCeVQ7Xcz8/31+dcC387MhRGxCPg0pYvBjwDqd30d8L6IOBp4PfA+4DTg/sAJEfG1zHxSlFHT96L8g3DGxMYxZ3Cp1c49iTON1S9deX4SmA98njLo4XMi4qrMvC4zvxPl5OGnADcNKN0VeRFwfe2HT0R8B0jglsy8ICKWAc+iHOofuIgIYOvMvAr40MRzETEXOBT4VGb+2L39aps1sj+GpT6CNXKqZkF9BGvkXTzytAoRMSciXgRlQ1Gfnlhmu1CuJHPGIHKbrerejYiIkymXNX0J/MNJiisVEYdExPMj4p/r4fEl9aU1gavqNL8EfpOZ/xJlvIKnAzdl5uco/bPfUnM5hNJv+wN1/jMz8+bMPJvSLWWLiHhuZv4hMz+XmV/tRVGo751Ruk9MLJNNZhKvH7q+u6uBJ2bmF4D9gccAR0fp1kFmngG8JDP/MriM/8FalC5GRMTngK0ogzmuGRGPzsyLgGMy8+oB5giUbROly8TbImJ+fW6iAKwJPJr6WWw4qS3WyP4adH2s739obSxZI6dgFtRHsEbexcbTSkTpv/0L4MlRBnkD7jYS9muBGzs2PuqdjSgr9quALepzk66rEXEE5cTkbSn9b9/e8fIfgddExCXA/2bmM+rzxwMPn9iwUU5kfWXd80ZmPgX4HeXE5z3rHguA6yiHqDfvzqMXe8Dq+vf1iHhllCtWXR6lW8Sw24iyMT2K+t1l5u8ohfQAypW3NqvP/31QSXZ8j51uAg6LiK8CO1K6Hi2h7FF9WkTMz3rJ5UGqReFc4FJKgb0N7lYAnkfZNn15MBlqdWCNHJiB1McaIyjdtV5ljZyWkaiPYI2cVGZ6W8GN0t/30x2PNwHmUTZSDwQ+0/FaTCP+2Eqen3KsprGH9baifIHNgF8BP2jyuYB1KP2vH1sfHw/s3TXNO4FxygmrO1MKyQXA3I5p5gOHU/rtv77j+S9TTozcB1i3Pvct4NiWlsk8yhVhrgZuAI6sz88Z9PfV8Lu7CPhe1/MPAhYAmw8o1y2A+3TmXpfxMyhFbU3gI5STSner07yMct7GAwe9rDvyfgzw/Y7HRwEnAC+l9O3eBHjAMK4v3mbPzRrZt+U8NPWxTmeNnPl3N3T1seZgjWx485ynFYgyuNvNwMfr4xMp17OfD7wpM386cdh6On0l4+5XLXkZ5eofi7Mc0p5xF5tcfhRle+DyXsRsywqWxSLKsvjPKCennhUR38/MA+t0KzvcP4cy3sDW9fHuwEURcTXlhz4fOInSNeFE4HLKVYQelrUbAUBm3hYRvwA+RjmMfktmfjwznx4Rp1P6Kl8VEZdTTupsZQydzLwjIs4HtqzLZGl9frwXXR56YZLv7gA6vjuAzLwwIvbOAeyZioi1gbcAW0XEqzLzSso/BospXRHeARxNuYzu34EzImIB5Ts+KEt3hGHxF8rVgY6j/MZ3pJzkfTzwpyyDFd4Iq9dVrNQ/1sj+GKb62BHfGtnAKNXHmq81cgpiSLcZAxXlykEnUEbXXkq5JOPzKS3uKzLzRT14jznA+cBfKZcFfSrlqiWv6MWGvK40BwHPBS4a1uIAK10W/5WZL41yicnvAXdk5sMmifMKSneEgyl7Pk+mFIbFlBNVzwFeCQRlQ0ZmZm04jdf7DwS+A3yFsvzuAXwkM4+r7/Fx4AWUPW9fn6RgTWdZ3HUZ0/r4wZS9JP8BnJiZH+jF+/TKVL+76fwj1cNcD6EM+DhG2Vu5UWa+tL72Fsoldd+amd+PiAdRTpAez8wbB5Fvt7pdWgrcQSnG9wSWUfbsLomIz1C63DiejlpljeyfYaiPE41Na+TUjFJ9rO9vjWxqOoerZuONssHYB3gw5fDklpR+qE+lHnoFXgecAqzRg/c7Cvhyx+MzgLNnEG9Fh4e/TvlxPojedHWY03F/xvEmWRbnTLwf5VDyL4BtJ4mzLmXvyJXAfYFtKF0Q7g88FvgJsE/H9DsBG3Q8XptygvOb6uMtgefUeK/umO5dHbnNuIvACvLYgXJJzX2Azepz/0zZG/ia+vizwJN69R0M+rtrOcfouH8w5R+8C4G3d033QeDiXnynPc5/DvBV4IeUy62+peO1ufXvqyl70nYcdL7eZufNGtnoPXpeIwdVH+s81sgh+O76kKc1cqrvOegPPQy3uuDPBn5K2Uvwa+CAzoVP2WOzENi1R+95LPCBev8LdUVdo/6Y9phmzAA27HruG5SRq2eUN8v7MM8B1urx8l/Vsthz4n1X9t11PV6HMiDjn4D9u177Sd3QB+UqMdfX73X9jmm+CryyK96/UfbOHd0Vb0b95rvyeFNdtjvXH/jP67r4MWofZEq/41so4yz8gR78gzLI766POXb+Q/Mc4GmUPacXAjt3vLYD5VK8G/Q7x0nWke9R9qruQRmB/jJqQab8U/FO4BrgIYPO19vsvFkjG8VupUb2uz52LCdr5AC/uz7naY2c4s2r7RUnAb/PzEdQDjWfCnwzIvavVxx5FeWa+/tn5q+mGnzifJqOx0E5jLt2RHySMiL7Q7NcteQI4FkRMW8an+MYSj/mTSeeyMxDKT/WEyPiofW9p5r/64HHR8SGlD11B08jt4lYU10Wz4yItXIF/VJrV4DxiLhnRPxTRDyU0tPgzZSB+j4TEY+q055G6V7wxSz+DLwCeCHwoojYqB5ivxF4YJQBCCPLFW/OpWykHxwVzPyKQV15vICynj2TcqnPvSl7CdcC3hoRO2Xmlyh91T8E3DfLYei+nbfYy+9uiu87t+tx43U4lo85E1EGaNw7M0+jjEdyEfCOiHhInfxJLL961bDYjrLH+A2Z+YvM/CLwOOABEfEsylWyLgT+KTP/b4B5anazRq46/57UyGGoj/APtcka2cAo1sc6vTVyOgbdahyGG2VQt5fU+xPngb0D+FIub21vNs3YE90ZgnIFkIfUx1tTWse3AlvU515OOdHt/g1jR9fj7YEv0XXFFsremDspG8Z5DWPfs+P+8cAP6kr41Rks554ti47vaVfK5Sh/Sjn8/QuWH8Z/Z835aMplc9fozKPefzLlqkGvrY+3q+97AvXqMZRuKEd3vGevumN0Hip/Ss3jQuDpHc8/hjJA4anUq9t0L88+/UZaW4+bLCPKnu0FdFwJaIpx3ljX/87v/kDgdMqJ76dR9j4P1dEbyl7Xc6h7ievyX4Pyz89bBpmbt9XnhjVyRbF7WiN7uRyYYX3k7rXJGjnAdbjJ8mGG9bHGsEZO5X0H/cEHvNDXrn9PBt7T9do/U0bYnkn8u/r8UsZBOJdyItuxwHqUPsf/R+kH+13KpUcf3DB25+W1O/sCb045kfP8jo3kkZTB9LZtGHs+5bB+Z7eMiyh7nA7t3FAMcllQxpv4BfCi+nhHyojdf6T0yR+jDN73X10bmeDuh6kPp/TZfmN9vFONex7wy7os15jqZ26wTLrzOJRyaPm9dHQtoZyUewblRM2e5DDo726qOQJvpZxoO504j6T8Y7OYfyyue1P6SJ8CbNmr77YHn31+xzL/CvBNYIOO9fjfgXf0ep305q3zhjVyZbF7WiPbWA5Msz5O5I01cmDf3VTzYwb1sc5vjZzq+w96AQxooc+h9I88qD5+HKVl/XzgHvW5l9UvY50pxp7X+WWxfE/EyfXxoZTW+7/VL3p9yt6hh9Dw+v4s3zBPrDRnU/a8PKU+v1l9fjFl7IUlTHGPBHWvWsfG9FDKnqr/ofSJXb87n34si47PvjZlnIezOzdElEP4X2P5Ca0bdL3/xPzr1fdbsz5+IncvDhtSDrPv0zFPz/ZirSKPQ2seR1GudDMx/cPpzYm3G9OgsPdjPZ5Czp+l9L+e+L2ucjms6HuiFIdvU/ZGbdf12oG9znkGn3UO5R+c71IuA/1ESoP/l5Tt0XHA2yiDFe486Hy9zc4b1sgm7zGjGtnGcmCG9bErhjVyJTWyH+vwFPKdUn1cxfpojZxKHoNeEANa8BdS9gJsxPK9Bs+kHBL+MfDflNGxd5ti7K3rl7lxR9xvUS75+MKO6Q6gnJz5XmCnGXyO8ymHUg+g7EH6X+AFHdMcSSlwU16BOvL/AfDxjuc/QDk575n18ZMpVzHp7h7R82XRsaHaFfg+8AjK1VWe3PX6vwPHdc/bkccD67L7LmVv4SH1+SdS9sq9Fliva/5eFoWmebwG2HhF805zfflpXV6/pGvjOKj1uGHu76eMK/FWJulSw93/aXoypW/5ziy/UtiXKP8UbttWvjP4nFF/w5+lnPj6GuDPwH6UPd3HAp+gHAXYZdD5epudN6yRjePXv1OukW0sB2ZYH7s+kzVyJTWyn+tww7wb18fO7wlr5MxyGfTCGMDCfzfw2Y7HjwEeRdkTtS3lKiNHANtPI/YjgYO7ntuHcnj/tK7n96ccvn0bdY9Kg/idfVFfCHyl4/GX60p0Jh3FYRqfoXs08UdQrnTz0Y7njqsbmG9T9kY+oF/Lom6wjgdeVh+/o2609qB2zaD0233XSubfqubwhvpDPJZyhai96uuHUca+ePqq8ujBetjXPCgF/TRgt7qu3MBKug60vR43Xce7nj8auIRSNCdbR+ZQBvc7gzK2xrco/fPHKBvZz9Vlca82v+NpfPYH0zHqfF2Pf0HZg9z525/b79y8rT43rJGTvceMa2Rby4EZ1sf6ujVyFTWy7XW46frd9Xzj+lint0bONJdBL4w+LvSN69+3UE6A25zSwr6IspfmR/TosGT9oR9DubIKwJ6UQec+2DXdI5l8bIa1JmLWv2OUPqh7s/ykxP+g9D2+F2VPxyXAS6eRd2ff3eMpV7bZn7L35y/AxzqmfTLwIibZY9eLZdHx2TeoP/iz6Sjc9Ud/Uf0OJ/oUz+2ev97fm7sX1P8FPl3vr1P/7r+yjdRM14tB5FG/y69Tu9vU506h9M9/UD/W44Z5du4RO4myd+9zE7Epe9Z+T+kSMa9r3s5+8e8Cvlnvrw08gdJV5ej63FMoe6fuOdOce7RebFyX9Z7AefW5T9Z1euI8ghdSCxkr6UrizdtMblgjm+Td0xrZi+XADOtjZ4x63xqZk9fIXq7DDXOcdn3sXHfrfWvkTHMa9ELp48L/PmVPxX1Z3iXhq5SrcuxRH0+7MHD3Vu8udUX8zMQPr24IFlGv+d8w5q6Uwcom+lYH5YoqE+MGzKVstM+mnshXN5TvZJon9tX3OJey1+xrlL60r6if6S/Av/dzWbC8MGwC7EvpLnA78NSu6R5D2SP6PJb3QR/rmH9TymCAe9Tvf4e6LCfGAtisblA2WNHn6MH6N5A8KHuKP1mX2RO6XjsJGKdeMamt9XiKyycoXSb+k3LewI+Bs6gnZtd1+3rgsStZd3cCXk/HnlXKb/wllHMRJgrQur3Mf4af/X8ol4Ado/yDdxlwTsfrr6OMabLJoHP1NntvWCObvueMamQvlwMzrI9dMayRk9TINtbhKS6badfHjvmtkb3IadALpU8L/qmUvQoTe6jWA+7J8r1IL6YM+jfdS612XqFmN8reradSRmP+XMcPa8/6Q3x3g5ibANcCr6qP59SV+5Nd0/0TZW/TQygbzl8C20wn/3p/HnB8Rw7PphSJV1JODl1GV3/ptpZFx494c8pAeJ+hjOHw+rqBeNyq5u2a/yTKSYYHUjaUf+Tue7U+U9eRng9YN6g8KKO/b1DX9Q9TCv4juqb5MLU/dhvr8RSXTwAPA77V9fongJ91PD4a2GEFcV5DOUH3GOCKrtfuVX8bw9YN4amUf0onBtg8hLJn+CTKPw1vonQfmdK5Jd68TeWGNbJR/vX+tGtkL5cDM6yPK4hhjVxFjWxjHZ7isplRfayvWSN7dOvbAJsDtg9ltOnxiJibmYuARXXwuNcDzwUOzMy/TDVwHSRuaR087mzK3osNKCceJvA34HURcVxmnhMRe1JGwG7iPOCHddCzEyg/nA0iYuPMvAkgM38WEbdRTgLdmnJy6FVTyH9OR/4fpAyUt0VEbJSZN0bEtyl7755OOSFvF0pxaHVZ1FjLImJX6l4mSvFbTOmL+27g4xFxZGZ+t06fHSHGMzM75l+/zn8dcBtwOfCbiHgu8Oj6uR6WdbC4rljTtoLP0Xoedfn/gFLkt6dclegMykml742IN2TmzwEy81Udeba1Hq8q12cDX42I24G3U66Qs3dEbDbxe8zMf4mIKyLiiMw8NTPfvZJw11JO/j4AeFBE/ITSN/3vlEvZLqWcfzBM9qHsRbuzPv42pRC8iXIi8FLgUZl50WDS02rCGrny/HtSI4epPtZ5+16bGn6WoauRE+8z4vURrJE9M6efbzYIEXEIpe/xSZl5Z914jUXE4ZS9HGPAvjnNkYc7frifB/6QmftSDo8/grIBPYOyQXtXRDwgM8/NzN81CD1O+UH+K2XF2YjSb3sh8JyIWKcjh70oxe2hmXl+09xj+QjkQTnpbidKYdiUMvr1urUAnUE5PL0XcF1mXtL2sqgbqk0oP5QfUvZCHUkZM+H/UQ7jHw98PSL27N6ArmD+R1P2Sm5Xc/gRZfTvfSiXPd09y2jkY70qCoPIo36XP6dsXB5LOX/h7cBD63v9GPhEROzRnWe92+v1eFW53ovyz8j7KeNj3DszP0Ipas/rXMcpewSv6Zj3H7ZdWUaX/xllo/pmSoH7XUR8j7I37l8yc2gKQ/e2CSDLaPMbZOaTMvMpwLNtOKlN1siV62WNHKb6uJIY1siV1MhRq491fmtkm3IIDsu1cWP5oc7XsvwkuAdRRnc+jzLGw33pwRVQauwvsrxf7CeBi+r9dSlXJvowU+xjTdlQ30zZ87J1fe6NlI30K6gDGM4w76D0cz+x47ljKJeifSnLT8y8Bx19jPuxLChX2/kBHeOIUMYbuZhyKcp9KIViZVegWdH8E4d8P0HXuB4ri9ODZdy3PCgbms4rZZ1I6cJxIaVf8OMoxeI8Vk3vAAAgAElEQVTe/VqPJ8l3V0of8UtYPo7HqykDN36E8s/IKyiFYfuueSdO2D28a/34JnWMFUoh3p0pdtNp88bk26YzWN6V0otDeGvl1mA9tEZm72tkr5bDSupK4/q4ihjWyJXUyDbW4UlynXZ97Fh3rZFt5DjohdTyF7AJZWTnD1P2ylxD6e/5sh6+R1CuVnIB8AzKnoILWd53/HWUkzWnfPIdZUTwU+uK8sm68gTl0p1fpfRtnlFxoPTdHaeMpXDXoH+UPTH/Sekj2+g9er0sgC0oe5yeVR9P9L//XM33XcBW9bkVDfq2qvl/SRl/Yas+rId9y4NyVZqJk0c/Afyq3v8Y5Z+MF7OCQS3bXI8nyXcbShG4uP7duK5/z6Ds5TuPMmbKQ1Yw72bA+yh7mj8KPK0+/xPgw21/rzP83K1vm7x5m+zWj/WwzW0LI1Qje70cJqkrk9bHBjGskX1YhyfJddr1sc5vjWwrv0En0OKCn0PpxzpO6QP8MbquQEIPT3ykDCB4E/DnjudeBlzFNMbD6Iq9DeUSo5+mnJA6MebBF+gYYXsG8fegHA4/jFoEKF01PlB/oFN6j14uC8qevf+jDo5Xn/to3XidCzx+mvMf32T+Hq4ffcujrh+bUcYzmSieb6jvt8oC1OZ6PMn73pfSNeGjE+sb5cpFj6IWqFXMu3MtEGdRrnx1LGXsitZGF5/hZ+3rtsmbtxXdrJFTit+zGjlM9XGSGNbIPq7Dk+Q77fpYp7VG9vg2cfnDWan2GX0x8CHg9iwnwbb1XmtT+l6/gLIXbG3K9fafkJm/7EH8HSgnvF5L2QNxEWVMgptmGrvG35cyrsEbge9m5q0RMUb5od44xVg9WxYRsR7lUO2rKf2RN6OMR7JrRJxKOVx+eK5kRZ7p/L3S7zwiYkvKxvE44FbK4e+9MvPqSeZrdT2e5L13oYwv8xPKSaCHAXtk5p8azLsmZYP7AcpAeg8A7peZ17WX8fT1c9skrYw1ckrxe1Ijh6k+9ipGL4xCjRzV+ljnt0b20KxuPHXq5dVhVvEecymXqTyIMtr0WZl5aQ/j7wB8lnK4+FVZT6DrYfx9KS38dwFfy8zbZhCrZ8uinvi4J2Uci78DJ2fm7RFxInBbZr6uzfl7pd95RMTzKXvF7gD+NTMXNJyv1fV4kve+P2UP5IaUywE3Kkidv++I2I6yPAdeFOqJ0Otl5h9XMU3r2yZpMtbIRvF7UiOHqT72KkYvjEKNHMX6WOcduhrZpD7W6YauRq42jafZoq70S3MKl1qdYvwDKf2LHzVsLf0JdQP7Okp/9kdm5sX9nL9X+pFH3ZsXmTnjy6b2Sy1OkZlLpjjfUG1gI2Ie8C3guZn550HnI60OVvca2Yu6Yo0cXtOtj3XeoamRo14fV5dxnmaNyVroPYj//Yj4eWbe2ub7TFdErEU5TP4o4NHTaDjNaP5e6Vcew1jcJ5OZS6c531AUhQmZeUdEPGFYf0vSbLQ618he1BVr5HCbbn2s8w5NjRz1+uiRJ42c2u947nT3FM10/l4ZljwkSbNDL+rKsNSmYclD6mbjSZIkSZIa+IcRiCVJkiRJ/8jGkyRJkiQ1YONJkiRJkhqw8bQSEXHkKMUd1dijmPOoxh7FnEc19ijmLDU1quu3v/f+xB7FnEc19ijmPMqxJ9h4Wrm2Fn6bX+ooxh7FnEc19ijmPKqxRzFnqalRXb/9vfcn9ijmPKqxRzHnUY4N2HiSJEmSpEZWm0uVR0RrH3TNNec3nnbZsqWMjTUfm3iHnXZoPO3f/vpXNrrHPRpN+9tf/7pxXEmjKzNj0Dlo+LVZIx/60Ic2nvaGG25g0003bTz9+eefP52UJAmYXo208dQD2223a1uh+cr3/7uVuA/boXmjTNLosvGkJtqskW3+nxHRVgeaUcwZMsdbiy3NRtOpkXbbkyRJkqQGbDxJkiRJUgM2niRJkiSpARtPkiRJktSAjSdJkiRJasDGkyRJkiQ1YONJkiRJkhoYycZTRLwxIi6LiEUR8ZuIOGzQOUmSNAyskZLUnpFsPAGXAfsAGwDHAp+PiHsONiVJkoaCNVKSWjKSjafMPD0zr8nM8cw8DbgUeHj3dBFxZEQsiIgF/c9SkqT+s0ZKUntGsvEUEc+JiAsiYmFELAR2ATbpni4zT87M3TNz9/5nKUlS/1kjJak9cwedwFRFxLbAKcABwNmZuSwiLgBisJlJkjRY1khJatcoHnlaB0jgBoCIOIKyV02SpNWdNVKSWjRyjafM/A1wPHA2cD3wQOBnA01KkqQhYI2UpHaNXLc9gMw8Gjh60HlIkjRsrJGS1J6RO/IkSZIkSYNg40mSJEmSGrDxJEmSJEkN2HiSJEmSpAZsPEmSJElSA5GZg86hLyKitQ86NtbeRQtvvf22VuLOW2ONVuJKGi6Z6eComlSbNXLOnLG2QnPnkjtbiTt3rL2c2xTR3j7xzPHWYkuDMp0a6ZEnSZIkSWrAxpMkSZIkNWDjSZIkSZIasPEkSZIkSQ3YeJIkSZKkBmw8SZIkSVIDNp4kSZIkqQEbT5IkSZLUgI0nSZIkSWpg0sZTRBwREd/oeHxpRJze8fiqiNgtIvaOiPMi4ub6d++OaX4cEe+KiJ9HxOKI+EZEbBwRX4iIW+r09+6Y/iM17i0RcX5E7NPx2tsj4isR8dmIWBQRF0fE7r1YGJIkNWV9lKTVT5MjT2cB+0TEnIjYElgT2AsgIrYH1gX+BHwLOAHYGPgg8K2I2LgjztOBZwNbATsAZwOnAvcAfgu8rWPa84Dd6mtfBE6PiLU6Xn8C8GVgQ+DrwInNP7IkST1hfZSk1cykjafMvBxYRNlYPxI4E7gmInYG9gV+AjweuDQzP5eZSzPzS8DvgEM7Qp2amZdl5s3Ad4DLMvP7mbkUOB14cMd7fj4zb6qxjgfmAfftiPXTzPx2Zi4DPgc8aEW5R8SREbEgIhY0XB6SJDUyyvURrJGSNB1zG053FrAfsGO9v5BSGPaqj7cEruya50rKXrQJ13fcv20Fj9edeBARrwVeUOMmsD6wScf013XcvxVYKyLm1kJzl8w8GTi5xszJP6YkSVMykvURrJGSNB1NLxgxURz2qffPohSHfev9a4Btu+a5F/DnqSZU+2+/Hjgc2CgzNwRuBmKqsSRJapn1UZJWI1NpPO0PzM/MqyldEQ6m9N/+JfBtYKeIeGZEzI2IpwH3B745jZzWA5YCNwBzI+IYyp41SZKGjfVRklYjjRpPmXkJsJhSFMjMW4DLgZ9l5rLMvAk4BDgKuImyZ+yQzLxxGjmdCXwXuITSteF24KppxJEkqVXWR0lavUTm6tHNuc3+3GNjTU8dm7pbb7+tlbjz1lijlbiShktm2qVLk2qzRs6ZM9ZWaO5ccmcrceeOtZdzmyLaG74zc7y12NKgTKdGOkiuJEmSJDVg40mSJEmSGrDxJEmSJEkN2HiSJEmSpAa8YMRqqs3vPcLz06Vh4QUj1EREZFsXP1q2bFkrcYt2atm6627USlyAxYsXtha7reUB7V2M4n7326uVuAC33HJTa7Gvvvp3rcVu0xvec1IrcY8/5pWtxAUYa+miM3cuuZ3x8XEvGCFJkiRJbbDxJEmSJEkN2HiSJEmSpAZsPEmSJElSAzaeJEmSJKkBG0+SJEmS1ICNJ0mSJElqYOgaTxGxOCK2X8XrV0TEgf3MSZKkYWCNlKTBamdEvBnIzHUn7kfEp4GrM/Mtg8tIkqThYI2UpMEauiNPkiRJkjSM+tZ4iogjIuIbHY8vjYjTOx5fFRG7RURGxI4RcSTwLOD1tZvCNzrC7RYRv4qImyPitIhYq1+fQ5KkXrNGStJo6OeRp7OAfSJiTkRsCawJ7AVQ+2+vC/xqYuLMPBn4AvD+zFw3Mw/tiHU4cDCwHbAr8Ly+fAJJktphjZSkEdC3c54y8/KIWATsBuwEnEnZO7YzpUD8JDPHI6JJuBMy8xqAurdttxVNVPfMHdmL/CVJaos1UpJGQ78vGHEWsB+wY72/ENiXUhjOmkKc6zru3wpsuaKJ6p65kwEiIqeeriRJfWONlKQh1+8LRkwUhn3q/bMohWFfVlwY3JhLklYX1khJGnKDaDztD8zPzKuBn1D6ZW8M/HIF018PrHQ8C0mSZhFrpCQNub42njLzEmAxpSCQmbcAlwM/y8xlK5jlk8D9I2JhRHytf5lKktRf1khJGn59HyQ3M+/Z9Xj3rsfRcf9Suk50zcx7dz1+e8+TlCRpAKyRkjTcHCRXkiRJkhqw8SRJkiRJDdh4kiRJkqQGbDxJkiRJUgORuXoME+EAgP0Uk08yTYtvv7W12OuuNb+12Lq7iPb228yZ007sZcuWthK3bZ0XGJBWJiKyzd9lWzLHW4k7d+6arcQF+MS3v9Na7BccdGBrsUdxWLE21+nNNrtXa7H/9rfrJp9omu5xjxWOmT1j1113eStxi7bKWE6rRo7ellKSJEmSBsDGkyRJkiQ1YONJkiRJkhqw8SRJkiRJDdh4kiRJkqQGbDxJkiRJUgM2niRJkiSpARtPkiRJktSAjSdJkiRJasDGkyRJkiQ1MJKNp4h4Y0RcFhGLIuI3EXHYoHOSJGkYWCMlqT1zB53ANF0G7ANcBzwV+HxE7JiZ13ZOFBFHAkcOID9JkgbFGilJLRnJI0+ZeXpmXpOZ45l5GnAp8PAVTHdyZu6embv3P0tJkvrPGilJ7RnJxlNEPCciLoiIhRGxENgF2GTQeUmSNGjWSElqz8h124uIbYFTgAOAszNzWURcAMRgM5MkabCskZLUrlE88rQOkMANABFxBGWvmiRJqztrpCS1aOQaT5n5G+B44GzgeuCBwM8GmpQkSUPAGilJ7Rq5bnsAmXk0cPSg85AkadhYIyWpPSN35EmSJEmSBsHGkyRJkiQ1YONJkiRJkhqw8SRJkiRJDURmDjqHvoiI1eODDoX2hhPJHG8tdoTDoGj2yUxXbE3KGtk/Ee3tt16ydElrseeOjbUWW3fX5jrS5v9Ro2g6NdIjT5IkSZLUgI0nSZIkSWrAxpMkSZIkNWDjSZIkSZIasPEkSZIkSQ3YeJIkSZKkBlprPEXEFRFxYFvxJUkaRdZHSRpdHnmSJEmSpAaGqvEUEXMHnYMkScPG+ihJw6H1xlNEPDwizo6IhRFxbUScGBFrdryeEfGyiLgUuLQ+9/o67TUR8cI6zY71tXkR8W8R8aeIuD4iPh4R89v+HJIk9ZL1UZJGTz+OPC0DXg1sAuwFHAC8tGuaJwF7APePiIOB1wAHAjsC+3VN+z5gJ2C3+vpWwDEt5S5JUlusj5I0YlpvPGXm+Zl5TmYuzcwrgE8A+3ZN9t7M/Gtm3gYcDpyamRdn5q3A2ycmiogAjgReXadfBLwHePqK3jsijoyIBRGxoPefTJKk6RtkfazzWCMlaYpa70MdETsBHwR2B9au73l+12RXddzfEliwktc2rTHOL3WivAUwtqL3zsyTgZNrHjm9TyBJUu8Nsj6CNVKSpqMf3fZOAn4H3Ccz1wfeTNmgd+rcaF8LbN3xeJuO+zcCtwEPyMwN622DzFy3hbwlSWqT9VGSRkw/Gk/rAbcAiyNiZ+Alk0z/FeCIiLhfRKwNvHXihcwcB04BPhQRmwFExFYR8Zh2UpckqTXWR0kaMf1oPL0WeCawiLJhP21VE2fmd4ATgB8BfwDOqS/dUf++YeL5iLgF+D5w396nLUlSq6yPkjRiInO4uzlHxP2AXwPzMnPpDOIM9wedVbp7nfRO2bnajo7zBKRZIzNdsWepXtXHGssa2ScR7e23XrJ0SWux546t9PQ59Vib60ib/0eNounUyKEaJHdCRBxWx6vYCDgO+MZMC4MkSaPO+ihJgzWUjSfgX4C/AJdRxsGYrB+4JEmrA+ujJA3Q0Hfb6xW7JPST3fakYWG3PTVhjewfu+1pMnbb659Z021PkiRJkoaNjSdJkiRJamDuoBPon2Du3DVaibx06Z2txG3TnDntHX7ffvvdWov95uNObi32/PnrtRJ3zTXXaiUuwM0339Ba7Da7X2666TaTTzRNN9zwp1bittmNYmysnU3x0ha78Gh2mTNnjPnz2xlP9+9/v7mVuG1ad92NWou9ePHfWou94QabtBa7rZqw2277txIX4Fe/Oqu12OPj7XV/e+az39ha7C989j2txN1ss21biQuwbFk7tWzhwun9D+WRJ0mSJElqwMaTJEmSJDVg40mSJEmSGrDxJEmSJEkN2HiSJEmSpAZsPEmSJElSA0PTeIqIT0fEuwadhyRJw8YaKUnDYWgaT5IkSZI0zHrSeIqIzXsRZxXxN42I9kbslCSpJdZISZo9pt14iogNI+IlEXEu8On6XEbEjh3T3NXNICL2i4irI+KoiPhLRFwbEUesJPZ6EfGjiDihFoTnA3+MiGMjYrvp5ixJUj9YIyVpdppS4yki5kTEQRHxJeBK4CDg3cATGobYAtgA2Ap4AfDRiNio6z02Bn4A/Cwz/zWL44CnA5sBC2rReHZErD2V/CVJaos1UpJmv8aNp4h4OXAF8D7gbGCHzDwsM8/IzCUNwywB3pGZSzLz28Bi4L4dr28JnAWcnplv6ZwxM8/JzJfUaU4CngFcHRH/sYqcj4yIBRGxALJhipIkTc2o18jM8YYpStLqbSpHnrYDNgIuAC4EbprG+92UmUs7Ht8KrNvx+PHAfODjKwuQmXcAv6p53AnssoppT87M3TNzd7A7uCSpNSNdIyO8fpQkNdF4a5mZRwE7AL8G/p3Sv/qdEXGfjsluBTq7CWwxxXxOAb4LfDsi1ul8ISI2joiX1/7jPwTGgP0zc88pvockST1ljZSk1cOUdjVl5l8y84OZuSvwFGBD4OyI+FSd5ALgmRExFhEHA/tOI6eXA78HvhER8wEi4gWU7hD7AscC22TmGzLzt9OIL0lSz1kjJWn2m/Zx+sw8PzNfQelfPdGF4JXAocBC4FnA16YRN4EjgauBMyJiLUr/8W0z86mZ+a3MXDbdvCVJaps1UpJmp7kzDZCZdwLn1vsLgAesZLofA1t3PXfvjvvP67g/DjynY9LfzDRPSZL6zRopSbOLZ4hKkiRJUgM2niRJkiSpARtPkiRJktSAjSdJkiRJasDGkyRJkiQ1EOWqp7Pf2NjcXGedDVqJvWjRX1uJq/7addf9Won7tJe+sJW4AMe87HmtxV62rL2rHY+NjbUWu628119/41biAtx226JW4i5deifj4+PRSnDNKnPmzMm5c9dsJfaSJXe0Erddbf5sRvX/rnaWyRnnL2glLsDhe+/TWuw77ri1tdht/RYBxua0U3/nr71+K3EBbrv1llbi3rnk9mnVSI88SZIkSVIDNp4kSZIkqQEbT5IkSZLUgI0nSZIkSWrAxpMkSZIkNWDjSZIkSZIaGKrGU0S8PSI+X+/fOyIyIuYOOi9JkgbJ+ihJw2GoGk+SJEmSNKxsPEmSJElSAz1rPEXEERHxjY7Hl0bE6R2Pr4qI3SLiI/X+LRFxfkQ0Gv45Ip4SEVdExC4RsVZEfD4iboqIhRFxXkRs3qvPIklSr1gfJWn26OWRp7OAfSJiTkRsCawJ7AUQEdsD6wK/As4DdgPuAXwROD0i1lpV4Ig4AjgOODAzfw08F9gA2AbYGHgxcFsPP4skSb1ifZSkWaJnjafMvBxYRNnwPxI4E7gmInYG9gV+kpnjmfn5zLwpM5dm5vHAPOC+qwj9KuB1wH6Z+Yf63BJKUdgxM5dl5vmZeUv3jBFxZEQsiIgFmdmrjypJUmPDWB/BGilJ09Hrc57OAvajFIezgB9TCsO+9TER8dqI+G1E3BwRCyl7yDZZRczXAR/NzKs7nvscpfh8OSKuiYj3R8Qa3TNm5smZuXtm7h4RM/90kiRNz1DVR7BGStJ0tNV42qfeP4uO4lD7b78eOBzYKDM3BG4GVrXVPgh4S0Q8ZeKJzFySmcdm5v2BvYFDgOf0+LNIktQr1kdJmgXaaDztD8yve8J+AhxM6ULwS2A9YClwAzA3Io4B1p8k5sU1xkcj4gkAEbF/RDwwIsaAWyjdFMZ7/FkkSeoV66MkzQI9HWAvMy+JiMWUokBm3hIRlwM3ZOayiDgT+C5wCfB34EPAVQ3iXhgRhwDfioglwIbAx4GtgcXAaZSuCpIkDR3royTNDj0fnTwz79n1ePeO+8uA59fbhPd3vP72jvtX0NFdITMXAJ2XW/1Sr3KWJKlt1kdJGn0OkitJkiRJDdh4kiRJkqQGbDxJkiRJUgM2niRJkiSpgVhdRhWPiIxop62Y6VVgZ4M5c8ZaiTs+vqyVuADz5q3dWuw77rittdjQ3nbnkmuvbSXuLttu10pcgDvvvKOlyElmOvqpJhURueohpWZi9fg/Y/YbvfXj0Y9+Xmuxv/e9T7cWu00nfvWbrcR95eFPaiUuwLJlbf0fNb0a6ZEnSZIkSWrAxpMkSZIkNWDjSZIkSZIasPEkSZIkSQ3YeJIkSZKkBmw8SZIkSVIDNp4kSZIkqQEbT5IkSZLUwMg2niLieRHx00HnIUnSsLFGSlI7RrbxJEmSJEn91LfGU0RcERFviojfRMTfIuLUiFgrIjaKiG9GxA31+W9GxNYd8z0vIi6PiEUR8ceIeFZE3A/4OLBXRCyOiIX9+hySJPWaNVKSRkO/jzw9C3gMsAOwE/CWmsOpwLbAvYDbgBMBImId4ATgsZm5HrA3cEFm/hZ4MXB2Zq6bmRuu6M0i4siIWBARC9r9WJIkzZg1UpKGXL8bTydm5lWZ+Vfg3cAzMvOmzPxqZt6amYvq8/t2zDMO7BIR8zPz2sy8uOmbZebJmbl7Zu7e248hSVLPWSMlacj1u/F0Vcf9K4EtI2LtiPhERFwZEbcA/wtsGBFjmfl34GmUPWjXRsS3ImLnPucsSVI/WCMlacj1u/G0Tcf9ewHXAEcB9wX2yMz1gUfW1wMgM8/MzEcD9wR+B5xSX8++ZCxJUn9YIyVpyPW78fSyiNg6Iu4BHA2cBqxH6cO9sD7/tomJI2LziHhi7dd9B7CY0kUB4Hpg64hYs6+fQJKkdlgjJWnI9bvx9EXgf4DLgcuAdwEfBuYDNwLnAN/tyu81lL1vf6X0835Jfe2HwMXAdRFxYz+SlySpRdZISRpyc/v8fudl5nu7nrsV2K/ruU/Uv9dy9xNj75KZdwKP72l2kiQNjjVSkoacg+RKkiRJUgM2niRJkiSpgb5128vMe/frvSRJGiXWSEkaDR55kiRJkqQGInP1GAoiIlaPDzrLRbTX3s8cn3yiITNnzthIxv6fC3/ZWuyn7ffYVuLecvMNrcQFuOPO21uKnGRmtBRcs0ipkW2tKpbf2WCttdZtJe7tty9uJS7AvDXntxb7Cz89q7XYzz+gnToGMG+tdVqJe8MNf2olbtumUyM98iRJkiRJDdh4kiRJkqQGbDxJkiRJUgM2niRJkiSpARtPkiRJktSAjSdJkiRJasDGkyRJkiQ1YONJkiRJkhqw8SRJkiRJDYxk4yki3hgRl0XEooj4TUQcNuicJEkaBtZISWrPSDaegMuAfYANgGOBz0fEPQebkiRJQ8EaKUktGcnGU2aenpnXZOZ4Zp4GXAo8vHu6iDgyIhZExIL+ZylJUv9ZIyWpPSPZeIqI50TEBRGxMCIWArsAm3RPl5knZ+bumbl7/7OUJKn/rJGS1J65g05gqiJiW+AU4ADg7MxcFhEXADHYzCRJGixrpCS1axSPPK0DJHADQEQcQdmrJknS6s4aKUktGrnGU2b+BjgeOBu4Hngg8LOBJiVJ0hCwRkpSu0au2x5AZh4NHD3oPCRJGjbWSElqz8gdeZIkSZKkQbDxJEmSJEkN2HiSJEmSpAZsPEmSJElSAzaeJEmSJKmBkbzanlZfe+75hNZi/+IX32gl7vj4slbith173ry1W4v9u0uvbC32DTf8qaXIbY4xmi3GlrS6eNtHTm0t9vte99LWYrfljjtvay32c/d/dGuxH/GI/9da7DPP/GQrcefMGWslLkBmOzUyc3xa83nkSZIkSZIasPEkSZIkSQ3YeJIkSZKkBmw8SZIkSVIDNp4kSZIkqQEbT5IkSZLUQGuNp4i4IiIObCu+JEmjyPooSaPLI0+SJEmS1MBQNZ4iwkF7JUnqYn2UpOHQeuMpIh4eEWdHxMKIuDYiToyINTtez4h4WURcClxan3t9nfaaiHhhnWbH+tq8iPi3iPhTRFwfER+PiPltfw5JknrJ+ihJo6cfR56WAa8GNgH2Ag4AXto1zZOAPYD7R8TBwGuAA4Edgf26pn0fsBOwW319K+CYlnKXJKkt1kdJGjGtN54y8/zMPCczl2bmFcAngH27JntvZv41M28DDgdOzcyLM/NW4O0TE0VEAEcCr67TLwLeAzx9Re8dEUdGxIKIWND7TyZJ0vQNsj7WeayRkjRFrfehjoidgA8CuwNr1/c8v2uyqzrubwksWMlrm9YY55c6Ud4CGFvRe2fmycDJNY+c3ieQJKn3BlkfwRopSdPRj257JwG/A+6TmesDb6Zs0Dt1brSvBbbueLxNx/0bgduAB2TmhvW2QWau20LekiS1yfooSSOmH42n9YBbgMURsTPwkkmm/wpwRETcLyLWBt468UJmjgOnAB+KiM0AImKriHhMO6lLktQa66MkjZh+NJ5eCzwTWETZsJ+2qokz8zvACcCPgD8A59SX7qh/3zDxfETcAnwfuG/v05YkqVXWR0kaMa2d85SZ9+54uHPXy8d0TNfdRYHMfC/wXoCIuB8wTumuQGbeTuna8ObeZixJUvusj5I0uoZqkNwJEXFYHa/i/7N332GWVGXix79vzzADOMAMWUBQQWAVBRUDugiKAVfQBRRR14CumCPBALqimHPAgGtYZY3L/mRRFNfEIoIwLBhAlyQsCCLgDgySZrrf3x/ntFyuE6q7b93Q/f08z3363rpV7z1Vt269fU6dOs5keXgAACAASURBVLUEeC9wSmauHHS5JEkaJPOjJA3WUFaegJcAfwQuo9wHY239wCVJmgvMj5I0QK0PVT4dmbnvoMsgSdKwMT9K0mAN65knSZIkSRoqVp4kSZIkqYGh7LbXlrGx1d5ofUYmJsZbiau/dtZZJ7cWe/v77tpK3Msuv6CVuG2bN6+9w8OCdRe0FrstixYtbi32Lbcsaylyrn0WqZo3r50cOT7ueBb98vbXvqi12BtssHErcVeOr2glLkBme8fA8fH2/vd71lH/0Frs0077XCtxN910m7XPNE3Lll3XStwVK+5Y+0yr4JknSZIkSWrAypMkSZIkNWDlSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDPa08RcROEXFBRCyPiFf3MrYkSaPMHClJo6/XN3I5CvhxZu7W47iSJI06c6Qkjbhed9vbDriwxzElSZoNzJGSNOJ6VnmKiB8BjwU+ERG31O4JH4iI/42I6yLi0xGxXp1374i4OiIOj4g/RsS1EXFoR6z1IuKDEXFlRNwUET/tWPaREfGziFgWEb+IiL17tQ6SJLXBHClJs0PPKk+Z+TjgDOCVmbkIeCmwI7AbsAOwNfDWjkW2BDaq018EHB8RS+p7HwAeCjwK2JjS1WEiIrYGvgMcV6cfAZwUEZutqkwRcVhELI2Ipb1aT0mSpsocKUmzQyuj7UVEAIcBr8vMP2XmcuBdwCEds60A3p6ZKzLzVOAWYKeIGANeCLwmM3+fmeOZ+bPMvAP4B+DUzDw1Mycy8z+BpcDfraocmXlCZu6embu3sZ6SJE2VOVKSRlevB4yYtBmwPnBeyREABDCvY54bM3Nlx+tbgUXApsC6wGWriLsd8IyI2L9j2jrAj3tUbkmS2maOlKQR1Vbl6QbgNuABmfn7aSx7O7A98Iuu964CvpyZL555ESVJGghzpCSNqFa67WXmBPBZ4MMRsTlARGwdEU9quOzngQ9FxFYRMS8i9oiIhcCJwP4R8aQ6fd16Ye02bayHJEm9Zo6UpNHVSuWpegNwKXB2RNwM/ADYqeGyRwC/As4F/gS8FxjLzKuApwFvBq6ntLIdSbvrIUlSr5kjJWkE9bTbXmbu3fH8dsoB/M2rmO8nwDZd0+7d8fw24LX10b3sz4G9elRkSZL6whwpSaPP1ihJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDbR1n6ehs846C9lss3u1Evuaay5tJa5WJVuLvHjJFi1FjrXPMk3rrbeotdiLFi1pLfaLn/zE1mK3tb2fdegRrcQF+Nzxb20l7sTEeCtxNfvMn7+ATTdtZ0TzP/zh8lbi6q+VkezbsWLFHa3EnZhor8xt7dMAN9441Vu0Nfeqvz+otdgLFqzbStyXH3NsK3EBPvyWI1uJOz6+cu0zrYJnniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJamD+oAvQpog4DDgMYN68Wb2qkiRNSWeOHBszR0pSE7P6zFNmnpCZu2fm7mNj8wZdHEmShsbdc+Ss/ndAknrGo6UkSZIkNWDlSZIkSZIamBWVp4j4bkS8edDlkCRpmJgfJam3ZsUVopn55EGXQZKkYWN+lKTemhVnniRJkiSpbVaeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDUQmTnoMvTFOusszI03vmcrsTfddJtW4gLcfPONrcS9+urfthK3iBZjt7e/LlmyZStxb7rp+lbiAmy00Watxd5ii3u3FvvSS/+7tdjz5y9oJe6SJVu0EhfgTzde00rcO1fczsTERJs/SM0SCxeun1tttUMrsZcvbyePtenPf765tdgLF67XWuxbbvm/1mKPjc1rJe7ExHgrcQHmz1untdi7PPAxrcVuM0e2tY9svKSd/7EB7rjj1lbi3vLnZYyPr5xyjvTMkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJaqBnlaeIaO0Okm3GliSpbeZISZodZlR5iojFEfGyiDgH+GKdtlVEnBQR10fE7yLi1R3zL4yIj0TENfXxkYhYWN/bNCK+HRHLIuJPEXFGREyW74sRcU5EvDQiFs+kzJIk9YM5UpJmnylXniJiLCKeGBFfBa4Engi8E3hqPZCfAvwC2BrYB3htRDypLn408EhgN2BX4OHAMfW9w4Grgc2ALYA3A1nfeyrwLuBJwJUR8ZWIeEJH4pAkaeDMkZI0u03pwBoRrwSuAN4DnAVsn5kHZObJmbkCeBiwWWa+PTPvzMzLgc8Ch9QQzwHenpl/zMzrgWOB59b3VgD3BLbLzBWZeUZmJkB9/a3MPADYHjgbeC9wRS3T6sp7WEQsjYilExPjU1lVSZKmZJRz5Pj4yt5uDEmapabaKnUfYAlwAaXl7Mau97cDtqrdCpZFxDJK69hkf+ytKC1xk66s0wDeD1wKfD8iLo+IN66mDDcCv6xlWFLLtEqZeUJm7p6Zu4+NzWu6jpIkTcfI5sh58+Y3XUdJmtOmVHnKzMMprVq/Bj4O/C4i3hER96uzXAX8LjMXdzw2yMy/q+9fQ0kek7at08jM5Zl5eGbel9IF4fURsc/kjBFxv4h4B/A74KPAr4D71jJJkjRQ5khJmv2m3B+6dif4UGY+CDgIWAycFRGfB84BlkfEGyJivYiYFxG7RMTD6uJfBY6JiM0iYlPgrcCJABGxX0TsEBEB3ASMAxP1vc9TukAsBg7MzF0z88O1W4MkSUPBHClJs9uMztNn5nnAeRFxOLBbZo5HxH7ABymtXwuB/+GuC16PAzakdCkA+GadBnA/4BOUi2H/D/hkZv64vvdp4KWZeedMyitJUr+YIyVp9ulJJ+d6wD6nPr8GeNZq5rsdeHV9dL/3YeDDq1nunF6UU5KkfjNHStLs4TCmkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDURmDroMfRER13P3O7evzabADS0Upa24oxp7FMs8qrFHscyjGntYyrxdZm7WUjk0i0wxRw7L/j0ssUexzG3GHsUyj2rsUSzzMMWeVo6cM5WnqYqIpZm5+6jEHdXYo1jmUY09imUe1dijWGapqVHdv/299yf2KJZ5VGOPYplHOfYku+1JkiRJUgNWniRJkiSpAStPq3fCiMUd1dijWOYZx46IaCv2AOIau39x244tNTGq+7e/9/7EnnHcNeTIUdwebcYexTKPcmzAa56kvoqIscycWMX0SH+MkqQ5zBypUWDlSeqTiJiXmeMRcT/gWcBNwLWZ+Y0BF02SpIEyR2pU2G1P6pOaFP4G+BmwLbA1cGxEfHiwJZMkabDMkRoV8wddAGkuqP235wOvAz6amcdFxALgAGDdgRZOkqQBMkdqlHjmSeqDLFYA84ALI2IMOAc4JzNfFhEPjIiHD7aUkiT1nzlSo8TKk9SSiJhX/24YEevVZDAP2A/4L+BXmfmcOvvhwCMHU1JJkvrLHKlR5YARUgs6Lnx9IPA54M2Z+YOI2Bo4GyAz71Xn/SLwIODhmblyUGWWJKkfzJEaZVaepJZExP2BnwIfzsx3RMT8zFxZRxL6D+BqYPIH+JTMXDGZUAZVZkmS+sEcqVFl5Unqoc57VETECcDVmfn22h3hVcAdlH7cvwLuWxe7JDMnJhPHQAouSVLLzJGaDRxtT+qRehO/iYjYBbgZ+DXwjIj4BfA24HrKb25/4HmZ+T8dy46ZFCRJs5U5UrOFA0ZIPVC7EmREbAWcC/wd8EPgN8BBwLcy84nAS4EF9fEXq7qjuiRJs4E5UrOJ3fakGagtaVmf7wA8BtgiM9/dMc/CzLyjPv88sB3wBJOBJGk2M0dqNvLMk+a02s96ta/XsNw/RsSBeffWh8OAfwYeFhHzOmKNR8SuEfEt4KHAvrXrgr+/IRURiwZdBkkapOnmxzqvOXIWm+s50h1Tc1btRjAREVtGxIMiYssmLV0RsQWwDLiuc3pmHgW8H3gisOtkrI5+2j8EHlpHDJpvq9pwqt1Kjo+I7Uzekuai6ebHuqw5chYzR9ptT3PUZFeCeo+J0yn9rncBDs7M09aw3B7AK4BXZeb/RcR2wBbAJpn53TrPJ4FDgMdn5n+vIsacGmq1q9tG5JAfdOo+sVlm/mjQZZmujnuoDP32ljRcppsf67LmyCkYtfwI5kiw8jRtk8NtRsSCzLyzThuJHX+u60gMmwNvAS4GTgBeB7wDeGZm/vtqln0QsDwzfxcRuwLfqsvvSGlle1Z97zPAAcBTM/Ps9tdqOHUOLds5RO0oiIgAjgPOysxvD7o8UxURi4H9gJ9m5hUDLo7mGHPkaJpJfqzLmyMbGuX8CHM7R1p5mqaIWAhsDrwWuCgzPzfgImkK6o/m88A9KTff+1Od/lrgfZQE8f/WsPxGwKnAiZn5qYhYAJwMbALsUVs0vgFskJlPbnl1hlpEzANOBJYA3wR+nJmXD7ZUq9bVCrgT8GZgfeAzmfmDgRauoYjYC9gBeCWwK/C6zPzoYEulucYcObpmmh/rvObIBkYpP4I5ctKc7Ks4UxHxbOCtwDcorTGPGmyJNA0bARcCDwAOnJyYmR8BjgBOqj+w1VmXcsA4vS53Z00A6wIfqtMOBp7SSumHXG2RmnRy/fsT4NnAS6Pc52Oo1NP4f2lNynKPkfcA1wIvj4gnDKxwDUTE3hHxAeCzlFbeSyn/vPhPq/rKHDnyZpofwRy5WqOYH8Ec2cmb5DZUWwdeRjmYHAAcS2kl+DXwrjqPXRJ6pLvP80y3bUf/1oWZeUdmXhkR7wJWUA5Wf87MrwJk5sci4g/AmR3Ld59Sv4FywNg3Ii7LOswq8K/AxpMzTY4YNGqn42eiqyvCIsop8ffU1xcALwZeEBFfyMwLB1jUv+jYP8aAwymto2cCZwHvBd5I2U8mMvOHAyzqX6kXZ/8LcDvlxpMHZuavI+JVwKbA7XNtH1T/mSP7Z9jyY41hjmxgFPMjmCO7eeapgYjYkNJv9/HAFcAjMvNTwB8pp1pvBTAp9EZNBOMRsU5EvLMe0GeSGMZqvF2Az0fENyLibZR7SbwTOAl4dUQ8c3KZzPwGsEFErFNfT446tEtE7FgT17eBfYCn1S4KAA+nq1FiriQF+Mt3tzLKMLT/DnwNOCIiHgaQmd+jDFW7LfDKiPibARb3LzqSwrnAw4DFwNOAfwNuobRMXQm8KSL2HlQ5V2Md4DTKzSVfUpPCw4A3UbqArJxL+6D6zxzZP8OSH+txftN63DRHNjCq+RHMkd0889RAZt4cEe/OzJ/FXRfB7ky5ePLIzLxubTHUTFeL2j8D8zparKalfl/bA/8JfJjSGvaQ+vqBwJeBceBdEXFjZv4gIh4FfAx4FXBWTSz/AdwE3BERF2XmCyNiU+AFwHER8Vvg3sAzmYO6vrvPAetR+nI/FHheRFyVmX/IzO9G6f9+EHDjgIq7Ki8GrqtdSYiI7wIJ3JyZF0TEOPAcyqn+gatdP7bJzKso+zVRzAf2Bz6fmT+xtV9tM0f2x7Dkx1oWc+QUzIL8CObIu2Smj9U8KGfmXtw1bX79+3TgeGCdQZdztj2AoIzu8z7KxaRQBzeZRqx16t/XAF/omP4LygWOAPMoFzY/l5KMJuf5NqW/9t9SksJhlL7ge1BaX75V59uhLntQx/4xbzrl7dwGHc9nFGtA391xwII6bWfKaEsfB7bsmPcegy5vV9lfA7yvPv8y8EtKi9VCyt3umVynQT/qselnlH+g1uvcZyjXGZwFHDLocvqY3Q9z5EC2+VDkx/qeOXJ6393I5ceO/cQcmWm3vdWp/bd/DhwY5T4FwN1u5nYEcENmrhhE+Wa5JcATKKM0bVmnNdpXI2K/iHhhRPxD7Vs8+f0sAK6q85xPGf3pJVFu9nYIcGNmfpnSP/sYgMzcj9Jv+/11+dMy86bMPIvSp3/LiHh+Zl6amV/OzJOynpLPGd6jIjOzo8vgeG29GwVLKDdAPJz63WXmbynbax/g3VGGwCUz/zyoQtaWp243AgdExEmUZP+Iuv8cBTwzItbLOuTyINWuE+cAlwAvy8zb4G5dol5AOTZ9bTAl1FxgjhyYgeXHmov2j4ijwRw5DSORH8EcuVaDrh0O64NyyvqLHa83pdSuxyinsv+l470pt/qwmpaS6cRqGntYH6sqL6Wl65fAD5uuF3AocD3lQuVfA8d1vPd0Sr/ci4GPdkz/6uRrSqvQc2uMIzrm+QJwJ6WrwWSr2Xzg68BRbW0T4LuUlp57US5y3GXQ39UUvrtfAf/ZNX1XYCmwxYDKuiVwv65t/FTgWZSktgD4KOWi0t3qPK+gXLfxwEFv645yPwn4QcfrwyndZ15OaQXcFHhAfW9s0OX1MTsf5si+beehyI+T294c2ZPvbujyYy2DObLpZwx6JYfxQTnt/G/AI+vrTwDfB84A/rZO26b+nXZSqEnmVcDzgYN6vA4BbN+LRNPytu7eFi8Anl6nbU65s/kPuudfRZx7AN8DnlxffxB4VNc87wAmKBes7lwP+BdMHuzrPOsBB1Muej6qY/rXKKd59wQW1WnfAY5tabssrAetq2uiOmxyOw36O5vud1enD+SUPuU0/SeA/wdsV6f9qn6n51NaqA6u+8a7KBe+nkTperLboLd117o8mNJd5r2U0czOB94O3AbsN+jy+Zj9D3Nk37bzUOXHOp85soXvrk4fWJc3c+TUHg4YsWpJaYU5NiJWUoZkfCGlxv18ytCSV8P0Rg/Ku0YtOQ/4E3AR8IyIeCzwqunEXIX3UE4PPz8iftWjmD23hm3xuMx8eR215T8j4tzMfFiu/lT/GLBNfQDsDvwqIq6mtJKsB3yK0jXhE8DllGFYH5a1G0Etz20R8XPgk8DREXFzZn46Mw+JiG9SLvS8KiIup4yI87QebxJqOe6IiPOArYDlwMo6faIXXR56YarfXV1sIF14MvPWiPge5Z4lb6/7xRmZ+XKA2lXzdcBbMvPNEfF1yoXTE5l5wyDK3C3KiGYrKS3O36fcwPJ/gGdn5oradWqLARZRc4c5sg+GKT/WsswzRzYu38jkRzBHTtmga4jD8qC0Qu1JqbEuofwgDwCewV0tCEdSbq414wtgKacRv9bx+mTgrBnEW9Xp4f+gdK3Yld50dRjreN6z1rrVbIuzJz+Pcir559TWkDXEeRXwe0qXkWdTkvsNlBayiyh3TN8A2LB+35MXD87reP5ASkvWhyhdG64B3tDxGZ+mHOAO6NgvetYFhL9u5XswpX/7lZRRqwb+W2nju2u5jJ0XFu9L+QfvF8Dbuub7EOXGkEPRctlRrjFKC9+PKMOtHtO9v1CS2g3ADoMur4/Z+TBHNvqMnufIYciPnetjjuz/d9eHcpojp/qZg17pYXjUDX8W8FNKK8GvgX06Nz7lBmDLgAf16DOPBd5fn/9r3VHXqT+mR0wzZgCLu6adQrlz9YzKzV2n4ceAdXu8/de0LSa7haz1xwosopyavRLYidIP+uHA/YEnU7qU7Nkx/47ARh2v16fcbfpN9fVWwPNqvNd1zHdcx8FvxgeRVZRje8r9CPYENq/T/oHSGvj6+vpLwN/38nsY5HfXchk7/6F5HqVf/ndrWXfu2u4/7vwuBv2ov+n/pIwY9AjKPz2XURNy3XfeQfkH5iGDLq+P2fkwRzaK3UqOHFR+rMuYI4fgu+tDOc2RU/3cQa/4MDyAz1AvfK07x+GUvpGPrUnhCEq/zgdPM3738J5BucjxeMop7r/0K6bctOtjwMJpfM4/UU67b9Y1/Uzgvyj3E5hO//Oj6o9pMaUf6bQPSNPcFqtNRN0HHkrf7ncB/ws8tuu9M+qBIYCtgesoCX/DjnlOAl7TFe8DlFa6o9e0LtPYFp3leBMl6e5MaR35GeWflE9SL+CkXLR5c90XL6XPQwD3+rubwud2tzQ23oe5K4EHpZX50/X1XpR7bHxj8oBaf/e/YbgSw30p/7Ru0jFtJ0pf9OdQEvHTgfsMuqw+Zu8Dc+Ta4vYkR/b6GMs08mPH55ojB/jdTeFzp50fO/cRzJFT+9xBr/gwPCgjwrxscgeqf98OfLU+357aujGN2H855U0ZAWRyJ9yGUju+lTq2P/BKyqgl928YO7pe35cyOs7dRmypB5Q7ga/QMOEA9+x4/kHgh8DvgJNmsJ17ui064t0TeDQl8a1fp72TkiAe1/EdX8jdB4c4uB5gD6d0Qxmj/JPwz/UHFx3znU65QPov3Rl6uP9NluP1db97eZ3+NEpXii8BO9ZpO1JaVibXfX4vyzKI/bjJPk75B20pHSMBTTHOG+v+33kfr8dTLia9qe4fP2HIzt5Q/nE4m/qPTt3+61DusXHMIMvmY+48MEeuKnZPc2SvtwMzzI91ujlywPtwk/2bGebHGsMcOZXPHfSKD3ijTx5ITgDe1fXePwCnzjD+X05bA7+ljDt/B+VU7gaU0+b/TekH+z3KRW6NWu64eyWg83T2FpSWgvO463T2YcDLaNivlnLx6P9y924Zv6K09uxPVx/oQWyLjoPGgyijwPyU0nf45x3r/Q5KMjua0jI6eUPAzoPDgZQ+30fU1/epB7aPUYfepPThP7rjM3vVl72zn/FBtRy/oOPGbZQD8RcpIx/t1rX8tFv1BvndTXP7vAX492nGeQzlH5tbVrENH0XpI/1ZYKtefK892jcmb+o3Vn/P36aMcDa5D34ceHsv90cfProfmCNXF7unObLX24EZ5seuY685ckD78BS2zbTzY13eHDnVzx/0BhjQRh+jtJw8sb7+O0rN+oXAxnXaK+qXMaW7PFNbrToPIvXHfUJ9vT+l9v6B+kVvWA9wD6Hh+P7cdWCe3GnOqgePg+r0zev0WyjDh65gii0S1FY17jrNvD/lYPt9Ste3DbvL0+9tQeki8XPqHe4pN207kZIQllCSwPuBf+fuLTTB3fv4Hkzps/3G+nrHGvdcyhCX53FXYunlQBnd5dif0i/33XT0y6eMCHUyZZSbGZWBMirWWhN7P/bjKZT5S5T+15O/1zX2EV/V/khJDqdSWqPu0/Xe43td5hms61jdh79HufD6aXWfPZ+7hl79J8rNCncedHl9zM4H5sgmnzGjHNn2dmCa+bGjLObI1cTpxz48hfJOKT+uYX80R06lHIPeEAPa8L+oP7Ql3NVq8GxKq8ZPKOPc/4Epjl1POUX76frjm4z7HcrINP/YMd8+lP7F76aeap7mepxHOZW6Tz0I/hfwoo55DqMkuCnvQB3l/yG1D2x9/X7KxXnPrq8PpIxi0t09opVt0XFQW59yn4ez6GjFAdYFvsVdF7R2tkREx/KTowotqK+fxt2Tw2LgAZQLUtsYMWh15di/luNwYEnH/A+nwUFxLfvLTymj0ZzPGvr/9nM/blj29wF/prSurbFLDXf/p+lAyo0hd+aukcK+SvmncLu2yjuD9Yz6G/4S5cLX11NGx9qb0tJ9LKXLzAkM2c0gfcyeB+bIxvHr3ynnyLa2AzPMj10xzJED3ocblrtxfuzarubImZRl0BtjABv/ncCXOl4/CXgcpSVqO8pFn4cC951G7McA+3ZN25Nyev/rXdMfSzl9+080vDEad+9u9o/ANzpef63uRKfRkRymsQ7dfZ7/lnKx5vEd095bDzCnUlojH9CPbcHduyL8oJbtR8CBXe9/HHhv97IdB7oHUpLq9yhdLfar059GaZU7Athgddu+B/tg03K8no6LIDuXncZnvp/yT8RudV+5ntV0HWh7P266j3dNPxq4uG6bNX4OJSlcQPnn7yJKYvsYpaV1b0rL2teBbXv1nfZov3gwHXedp/Q//znln6DO335frnPzMTcfmCPX9hkzzpFtbAdmmB/rX3PkWnJk2/tw0/27a3rj/Di5jTBHzqwsg94Yfdzom9S/x1AugNuCUsP+VT3Q/JgenZak/KP+VuCh9fUjKTdx+1DXfI9h7fdmWHcyZv07j9IH9VHcdVHiP1NOn29Laem4mHpB5RTL3dl394PAa+uP/4GUPs6f7Jj3QODFrKXFrpfbYvJ7rGV7RX39dkqLzyOo/drrD+q41Sy/dT3IvaGW7VjK8Lp71PcPoNw08JC1lWWG+0jfylG3139Qu9vUaZ+l9M/ftV/fXYNydraIfYrSuvdl7rrb+VsoN7zbn64WNrhb147jgG/X5+tT7kL/LepIUJS+85+h44LvQT7qPh11+55bp32Ocmya7Arzj9REhtc4+WjhgTmySbl7miN7fYxlhvmxvm+OzLXnyF5/dw3KOO382Lnv1ufmyJmWadAbpY8b/wf1x7YTd3VJOIkyKscj6utpJwbuXuvdpe6I/zL5w6McyJdTx/xvGPNBlJuVTfatDsqIKpP3DZhPOWifRb2Qj9J68A6meWFf/YxzKC1W36L0pX1VXac/Ah/v97bgrqS4EaW15Cw6Wj3rOv+KktwnL8ic3718x2d3tkb+F3cNwXuP+vex9LAVbZDloLQUf46SZJ7a9d6ngAnqRb9t7cdT/I6D0mXi3yjXDfyEMorTPvX9d1BaeZ+8mn13R8qwwcd1TF+HcjH497krAS3q9fc7g3X/PvD3lH/6zqWMznR2x/tHUobl3XTQZfUxex+YI5t+5oxyZK+3AzPMj50xOj7fHJl/nSPb2Ien+B1POz92LG+O7EWZBr1R+rThn0FpVZhsodqAMnznZCvSSyk3/ZvuUKudF1nuRmndegblbsxf7vhhPbL+EN/ZIOamwLXAa+vrsbpzf65rvkdTDpgPoZxKPx+413TKX58vBD7YUYbnUpLEayj9m8fpOuXf5rboOGhsSrnvwBGUg9wzuuZ7EqU7yQs6yjCvY/nNKDcDfATlH4PtKUl28kZqm1Na/EQDEAAAIABJREFUYzpHZeplN4S+l4NyA8ON6r7+EUrC/9uueT7CXUO89nw/bljOzmFeHwZ8p+v9zwBndrw+Gth+FXFeT7lA963AFV3vbVt/G8PWDeEZlH9KJ2+wuR/ln5tP1X3jTZTuI1O6tsSHj6k8MEc2Kn99Pu0c2evtwAzzY1cMc+QacmQb+3DDMvYkP9b3zJE9esxnbtiTco+AiYiYn5nLgeURcc+IOAp4PvD4zPzjVANHRGTmyoiYvAP77ZQf40+BBP4PODIi3puZZ0fEIyk3cWviXOBHERGU1qOHARtFxCaZeSNAZp4ZEbdR+jFvQ+nffNUUyj/WUf4PARsDW0bEksy8ISJOpbTeHUK5IG8XSnJofVtExLzMHI+ILSh9hu9R1/OtwCci4s+ZeWrdDqd1L1unZ13+bZR/CL5IORD/gHL695C6yPspF0cvn4yRmatcz6nqWo/Wy1G3/w8pSf6+lAtrT6ZcVPruiHhDZv6sxn5tXabN/XhNZX0ucFJE3E7ZNjcCj4qIzSd/j5n5koi4IiIOzcwvZOY7VxPuWsrF3/sAu0bEGZS+6X+mjMa0knL9wTDZk9KKdmd9fSolEbyJciHwSsq9WH41mOJpjjBHrr78PcmRw5Yf67J9zU0N16X1ckw1R9bvLkc8P4I5smfG+vlhgxAR+1H6Hn8qM++sB695EXEwpU/3PGCvzPzv6cTPWhWmDJ14aWbuRWnh+VvKAfRkynCox0XEAzLznMz8bYPQE5Qf5KspO84SSr/tZcDzIuIeHWXYg5LcHpqZ5zUtez1gTdTE83NK68rGlJafZ0fEopqATqacnt4D+ENmXtz2tqgHq/GIeBCl7/EDuGtkmLMpFzV/OiL2nZy/K8RE1/J/Q+nXuy9wG3A5cFFEPD8iTgR2pQxjO7GKWNO2ivVotRx1mZ9RDi5Pply/8DbKDRJ/TDnN/5mIeETnci3ux2sq67aUf0beR7k/xr0z86OUpPaCzn2c0iJ4Tceyf3XsysyvAmdSDqpvpiS430bEf1Ja416SmUOTGLqPTQCZOUFpUf37zDwIeK4VJ7XJHLl6vcyRw5Qf67J9zU2rMwo5ctTyY13eHNmiWXvmabIlgzIE46cy85KI2JVSi30+ZaSWt1Bu/HbnGkJNxXvq36Mop1pfGxGLKKdDd6Uc6BvJzD9FxDspLWs3Am/IzKsjYhPK6fmVEfG5zLy1zn/pVAs7eQCl9CX9eWa+EiAi3koZ1z8j4l9qWb4OfGUKP6wZbYvayrMppZXhA8DxwNMpd4J/OmU0nA8C/xERj8nMsxssfzDlh3gJ5SB5M2V/uBJ4weQ/Db1qTRtQOQ4ALs7M5wFExG6UYV6fSzntfxblBn7XrSFGz/bjNcnM/42IfSgH82spF3xSX+8JbB0RXwN2pySpN3YsO5k43wL8NjO/Ud/6GvBy4NrMfEJEPKGW97qcQmtzmxocm66JiCvqP2ArBllWzV7myEaf0VaOHGh+XEMMc+Tac+TQ58e6vDmyTTkEfRrbelD6Af83pc/qYZSa+QeoI9H06DOCMlrJBZQD14co/XQn+44fSelvPOWL7yg3tfsCpVXjc5QfZlBGnzmJ8uNdf4bl343SgnceHTf9o7TE/Bulj2yjz+j1tqCMtvNDOm7CSLlZ44WUcfz3pCSK1Q3fuarlJ/vLfoaumyKuLk4P9pG+lYMyKs3kxaOfAX5Zn3+Scjr+pazippZt7sdrKe+9KBd8X1j/blL3v2dRWvnOpdwz5SGrWHZzSiJbRkm4z6zTzwA+0sZ32cP1bv3Y5MPH2h792A/bPLYwQjmy19thNXmlcX5cQwxzZFeObHMfXktZp50f6/LmyLbKN+gCtLjhxyj9WCco49V/kq4RSJjBDdVW8XnPprR+/b5j2iuAq5jG/TC6Yt+LMkrOFymn5yeH7fxXOm4SN4P4j6C06BxATQLcdffxE6f6Gb3aFsCWwJ+A53R+X5SLM8+jXDS69WR5p7j8+ZSb123dh32xr+Wo+8fmlPuZTG6fN9SD5xo/p839eC2fuxOla8Lxk/sbZeSix1ET1BqW3bkmiNMpI18dS7l3RWt3F5/huvb12OTDx6oe5sgpxe9ZjhyW/Ngghjmyj/vwWso77fxY5zVH9rqMg95ILX8B2wLvovRP3qDlz1qfctr0Ekrt+JP1B7XKG5FOI/72lNPan6MMzxp03RxuhvH3otz74sCu5DDloR97uS0op5f/m3pzvDrteErLzznAU6a5/AebLN/D7dvXclBGEVpG6dv8mrr9txn0fryWz94F+A2l1fSTlK4KjUb+ARYA61IumP4ppRvClv34bqe5rn07NvnwsbqHOXJK8XuSI4cpP64lhjmyz/vwWj572vmxLm+O7OFjcmjIWa9elNjqykbEfMowlU+k3Nzt9My8pIfxtwe+RDld/NrsXT/0yfh7UX6UxwHfyszbZhCrJ9siIjYAXgm8jtL3eXNKQnxQRHwBWAQcvLrvdqbL98ogyhERL6S0it0BvDozlzZcrtX9eC2ffX9KEl1MGQ74/IbL/eX3HRH3AW7LzD+0V9Jmal/+DTLzd2uYp/Vjk7Q25shG8XuSI4clP/YqRi+MSo4cxfxYlx26HNkkP9b5hi5HzpnK02xRd/qV2dKFfRHxeMop8sdlGa524OqoMY+ktPz9GTghM2+PiE9QDgBHtrl8rwyiHDUhRWbOeNjUfqnJKTJzSheCDtsBNiIWAt8Bnp+Zvx90eaS5YK7lyF7kFXPk6OTI6ebHuuzQ5MhRz49WnvRXImL9rCMUDaN6gD2ScjHwYzLzwn4u3yvDUg61Z9h/S5Kmbph/173IK8OSm4alHGrHMP+O1mbWDlWu6RvmnTki1gX2p1wo+YRpVJxmtHyvDEs51K5h/i1Jmp5h/V33Iq8MS24alnKoPcP6O2rCM08aORGxPjB/uqfZZ7p8rwxLOSRJs0Mv8sqw5KZhKYfUzcqTJEmSJDUwNugCSJIkSdIosPIkSZIkSQ1YeVqNiDhslOKOauxRLPOoxh7FMo9q7FEss9TUqO7f/t77E3sUyzyqsUexzKMce5KVp9Vra+O3+aWOYuxRLPOoxh7FMo9q7FEss9TUqO7f/t77E3sUyzyqsUexzKMcG7DyJEmSJEmNzJnR9iKitRXdYIONG8975513sGDBwsbzb7zFJo3nXX7zTWyw4UaN5r3y0ksax5U0ujIzBl0GDb82c+RDH/rQxvNef/31bLbZZo3nP++886ZTJEkCppcjrTz1wOMe9w9theaQI57XStzD/u6JrcQFGBub11rsiYnx1mJLs5GVJzXRZo5s8/+MCHdvSdM3nRxptz1JkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpAStPkiRJktTASFaeIuKNEXFZRCyPiIsi4oBBl0mSpGFgjpSk9oxk5Qm4DNgT2Ag4FjgxIu452CJJkjQUzJGS1JKRrDxl5jcz85rMnMjMrwOXAA/vni8iDouIpRGxtP+llCSp/8yRktSekaw8RcTzIuKCiFgWEcuAXYBNu+fLzBMyc/fM3L3/pZQkqf/MkZLUnvmDLsBURcR2wGeBfYCzMnM8Ii4AYrAlkyRpsMyRktSuUTzzdA8ggesBIuJQSquaJElznTlSklo0cpWnzLwI+CBwFnAd8EDgzIEWSpKkIWCOlKR2jVy3PYDMPBo4etDlkCRp2JgjJak9I3fmSZIkSZIGwcqTJEmSJDVg5UmSJEmSGrDyJEmSJEkNWHmSJEmSpAYiMwddhr6IiNZWdPHiLdoKzYWX/6aVuFtvvHErcdvX5n0e58ZvQXNLZnpzVK1Vmzlyk022ais0S39zQStx77P55q3EbZ85UpqK6eRIzzxJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNrrTxFxKERcUrH60si4psdr6+KiN0i4lERcW5E3FT/Pqpjnp9ExHER8bOIuCUiTomITSLiXyPi5jr/vTvm/2iNe3NEnBcRe3a897aI+EZEfCkilkfEhRGxey82hiRJTZkfJWnuaXLm6XRgz4gYi4itgAXAHgARcV9gEfC/wHeAjwGbAB8CvhMRm3TEOQR4LrA1sD1wFvAFYGPgN8A/dcx7LrBbfe8rwDcjYt2O958KfA1YDPwH8IlVFTwiDouIpRGxtMF6SpI0FSObH2sZzZGSNEVrrTxl5uXAcsrB+jHAacA1EbEzsBdwBvAU4JLM/HJmrszMrwK/BfbvCPWFzLwsM28Cvgtclpk/yMyVwDeBB3d85omZeWON9UFgIbBTR6yfZuapmTkOfBnYdTVlPyEzd89MW94kST01yvmxxjJHStIUzW843+nA3sAO9fkySmLYo77eCriya5krKa1ok67reH7bKl4vmnwREUcAL6pxE9gQ2LRj/j90PL8VWDci5tdEI0lSv5gfJWkOaTpgxGRy2LM+P52SHPaqz68BtutaZlvg91MtUO2/fRRwMLAkMxcDNwEx1ViSJLXM/ChJc8hUKk+PBdbLzKspXRH2pfTfPh84FdgxIp4dEfMj4pnA/YFvT6NMGwArgeuB+RHxVkrLmiRJw8b8KElzSKPKU2ZeDNxCSQpk5s3A5cCZmTmemTcC+wGHAzdSWsb2y8wbplGm04DvARdTujbcDlw1jTiSJLXK/ChJc0tk5qDL0BcR0dqKLl68RVuhufDy37QSd+uNN24lbvva7J0yN34Lmlsy0y5dWqs2c+Qmm2zVVmiW/uaCVuLeZ/PNW4nbPnOkNBXTyZHeJFeSJEmSGrDyJEmSJEkNWHmSJEmSpAasPEmSJElSA3NmwIixsXm5YMG6gy7GlN1xx22txL3PfR7YSlyA577qNa3FfueRL2kt9vh4O/eQ3HTTbVqJC3DDDVO+VcwUjOaxYaONNmsl7k03TWdwtGYi2rnIO3PCASPUSETk2Ni8VmJPTEy0Erdo5zj1/i9+o5W4AMcc9vzWYt9xZzv/MwBEtNPevtVW27cSF+C667rvT907K1fe2VrsNm277f1biXvVVb9tJS7AvHnzW4m7cuUKMiccMEKSJEmS2mDlSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDVp4kSZIkqYGhqzxFxC0Rcd81vH9FRDy+n2WSJGkYmCMlabDaGTh9BjJz0eTziPgicHVmHjO4EkmSNBzMkZI0WEN35kmSJEmShlHfKk8RcWhEnNLx+pKI+GbH66siYreIyIjYISIOA54DHFW7KZzSEW63iPhlRNwUEV+PiHX7tR6SJPWaOVKSRkM/zzydDuwZEWMRsRWwANgDoPbfXgT8cnLmzDwB+FfgfZm5KDP374h1MLAvcB/gQcAL+rIGkiS1wxwpSSOgb9c8ZeblEbEc2A3YETiN0jq2MyVBnJGZExHRJNzHMvMagNrattuqZqotc4fVVzNdBUmSWjH4HClJaqLfA0acDuwN7FCfLwP2oiSG06cQ5w8dz28FtlrVTLVl7gSAsbF5OfXiSpLUNwPLkRFhjpSkBvo9YMRkYtizPj+dkhj2YtWJwYO5JGmuMEdK0pAbROXpscB6mXk1cAalX/YmwPmrmP86YLX3s5AkaRYxR0rSkOtr5SkzLwZuoSQEMvNm4HLgzMwcX8UinwPuHxHLIuJb/SupJEn9ZY6UpOHX95vkZuY9u17v3vU6Op5fQteFrpl5767Xb+t5ISVJGgBzpCQNN2+SK0mSJEkNWHmSJEmSpAasPEmSJElSA1aeJEmSJKkBK0+SJEmS1EBkzo177EWM5fz567QSe+XKO1uJW8TaZ5lO1GgnLsBBB72utdg7PWLn1mK/88gXtxZ7FEW017bS5nFn3rx5rcQdH1/ZSty2dY7OJq1ORMyNfwYa2mab9nLNZ075amuxn/LgB7cWexS1mccWrLOwtdh33Hl7a7HHxtrZJhMTq7qbQm+09T1mTkwrR3rmSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDVp4kSZIkqQErT5IkSZLUgJUnSZIkSWrAypMkSZIkNWDlSZIkSZIaGMnKU0S8MSIui4jlEXFRRBww6DJJkjQMzJGS1J6RrDwBlwF7AhsBxwInRsQ9B1skSZKGgjlSkloykpWnzPxmZl6TmROZ+XXgEuDh3fNFxGERsTQilkL2v6CSJPXZ9HKkJKmJkaw8RcTzIuKCiFgWEcuAXYBNu+fLzBMyc/fM3B2i/wWVJKnPppcjJUlNzB90AaYqIrYDPgvsA5yVmeMRcQHWjiRJc5w5UpLaNYpnnu5B6YN3PUBEHEppVZMkaa4zR0pSi0au8pSZFwEfBM4CrgMeCJw50EJJkjQEzJGS1K6R67YHkJlHA0cPuhySJA0bc6QktWfkzjxJkiRJ0iBYeZIkSZKkBqw8SZIkSVIDVp4kSZIkqQErT5IkSZLUQGTmoMvQFxExN1Z0KLR3L8bMidZiR3gPSc0+memOrbUyR3Zr72czf/46rcX+1CnfaS32i5/8hNZi6+4e/egDW4t95pn/3lrsUTSdHOmZJ0mSJElqwMqTJEmSJDVg5UmSJEmSGrDyJEmSJEkNWHmSJEmSpAasPEmSJElSA61VniLiioh4fFvxJUkaReZHSRpdnnmSJEmSpAaGqvIUEfMHXQZJkoaN+VGShkPrlaeIeHhEnBURyyLi2oj4REQs6Hg/I+IVEXEJcEmddlSd95qI+Mc6zw71vYUR8YGI+N+IuC4iPh0R67W9HpIk9ZL5UZJGTz/OPI0DrwM2BfYA9gFe3jXP3wOPAO4fEfsCrwceD+wA7N0173uAHYHd6vtbA29tqeySJLXF/ChJI6b1ylNmnpeZZ2fmysy8AvgMsFfXbO/OzD9l5m3AwcAXMvPCzLwVeNvkTBERwGHA6+r8y4F3AYes6rMj4rCIWBoRS3u/ZpIkTd8g82NdxhwpSVPUeh/qiNgR+BCwO7B+/czzuma7quP5VsDS1by3WY1xXskT5SOAeav67Mw8ATihliOntwaSJPXeIPMjmCMlaTr60W3vU8Bvgftl5obAmykH9E6dB+1rgW06Xt+r4/kNwG3AAzJzcX1slJmLWii3JEltMj9K0ojpR+VpA+Bm4JaI2Bl42Vrm/wZwaET8TUSsD7xl8o3MnAA+C3w4IjYHiIitI+JJ7RRdkqTWmB8lacT0o/J0BPBsYDnlwP71Nc2cmd8FPgb8GLgUOLu+dUf9+4bJ6RFxM/ADYKfeF1uSpFaZHyVpxETmcHdzjoi/AX4NLMzMlTOIM9wrOqt09zrpndK42o6O6wSkWSMz3bFnqV7lxxrLHHk37f1s5s9fp7XYnzrlO63FfvGTn9BabN3dox99YGuxzzzz31uLPYqmkyOH6ia5kyLigHq/iiXAe4FTZpoYJEkadeZHSRqsoaw8AS8B/ghcRrkPxtr6gUuSNBeYHyVpgFofqnw6MnPfQZdBkqRhY36UpMEa1jNPkiRJkjRUrDxJkiRJUgNDP9per0SMZVsj3KxceWcrcdu0ySZbtRZ73rz2RhJaseKOtc80TfPnL2gl7vj4ilbiAtx00/WtxV60aElrsZ/8lBe1FvtrX3lPK3Ej2mtrmjevnR7UK1euIHPC0fa0VhGRY2PzWok9MTHeStw2bbDBxq3FvuWWZa3FXmeddvIYtHcMbDOvt/k/bpsj9J689NzWYu//kIe0ErfNHDk21k7s8fGVs2e0PUmSJEkaNlaeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDVg5UmSJEmSGhiaylNEfDEijht0OSRJGjbmSEkaDkNTeZIkSZKkYdaTylNEbNGLOGuIv1m0eTcySZJaYo6UpNlj2pWniFgcES+LiHOAL9ZpGRE7dMzzl24GEbF3RFwdEYdHxB8j4tqIOHQ1sTeIiB9HxMdqQngh8LuIODYi7jPdMkuS1A/mSEmanaZUeYqIsYh4YkR8FbgSeCLwTuCpDUNsCWwEbA28CDg+IpZ0fcYmwA+BMzPz1Vm8FzgE2BxYWpPGcyNi/bWU97CIWBoRSyGnsqqSJE3JaOdISVITjStPEfFK4ArgPcBZwPaZeUBmnpyZKxqGWQG8PTNXZOapwC3ATh3vbwWcDnwzM4/pXDAzz87Ml9V5PgU8C7g6Iv55dR+WmSdk5u6ZuTvYo0GS1I7Rz5GSpCamcubpPsAS4ALgF8CN0/i8GzNzZcfrW4FFHa+fAqwHfHp1ATLzDuCXtRx3ArtMoxySJPWSOVKS5oDGlafMPBzYHvg18HFK/+p3RMT9Oma7FejsJrDlFMvzWeB7wKkRcY/ONyJik4h4Ze0//iNgHvDYzHzkFD9DkqSeMkdK0twwpWueMvOPmfmhzHwQcBCwGDgrIj5fZ7kAeHZEzIuIfYG9plGmVwL/A5wSEesBRMSLKN0h9gKOBe6VmW/IzN9MI74kST1njpSk2W/ao+1l5nmZ+SpK/+rJLgSvAfYHlgHPAb41jbgJHAZcDZwcEetS+o9vl5nPyMzvZOb4dMstSVLbzJGSNDvNn2mAzLwTOKc+Xwo8YDXz/QTYpmvavTuev6Dj+QTwvI5ZL5ppOSVJ6jdzpCTNLj25Sa4kSZIkzXZWniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1EGXU09kvIhKipehzYxvOdltsce9W4h5z/MdbiQvw3tce2Vrs3//+4tZit2nRosWtxL3llmWtxAVo7zicZGZbBz7NIhFjOX/+Oq3EXrnyzlbitmlsbF5rsScmRnMk+R13fFgrcQ889EWtxAX42DuOai32rbcuby32hhtu0lrsttx++y2txY6W/n+/c8XtTExMTDm4Z54kSZIkqQErT5IkSZLUgJUnSZIkSWrAypMkSZIkNWDlSZIkSZIaGKrKU0S8LSJOrM/vHREZEfMHXS5JkgbJ/ChJw2GoKk+SJEmSNKysPEmSJElSAz2rPEXEoRFxSsfrSyLimx2vr4qI3SLio/X5zRFxXkTs2TD+QRFxRUTsEhHrRsSJEXFjRCyLiHMjYoterYskSb1ifpSk2aOXZ55OB/aMiLGI2ApYAOwBEBH3BRYBvwTOBXYDNga+AnwzItZdU+CIOBR4L/D4zPw18HxgI+BewCbAS4HbergukiT1ivlRkmaJnlWeMvNyYDnlwP8Y4DTgmojYGdgLOCMzJzLzxMy8MTNXZuYHgYXATmsI/VrgSGDvzLy0TltBSQo7ZOZ4Zp6XmTd3LxgRh0XE0ohY2qv1lCRpKoYxP0J3jsyerKskzXa9vubpdGBvSnI4HfgJJTHsVV8TEUdExG8i4qaIWEZpIdt0DTGPBI7PzKs7pn2Zkny+FhHXRMT7ImKd7gUz84TM3D0zd5/5qkmSNG1DlR+hO0fGzNZOkuaItipPe9bnp9ORHGr/7aOAg4ElmbkYuIk1H7WfCBwTEQdNTsjMFZl5bGbeH3gUsB/wvB6viyRJvWJ+lKRZoI3K02OB9WpL2BnAvpQuBOcDGwArgeuB+RHxVmDDtcS8sMY4PiKeChARj42IB0bEPOBmSjeFiR6viyRJvWJ+lKRZoKc32MvMiyPiFkpSIDNvjojLgeszczwiTgO+B1wM/Bn4MHBVg7i/iIj9gO9ExApgMfBpYBvgFuDrlK4KkiQNHfOjJM0OPb87eWbes+v17h3Px4EX1sek93W8/7aO51fQ0V0hM5cCncOtfrVXZZYkqW3mR0kafd4kV5IkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDVp4kSZIkqYHIzEGXoS8iYm6sqGZgTfeinIn2dr3ttntAa7GvvPKi1mK3uU3OvPjiVuI+esedWolbtLc9MrOtHVuzSERkRDvtqZneZmo2WHfdRa3EjWjvEHX77X9uLXab/z+3uU3OvezSVuI+fIf7tRK3TRMT49PKkZ55kiRJkqQGrDxJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhoY2cpTRLwgIn466HJIkjRszJGS1I6RrTxJkiRJUj/1rfIUEVdExJsi4qKI+L+I+EJErBsRSyLi2xFxfZ3+7YjYpmO5F0TE5RGxPCJ+FxHPiYi/AT4N7BERt0TEsn6thyRJvWaOlKTR0O8zT88BngRsD+wIHFPL8AVgO2Bb4DbgEwARcQ/gY8CTM3MD4FHABZn5G+ClwFmZuSgzF/d5PSRJ6jVzpCQNuX5Xnj6RmVdl5p+AdwLPyswbM/OkzLw1M5fX6Xt1LDMB7BIR62XmtZl5YdMPi4jDImJpRCzt7WpIktRz5khJGnL9rjxd1fH8SmCriFg/Ij4TEVdGxM3AfwGLI2JeZv4ZeCalBe3aiPhOROzc9MMy84TM3D0zd+/pWkiS1HvmSEkacv2uPN2r4/m2wDXA4cBOwCMyc0PgMfX9AMjM0zLzCcA9gd8Cn63vZ19KLElSf5gjJWnI9bvy9IqI2CYiNgaOBr4ObEDpw72sTv+nyZkjYouIeFrt130HcAuliwLAdcA2EbGgr2sgSVI7zJGSNOT6XXn6CvB94HLgMuA44CPAesANwNnA97rK93pK69ufKP28X1bf+xFwIfCHiLihH4WXJKlF5khJGnLz+/x552bmu7um3Qrs3TXtM/Xvtdz9wti/yMw7gaf0tHSSJA2OOVKShpw3yZUkSZKkBqw8SZIkSVIDfeu2l5n37tdnSZI0SsyRkjQaPPMkSZIkSQ1YeZIkSZKkBiJzbtxHLyJybGxeK7EnJsZbiau/Nn9+e7csyZxY+0zTMD6+spW4AFtsce/WYq+zzrqtxd7j0fu3Fvv3V13aStzzz/9BK3EB7rjj1lbiTkyMk5nRSnDNKhGREe20p7Z1bFV/bbDBxq3EXb78T63EBVi4cP3WYn/iWye3FvtqltZcAAAgAElEQVTwgw9pLfbCde/RStwbb/x9K3HbNN0c6ZknSZIkSWrAypMkSdL/Z+++wySrysSPf9/JQ0aCCCgSBFQEVAywi4CgoIKK/mQNa8Aw5ghiQF1RDKiY1jiuEQyI7ooBdY2ICsKwgKirIkFBJLokSTPT7++Pc9opypnp2911K3R/P89TT1e4961Tt27dt8+5554jSQ1YeZIkSZKkBqw8SZIkSVIDVp4kSZIkqQErT5IkSZLUgJUnSZIkSWrAypMkSZIkNWDlSZIkSZIaGMnKU0S8NiIuioibIuI3EXHooMskSdIwMEdKUntGsvIEXATsDWwIHAOcGBF3G2yRJEkaCuZISWrJSFaeMvPkzLwiM8cy8yTgQuDB3ctFxJKIWBYRy/pfSkmS+s8cKUntGcnKU0Q8IyLOi4jrI+J6YBdg0+7lMnNpZu6RmXv0v5SSJPWfOVKS2jNv0AWYrIjYBvgEsD9wRmaujIjzgBhsySRJGixzpCS1axTPPK0LJHANQEQcTmlVkyRptjNHSlKLRq7ylJm/AY4HzgCuAu4H/GyghZIkaQiYIyWpXSPXbQ8gM48Gjh50OSRJGjbmSElqz8ideZIkSZKkQbDyJEmSJEkNWHmSJEmSpAasPEmSJElSA1aeJEmSJKmByMxBl6EvImJ2fNAZ7v73f0Rrsf/2t+tbifv735/dSlyAiPbaP+bMaS/20e/+eGux3/Kq57QUeRTnGE0ycxQLrj4zR84M22+3e2uxL77kl63EzRxrJW7b5syZ21rs711wfmux979vW9O+tZlq2js8TSVHeuZJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpgdYqTxFxaUQc0FZ8SZJGkflRkkaXZ54kSZIkqYGhqjxFxLxBl0GSpGFjfpSk4dB65SkiHhwRZ0TE9RHxl4j4UEQs6Hg9I+LFEXEhcGF97qi67BUR8dy6zA71tYUR8Z6I+FNEXBURH4uIxW1/DkmSesn8KEmjpx9nnlYCrwQ2BfYE9gde1LXM44GHAPeJiIOAVwEHADsA+3Yt+05gR2D3+vpWwJtaKrskSW0xP0rSiGm98pSZ52TmmZm5IjMvBT4O7NO12Dsy86+ZeStwGPDpzPx1Zt4CvHl8oYgIYAnwyrr8TcDbgSev7r0jYklELIuIZb3/ZJIkTd0g82NdxxwpSZPUeh/qiNgReC+wB7BOfc9zuha7rOP+lsCyNby2WY1xTskT5S2Auat778xcCiyt5cipfQJJknpvkPkRzJGSNBX96Lb3UeC3wL0ycwPg9ZQDeqfOg/ZfgK07Ht+94/61wK3AfTNzo3rbMDPXa6HckiS1yfwoSSOmH5Wn9YEbgZsjYmfghRMs/2Xg8Ii4d0SsA7xx/IXMHAM+AbwvIjYHiIitIuLAdoouSVJrzI+SNGL6UXk6EngqcBPlwH7S2hbOzG8DHwR+BPwBOLO+dHv9+5rx5yPiRuD7wE69L7YkSa0yP0rSiGntmqfMvGfHw527Xn5Tx3LdXRTIzHcA7wCIiHsDY5TuCmTmbZSuDa/vbYklSWqf+VGSRtdQTZI7LiIOrfNVbAwcB3wjM1cMulySJA2S+VGSBmsoK0/A84GrgYso82BM1A9ckqTZwPwoSQPU+lDlU5GZBw26DJIkDRvzoyQN1rCeeZIkSZKkoWLlSZIkSZIasPIkSZIkSQ0M5TVP0pqce+73Wov9udN+0krcZ+23Xytx25aZrcU+9QtfaS12e9rbHtJw+IeR0XvE306/XHTx+a3F3mKLbVuJe+WVl7QSt23z5y9sLXaL6bc18+bNby32ihV3tBZ7KjzzJEmSJEkNWHmSJEmSpAasPEmSJElSA1aeJEmSJKkBK0+SJEmS1EBPK08RsVNEnBcRN0XEy3oZW5KkUWaOlKTR1+uhyo8CfpSZu/c4riRJo84cKUkjrtfd9rYBft3jmJIkzQTmSEkacT2rPEXED4H9gA9FxM21e8J7IuJPEXFVRHwsIhbXZfeNiMsj4oiIuDoi/hIRh3fEWhwRx0fEHyPihoj4ace6D42In0fE9RFxfkTs26vPIElSG8yRkjQz9KzylJkPB04HXpKZ6wEvAHYEdgd2ALYC3tSxyhbAhvX55wAfjoiN62vvAR4I7AXchdLVYSwitgK+BRxbnz8S+GpEbNarzyFJUq+ZIyVpZmhltL2ICGAJ8MrM/Gtm3gS8HXhyx2LLgbdk5vLMPBW4GdgpIuYAzwZenpl/zsyVmfnzzLwd+Ffg1Mw8NTPHMvN7wDLg0Wsox5KIWBYRy9r4nJIkTZY5UpJGV68HjBi3GbAOcE7JEQAEMLdjmesyc0XH41uA9YBNgUXARauJuw3wpIg4pOO5+cCPVleIzFwKLAWIiJz8x5AkqefMkZI0otqqPF0L3ArcNzP/PIV1bwO2B87veu0y4ITMfN70iyhJ0kCYIyVpRLXSbS8zx4BPAO+LiM0BImKriDiw4bqfAt4bEVtGxNyI2DMiFgInAodExIH1+UX1wtqt2/gckiT1mjlSkkZXK5Wn6jXAH4AzI+JG4PvATg3XPRK4ADgb+CtwHDAnMy8DHge8HriG0sr2atr9HJIk9Zo5UpJGUGTOjm7O9ufWRD532k9aifus/fZrJe4oe8ADHtla7GXLvt1a7FGUmTHxUprtSo5sa1cx/fZPez/3LbbYtpW4V155SStx27Zw4eLWYn/rnLNai33ALru0EnfevAWtxAVYseKO1mJPJUfaGiVJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpAStPkiRJktRAW5PkSiNnhy3v1krcNke0vMtdtmgt9g03XNta7HPP/V5rsefObeewNjY21kpcaHMfcZQzNTNnzlzWWWeDVmLffPP/tRJXq9Peb37rrZuOpD85t932t1biAvztb9e3FnvRonVbi/3MAw9tLfbCheu0EvfAA5/TSlyAU0/9eCtxV6xYPqX1PPMkSZIkSQ1YeZIkSZKkBqw8SZIkSVIDVp4kSZIkqQErT5IkSZLUgJUnSZIkSWrAypMkSZIkNWDlSZIkSZIasPIkSZIkSQ1YeZIkSZKkBqw8SZIkSVID8wZdgDZFxBJgyaDLIUnSsOnMkRG2pUpSEzO68pSZS4GlABGRAy6OJElDozNHzp07zxwpSQ3Y1CRJkiRJDVh5kiRJkqQGZkTlKSK+HRGvH3Q5JEkaJuZHSeqtGXHNU2Y+atBlkCRp2JgfJam3ZsSZJ0mSJElqm5UnSZIkSWrAypMkSZIkNWDlSZIkSZIasPIkSZIkSQ1E5uyYVHzOnLm5YMGiVmKvWHFHK3EBxsbGWom7wQabtBIX4IYbrm0tdkS0Fnu7bXdtJe7Fl/yylbjQXpkB7lh+e2uxt9hi29Zi/+pXp7cSd968+a3EBRgbW9lK3FtvvZmVK1e096PRjLHeehvl/e63byuxzz//h63EBVh33Q1biXvddVe0Ehdg/vwFrcXeaKO7thb7ppv+2lrstrR1bIV289jVV/+ptdi3335LK3Hb3B7XXHNZK3FXrLiDsbGxSedIzzxJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGelZ5iojWZmZrM7YkSW0zR0rSzDCtylNEbBQRL4yIs4DP1Oe2jIivRsQ1EXFJRLysY/mFEfH+iLii3t4fEQvra5tGxDcj4vqI+GtEnB4R4+X7TEScFREviIiNplNmSZL6wRwpSTPPpCtPETEnIh4ZEV8E/gg8Engb8Nh6IP8GcD6wFbA/8IqIOLCufjTwUGB3YDfgwcAb6mtHAJcDmwF3BV4PZH3tscDbgQOBP0bEFyLiER2JY01lXRIRyyJiWWaubVFJkqZtVHPk8uV39ODTS9LMN6nKU0S8BLgUeCdwBrB9Zh6amadk5nLgQcBmmfmWzLwjMy8GPgE8uYZ4GvCWzLw6M68BjgGeXl9bDtwN2CYzl2fm6VlrPPXx1zLzUGB74EzgOODSWqbVysylmblHZu4REZP5qJIkTcoo58j58xf0dmNI0gw12TNP2wIbA+dRWs6u63p9G2DL2q3g+oi4ntI6Nt4fe0tKS9y4P9bnAN4N/AH474i4OCJeu4YyXAf8spZh41omSZIGzRwpSTPcpCpPmXkEpVXrV8C/A5dExFsj4l51kcuASzJzo47b+pn56Pr6FZTkMe4e9Tky86bMPCIzt6N0QXhVROw/vmBE3Csi3gpcAnwAuADYrpZJkqSBMkdK0sw36WueaneC92bmrsATgY2AMyLiU8BZwE0R8ZqIWBwRcyNil4h4UF39i8AbImKziNgUeBNwIkBEHBwRO0TpX3cDsBIYq699itIFYiPgCZm5W2a+r3ZrkCRpKJgjJWlmmzedlTPzHOCciDgC2D0zV0bEwcDxlNavhcDvWHXB67HABpQuBQAn1+cA7gV8iHIx7P8BH8nMH9XXPga8IDO9olWSNBLMkZI080yr8jSuHrDPqvevAJ6yhuVuA15Wb92vvQ943xrWO6sX5ZQkqd/MkZI0c/RsklxJkiRJmsmsPEmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqIDJz0GXoi4i4hjvP3D6RTYFrWyhKW3FHNfYolnlUY49imUc19rCUeZvM3KylcmgGmWSOHJb9e1hij2KZ24w9imUe1dijWOZhij2lHDlrKk+TFRHLMnOPUYk7qrFHscyjGnsUyzyqsUexzFJTo7p/+3vvT+xRLPOoxh7FMo9y7HF225MkSZKkBqw8SQMQETHoMkiSNIzMkRpmVp7WbOmIxR3V2KNY5inHjog5ANnVX7YrUbitRz/2KJZZampU929/7/2JPeW4DXLkKG6PNmOPYplHOTbgNU9S30TE3MxcGRH3Ap4C3AD8JTO/POCiSZI0UOZIjQrPPEl9UpPCvYGfA/cAtgKOiYj3DbZkkiQNljlSo2LeoAsgzQa1y8E84JXABzLz2IhYABwKLBpo4SRJGiBzpEaJZ56kPshiOTAX+HXt130WcFZmvjAi7hcRDx5sKSVJ6j9zpEaJlSepJRExt/7dICIW12QwFzgY+AlwQWY+rS5+BPDQwZRUkqT+MkdqVDlghNSCjgtf7wd8Enh9Zn4/IrYCzgTIzLvXZT8D7Ao8ODNXDKrMkiT1gzlSo8zKk9SSiLgP8FPgfZn51oiYl5kr6khCXwcuB8Z/gI/JzOXjCWVQZZYkqR/MkRpVVp6kHoqIOZk5Vu8vBS7PzLfU7ggvBW6n9OO+ANiurnZhZo6NJ46BFFySpJaZIzUTONqe1CMREfUAvwtwI/Ar4EkRcT7wZuAaym/uEOAZmfm7jnXnmBQkSTOVOVIzhQNGSD1QuxJkRGwJnA08GvgB8L/AE4GvZeYjgRcAC+rt78Zb4iRJmmnMkZpJ7LYnTUNtSct6fwfgYcBdM/MdHcsszMzb6/1PAdsAjzAZSJJmMnOkZiLPPGlWq/2s1/h4Les9NyKekHdufVgC/AfwoIiY2xFrZUTsFhFfAx4IHFS7Lvj7kyQNpanmx7qsOVIzljumZq3ajWAsIraIiF0jYosmLV0RcVfgeuCqzucz8yjg3cAjgd3GY3X00/4B8MA6YtA8W9WGV0SsN+gySNKgTDU/1nXNkTPcbM+RdtvTrDTelaDOMXEapd/1LsBhmfndtay3J/Bi4KWZ+X8RsQ1wV2CTzPx2XeYjwJOBAzLzf1YTw6FWh1jtk/8O4E3AZSZwSbPJVPNjXdccOcOZI608aRbqSAybA28Efg8sBV4JvBX4l8z8zzWsuytwU2ZeEhG7AV+r6+9IaWV7Sn3t48ChwGMz88z2P9Xw6urz/vf7w6r+w7BZZv5w0GWZqo4JKId+e0saHtPJj3V9c+QkjFp+BHMkWHmasvG5CiJiQWbeUZ8biR1fEBEbAZ8C7kaZfO+v9flXAO+iJIj/Wsv6GwKnAidm5kcjYgFwCrAJsGf9UX4ZWD8zH9XyxxlanfNydM7vMQoiIoBjgTMy85uDLs9k1X38YOCnmXnpgIujWcYcObqmmx/rsubICYxyfoTZnSO95mnq5kfE3YF3RMRzAEwKI2VD4NfAfYEnjD+Zme8HjgS+GhH7rGX9RcA6lC4NZOYdNQEsAt5bnzsMeEwrpR8RWWaLnxsRXwROjYjnRMR2E644IDUZjNsR2Bp4ZkQcMKAiTVpE7FOPST8CPgc8bsBF0uxkjhxd082PYI6c0KjlRzBHjnOS3CmIiKdSDioPBx4CfBr45EALpbXqOEW7MDNvz8w/RsTbgeXACyLib5n5RYDM/GBEXAn8rGP97laha4G/AAdFxEXjw6wCnwfuMr7Q+IhBo9aiNF1dLcynADcBPwaeCuwUEZ/LzF8Nqnyr093PPjN/FxHvBF4IvKh+pu8NroRrFxH7UlrRHgv8F/AH4M94bFKfmSNHy3TzY41hjmxoFPMjmCM7WXlqKCLmUnaQ+1L66R4DnEyZIfvtdRm7JPRI9490Otu2HphXRpnV/HURMR/4DfAl4G2UBPGyiBjLzJMAMvPLEbFxRNycmcvrAX4LYFPgjsz8fUR8k9JqdnlEfDczbwAeDFza+f6zKSnAP3RFWI9ySvyd9fF5wPOAZ0XEpzPz1wMs6t91/PMwBziC0rXkZ8AZwHHAayn/RIxl5g8GWNR/EGVkq88CtwE3Ak/IzF9FxEsp++tts+2fE/WfObJ/hiE/1nU3Bf6amWPmyGZGMT+CObKb3fYaiIgNKBc9HkD50T8kMz8KXA1sDNwCdknolZoIVkbE/Ih4W20Nm/K2rQf17YHvAecD36B0S/gesD5wAqUV4u3jp54jYq/6+h718S7Az+uyn4uIT2XmR4BfAM8Czo6IrwP3Al4z1bKOuvrdjXdF+E9KAj4yIh4EkJnfoczzcQ/gJRFx7wEW9+86ksLZwIOAjSin8r8C3Expmfoj5Z+LfQdVzjWYD3wXeAHw/JoUHgS8DvhRZq6YTf+cqP/Mkf0zDPmxlmMv4DuUM4vmyAZGNT+CObKbZ54ayMwbI+IdmfnzWHUR7M6UkWdenZlXTRRDzXS1qP0HMLfjdP9U4s3PzOWUU7Xfycx31eePBE7NzOsj4ibgM5QuBj8CqN/1lcA7I+Jo4CjgncBJwH2AD0bE1zLz8VFmTd+T8g/CKeMHx5zGUKudLYnTjdUvXeX8JLAYOJEy6eEzIuKyzLwyM78d5eLhJwLXDai4q/M84KraD5+I+DaQwI2ZeV5ErASeRjnVP3AREcDWmXkZ8L7x5yJiHnAI8KnM/LGt/WqbObI/hiU/gjlysmZAfgRz5N955mktImJORDwPyoGiPj2+zXahjCRzyiDKNlPV1o2IiKWUYU1fCP9wkeIaRcTBEfHsiPjXenp8eX1pAXBZXeZc4DeZ+fwo8xU8GbguM0+g9M9+Qy3LwZR+2++u6383M2/IzDMo3VK2iIhnZuYfMvOEzPxqL5JCfe+M0n1ifJtsOp14/dD13V0OPC4zPw/sBxwIHB2lWweZeQrwwsy8enAl/geLKF2MiIgTgK0okzkuiIhHZOYFwJsy8/IBlhEoxyZKl4l/i4jF9bnxBLAAeAT1s1hxUlvMkf016PxY3/+QWlkyR07CDMiPYI78OytPaxCl//YvgCdEmeQNuNNM2EcC13YcfNQ7G1N27FcAW9TnJtxXI+JwyoXJ21D637654+VLgFdFxO+Bn2TmU+rzxwMPHj+wUS5kfXlteSMznwj8lnLh80NriwXAlZRT1HftLkcvWsDq/vf1iHh5lBGrLo7SLWLYbUw5mB5B/e4y87eURLo/ZeStzevzfxtUITu+x07XAYdGxFeBHShdj5ZTWlT/JSIWZx1yeZBqUjgLuJCSYG+FOyWAZ1GOTV8aTAk1G5gjB2Yg+bHGCEp3rVeYI6dkJPIjmCMnlJneVnOj9Pf9TMfjTYGFlIPU/YDPdrwWU4g/dw3PTzpW09jDeltdeYHNgV8CP2jyuYB1Kf2vH1UfHw/s1bXMW4ExygWrO1MSyXnAvI5lFgOHUfrtH9Xx/JcoF0buDaxXn/sWcExL22QhZUSYy4FrgCX1+TmD/r4afncXAN/ren43YBlw1wGVdQvgXp1lr9v4KZSktgD4AOWi0t3rMi+mXLdxv0Fv645yHwh8v+PxEcAHgRdR+nZvCtx3GPcXbzPnZo7s23YemvxYlzNHTv+7G7r8WMtgjmx485qn1YgyudsNwMfq4w9RxrNfDLwuM386ftp6Kn0l486jlryYMvrHzVlOaU+7i02uOouyHXBxL2K2ZTXb4ibKtvhKlItTT4uI72fmAXW5NZ3un0OZb2Dr+ngP4IKIuJzyQ18MfJTSNeFDwMWUUYQelLUbAUBm3hoRvwA+QjmNfmNmfiwznxwRJ1P6Kl8WERdTLupsZQ6dzLw9Is4BtqzbZEV9fqwXXR56YYLvbn86vjuAzDw/IvbKAbRMRcQ6wBuArSLiFZn5R8o/BjdTuiK8BTiaMozu34BTImIZ5Tt+ZJbuCMPiasroQMdRfuM7UC7yPh74U5bJCq+F2TWKlfrHHNkfw5QfO+KbIxsYpfxYy2uOnIQY0mPGQEUZOeiDlNm1V1CGZHw2pcZ9aWY+rwfvMQc4B/grZVjQJ1FGLXlpLw7kdad5JPBM4IJhTQ6wxm3xn5n5oihDTH4PuD0zHzRBnJdSuiMcRGn5XEpJDDdTLlQ9E3g5EJQDGZmZteI0Vu/fD/g28GXK9rsL8IHMPK6+x8eA51Ba3r4+QcKayrb4+zCm9fH9Ka0k/wF8KDPf3Yv36ZXJfndT+Ueqh2U9mDLh41xKa+XGmfmi+tobKEPqvjEzvx8Ru1EukB7LzGsHUd5u9bi0AridkozvBqyktOwuj4jPUrrcOJ+OWmWO7J9hyI/jlU1z5OSMUn6s72+ObGoqp6tm4o1ywNgbuD/l9OSWlH6oT6KeegVeDXwCmN+D9zsC+FLH41OAM6YRb3Wnh79O+XHuRm+6OszpuD/teBNsizPH349yKvkXwDYTxFmP0jryR2An4O6ULgj3AR4FnA7s3bH8jsCGHY/XoVzg/Lr6eEvgGTXeKzuWO7ajbNPuIrCacmxPGVJzb2Dz+ty/UloDX1Uffw54fK++g0F/dy2XMTruH0T5B+984M1dy70X+HUvvtMel38O8FXgh5ThVt/Q8dq8+veVlJa0HQZdXm8z82aObPQePc+Rg8qPdR1z5BB8d30opzlysu856A89DLe64c8AfkppJfgVsH/nxqe02FwP7Nqj9zwGeHe9//m6o86vP6aHTDFmABt1PfcNyszV0yo3q/owzwEW9Xj7r21bPHT8fdf03XU9XpcyIeOfgP26Xju9HuiDMkrMVfV73aBjma8CL++K9x5K69zRXfGm1W++qxyvq9t25/oD/3ndFz9C7YNM6Xd8I2WehT/Qg39QBvnd9bGMnf/QPAP4F0rL6fnAzh2vbU8ZinfDfpdxgn3ke5RW1YdQZqC/iJqQKf9UvBW4AnjAoMvrbWbezJGNYreSI/udHzu2kzlygN9dn8tpjpzkzdH2io8Cv8vMf6acav408M2I2K+OOPIKypj7+2XmLycbfPx6mo7HQTmNu05EfJIyI/sDs4xacjjwtIhYOIXP8SZKP+bNxp/IzEMoP9YPRcQD63tPtvxHAY+JiI0oLXUHTaFs47Emuy2eGhGLcjX9UmtXgLGIuFtE/FNEPJDS0+D1lIn6PhsRD6/LnkTpXvCFLP4MvBR4LvC8iNi4nmK/FrhflAkII8uIN2dRDtL3jwqmP2JQVzmeQ9nPnkoZ6nMvSivhIuCNEbFjZn6R0lf9fcBOWU5D9+26xV5+d5N833ldjxvvw7FqzpmIMkHjXpl5EmU+kguAt0TEA+rij2fV6FXDYltKi/FrMvMXmfkF4NHAfSPiaZRRss4H/ikz/2eA5dTMZo5ce/l7kiOHIT/CP+Qmc2QDo5gf6/LmyKkYdK1xGG6USd1eWO+PXwf2FuCLuaq2vfkUY493ZwjKCCAPqI+3ptSObwG2qM+9hHKh230axo6ux9sBX6RrxBZKa8wdlAPjwoax79Zx/3jgB3Un/Oo0tnPPtkXH97QrZTjKn1JOf/+CVafx31rLfDRl2Nz5neWo959AGTXoyPp42/q+H6SOHkPphnJ0x3v2qjtG56nyJ9ZynA88ueP5AykTFH6aOrpN9/bs02+ktf24yTaitGwvo2MkoEnGeW3d/zu/+wOAkykXvp9EaX0eqrM3lFbXM6mtxHX7z6f88/OGQZbN2+y5YY5cXeye5shebgemmR+5c24yRw5wH26yfZhmfqwxzJGTed9Bf/ABb/R16t+lwNu7XvtXygzb04n/9z6/lHkQzqJcyHYMsD6lz/H/UPrBfocy9Oj9G8buHF67sy/wXSkXcp7TcZBcQplMb5uGsRdTTut3dsu4gNLidEjngWKQ24Iy38QvgOfVxztQZuy+hNInfy5l8r7/7DrIBHc+TX0Ypc/2a+vjHWvcs4Fz67acP9nP3GCbdJfjEMqp5XfQ0bWEclHuKZQLNXtShkF/d5MtI/BGyoW2U4nzMMo/Njfzj8l1L0of6U8AW/bqu+3BZ1/csc2/DHwT2LBjP/534C293ie9eeu8YY5cU+ye5sg2tgNTzI/j5cYcObDvbrLlYxr5sa5vjpzs+w96Awxoo8+h9I98ZH38aErN+tnAXepzL65fxrqTjL2w88tiVUvE0vr4EErt/T31i96A0jr0ABqO78+qA/P4TnMGpeXlifX5zevzN1PmXljOJFskqK1qHQfTQygtVf9N6RO7QXd5+rEtOj77OpR5Hs7oPBBRTuF/jVUXtG7Y9f7j669f329Bffw47pwcNqKcZt+7Y52etWKtpRyH1HIcQRnpZnz5B9ObC283oUFi78d+PIkyf47S/3r897rW7bC674mSHE6ltEZt2/XaAQWrTE8AACAASURBVL0u8zQ+6xzKPzjfoQwD/ThKhf9cyvHoOODfKJMV7jzo8nqbmTfMkU3eY1o5so3twDTzY1cMc+QacmQ/9uFJlHdS+XEt+6M5cjLlGPSGGNCGP5/SCrAxq1oNnko5Jfxj4L8os2PvPsnYW9cvc5OOuN+iDPn43I7l9qdcnPkOYMdpfI5zKKdS96e0IP0EeE7HMksoCW7SO1BH+X8AfKzj+XdTLs57an38BMooJt3dI3q+LToOVLsC3wf+mTK6yhO6Xv934LjudTvKcb+67b5DaS08uD7/OEqr3JHA+l3r9zIpNC3Hq4BNVrfuFPeXn9btdS5dB8dB7ccNy/4uyrwSb2SCLjXc+Z+mJ1D6lu/MqpHCvkj5p3Cbtso7jc8Z9Tf8OcqFr68C/gzsS2npPgb4OOUswC6DLq+3mXnDHNk4fv076RzZxnZgmvmx6zOZI9eQI/u5Dzcsd+P82Pk9YY6cXlkGvTEGsPHfBnyu4/GBwMMpLVHbUEYZORzYbgqxHwYc1PXc3pTT+yd1Pb8f5fTtv1FbVBrE7+yL+lzgyx2Pv1R3ou/SkRym8Bm6ZxP/Z8pINx/ueO64eoA5ldIaed9+bYt6wDoeeHF9/JZ60HoItWsGpd/usWtYf6tahtfUH+IxlBGi9qyvH0qZ++LJaytHD/bDvpaDktBPAnav+8o1rKHrQNv7cdN9vOv5o4HfU5LmRPvIHMrkfqdQ5tb4FqV//lzKQfaEui3u0eZ3PIXPfn86Zp2v+/EvKC3Inb/9ef0um7fZc8McOdF7TDtHtrUdmGZ+rK+bI9eSI9veh5vu313PN86PdXlz5HTLMuiN0ceNvkn9+wbKBXB3pdSwL6C00vyIHp2WrD/0N1FGVgF4KGXSufd2LfcwJp6bYdF4zPp3LqUP6l6suijxPyh9j+9Baen4PfCiKZS7s+/u8ZSRbfajtP5cDXykY9knAM9jgha7XmyLjs++Yf3Bn0FH4q4/+gvqdzjep3he9/r1/l7cOaH+BPhMvb9u/bvfmg5S090vBlGO+l1+ndrdpj73CUr//N36sR83LGdni9hHKa17J4zHprSs/Y7SJWJh17qd/eKPBb5Z768DPJbSVeXo+twTKa1Td5tumXu0X2xSt/VDgbPrc5+s+/T4dQTPpSYy1tCVxJu36dwwRzYpd09zZC+2A9PMj50x6n1zZE6cI3u5Dzcs45TzY+e+W++bI6dbpkFvlD5u/O9TWip2YlWXhK9SRuV4SH085cTAnWu9u9Qd8bPjP7x6ILiJOuZ/w5i7UiYrG+9bHZQRVcbnDZhHOWifQb2Qrx4o38oUL+yr73EWpdXsa5S+tC+tn+lq4N/7uS1YlRg2BfahdBe4DXhS13IHUlpEn8WqPuhzO9bfjDIZ4EPq97993ZbjcwFsXg8oG67uc/Rg/xtIOSgtxZ+s2+yxXa99FBijjpjU1n48ye0TlC4TX6FcN/Bj4DTqhdl1374KeNQa9t0dgaPoaFml/MZfSLkWYTwBrdfL8k/zs/83ZQjYuZR/8C4Czux4/dWUOU02HXRZvc3cG+bIpu85rRzZy+3ANPNjVwxz5AQ5so19eJLbZsr5sWN9c2QvyjTojdKnDf8kSqvCeAvV+sDdWNWK9ALKpH9THWq1c4Sa3SmtW0+izMZ8QscP66H1h/i2BjE3Bf4CvKI+nlN37k92LfdPlNamB1AOnOcCd59K+ev9hcDxHWV4OiVJvJxycehKuvpLt7UtOn7Ed6VMhPdZyhwOR9UDxKPXtm7X+h+lXGR4AOVAeQl3btX6bN1Hej5h3aDKQZn9fcO6r7+fkvD/uWuZ91P7Y7exH09y+wTwIOBbXa9/HPhZx+Ojge1XE+dVlAt03wRc2vXaPepvY9i6ITyJ8k/p+ASbB1Nahj9K+afhdZTuI5O6tsSbt8ncMEc2Kn+9P+Uc2cvtwDTz42pimCPXkiPb2IcnuW2mlR/ra+bIHt36NsHmgO1NmW16LCLmZeZNwE118rijgGcCB2Tm1ZMNXCeJW1EnjzuD0nqxIeXCwwT+D3h1RByXmWdGxEMpM2A3cTbwwzrp2QcpP5wNI2KTzLwOIDN/FhG3Ui4C3Zpycehlkyj/nI7yv5cyUd4WEbFxZl4bEadSWu+eTLkgbxdKcmh1W9RYKyNiV2orEyX53Uzpi/s24GMRsSQzv1OXz44QY5mZHetvUNe/ErgVuBj4TUQ8E3hE/VwPyjpZXFesKVvN52i9HHX7/4CS5LejjEp0CuWi0ndExGsy8+cAmfmKjnK2tR+vraxPB74aEbcBb6aMkLNXRGw+/nvMzOdHxKURcXhmfjoz37aGcH+hXPy9P7BbRJxO6Zv+N8pQtiso1x8Mk70prWh31MenUhLB6ygXAq8AHp6ZFwymeJolzJFrLn9PcuQw5ce6bt9zU8PPMnQ5cvx9Rjw/gjmyZ+b0880GISIOpvQ9/mhm3lEPXnMj4jBKK8dcYJ+c4szDHT/cE4E/ZOY+lNPj/0w5gJ5COaAdGxH3zcyzMvO3DUKPUX6QL6PsOBtT+m1fDzwjItbtKMOelOT2wMw8p2nZY9UM5EG56G5HSmLYjDL79Xo1AZ1COT29J3BlZv6+7W1RD1SbUn4oP6S0Qi2hzJnw/yin8Y8Hvh4RD+0+gK5m/UdQWiW3rWX4EWX2770pw57ukWU28rm9SgqDKEf9Ln9OObg8inL9wpuBB9b3+jHw8Yh4SHc5691e78drK+s9KP+MvIsyP8Y9M/MDlKT2rM59nNIieEXHuv9w7Moyu/zPKAfV11MS3G8j4nuU1rjnZ+bQJIbuYxNAltnmN8zMx2fmE4GnW3FSm8yRa9bLHDlM+XENMcyRa8iRo5Yf6/rmyDblEJyWa+PGqlOdR7LqIrjdKLM7n02Z42EnejACSo39BVb1i/0kcEG9vx5lZKL3M8k+1pQD9Q2Ulpet63OvpRykX0qdwHCa5Q5KP/cPdTz3JspQtC9i1YWZd6Gjj3E/tgVltJ0f0DGPCGW+kV9ThqLcm5Io1jQCzerWHz/l+3G65vVYU5webOO+lYNyoOkcKetDlC4c51P6BT+akizu2a/9eILy7krpI/57Vs3j8UrKxI0foPwz8lJKYtiua93xC3YP69o/vkmdY4WSiPdgkt102rwx8bHpFFZ1pXRwCG+t3Brsh+bI7H2O7NV2WENeaZwf1xLDHLmGHNnGPjxBWaecHzv2XXNkG2Uc9EZq+QvYlDKz8/sprTJXUPp7vriH7xGU0UrOA55CaSk4n1V9x19NuVhz0hffUWYE/3TdUT5Zd56gDN35VUrf5mklB0rf3THKXAp/n/SP0hLzFUof2Ubv0ettAWxBaXF6Wn083v/+hFreY4Gt6nOrm/RtbeufS5l/Yas+7Id9KwdlVJrxi0c/Dvyy3v8I5Z+MF7CaSS3b3I8nKO/dKUng1/XvJnX/ewqlle9sypwpD1jNupsD76S0NH8Y+Jf6/OnA+9v+Xqf5uVs/NnnzNtGtH/thm8cWRihH9no7TJBXJsyPDWKYI/uwD09Q1innx7q+ObKt8g26AC1u+DmUfqxjlD7AH6FrBBJ6eOEjZQLB64A/dzz3YuAypjAfRlfsu1OGGP0M5YLU8TkPPk/HDNvTiP8QyunwQ6lJgNJV4931Bzqp9+jltqC07P0PdXK8+tyH68HrLOAxU1z/+Cbr93D/6Fs56v6xOWU+k/Hk+Zr6fmtNQG3uxxO8706UrgkfHt/fKCMXPZyaoNay7s41QZxGGfnqGMrcFa3NLj7Nz9rXY5M3b6u7mSMnFb9nOXKY8uMEMcyRfdyHJyjvlPNjXdYc2ePb+PCHM1LtM/oC4H3AbVkugm3rvdah9L1+DqUVbB3KePuPzcxzexB/e8oFr3+htEBcQJmT4Lrpxq7x96HMa/Ba4DuZeUtEzKX8UK+dZKyebYuIWJ9yqvaVlP7Im1PmI9k1Ij5NOV1+WK5hR57u+r3S73JExJaUg+NxwC2U0997ZublE6zX6n48wXvvQplf5nTKRaCHAg/JzD81WHcB5YD7bspEevcF7p2ZV7ZX4qnr57FJWhNz5KTi9yRHDlN+7FWMXhiFHDmq+bGub47soRldeerUy9Fh1vIe8yjDVD6SMtv0aZl5YQ/jbw98jnK6+BVZL6DrYfx9KDX8Y4GvZeat04jVs21RL3x8KGUei78BSzPztoj4EHBrZr66zfV7pd/liIhnU1rFbgdelpnLGq7X6n48wXvfh9ICuRFlOOBGCanz9x0R21K258CTQr0Qev3MvGQty7R+bJImYo5sFL8nOXKY8mOvYvTCKOTIUcyPdd2hy5FN8mNdbuhy5KypPM0UdadfkZMYanWS8Q+g9C9++LDV9MfVA+yrKf3ZH5aZv+7n+r3Sj3LU1rzIzGkPm9ovNTlFZi6f5HpDdYCNiIXAt4BnZuafB10eaTaY7TmyF3nFHDm8ppof67pDkyNHPT/OlnmeZoyJaug9iP/9iPh5Zt7S5vtMVUQsopwmfzjwiClUnKa1fq/0qxzDmNwnkpkrprjeUCSFcZl5e0Q8dlh/S9JMNJtzZC/yijlyuE01P9Z1hyZHjnp+9MyTRk7tdzxvqi1F012/V4alHJKkmaEXeWVYctOwlEPqZuVJkiRJkhr4hxmIJUmSJEn/yMqTJEmSJDVg5UmSJEmSGrDytAYRsWSU4o5q7FEs86jGHsUyj2rsUSyz1NSo7t/+3vsTexTLPKqxR7HMoxx7nJWnNWtr47f5pY5i7FEs86jGHsUyj2rsUSyz1NSo7t/+3vsTexTLPKqxR7HMoxwbsPIkSZIkSY3MmqHKI6K1D/rABz6w8bLXXHMNm222WePlzznnnKkUSZIAyMwYdBk0/NrMkW16wCTy77XXXMOmDfPv/5h7pVlhKjnSylMPtLkNI9o6OTg7vndptrPypCbarTy1twvesWJ5K3EXzJvXSty2tfc/A2SOtRZbGpSp5Ei77UmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDVg5UmSJEmSGrDyJEmSJEkNjGTlKSJeGxEXRcRNEfGbiDh00GWSJGkYmCMlqT0jWXkCLgL2BjYEjgFOjIi7DbZIkiQNBXOkJLVkJCtPmXlyZl6RmWOZeRJwIfDg7uUiYklELIuIZf0vpSRJ/WeOlKT2jGTlKSKeERHnRcT1EXE9sAuwafdymbk0M/fIzD36X0pJkvrPHClJ7Zk36AJMVkRsA3wC2B84IzNXRsR5QAy2ZJIkDZY5UpLaNYpnntYFErgGICIOp7SqSZI025kjJalFI1d5yszfAMcDZwBXAfcDfjbQQkmSNATMkZLUrsjMQZehLyKitQ/a5jaMaKt+Ozu+d2m2y0y7a2lCbebINnsM3rFieStxF8wbuasagDb/Z4DMsdZiS4MylRw5cmeeJEmSJGkQrDxJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJamA0h5MZMosXr99a7BtvvaWVuBssXtxKXEmS7qy9gfzmhINJSuovzzxJkiRJUgNWniRJkiSpAStPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgMTVp4i4vCI+EbH4wsj4uSOx5dFxO4RsVdEnB0RN9S/e3Us8+OIODYifh4RN0fENyJik4j4fETcWJe/Z8fyH6hxb4yIcyJi747X3hwRX46Iz0XETRHx64jYoxcbQ5KkpsyPkjT7NDnzdBqwd0TMiYgtgQXAngARsR2wHvAn4FvAB4FNgPcC34qITTriPBl4OrAVsD1wBvBp4C7A/wL/1rHs2cDu9bUvACdHxKKO1x8LfAnYCPg68KHmH1mSpJ4wP0rSLDNh5SkzLwZuohysHwZ8F7giInYG9gFOBx4DXJiZJ2Tmisz8IvBb4JCOUJ/OzIsy8wbg28BFmfn9zFwBnAzcv+M9T8zM62qs44GFwE4dsX6amadm5krgBGC31ZU9IpZExLKIWNZwe0iS1Mgo50cwR0rSVMxruNxpwL7ADvX+9ZTEsGd9vCXwx651/khpRRt3Vcf9W1fzeL3xBxFxJPCcGjeBDYBNO5a/suP+LcCiiJhXE83fZeZSYGmNmRN/TEmSJmUk8yOYIyVpKpoOGDGeHPau90+jJId96v0rgG261rkH8OfJFqj23z4KOAzYODM3Am4AYrKxJElqmflRkmaRyVSe9gMWZ+bllK4IB1H6b58LnArsGBFPjYh5EfEvwH2Ab06hTOsDK4BrgHkR8SZKy5okScPG/ChJs0ijylNm/h64mZIUyMwbgYuBn2Xmysy8DjgYOAK4jtIydnBmXjuFMn0X+A7we0rXhtuAy6YQR5KkVpkfJWl2iczZ0c25zf7cixatN/FCU3T1/13TStwNFi9uJa6k4ZKZdunShEb1mqcVK1e2Enfe3LmtxG1bRHvTd2aOtRZbGpSp5EgnyZUkSZKkBqw8SZIkSVIDVp4kSZIkqQErT5IkSZLUgANGzFI33npra7E3WLxOa7HLnJCjZf78ha3FXr789tZij662xkcYvX0PHDBCzZgj76ytgShgdAejkGYiB4yQJEmSpJZYeZIkSZKkBqw8SZIkSVIDVp4kSZIkqQErT5IkSZLUgJUnSZIkSWrAypMkSZIkNTB0laeIuDkitlvL65dGxAH9LJMkScPAHClJgzVv0AXolpnrjd+PiM8Al2fmGwZXIkmShoM5UpIGa+jOPEmSJEnSMOpb5SkiDo+Ib3Q8vjAiTu54fFlE7B4RGRE7RMQS4GnAUbWbwjc6wu0eEb+MiBsi4qSIWNSvzyFJUq+ZIyVpNPTzzNNpwN4RMScitgQWAHsC1P7b6wG/HF84M5cCnwfelZnrZeYhHbEOAw4CtgV2BZ7Vl08gSVI7zJGSNAL6ds1TZl4cETcBuwM7At+ltI7tTEkQp2fmWEQ0CffBzLwCoLa27b66hWrL3JJelF+SpLaYIyVpNPR7wIjTgH2BHer964F9KInhtEnEubLj/i3AlqtbqLbMLQWIiJx8cSVJ6htzpCQNuX4PGDGeGPau90+jJIZ9WH1i8GAuSZotzJGSNOQGUXnaD1icmZcDp1P6ZW8CnLua5a8C1jifhSRJM4g5UpKGXF8rT5n5e+BmSkIgM28ELgZ+lpkrV7PKJ4H7RMT1EfG1/pVUkqT+MkdK0vCLzNlx1t/+3Hd24623thZ7g8XrtBZ7FHupzJ+/sLXYy5ff3lrs0dXogvopGL19DyAz29ogmkHMkXe2YuXq6qq9MW/u3NZiS5qcqeRIJ8mVJEmSpAasPEmSJElSA1aeJEmSJKkBK0+SJEmS1MCsGjBi7tx25gQeGxtrJS5AZjuxI9qrNx9/4ldai/26w5/WWuzb72hvEI22tPk9RrQ3zsCCBYtbi32vez2wlbi/+tXprcSF9rb12NhKB4xQIw4Y0a3Nn017m/pvt93WWux1Fy1qLbbubM6c9gYVGRtrbzCUUeSAEZIkSZLUEitPkiRJktSAlSdJkiRJasDKkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpAStPkiRJktTASFaeIuK1EXFRRNwUEb+JiEMHXSZJkoaBOVKS2jNv0AWYoouAvYErgScBJ0bEDpn5l86FImIJsGQA5ZMkaVDMkZLUkpE885SZJ2fmFZk5lpknARcCD17Nckszc4/M3KP/pZQkqf/MkZLUnpGsPEXEMyLivIi4PiKuB3YBNh10uSRJGjRzpCS1Z+S67UXENsAngP2BMzJzZUScB8RgSyZJ0mCZIyWpXaN45mldIIFrACLicEqrmiRJs505UpJaNHKVp8z8DXA8cAZwFXA/4GcDLZQkSUPAHClJ7Rq5bnsAmXk0cPSgyyFJ0rAxR0pSe0buzJMkSZIkDYKVJ0mSJElqwMqTJEmSJDVg5UmSJEmSGojMHHQZ+iIiZscHHQIR7dXJb19+R2uxF8wbyfFTRlKb+0jmWGuxR1FmOr+PJmSOnBna/J8uwkOJZp6p5EjPPEmSJElSA1aeJEmSJKkBK0+SJEmS1ICVJ0mSJElqwMqTJEmSJDVg5UmSJEmSGmit8hQRl0bEAW3FlyRpFJkfJWl0eeZJkiRJkhoYqspTRDhLqSRJXcyPkjQcWq88RcSDI+KMiLg+Iv4SER+KiAUdr2dEvDgiLgQurM8dVZe9IiKeW5fZob62MCLeExF/ioirIuJjEbG47c8hSVIvmR8lafT048zTSuCVwKbAnsD+wIu6lnk88BDgPhFxEPAq4ABgB2DfrmXfCewI7F5f3wp4U0tllySpLeZHSRoxrVeeMvOczDwzM1dk5qXAx4F9uhZ7R2b+NTNvBQ4DPp2Zv87MW4A3jy8UEQEsAV5Zl78JeDvw5NW9d0QsiYhlEbGs959MkqSpG2R+rOuYIyVpklrvQx0ROwLvBfYA1qnveU7XYpd13N8SWLaG1zarMc4peaK8BTB3de+dmUuBpbUcObVPIElS7w0yP4I5UpKmoh/d9j4K/Ba4V2ZuALyeckDv1HnQ/guwdcfju3fcvxa4FbhvZm5Ubxtm5notlFuSpDaZHyVpxPSj8rQ+cCNwc0TsDLxwguW/DBweEfeOiHWAN46/kJljwCeA90XE5gARsVVEHNhO0SVJao35UZJGTD8qT0cCTwVuohzYT1rbwpn5beCDwI+APwBn1pdur39fM/58RNwIfB/YqffFliSpVeZHSRoxkTnc3Zwj4t7Ar4CFmbliGnGG+4POIBHt1clvX35Ha7EXzHMalX5pcx8pDfAal5nd3cA0Q/QqP9ZY5sgZoM3/6TqupZNmjKnkyKGaJHdcRBxa56vYGDgO+MZ0E4MkSaPO/ChJgzWUlSfg+cDVwEWUeTAm6gcuSdJsYH6UpAEa+m57vWKXhP6x254mYre9/rHbnpowR84MdtuTJmfGdNuTJEmSpGFj5UmSJEmSGphF/ZSCefPmtxJ5xYr2upK1Zc6cNU46P23z5y1oLfa229y7tdhtWbx4/dZi33rrza3FbtPHvn5qa7Gff8hBrcRt83tcvvz2iReaghUrlrcSVzNTW3lhbGxlK3Hb1V4XtTlz2mu33nCDTVuLPXduO/8yrlzZ3v7R5rZua3sAfH3ZWa3FftRuu7USdxT/r7xj+W1TWs8zT5IkSZLUgJUnSZIkSWrAypMkSZIkNWDlSZIkSZIasPIkSZIkSQ1YeZIkSZKkBoam8hQRn4mIYwddDkmSho05UpKGw9BUniRJkiRpmPWk8hQRd+1FnLXE3ywi2puxTpKklpgjJWnmmHLlKSI2iogXRsRZwGfqcxkRO3Qs8/duBhGxb0RcHhFHRMTVEfGXiDh8DbHXj4gfRcQHa0J4NnBJRBwTEdtOtcySJPWDOVKSZqZJVZ4iYk5EPDIivgj8EXgk8DbgsQ1DbAFsCGwFPAf4cERs3PUemwA/AH6WmS/L4jjgycDmwLKaNJ4eEetMpvySJLXFHClJM1/jylNEvAS4FHgncAawfWYempmnZObyhmGWA2/JzOWZeSpwM7BTx+tbAqcBJ2fmGzpXzMwzM/OFdZmPAk8BLo+I/1hLmZdExLKIWAbZsIiSJE3O6OdISVITkznztC2wMXAecD5w3RTe77rMXNHx+BZgvY7HjwEWAx9bU4DMvB34ZS3HHcAua1l2aWbukZl7gN3BJUmtGfEcKUlqonHlKTOPALYHfgX8O6V/9Vsj4l4di90CdHYT2GKS5fkE8B3g1IhYt/OFiNgkIl5S+4//EJgL7JeZD53ke0iS1FPmSEmaHSZ1zVNmXp2Z783MXYEnAhsBZ0TEp+oi5wFPjYi5EXEQsM8UyvQS4HfANyJiMUBEPIfSHWIf4Bjg7pn5msz83ynElySp58yRkjTzTXm0vcw8JzNfSulfPd6F4OXAIcD1wNOAr00hbgJLgMuBUyJiEaX/+DaZ+aTM/FZmrpxquSVJaps5UpJmpnnTDZCZdwBn1fvLgPuuYbkfA1t3PXfPjvvP6rg/BjyjY9HfTLeckiT1mzlSkmaWnkySK0mSJEkznZUnSZIkSWrAypMkSZIkNWDlSZIkSZIasPIkSZIkSQ1EGfV05ouIjGinrlgGPtKo22yze7QS99hPLW0lLsDL/t8TWot9xx23tRZ7vfU2ai327bff0krcuXPntxIXYMWKO1qLOzY2Fq0E14wSMSfnzWtnH29r/27TnDlzW4s9NjaaI8lvsMGmrcR96yc+2UpcgKOe8ZTWYi9ffntrsRctWnfihTRtt912MytXrpx0jvTMkyRJkiQ1YOVJkiRJkhqw8iRJkiRJDVh5kiRJkqQGrDxJkiRJUgNWniRJkiSpgaGqPEXEmyPixHr/nmV48Zg36HJJkjRI5kdJGg5DVXmSJEmSpGFl5UmSJEmSGuhZ5SkiDo+Ib3Q8vjAiTu54fFlE7B4RH6j3b4yIcyJi74bxnxgRl0bELhGxKCJOjIjrIuL6iDg7Iu7aq88iSVKvmB8laebo5Zmn04C9I2JORGwJLAD2BIiI7YD1gF8CZwO7A3cBvgCcHBGL1hY4Ig4HjgMOyMxfAc8ENgTuDmwCvAC4tYefRZKkXjE/StIM0bPKU2ZeDNxEOfA/DPgucEVE7AzsA5yemWOZeWJmXpeZKzLzeGAhsNNaQr8CeDWwb2b+oT63nJIUdsjMlZl5Tmbe2L1iRCyJiGURsaxXn1OSpMkYxvwI3Tkye/JZJWmm6/U1T6cB+1KSw2nAjymJYZ/6mIg4MiL+NyJuiIjrKS1km64l5quBD2fm5R3PnUBJPl+KiCsi4l0RMb97xcxcmpl7ZOYe0/9okiRN2VDlR+jOkTG9TydJs0Rblae96/3T6EgOtf/2UcBhwMaZuRFwA2s/aj8SeENEPHH8icxcnpnHZOZ9gL2Ag4Fn9PizSJLUK+ZH/f/27j1Ir7q+4/j7JujgrQAAEnpJREFUk6yBxMQkDvECCBYUFEVxGrww06JFRa037LRirR21NsXqWBXrBS31jrQq1GKLoaPOSL2OM1rxWq/FCmKwoMVa5Wo0KAmQkAjktt/+8Typ6zY0Z5Pnt7tn837N7GT3Oef5PN+zk93vfs85zzmS5oAWw9PjgIXDPWEXA09icArBfwBLgB3AemAsyZnAPfaQedUw471Jng6Q5HFJjk0yH7iNwWkK4yPeFkmSRsX+KElzwEhvsFdVP0qyhUFToKpuS3ItsL6qdib5IvAF4EfAL4FzgLUdcq9M8lTgs0m2A8uA84FDgS3AxxicqiBJ0qxjf5SkuWHkdyevqvtO+nrlhM93Ai8cfuzyNxOWv3HC59cz4XSFqloDTLzc6kdGVbMkSa3ZHyWp/7xJriRJkiR14PAkSZIkSR04PEmSJElSBw5PkiRJktRBqvaPu4on2T82VHvtXvc6vEnuTTf9pEkuwCOOO6lZ9hVXfrVZdlW7Kyef+5FPNcl9+XNOaZLbVlFV3v1UezRv3rwaG1vQJHv79q1NcltavHh5s+wtW25tlt3yZscnnPDMJrnf/vZnmuQCHHHEcc2y1637cbPsu93twGbZj3jE45vkXnzxJ5rkQru/GXbu3LFXPdIjT5IkSZLUgcOTJEmSJHXg8CRJkiRJHTg8SZIkSVIHDk+SJEmS1IHDkyRJkiR14PAkSZIkSR04PEmSJElSB70dnpI8P8k3Z7oOSZJmG3ukJLXR2+FJkiRJkqbTtA1PSa5P8rokP0hya5IPJDkwyfIkFyVZP3z8oiSHTnje85Ncm2RzkuuSPDfJg4Hzgcck2ZJk43RthyRJo2aPlKR+mO4jT88FTgaOBI4C3jCs4QPA4cBhwB3AeQBJ7g68B3hyVS0BTgCuqKr/Ak4DLqmqxVW1bHcvlmRVkjVJ1rTdLEmS9tmM9ciqartlkjRHTPfwdF5Vra2qW4C3Ac+pqpur6pNVdXtVbR4+fuKE54wDD02ysKpurKqrur5YVa2uqpVVtXK0myFJ0sjNWI9MMtotkaQ5arqHp7UTPr8BODjJoiTvS3JDktuAfwOWJZlfVb8Ens1gD9qNST6b5EHTXLMkSdPBHilJs9x0D0/3m/D5YcA64HTgaOBRVXUP4LeHywNQVV+sqicA9wV+CFwwXO45BpKkucQeKUmz3HQPTy9JcmiSewKvBz4GLGFwDvfG4eN/vWvlJPdO8ozhed1bgS0MTlEA+AVwaJIF07oFkiS1YY+UpFluuoenDwNfAq4FrgHeCpwLLAQ2AJcCX5hU3ysZ7H27hcF53i8eLvsqcBXw8yQbpqN4SZIaskdK0iw3Ns2v952qOmvSY7cDj5302PuG/97Ir78x9n9V1Tbgd0danSRJM8ceKUmznDfJlSRJkqQOHJ4kSZIkqYNpO22vqu4/Xa8lSVKf2CMlqR888iRJkiRJHaRq/7gVRJL9Y0O1197wzn9qkvvWV72oSS7AkiX3bJZ94IGLm2Vv2nRTs+yHPexxTXIvv/yLTXIBqsb3vNJeZ1eahWvOGPTIVv9VbL/T5T73OaJZ9i9/ubFJ7ubNtzTJBRgba3el/nnz2h1/OOv9/9ws+12vfk2T3J///LomuQDj4616ZO1Vj/TIkyRJkiR14PAkSZIkSR04PEmSJElSBw5PkiRJktSBw5MkSZIkdeDwJEmSJEkdODxJkiRJUgcOT5IkSZLUgcOTJEmSJHXQy+EpyWuTXJNkc5IfJDllpmuSJGk2sEdKUju9HJ6Aa4DfApYCbwIuTHLfmS1JkqRZwR4pSY30cniqqk9U1bqqGq+qjwE/Bh45eb0kq5KsSbJm+quUJGn62SMlqZ1eDk9J/jjJFUk2JtkIPBQ4aPJ6VbW6qlZW1crpr1KSpOlnj5SkdsZmuoCpSnI4cAFwEnBJVe1McgWQma1MkqSZZY+UpLb6eOTp7kAB6wGSvIDBXjVJkvZ39khJaqh3w1NV/QB4F3AJ8AvgWODfZ7QoSZJmAXukJLXVu9P2AKrq9cDrZ7oOSZJmG3ukJLXTuyNPkiRJkjQTHJ4kSZIkqQOHJ0mSJEnqwOFJkiRJkjpweJIkSZKkDnp5tT3tv97z8U83y375qc9qlt3K5s23NMu+444tzbJPPPHZzbK/8pUPNUr2HqOa62qmC9A+OuaYE5plf+MbH22W3cqOHduaZS9YcGCz7LG7tfvzfN26q5tl7y888iRJkiRJHTg8SZIkSVIHDk+SJEmS1IHDkyRJkiR14PAkSZIkSR04PEmSJElSB82GpyTXJ3l8q3xJkvrI/ihJ/eWRJ0mSJEnqYFYNT0m8aa8kSZPYHyVpdmg+PCV5ZJJLkmxMcmOS85IsmLC8krwkyY+BHw8fe/Vw3XVJXjRc5wHDZQckeWeSnyT5RZLzkyxsvR2SJI2S/VGS+mc6jjztBF4BHAQ8BjgJ+PNJ6zwTeBRwTJInAa8EHg88AHjspHXfARwFHDdcfghwZqPaJUlqxf4oST3TfHiqqsur6tKq2lFV1wPvA06ctNpZVXVLVd0B/AHwgaq6qqpuB964a6UkAVYBrxiuvxl4O3Dq7l47yaoka5KsGf2WSZK092ayPw6fY4+UpClqfg51kqOAdwMrgUXD17x80mprJ3x+MLDmLpatGGZcPugTg5cA5u/utatqNbB6WEft3RZIkjR6M9kfwR4pSXtjOk7b+0fgh8ADq+oewBkMfqFPNPGX9o3AoRO+vt+EzzcAdwAPqaplw4+lVbW4Qd2SJLVkf5SknpmO4WkJcBuwJcmDgBfvYf2PAy9I8uAki4C/2rWgqsaBC4BzktwLIMkhSU5uU7okSc3YHyWpZ6ZjeHoV8IfAZga/2D/2/61cVZ8H3gN8DbgauHS4aOvw39fsejzJbcCXgaNHX7YkSU3ZHyWpZ5q956mq7j/hywdNWnzmhPUmn6JAVZ0FnAWQ5MHAOIPTFaiqOxmc2nDGaCuWJKk9+6Mk9desuknuLklOGd6vYjlwNvCZqtox03VJkjST7I+SNLNm5fAE/BlwE3ANg/tg7Ok8cEmS9gf2R0maQc0vVb43qupJM12DJEmzjf1RkmbWbD3yJEmSJEmzisOTJEmSJHWQqv3jpuLePX2u+D8XnxqZxYuXNcndsuXWJrkASbv9H4sWLWmW/fXvfbdZ9vFHHtkkd2xsQZNcgB07tjXL3t0V26TJ7JHak6VLVzTJ3bRpfZNcaNsjV6y4355X2ksXfuWiZtlPPPbYJrktv9eD29i1yp56j/TIkyRJkiR14PAkSZIkSR04PEmSJElSBw5PkiRJktSBw5MkSZIkdeDwJEmSJEkdjHR4SnJ0kiuSbE7yslFmS5LUZ/ZISeq/sRHnvRr4WlUdN+JcSZL6zh4pST036tP2DgeuGnGmJElzgT1SknpuZMNTkq8CjwPOS7JleHrCO5P8JMkvkpyfZOFw3ccm+WmS05PclOTGJC+YkLUwybuS3JBkU5JvTnjuo5N8K8nGJFcmeeyotkGSpBbskZI0N4xseKqq3wEuBl5aVYuB04CjgOOABwCHAGdOeMp9gKXDx/8EeG+S5cNl7wR+EzgBuCeDUx3GkxwCfBZ46/DxVwGfTLJidzUlWZVkTZI1o9pOSZKmyh4pSXNDk6vtJQmwCnhFVd1SVZuBtwOnTlhtO/DmqtpeVZ8DtgBHJ5kHvBD4i6r6WVXtrKpvVdVW4I+Az1XV56pqvKr+FVgDPGV3dVTV6qpaWVUrW2ynJElTZY+UpP4a9QUjdlkBLAIuH/QIAALMn7DOzVW1Y8LXtwOLgYOAA4FrdpN7OPD7SZ424bG7AV8bUd2SJLVmj5Sknmo1PG0A7gAeUlU/24vn3gkcCVw5adla4ENV9af7XqIkSTPCHilJPdXktL2qGgcuAM5Jci+AJIckObnjc98PvDvJwUnmJ3lMkgOAC4GnJTl5+PiBwzfWHtpiOyRJGjV7pCT1V5Phaeg1wNXApUluA74MHN3xua8Cvg98B7gFOBuYV1VrgWcAZwDrGexl+0vabockSaNmj5SkHkpVzXQN0yLJ/rGhc172vMpeWrx4WZPcLVtubZILMHjveBuLFi1plv317323WfbxRx7ZJHdsbEGTXIAdO7Y1y66qdj80mjPskdqTpUt3e9HGfbZp0/omudC2R65Ycb9m2Rd+5aJm2U889tgmuS2/14MD7q2yp94j3RslSZIkSR04PEmSJElSBw5PkiRJktSBw5MkSZIkdbCfXTCi1fum94/v4Vx32svPapJ7/rmva5ILcNRRxzfLbnnBiGuvmXx7mtHZtn1rk9wa39kkF2DrtjsbJZcXjFAnSWr+/Da3fmz5d8Z4s5/Llj82/fyb4dGPfkaT3O9857NNcgEOO+yYZtnLl9+nWfbVV7e7qNLChYub5M6bN3/PK+2lDRt+2iR3x45tjI+Pe8EISZIkSWrB4UmSJEmSOnB4kiRJkqQOHJ4kSZIkqQOHJ0mSJEnqwOFJkiRJkjpweJIkSZKkDhyeJEmSJKkDhydJkiRJ6sDhSZIkSZI6cHiSJEmSpA7GZrqAlpKsAlbNdB2SJM029khJmro5PTxV1WpgNUCSmuFyJEmaNeyRkjR1nrYnSZIkSR04PEmSJElSB3NieEry+SRnzHQdkiTNJvZHSRqtOfGep6p68kzXIEnSbGN/lKTRmhNHniRJkiSpNYcnSZIkSerA4UmSJEmSOnB4kiRJkqQOHJ4kSZIkqYNU7R83FR8bW1BLlx7UJPv22zc3yQWYP39+k9ytW+9oktva/Hltvh8ABxywqEnubZtvbpILsGzZvZtlH3PMCc2yN2z4abPstWt/2CR38eLlTXIBNm26qUnu9u1bGR8fT5NwzSlJKmmzP/XEE09tkgtw6aX/0iT3zju3NMkFGBtb0Cx7x45tzbKPOOLhTXKvvfZ7TXIB5s1rd4zgOc97bbPsj154drPsnTt3NMmdP7/dBbzHx8eb5FaNU1VT7pEeeZIkSZKkDhyeJEmSJKkDhydJkiRJ6sDhSZIkSZI6cHiSJEmSpA4cniRJkiSpA4cnSZIkSerA4UmSJEmSOhjZ8JSk2d06W2ZLktSaPVKS5oZ9Gp6SLEvy4iSXAR8cPnZwkk8mWZ/kuiQvm7D+AUnOTbJu+HFukgOGyw5KclGSjUluSXJxfnW78w8muSzJaUmW7UvNkiRNB3ukJM09Ux6eksxL8sQkHwFuAJ4IvA14+vAX+WeAK4FDgJOAlyc5efj01wOPBo4DHg48EnjDcNnpwE+BFcC9gTOAGi57OvB24GTghiQfTvKECY1DkqQZZ4+UpLltSr9Yk7wUuB54B3AJcGRVnVJVn66q7cDxwIqqenNVbauqa4ELgFOHEc8F3lxVN1XVeuBNwPOGy7YD9wUOr6rtVXVxVRXA8OtPVdUpwJHApcDZwPXDmu6q3lVJ1iRZUzU+lU2VJGlK+twjR/udkKS5a6p7pX4DWA5cwWDP2c2Tlh8OHDw8rWBjko0M9o7tOh/7YAZ74na5YfgYwN8CVwNfSnJtktfeRQ03A98b1rB8WNNuVdXqqlpZVSvdASdJaqy3PbLrBkrS/m5KE0VVnc5gr9Z/An8PXJfkLUkeOFxlLXBdVS2b8LGkqp4yXL6OQfPY5bDhY1TV5qo6vaqOYHAKwiuTnLRrxSQPTPIW4Drg74DvA0cMa5IkaUbZIyVp7pvy4Zjh6QTvrqqHAb8HLAMuSfJ+4DJgc5LXJFmYZH6ShyY5fvj0jwBvSLIiyUHAmcCFAEmemuQBSQJsAnYC48Nl72dwCsQy4FlV9fCqOmd4WoMkSbOCPVKS5raxfXlyVV0OXJ7kdOC4qtqZ5KnAuxjs/ToA+G9+9YbXtwL3YHBKAcAnho8BPBA4j8GbYW8F/qGqvjZcdj5wWlVt25d6JUmaLvZISZp79ml42mX4C/uy4efrgOfcxXp3Ai8bfkxedg5wzl0877JR1ClJ0nSzR0rS3OFVFCRJkiSpA4cnSZIkSerA4UmSJEmSOnB4kiRJkqQOHJ4kSZIkqYNU1UzXMC2SrOfX79y+JwcBGxqU0iq3r9l9rLmv2X2sua/Zs6Xmw6tqRaM6NIdMsUfOlv/fsyW7jzW3zO5jzX3N7mPNsyl7r3rkfjM8TVWSNVW1si+5fc3uY819ze5jzX3N7mPNUld9/f/tz/v0ZPex5r5m97HmPmfv4ml7kiRJktSBw5MkSZIkdeDwdNdW9yy3r9l9rLmv2X2sua/ZfaxZ6qqv/7/9eZ+e7D7W3NfsPtbc52zA9zxJkiRJUiceeZIkSZKkDhyeJEmSJKkDhydJkiRJ6sDhSZIkSZI6cHiSJEmSpA7+BwAZlrDeidXnAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 1080x1800 with 8 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ViV7xRAAmv0j",
        "outputId": "6375fe26-af25-47ac-eb4c-d57c5dfc255b"
      },
      "source": [
        "example_idx = 6\n",
        "\n",
        "src = vars(valid_data.examples[example_idx])['src']\n",
        "trg = vars(valid_data.examples[example_idx])['trg']\n",
        "\n",
        "print(f'src = {src}')\n",
        "print(f'trg = {trg}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "src = ['ein', 'brauner', 'hund', 'rennt', 'dem', 'schwarzen', 'hund', 'hinterher', '.']\n",
            "trg = ['a', 'brown', 'dog', 'is', 'running', 'after', 'the', 'black', 'dog', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8TBKafFwm_-h",
        "outputId": "5f2c0b25-875f-4a53-ebf2-49075792ee8e"
      },
      "source": [
        "translation, attention = translate_sentence(src, SRC, TRG, model, device)\n",
        "\n",
        "print(f'predicted trg = {translation}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "predicted trg = ['a', 'brown', 'dog', 'running', 'the', 'black', 'dog', '.', '<eos>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xgzNlSVZnDHi",
        "outputId": "7d84cd30-4362-4a71-ba44-366be3d315f8"
      },
      "source": [
        "example_idx = 10\n",
        "\n",
        "src = vars(test_data.examples[example_idx])['src']\n",
        "trg = vars(test_data.examples[example_idx])['trg']\n",
        "\n",
        "print(f'src = {src}')\n",
        "print(f'trg = {trg}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "src = ['eine', 'mutter', 'und', 'ihr', 'kleiner', 'sohn', 'genießen', 'einen', 'schönen', 'tag', 'im', 'freien', '.']\n",
            "trg = ['a', 'mother', 'and', 'her', 'young', 'song', 'enjoying', 'a', 'beautiful', 'day', 'outside', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oCgtHAP-nH4k",
        "outputId": "266728d3-e9c3-4cfb-ba72-cbd4260fccb9"
      },
      "source": [
        "translation, attention = translate_sentence(src, SRC, TRG, model, device)\n",
        "\n",
        "print(f'predicted trg = {translation}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "predicted trg = ['a', 'mother', 'and', 'little', 'mother', 'are', 'enjoying', 'a', 'beautiful', 'day', 'outside', '.', '<eos>']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6xAlG1l-nRSt"
      },
      "source": [
        "## BLEU SCORE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s74NuPyXnKZH"
      },
      "source": [
        "from torchtext.data.metrics import bleu_score\n",
        "\n",
        "def calculate_bleu(data, src_field, trg_field, model, device, max_len = 50):\n",
        "    \n",
        "    trgs = []\n",
        "    pred_trgs = []\n",
        "    \n",
        "    for datum in data:\n",
        "        \n",
        "        src = vars(datum)['src']\n",
        "        trg = vars(datum)['trg']\n",
        "        \n",
        "        pred_trg, _ = translate_sentence(src, src_field, trg_field, model, device, max_len)\n",
        "        \n",
        "        #cut off <eos> token\n",
        "        pred_trg = pred_trg[:-1]\n",
        "        \n",
        "        pred_trgs.append(pred_trg)\n",
        "        trgs.append([trg])\n",
        "        \n",
        "    return bleu_score(pred_trgs, trgs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dIK2-GETnY9W",
        "outputId": "eb0eed27-7234-49d3-975d-b8f84c441b97"
      },
      "source": [
        "bleu_score = calculate_bleu(test_data, SRC, TRG, model, device)\n",
        "\n",
        "print(f'BLEU score = {bleu_score*100:.2f}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU score = 25.68\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kw5mAgeonZYj"
      },
      "source": [
        "## Appendix \n",
        "\n",
        "The calculate_bleu function above is unoptimized. Below is a significantly faster, vectorized version of it that should be used if needed. Credit for the implementation goes to @azadyasar."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ut3Z-k9nhYb"
      },
      "source": [
        "def translate_sentence_vectorized(src_tensor, src_field, trg_field, model, device, max_len=50):\n",
        "    assert isinstance(src_tensor, torch.Tensor)\n",
        "\n",
        "    model.eval()\n",
        "    src_mask = model.make_src_mask(src_tensor)\n",
        "\n",
        "    with torch.no_grad():\n",
        "        enc_src = model.encoder(src_tensor, src_mask)\n",
        "    # enc_src = [batch_sz, src_len, hid_dim]\n",
        "\n",
        "    trg_indexes = [[trg_field.vocab.stoi[trg_field.init_token]] for _ in range(len(src_tensor))]\n",
        "    # Even though some examples might have been completed by producing a <eos> token\n",
        "    # we still need to feed them through the model because other are not yet finished\n",
        "    # and all examples act as a batch. Once every single sentence prediction encounters\n",
        "    # <eos> token, then we can stop predicting.\n",
        "    translations_done = [0] * len(src_tensor)\n",
        "    for i in range(max_len):\n",
        "        trg_tensor = torch.LongTensor(trg_indexes).to(device)\n",
        "        trg_mask = model.make_trg_mask(trg_tensor)\n",
        "        with torch.no_grad():\n",
        "            output, attention = model.decoder(trg_tensor, enc_src, trg_mask, src_mask)\n",
        "        pred_tokens = output.argmax(2)[:,-1]\n",
        "        for i, pred_token_i in enumerate(pred_tokens):\n",
        "            trg_indexes[i].append(pred_token_i)\n",
        "            if pred_token_i == trg_field.vocab.stoi[trg_field.eos_token]:\n",
        "                translations_done[i] = 1\n",
        "        if all(translations_done):\n",
        "            break\n",
        "\n",
        "    # Iterate through each predicted example one by one;\n",
        "    # Cut-off the portion including the after the <eos> token\n",
        "    pred_sentences = []\n",
        "    for trg_sentence in trg_indexes:\n",
        "        pred_sentence = []\n",
        "        for i in range(1, len(trg_sentence)):\n",
        "            if trg_sentence[i] == trg_field.vocab.stoi[trg_field.eos_token]:\n",
        "                break\n",
        "            pred_sentence.append(trg_field.vocab.itos[trg_sentence[i]])\n",
        "        pred_sentences.append(pred_sentence)\n",
        "\n",
        "    return pred_sentences, attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TVnz8bTznisn"
      },
      "source": [
        "from torchtext.data.metrics import bleu_score\n",
        "\n",
        "def calculate_bleu_alt(iterator, src_field, trg_field, model, device, max_len = 50):\n",
        "    trgs = []\n",
        "    pred_trgs = []\n",
        "    with torch.no_grad():\n",
        "        for batch in iterator:\n",
        "            src = batch.src\n",
        "            trg = batch.trg\n",
        "            _trgs = []\n",
        "            for sentence in trg:\n",
        "                tmp = []\n",
        "                # Start from the first token which skips the <start> token\n",
        "                for i in sentence[1:]:\n",
        "                    # Targets are padded. So stop appending as soon as a padding or eos token is encountered\n",
        "                    if i == trg_field.vocab.stoi[trg_field.eos_token] or i == trg_field.vocab.stoi[trg_field.pad_token]:\n",
        "                        break\n",
        "                    tmp.append(trg_field.vocab.itos[i])\n",
        "                _trgs.append([tmp])\n",
        "            trgs += _trgs\n",
        "            pred_trg, _ = translate_sentence_vectorized(src, src_field, trg_field, model, device)\n",
        "            pred_trgs += pred_trg\n",
        "    return pred_trgs, trgs, bleu_score(pred_trgs, trgs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AiVZnl5LnkRl"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}