vi2en.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "vi2en.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "colab_type": "code",
        "id": "QQTKmCqMLfv2",
        "outputId": "c254b3b0-09e4-4abb-945c-290d2c4c1a4f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "Tvqp0ME6R1u1",
        "outputId": "2029d7ab-415c-4616-f196-f4cc3499ddc2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "!pip3 install sacrebleu"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: sacrebleu in /usr/local/lib/python3.6/dist-packages (1.2.12)\n",
            "Requirement already satisfied: typing in /usr/local/lib/python3.6/dist-packages (from sacrebleu) (3.6.6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "O47m-P3N1DJw",
        "outputId": "c6043a9c-bdab-464d-af84-94891877e34d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "from os import path\n",
        "from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
        "platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
        "\n",
        "accelerator = 'cu80' if path.exists('/opt/bin/nvidia-smi') else 'cpu'\n",
        "\n",
        "!pip install -q http://download.pytorch.org/whl/{accelerator}/torch-0.4.1-{platform}-linux_x86_64.whl torchvision\n",
        "import torch\n",
        "print(torch.__version__)\n",
        "print(torch.cuda.is_available())"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.4.1\n",
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "xGw8DDZT0d5X",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from __future__ import unicode_literals, print_function, division\n",
        "from io import open\n",
        "import unicodedata\n",
        "import string\n",
        "import re\n",
        "import random\n",
        "import operator\n",
        "from torch.utils.data import Dataset\n",
        "from collections import Counter\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch import optim\n",
        "import torch.nn.functional as F\n",
        "from sacrebleu import corpus_bleu, TOKENIZERS, DEFAULT_TOKENIZER\n",
        "from torch.optim.lr_scheduler import ExponentialLR\n",
        "from torch.autograd import Variable\n",
        "import random\n",
        "from torch.nn import functional"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "EYDlc9JJ0d5Z",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "PAD_token = 0\n",
        "SOS_token = 1\n",
        "EOS_token = 2\n",
        "hidden_size = 1024\n",
        "dropout_p = 0.1\n",
        "teacher_forcing_ratio = 1\n",
        "BATCH_SIZE = 64\n",
        "MIN_LENGTH = 1\n",
        "MAX_LENGTH = 55\n",
        "source_vocab_size = 19000\n",
        "target_vocab_size = 22000\n",
        "n_layers = 4\n",
        "lr_rate_en = 0.0001\n",
        "lr_rate_de = 0.0005\n",
        "lr_decay = False\n",
        "gamma_encoder = 0.9\n",
        "gamma_decoder = 0.9\n",
        "n_epochs = 20\n",
        "plot_every = 100\n",
        "print_every = 100\n",
        "evaluate_every = 500\n",
        "attn_model = 'dot'\n",
        "Attention = True\n",
        "search_method = 'greedy'\n",
        "beam_size = 10\n",
        "n_best = 5\n",
        "dynamic_sentence_length = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UWhY1ARnf0mM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def filter_pairs(pairs):\n",
        "    filtered_pairs = []\n",
        "    for pair in pairs:\n",
        "        if len(pair[0].split()) >= MIN_LENGTH and len(pair[0].split()) <= MAX_LENGTH \\\n",
        "            and len(pair[1].split()) >= MIN_LENGTH and len(pair[1].split()) <= MAX_LENGTH:\n",
        "            filtered_pairs.append(pair)\n",
        "    return filtered_pairs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "0-6BiXr1MaHT",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def sequence_mask(sequence_length, max_len=None):\n",
        "    \"\"\"\n",
        "    Code paraphrased from \n",
        "    https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/masked_cross_entropy.py\n",
        "    \"\"\"\n",
        "    if max_len is None:\n",
        "        max_len = sequence_length.data.max()\n",
        "    batch_size = sequence_length.size(0)\n",
        "    seq_range = torch.arange(0, max_len).long()\n",
        "    seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len).contiguous()\n",
        "    seq_range_expand = seq_range_expand.to(device)\n",
        "    seq_length_expand = (sequence_length.unsqueeze(1)\n",
        "                         .expand_as(seq_range_expand))\n",
        "    return seq_range_expand < seq_length_expand\n",
        "\n",
        "\n",
        "\n",
        "def masked_cross_entropy(logits, target, length):\n",
        "    length = torch.LongTensor(length).to(device)\n",
        "    \"\"\"\n",
        "    Code paraphrased from \n",
        "    https://github.com/spro/practical-pytorch/blob/master/seq2seq-translation/masked_cross_entropy.py\n",
        "    \"\"\"\n",
        "    \n",
        "    \"\"\"\n",
        "    Args:\n",
        "        logits: A Variable containing a FloatTensor of size\n",
        "            (batch, max_len, num_classes) which contains the\n",
        "            unnormalized probability for each class.\n",
        "        target: A Variable containing a LongTensor of size\n",
        "            (batch, max_len) which contains the index of the true\n",
        "            class for each corresponding step.\n",
        "        length: A Variable containing a LongTensor of size (batch,)\n",
        "            which contains the length of each data in a batch.\n",
        "\n",
        "    Returns:\n",
        "        loss: An average loss value masked by the length.\n",
        "    \"\"\"\n",
        "\n",
        "    logits_flat = logits.view(-1, logits.size(-1))\n",
        "    log_probs_flat = functional.log_softmax(logits_flat, dim=1)\n",
        "    target_flat = target.view(-1, 1)\n",
        "    losses_flat = -torch.gather(log_probs_flat, dim=1, index=target_flat)\n",
        "    losses = losses_flat.view(*target.size())\n",
        "    mask = sequence_mask(sequence_length=length, max_len=target.size(1))\n",
        "    losses = losses * mask.float()\n",
        "    loss = losses.sum() / length.float().sum()\n",
        "    return loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "OESV2zjg0d5c"
      },
      "cell_type": "markdown",
      "source": [
        "## Loading Data"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "X0G2eDBP0d5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "4403fcd5-4213-459d-a0cd-022163991662"
      },
      "cell_type": "code",
      "source": [
        "class Lang:\n",
        "    '''\n",
        "    Part of the codes are paraphrased from\n",
        "    https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html\n",
        "    '''\n",
        "    def __init__(self, name):\n",
        "        self.name = name\n",
        "        self.word2index = {}\n",
        "        self.word2count = {}\n",
        "        self.index2word = {0: \"<PAD>\", 1: \"<SOS>\", 2:\"<EOS>\", 3:\"<UNK>\"}\n",
        "        self.n_words = 4  # Count SOS and EOS\n",
        "\n",
        "    def addSentence(self, sentence):\n",
        "        for word in sentence.split(' '):\n",
        "            self.addWord(word)\n",
        "\n",
        "    def addWord(self, word):\n",
        "        if word not in self.word2index:\n",
        "            self.word2index[word] = self.n_words\n",
        "            self.word2count[word] = 1\n",
        "            self.index2word[self.n_words] = word\n",
        "            self.n_words += 1\n",
        "        else:\n",
        "            self.word2count[word] += 1\n",
        "\n",
        "\n",
        "other_punctuations = string.punctuation.replace('!','').replace('.','').replace('?','').replace(',','').replace('-','')\n",
        "\n",
        "def normalizeEnString(s):\n",
        "#     s = unicodeToAscii(s.strip())\n",
        "    s = s.replace(\"&apos\", \"\").replace(\"&quot\",\"\")\n",
        "#    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
        "    s = re.sub(r\"[^a-zA-Z,.!?0-9]+\", r\" \", s)\n",
        "    s = re.sub( '\\s+', ' ', s).strip()\n",
        "    return s\n",
        "\n",
        "def normalizeViString(s):\n",
        "    s = s.replace(\"&apos\", \"\").replace(\"&quot\",\"\").replace(\"_\",\"\").replace('-','')\n",
        "    s = re.sub(r'[{}]'.format(other_punctuations), '', s)\n",
        "    s = re.sub( '\\s+', ' ', s).strip()\n",
        "    return s\n",
        "\n",
        "\n",
        "print(normalizeEnString(\"It &apos;s very pretty , and it has rapidly started to overgrow the \\\n",
        "                  once very rich biodiversity of the northwestern Mediterranean .\"))\n",
        "normalizeViString('húng_ta đã đạt được điều này qua công_nghệ đến_mức mili - giây . và điều này cho_phép Peter nhìn_thấy não_bộ anh ấy dưới thời_gian thực khi anh ta ở trong máy_quét .')               "
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "It s very pretty , and it has rapidly started to overgrow the once very rich biodiversity of the northwestern Mediterranean .\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'húngta đã đạt được điều này qua côngnghệ đếnmức mili giây . và điều này chophép Peter nhìnthấy nãobộ anh ấy dưới thờigian thực khi anh ta ở trong máyquét .'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "metadata": {
        "id": "qj85Ak3cpsf9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# open('/content/drive/My Drive/iwslt-vi-en/{}.tok.vi'.format('train')).read().split('\\n')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "sdkyHn8Ff0mX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "b6a92b5c-1215-4212-8ef0-b01c9870cf57"
      },
      "cell_type": "code",
      "source": [
        "def readLangs(lang1, lang2, data='train'):\n",
        "    '''\n",
        "    Part of the codes are paraphrased from\n",
        "    https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html\n",
        "    '''\n",
        "    #data: train/dev/test\n",
        "    print(\"Reading lines...\")\n",
        "    # Read the file and split into lines\n",
        "    zh_lines = open('/content/drive/My Drive/iwslt-vi-en/{}.tok.vi'.format(data)).read().split('\\n')\n",
        "    en_lines = open('/content/drive/My Drive/iwslt-vi-en/{}.tok.en'.format(data)).read().split('\\n')\n",
        "    # Split every line into pairs and normalize\n",
        "    pairs = [[normalizeViString(element[0]), normalizeEnString(element[1])] for element in zip(zh_lines, en_lines)]\n",
        "\n",
        "    input_lang = Lang(lang1)\n",
        "    output_lang = Lang(lang2)\n",
        "\n",
        "    return input_lang, output_lang, pairs\n",
        "\n",
        "def prepareData(lang1, lang2):\n",
        "    input_lang, output_lang, pairs = readLangs(lang1, lang2)\n",
        "    print(\"Read %s sentence pairs\" % len(pairs))\n",
        "    pairs = filter_pairs(pairs)\n",
        "    print(\"Trimmed to %s sentence pairs\" % len(pairs))\n",
        "    print(\"Counting words...\")\n",
        "    for pair in pairs:\n",
        "        input_lang.addSentence(pair[0])\n",
        "        output_lang.addSentence(pair[1])\n",
        "    print(\"Counted words:\")\n",
        "    print(input_lang.name, input_lang.n_words)\n",
        "    print(output_lang.name, output_lang.n_words)\n",
        "    return input_lang, output_lang, pairs\n",
        "\n",
        "def build_topwordVocab(lang, vocab_size):\n",
        "    print(\"Build vocabulary by top {} frequent word...\".format(vocab_size))\n",
        "    sorted_word2Count = sorted(lang.word2count.items(),\n",
        "        key=operator.itemgetter(1),\n",
        "        reverse=True)\n",
        "    sorted_words = [x[0] for x in sorted_word2Count[:vocab_size]]\n",
        "    \n",
        "    lang.word2index = {}\n",
        "\n",
        "    for ind, word in enumerate(sorted_words):\n",
        "            lang.word2index[word] = ind + 4\n",
        "    lang.index2word = {}\n",
        "    lang.index2word[0] = \"<PAD>\"\n",
        "    lang.index2word[1] = \"<SOS>\"\n",
        "    lang.index2word[2] = \"<EOS>\"\n",
        "    lang.index2word[3] = \"<UNK>\"\n",
        "\n",
        "    for ind, word in enumerate(sorted_words):\n",
        "        lang.index2word[ind + 4] = word\n",
        "    \n",
        "    lang.n_words = len(lang.index2word)\n",
        "    \n",
        "    print(lang.name, lang.n_words)\n",
        "    return lang\n",
        "\n",
        "input_lang, output_lang, pairs = prepareData('vi', 'eng')\n",
        "\n",
        "input_lang = build_topwordVocab(input_lang,vocab_size=source_vocab_size)\n",
        "output_lang = build_topwordVocab(output_lang, vocab_size=target_vocab_size)\n",
        "print(random.choice(pairs))"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading lines...\n",
            "Read 133318 sentence pairs\n",
            "Trimmed to 128908 sentence pairs\n",
            "Counting words...\n",
            "Counted words:\n",
            "vi 40038\n",
            "eng 46550\n",
            "Build vocabulary by top 19000 frequent word...\n",
            "vi 19004\n",
            "Build vocabulary by top 22000 frequent word...\n",
            "eng 22004\n",
            "['Mọi việc tiếntriển tốtđẹp từ đó , và chúngtôi trởthành những nhà khoahọc tuyệtvời .', 'Everything went well from there , and we became amazing scientists .']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "I_PL2_xh0d5y",
        "outputId": "7ebf0f55-6556-48b6-a48b-447603d192e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "_, _, val_pairs = readLangs('vi', 'eng', 'dev')\n",
        "val_pairs = filter_pairs(val_pairs[:-1])"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading lines...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "PMl_EFTVu1F8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "61d81d17-6caa-4dc8-cfdf-1c0ccd40dae6"
      },
      "cell_type": "code",
      "source": [
        "_, _, test_pairs = readLangs('vi', 'eng', 'test')\n",
        "test_pairs = filter_pairs(test_pairs)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading lines...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "FJv4sN4W0d52",
        "outputId": "e5da207d-9c20-4de8-c4fe-bd7065cb7b07",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "print(random.choice(test_pairs))"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Giờ là câuhỏi đầutiên của ngày hômnay , Bạn có sẵnsàng để nghe về vấnđề quátải trong lựachọn ?', 'So for my first question for you today Are you guys ready to hear about the choice overload problem ?']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "h0KF3UmvulaQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "vmMewXau0d6E"
      },
      "cell_type": "markdown",
      "source": [
        "# Preparing Training Data"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "SPnqKF_F0d6F",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def indexesFromSentence(lang, sentence):\n",
        "    idxs = []\n",
        "    for word in sentence.split(' '):\n",
        "        try:\n",
        "            idxs.append(lang.word2index[word])\n",
        "        except KeyError:\n",
        "            idxs.append(3)  # 3 is the id of 'UNK'\n",
        "    idxs.append(EOS_token)\n",
        "    return idxs\n",
        "\n",
        "\n",
        "class VocabDataset(Dataset):\n",
        "    def __init__(self, pairs):\n",
        "        \n",
        "        self.source_sent_list = [indexesFromSentence(input_lang,pair[0]) for pair in pairs]\n",
        "        self.target_sent_list = [indexesFromSentence(output_lang,pair[1]) for pair in pairs]\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.source_sent_list)\n",
        "        \n",
        "    def __getitem__(self, key):\n",
        "        token1_idx = self.source_sent_list[key]\n",
        "        token2_idx = self.target_sent_list[key]\n",
        "        return [token1_idx,token2_idx, len(token1_idx), len(token2_idx)]\n",
        "\n",
        "    \n",
        "def Vocab_collate_func(batch):\n",
        "    source_sent_list = []\n",
        "    target_sent_list = []\n",
        "    source_len_list = []\n",
        "    target_len_list = []\n",
        "\n",
        "    for datum in batch:   ### batch = sample\n",
        "        source_len_list.append(datum[2])\n",
        "        target_len_list.append(datum[3])\n",
        "    \n",
        "    max_len_src = max(source_len_list)\n",
        "    max_len_trg = max(target_len_list)\n",
        "    \n",
        "    # padding\n",
        "    for datum in batch:\n",
        "        \n",
        "        # source sentence processing\n",
        "        padded_source = np.pad(np.array(datum[0]), \n",
        "                                pad_width=((0,max_len_src-datum[2])),          \n",
        "                                mode=\"constant\", constant_values=PAD_token)\n",
        "        source_sent_list.append(padded_source)\n",
        "        \n",
        "        # target sentence processing\n",
        "        padded_target = np.pad(np.array(datum[1]), \n",
        "                                pad_width=((0,max_len_trg-datum[3])),        \n",
        "                                mode=\"constant\", constant_values=PAD_token)\n",
        "        target_sent_list.append(padded_target)\n",
        "        \n",
        "    #sort sentences for the batch\n",
        "    sort_idx = sorted(range(len(source_len_list)), key=source_len_list.__getitem__, reverse=True)\n",
        "    source_sent_list = np.array(source_sent_list)[sort_idx]\n",
        "    target_sent_list = np.array(target_sent_list)[sort_idx]\n",
        "    source_len_list = np.array(source_len_list)[sort_idx]\n",
        "    target_len_list = np.array(target_len_list)[sort_idx]\n",
        "        \n",
        "    return [torch.tensor(source_sent_list).to(device), \n",
        "            torch.tensor(target_sent_list).to(device),\n",
        "            torch.LongTensor(source_len_list), \n",
        "            torch.LongTensor(target_len_list)]\n",
        "\n",
        "train_dataset = VocabDataset(pairs)\n",
        "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
        "                                           batch_size=BATCH_SIZE,\n",
        "                                           collate_fn=Vocab_collate_func,\n",
        "                                           shuffle=True)\n",
        "\n",
        "val_dataset = VocabDataset(val_pairs)\n",
        "val_loader = torch.utils.data.DataLoader(dataset=val_dataset,\n",
        "                                        batch_size=BATCH_SIZE,\n",
        "                                        collate_fn=Vocab_collate_func,\n",
        "                                        shuffle=False)\n",
        "\n",
        "\n",
        "test_dataset = VocabDataset(test_pairs)\n",
        "test_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n",
        "                                        batch_size=BATCH_SIZE,\n",
        "                                        collate_fn=Vocab_collate_func,\n",
        "                                        shuffle=False)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "PhwZoZz_0d6Q"
      },
      "cell_type": "markdown",
      "source": [
        "# Build Encoder-Decoder"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "E4lH5BjW10Jj",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class EncoderRNN(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, n_layers=1, dropout=0.1):\n",
        "        super(EncoderRNN, self).__init__()\n",
        "        \n",
        "        self.input_size = input_size\n",
        "        self.hidden_size = hidden_size\n",
        "        self.n_layers = n_layers\n",
        "        self.dropout = dropout\n",
        "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "        self.lstm = nn.LSTM(hidden_size, hidden_size, n_layers, dropout=self.dropout, bidirectional=True)\n",
        "        \n",
        "    def forward(self, input_seqs, input_lengths, hidden):\n",
        "        # Note: we run this all at once (over multiple batches of multiple sequences)\n",
        "        embedded = self.embedding(input_seqs)\n",
        "        packed = torch.nn.utils.rnn.pack_padded_sequence(embedded, input_lengths)\n",
        "        outputs, hidden = self.lstm(packed, hidden)\n",
        "        outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(outputs) # unpack (back to padded)\n",
        "        outputs = outputs[:, :, :self.hidden_size] + outputs[:, : ,self.hidden_size:] # Sum bidirectional outputs\n",
        "        return outputs, hidden, output_lengths\n",
        "      \n",
        "    def initHidden(self,batch_size):\n",
        "        return (torch.zeros(2 * self.n_layers, batch_size, self.hidden_size, device=device),torch.zeros(2 * self.n_layers, batch_size, self.hidden_size, device=device))\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "EF4Fj5_pN9Lv",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class DecoderRNN(nn.Module):\n",
        "    def __init__(self, hidden_size, output_size,dropout_p=0.1, n_layers=1, max_length=MAX_LENGTH):\n",
        "        super(DecoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.n_layers = n_layers\n",
        "        self.dropout_p = dropout_p\n",
        "        self.output_size = output_size\n",
        "        \n",
        "        \n",
        "        self.embedding = nn.Embedding(output_size, hidden_size,padding_idx=PAD_token)\n",
        "        self.gru = nn.GRU(self.hidden_size, self.hidden_size, n_layers, dropout=dropout_p)\n",
        "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
        "        self.dropout = nn.Dropout(self.dropout_p)\n",
        "        self.softmax = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    def forward(self, input_seqs, hidden, batch_size):\n",
        "        embedded = self.embedding(input_seqs).view(1, batch_size, -1)\n",
        "        embedded = self.dropout(embedded)\n",
        "        output = F.relu(embedded)\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "        output = self.softmax(self.out(output[0]))\n",
        "        return output, hidden"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "SbhB0-fFEd8Q",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class Attn(nn.Module):\n",
        "    '''\n",
        "    Part of the codes are paraphrased from\n",
        "    https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html\n",
        "    '''\n",
        "    def __init__(self, method, hidden_size):\n",
        "        super(Attn, self).__init__()\n",
        "        \n",
        "        self.method = method\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        if self.method == 'concat':\n",
        "            self.attn = nn.Linear(self.hidden_size * 2, hidden_size)\n",
        "            self.v = nn.Parameter(torch.FloatTensor(hidden_size))\n",
        "\n",
        "    def forward(self, hidden, encoder_outputs):\n",
        "        max_len = encoder_outputs.size(0)\n",
        "        this_batch_size = encoder_outputs.size(1)\n",
        "\n",
        "        # Create variable to store attention energies\n",
        "        attn_energies = Variable(torch.zeros(this_batch_size, max_len)).to(device) # B x S\n",
        "        \n",
        "        if self.method == 'dot':\n",
        "\n",
        "            attn_energies = torch.matmul(encoder_outputs.permute(1,0,2), hidden.permute(1,2,0)).squeeze()\n",
        "            \n",
        "        if self.method == 'concat':\n",
        "            hidden_expand = hidden.expand(max_len, -1, -1).permute(1, 0, 2)  # shape of (B, S, N)\n",
        "            enc_cat_hid = torch.cat([encoder_outputs.permute(1,0,2), hidden_expand], dim=-1)  # shape of (B, S, 2*N)\n",
        "            # After nn.Linear(2*N, N), enc_cat_hid with shape (B, S, N)\n",
        "            # v is shape of (N)\n",
        "            attn_energies = torch.matmul(self.attn(enc_cat_hid), self.v)  # shape of (B, S)\n",
        "        # Normalize energies to weights in range 0 to 1, resize to 1 x B x S\n",
        "        # Dangerous\n",
        "        if attn_energies.dim() == 1:\n",
        "            attn_energies = attn_energies.unsqueeze(0)\n",
        "        return F.softmax(attn_energies, dim=1).unsqueeze(1)\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "LqklCWyTEnpG",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class LuongAttnDecoderRNN(nn.Module):\n",
        "    '''\n",
        "    Part of the codes are paraphrased from\n",
        "    https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html\n",
        "    '''\n",
        "    def __init__(self, attn_model, hidden_size, output_size, n_layers=1, dropout=0.1):\n",
        "        super(LuongAttnDecoderRNN, self).__init__()\n",
        "\n",
        "        # Keep for reference\n",
        "        self.attn_model = attn_model\n",
        "        self.hidden_size = hidden_size\n",
        "        self.output_size = output_size\n",
        "        self.n_layers = n_layers\n",
        "        self.dropout = dropout\n",
        "\n",
        "        # Define layers\n",
        "        self.embedding = nn.Embedding(output_size, hidden_size).to(device)\n",
        "        self.embedding_dropout = nn.Dropout(dropout)\n",
        "        self.lstm = nn.LSTM(hidden_size, hidden_size, n_layers, dropout=dropout)\n",
        "        self.concat = nn.Linear(hidden_size * 2, hidden_size)\n",
        "        self.out = nn.Linear(hidden_size, output_size)\n",
        "        \n",
        "        # Choose attention model\n",
        "        if attn_model != 'none':\n",
        "            self.attn = Attn(attn_model, hidden_size)\n",
        "\n",
        "    def forward(self, input_seq, last_hidden, encoder_outputs):\n",
        "        # Note: we run this one step at a time\n",
        "\n",
        "        # Get the embedding of the current input word (last output word)\n",
        "        batch_size = input_seq.size(0)\n",
        "        embedded = self.embedding(input_seq)\n",
        "        embedded = self.embedding_dropout(embedded)\n",
        "        embedded = embedded.view(1, batch_size, self.hidden_size) # S=1 x B x N\n",
        "\n",
        "        # Get current hidden state from input word and last hidden state\n",
        "        rnn_output, hidden = self.lstm(embedded, last_hidden)\n",
        "\n",
        "        # Calculate attention from current RNN state and all encoder outputs;\n",
        "        # apply to encoder outputs to get weighted average\n",
        "        attn_weights = self.attn(rnn_output, encoder_outputs)\n",
        "        context = attn_weights.bmm(encoder_outputs.transpose(0, 1)) # B x S=1 x N\n",
        "\n",
        "        # Attentional vector using the RNN hidden state and context vector\n",
        "        # concatenated together (Luong eq. 5)\n",
        "        rnn_output = rnn_output.squeeze(0) # S=1 x B x N -> B x N\n",
        "        context = context.squeeze(1)       # B x S=1 x N -> B x N\n",
        "        concat_input = torch.cat((rnn_output, context), 1)\n",
        "        concat_output = torch.tanh(self.concat(concat_input))\n",
        "\n",
        "        # Finally predict next token (Luong eq. 6, without softmax)\n",
        "        output = self.out(concat_output)\n",
        "\n",
        "        # Return final output, hidden state, and attention weights (for visualization)\n",
        "        return output, hidden, attn_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "d1GxY0qw0d6V"
      },
      "cell_type": "markdown",
      "source": [
        "# Training Model"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "4ICvoIhH0d6W",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import time\n",
        "import math\n",
        "\n",
        "\n",
        "def asMinutes(s):\n",
        "    m = math.floor(s / 60)\n",
        "    s -= m * 60\n",
        "    return '%dm %ds' % (m, s)\n",
        "\n",
        "\n",
        "def timeSince(since, percent):\n",
        "    now = time.time()\n",
        "    s = now - since\n",
        "    es = s / (percent)\n",
        "    rs = es - s\n",
        "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "qiiPjqOy0d6Y",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train(input_tensor, target_tensor, input_lengths, target_lengths, encoder, decoder, \n",
        "          encoder_optimizer, decoder_optimizer, clip=10.0):\n",
        "    encoder_optimizer.zero_grad()  # zero out the accumulated gradient over mini-batch\n",
        "    decoder_optimizer.zero_grad()\n",
        "    \n",
        "    \n",
        "    batch_size = input_tensor.size(1)\n",
        "    input_tensor = input_tensor.to(device)\n",
        "    target_tensor = target_tensor.to(device)\n",
        "\n",
        "    encoder_hidden = encoder.initHidden(batch_size)\n",
        "    encoder_outputs = torch.zeros(input_lengths.max(), batch_size, encoder.hidden_size, device=device) \n",
        " \n",
        "\n",
        "    encoder_outputs, encoder_hidden, encoder_output_lengths = encoder(input_tensor, input_lengths, encoder_hidden)\n",
        "    #encoder_outputs:  # max_len x batch_size x hidden_size\n",
        "    #hidden: n_layers * 2 x batch_size x hidden_size\n",
        "    loss = 0\n",
        "\n",
        "    \n",
        "    decoder_input = torch.tensor([SOS_token]*batch_size).to(device)  # decoder_input: torch.Size([1, 32])\n",
        "    decoder_hidden = (encoder_hidden[0][:decoder.n_layers], encoder_hidden[-1][:decoder.n_layers]) # Use last (forward) hidden state from encoder\n",
        "    all_decoder_outputs = Variable(torch.zeros(target_lengths.max(), batch_size, decoder.output_size)).to(device)\n",
        "    \n",
        "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
        "    \n",
        "    if use_teacher_forcing:\n",
        "        # Teacher forcing: Feed the target as the next input\n",
        "        for di in range(target_lengths.max()):\n",
        "            if Attention:\n",
        "                decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                    decoder_input, decoder_hidden, encoder_outputs)\n",
        "     \n",
        "            else:\n",
        "                decoder_output, decoder_hidden = decoder(\n",
        "                    decoder_input, decoder_hidden, batch_size)\n",
        "            \n",
        "            decoder_input = target_tensor[di]  # Teacher forcing\n",
        "            all_decoder_outputs[di] = decoder_output\n",
        "            \n",
        "    else:\n",
        "        # Without teacher forcing: use its own predictions as the next input\n",
        "        for di in range(target_lengths.max()):\n",
        "          \n",
        "            if Attention:\n",
        "                decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                    decoder_input, decoder_hidden, encoder_outputs)\n",
        "     \n",
        "            else:\n",
        "                decoder_output, decoder_hidden = decoder(\n",
        "                    decoder_input, decoder_hidden, batch_size)\n",
        "            \n",
        "            topv, topi = decoder_output.topk(1)\n",
        "\n",
        "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
        "            all_decoder_outputs[di] = decoder_output\n",
        "           \n",
        "            \n",
        "    # Loss calculation and backpropagation\n",
        "\n",
        "    loss = masked_cross_entropy(\n",
        "        all_decoder_outputs.transpose(0, 1).contiguous(), # -> batch x seq\n",
        "        target_tensor.transpose(0, 1).contiguous(), # -> batch x seq\n",
        "        target_lengths\n",
        "    )\n",
        "#     loss = loss.sum()/batch_size \n",
        "    loss.backward()\n",
        "    #    ave_loss.backward()\n",
        "    \n",
        "    # Clip gradient norms\n",
        "    torch.nn.utils.clip_grad_norm_(encoder.parameters(), clip)\n",
        "    torch.nn.utils.clip_grad_norm_(decoder.parameters(), clip)\n",
        "    \n",
        "    encoder_optimizer.step()   # update parameters\n",
        "    decoder_optimizer.step()\n",
        "\n",
        "    return loss.item()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "HmHvipVe0d6a",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def trainIters(encoder, decoder, n_iters, lr_decay=True, gamma_encoder=0.9, gamma_decoder=0.9, print_every=100, plot_every=100, learning_rate_encoder=0.0005, learning_rate_decoder=0.002,evaluate_every=3000):\n",
        "    start = time.time()\n",
        "\n",
        "    encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate_encoder)\n",
        "    decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate_decoder)\n",
        "    \n",
        "    scheduler_encoder = ExponentialLR(encoder_optimizer, gamma_encoder, last_epoch=-1) \n",
        "    scheduler_decoder = ExponentialLR(decoder_optimizer, gamma_decoder, last_epoch=-1) \n",
        "    \n",
        "    encoder.to(device)\n",
        "    decoder.to(device)\n",
        "    score_max = 0\n",
        "    plot_losses = []\n",
        "    validation_scores = []\n",
        "    \n",
        "    for epoch in range(1, n_iters + 1):\n",
        "        print_loss_total = 0  # Reset every print_every\n",
        "        plot_loss_total = 0  # Reset every plot_every\n",
        "        if lr_decay:\n",
        "            scheduler_encoder.step()\n",
        "            scheduler_decoder.step()\n",
        "        \n",
        "        for i, (input_sentences, target_sentences,len1,len2) in enumerate(train_loader): \n",
        "            encoder.train()\n",
        "            decoder.train()\n",
        "            \n",
        "            input_tensor = input_sentences.transpose(0,1)   # 13*100 to 100*13\n",
        "            target_tensor = target_sentences.transpose(0,1)\n",
        "            loss = train(input_tensor, target_tensor, len1, len2, encoder,\n",
        "                         decoder, encoder_optimizer, decoder_optimizer)\n",
        "            print_loss_total += loss\n",
        "            plot_loss_total += loss\n",
        "            \n",
        "            if i > 0 and i % evaluate_every == 0:\n",
        "                bleu_score, (src_sents, sys_sents, ref_sents) = test_model(encoder, decoder, val_loader)\n",
        "                print('Validation Score: {} \\n source sentence {} \\n predicted sentence {} \\n Reference sentence: {}'.format(bleu_score,src_sents, sys_sents, ref_sents))\n",
        "                validation_scores.append(bleu_score)\n",
        "                \n",
        "                if bleu_score > score_max:\n",
        "                    score_max = bleu_score\n",
        "                \n",
        "                    torch.save({\n",
        "                                'epoch': epoch,\n",
        "                                'encoder': encoder.state_dict(),\n",
        "                                'encoder_optimizer': encoder_optimizer.state_dict(),\n",
        "                                'decoder': decoder.state_dict(),\n",
        "                                'decoder_optimizer': decoder_optimizer.state_dict()\n",
        "                                }, \"/content/drive/My Drive/saved_model/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                        .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size,\n",
        "                                target_vocab_size,lr_decay,teacher_forcing_ratio))   \n",
        "                    \n",
        "            if i > 0 and i % print_every == 0:\n",
        "                print_loss_avg = print_loss_total / print_every\n",
        "                print_loss_total = 0\n",
        "#                bleu_score, (sys_sents, ref_sents) = test_model(encoder, decoder, val_loader)\n",
        "                print('Time: {}, Epoch: [{}/{}], Step: [{}/{}], Train Loss: {}'.format(\n",
        "                    timeSince(start, i + 1/len(train_loader)), epoch, n_iters, i, \n",
        "                    len(train_loader),print_loss_avg))\n",
        "\n",
        "            if i > 0 and i % plot_every == 0:\n",
        "                plot_loss_avg = plot_loss_total / plot_every\n",
        "                plot_losses.append(plot_loss_avg)\n",
        "                plot_loss_total = 0\n",
        "                torch.save({\n",
        "                            'plot_losses': plot_losses,\n",
        "                            'validation_scores': validation_scores\n",
        "                            }, \"/content/drive/My Drive/saved_scores/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                    .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size,\n",
        "                            target_vocab_size,lr_decay,teacher_forcing_ratio))   \n",
        "                \n",
        "            torch.cuda.empty_cache()    \n",
        "        print(\"plot_losses:\",plot_losses)\n",
        "        print(\"validation_scores:\",validation_scores)\n",
        "    showPlot(plot_losses)\n",
        "    showPlot(validation_scores)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "mj8OwEU_0d6d"
      },
      "cell_type": "markdown",
      "source": [
        "# Plotting results"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "bDqy1HHH0d6e",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.switch_backend('agg')\n",
        "import matplotlib.ticker as ticker\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "def showPlot(points):\n",
        "    plt.figure()\n",
        "    fig, ax = plt.subplots()\n",
        "    # this locator puts ticks at regular intervals\n",
        "    loc = ticker.MultipleLocator(base=0.2)\n",
        "    ax.yaxis.set_major_locator(loc)\n",
        "    plt.plot(points)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "L_-Y9XII0d6g"
      },
      "cell_type": "markdown",
      "source": [
        "# Evaluation"
      ]
    },
    {
      "metadata": {
        "id": "rwNlj-Hi9JTV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class beam_search(object):\n",
        "    \"\"\"\n",
        "    Some code is paraphrased from\n",
        "    https://github.com/OpenNMT/OpenNMT-py/blob/master/onmt/translate/beam.py\n",
        "    \"\"\"\n",
        "    def __init__(self, encoder, decoder, max_length, beam_size, attention = True,sentence_ratio = False): \n",
        "        super(beam_search, self).__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.attention = attention\n",
        "        self.max_length = max_length\n",
        "        self.beam_size = beam_size\n",
        "        self.sentence_ratio = sentence_ratio\n",
        "        \n",
        "    def search(self, encoder_outputs, decoder_input, decoder_hidden, src_len):\n",
        "\n",
        "        prob = {k:0 for k in range(self.beam_size)}\n",
        "        bestSent = []\n",
        "        bestScore = []       \n",
        "        decoder_word_choices = {k:[] for k in range(self.beam_size)}\n",
        "        decoder_hidden_choices = {}\n",
        "        decoder_input_choices = {}\n",
        "        decoder_output_choices = {}\n",
        "        \n",
        "        # Initialize beam serach\n",
        "        if self.attention == True:\n",
        "            decoder_output, decoder_hidden, decoder_attention = self.decoder(decoder_input.contiguous(), decoder_hidden, encoder_outputs)\n",
        "            decoder_output = F.log_softmax(decoder_output, dim=1)\n",
        "            topv, topi = decoder_output.data.topk(self.beam_size)\n",
        "        else: \n",
        "            print(\"Only available when attention = True\")\n",
        "        \n",
        "        # Initialize beam candidates \n",
        "        for i in range(self.beam_size):\n",
        "            decoder_word_choices[i].append(topi.squeeze()[i].item())\n",
        "            decoder_input_choices[i] = topi.squeeze()[i].detach()\n",
        "            decoder_hidden_choices[i] = decoder_hidden\n",
        "            prob[i] += topv.squeeze()[i].detach()\n",
        "            \n",
        "        ## running beam search\n",
        "        cur_len = 0\n",
        "        max_length = 2*src_len if self.sentence_ratio else self.max_length\n",
        "        # delete\n",
        "#         print(self.sentence_ratio)\n",
        "#         print(src_len)\n",
        "#         print(max_length)\n",
        "        \n",
        "        while decoder_hidden_choices and cur_len <= max_length:\n",
        "            cur_len += 1\n",
        "            topi = {}\n",
        "            key_list = list(decoder_hidden_choices.keys())\n",
        "            scores = []\n",
        "            for key in key_list:\n",
        "                    \n",
        "                decoder_output, decoder_hidden_choices[key],decoder_attn  = self.decoder(decoder_input_choices[key].unsqueeze(0), decoder_hidden_choices[key],encoder_outputs)\n",
        "                decoder_output_choices[key] = F.log_softmax(decoder_output, dim=1)\n",
        "                topv, topi[key] = decoder_output_choices[key].data.topk(len(decoder_hidden_choices))\n",
        "                scores.extend((topv+prob[key]).tolist()[0])\n",
        "                \n",
        "            scores = np.array(scores)   \n",
        "            max_candidate_score = scores.argsort()[-len(decoder_hidden_choices):][::-1]\n",
        "            decoded_sent_score = scores[max_candidate_score]\n",
        "\n",
        "            choice_sentence = {}\n",
        "            choiceHidden = {}\n",
        "            trashOfKeys = []\n",
        "            \n",
        "            for j in range(len(max_candidate_score)):\n",
        "                prev_choice_idx = key_list[int(np.floor(max_candidate_score[j]/len(decoder_hidden_choices)))]\n",
        "                if topi[prev_choice_idx].squeeze().dim() == 0:\n",
        "                    next_idx = topi[prev_choice_idx].squeeze()\n",
        "                else:\n",
        "                    next_idx = topi[prev_choice_idx].squeeze()[max_candidate_score[j] % len(decoder_hidden_choices)]\n",
        "                \n",
        "                s_choice = decoder_word_choices[prev_choice_idx].copy()\n",
        "                s_choice.append(next_idx.item())\n",
        "                choice_sentence[j] = s_choice\n",
        "                h_choice = decoder_hidden_choices[prev_choice_idx]\n",
        "                choiceHidden[j] = h_choice\n",
        "                decoder_input_choices[j] = next_idx.detach()   \n",
        "                prob[j] = decoded_sent_score[j] \n",
        "   \n",
        "            decoder_word_choices = choice_sentence\n",
        "            decoder_hidden_choices = choiceHidden\n",
        "            \n",
        "            for key, s in decoder_word_choices.items():\n",
        "                if EOS_token in s:\n",
        "                    bestSent.append(s)\n",
        "                    bestScore.append(prob[key]) \n",
        "                    trashOfKeys.append(key)\n",
        "                    \n",
        "            for k in trashOfKeys:\n",
        "                decoder_hidden_choices.pop(k)\n",
        "                decoder_word_choices.pop(k)\n",
        "\n",
        "        if len(bestScore) == 0:\n",
        "            max_prob = prob[0]\n",
        "            max_prob_idx = 0\n",
        "            for k in prob.keys():\n",
        "                if prob[k] > max_prob: \n",
        "                    max_prob_idx = k\n",
        "                    max_prob = prob[k]\n",
        "            bestScore.append(max_prob)\n",
        "            bestSent.append(decoder_word_choices[max_prob_idx])\n",
        "                \n",
        "        return bestSent, bestScore"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "muc68_Oz0d6h",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_batch_outputs(encoder, decoder, input_sentences, input_lengths, output_lengths): \n",
        "    with torch.no_grad():\n",
        "        input_tensor = input_sentences.transpose(0,1).to(device)   # 32*100 to 100*32\n",
        "        batch_size = input_tensor.size(1)\n",
        "        encoder_hidden = encoder.initHidden(batch_size)\n",
        "        encoder_outputs, encoder_hidden,encoder_output_lengths = encoder(input_tensor, input_lengths, encoder_hidden)\n",
        "       \n",
        "        decoder_hidden = (encoder_hidden[0][:decoder.n_layers], encoder_hidden[-1][:decoder.n_layers]) # Use last (forward) hidden state from encoder\n",
        "\n",
        "        decoder_input = Variable(torch.tensor([SOS_token]*batch_size)).to(device)  # decoder_input: torch.Size([1, 32])\n",
        "        decoded_words = np.empty((output_lengths.max(), batch_size), dtype=object)\n",
        "\n",
        "        for di in range(output_lengths.max()):\n",
        "            if Attention:\n",
        "                decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                    decoder_input, decoder_hidden, encoder_outputs)\n",
        "\n",
        "            else:\n",
        "                decoder_output, decoder_hidden = decoder(\n",
        "                    decoder_input, decoder_hidden, batch_size)\n",
        "\n",
        "            topv, topi = decoder_output.topk(1)\n",
        "            decoder_input = topi.squeeze().detach().to(device)  # detach from history as input\n",
        "\n",
        "            decoded_words[di:] = np.array(['<EOS>' if idx==EOS_token else output_lang.index2word[idx] for idx in decoder_input.tolist()])\n",
        "\n",
        "        return decoded_words.transpose()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5-3nvuRwf0nD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "sentence_ratio = True\n",
        "def pad(l, max_length):\n",
        "    while len(l) < max_length + 2:\n",
        "        l.append(PAD_token)\n",
        "    return l\n",
        "  \n",
        "  \n",
        "def get_beam_batch_outputs(encoder, decoder, input_sentences, input_lengths): #####\n",
        "    with torch.no_grad():\n",
        "        input_tensor = input_sentences.transpose(0,1).to(device)   # 32*100 to 100*32\n",
        "        batch_size = input_tensor.size(1)\n",
        "        encoder_hidden = encoder.initHidden(batch_size)\n",
        "        encoder_outputs, encoder_hidden,encoder_output_lengths = encoder(input_tensor, input_lengths, encoder_hidden)\n",
        "\n",
        "        decoder_hidden = (encoder_hidden[0][:decoder.n_layers], encoder_hidden[-1][:decoder.n_layers])\n",
        "        my_beam_search = beam_search(encoder, decoder,input_sentences.max().item(), beam_size, True, sentence_ratio)\n",
        "        beam_search_result = []\n",
        "        for i in range(batch_size):\n",
        "            decoder_input = torch.tensor([SOS_token], device=device, requires_grad=False).unsqueeze(0)#.view(1,-1) # take care of different input shape\n",
        "            sentences, probs = my_beam_search.search(encoder_outputs[:,i,:].unsqueeze(1), decoder_input, \n",
        "                                                     (decoder_hidden[0][:,i,:].unsqueeze(1).contiguous(),decoder_hidden[1][:,i,:].unsqueeze(1).contiguous()), input_lengths[i].item())\n",
        "\n",
        "            beam_search_result.append(sentences[probs.index(max(probs))])\n",
        "\n",
        "        padded_beam_search_result = []\n",
        "\n",
        "        max_length = 0\n",
        "        for sentence in beam_search_result:\n",
        "            if len(sentence) > max_length:\n",
        "                max_length = len(sentence)\n",
        "\n",
        "        for sentence in beam_search_result:\n",
        "            while len(sentence) < max_length + 2:\n",
        "                sentence.append(PAD_token)\n",
        "            padded_beam_search_result.append(sentence)\n",
        "\n",
        "        batch_sentences = []\n",
        "\n",
        "        for sentence in padded_beam_search_result:\n",
        "            sentence = [output_lang.index2word[k] for k in sentence]\n",
        "            try:\n",
        "                end_idx = sentence.index('<EOS>')\n",
        "                batch_sentences.append(' '.join(sentence[:end_idx]))\n",
        "            except ValueError:\n",
        "                batch_sentences.append(' '.join(sentence))\n",
        "\n",
        "    return batch_sentences\n",
        "  \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "ce-QyODD0d6l",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test_model(encoder, decoder, loader, search_method = 'greedy'):\n",
        "    \n",
        "    encoder.eval()\n",
        "    decoder.eval()\n",
        "    \n",
        "    score = []\n",
        "    src_sentences = []\n",
        "    sys_sentences = []\n",
        "    ref_sentences = []\n",
        "    encoder.train(False)\n",
        "    decoder.train(False)\n",
        "    for i, (input_sentences, target_sentences, len1, len2) in enumerate(loader):\n",
        "        for sentence in target_sentences:\n",
        "            trg_list = []\n",
        "            for idx in sentence:\n",
        "                if idx.item() == EOS_token:\n",
        "                    break\n",
        "                else:\n",
        "                    trg_list.append(output_lang.index2word[idx.item()])\n",
        "            ref_sentences.append(' '.join(trg_list))\n",
        "        for sentence in input_sentences:\n",
        "            src_list = []\n",
        "            for idx in sentence:\n",
        "                if idx.item() == EOS_token:\n",
        "                    break\n",
        "                else:\n",
        "                    src_list.append(input_lang.index2word[idx.item()])\n",
        "            src_sentences.append(' '.join(src_list))\n",
        "\n",
        "        batch_size = input_sentences.size(0)\n",
        "        if search_method == 'greedy':\n",
        "            for sentence in get_batch_outputs(encoder, decoder, input_sentences, len1, len2):\n",
        "                try:\n",
        "                    end_idx = sentence.tolist().index('<EOS>')\n",
        "                    sys_sentences.append(' '.join(sentence[:end_idx]))\n",
        "                except ValueError:\n",
        "                    sys_sentences.append(' '.join(sentence))\n",
        "                    \n",
        "        elif search_method == 'beam':\n",
        "            translation_output = get_beam_batch_outputs(encoder, decoder, input_sentences, len1)\n",
        "            sys_sentences.extend(translation_output)\n",
        "            \n",
        "    encoder.train(True)\n",
        "    decoder.train(True) \n",
        "    \n",
        "    score = corpus_bleu(sys_sentences,[ref_sentences], smooth=\"floor\", smooth_floor=0.01, lowercase=False, use_effective_order=True, tokenize=DEFAULT_TOKENIZER).score\n",
        "    return score, (src_sentences[0:5], sys_sentences[0:5], ref_sentences[0:5])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "colab_type": "text",
        "id": "hjafDuaU0d6s"
      },
      "cell_type": "markdown",
      "source": [
        "# TRAINING AND EVALUATING  (Only one example is shown )"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "T3iWCiIQ0d6t",
        "scrolled": true,
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "encoder1 = EncoderRNN(input_lang.n_words, hidden_size, n_layers=n_layers).to(device)\n",
        "if Attention:\n",
        "    decoder1 = LuongAttnDecoderRNN(attn_model, hidden_size, output_lang.n_words, dropout=dropout_p, n_layers=n_layers).to(device)\n",
        "else:\n",
        "    decoder1 = DecoderRNN(hidden_size, output_lang.n_words, dropout_p=0)\n",
        "\n",
        "trainIters(encoder1, decoder1, n_iters=n_epochs, print_every=print_every, plot_every=plot_every, evaluate_every=evaluate_every, learning_rate_encoder=lr_rate_en, learning_rate_decoder=lr_rate_de, lr_decay=lr_decay, gamma_encoder=gamma_decoder,\n",
        "          gamma_decoder=gamma_decoder)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8_R-uE7bf0nM",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Perform bean-search  (only a small part is shown because some experiments were run on other matchines)"
      ]
    },
    {
      "metadata": {
        "id": "dZRY9Qa4qGsf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "hidden_size = 512\n",
        "n_layers = 1\n",
        "teacher_forcing_ratio = 1\n",
        "n_iters = 20\n",
        "source_vocab_size = 19000\n",
        "target_vocab_size = 22000\n",
        "\n",
        "\n",
        "encoder_current = EncoderRNN(input_lang.n_words, hidden_size,n_layers=n_layers).to(device)\n",
        "decoder_current = LuongAttnDecoderRNN(attn_model, hidden_size, output_lang.n_words, dropout=dropout_p, n_layers=n_layers).to(device)\n",
        "checkpoint = torch.load(\"/content/drive/My Drive/saved_model/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                    .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size, target_vocab_size,lr_decay,teacher_forcing_ratio))\n",
        "encoder_current.load_state_dict(checkpoint['encoder'])\n",
        "decoder_current.load_state_dict(checkpoint['decoder'])\n",
        "encoder_current.eval()\n",
        "decoder_current.eval()\n",
        "\n",
        "test_score_greedy, _ = test_model(encoder_current, decoder_current, val_loader)\n",
        "#greedy_score_vocab.append(test_score_greedy)\n",
        "beam_list = []\n",
        "for beam_size in range(2,15):\n",
        "    print(\"beam_size: \",beam_size)\n",
        "    test_score_beam, _ = test_model(encoder_current, decoder_current, val_loader, search_method='beam')\n",
        "    print('beam score:', test_score_beam)\n",
        "    beam_list.append(test_score_beam)\n",
        "torch.save({'beam_score': beam_list,'greedy_score': test_score_greedy},\n",
        "           \"/content/drive/My Drive/saved_scores/val_scores/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size,\n",
        "                        target_vocab_size,lr_decay,teacher_forcing_ratio))   \n",
        "\n",
        "#beam_score_vocab.append(max(beam_list))\n",
        "print('greedy score', test_score_greedy)\n",
        "print ('best beam score', max(beam_list))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZLdxIWmg6WG7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "hidden_size = 512\n",
        "n_layers = 4\n",
        "teacher_forcing_ratio = 1\n",
        "n_iters = 20\n",
        "source_vocab_size = 19000\n",
        "target_vocab_size = 22000\n",
        "\n",
        "\n",
        "encoder_current = EncoderRNN(input_lang.n_words, hidden_size,n_layers=n_layers).to(device)\n",
        "decoder_current = LuongAttnDecoderRNN(attn_model, hidden_size, output_lang.n_words, dropout=dropout_p, n_layers=n_layers).to(device)\n",
        "checkpoint = torch.load(\"/content/drive/My Drive/saved_model/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                    .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size, target_vocab_size,lr_decay,teacher_forcing_ratio))\n",
        "encoder_current.load_state_dict(checkpoint['encoder'])\n",
        "decoder_current.load_state_dict(checkpoint['decoder'])\n",
        "encoder_current.eval()\n",
        "decoder_current.eval()\n",
        "\n",
        "test_score_greedy, _ = test_model(encoder_current, decoder_current, val_loader)\n",
        "#greedy_score_vocab.append(test_score_greedy)\n",
        "beam_list = []\n",
        "for beam_size in range(2,15):\n",
        "    print(\"beam_size: \",beam_size)\n",
        "    test_score_beam, _ = test_model(encoder_current, decoder_current, val_loader, search_method='beam')\n",
        "    print('beam score:', test_score_beam)\n",
        "    beam_list.append(test_score_beam)\n",
        "torch.save({'beam_score': beam_list,'greedy_score': test_score_greedy},\n",
        "           \"/content/drive/My Drive/saved_scores/val_scores/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size,\n",
        "                        target_vocab_size,lr_decay,teacher_forcing_ratio))   \n",
        "\n",
        "#beam_score_vocab.append(max(beam_list))\n",
        "print('greedy score', test_score_greedy)\n",
        "print ('best beam score', max(beam_list))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wsbN85nY6PmH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "hidden_size = 256\n",
        "n_layers = 2\n",
        "teacher_forcing_ratio = 1\n",
        "n_iters = 20\n",
        "source_vocab_size = 19000\n",
        "target_vocab_size = 22000\n",
        "\n",
        "\n",
        "encoder_current = EncoderRNN(input_lang.n_words, hidden_size,n_layers=n_layers).to(device)\n",
        "decoder_current = LuongAttnDecoderRNN(attn_model, hidden_size, output_lang.n_words, dropout=dropout_p, n_layers=n_layers).to(device)\n",
        "checkpoint = torch.load(\"/content/drive/My Drive/saved_model/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                    .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size, target_vocab_size,lr_decay,teacher_forcing_ratio))\n",
        "encoder_current.load_state_dict(checkpoint['encoder'])\n",
        "decoder_current.load_state_dict(checkpoint['decoder'])\n",
        "encoder_current.eval()\n",
        "decoder_current.eval()\n",
        "\n",
        "test_score_greedy, _ = test_model(encoder_current, decoder_current, val_loader)\n",
        "#greedy_score_vocab.append(test_score_greedy)\n",
        "beam_list = []\n",
        "for beam_size in range(2,15):\n",
        "    print(\"beam_size: \",beam_size)\n",
        "    test_score_beam, _ = test_model(encoder_current, decoder_current, val_loader, search_method='beam')\n",
        "    print('beam score:', test_score_beam)\n",
        "    beam_list.append(test_score_beam)\n",
        "torch.save({'beam_score': beam_list,'greedy_score': test_score_greedy},\n",
        "           \"/content/drive/My Drive/saved_scores/val_scores/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size,\n",
        "                        target_vocab_size,lr_decay,teacher_forcing_ratio))   \n",
        "\n",
        "#beam_score_vocab.append(max(beam_list))\n",
        "print('greedy score', test_score_greedy)\n",
        "print ('best beam score', max(beam_list))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "A3uXnn0ABMRI",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Plot beam on validation set"
      ]
    },
    {
      "metadata": {
        "id": "jiYyZEE3-UgU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "score = torch.load(\n",
        "           \"/content/drive/My Drive/saved_scores/val_scores/LSTM_attnIsTrue_hiddenSize512_nLayer2_batchSize64_epoch20_srcVocSize19000_tgtVocSize22000_lrDecayFalse_teacherF1\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UJkG2y63Cbw8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 378
        },
        "outputId": "0c38d206-96d6-4952-adfa-9ed115902be6"
      },
      "cell_type": "code",
      "source": [
        "plt.plot(range(1,15),[score['greedy_score']]+score['beam_score'])\n",
        "plt.xlabel('beam width')\n",
        "plt.ylabel('BLEU score')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0,0.5,'BLEU score')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfUAAAFYCAYAAABKymUhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8lNWh//HPZCb7njBJ2BL2nci+\niYjI4tJaUQuWH1h7e1vp5nKh1Vra2ktbL9FetGiNSCleqIpN3QGLKGi0CZssIgQSkEBYkkwymQSS\nyTKZ3x+BKJKQAJn9+369fJnJzPM85yRhvnPOcxaD0+l0IiIiIj4vyNMFEBERkY6hUBcREfETCnUR\nERE/oVAXERHxEwp1ERERP6FQFxER8RMmTxfgapWWVnm6CB0mPj4Cq7Xa08XwiECte6DWGwK37oFa\nbwjcund0vc3m6FafU0vdi5hMRk8XwWMCte6BWm8I3LoHar0hcOvuznor1EVERPyEQl1ERMRPKNRF\nRET8hEJdRETETyjURURE/IRCXURExE8o1EVERPyEQl1ERMRPKNRFRET8hEJdRETETyjURSSgfHak\njJIAXH9cAoNCXUQCgtPp5I3sIyx9dQ9LX91DY6PT00US6XAKdRHxe06nk6wPD/PWJ0cxAMXWGnYe\nKvV0sUQ6nEJdRPya0+nk5ffz2ZB7jOSECH7+neEYDLAu5yhOp1rr4l8U6iLitxqdTlZvPMSmHUV0\n6RTJI3OGMyAtnlH9kzhWfIbPvyj3dBFFOpRCXUT8UmOjk1Xr89iy6wTdk6L4xZzhxEaFAnDLuDQA\n1uUUerKIIh1OoS4ifsfR2MiKd/bz8Wen6JESzc+/M5yYiJDm59NSohnSM4GDxysoOGHzYElFOpZL\nQz0jI4PZs2dz5513snHjxubvZ2dn079//xaPeeutt7jtttu444472LJliyuLJyJ+qMHRSOabn5O7\nv5jeXWNYePdwosKDL3rdreObWuvr1VoXP2Jy1Ylzc3PJz89n7dq1WK1WZs6cyfTp06mtrWX58uWY\nzeaLjrFarTz77LP885//pLq6mmXLljF58mRXFVFE/Ex9QyPPvbGP3QUW+neP4/670gkPbfltrl/3\nOHp3jWF3gYWi0jN0M0e5ubQiHc9lLfXRo0fz9NNPAxATE0NNTQ0Oh4PMzEzmzJlDSEjIRcfk5OQw\nfvx4oqKiSEpKYvHixa4qnoj4mbp6B8v+uZfdBRYG94jnwVnXtBroAAaDgVvH9QBgQ65a6+IfXNZS\nNxqNREREAJCVlcWkSZM4duwYeXl5PPDAAzzxxBMXHVNUVITdbmf+/PlUVlbys5/9jPHjx1/yOvHx\nEZhMRpfUwRPM5mhPF8FjAq3utfUOqu31AVfvr+qouttrG1i8civ7vihn1MBkfvnd0YQEt/2+cGNi\nFG9+8gVbD5TwH98aSkpiZIeUpy36nQced9XbZaF+3qZNm8jKymLlypUsWLCARYsWXfL1FRUVPPPM\nM5w8eZJ77rmHzZs3YzAYWn291Y+WezSboyktrfJ0MTzCH+vudDqxna2jtKLm3H/2r3xdQ8WZOgwG\n6G6OYkBaPANS4+nXPY6IMJf/s/QKHfU7r6lt4Kl/7CG/yMaIfmZ++I2B2Cra/74wfXR3Xnh7Py+9\ne4B501se69OR/PFvvb0Cte4dXe9LfUBw6btHdnY2mZmZrFixgurqao4cOcLChQsBKCkpYe7cuaxZ\ns6b59YmJiQwfPhyTyURqaiqRkZGUl5eTmJjoymKKXLHaegeWFgK71GbHUlFDXUPjRccYDJAYE8aA\n1DhMwUbyjlo5VnKGjduPYzBAj5RoBqTGMyAtnr7dYgkLCYyQvxLV9nr+99U9HDlZyZiBSfznNwZh\nMl7eXcUxA5N4/aMjfLz3FLdd25PYyItvDYr4Cpe9W1RVVZGRkcGqVauIi4sDmlrt502ZMuWCQAeY\nOHEijzzyCD/4wQ+w2WxUV1cTHx/vqiKKtKnR6aSiqvbClratBsu5r21n61o8LjzUROfESMxxYZjj\nwr/yXxgJMWHNwWM2R3PiZAWHT1ZyoNBK3jErX5ys5ItTVWzYegxjkIGenWMYkBbHgNR4+nSNbVe3\nciA4U1PPn17ZTWFxFeMHp/D9WwcSFNR6r15rjEFB3Dw2ldUbD/He9uPcNbm3C0or4h4uC/X169dj\ntVp58MEHm7+3ZMkSunTpctFrH3roIR5//HGSk5OZMWMGs2bNAmDRokUEBWkqvbhWTW0DFtvXWtrn\nQttis9PguLi1bQwykBgTxuAe8XT6Wmib48KJDLt4ClVrQoKNDEyLZ2Ba0wfY2joH+ScqyCus4ECh\nlcMnbRScsPHOvwsxGQ307hJ7rrs+jl5dYgk2Bd6/kcqzdTz5yi6KSs8y6ZrO3DNjwBUF+nkT0zvz\n5idH2byriFvGpQXMLRDxPwanjy9+7E/3ZwL1fhN4pu41tQ0889pnHCi0tvh8VHjwxS3t2KbH8TGh\nGDvgA2d76l1T28Ch4xXkHbOSV1jBseIqzv+jDTEF0adbbHN3fY+U6MvufvaUK/2dW6tqefKVXZwq\nq2bKiK7MmdaPoEuMu2mv9bmFZG05zJ3X9+LW8T2u+nyt0b/zwKu739xTF/FW9Q2NzYHes3M0PVJi\nLmhpd4oN95rWWnioiWv6dOKaPp2Apm7nQ8cryDvXXb//aNN/AKEhRvp2i2XguYF3acnRV9WC9Tbl\nlXYyXt5FibWG6aO7M3tKn0sOpL0ck4d1ZV3OUd7bfpxpo7rrNof4JO941xJxo8ZGJy+8s58DhVaG\n9enET+4Y0iGtbneJCg9mRD8zI/o1LeBUWV3HwWNfhvy+I+XsO9K0UUl4qIn+3eOau+u7JUV1SKvW\nE0oranji5V1YbHZuHZ/GHZN6dVigA0SEmZgyohvrcgr5+LNTTBnRrcPOLeIuCnUJKE6nk79vOsSO\nvBL6dotl/rcG+1SgtyQmIoTRA5IYPSAJaOqePnjM2txdv7vAwu4CC9D0geB8yA9Mi6dLJ/fMy75a\nxeXVZLy8C2tVLbdf15Pbru3pkutMG9WdjduP8+7WY1w/rIvP/21I4FGoS0B58+Mv2PzpCbqZo3jg\nrnS/7GKNjw5l3OAUxg1OAaDMZj8X8E1Bv/NQKTsPlQKQmhzFdeldGDsoucX10b3BCctZnnx5F7az\ndXx7cm9uPrfDmivERIZwXXpnPvj0BNv2lzB+SIrLriXiCgp1CRjv7yzirU+O0ik2jP+afQ0RlzFC\n3ZclxoZx7dDOXDu0M06nk1KbnbxCK7vzLew9XMbf3zvE2g8KGNGvE9eld2Fgj3iv6aI/XnKGJ1/Z\nRVV1Pd+Z2pdpo7q7/Jo3jUlly66TrM8tZOzgZK/5WYi0h0JdAsK2A8W89N4hYiKCWXD3MOLO7asd\naAwGA0lx4STFhTPpmi7YztSS83kx2XtPsu1ACdsOlJAYE9r8IcAcF+6xsh49XcmfXtnNWXsD98zo\nz+ThXd1y3U5x4YwdlEzO56fZU2BheN+LN58S8VYKdfF7+74o44W39xMWauShWcNIjo/wdJG8RmxU\nKDeNTWXGmO4cPlnJx3tPsvVACW99cpS3PjnKwLR4rkvvzIh+Zrfeqjh8wsb/vroHe20D/3HLQCam\nd3bbtQFuGZdKzuenWZ9TyLA+nTp0QJ6IKynUxa8dOVnJs6/tw2Aw8LM70klLCczNJNpiMBjo0zWW\nPl1j+c6N/dhxsITsvac4UGjlQKGV8FAT4wYlMzG9Mz1Sol0acoeOV7D0H3uor2/kB98c1Dw2wJ26\nmqMY3rcTu/ItHDxWwYA0rWwpvkGhLn7rVNlZnvrHHuoaHPz49qF6Y26n0BBjc/d7cXk1H392ik8+\nO8XmXSfYvOsE3cyRTEzvwvjByURHdOw66fuPlvPnf+7F4XAy/1uDGXVuRL8n3DI+jV35FtblFupv\nR3yGQt1LbDtQzKt/+YSpI7szbXQ3TaW5SuWVdv60djdnauq59+YBjOyv+6JXIjkhgjuv783t1/Xk\n8y/Kyd57it35Fl55P59/bC5geN9OTEzvwpCeCVe9yM1nR8p45rXPcDqd/GTmUIb17dRBtbgyvbvE\nMiA1js+/KOfo6Up6pMR4tDwi7aFQ9xJ7Csoor6zl1c0F5O4/zb03D9CbyBU6U1PPn9bupryyljuv\n78Wkay7eb0AujzEoiPTenUjv3YnK6jpy950me+8pdhwsZcfBUuKjQ5kwJIWJ6Z2vaMzCrvxSnnvj\n3G2SO9MZ2ss7dma8ZXwaeccqWJ97jB/fPsTTxRFpk0LdS5TZaggywLjBKfx732kWv7iDqSO7M3NS\nT229eRlq6xw89Y89nCqrZvro7tziwjnNgSomIoTpY1KZNro7R09Xkb33FFv3n2ZdTiHrcgrp1z2O\n69I7M6p/EqEhbQ+u25FXwvNvfY7RaOCBO9MZ2CPBDbVon8E9EkhLjmZnXgmny6tJSdAgS/Fuxsce\ne+wxTxfialRXt7z1pa95PfsLoiJCeHjOCPp1iyX/hI3PjpSR8/lpkuIiSEn07zeTyMjQq/5dNjia\n1nM/eLyC8YOTmTejv9fPMe6IenuKwWAgPjqUa/p0Yuqo7nRJjKTaXs/B4xXsyrfw/s4iSivsREcE\nEx8detHgusjIUN7ffozlb39OSHDTzARvu3dtMBiICg9me14JdfWODpne5su/86sVqHXv6HpHRrY+\nJVeh7gUaHI1kbT5Mz66xjB+cgjkunOuHdQEM7DtSTu7+Yk6UnqFv9zi/bbVf7R99o9PJincOsCvf\nQnrvRO67zTeWf/WXNzmTMYjuSVFcO7Qz44ekEB5i4lRZNQePVZC99xTb80qob2jEHB9O2LnW+7/3\nnSbz9c8ICzWx4O5h9O0W5+FatCwlIYKtB0o4eKyCiUM7Ex56df8G/eV3fiUCte4K9cvgD38gFlsN\nm3YWMaR3J9LP3Us0BgUxMC2ekf3MHC85w74vyvloz0kiQk2kuXhKkSdczR+90+nk5U35ZO89RZ+u\nsdx/VzohJt9Y/tUf3+Qiw4IZmBbPtFHd6dMtlgaHk8MnbXx2pJxNO45TeLqKotIzrPnXQSLDTPz8\nO8Pp2dl7x48YDAZCTUF8mt+0fv6Qq7zf74+/8/YK1Lq7M9S9vykTAEptdqBppPHXdTVH8cjcEdwz\noz9gYPXGQ/zPmk8pKj3j5lJ6r3dyCnl/ZxFdO0Vy/13phPrheu6+KCjIwJCeifzo9iH8708nMmdq\nX7p0imyaJpZTSGxU0+0mX1g7YPyQFOKjQ/lw90nO1NR7ujgirVJL3QvkFVrZXWBh6uhUkltYltNg\nMNCjcwzXDk2hvLK2udVe39BIn66xGI2+/9nsSj/Jbtl1grUfFJAYE8Yv5owgNrJj5027WqC0XEKC\njfTqEssNw7syrE8n4qJC+dFdw4j3kd9XUJABg8HAngILwaagq7r3Hyi/8/NKrNVkfXiYFzfkceBo\nOQ5HI53iwjH5wftWe7mzpe6fN2h9jMVWA0BSGyNr46JC+dHtQ5hQYGHNxoOsyylke14J98zozyAv\nGjHsLjvySlj9r4NEhTet5x4fHZjrufuatJRo0lKiMZujKC2t8nRx2u36a7rwzr+P8v7OIm4am+q3\n41s6yvGSM6zLOcr2vBKcTggPNZK77zS5+04TGmxkWN9OjB2YzOCeCQSbAifgXU1/lV7AUnGu+z0+\nAhyONl9/TZ9O9E+N443sL3hvx3GefGU3E4akMHtKnw5f4ctbHTha3jRqOsTIf82+RlONxOVCQ4xM\nHdmNNz7+gg93n2TGmFRPF8kr5RdVsC6nkL2HywDonhTFrePTGNU/CXsjvPvvI2zdX9z8X0SoiRH9\nzYwdlMyA1DifGODqzRTqXsBisxNkMJAYG0Z5+dl2HRMWYuLuG/sybnAyL244yL/3nWbv4TJmT+nD\nhCEpfjeQ7quOnq7kz699BsD9dwzVIj3iNlNGdmPD1mP8a9sxpozophbmOU6nk31flLPu30c5VGQD\noG+3WG4dn8bQXonN70dpydHcMak3M6/rxdHTVWzdX8y2A8V8vPcUH+89RUxEMKMHJDNmUBK9u8Z6\n/ZRUb6RQ9wIWWw0JMaFXdG+8R0oMi747kvd3FPFa9hH+uu4A/953mntm9G9x4J2vO11ezdJX91BX\n5+BHtw/xqoVKxP9FhQdz/bAubNx+nJzPTwf8aoWNjU52HCxhfW4hx4qbBu+m907klnFp9Ove+hRF\ng8FAz84x9Owcw6wpfcg/XsHWAyXsyCvh/U+LeP/TIhJjQhk9MJmxA5NJTY7y64ZKR1Koe1h9g4OK\nM3UMSL3yObrGoCCmj0llRH8zazYeYu/hMn79123cdm0Pbhqb6jcDUqxVtfzpld1UVddzz4z+Ht3s\nQwLXjDGpvL+ziA25hUwc2vmq17z3RQ2ORv697zQbcgspttZgMMCYgUncMi6N1OTLm80QZDDQPzWe\n/qnxzJnalwOFVrbtL+bT/FLe3XqMd7ceIyUhgjEDkxg7KJnOiZEuqpV/UKh7WFllLQCdYi8e9X65\nOsWG88Bd6ew4WMpL7x3itY+a7l1996YB9OkWe9Xn96Sz9nr+99XdlFXamXldTyYP7+rpIkmAio8O\n5dqhKXy05xQ7D5UyOoA+XNbWOfhw9wn+tf041qpajEEGJl3TmZvHpnVIz6DJGMTQXokM7ZXIPQ0O\n9h4uZ+uBYvYUWHjrk6O89clRUpOiGDMomTEDkzrkfdPfKNQ97PzI905xYR1yPoPBwOgBSQzuEU/W\nlsNs2X2SP67ZyeThXbnr+l5EhAV3yHXcqbbewdNZezlRepYbR3bjGxN6eLpIEuBuHptG9t5TrMs5\nyqj+Zr/vGj5TU88HO4vYtLOIMzX1hAYbmT66OzPGpLps1kmwycjI/mZG9jdTU9vA7gIL2/YXs++L\ncrK2HCZry2F6d41hzMBkxgxIIjZKs19Aoe5xlnMLz3SK7ZhQPy8iLJh7bhrA+CEpvPjuQbbsOsGu\n/FL+39R+jPShN6EGRyPPvbGPgiIbYwcl852pfX2m7OK/khMiGNU/ie15JXz+RflVrzLnrSrO1LJx\n23E27z5BbZ2DyDATt13bg6mjuhMV7r4GQnioifGDUxg/OIUzNfV8eqiUrfuLyTtm5fCJSl55P58B\nqfGMHZTMiH5mt5bN2yjUPez8dDZXdSP17RbHY98bzYatx3j7k6P85Y19XNM7kbnT+5PYwR8kOlqj\n08nf1uex93AZQ3om8P1bB2o0rHiNW8alsT2vhHU5hX4X6iXWajZsPcYnn52iweEkLiqE2yf25Pph\nXTw+Pz8qPJhJ13Rh0jVdsJ2pZXteCVsPFHOg0MqBQiur/3WQIT0TGDMomeF9O3m8vO4WWLX1Qs3d\n7y4MWJMxiG9O6MHoAUn837t57DlcRt6Krcyc1IupI7t55UAfp9PJqx8UkPP5aXp1ieEnM4f6zYA/\n8Q9pKdEM6ZXAviPlFJyw0aerb49bgaYFY9bnFrLtQDFOJyTFhXPzuFQmDOnsldP3YqNCmTqqO1NH\ndcdSUcO2vBK27S9mz+Ey9hwuI8QURHqfTowdmERSfATGIANGowGjwYDRGIQxyEBQkAFjkAGT0YAx\nKAiDAZ/uDVSoe1iZzY4xyECcG+4HpSRE8PPvDOff+06z9oMCXnk/n5zPT3PvTQO8bv3tDVuPsXH7\ncTonRvDgt69p177cIu5267g09h0pZ31OIfffle7p4lyxgiIb7+QcbV4wppv53IIxA8w+sxhMp7hw\nbhmXxi3j0jhVdrZpcZtz0+R25JVc1rmawz+oKfi/fGwgKCgIU5Dhyw8EX39d0JcfGM7/N3VcD9I6\nuWeKsULdw0ptdhJjwtzWWjYYDFw7tDNDeyey9v2mlvB/v7idaaO6M7K/meT4CKIjgj36SfWjPSfJ\n2nKYhJhQFsweFtD3x8S79eseR5+usewusFBUeoZu5ihPF6ndmheMySnk0PEKoOUFY3xR58RIbr+u\nF9+a2JNjxWfYlV/KmZp6GhudNDQ6cTicNDqdOByNOBqdX/537vFXX+dobKTx3PMNDieO+oYLjjv/\n3CUFBfH9Wwa4pe4KdQ+qq3dQebaObj2ufHOIKxUTEcIPvjmICUNTWP3uQTZuP87G7ceBpjWak+Ii\nSE4IJyk+guT4cJITIkiKDyc63LWB/+mhUl58N69pPffZw0iI8e77/hLYDAYDt4xP489Ze9mQW8gP\nvjnY00VqU2Ojk52HSlmXc7R5wZihvRK5dfylF4zxRQaDoXmvAVdyOi8M+K9/SOjfq1O7Vwu9Wgp1\nDyqrdM3I98sxuEcC//39MWw9UMypsmqKy6spsdZwwnKWwuKLN9sIDzU1h3xyfDjJ8U1hn5wQcdUt\n6oPHrGS++TkhJiMPfvsaLTIhPiG9dyJdzZFs3V/C7df1wtzCToveoMHRSM6+06zfeozi8moMwOgB\nTQvGeNvtN19jMDTdk6eVu4Tu3ElToe5BpedGvid6eAGFkGAj16VfuNxlo9OJtbKWYmtTyBdbqyku\nb/p/UekZjp6+OPAjw0xNLfuEr4T9uceRbcyPP3LCxp//uRen08lP70inVxet5y6+Ichg4JZxabzw\n9n7e3XaMedP7e7pIFyk4YePFDXmcsJzt8AVjxLso1D2ozA0j36/U+Q1mEmPDGNTjwucaG52UV9kp\nttZQUl5NsbWmqYVfUcOx4iq+OFV50fmiwoNJjg9vDvqkc8GfHB/BmZo6/uelXdhrHdz3rcEM7qn1\n3MW3jBmYxOsfHeHjvae47dqexHrJPvE1tQ1kfXiYLZ+ewAlcP6wL35zQQ7e1/JhC3YPOLzxj9rGl\nDoOCDHSKDadTbDiDv7ahSmOjk7JKe3PLvrmVb63h6OkqDp+8OPCNQQYcjU7+37R+jBmY7K5qiHQY\nY1AQN49NZfXGQ7y3/Th3Te7t6SLx6aFS/v7eIaxVtXROjODemwfQt5t/3TOXiynUPajUdr773X8+\nNQcFGTDHhWOOC2dIzwufczQ2UmY718I/17ovttZQXmXnpvE9uXZQ4KyhLf5nYnpn3vzkKJt3FXHL\nuDQiwjzz9mqtquWl9w6x81ApJqOB2yf25OZxaV45z1w6nkLdg8psNZiMQcRGeUdXnasZg4JIio8g\nKf7i+3hmczSlpRffpxfxFcGmpvXQs7YcZvOuIm4d38Ot1290Ovlw90mythRQU+ugX7dYvnvzAA04\nDTAKdQ+y2OwkxoZp6VMRP3HD8K6syynkve3HmTaqOyHB7lk06YTlLC++m0dBkY3wUBPfvak/113T\nRe8tAUih7iG1dQ6qqusve+9hEfFe4aEmpoxoCvbsvae4cWQ3l16vvqGRdTlHWZdTiKPRyaj+ZuZM\n6+eWFSrFOynUPcQda76LiPtNG9WdjduP8+7WY1w/rIvL9iw4dLyCF9/N41RZNfHRocyd3o/hfc0u\nuZb4DoW6h7hqy1UR8ayYyBAmpXfh/U+L2H6ghPFDUjr0/NX2ev6x5TAf7j6JAbhxZDfumNSL8FC9\nnYtC3WO+DHXfms4mIm2bMaY7m3edYH1uIWMHJ3fIvW2n08nOg03T1Gxn6+hqjuTemwbQ2w92h5OO\n49I5DhkZGcyePZs777yTjRs3Nn8/Ozub/v1bX3XJbrczdepUXnvtNVcWz6PU/S7ivzrFhTN2UDIn\nLGfZU2C56vOVV9pZ9s/P+Msb+zhrb+COSb347b2jFehyEZe11HNzc8nPz2ft2rVYrVZmzpzJ9OnT\nqa2tZfny5ZjNrd/7ee6554iN9e8/VnW/i/i3W8alkvP5adbnFDKsT6cr2gipsdHJ5l0nyPrwMLV1\nDgakxvHdmwZoeVdplctCffTo0aSnN+0vHBMTQ01NDQ6Hg8zMTObMmcMTTzzR4nGHDx+moKCAyZMn\nu6poXsFisxNsCiLGS5aTFJGO1dUcxfC+ndiVb+HgsQoGpF3eboxFJWd48d08Dp+sJDLMxJybBzAx\nvbNPb4kqruey7nej0UhERNOnyaysLCZNmsSxY8fIy8vj5ptvbvW4JUuW8Mgjj7iqWF7DUlFDp9gw\n/QMV8WO3jE8DYF1uYbuPqW9w8NpHh/ndqu0cPlnJmIFJ/P4H47jumi56v5A2uXyg3KZNm8jKymLl\nypUsWLCARYsWtfraN954g2HDhtG9e/d2nz8+PgKTyT0LPHSUans9Z+0N9O+RgNl84Tz1rz8OJIFa\n90CtN/h/3c3maNL7FLK3wILN7qDPuf3KW6v33oJSnv3HHk5azmKOD+fHd17DKD/bD8Hff+etcVe9\nXRrq2dnZZGZmsmLFCqqrqzly5AgLFy4EoKSkhLlz57JmzZrm12/ZsoXjx4+zZcsWTp8+TUhICCkp\nKUyYMKHVa1it1a6sgksUlZwBIDY8+IKlUQN5qdRArXug1hsCp+7TRnZjb4GFv2/Yz49nDm2x3mdq\n6nl1cwEf7z2FwQDTR3fn9ut6EhZi8qufUaD8zr+uo+t9qQ8ILgv1qqoqMjIyWLVqFXFxTZ9ON23a\n1Pz8lClTLgh0gKeeeqr562XLltG1a9dLBrqvKtXId5GAMahHPGkp0ew8WMrp8uoL3pCdTifbDpTw\n8qZDVFbX0z0pintvHkDPzjEeLLH4MpfdU1+/fj1Wq5UHH3yQefPmMW/ePE6ePNniax966CHsdrur\niuJ1LH64O5uItMxgMHDruDScwIav3Fu32Gp4Omsvz7/1OTV1Dr49uTe//u4oBbpcFZe11GfPns3s\n2bNbff6DDz5o/nrp0qUXPf+zn/3MJeXyBmXn91GP08IzIoFgRD8zyQkR/HvfaUrKq3lv2zFeyz5C\nXX0jg3rEc8+M/i3uXihyubSinAeUVjR1v6ulLhIYgoIM3DI2lb9tyONnf9pMtb2BqPBg7pnRn/GD\nUzSqXTqMQt0Dymx2QoKDiA4P9nRRRMRNxg9J4c1PvqC8spbxg1OYfWMfYiK0ToV0LIW6B1hsdsyx\n4fp0LhJATMYgfjFnBCFhwcSF6a1XXMOla7/Lxart9VTXNqjrXSQAJcWF07f75a0sJ3I5FOpupjXf\nRUTEVRTqbqYtV0VExFUU6m6mlrqIiLiKQt3NLOems3WKU6iLiEjHUqi7mbrfRUTEVRTqbmax2QkL\nMRKpKS0iItLBFOpu5HQ6sdjs2MB6AAAfn0lEQVS0j7qIiLiGQt2NztobsNc51PUuIiIuoVB3ozKN\nfBcRERdSqLvR+Y1cFOoiIuIKCnU3+nIfdXW/i4hIx1Oou9GX+6irpS4iIh1Poe5GFpu630VExHUU\n6m5ksdkJDzUREaZ91EVEpOMp1N2kaY66Xa10ERFxGYW6m5ypqae23qFQFxERl1Gou4nWfBcREVdT\nqLuJtlwVERFXU6i7SfPId01nExERF1Gou4mlQt3vIiLiWgp1N1H3u4iIuJpC3U0sthoiw0yEh2of\ndRERcQ2Fuhs4nU7KbHZ1vYuIiEsp1N2gsrqeuoZGdb2LiIhLKdTd4PzI90SFuoiIuJBC3Q2+3J1N\n3e8iIuI6CnU3KK1QS11ERFxPoe4GZZrOJiIibqBQdwPNURcREXdQqLtBqc1OVHgwYSGaoy4iIq6j\nUHexxuY56mqli4iIaynUXazybB0NjkY6aeS7iIi4mELdxb7cyEUtdRERcS2Fuos1b7mqUBcRERdT\nqLvYlyPf1f0uIiKupVB3MbXURUTEXRTqLna+pa7V5ERExNUU6i5msdmJiQgmNNjo6aKIiIifc2mo\nZ2RkMHv2bO688042btzY/P3s7Gz69+9/Wcf4ouY56prOJiIibtCuJc6sVitFRUUMHTqUxsZGgoLa\n/iyQm5tLfn4+a9euxWq1MnPmTKZPn05tbS3Lly/HbDa3+xhfVVFVi6PRqfvpIiLiFm2m8zvvvMPs\n2bP55S9/CcDixYv5xz/+0eaJR48ezdNPPw1ATEwMNTU1OBwOMjMzmTNnDiEhIe0+xlfpfrqIiLhT\nm6H+t7/9jTfffJP4+HgAHn74YV599dU2T2w0GomIiAAgKyuLSZMmcezYMfLy8rj55pvbfYzR6Lv3\nopv3Udd0NhERcYM2u9+jo6MJD/8ylMLCwggODm73BTZt2kRWVhYrV65kwYIFLFq06LKOaUt8fAQm\nk3cGf3XDSQB6pyZgNke365j2vs4fBWrdA7XeELh1D9R6Q+DW3V31bjPU4+Pjef3116mtreXzzz9n\n/fr1JCQktOvk2dnZZGZmsmLFCqqrqzly5AgLFy4EoKSkhLlz57JmzZpWj4mObvuHYLVWt6ssnlB4\n0gaAiUZKS6vafL3ZHN2u1/mjQK17oNYbArfugVpvCNy6d3S9L/UBoc1Q/93vfsdTTz3F2bNnWbRo\nESNHjuT3v/99mxetqqoiIyODVatWERcXBzS1wM+bMmXKRYHe0jG+rEz7qIuIiBu1Geq7du3iN7/5\nzWWfeP369VitVh588MHm7y1ZsoQuXbpc9NqHHnqIxx9//LKO8QWlFTXERoUQ7KW3B0RExL8YnE6n\n81Iv+N73vscLL7yAydSu2W9u561dOY7GRuY/+SE9Okfzq3mj2nVMoHZNQeDWPVDrDYFb90CtNwRu\n3b2q+z06Oppbb72VQYMGXTBALiMjo2NK56cqqupwNDo18l1ERNymzVC/4YYbuOGGG9xRFr9yfiMX\nzVEXERF3aTPUZ86cSVFREfv378dgMDB48GCfvcftThYNkhMRETdrc/GZl19+mXvuuYd169bx9ttv\nM2/ePF5//XV3lM2naR91ERFxtzZb6m+++SYbNmwgNDQUgOrqar73ve8xc+ZMlxfOl1kqzu2jHqeW\nuoiIuEebLXWTydQc6AARERGXtaJcoLLY7BiAhGiFuoiIuEebLfWUlBQWL17MhAkTgKYV3zp37uzy\ngvk6i81OXHQowSZtWS8iIu7RZqgvXryY1atX89prr2EwGBg2bBhz5851R9l8lqOxEWtVLb27xni6\nKCIiEkDaDPXQ0FBGjBjBD3/4QwA++OCDFrdNlS+VV9bS6NQ+6iIi4l5t9g3/5je/4cMPP2x+vG3b\nNn71q1+5tFC+7st91DXyXURE3KfNUD969CgLFixofvzII49QVFTk0kL5uvMLz5jVUhcRETdqM9Tt\ndjsVFRXNj4uLi6mtrXVpoXydpUILz4iIiPu1eU/9Jz/5Cd/4xjfo3LkzDoeDkpIS/vCHP7ijbD6r\nufs9Tt3vIiLiPu1a+33Tpk0UFBRgMBjo1asX4eEKq0sps9VgMEBCdGjbLxYREekgbXa/79u3j5yc\nHIYMGcLGjRuZP38+O3bscEfZfFapzU5CdCgmo+aoi4iI+7SZOr///e/p2bMnO3bs4LPPPuPXv/41\nf/7zn91RNp/U4GikoqpWI99FRMTt2gz10NBQevTowfvvv8+sWbPo06cPQUFqgbamvNKOEw2SExER\n92sznWtqatiwYQObNm1i4sSJVFRUUFlZ6Y6y+aRSbbkqIiIe0maoL1iwgLfffpuHHnqIqKgoVq9e\nzb333uuGovmmMm25KiIiHtLm6PexY8cyduzY5sc/+9nPXFogX3d+4Rm11EVExN10c7yDnZ+jrn3U\nRUTE3RTqHcxSYSfIYCBec9RFRMTNFOodzGKrISEmFKNmCIiIiJu1ek99ypQpGAyG5scGg4Ho6Gi+\n+c1v8h//8R9uKZyvqW9opOJMHQNS4zxdFBERCUCthvqqVasu+p7FYuHvf/87f/nLX/jxj3/synL5\npLJKjXwXERHPaTXUU1NTW/xeeno68+bNU6i3QCPfRUTEky77xq/JZCI4ONgVZfF5zbuzKdRFRMQD\nLjvUjx49qmViW3F+H3WztlwVEREPaLX7/ec///kFA+UAbDYbBQUFLF261OUF80XqfhcREU9qNdQn\nTJhw0fciIyMZM2YMcXEa3d2SMpsdY5CBuCjNURcREfdrNdQnT55MfHx8i8/t2LGDUaNGuaxQvqrU\nZicxJoygIEPbLxYREelgrd4cf+CBBy54/N///d/NX2s/9YvV1TuoPFunQXIiIuIxrYa60+m84HF+\nfn6rz8lX56gr1EVExDNaDfWvD5Jr73OB6suNXDTyXUREPKPdc9MU5JdmqdDIdxER8axWB8qVlJSQ\nlZXV/Li0tJSsrCycTielpaVuKZwvaW6pK9RFRMRDWg314cOHs3PnzubHw4YNa348bNgw15fMx3wZ\n6up+FxERz2g11B9//PFWD6qsrHRJYXyZxVaDyWggNirE00UREZEAdUXrvf70pz/t6HL4PMv5Oeoa\neyAiIh5yRaGuKW0Xqq1zUFVdr/vpIiLiUVcU6hoJf6HmNd81nU1ERDyo1XvqOTk5rR6ke+oX0sh3\nERHxBq2G+l/+8pdWD4qOjm7XyTMyMti5cycNDQ3cd999TJ8+HYDs7Gz+8z//k4MHD150zB//+Ef2\n7NmDwWDg0UcfJT09vV3X8iTtoy4iIt6g1VBfvXr1VZ04NzeX/Px81q5di9VqZebMmUyfPp3a2lqW\nL1+O2Wy+6Jht27ZRWFjI2rVrOXz4MI8++ihr1669qnK4w/nud7Oms4mIiAdd8p56Xl4eFosFgL//\n/e/86Ec/YunSpdjt9jZPPHr0aJ5++mkAYmJiqKmpweFwkJmZyZw5cwgJuXjqV05ODlOnTgWgd+/e\n2Gw2zpw5c9mVcjd1v4uIiDdotaX+pz/9iY0bN9LQ0MCsWbMoKCjgrrvuYseOHfzmN78hIyPjkic2\nGo1EREQAkJWVxaRJkzh27Bh5eXk88MADPPHEExcdY7FYGDx4cPPjhIQESktLiYqKavU68fERmEzG\nNivqSrazdYSYgujdI/GqBxGaze27teGPArXugVpvCNy6B2q9IXDr7q56txrqubm5bNiwAavVyq23\n3srHH3+MyWTixhtv5O677273BTZt2kRWVhYrV65kwYIFLFq0qN3HtmfqnNVa3e7zucopy1kSY8Ow\nWK6uV8Fsjqa0tKqDSuVbArXugVpvCNy6B2q9IXDr3tH1vtQHhFa738PDwwkKCiIxMZE+ffpgMn2Z\n/8HBwe26cHZ2NpmZmbzwwgtUV1dz5MgRFi5cyKxZsygpKWHu3LkXvD4pKam5ux+a1p9v6d67N6mp\nbeCsvUGD5ERExONabal/VVDQhdnfni7mqqoqMjIyWLVqFXFxcUBTq/28KVOmsGbNmguOufbaa1m2\nbBl33303n3/+OUlJSZfsevcGZVrzXUREvESrob5r1y4mT54MQFlZWfPXTqcTq9Xa5onXr1+P1Wrl\nwQcfbP7ekiVL6NKly0Wvfeihh3j88ccZMWIEgwcP5u6778ZgMPDb3/72MqvjfhokJyIi3qLVUH/3\n3Xev6sSzZ89m9uzZrT7/wQcfNH+9dOnS5q8XLlx4Vdd1t1Kb9lEXERHv0Gqod+3a1Z3l8FnqfhcR\nEW9xRWu/y5fU/S4iIt5CoX6VLBU1hAQHER3RvhkBIiIirqJQv0oWm51OseHauU5ERDxOoX4Vqu31\nVNc2qOtdRES8gkL9Kuh+uoiIeBOF+lWwaOS7iIh4EYX6VVBLXUREvIlC/SpYKs4tPBOnUBcREc9T\nqF8Fdb+LiIg3UahfBYvNTmiIkciwdu2LIyIi4lIK9SvkdDopq6yhU2yY5qiLiIhXUKhfobP2Bmpq\nHZjV9S4iIl5CoX6Fzm/kkqiR7yIi4iUU6lfIoi1XRUTEyyjUr1BphUa+i4iId1GoX6EyLTwjIiJe\nRqF+hZq737XwjIiIeAmF+hWy2OyEh5qIDNM+6iIi4h0U6lfA6XSe20ddrXQREfEeCvUrcKamntp6\nh0JdRES8ikL9Clg0R11ERLyQQv0KnA91rSYnIiLeRKF+BbTwjIiIeCOF+hVQ97uIiHgjhfoVsGg1\nORER8UIK9StgsdUQGWYiQvuoi4iIF1GoXyan00mZza6udxER8ToK9ctUWV1PXUOjRr6LiIjXUahf\npvMj39VSFxERb6NQv0zanU1ERLyVQv0ylVac351N3e8iIuJdFOqXSS11ERHxVgr1y9S88EyMQl1E\nRLyLQv0yldrsRIUHEx6qOeoiIuJdFOqXofHcHHV1vYuIiDdSqF+GyrN1NDgaFeoiIuKVFOqXwWLT\nmu8iIuK9FOqXwdI8nU0tdRER8T4K9ctg0XQ2ERHxYgr1y/DlPurqfhcREe/j0nlZGRkZ7Ny5k4aG\nBu677z7MZjMZGRmYTCZCQkJ44oknSEhIaH792bNnefjhh7HZbNTX1/OTn/yE6667zpVFvCzn131X\nS11ERLyRy0I9NzeX/Px81q5di9VqZebMmaSnp5ORkUH37t155plnePXVV5k/f37zMa+//jo9e/Zk\nwYIFFBcX893vfpd3333XVUW8bBabnZiIYEKDjZ4uioiIyEVcFuqjR48mPT0dgJiYGGpqali6dClG\noxGn00lxcTEjR4684Jj4+HgOHjwIQGVlJfHx8a4q3mU7P0c9NTna00URERFpkcvuqRuNRiIiIgDI\nyspi0qRJGI1GPvroI2666SYsFgu33XbbBcfceuutnDx5kmnTpjF37lwefvhhVxXvslVU1eJodGLW\nyHcREfFSBqfT6XTlBTZt2sTzzz/PypUriY5uauU6nU6efPJJoqOjL+h+f/PNN9mxYweLFy8mLy+P\nRx99lNdee+2S529ocGAyub47/PMjZTzy7MfceUMf7v3GYJdfT0RE5HK5dKBcdnY2mZmZrFixgujo\naN577z2mTZuGwWBgxowZLFu27ILXf/rpp0ycOBGAAQMGUFJSgsPhwGhsPbSt1mpXVqHZ4cJyACJC\njJSWVrnkGmZztMvO7e0Cte6BWm8I3LoHar0hcOve0fU2m1u/Deyy7veqqioyMjJ4/vnniYuLA2DZ\nsmUcOHAAgD179tCzZ88LjklLS2PPnj0AnDhxgsjIyEsGujuVauS7iIh4OZe11NevX4/VauXBBx9s\n/t6vf/1rfve732E0GgkLCyMjIwOAhx56iMcff5zZs2fz6KOPMnfuXBoaGnjsscdcVbzLpoVnRETE\n27n8nrqruasr54mXd3Gg0ErmgusJcdGUtkDtmoLArXug1hsCt+6BWm8I3Lr7Rfe7vymtqCE2MsRl\ngS4iInK1FOrt4GhsxFpVq41cRETEqynU26Giqg5Ho1NbroqIiFdTqLeD1nwXERFfoFBvB418FxER\nX6BQb4cvQ13d7yIi4r0U6u2g7ncREfEFCvV2sFTYMQAJMQp1ERHxXgr1drDY7MRFhxJs0o9LRES8\nl1KqDefnqCeq611ERLycQr0N5ZW1NDqdup8uIiJeT6HeBo18FxERX6FQb4NGvouIiK9QqLfBUqGF\nZ0RExDco1NvQ3P0ep+53ERHxbgr1NpTZajAYICE61NNFERERuSSFehsslXbio0MxGfWjEhER76ak\nuoQGRyPWylqNfBcREZ+gUL+E8ko7TjRITkREfINC/RK05aqIiPgShfolnA91LRErIiK+QKF+CecX\nnjHrnrqIiPgAhfolqPtdRER8iUL9EiwVdoIMBuJjNEddRES8n0L9Eiy2GhJiQjEG6cckIiLeT2nV\nivqGRirO1KnrXUREfIZCvRVllRr5LiIivkWh3gqNfBcREV+jUG+F5qiLiIivUai3okzT2URExMco\n1FtRWnGu+137qIuIiI9QqLeizGbHGGQgLkpz1EVExDco1FthsdlJiAklKMjg6aKIiIi0i0K9BXX1\nDmxn67SPuoiI+BSFegvOz1HXIDkREfElCvUWaCMXERHxRQr1FljOjXxX97uIiPgShXoLmlvqcWqp\ni4iI71Cot+DL7ne11EVExHco1FtgsdVgMhqIjQrxdFFERETaTaHeAovNTmJMGEEGzVEXERHfoVD/\nmto6B1XV9Rr5LiIiPsfkypNnZGSwc+dOGhoauO+++zCbzWRkZGAymQgJCeGJJ54gISHhgmPeeust\nVqxYgclk4v7772fy5MmuLOJFzm+5mqj76SIi4mNcFuq5ubnk5+ezdu1arFYrM2fOJD09nYyMDLp3\n784zzzzDq6++yvz585uPsVqtPPvss/zzn/+kurqaZcuWeSDUNUddRER8k8tCffTo0aSnpwMQExND\nTU0NS5cuxWg04nQ6KS4uZuTIkRcck5OTw/jx44mKiiIqKorFixe7qnit0nQ2ERHxVS67p240GomI\niAAgKyuLSZMmYTQa+eijj7jpppuwWCzcdtttFxxTVFSE3W5n/vz5zJkzh5ycHFcVr1Vlms4mIiI+\nyqX31AE2bdpEVlYWK1euBGDSpElcd911PPnkkyxfvvyC7neAiooKnnnmGU6ePMk999zD5s2bMVxi\nFHp8fAQmk7HDyltprwegf69OJMS4v7VuNke7/ZreIlDrHqj1hsCte6DWGwK37u6qt0tDPTs7m8zM\nTFasWEF0dDTvvfce06ZNw2AwMGPGDJYtW3bB6xMTExk+fDgmk4nU1FQiIyMpLy8nMTGx1WtYrdUd\nWuYTJWcINgXRYK+jtLa+Q8/dFrM5mtLSKrde01sEat0Dtd4QuHUP1HpD4Na9o+t9qQ8ILut+r6qq\nIiMjg+eff564uDgAli1bxoEDBwDYs2cPPXv2vOCYiRMnkpubS2NjI1arlerqauLj411VxBaVnZuj\nfqneAREREW/kspb6+vXrsVqtPPjgg83f+/Wvf83vfvc7jEYjYWFhZGRkAPDQQw/x+OOPk5yczIwZ\nM5g1axYAixYtIijIfVPpa2obOFNTT4+UwOweEhER32ZwOp1OTxfianRkl0ZRyRl+s3Ibk4d35Z4Z\n/TvsvO0VqF1TELh1D9R6Q+DWPVDrDYFbd7/ofvdFmqMuIiK+TKH+FaW28/uoK9RFRMT3KNS/QnPU\nRUTElynUv0Ld7yIi4ssU6l8RExFMSkIE0RHBni6KiIjIZXP5inK+ZO6M/uBEc9RFRMQnKdS/Ishg\nAOW5iIj4KHW/i4iI+AmFuoiIiJ9QqIuIiPgJhbqIiIifUKiLiIj4CYW6iIiIn1Coi4iI+AmFuoiI\niJ9QqIuIiPgJhbqIiIifUKiLiIj4CYPT6XR6uhAiIiJy9dRSFxER8RMKdRERET+hUBcREfETCnUR\nERE/oVAXERHxEwp1ERERP6FQ9xIZGRnMnj2bO++8k40bN3q6OG5lt9uZOnUqr732mqeL4lZvvfUW\nt912G3fccQdbtmzxdHHc4uzZs/z0pz9l3rx53H333WRnZ3u6SC536NAhpk6dypo1awA4deoU8+bN\nY86cOTzwwAPU1dV5uISu01Ld7733XubOncu9995LaWmph0voGl+v93nZ2dn079/fpddWqHuB3Nxc\n8vPzWbt2LStWrOCPf/yjp4vkVs899xyxsbGeLoZbWa1Wnn32WV566SUyMzN5//33PV0kt3j99dfp\n2bMnq1ev5umnn+YPf/iDp4vkUtXV1SxevJjx48c3f+/Pf/4zc+bM4aWXXiItLY2srCwPltB1Wqr7\nU089xaxZs1izZg3Tpk3jb3/7mwdL6Bot1RugtraW5cuXYzabXXp9hboXGD16NE8//TQAMTEx1NTU\n4HA4PFwq9zh8+DAFBQVMnjzZ00Vxq5ycHMaPH09UVBRJSUksXrzY00Vyi/j4eCoqKgCorKwkPj7e\nwyVyrZCQEF544QWSkpKav7d161ZuvPFGAG644QZycnI8VTyXaqnuv/3tb5kxYwZw4d+CP2mp3gCZ\nmZnMmTOHkJAQl15foe4FjEYjERERAGRlZTFp0iSMRqOHS+UeS5Ys4ZFHHvF0MdyuqKgIu93O/Pnz\nmTNnjt++sX/drbfeysmTJ5k2bRpz587l4Ycf9nSRXMpkMhEWFnbB92pqaprf2BMTE/22C7qlukdE\nRGA0GnE4HLz00kt885vf9FDpXKelen/xxRfk5eVx8803u/76Lr+CtNumTZvIyspi5cqVni6KW7zx\nxhsMGzaM7t27e7ooHlFRUcEzzzzDyZMnueeee9i8eTMGg8HTxXKpN998ky5duvDXv/6VvLw8Hn30\n0YAbS/FVgbhKt8Ph4Be/+AXjxo27qIvaXz3++OMsWrTILddSqHuJ7OxsMjMzWbFiBdHR0Z4ujlts\n2bKF48ePs2XLFk6fPk1ISAgpKSlMmDDB00VzucTERIYPH47JZCI1NZXIyEjKy8tJTEz0dNFc6tNP\nP2XixIkADBgwgJKSEhwOR8D0TEFTa9VutxMWFkZxcfFF3bT+7pe//CVpaWn89Kc/9XRR3KK4uJgj\nR46wcOFCAEpKSpg7d+5Fg+g6ikLdC1RVVZGRkcGqVauIi4vzdHHc5qmnnmr+etmyZXTt2jUgAh1g\n4sSJPPLII/zgBz/AZrNRXV3t9/eXAdLS0tizZw8zZszgxIkTREZGBlSgA0yYMIF//etffOtb32Lj\nxo1cd911ni6S27z11lsEBwdz//33e7oobpOcnMymTZuaH0+ZMsVlgQ4Kda+wfv16rFYrDz74YPP3\nlixZQpcuXTxYKnGl5ORkZsyYwaxZswBYtGgRQUH+P8Rl9uzZPProo8ydO5eGhgYee+wxTxfJpfbt\n28eSJUs4ceIEJpOJf/3rXzz55JM88sgjrF27li5dunD77bd7upgu0VLdy8rKCA0NZd68eQD07t3b\n7/4GWqr3smXL3NZg09arIiIifsL/mwYiIiIBQqEuIiLiJxTqIiIifkKhLiIi4icU6iIiIn5CoS7i\nJ7Zu3cp3vvMdTxejRcuXL29xJ7qlS5eybNkyAD788MPmtcCnTJlCYWGhO4so4hcU6iLicj/84Q/b\n3LRn1apV2Gw29xRIxE9p8RkRP1JXV8cvfvELjh07RmRkJE8//TRRUVGsX7+eNWvW4HQ6SUhI4Pe/\n/z3x8fG89NJLvPnmmwQHBxMaGsrSpUuJiYlhypQpzfudl5aW8vDDD7N27VoKCgr4yU9+wsyZM5uv\nmZuby5o1a3jmmWeoqqpi3Lhx/O1vf2PMmDEsX74co9FIfn4+I0eO5Nvf/jZLly5l8+bNdO7cmfDw\ncHr37s1LL73Ejh07WLhwIY8//jgA77zzDjt37uTEiRP89re/DZjVBkWuhlrqIn7k0KFD/Nd//Rev\nvPIKCQkJvPHGG5w6dYrMzExWrVrFyy+/zJgxY3j++eeBpj2e//rXv7JmzRq6du3KW2+91Xyu+Ph4\nVq9ezbBhw3jxxRd57rnn+MMf/sCqVasuuOaIESPYv38/ANu3b2fcuHFs27YNaLolcH6td2jarert\nt98mKyuLZ599trmLfc6cOZjNZp588kn69OkDQEJCAitXruTHP/4x//d//+eyn5mIP1FLXcSP9OrV\ni5SUFACGDx/OwYMHSUhIoLS0lO9///tAU2u+W7duAMTFxfHDH/6QoKAgTpw4gdlsbj7XiBEjgKYl\nbZOTkzEYDKSkpFBVVXXBNUNCQujVqxcFBQVs3bqVe++9l1WrVlFfX8/x48fp379/82sPHTrE4MGD\nm7ceHTVqVKt1GTNmDAApKSlUVlZe7Y9GJCAo1EX8yFfXj3c6nRgMBkJCQkhPT29unZ93+vRplixZ\nwrp160hMTGTJkiUXPG8ymVr8uiUTJ05k+/bt7N27l4ULF5KZmcnOnTubPxh8vUznNTY2tnrOr15T\nq1mLtI+630X8yJEjRyguLgaatjnt168fQ4cOZe/evZSWlgKwYcMGNm3aRFlZGfHx8SQmJlJRUcHH\nH39MXV3dFV13woQJfPDBB0RERBAcHMyQIUNYtWrVBV3v0LSBx/79+6mrq6O+vr65mx7AYDDQ0NBw\nhTUXEVBLXcSvDBo0iKeeeorCwkKioqL41re+RWRkJL/61a+47777CA8PJywsjCVLlpCQkEBaWhp3\n3XUXqamp3H///Tz22GNcf/31l33dfv36cfDgweYpdaNHj2b16tX88Y9/vOB1ffv2ZerUqcyaNYsu\nXbowcODA5ucmTpzI/PnzL+oxEJH20y5tIiIifkLd7yIiIn5CoS4iIuInFOoiIiJ+QqEuIiLiJxTq\nIiIifkKhLiIi4icU6iIiIn5CoS4iIuIn/j964yJHawhedAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f8f35671cc0>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "NLszAj0oCupK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Plot training loss for GRU/LSTM to compare speed"
      ]
    },
    {
      "metadata": {
        "id": "kteP61XhBapr",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "score1 = torch.load(\n",
        "           \"/content/drive/My Drive/saved_scores/LSTM_attnIsTrue_hiddenSize512_nLayer2_batchSize64_epoch20_srcVocSize19000_tgtVocSize22000_lrDecayFalse_teacherF1\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bK_0qylOT9kA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "score2 = torch.load(\n",
        "           \"/content/drive/My Drive/saved_scores/attnIsTrue_hiddenSize512_nLayer2_batchSize64_epoch20_srcVocSize20000_tgtVocSize32000_lrDecayFalse_teacherF1\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fzv0j3zuFDx4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# This is the loss of each epoch from self-attention, for convenience\n",
        "score3 = [4.545370580658081,\n",
        " 3.537611612251827,\n",
        " 3.0360496749953616,\n",
        " 2.672862425115373,\n",
        " 2.394343628107555,\n",
        " 2.1719146188289393,\n",
        " 1.9884819965513922,\n",
        " 1.8350102686692797,\n",
        " 1.707337155985454,\n",
        " 1.5958151286556606,\n",
        " 1.4975455459148164,\n",
        " 1.4124447373170703,\n",
        " 1.3285594105247467,\n",
        " 1.2610603803207003,\n",
        " 1.1966657795603315]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3tqfVcOxC1J_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 311
        },
        "outputId": "0a187f96-f221-49dd-c566-c504e0192458"
      },
      "cell_type": "code",
      "source": [
        "CHAR_GRU_H512_L4 = score1['plot_losses']\n",
        "CHAR_LSTM_H512_L4 = score2['plot_losses']\n",
        "\n",
        "def average_loss(losses):\n",
        "    average_losses = []\n",
        "    for i in range(0,320,20):\n",
        "        average_losses.append(np.mean(losses[i:i+20]))\n",
        "    return average_losses\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(6, 4))\n",
        "# this locator puts ticks at regular intervals\n",
        "loc = ticker.MultipleLocator(base=1)\n",
        "ax.xaxis.set_major_locator(loc)\n",
        "ax.set_xlim(xmin = 0,xmax = 14)\n",
        "plt.plot(average_loss(score1['plot_losses']), marker = '*',label = 'GRU_H512_L2')\n",
        "plt.plot(average_loss(score2['plot_losses']), marker = '*',label = 'LSTM_H512_L2')\n",
        "plt.plot(score3, marker='*', label='self_attention')\n",
        "\n",
        "plt.xlabel(\"Epoches\")\n",
        "\n",
        "plt.title(\"Training Loss Over Steps\")\n",
        "plt.legend()"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f8ee8c8fd68>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEVCAYAAAAb/KWvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd4FNX6wPHvbE3fTW+ETgIBQgkg\nPRApoSlVEMEO+hOucEHUq4KoqIgKKKIgRfTqFRAxKiogAiI9CTVAEhJaeu89m/n9EVkJkLIhFc7n\neXye7OzMe96d4JvZM2fOkWRZlhEEQRDuKoqGTkAQBEGofaK4C4Ig3IVEcRcEQbgLieIuCIJwFxLF\nXRAE4S4kirsgCMJdSNXQCQgN6/XXX+fYsWMAREdH4+TkhFarBWDbtm1YWVlVO1ZAQABff/01Dg4O\nFe7z4Ycf4ubmxsMPP3xnif/Ny8uLP//8ExcXl1qJVx1JSUksX76ckJAQlEolWq2WKVOm1Npnup3Q\n0FDef/99EhMTkWUZvV7PggUL6NGjBwBbt27loYceqrP2hSZIFoS/DR48WA4KCmroNEzi6ekpx8fH\n11t7ubm58rBhw+SVK1fKxcXFsizLcnR0tDx27Fh51apVddJmaWmp3K9fP3nfvn3Gbbt27ZJ79eol\n5+XlyUlJSfLQoUPrpG2h6RLdMkKlpk+fzooVKxgxYgQnTpwgJSWFp556ioCAAPz9/fniiy+M+3p5\neZGQkMCxY8eYPHkyH374ISNGjMDf35/jx48D8PLLL/Ppp58C4O/vz+bNm5k4cSL9+/dn6dKlxlhr\n1qyhT58+TJgwgW+++QZ/f3+T8i4sLGTRokUMHz6cESNGsHTpUgwGAwBff/01I0aMICAggIkTJ3Lx\n4sVKt9/ohx9+wM7Ojjlz5qBSlX3xbdasGUuXLmX9+vVkZ2fTv39/QkNDjcds2rSJf//73wBs2bLF\neO7mzZtHQUGB8by8++67jBkzht9++61cm+np6SQnJ9OlSxfjtmHDhvHjjz9ibm7OlClTiIuLIyAg\ngKKiIiIjI5k2bRrDhw9nzJgxnD17FoDt27czY8YMFixYwJAhQxg9ejRXrlwB4Pjx44wbN46RI0cy\nYsSIW3IQmqCG/usiNB63u3KfNm2a/OSTT8oGg0GWZVl+88035UWLFsmyLMvXrl2TO3bsKMfFxcmy\n/M9V9NGjR+VOnTrJv//+uyzLsrxu3Tr58ccfl2VZll966SV59erVxvbmzZsnl5SUyAkJCXLHjh3l\n+Ph4OSIiQvb19ZUTExPlgoICedq0afLgwYNvm3NFV+5r166VZ8yYIRcXF8v5+fnyhAkT5MDAQDk7\nO1vu0aOHnJ2dLcuyLP/666/y559/XuH2mz3//PPy2rVrKzx/Bw8elF9//XX5o48+Mm5/5JFH5F27\ndslBQUFynz595ISEBFmWZXnhwoXy0qVLjedlzJgxckFBwS1xS0tL5QkTJsijR4+Wt27dKl+7dq3c\n+0ePHpWHDBkiy7IsGwwGediwYfLWrVtlWZbl4OBguX///nJxcbH8/fffy97e3vLJkydlWZbl5cuX\ny88995wsy7I8fvx4+dixY7Isy/Lly5flefPm3fYzCk2HuHIXquTn54dCUfZP5bXXXmPhwoUAeHh4\n4OjoSExMzC3HWFpaMmTIEAA6duxIXFzcbWOPGTMGpVKJs7Mz9vb2xMfHExQURK9evYz9/xMmTDA5\n5/379/PQQw+hUqkwMzNjzJgxHDp0CK1WiyRJbNu2jZSUFEaMGMGMGTMq3H6zzMxMbG1tb9umg4MD\nmZmZDB8+nL179wKQlpZGWFgYfn5+7N27l5EjR+Ls7AzAww8/zO7du43H9+nTx3i/40aSJPHFF18w\ndOhQvvrqK4YMGcKoUaPKHXvdpUuXSE1NZeLEiQD4+vpiZ2fHyZMnAWjTpg1du3YFYPjw4cbt9vb2\nBAYGEhUVRcuWLfnwww+rfa6FxkkUd6FKOp3O+PPZs2d56qmnGDZsGAEBASQnJ1NaWnrLMdbW1saf\nFQrFbfcByt2wVSqVGAwGsrKyyrV5vRiaIi0trVwMnU5HamoqarWaTZs2ceLECYYPH87UqVMJDw+v\ncPvNbG1tSUpKum2bKSkp2NnZ0atXLxITE4mLi2Pv3r34+fmh1WrJzs5mx44dBAQEEBAQwNy5cyku\nLi6XY0Wsra15/vnn+fnnnzl06BAPPvgg8+bNIyoqqtx+WVlZFBQUGLuXAgICSE1NJSMj45Y2bGxs\nyMrKAuCdd97B3NycJ554gmHDhrFz585qnGWhMRPFXTDJggULGD58OLt27WLnzp0VXsXeCSsrK/Ly\n8oyvKyqmlXFwcDAWNICMjAzjKB5vb28+/vhjjhw5Qv/+/Xn99dcr3X6jgQMH8scff9yyPSIigszM\nTHx8fFAqlQwZMoR9+/axZ88eRowYAYCTkxPjxo1j586d7Ny5k127dnHgwIEqP0tCQgLBwcHlPtvM\nmTPx9PS85b6Ak5MTlpaWxjZ27tzJwYMHGTp0qPE8XJeZmWks9g4ODixcuJADBw6waNEi/vOf/5Cb\nm1tlbkLjJYq7YJLU1FQ6deqEJEn88MMP5OfnlyvEtcHHx4djx46RlpZGUVERgYGBJscYNGgQ27Zt\nw2AwkJeXx48//oifnx/h4eE8//zzFBUVodFojJ+lou03e+CBBygpKWHp0qXGq+64uDhefvllnnvu\nOSwsLACMXTNnz55l4MCBQNkN5N27d5OWlgbAnj17+Pzzz6v8LPHx8cyaNavcTdozZ84QFxdH586d\nUalU5OXlUVJSgru7Oy4uLsYr77S0NObNm2f8HV2+fJnz588DsGvXLnx9fSkuLmb69OnGP6IdO3ZE\npVIZu+KEpkmMcxdMMmfOHGbNmoVer2fKlClMnjyZhQsX8r///a/W2vDx8WHcuHGMGzcOV1dXRo4c\nyaZNmyrcf/r06SiVSuPrJUuWMH36dKKjoxk1ahSSJBEQEGC8gm7WrBmjR49GrVZjaWnJokWL8PT0\nvO32mymVSr744gs++OADRowYgUqlQqvVMm3aNCZNmmTcr3fv3syfP5+BAwei0WiAsqL57LPPMn36\ndEpLS7G3t+eNN96o8nx069aNt956i8WLF5OdnU1paSkODg6sWLECd3d3dDodOp2Ofv368cMPP7B8\n+XIWL17MypUrUSgUPPHEE8Y/Ot26dWPTpk0EBwdjYWHBZ599hlqtZuLEiTz++ONAWTfaa6+9hrm5\neZW5CY2XJMtiPneh8ZFl2XjlvH//flauXFmjK3jhH9u3b+enn36q9A+lcPcQ37uERictLY3evXsT\nGxuLLMv89ttvxhEegiBUj+iWERodOzs75s6dy+OPP44kSbRu3ZoXX3yxodMShCZFdMsIgiDchUS3\njCAIwl2o3rtlSkoMpKfX7tC5G9naWoj4Iv49Gb8p5y7iV83R0brqnW5Q71fuKpWy6p1EfBFfxG9U\nsUX8ho9vKtEtIwiCcBcSxV0QBOEuJIq7IAjCXUgUd0EQhLuQKO6CIAh3oXov7mcjU+q7SUEQhHtO\nvRf3/+0Oq+8mBUEQ7jn1XtxDo1J575sThF1Nr++mBUEQ7hkN0uc+uLs77VvU/go+giAIQpkGmRXy\nj5AYenUwfV1MQRDqx/Vv1rV1ERYTE82qVcuNq1C5uLgyf/7LHD78F+vXr8HNzR2NRkVWVg6jRz/A\n2LETOXEimO3bt7JkyTJjnA0b1qLX65kwYfJt27ndMW+/vZhBg+6nRw8fRo8eg5dXewD0eluWLHkP\ngL179/Duu2+wdu0XtG7d1hhrzZpPUCoVeHi04OWXF1a4OtWGDWtp1syF4cMfLLf9jz92s3nz10iS\nAl/fnjzzzKwankHT1Xtx12qUJKXnl1uMQRCExuXHg5eB2inuBoOBV199kXnzXqJLl7J5+b/+ehMr\nV75Pr1698fcfyuzZc3F0tCY2NpUnn3yE++7re8ft3k7z5i345JPySxuePBnC0aOHaNOmXbnty5a9\nzccfr8HJyZnXXnuJY8cO06dP/2q3VVBQwGefreKrrzZjbm7BzJmPM2zYCFq1al0rn6Uq9V7ce7R3\n5tCZOGKTc2nmZFXfzQvCPW3r3kiCwipecLy4xEBeYQklhrKZwGe+vw8LrQq1SolSKWEw3DpDeM/2\nTjzk37bCmEFBx2jduo2xsANMnfoosiyza9ev5fbVaDS0bt2WuLjYerv48/JqT7duvsyePbPc9g0b\n/oulZVmN0uttyczMNCmumZkZX321GQsLSwB0Oh1ZWabFuBP13ufe18cVgOBw01e0FwShbqlVSizN\n1MbXlmZq1Hc4Ida1a1eMXR3XKRSKcuveXpeWlsqFC+do3bpNjds7deoEs2fPNP537NiRcvFfe+1F\nnn32SXbv/g3AWHxvdr2wp6SkEBR0lD59+pmcy/XYUVGRJCTE07FjZ5Nj1FT9X7l3cEalVBASkczY\nAfXz9UQQhDIP+bet9CobIPCvS8afJUniwf6tgLIpZ5OTs01uU5IUGAwlxtcvvzyPnJwckpOTmDz5\nEfbu/Z2wsPPIsoHExCTmzl2Ara0dly9fqihipe117dr9lj53AL1ez9NPP8vw4SPJyclhxozH6N69\nJw4ODhXGSk9P46WX/s38+S+j0+mr/ZlvFB19jTfeeJXXX1+CSlV/Jbfer9yLL4bTqZUdscm5JKTV\n3dzHgiDUjLujFWMHtGbsgNa4Odz+qtYUrVq1JizsvPH10qXL+eSTzzEYDMhyKf7+Q/nkk8/58ssv\n0Wq1eHp6AWVdIdnZOeViZWRkVFqMK2NlZcWoUQ+gUqnQ6/W0b9+Ba9euVLh/bm4O8+c/z4wZ/0ev\nXr1r1GZSUiL/+c8LvPrqG7Rr51WjGDVV78X9ypf/xdfLEYAQ0TUjCI1Oz/ZOt/25pnx9e5KUlMjB\ngweM28LDw8jLy0Oh+KdrxtzcnMcff5qPP14OlN38TE5OJCYmGoD09HROngymc+cuNcrj6NGjrFpV\nFjs/P5+LFyPw8Ghe4f6ffLKSyZOn0rt3zW/uLl36Fi+88LJxhE59qtZ3hIKCAkaPHs1zzz3H+PHj\njdv9/f1xcXEx9p198MEHODtXPsQx52IkboGf09LgSXC4NaP6tKx59oIgNHqSJPHhh6tYvnwZmzat\nR61WYWZmznvvLSc6+lq5fYcODWD79q0cP36UXr16s2jREpYte5vS0lIA5sx5ATs7+xrl0aNHDzZv\n/o5nnnmC0lID06c/jqOjEzt2BLJz569ERkbwzjtv0qJFSxYseIWdO38hOvoaP/8caMztwQfHVxj/\nq6++4ueffwHAxkbHs8/O5vTpk6xfv8a4z5Qpj9C/v1+N8jdVtRbIXrFiBQcPHuSRRx65pbj//PPP\nWFpW/6vboQcnYNXzPjbr+xB6OY1lz/bBQW9es+xvo6b9giK+iN/U4zfl3EX86sU3RZVX7lFRUURG\nRjJo0KCa5lSO0sKCnJAgej7uT+hlCIlIZnivir8aCYIg3OyVVxbcMqzQysqKpUuX11mbCQkJLFmy\n6Jbt3br58tRTz9RZuzVV5ZX7zJkzWbhwIYGBgbi7u99y5d69e3diY2Px9fVl/vz5VY5Njf/lVy59\nvgHHceNYEGqNVws7lv1rQO18GkEQBAGo4so9MDCQrl274uHhcdv3n3/+eQYMGIBOp2PWrFns2rWL\ngICASht0ut+fK//bQuru3XTo8SjnrqQRcSkFW2ttzT/FDe6Gr14ivojf2GKL+I0jvikqLe779+8n\nOjqa/fv3k5CQgEajwcXFhb59y+4ejx071rjvwIEDiYiIqLK4K83M0A++n7Sff6S/4RrnsOdERDL3\n+zYzKXFBEAShYpUOhVy5ciXff/89W7duZdKkSTz33HPGwp6dnc1TTz1FUVERAEFBQbRr166ycEZ6\n//uR1Grszx1FkkvFkEhBEIRaZvLjUtu3b8fa2pqhQ4cycOBAJk+ejFarxdvbu8qrdmOj1jbY9B9A\n5r69+Lkk8We0gqy8ImwsNCZ/AEEQBOFW1S7u//rXv27Z9thjj/HYY4/VqGHboQFk7t9Ht6Sz7Nc7\nczIiGb+u7jWKJQhC7YpIjwLA07bmc7xcFx8fx2uvvcSGDf81bsvNzeHdd98iPT2N0lIDOp2eFSs+\nZOfOX9mx40eKioq4fPmS8eGf1157kyVLFhnHoF/3/fdbWLHifQ4eDK6w/etT/o4dO9K4bdSo+/nl\nlz/YsGEtv/++EweHsgcrAwJGMnr0WAoLC3n//Xe4fPlSubw//fQjTp8+hcFQNk7ez8+/wnYnThzD\nV19twcLCwritpKSEpUvfIjY2BoPBwKxZc8tNqFabGmQ+dwCNkxNWvj0gOIgW2nhCwu1FcReERuLX\ny78DtVPcb2fLlv/h7d2RqVMfBWDTpvX8/PPPBAQ8QEDAKOMfhJun542ICKekpMQ4R8vBgwewt6/Z\ndATXTZo05Zb54T/99CPatfMsN7/NiRPBXLoUxdq1X5CZmcETTzxSaXG/nV27fsXMzJzPPtvApUtR\nvPvuG6xb99Ud5V+Rei/u55IicJLKZoa0Gz6CnOAgBuWF8/VVd3ILisvNSCcIQu3aHrmDk0lnK3y/\n2FBMfkkBJXLZRF9z9r2CucoMtVKNUiFhKL115HQ3p86MbzvapDxycrIpKflnMrHHH3+6WqNNvL07\ncvz4Ufr27U9iYgIqlQq1uvZrxjPPzCIzM5Pdu3cat3Xp0o0OHToCYGVlTUFBAQaD4bazW1Zk+PCR\nDBkyHABbW9OnETZFvc8t8/Xp7cafzVq1xrx9B1wzonHIT+HUxZT6TkcQhBuolWos1P90I1iqLVAr\na794jh//EL//vosnn3yENWs+4eLFiGodN2jQ/ezZswuAP/74nYEDB1fruLVrP2H69OnGaYBvtG/f\nH8yd+xwvvjiXuLhY4PbTACuVSszNy56m37HjR/r06WtSYQdQqVRotWXDvrdu/ZahQ6t3n7Im6v3K\nPSrtKu8Hf8KDbUbgadsGu+EjiA27QK/084SEe9Gvs2t9pyQI94zxbUdXeZX9y6Xd/7yQJEa1GgrU\n7jjuZs08+Pbb7zlxIphjx44wd+7/8eKLL+LnN7zS47p06cZ77y2hsLCAP//cy3vvreDLLzdU2d4z\nz8xm7NiRxvxHjbofgD59+uHr25OuXbuzZ88uVq58n2XLVlYa66+/9rNjx4+sWLG6mp/2Vt9/v5Xw\n8DCWLVtR4xhVaZA+9xbWHsa+PItOndG4N6ND7BUORVwlv9Abc22D3QoQhHueq5UL3Z18ADiRdKZO\n2igsLECrNaNXr9706tWb/v0H8t//bqiyuCsUCnr27M3333+HmZk5en3N5li/ztu7k/Hn/v39+Oyz\nVZXuf+zYEb76aiMffrgKK6uarSS3Y0cghw79xbvvflCn87vXe7eMQlIQlHiC67MeSJKEXcAIFMh0\nTz3H2Uup9Z2SIAg3uF7Yb/65Ns2dO4ugoGPG18nJSRU+CX+zwYPv5+uvNzFokGk3M29n5coPOH36\nJAAnTwZXugJUTk4On376EcuWrcTGRlej9mJjYwgM3M4777xv7J6pK/V+idyveQ/+unqcqMwrtNWX\nrfBi3fM+ErdtwycrkiNnr9GrQ+XTBguC0LRcu3a1XF/37Nn/Zv36z9i0aT1KpRIrK2vefXcJVc9R\nW7bSkkajwc+vev3tlRkzZizvv/8OKpUKSZJ46aXXAHjttZdISko05v3AA+PJz88jIyODhQtfNh7/\n2mtv4uLiUmH8F154HoWi7Bp66NAAEhLiyczM5IUXnjfus2LF6jq5KVytKX9rU2hiGG/u/4jerj2Y\n3uEh4/a0XTtJ+W4zhxy78cibs9Goa7Zu490wf4SIL+I3ttgifuOIb4p6v3L3dvLE3syWE0lnmNTu\nQcxUZV9N9H5+JAb+QNe0C4RGJNK9o1t9pyYIQhN1/nwon3768S3b779/GOPGTayzdn/8cTu//142\nXFKjUVFUVDa889lnZ9OpU910aVVXvRd3haTgPhdffr2yh5PJZ+nj2qNsu5k56j4DUR74nYg/9tG9\n4yP1nZogCE2Ut3enWx54qg8PPjjeuDpTXV+5m6reb6gC3Pd3QT8aH1Rue4sHRmGQFDiFHaO4uOR2\nhwqCIAjV0CDF3cHcDk99GyIzLpOc98/oGLVeT1orH/RFWYTv/qshUhMEQbgrNEhxB+h9/eo9ofyE\nP44jRiADhft3U8/3egVBEO4aDVbcuzp1xkyp5Vh8CKVyqXF7266eXLZpgXV6PLnhYQ2VniAIQpPW\nYMVdq9TQ3cmH9MIM4/SiAApJIt93IACxgT83VHqCcE/LC7tAXtiFem1z+vTpXLoUSUZGBtOmPcSa\nNZ+YdHxubg7Hjx8FyhazPn8+1OQcbjzuo48+NM410xQ1WHEH6O3aE4AjN91Y9erXjWgzR6TI8xTG\nxjREaoJwT0v9KZDUnwIbpO0rVy7h4eHBs8/ONum48PAwY3E/cSKICxfOmdz2jcfNmTMfN7emOw15\ng07i0lrXAicLB04nh5JXnI+FumzGNa/menY6d8Hj6h7Sdv6G61MzGjJNQbhrJH+3mezgoArfLy0u\npjQvD/6ejjfi2adRWFigUKu5qlRgMJTecox1j544TppSYcyEhATeemshCoUCg8HAokVv8cUX64iL\ni6WkpISnn34WX9+exv0//ng5SUkJrFnzSYUF/ttvv2b//j8oLS2lT59+PPnkTJYvX0ZeXi56vZ7A\nwO9RqVQ4O7vg7u7BihXL0GhUqNVaXnllMTk52bz99mLc3NyJjLyIp6cXzzwzm40bPzcet3nzN8yb\n9yJOTi68/fZi4zTFc+cuwMurPZMnj2XAgEGcPXsaKytrNm2qegKz+lStK/eCggKGDBnC9u3by20/\nfPgwEydOZPLkyaxebfoMaZIk0dulB8WlJYQknTZuVyoU2HbvTopaR/axoxSnpZkcWxAE0ynUapSW\n/0x3q7S0RHGHj8bv37+Hnj3vY9WqtcyZ8wI7d/6Cvb0Dq1at5d13P+Tjjz8st//s2XPp2rV7lVfu\nn366ns8/38Rvv+0gNzeHqVOn4+8/lGnTHmfEiNFMmjSF/v39WLnyfRYseIUvv/ySnj17s337VgDC\nwy/wzDOzWL/+K44cOYRKpSp33HXfffctHTt2+jv/+axatRyAuLhYAgJGsXbtF2RnZxEeHn5H56m2\nVevK/bPPPkOnu3WinCVLlrBhwwacnZ2ZNm0aw4cPp23btiYlcJ+rLz9f2sXR+GAGuPc2bvdt78ze\nQ96MTDpCxh+7K70yEAShehwnTany/6WUH38w/ixJEvYPjC07toYP6fTq1ZtXXllAdnY2gwffT0pK\nMqdPn+TMmVMAFBYWUlxcbFJMMzMzZs+eiVKpJCMjg6ysrAr3PX/+HO+9twSNRkVubj4dOngD4O7u\nYVzFycHBkdzcnNseHxZ2nkcffQqA9u29iYmJBsDS0pK2bdsB4OTkRHZ2Ng53tihUraqyuEdFRREZ\nGcmgQYPKbY+Ojkan0+HqWjb/up+fH0eOHDG5uOu1OjrYeXI+LZyE3ERcLMsmDfNuact6e0/y0k6h\n+HM/dqMeQHnDWoSCINQNrbs71j16AZAdfPyO47Vu3ZZNm77l+PGjrFnzCYmJCcyc+VyNF6pISIhn\ny5Zv2LjxGywsLJg+/aFK9zczM2PVqrU4OdkY/zjFx8fdstBGRUOvJUkq915paVnXVHWPbyhVFvf3\n3nuPhQsXEhhY/uZKcnIydnZ2xtd2dnZER0dXq9GbJ8AZ5jWA80fCOZ15hs4txxu39/Bx51hSBwan\nnqAk5Agu48fWKH5tE/FF/MYavzZiO464/7Y/1zT+L7/8goeHBxMmjKFFC1deffVVgoIOM3XqJFJT\nU/nyyy+ZN28eALa2lkARWq26wrYSE6/i6OhAixbOnDt3jsTEBKytNeh0Fmg0ChwdrbGyMsPMTIWj\nozXe3h24cOEkTk5+HD9+ADs7Ozw8PFCpFMY2VCoFdnaW5Y7TaFTY2lri69uNiIizDB7cl1OnTuHl\n5YmjozWSJBmP12rVNT4/daXS4h4YGEjXrl2rPc9ydd381a6lphXmKnP2XzrKEBd/lIqyv4idWtiy\nzsaTgVmhxAT+jLr3wCr7/+6Gmd9EfBG/scW+k/g6nROLFr2OubkFCoWCN95YynfffcuECZMwGAw8\n+eRMY9z09FwyMvIoLCyusC0Hh2ao1VomTpxE585deeCB8bz66iLmzJnHsmXLsLKypV07L5YsWYxa\nbcH//d9cli17m3Xr1iFJKhYvXkJaWi4lJaXGNkpKSklLy6V163+OKyoqIT09l1GjxvPOO2/w8MOP\nUFpayrx5L5GcnI0sy8bjCwvLupUa06yQlU75O3fuXKKjo1EqlSQkJKDRaHjzzTfp27cvMTExzJ8/\nny1btgDwySefoNfrmTZtWpWN3u4EbAn/gQOxR/g/nyfo5NABgKJiA3M+PsiQtBA6J57B+fEn0fUf\nWGnsxvoPXMQX8es6flPOXcSvXnxTVHrlvnLlP2sJrlq1Cnd3d/r27QtAs2bNyMnJISYmBhcXF/bt\n28cHH3xQg5TL9HbtwYHYIxyJDzYWd41aSec29hzIb0dnRSjpO3/Dpm9/JEWDDs8XBKEeHDz4J5s3\nf3PL9kmTHq6VhTrudiaPc9++fTvW1tYMHTqUxYsXM3/+fABGjhxJq1atapxIc+tmuFm6cDblPDlF\nuVhpyoZj9fByJDgsiYxWndFHnSb3zGmsunarcTuCIDQN/fv7lRuSKJim2sX9X//61y3bevbsaeyW\nuVOSJNHbtQfbI3cQlHiSwR79Aejc2h6VUsFfWi/GcJr0Xb+J4i4IglCFRtW/0culOwpJwdH4f2aK\nNNeq6NTKjnN5ZijbdyT/YgT5UZENmKUgCELj16iKu7XGik72HYjJiSM6O8643dfLEYArrcoeUU7f\n+VuD5CcIgtBUNKriDjfM837DZGJd2zmgVEgczLTArFVrck6doCghvqFSFARBaPQaXXHvZN8eK7Ul\nQYknKSktm7zI0kxNhxa2XE3KQdn/fpBl0nfvbOBMBUEQGq9GV9yVCiW9XLqTW5xHaMo/80lf75o5\nq3FH7ehE1uFDlGRmNFSagiBMLPjtAAAgAElEQVQIjVqjK+7wT9fMkRturHbzdESSICQyBdvhAcgl\nJWT8saehUhQEQWjUGmVxd7dypbm1O+fTwsksLHviy8ZCg5eHnqjYLEo790RpbU3G/r2UFuQ3cLaC\nIAiNT6Ms7lC2SlOpXMrxhBDjNl8vJwBOXslE7z+E0rw8Mg8caKgUBUEQGq1GW9x7OHdFJSk5Gh9s\nnEqzu2dZv3tIeBL6wfcjaTSk79mF/PeqMYIgCEKZRlvcLdUW+Dh2JCEviStZZVMJ21praeNuQ3h0\nBrkKDboBfpSkpZEddOdzTguCINxNGm1xh38W0L5xzLuvpxOyDCcjkrEdOgwUCtJ2/troJsoXBEFo\nSI26uHewa4dOY0NI0mmKDGXzJV8fEhkSnozawRHrHr0oio0h79zZhkxVEAShUWnUxV0hKbjP1Zf8\nkgLOJIcC4Kg3p4WzNReuppNbUIxtwAgA0sSUBIIgCEaNurjD7ce8+3o5YiiVOXUxBbPmLbDw7kh+\n2AUKrlxuqDQFQRAalUZf3J0tHGmta0F4eiRpBelA+a4ZANvh4updEAThRo2+uEPZ1buMzLH4EwC4\n2lvi7mBJ6OU08gtLsPDuiNajOTnBx0n+U4x7FwRBqLK45+fnM2fOHKZNm8akSZPYt29fuff9/f2Z\nOnUq06dPZ/r06SQmJtZ6kt2duqBWqDma8M+Yd18vR0oMpZyJSkWSJGwDRgJwedNXtd6+IAhCU1Pl\nSkz79u2jU6dOzJgxg9jYWJ588kkGDy6/fuG6deuwtLSssyTNVWZ0c+rM8YQTRGZcpp1ta3p4OfHT\noSuEhCfRWZFG5p9lf3SK09K5snghTlOmYtG+Q53lJAiC0JhVeeU+cuRIZsyYAUB8fDzOzs51ntTt\n9DHO8152Y9Xd0RJnW3POXEpF2cYTp0ceNe5blJiIQqttkDwFQRAag2r3uU+ZMoUXXniBV1555Zb3\nXn/9dR5++GE++OCDOnuYqK2+NfZmtpxIPkNBSSGSJOHr5URRcSmhl9LIDj6O3ZgHse/XF4qLiPlo\nOYVxcVUHFgRBuAtJsgnV+MKFC7z44ov89NNPSJIEQGBgIAMGDECn0zFr1izGjRtHQEBAnSS7NXQH\n2879wnO9HmVQqz5cjE5n3soDDOrejMdaFuLQry8AUZ+tJWHnbjT29vi89zZaR8c6yUcQBKGxqrLP\nPTQ0FHt7e1xdXenQoQMGg4G0tDTs7e0BGDt2rHHfgQMHEhERUWVxT07OrlGynW06s41f2B3xFx2t\nOqHTKrG3MePYuXge9h9AcnI2jo7W2EycSomVnpRtWzn92mI8XnoFlbVNjdq8maOjdY3zF/FF/LqM\n35RzF/GrF98UVXbLBAcHs3HjRgBSUlLIy8vD1tYWgOzsbJ566imKiooACAoKol27dqbmXG0O5nZ4\n6tsQmXGZ5LzUv7tmHMkvNHDhalq5fe0CRmIbMJLihARiVy7HkC/mfRcE4d5RZXGfMmUKaWlpTJ06\nlZkzZ7Jo0SICAwP5/fffsba2ZuDAgUyePJkpU6ZgZ2dXZ10y1xkX0E4ou7F6/YGm4L8faLqRw4RJ\n2PQfSOHVK8St/pjS4qI6zU0QBKGxqLJbxszMjA8//LDC9x977DEee+yxWk2qMt2cOrM1IpBj8SGM\najWUNu46dFYaTl1MwVBaWm5fSZJwnv4YpXm55JwIIf7zNbg9OwtJqay3fAVBEBpCk3hC9UYapYbu\nTl1IL8wgPD0ShSTR3dORnPxiwq/dumC2pFTiMuNZLDp4k3vyBIlfbRLTAwuCcNdrcsUdoI9b+THv\nPTzLzzVzM4Vajdusf6Ft2YqsQ3+R8t0WUeAFQbirNcni3sqmBU4WDpxODiWvOB/P5nqszNWciEim\ntPT2RVthZk6zOfPQuLiSvnsn6b/9Us9ZC4Ig1J8mWdwlSaK3Sw+KS0sISTqNUqGgWzsHMnOL2HHo\nUoXHKa2tcZ/3Aio7O1K2byPjz/31l7QgCEI9apLFHeA+V18kJGPXjK+XEwDb90VWepzazp5m8xag\ntLIm6esvyQ4OqnR/QRCEpqjJFne9VkcHO0+uZF3j8MWL/Hr0CgCpmQW8+3UIYVfTKzxW4+KK+9z5\nKLRa4tetIfdcaD1lLQiCUD+abHGHf8a8J0oRTB/mZdyukMDTQ1/psWYtW+I2ew6SJBH36SryL0XV\naa6CIAj1qUkXdx8HbyxU5hxPOMGxCwmM7tsCZzsLwqMz2ba/6mJt0b4Drs/8H3JREbEfLacwNrYe\nshYEQah7Tbq4q5Vqejh3I6soG2ySGT+wDSv/7YfOSsPO49f463TVs0JadfPF+bEnKc3NJWbF+xSn\n3H44pSAIQlPSpIs7/DPPe6qy7EaqlYWGl6d2x9JMxVe7wgm/VnHf+3W6/gNwmDQZQ0YGMSs+oCQz\ns05zFgRBqGtNvrh7WLvjZunC2ZTz5BTlAuBsZ8Fz4zoDsPqHUJLS86qMYzd8BHYjR1OcmEjsR8sx\n5FV9jCAIQmPV5Iu7JEn0du2BQTYQlHjSuL1DC1seGeZJTn4xH207Q15BSZWx7MdNQDdwEIXXrhL3\nyUeUFomJxgRBaJqafHEH6OXSHYWkMI55v25QV3eG9vAgPjWPNT+G3jKx2M0kScJp2qNY+fYgPyKc\n+LWfIhsMdZm6IAhCnbgriru1xopO9h2IyYnjj6iD5d57yL8NnVvbE3o5jS17K3/ACUBSKHB5+hks\nvDuSe/oUiZs2IlfxR0EQBKGxuSuKO/wz5v27c+XnjFEqFDzzQEfcHCzZExzD/pNVD3dUqNW4Pfcv\nzFq3JuvIIZK3bhYTjQmC0KTcFcU9Ij2KvdcOAJCWn8GyoFVEpP8zzt3CTMXzE32wMlfz9e4ILlxJ\nqyiUkcLMDPfn56FxcyNjz27SfvmZvLALZJ4VT7MKgtD43RXF3dO2DZO9xhlfZxVl4WrpXG4fJ705\ns8d3RpLg08BQEtKqHg2jtLLC/d8LUNnbkxq4ncSvNnFt89Zaz18QBKG2VVnc8/PzmTNnDtOmTWPS\npEns27ev3PuHDx9m4sSJTJ48mdWrV9dZolU5mXSGkS2H0N6hDemFmXx6egMFJQXl9vH00PNYQHty\nC0r4aNsZcguKq4yrtrXFfux4UCopTkokK/Qc0cveJS/sQl19FEEQhDtWZXHft28fnTp14uuvv2bl\nypUsXbq03PtLlixh1apVfPvttxw6dIjIyKpvWtYFVysXRrUexhv+8/HUt+Fadizrzv6XktLyQyD7\n+7gy4r7mJKbl8ekPoZQYqr5ZquvTD5cZzxpfa9zdMff0quQIQRCEhlVlcR85ciQzZswAID4+Hmfn\nf7o7oqOj0el0uLq6olAo8PPz48iRI3WXbSW6O/kAZcMZZ3d9ms4OHQhLv8h/L2ylVC5fwCf4taFr\nWwcuXE3nf3suVutmaVFsDDYD/VBaWJC5by9xq1ZiyM2tk88iCIJwp6pcIPu6KVOmkJCQwJo1a4zb\nkpOTsbOzM762s7MjOjq6yliOjtYmpmkaF2c9L9o/y1v7PyI48RQuOnse7Tax3D6vPHkfL676i/0n\nY/FsYceYAa0rjSl1aItDv74UZ2UTuvB1cs+eIfbdN2n/8otYtmpZq/nX9fkR8e/e+E05dxG/dlW7\nuG/evJkLFy6wYMECfvrpJyRJqnGjycnZNT62Ko6O1sb4T3s/yvKQT9kR8QfqUjOGNPcrt++ssZ14\n68sg1v14FkuNgs6t7SsO7NmZ5ORsHB2tcX/ldVJ/+oG0HT9z+sX/4Dz9cWz69K31/OuCiH/3xm/K\nuYv41Ytviiq7ZUJDQ4mPjwegQ4cOGAwG0tLKhhI6OTmRkpJi3DcxMREnJyeTEqhLlmoLZnd9Gr1W\nxw+Rv3AsPqTc+/Y6M2ZP8EGpULDmx1DiUqrXzSIpFDiMnVA2H7xSScKGz0n85r/IJVVPcSAIglAf\nqizuwcHBbNy4EYCUlBTy8vKwtbUFoFmzZuTk5BATE0NJSQn79u2jX79+dZuxiWzN9Mzq8hTmKnO+\nDvuOc6nh5d5v667jiZHtyS808NG202TnVX8+Gauu3Wj+2uto3JuRue8Pot9fSnF61bNQCoIg1LUq\ni/uUKVNIS0tj6tSpzJw5k0WLFhEYGMjvv/8OwOLFi5k/fz6PPPIII0eOpFWrVnWetKncrFx41udx\nlJKC9aH/5WpW+fsCfTq6MLpvS5IzClhdzRE012mcXWj+ykKse/WmICqSa2+9Tl54WG1/BEEQBJNI\ncgM8V99Q/V6nk8+x7uxXWKotmOf7HM4Wjsb3SmWZzwJDCQlPpr+PK0+MaH/b+woVxZdlmYw/fif5\nuy0gyzhOnIx+6DCT703cDf2CIn7DxG/KuYv41YtvirviCdXq6uLYkYe9xpNTnMvqU+vJLMwyvqeQ\nJJ4e5U0LZ2sOnolnd1DVo35uJEkStkOG0Wz+iyitrUne+i0Jn39GaUFB1QcLgiDUsnuquAP0c7+P\nUa2GklqQzurTG8gvyTe+p9UoeX6iDzorDVv3RnIqMqWSSLdn4elFi4VvYNa2HdlBx7n2zlsUJSTU\n5kcQBEGo0j1X3AFGtBxCf/fexObEs/bMlxTf8BSrrbWW5yf4oFIpWPvTOWKSckyOr9Lr8XjhJfT3\nD6UoLpZrSxaTczKkyuMEQRBqyz1Z3CVJYrLnWLo6duJixiW+PL+53FOsrVxteHq0N4VFBj7adoas\nXNNXZJJUKpwefgSXp2cil5YSt3oVKdu3ibnhBUGoF/dkcQdQSAoe936YtvpWnEw6w7aLP5WbhqBn\neyfGDmhFalYBn2w/S3FJzVZksundl+b/WYja0Ym0X3cQu+JDDNl1d9NFEAQB7uHiDqBWqnmm8+O4\nWbrwZ8xhdl0tP+PlmL4t6dXBicjYTDb9Fl7jBTu0Hh40X/g6lj5dyLtwjqtvLabg8qXa+AiCIAi3\ndU8XdwALtTmzuj6FrVbPz5d2cjguyPieJEk8ObIDrd1sOHIugS93hnG2BjdZAZQWlrjNnoP92PGU\npKcR/d47ZB74s7Y+hiAIQjn3fHEH0Gt1zO76NJZqC74N/56zKeeN72nUSv41vjN2NloOnI5nzfYz\nNW5HUiiwH/0A7nP+jaTRkvjVFyR8uZHSYtP79AVBECojivvfXCyd+D+fJ1BKSjaEfsOlzKvG9+JT\n87C20ABwLTGbBZ8e5vyV1Bq3ZdnJhxYLF6Nt3oKsvw4Q/d67FKemiGX8BEGoNaK436CVrgVPd5qG\nQTaw5vQXJOQmAtC+hS1Pj+pg3C81q4DAg1dIycivKFSV1I6OeLz8KjZ9+1N45TJX31pM0ub/iWX8\nBEGoFaK436STQwemtp9Ibkken5zaQHpBBgBBYUk80K8lEwa3xc3BksiYTF7/4jjHzifWuC2FRoPz\nE0+hHzKM0pwcimKiyQo9x7X33hHL+AmCcEdEcb+NPq49eLD1CNILM1h9egN5xXm4O1oxdkBrHh/d\nkQf6teSJke0pLYW1P51j/Y7z5BfWbLpfSZJwmjIVlxnPGLeVpKYil1S9vqsgCEJFRHGvwNAWgxjU\nrB/xuYmsObOJLu1sje/16uDMAB83Fj/Rk5Yu1hwOTeCNL4K4FJdVScTKFSUkYBswEivPdpSkpRK7\ncjlxq1dRnFqz0TmCINzblIsXL15c343mmTBnuqksLbW1El+SJDrYeZKUl8y5tHASchOxVFmSU5qN\npVQ2O5uVuZp+nV0xlMqcjkzh0Nl4FJJEW3edybNBGnKysR0yjNZjR5FfIiEXF5N3LpTMA/sBMGvV\nGkmpvOPPVVvnR8RvfPGbcu4ifvXim0IU90pIkkQnB28uZ17lXFo4YekXicmKo5ezr3EfhULCu6Ud\nnh56zl1J4+TFFMKvZeDd0hZzbbVXMUTr5m7MX3Zrjk2//micnMgPDyf39Cmyg4+jcXZB4+RcRaTK\n3Q3/wEX8+o8t4jeO+KYQ3TJVUCtUDPYYgFahIasom/PJF1lx4jMi0qPK7dehhS1vPNmL7p6OhEdn\n8PrG4wSHJdW4XUmSsOnTj5Zvv4v+/qEUJyURu/JD4j77hOLUmg/DFATh3iCKezV0dujAs12eML42\nlBpwtrh1rVgrczWzxnXi0QAviktK+TQwlC9+vUBhUc3mpYGyJ1udHn6EFovKphHOCQnmysL/kPbr\nDkqLxU1XQRBur1r9BsuWLSMkJISSkhKeeeYZhg0bZnzP398fFxcXlH/3B3/wwQc4O99Z10FjdDE9\nivs9BnIu/QKXs67xzvHlPOo9mY727cvtJ0kSg7q64+WhZ+2P5/jrTDwRMZk884A3LV1saty+1qM5\nHi/+h6wjh0nZtpWU7dvIPHQQp6nTsOzY6U4/niAId5kqi/vRo0e5ePEiW7ZsIT09nXHjxpUr7gDr\n1q3D0tKyzpJsDFytXOju5MNMhymsOLCR4MSTfHp6I/4eA3igzQjUivKn0tXeklcf7cH2A1HsOh7N\n21+FMH5ga4bf1xyFiTdbr5MUCnT9+mPVrRupgT+Qse8PYld8gJVvDxwnP4zazr42PqogCHeBKot7\nz5498fHxAcDGxob8/HwMBoPxSv1e0d2p7BxIksR074cY5NGfL879j73Rf3ExPYonOk7F2bJ8V41a\npWCyfzs6tbJn/Y7zfLc/itDLaTw92htba9NujtxIaWGJ09Rp2PQfQNI3/yUnJJjcs2ewH/MgtkOH\nI6mqfyNXEIS7k0kLZG/ZsoXg4GDef/994zZ/f3+6d+9ObGwsvr6+zJ8/3+RhgE1VQUkhm05+x95L\nh9AqNTzZfTKDWvW57efPzCnk4y2nOH4+AWsLDc9P7krvTq53nINcWkrSvv1c/fK/FGdmYe7uRuuZ\nT6Pv2uWOYwuC0HRVu7jv2bOHtWvXsnHjRqyt/1mFOzAwkAEDBqDT6Zg1axbjxo0jICCg0lhNfQXy\nm+OHJJ7m2/DvyS8pwNepCw+3H4+5yvyWY2VZZt/JWLbsjaS4pJRB3dyZ7N8Wrfqfb0E1zd+Qm0vq\nj9vJ2LcXZBmrHr1wfGgKaju7KvOvTSJ+w8VvyrmL+NWLb4pqjXP/66+/WL16NevXr0en05V7r337\n9lhYWKBQKMjKyiImJob77ruv0nhNfazpzfHdrFzwderClaxozqeFE5J4mpY2zbE105fbT5IkWrna\n0L2dAxdjMjgTlcqJiGTaNdOhs9LeUf4KjQbLzl2w7NqNwpgY8s6dJfPAfiSFErNWrZAUijuKX10i\nfsPFb8q5i/jVi2+KKot7dnY28+bNY8OGDdjddBWYnZ3Nc889x4gRI1AqlWzcuJFevXrRrl27Shtt\n6if4dvEt1Obc51L2cNPZlAscTQhGISlorWtxSzeNjaWG/j6uFBQaOBOVysGz8WjVSgpLDGTnlWBl\nVvM+c5VOj02//qjtHcgPDyP39ElygoPQuLpRnJqClJWOwUpfdaAauhv+B2qq8Zty7iJ+9eKbosoq\n8uuvv5Kens7cuXON2+677z68vLwYOnQoAwcOZPLkyWi1Wry9vavskrmbKRVKRrcejqdtW748v5mf\nL+0kPO0ij3Wcgl5b/huPWqVk6lBPOrW2Z+Mv59m8NxILMxXNna158eFud5SHpFCg6z8Aq27dSQn8\nnsz9+4j5cBlKa2syXF1xe/GVO4ovCELjZ9IN1drS1Pu9qhM/pziXby5s40zKOSzVFkzv8BCdHbxv\nu29IeBKbfgsjt6BsZkk7Gy2PDPWkWzvHWsk54899JH+3BbmgAAClTofDxMno+vStlfg3aizn/16M\n35RzF/GrF98U4gnVOmKltmRm50eZ7DmWQkMRa85sYmvEjxQbbn2q1NfLiZemdje+TssqZP2O8wT+\ndYm8gjt/ClXvNxiPl181vjZkZpK4cR3x69ZQGBtzx/EFQWh8xIDoOiRJEgOb9aWNvhUbz/2PP2MO\nEZlxiSc6TsXVsvxTvMHhZYuBaM3UnL+UyrXEbH46dIU9wTEM7+XBkB4eJk1EdrOckGDsxjyIhbma\njKirFCUmkH3sKNnHjmLZrTv2o8Zg1rLVnX5kQRAaCTErZD3Et9FY08e1B7kleZxLDeNIfDDWais8\nrN2NN1uz84u537cZfbs2o6iwhGlDvbDQqoiKy+JMVCp/nopFlmWaO1uhUpr+hcuQk42t/xDcevuS\nl1eI08PTMGvZiuKUZPIvnCfzwJ/kR0WisrdHbe9gcvzrGuP5v1fiN+XcRfzqxTeFKO71FF+pUNLZ\noQPuli6cSw3jRPIZ4nMTaW/XDrVSjbuDpTG+3kKNSqmgXTM9g7q5o9UoiYzJ5ExUKgdOxyEh4WFi\nkb9xSuESvSOSJKFxccGm/0AsPL0oTk8j/8J5sg4dJO/CeVQ6HWonJ5MfSGus5/9eiN+Ucxfxqxff\nFKK413N8F0tnejp342pWDOfTwglOPEULGw/s/h4Tf3N8tUqBl0dZkVerFFyMyeB0ZCp/nYlHqZBo\n7myFUlH9In9zfEmSUDs6ouvbH4uOnTBkZZJ34TzZx46Qe/oUSisrNC6u1S7yjf38383xm3LuIn71\n4ptCFPcGiG+uMuM+V18UklQ2Jj4+GIBSuZTsG1Z6upFapaB9c1sGdXNHqVAQEZPBqcgUDp6JR6VU\n4OFkhVJRdQGuLH+1nR029/XBqlt3SvNyyQu7QE7QcXKCg1CYm6Fxczc+DFWT+LVBxG+Y2CJ+44hv\nClHcGyi+JEm0s22Dp21bwtIuciblPKeTz3MtM5beLj0rPE6jUtKhhS1+XdyQJIiIzuDUxRQOh8aj\nUStp5miFopIiX538VTod1j16Yt2rN6WFheSFh5ETEkz20SNIajUa92YVLvnXVM7/3Ri/Kecu4lcv\nvilEcW/g+HZmtjia23EhPZL8knzS8jMITjiJq6UTDuYVT+GrVSvp2NKOgV3ckJEJv5bByYgUjpxL\nwEyjxN3R8rZF3pT8lVZWWHXrjk3ffsgGA/kRYeSeOknmob+QAG0zj1tmoGxq5/9uit+Ucxfxqxff\nFKK4N4L4zpZOdLT34q/YowDkluSRVpBBa11LLNUWlR6r1Sjp1MqeAT6uGAwyYdcyOBGRzLHziZhr\nVWVF/ob+8prkr7SwwMqnC7oBA0GhIP/iRXLPnCLjwH7kkhK0zZqhUGvIC7sgpjdowPhNOXcRv3rx\nTSGKeyOJfyDmCO30rfF0akVmfg7R2bEcjjuGUlLS0sYDhVR5X7eZRkXnNvb06+xCsaGUsKvphEQk\nE3QhCUtzFW72loRfyyArr7jGc9cozMyw9O6I3m8QCo2Ggqgo8s6eIXP/PkoLCsg8+Bc54eFY9e5X\no/jV0VR/v/URvynnLuJXL74pRHFvJPFzinPxa9aXvm26oShW083Jh4j0KM6knONcahgtbZpjo636\n8WNzrYoubRzo18mVohIDYVfTCQ5LJjg8mTNRKUTGZNKno8sd5arQaLDwao9+sD8KC0vyIy+Sd/4c\nJelpFCYlk3PqJBpnF9QOtTN9wo2a6u+3PuI35dxF/OrFN4Uo7o0k/vUnVi0ttegkW9ysXOjt1oPs\nohzOp4VzOP44BtlAa11LlFVcxQNYmKno2taBPh1dSEjLIyo2i+z8YpLS8zkcGo+dtRY3hztbGlFS\nqTFv2w79/UNBlsm/GAGUTW9QGBMDsoza2QWFWn1H7dyoqf5+6yN+U85dxK9efFOI4t6I42uUGro4\ndqKlTXMupl8iNPUCJ5PO0szKzTguvsp4Zmr6dHShlYs1R88nApBXUMKJiGRik3OxNFNhrzO7o9Wz\nJKWSvIhwzD29sHRxoriwmKL4OHJPnyLjj98pTkpCaWODytb2jlfpupt+v00ptojfOOKbQhT3JhDf\nycKBvm49KTQUcj41nKPxweQU59FG1wqVonr950fOJeDVXE+ntg6oFKBUKgi7lsHh0ASOnEugqNiA\ns605Zpqa9cdfn96g+RA/ClVanCZPRWllRXFCAvnhF8g6eICcEyHIBgMaZxcUGk2N2rkbf79NIbaI\n3zjim0IU9yYSX6VQ0dG+PV627biUeZVzqWEEJZzExdIJR4uq54K5PndNv27NKCws4ZGhnnRubY8s\nw6W4LEIvp7EnOIbopBwszFQ46M1Nusq+eXoDhZkZ5u080fsPwbydJ3JxCfmREeSdPU3GH79TlBCP\n0soalZ29Se3crb/fxh5bxG8c8U0hinsTi29npqeva9lDTufTwjmecILU/DTa6FuhUVZ8NXzz3DWS\nJGFnY0Y3T0f8u7tjZ2NGamYh4dcyOHIukcOhCRQUG3DSm5s0G+XN+UuShMbRCesePdH5DUZpY0Nx\nUhL54WFkHTpIdtAx5OLisqt5bdX/eBv6/Dfm+E05dxG/evFNIYp7E4yvVCjxsmuLj4M317LL1m09\nFh+CnbktLhaVT/Z1u/hqlZJWrjYM6uaGTxsHZFnmUnwW5/6+mr+amI25VoVjNa7mK8tfodWW3YD1\nvx+L9h2QDQYKoiLJCz1L+p7dFMXForS0RGVf8dV8Yzj/jTV+U85dxK9efFNUq7gvW7aMjz/+mM2b\nN2Nra0ubNm2M7x0+fJh///vffP/99yQlJdGrV68qG23qJ7ixxLfRWtPHtSdapZYLf09CFpMTT1t9\nK8xUZibHlyQJW2st3do5cr9vMxx0ZqRlFxB+LYOj5xM5dDaegiIDTrYWFV7NVyd/SZJQOzhg7dsD\n/SB/VHo9JampZVfzRw6RffQIclEhaicnFGblP0djOv+NLX5Tzl3Er158U1S5zN7Ro0fZsGED69at\nIz09nXHjxrF//37j+yNHjmTDhg04Ozszbdo03nzzTdq2bVtpo019qavGGD8pL5n/hX3PxYxLmKvM\nGNd2FH1de91yBVyT+FcSsvjzVBxHzydSWGRAkqBLGwcGdnXDp7W9cZqDsKvp6PUWuOhM+0cIIMsy\nBVGRZB74k+zg48hFRaBUYuXTFZ2fHxbenciPCEevt6DIpYXJ8aursf5+Gzq2iN844puiys7Unj17\n4uPjA4CNjQ35+fkYDDCqqUUAACAASURBVAaUSiXR0dHodDpcXV0B8PPz48iRI1UWd6H2OVk48ny3\nmRyOO84Pkb/yv7DvCU44xdT2E3G0qHiOmupo6WJDywAbHhrcluMXEvnzVBynIlM4FZmCrbWWAT6u\nDPBx48eDl1FrlMyb1MXkNiRJwrxtO8zbtsNxylSyjx0h88Cf5JwMIedkCCo7e5AkMuxtcXvx1aoD\nCsI9zqQFsrds2UJwcDDvv/8+ACdOnGDDhg2sXr0agO+++47o6GjmzZtXN9kK1ZKal866kG85EXcW\njVLN5E4PMMrTnwspkQB0dPK84zaiYjLYdewq+0NiyC8sKfeedys7pgV0oHPbmq/oBGVX8zmRUURv\n3kJ6yEn4+5+qwswMx4H9cR8/DnPXO3vaVhDuVtUeBrFnzx62bdvGxo0b77jRpv7VqPHHV/Gk1zS6\n2J7mu4gf+f/2zjy6rfLM/x9Zq7XYlmwtXmPHjp3ESQjZIBRCoQlbp/ygU0qaSaacYTjDAL+WFmjL\nzhRaSIbDoQQOUEopQwskDSXQGRq2CT/SkoQkhDh2Fide4l2LLdvavEn398eVFa+xZcshdt/POTqS\nfa++ei3f+9Wj533u87526C0+rf4cCQmjLhnbwlsmPc4UrZIbVs3m2gtn8flRJx/ur6fBHQDgRJ2X\ntz6upLbRy3mFGZNa+5U0O9Zbf4Cxpor6XzwKQKSnB+cHH+H84CO0ebMwLVuOcekyNPbJG/30+P+e\nfW2hf27ox8O4zrpdu3bxwgsv8Jvf/AaT6fQL2Gw2PB5P7Gen04nNZotrAIKpQaFQsMy+mLnmOfy2\n4nWOe0/IG3zwxOe/4vqib1JimXz6TKtRcsl5WbR2dlGUk4q7o5tTzZ0cqHRzoNKNSpnEwtkWlpXY\nOK8oA/0Em5YFysqwfOv/YDBo8bf70Tgy8e3fR/BoBd11p/D8aRva3FyMS5djWrYCjUNE9IK/b8Y8\n03w+H5s2beJ3v/sdaWmDL3nPycnB7/fT0NCAw+Fg586dPPnkk1M2WEH8GDUGfnD+Lexq3M2bx98G\noN7fyLYT73Jh5jKWO84nRRNfRDAS2VYj1821YbWaeG9XFVkZBvYfc7H/uIuDJzwcPOFBpVRQmm9h\n2Vwb58/JQK8bf88ZbXY2pmUrsFpNVP/lY0zLVpB68SWEAwH8Xx7Ef2AfgYpyuuv/ROv2P6HJzpEX\nHFm2HE1m1qT/PoFgujFmzn3Lli1s3ryZgoKC2O8uuOACSkpKWLNmDfv27YsZ+hVXXMHNN9885otO\n969G01H/f6o/ICJJ+KQOqjx1uEOthKUwSYokStPncmHmMhakzx13O4PRGGn8za2BqNG7qXf5AVAm\nKZifb2HZXCvnz7FiTB6f0Z/p/QkHAwS+/BLfgX0EK8qR+uS5AE12DqalyzAuWx67knYi+olApGWE\n/mT04yGuCdVEMd3f4Omo/4WrjCW2RVitJt6v+BvFaYXsd37JnuZ91PubADCqDSy3n88FmcvINU0s\n2h1r/C1tQQ4cd7HvmIs652mjnzfLHIvoTfrRr7Qd7/sTDgYJHIoaffnh00aflRVN3SyX14QdUCoa\nPHZ0WpdaTtdjU+iPXz8ehLkLfRp8TextOcDnLV/g75UnRbONmXLaxn4+Jo1xUvqj4fIGOXDczb5j\nLmpb5OckKRTMnZXGshIbS4qtpBhOG/1E6+jDoRCBsi/lHP3hstNG78jE2J+6yc6h4T+fQK1R4bjz\nnrj040GYu9CfjH48CHMX+jH6In1UtB5nb/N+DrceJSJFSFIksTB9HhdE0zbKpJEXxh6P/pnwtIfY\nf9zN/uMuqps6AVAooCQ3jWVzbSwttvLCOxUTrqPvJ9IVwl92CP/+fQQOlyH19sqvpVbHHicXl5B+\n7XXo586b8OuMhjB3oT8Z/XgQ5i70R8TX42ef8yB7mvfT6G8G5LTNCscSLsxcRrYxc1L6Z6K1o4sD\nx+Uc/cnGjmHb8x0mbriskHmzLJN6nUhXF4GyQ/gO7MN/6EuIRvQKnQ5D6QL0pQswlC5EnT65i8AG\nIsxd6E9GPx6EuQv9Man3NbKneT/7nAcJ9AYByDVlc6FjGcvsizFq5I6Tld4q0tL02BQjG/9EaOvs\n4kClm88ON3MqmqMHSE/RsmB2OgtnpzNvlnlytfSA+0/b6HU5kbythJqaiYRCsW0aRyb60gXoS0vR\nl8wbV/fK0RDmLvQnox8PwtyF/rjpi/RR7jnKnpb9VLQeJyJFUCqULMyYz4WZS/no1P9Do1FxewIu\nkhrK9l3VdPeEaQ/20uD00e7vJtAlR9rKJAVzclJZGDX7bKsh7hWffPs/j5VaVr33MbrcPAIVhwlW\nlBM8fgypuxsAhUqFrmhOLLLX5uSiSBp72cN+hLkL/cnox4Mwd6E/ITp7fHze8gV7mvfTHHAO2pZl\ncPDton9gXvrk2xz0s++Yi+UD6uiXFlupbu7kcFUrh6tbYxOyAGaTlgUFFhbOTmd+viWuC6dGen8i\nvb10VZ0kUFFOsKKc7rpTsW1KUwr60lLZ7OcvQJWaGrd+opgux47Qn7h+PAhzF/qTQpIk6n2NfFz/\nKfudX8Z+r1PqmJ9ezKKMUkrTS9Cr9Ql5vdHG3xnooaKmjcPVrZTXtOEPyZOjSQoFRdkpLCyUo/pc\nm/GMUf143p++zk6CRysIlpcTOFJOuOP0vIA2Nxd96UIMpQvQFc0ZtDj4VJdaTrdjR+jHrx8PwtyF\nfkLov0gqiJ86bzO+Hh+tXV4AkhRJFKXNZlHGfBZmzCcjeeIToeMZfyQiUdvi43C1HNXXNHXSf5Cn\nGjQsmC1H9aUFFgwDrpKdSKmlJEn0NDREUzgVhE4cj5VaKjQakovnYlggR/Wu3786paWW0/XYEfrj\n148HYe5CPyEMvUjqfOtCmgItHPYcocx9hFO++ti+WQYHizLms8haSq4pmyTF1OasfcEeKmrbOFzV\nRnlNK75gtPxRAYVZqSycbWFhYTpvfnwSzWRLLbu7CZ04TqC8nOCRcnqamobto7bbsXzzWlJWXhT3\n3MCZmK7HjtAfv348CHMX+mdFv727g3LPUco8RzjuPUlfRI5uUzUmFmTMZ1HGfErMRaiVZ25DMNnx\nRySJUy0+yqtbOVzdRlVTB0PPALs5mX+4KJ+LFjgmbb69ba0Ej1TQue9zQhXlg7YpTSkkFxeTPKeE\n5OLiuCdnhzJTjx2hf1o/HoS5C/2zrt/V180x7wnK3BWUtx6NlVdqlBrmW4pZmDGfBenzYiWW/UxF\nqaU/1MuR2jb2Hmnh4InWQdtSDBpKctOYm5dGSZ6ZzHT9hM3e887bIElowt34GlpISk4mdOI4fV5v\nbJ+k5GR5wZI5xSQXl6DLL0ChmtxkcCIR+l+9fjxMrjhYIJgAOpWWxdYFLLYuICJFqO44RZmngsPu\nI3zpLudLdzkKFMxOzWeRVY7qbXor79V8iFqtTGippTFZzYp5dpo8AXJtJsIoaHB2otOoOFbnZd8x\nuQ8OQIpeTXGeOWb2WXGY/UhdLSVJos/jIVh5nNCJ44ROVBI4XEbgcBkgXzWrm11IcnGJbPiFRZOq\nsRf8fSEid6F/Tuk7Ay7KPEco8xyhpuMUUnQqVJ2kpjci58oLUwv4h9lXUGwuPJNUXAwttVw+14Yk\nSbi8IY7VeTle186xOi/t/tMLIJv0akpyZaOfm5dGVsbY9fVjvT997e2ETlQSOnGcYGUlPY0NsRWo\nUCrRzZolG/2cEpKL5qA0yn1/pnvTM6E/Pv14EOYu9M9ZfV+Pn3LPUQ57jlDRepw+KXrRkkLJ7NRZ\nFJsLKTYXkZ+SO+lWxf2cafySJOFqD8WM/nhdO15fd2y7MVlNSV4ac/PMlOSmkWU1kDTA7CdSjRMO\nBAidPEGoUo7su07VQjgc267JziG5uITQsaNoUoxk3XNf/H/0OJlOx85M1Y8HYe5Cf1rov1u1g9ZQ\nGx3hDpw+D74e/6CovjA1P2r2heSZcsZscDYa8YxfkiTc7SGO1bVzvM7LsZHMPjeN4qjh/+HDyoRU\n43RVV0VTOZWETp6I9cQBQKVCN3s2xkWL0RXMRjcrnySdbsKvN5DpeuzMJP14EDl3wbQgx5TFtYVX\nxUotS8xFnGyv5ri3ihPeKo55T3AsupSgVqmhKG22bPZpheSYsuIqtxwvCoUCm1mPzaxn1XlZstl3\ndHH8lGz0x+u9seUGB/KT5z/jG0uyWbU4O+6eOElaLfp589HPmw+A1NeHb/8+Wn7zorxdo6WrspKu\nysr+QaLJypaNvmA2uoICtNk5KJQT+/ATTB9E5C70Z4S+r8dPpbeKynbZ7J3B04aarEpmTr/ZmwvJ\nNNhHNftEjl+SJDwdXRyr83Kw0s2XJwdX4ygUkJ1hpCg7hcLsVAqzU7Gbk+OuyPG8Iy+faDBoCQS6\nSb34ErpqquVbdTVdp2qRek7PFSg0GnSz8tHlF6CbXYiuoABVesak5wsmi9AfWz8exhU2VFZWcttt\nt3HTTTexfv36Qdsuv/xyHA4Hymgk8OSTT2K32+MahEAwWUwaI0vt57HULqc82rs7qIxG9ZXeKso8\nFZR5KgC5dfFAs7frbSgUCiq9VbikxJVaKhQKrGnJWNOSae3oIs9uIkmlpNHpIz1VR1VjB7UtPhrc\nfj75MroaVrKa2Vmy2RdlpZCfmTJmdD+0EkednoE6PQPTshUASOEwPU1NhGqqoqZfI+fxT1TGNJSm\nFHQFBacj/PyC2GQtyBO2HS16mMIJW0FiGdPcg8Egjz76KCtXrhx1n5deegmDwTDqdoHgbJOmTWWF\nYwkrHEsAaA15Y1H9ce9JDroPc9B9GIAUjYlicyF1nQ2k6k3ced6/J3w82VbjsGocgL5whHqXn5ON\nHVQ1dlDV2ElZVStlVXKUP57o3rRsBcdOeWnp6MYRNfSBKJRKtLm5aHNzYdXXAbmXfVfdqWh0X0VX\nTQ2BskMEyg7Fnqe229Hly2bf+dkuOo0GHD/+acLfG8HUMGZapq+vj76+Pl566SXMZvOIkfuf//zn\nuMx9un81EvrTW1+SJNyhVjmqb6+iovUYob6u2HatUstS2yK+kbcqFtUnivGMv93fTVVjJ1VNHbHo\nvrcvEtsei+6jEX5BZgrPbCub9CpVfR3tdNXUnE7p1FQP6msP0ZROYRGGBQvR5uSizc1DlZIy4dcc\nyHQ4dr5q/XgYd8598+bNo5r7kiVLaGxsZOnSpdx1110JPRkEgqlGkiQONJWx6a8vAKBAEavEyTTZ\nWJ69mBXZ51GUnj8lE7Nj0dsXoaapQ56oPdXGsVNeXG3BEffNshq4/tIivrE8D7VqcmOVIhFCTc20\n7tlD3WuvA4OXI+xHbU7DkJ+PoSA/ej+L5OxsMWn7FTNpc9++fTuXXHIJqamp3H777Vx//fVcddVV\nZ9Sa7p+eQn/m6f9P9QcA6A1aOnwBsowODrkrONJ6jJ7oxVMmjZFFGaWcZy2l2FyEegK19Ykaf4e/\nm5PR6P5IbRt1A1apAlApFWRbjeQ7TMxymMh3mMjOME7I8IdO2KZcuJLu+nq6G+rk+/p6+tqGTBar\nVGiystHm5skpoWiUrxzlG764CGt8+vEw6VLI6667LvZ41apVVFZWjmnuAsG5RqbRMair5RLbIlY4\nltAT7uV4tA9OmecIf2vay9+a9qJVaihNn8t5GaWUZswlWZV8VsebatSytMTK0hIr23dVs6gwnd4w\ntLT6MRu1sYnaUwMWMVEmKci2GqKGn0K+w0SO1YBadeYIe+iErcbuQGN3YFq2PLZPOBCgu6F+kOn3\nNDYMWtgEQGWxxIxeNv081DYbre9up3MK2yH/PTIpc/f5fNx55508//zzaDQa9u3bx5VXXpmosQkE\nZ40ltkUjPtYo1SyM9qH/Xn8fHHcFh9zlfOEq4wtXGUqFkmJzIYsySllknU+a9syrMSWaM03WNroD\nnHL6ONXio7bFR73LL0f5h+RFz5VJCrIzDORFo/tZDhO5ViMa9WnDH2vCFkBpMKAvmYu+ZG7sd1I4\nTI+zJRrd18XMf+jELQoFSBIhIHTfT0n7xmpSVn4NpT4xC7z8vTJmWqa8vJyNGzfS2NiISqXCbrdz\n+eWXk5OTw5o1a3j11VfZvn07Wq2W+fPn8+CDD46Zc5/uX42EvtCXJInmgJND7grKPOXU+Rpj22al\n5HJeNH3jMJwuC56Krpbxjr0vHKHJE5DN3umjrsVHncs/aMI2SaEgK8MQM/tZDhN/3HkSrVY1qQnb\n2Bh8nYMMP1RdTZ+zZdh+qowMOcrPyUGbk4c2Jxe1zTbhtsjnyrEzGf14EBcxCX2hnwD9ti6v3PDM\nXcGJ9moikmyWdr01lqffXvUeGvXULCAOEx97OBKh2ROktkWO8E85fdS5fPT0Robta9KrWTLHypIS\nKzlWI2lGzaQLKDzvvA2RCOq+bgIuD2qbje6GBrrr6wh3dg7aV6HRoM3OQZOTczq9k50zai5/IOfq\nsROPfjwIcxf6Qj/B+oHeIBWtx4ZNyPZj0aaxMmsFKxxLSNeZE1Zdlsj3JhKRaG4NUNvio6KmjT1H\nnCPuZ0xWk2M1kGMzkhu9ZaUbBqV1xsK3//Nh7ZD76evooLtRNvqehga6G+rpaW6KLWXYTyyX33/L\nzUVts8cqdmbKhG08CHMX+kJ/CvX7J2T3NO3nS0/5sO3JKh05xixyjFlkm+T7TINtQl0up+q92b6r\nGgC9XoOnLUhJnpkGt58Gl596tx+Xd0gtvAIcFj05VqNs+lYjOTYD6Sm6UT/I4umYKfX1ybn82ASu\nfAu3tw8eh1otV+zk5BKqPI7akIz933+Aymye1IpXo3GumbtoHCYQTCH9E7J1nQ1kGR0kaSScHW1k\nGRw0+Jto8Ddxsr2GE+3VsecoFUoyDXbZ9E1Z5BgzyTFlnfWKnH6GTtj2V+n009XTR6M7QH3U8GXT\nD9DcenqhE4BkrYrcaJTfb/rZVgM6jYp3/loz7ouwFCoV2uwctNk5cMHpK+f7fJ2x6D5m+vV1dJ+q\nBaDXDTU/vQuUSjSOTDQ2O2q7HY3djtruQGOzo0xNnTHX6YjIXegL/bOgP3QB8YEVOd3hHpr8zTT4\nm6j3yYbf5G+mNzI49ZCus8TMPteUTY4xizRtaqwvzlRO1kL87ZBbO7tocA0wfbeflrbgsDVr1aqk\n2IRuVrqeqy7I48JSByrl5KNrKRzGf/gQzc8+A4B+wULCnZ30upxEurqG7a/Q6mSzt9nROOxobI7o\nB4BjUK+doZyttE88CHMX+kL/HNQPR8K4Qx4afE3U+5toiJq+vzcwaD+DSk+2KQtX0I1eo2PD3Btx\n6G1olJqvbOxnors3TJMnEEvpNLjkWvxQT3jQfsokBTZzMtkZBrKit+wMA3aLPm7TH3gRVjDYQ/q1\n1yFJEuHODnqcTnpdTnpaWuT76M9Dr8IFSNIbBhi/Q76P/tz07K9QT3GdvjD3c+TkFPpCP9H6kiTR\n0dMZM/oGXxPVHafo6Okctm+6zoLDYMNhsJGpt+Mw2HEYbCSrJr5wx1Tm9Lt6wnSHI3i8IcwmLc2e\nAI2eAF2jmH6/2fcbv+MMpu/b/zmN1jmkpekxnDg0aMJ2JKRIhL52L71OJz0uJ70tLfK900mP2zVo\nJayhKFNSMCxchH5+KWqrHY3NdsaIPx6EuZ/DJ6fQF/pToV/bUcd/HngWgKW2xfh6fDQHnfh6/MP2\nTdOmkmmw49DbouZvJ9Ngx6A+8wVDU5n2GWn9WpA/zLy+bpo8AZqiZt/UKj8OdQ822CSFArtFNv2s\ndAPZVvnebtGjViWx8Q9fTLqxGshpnt62VtnonS30Op1yd82TJ0Z9TpJeL0f5Vitqmx21zYbaaosr\nx9+f9klduGDcYxXmLvSF/jTXH9gXJxjs4ZsFawC5JLMl4KIl4KQ56KQl4KI54KS9u2OYhkljjEb4\n/YYv35vURhQKBU9/8QJqtXLKavQhvgvIvL5u2ejdsuE3ekY2fQWgGpDTz7ToWb08hwvnO+JeBWs0\n+tM+ep0Kf2sHhkWL6HG56HW56HU56XW76XW7hpVvgly332/0apts/hqbHbXVisqSHqvqqd/0OGqN\nioW/+Pm4xyWqZQSCac7Qvjj9GNR6CtPyKUzLH7R/qK9LNv2gbPwtASfNAReV7XIL5IHokrSggK6w\nvDbsL/Y+xZq8S1lqXzzhdWoni0KhwJKiw5KiY0FBeuz3kiTR7u85HeVHb/Wu099gmtuCvPZ+Ja+9\nX0mqQYPDoseRrsdh0ZMZvc9ITSYpafwVM0N77xgWLGLoJVVSJEKf10uv2yWneFwuet3yB0CPy0VP\nY8NwYaUSZUoKUk8PkUCA0PA9zvw+ichd6Av9maE/We3ucA/OoCsW4TsDLpqDTlxBz7B9kxRJZOgs\n2PQZWPUZ2JKt2PQZ2PQZpGlTJ9Qaeapz+r0Ridb2ELa0ZFragjS3Bmnr7GKoAaqU8tq4DsuAW9T4\njcnqEV8jnjr9ociTu52nzd4djfijjyP+0x9OX3vnrXHrishdIBAA8sLieaYc8kw5g37/btUOAr1B\nepO68QY6MWvTcAU9uEJuyluPweBuv6iTVFiT+00/I2r6VqzJGaRojCPmmBO9xOFARmusBtDTG8bp\nDdHSFqSlNSDfR29NnsAwLZNePczwHRY92/9ajUYzsd47CoUCVWoqqtRUkovmDNvu3raVcCBAanZ8\ny5cKcxcIBGckx5Q1ao1+oDeIK+jBHfLgCrqjpu/BHfTQFBjeDEyn1I5o+n+u3oFOo5mSnP5AMx/4\nGECjVsbaJgxEkiQ6Az1yhN8WpKU1avqtQU42dnCiYfi8BcD/ffpTSvLSmDfLgt2SjM2sJyNFF1ea\nZyi6/PxY2iceRFpG6Av9GaJ/Lo1dkiQ6e/wjmr4r5KEvMnxyEUCv0rMoo5TljsXMSslJ6FW5iXp/\nevsiuNpDUcMPUN3UwcETraPur0ySF0q3m2Wzt1uSsZv12M3JWMZp/P1pn4VFGeMep4jcBQJBwlEo\nFKRqTaRqTRSlFQzaFpEitHd3yIYf9FDTWcvnLQcBCPYF2dOyjz0t+1CgwK63kp+aR36KfMsy2L+y\nidx+1KoksqN19iAvlpJrM2EwaGnvDLGsxIazLYjTG8LpDeJsC+HyypH/0ByWStlv/Hps5ugHgGWA\n8UdTWP3tGR4vunjc4xTmLhAIzipJiiQsOjMWnZm5ljn4qn1ck78avUGLt9NHYVo+tZ311HbUccpX\nT0uziz3N+wHQJKnJS8mJmX1Bat5ZXxxlKENz+gWZKRRkDl803B/qxekN4mqTTd81wPybW4eviatS\nJpFiUNPdEybQNfI3nTMhzF0gEHylDC3lPM+6gPOs8sU6ESlCc8BJbWcdtR311HbWUdVey8n2mtjz\n07Sp5Kfkxgw/LyUH7ZD2C1M5YXumnP5AjMlqjMmpFGYN/jCSJAl/qHeQ2Tu9cuTv8gaH1e6Pl3GZ\ne2VlJbfddhs33XTTsAWyP/vsM5566imUSiWrVq3i9ttvn9BABALB3yejLXEIcpSfbcwk25jJ17Iu\nAKCrr4s6X0PM7Gs66/jSXc6X7vLYczINdjmyT8kjPzWP/6n5YEoXSpkMCoUCk16DSa+hMHu48W/d\neZJAqI/cEb4NnIkxzT0YDPLoo4+ycuXKEbc/9thjvPzyy9jtdtavX8+VV15JUVFRXIMQCASC8aJT\n6Sg2F1Fsln1GkiS83e2xVE5NZx31vgYa/c38rWnvoOf+bNfPmWuZQ2FaAWZtKhadGbMu9StrpzwW\nCoWC2VmpsbRPPIxp7hqNhpdeeomXXnpp2Lb6+npSU1PJzJS/6lx66aXs3r1bmLtAIDhrKBSKWA6/\nP/IPR8I0Bpqp7ajnSOtxDrceAcDX62ef8yD7nAcHaeiUOsy6VMy6NCzaNMy6NMzRe4sujTRt6hkX\nUDlbaZ94GNPcVSoVKtXIu7ndbiwWS+xni8VCfX39hAYiEAgEiUKZpIxdkOXr8ZFrykKXrMbr83Ge\ndQHe7na8Xe14uzvwdnlpiz5uDoy8nCBAisYUM32LLg2zNhVzNPJ/t2oHOq2GO86htM9XMqEa79cL\noS/0hf5Xrz1d9Uu68lmZuxSA3fUHWJk7+lWkod4uWoNePMG26E1+LP/OS6O/mVOdowewd35yHw6j\njdzUTCzJaVj0ZtL1aViS00hPNpOWnIpqAqWcFa5KXK5mSm3F437OpMzdZrPh8ZzuO+F0OrHZxv4K\nca5caCH0hf5M0p/OY59K/SJdMW63D6vVFHt8JrQYyVYZyU7JgyFzmBEpgr83IEf9Xe20dbdT72vk\n85YvADm909DZTH1n04jaChSYNEbStKkDbimnH+vk+6HVPq8ffAe1Wknp5T8e9989KXPPycnB7/fT\n0NCAw+Fg586dPPnkk5ORFAgEgnOWJEUSKRoTKRoTs1JyAbnlcn+dfjDYw1WzLsfX66e9u4P2rg7a\nuztp7+7A291OR3dnNP3TQp1vhE6QUZJVyZijef727g46e+L/0BvT3MvLy9m4cSONjY2oVCref/99\nLr/8cnJyclizZg2PPPIId911FwDXXHMNBQUFYygKBALBzGFonb4ySRmLxIdG/v1IkkSgLxg1//5b\n54DH8odBqG/4Oq/jZUxzX7BgAa+99tqo25cvX86WLVsmPACBQCCYzpypTn80FAoFRrUBo9pAjilr\n1P26+rp5p+o9uvq6ycuIrxJHXKEqEAgE5yg6lZY55sLYN4N4iL+jvkAgEAjOGuP9NjAUYe4CgUAw\nAxHmLhAIBDMQYe4CgUAwAxHmLhAIBDMQYe4CgUAwAxHmLhAIBDMQYe4CgUAwAxHmLhAIBDMQYe4C\ngUAwA1FIkiR91YMQCAQCQWIRkbtAIBDMQIS5CwQCwQxEmLtAIBDMQIS5CwQCwQxEmLtAIBDMQIS5\nCwQCwQxEmLtAIBDMQM6quf/yl7/kxhtvZO3atZSVlSVcv7KyktWrV/P73/8+4doAmzZt4sYbb+Qf\n//Ef+eCDDxKm5LiFygAACulJREFUGwqF+OEPf8j69eu54YYb2LlzZ8K0B9LV1cXq1av505/+lFDd\nvXv3cuGFF7JhwwY2bNjAo48+mlB9gHfffZdrr72Wb3/723zyyScJ1f7jH/8YG/uGDRs4//zzE6of\nCAS444472LBhA2vXrmXXrl0J1Y9EIjz44IOsXbuWDRs2UFVVlRDdoedTc3MzGzZsYN26dfzwhz+k\np6cnofoA//Vf/0VpaSmBQGBS2iPpNzc3c9NNN7F+/Xpuuukm3G53QvUPHjzI9773PTZs2MDNN99M\nW1tbQvX72bVrFyUlJWM+/6ytofr5559z6tQptmzZQlVVFffdd19CF9YOBoM8+uijrFy5MmGaA9mz\nZw8nTpxgy5YteL1err/+eq644oqEaO/cuZMFCxZwyy230NjYyL/8y79w2WWXJUR7IM8//zypqakJ\n1wVYsWIFzzzzzJRoe71ennvuOd566y2CwSCbN2/m61//esL0b7jhBm644QZAPk7/8pe/JEwb4O23\n36agoIC77roLp9PJ97//fXbs2JEw/Y8//hifz8ebb75JXV0dv/jFL3jxxRcnpTnS+fTMM8+wbt06\nrr76ap566im2bdvGunXrEqa/fft2Wltbsdlskxr7aPpPP/003/3ud7nmmmv4wx/+wCuvvMJPfvKT\nhOm/8sorbNq0idzcXJ599lm2bt3KrbfemjB9gO7ubn79619jtVrH1Dhrkfvu3btZvXo1AIWFhXR0\ndOD3+xOmr9FoeOmllxJyYIzE8uXL+dWvfgVASkoKoVCIcDicEO1rrrmGW265BZCjC7vdnhDdgVRV\nVXHy5MmEmuLZYvfu3axcuRKj0YjNZpuSbwb9PPfcc9x2220J1TSbzbS3twPQ2dmJ2WxOqH5tbS2L\nFsnrbObl5dHU1DTpY3Ok82nv3r184xvfAOCyyy5j9+7dCdVfvXo1P/rRj1AoFBMf+Bn0H374Ya68\n8kpg8P8kUfrPPPMMubm5SJKE0+nE4XAkVB/ghRdeYN26dWg0mjE1zpq5ezyeQQe1xWKZ9NeigahU\nKnQ6XcL0hqJUKtHr9QBs27aNVatWoVQqE/oaa9eu5e677+a+++5LqC7Axo0b+dnPfpZw3X5OnjzJ\nrbfeyve+9z3+9re/JVS7oaGBrq4ubr31VtatWzcpUzkTZWVlZGZmjisqiodvfvObNDU1sWbNGtav\nX89Pf/rThOoXFxfz17/+lXA4THV1NfX19Xi93klpjnQ+hUKhmKmkp6dP6vwdSd9oNE5Ybzz6er0e\npVJJOBzm9ddf51vf+lZC9QE+/fRTrrrqKjweD9dee21C9Wtqajh27BhXX331uDS+sgnV6drS5qOP\nPmLbtm089NBDCdd+8803ef7557nnnnsS+v5s376dxYsXk5ubmzDNgeTn53PHHXfw/PPPs3HjRu6/\n//5J52OH0t7ezrPPPssTTzzBvffeOyXHz7Zt27j++usTrvvOO++QlZXFhx9+yKuvvsrPf/7zhOpf\neumlLFy4kH/6p3/i1VdfZfbs2VN+fk3X8zccDvOTn/yECy+8cEpSuKtWrWLHjh3Mnj2bX//61wnV\nfvzxx7n33nvHvf9Zy7nbbDY8Hk/sZ5fLlfAIaarZtWsXL7zwAr/5zW8wmUwJ0y0vLyc9PZ3MzEzm\nzZtHOBymra2N9PT0hOh/8skn1NfX88knn9DS0oJGo8HhcHDRRRclRN9ut3PNNdcAclogIyMDp9OZ\nsA+T9PR0zj//fFQqFXl5eRgMhoS+P/3s3buXBx54IKGaAF988QUXX3wxAHPnzsXlchEOhxP6ze9H\nP/pR7PHq1asT/t6AHPl2dXWh0+lwOp1TlgKdSu69915mzZrFHXfckXDtDz/8kDVr1qBQKLjyyivZ\nvHlzwrSdTifV1dXcfffdgOyf69evP2PxyFmL3L/2ta/x/vvvA1BRUYHNZkvo17CpxufzsWnTJl58\n8UXS0tISqr1//35++9vfAnL6KhgMJjQv+/TTT/PWW2+xdetWbrjhBm677baEGTvIlSwvv/wyAG63\nm9bW1oTOG1x88cXs2bOHSCSC1+tN+PsD8sljMBjGlcuMl1mzZnHo0CEAGhsbMRgMCTX2Y8eOxSK6\nTz/9lPnz55OUlPhT+6KLLoqdwx988AGXXHJJwl9jKnn33XdRq9X84Ac/mBL9zZs3c/ToUQAOHTpE\nQUFBwrTtdjsfffQRW7duZevWrdhstjGrAs9a5L5kyRJKS0tZu3YtCoWChx9+OKH65eXlbNy4kcbG\nRlQqFe+//z6bN29OmBG/9957eL1e7rzzztjvNm7cSFZW1qS1165dy/3338+6devo6urioYcempKT\nc6q4/PLLufvuu/n444/p7e3lkUceSahJ2u12rrzySr773e8C8MADDyT8/XG73VgsloRq9nPjjTdy\n3333sX79evr6+njkkUcSql9cXIwkSXznO99Bq9Xy5JNPTlpzpPPpySef5Gc/+xlbtmwhKyuL6667\nLqH6F110EZ999hlut5tbbrmFxYsXT7iaZST91tZWtFotGzZsAOTCjon+L0bSf+yxx/iP//gPlEol\nOp2OTZs2TUh7NP14/Uz0cxcIBIIZyPQJDwUCgUAwboS5CwQCwQxEmLtAIBDMQIS5CwQCwQxEmLtA\nIBDMQM5aKaRAkGgaGhq46qqrhnVxvPTSS/nXf/3XSevv3buXp59+mjfeeGPSWgLB2UaYu2BaY7FY\neO21177qYQgE5xzC3AUzkvnz53Pbbbexd+9eAoEATzzxBMXFxRw6dIgnnngClUqFQqHgoYceoqio\niNraWh588EEikQharZbHH38ckHulP/zwwxw9ehSNRsOLL76IwWDgvffe4/e//z2SJGGxWHjssccw\nmUw88MAD1NTUoFAomDdvXsIv1hMIxo0kEExT6uvrpUsuuWTEbcXFxdKOHTskSZKkrVu3Srfffrsk\nSZJ0xRVXSIcOHZIkSZL+93//V1q/fr0kSZL0z//8z9LOnTslSZKk//7v/5ZeeeUVac+ePdLSpUsl\nt9stSZIkff/735d27NghNTU1Sd/61rek7u5uSZIk6Xe/+530+OOPSxUVFdJVV10VG8OWLVukzs7O\nxP/hAsE4EJG7YFrT1tYWu5y8n3vuuQcg1qxryZIlvPzyy3R2dtLa2hrrfb5ixQp+/OMfA3K73xUr\nVgByi16Qc+6zZ88mIyMDAIfDQWdnJwcPHsTtdnPzzTcD0NPTQ05ODoWFhZjNZm655RYuu+wyrr76\n6oQ2mBMI4kGYu2Bac6acuzSgs4ZCoRi2CIQ0pPNGJBIZpjFSgy+NRsOiRYtGXO3o9ddfp6Kigp07\nd/Kd73yHN954Y1p2TxRMf0QppGDGsmfPHgAOHDhASUkJJpMJq9Ua69C4e/duFi9eDMjRff/apu+9\n9x5PPfXUqLoLFy6krKwstljFX/7yFz766CMOHz7M22+/TWlpKXfccQelpaXU1tZO4V8oEIyOiNwF\n05qR0jI5OTkAHDlyhDfeeIOOjg42btwIyJ08n3jiCZRKJUlJSbGugA8++CAPPvggr7/+OiqVil/+\n8pfU1dWN+Jp2u53777+ff/u3fyM5ORmdTsfGjRtRq9U899xzbNmyBY1GQ15eHkuWLJm6P14gOAOi\nK6RgRlJSUkJFRQUqlYhfBH+fiLSMQCAQzEBE5C4QCAQzEBG5CwQCwQxEmLtAIBDMQIS5CwQCwQxE\nmLtAIBDMQIS5CwQCwQzk/wOsEsCxSAtrYwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f8ee8f73898>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "W5lBux2pD-Dl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Evaluate on test set"
      ]
    },
    {
      "metadata": {
        "id": "fvTiCHSoD-tc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "e55653b9-799d-4612-dae0-1e6afff0ff0e"
      },
      "cell_type": "code",
      "source": [
        "hidden_size = 1024\n",
        "n_layers = 2\n",
        "teacher_forcing_ratio = 1\n",
        "n_iters = 20\n",
        "source_vocab_size = 19000\n",
        "target_vocab_size = 22000\n",
        "beam_size = 7\n",
        "\n",
        "encoder_current = EncoderRNN(input_lang.n_words, hidden_size,n_layers=n_layers).to(device)\n",
        "decoder_current = LuongAttnDecoderRNN(attn_model, hidden_size, output_lang.n_words, dropout=dropout_p, n_layers=n_layers).to(device)\n",
        "checkpoint = torch.load(\"/content/drive/My Drive/saved_model/LSTM_attnIs{}_hiddenSize{}_nLayer{}_batchSize{}_epoch{}_srcVocSize{}_tgtVocSize{}_lrDecay{}_teacherF{}\"\\\n",
        "                    .format(Attention,hidden_size,n_layers,BATCH_SIZE,n_iters,source_vocab_size, target_vocab_size,lr_decay,teacher_forcing_ratio))\n",
        "encoder_current.load_state_dict(checkpoint['encoder'])\n",
        "decoder_current.load_state_dict(checkpoint['decoder'])\n",
        "encoder_current.eval()\n",
        "decoder_current.eval()\n",
        "\n",
        "test_score_beam, predicted_sentence = test_model(encoder_current, decoder_current, test_loader, search_method='beam')\n",
        "\n",
        "print ('test score', test_score_beam)\n"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:That's 100 lines that end in a tokenized period ('.')\n",
            "WARNING:root:It looks like you forgot to detokenize your test data, which may hurt your score.\n",
            "WARNING:root:If you insist your data is detokenized, or don't care, you can suppress this message with '--force'.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "test score 21.309190297919464\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Jnaut8XNEKJ2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}