0_noise_seq2seq_RNN.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "A3Bbm9MSP6Yk",
        "outputId": "fc2d2750-3635-44a2-c588-7722e474d772"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sat Jan 15 07:28:44 2022       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 495.46       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   39C    P0    28W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CuNqn2TkS1T5"
      },
      "source": [
        "## Load dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "PBEAq9r9wMRI",
        "outputId": "08166954-01a0-491f-ba37-deffd6ae7ef4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting torchtext==0.6.0\n",
            "  Downloading torchtext-0.6.0-py3-none-any.whl (64 kB)\n",
            "\u001b[?25l\r\u001b[K     |█████                           | 10 kB 23.6 MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 20 kB 10.9 MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 30 kB 8.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 40 kB 8.0 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 51 kB 5.4 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 61 kB 5.5 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 64 kB 1.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from torchtext==0.6.0) (1.10.0+cu111)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from torchtext==0.6.0) (4.62.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from torchtext==0.6.0) (1.15.0)\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 10.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchtext==0.6.0) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchtext==0.6.0) (2.23.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.6.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.6.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.6.0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.6.0) (2021.10.8)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->torchtext==0.6.0) (3.10.0.2)\n",
            "Installing collected packages: sentencepiece, torchtext\n",
            "  Attempting uninstall: torchtext\n",
            "    Found existing installation: torchtext 0.11.0\n",
            "    Uninstalling torchtext-0.11.0:\n",
            "      Successfully uninstalled torchtext-0.11.0\n",
            "Successfully installed sentencepiece-0.1.96 torchtext-0.6.0\n"
          ]
        }
      ],
      "source": [
        "!pip install torchtext==0.6.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "dfQk37UUQi2T"
      },
      "outputs": [],
      "source": [
        "!pip install tqdm --upgrade >> /dev/null 2>&1\n",
        "#!pip install torchtext --upgrade >> /dev/null 2>&1\n",
        "!pip install spacy --upgrade >> /dev/null 2>&1\n",
        "!python -m spacy download de >> /dev/null 2>&1\n",
        "!python -m spacy download en >> /dev/null 2>&1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "zku7rzrTSE1j"
      },
      "outputs": [],
      "source": [
        "import tqdm\n",
        "import spacy\n",
        "import warnings\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "from IPython.core.display import display, HTML\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "from torchtext.data import Dataset, Example, Field\n",
        "from torchtext.data.iterator import BucketIterator\n",
        "from torchtext.data.metrics import bleu_score\n",
        "from torchtext.datasets import Multi30k"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-rfDBpyrS7Pq",
        "outputId": "8ebcb12e-84f7-47c2-874a-e604afdce346"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Device: cuda\n"
          ]
        }
      ],
      "source": [
        "warnings.simplefilter(action='ignore', category=UserWarning)\n",
        "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
        "warnings.simplefilter(action='ignore', category=DeprecationWarning)\n",
        "\n",
        "SEED = 546\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.cuda.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "\n",
        "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print(f'Device: {DEVICE}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pFrpUtp8cMPw",
        "outputId": "948a8aca-f5f0-430d-c10f-d9192f5a64b7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "0_no_noise  1_noise  2_noise  3_noise  4_noise\t5_noise  6_noise  7_noise\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive',force_remount=True)\n",
        "!ls '/content/drive/MyDrive/data_europarl/'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qfjuxu1QTtoP"
      },
      "source": [
        "#### Build vocabularies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1GPY7yEdf3pL",
        "outputId": "7c5efa03-a519-4b24-b9ba-cce7cbe6606d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train set size: 29,824\n",
            "valid set size: 996\n",
            "test set size: 987\n",
            "CPU times: user 969 ms, sys: 65 ms, total: 1.03 s\n",
            "Wall time: 2.69 s\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "FR_TEXT = Field(lower=True,  tokenizer_language='fr', include_lengths=True)\n",
        "EN_TEXT = Field(init_token='<sos>', eos_token='<eos>', lower=True, tokenizer_language='en', include_lengths=True)\n",
        "train, valid, test = Multi30k.splits(exts=('.fr', '.en'),  fields=(FR_TEXT, EN_TEXT), root = '/content/drive/MyDrive/data_europarl/0_no_noise')\n",
        "print(f'train set size: {len(train.examples):,}')\n",
        "print(f'valid set size: {len(valid.examples):,}')\n",
        "print(f'test set size: {len(test.examples):,}')\n",
        "# print(vars(DE))\n",
        "# print(vars(EN))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PvuBC8xKTZJf",
        "outputId": "0862c6d8-0a68-483c-dedb-c56ad78b549c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Length of FR vocabulary: 10,931\n",
            "Length of EN vocabulary: 8,777\n",
            "CPU times: user 341 ms, sys: 4.4 ms, total: 346 ms\n",
            "Wall time: 342 ms\n"
          ]
        }
      ],
      "source": [
        "%%time\n",
        "MIN_COUNT = 2\n",
        "FR_TEXT.build_vocab(train, min_freq=MIN_COUNT, specials=['<unk>', '<pad>'])\n",
        "EN_TEXT.build_vocab(train, min_freq=MIN_COUNT, specials=['<sos>', '<eos>', '<unk>', '<pad>'])\n",
        "print(f'Length of FR vocabulary: {len(FR_TEXT.vocab):,}')\n",
        "print(f'Length of EN vocabulary: {len(EN_TEXT.vocab):,}')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C1-SpXqDT72P"
      },
      "source": [
        "# Modeling 0\n",
        "Using Vanilla data, i.e. data without adding noise."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LyEKiMbF8NfK"
      },
      "source": [
        "#### Encoder layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "QOZS6pPqT7Ba"
      },
      "outputs": [],
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "\n",
        "    def __init__(self, vocab_size, embedding_size, hidden_size, n_layers, embedding_dropout, recurrent_dropout):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.vocab_size = vocab_size\n",
        "        self.embedding_size = embedding_size\n",
        "        self.hidden_size = hidden_size\n",
        "        self.n_layers = n_layers\n",
        "        self.embedding_dropout = embedding_dropout\n",
        "        self.recurrent_dropout = recurrent_dropout if n_layers > 1 else 0\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        self.gru = nn.GRU(embedding_size, hidden_size, num_layers=n_layers, dropout=self.recurrent_dropout, bidirectional=True)\n",
        "\n",
        "    def forward(self, input_sequences, sequence_lengths):\n",
        "        \"\"\"\n",
        "        :param Tensor[seq_len, batch_size] input_sequences\n",
        "        :param Tensor[batch_size,] sequence_lengths\n",
        "        :return Tensor[n_layers * 2, batch_size, hidden_size] h_state\n",
        "        \"\"\"\n",
        "        embedded = self.embedding(input_sequences)\n",
        "        embedded = F.dropout(embedded, p=self.embedding_dropout)\n",
        "        packed = nn.utils.rnn.pack_padded_sequence(embedded, sequence_lengths.cpu())\n",
        "        _, h_state = self.gru(packed)\n",
        "        return h_state"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ALcFgkFzUMX3"
      },
      "source": [
        "#### Decoder layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "-MMyNs5aUOcM"
      },
      "outputs": [],
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "\n",
        "    def __init__(self, vocab_size, embedding_size, hidden_size, n_layers, embedding_dropout, recurrent_dropout):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        self.vocab_size = vocab_size\n",
        "        self.embedding_size = embedding_size\n",
        "        self.hidden_size = hidden_size\n",
        "        self.n_layers = n_layers\n",
        "        self.embedding_dropout = embedding_dropout\n",
        "        self.recurrent_dropout = recurrent_dropout if n_layers > 1 else 0\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        self.gru = nn.GRU(embedding_size, hidden_size, num_layers=n_layers, dropout=self.recurrent_dropout)\n",
        "        self.fc = nn.Linear(hidden_size, vocab_size)\n",
        "\n",
        "    def forward(self, input_word_index, h_state_prev):\n",
        "        \"\"\"\n",
        "        :param Tensor[batch_size,] input_word_index\n",
        "        :param Tensor[n_layers, batch_size, hidden_size] h_state_prev\n",
        "        :return Tensor[batch_size, vocab_size] logit\n",
        "        :return Tensor[n_layers, batch_size, hidden_size] h_state\n",
        "        \"\"\"\n",
        "        embedded = self.embedding(input_word_index.unsqueeze(0))\n",
        "        embedded = F.dropout(embedded, p=self.embedding_dropout)\n",
        "        outputs, h_state = self.gru(embedded, h_state_prev)\n",
        "        logit = self.fc(outputs.squeeze(0))\n",
        "        return logit, h_state"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eaB7LsPWUO94"
      },
      "source": [
        "#### Sequence-to-sequence model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "Ttc82wRxUTuj"
      },
      "outputs": [],
      "source": [
        "class SeqToSeqNet(nn.Module):\n",
        "\n",
        "    def __init__(self, encoder, decoder, device):\n",
        "        assert encoder.n_layers == decoder.n_layers, 'Encoder and Decoder must have the same number of reccurent layers'\n",
        "        assert encoder.hidden_size == decoder.hidden_size, 'Encoder and Decoder must have the same number of reccurrent hidden units'\n",
        "        super(SeqToSeqNet, self).__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.device = device\n",
        "        self.init_h0 = nn.Linear(encoder.n_layers * 2, decoder.n_layers)\n",
        "\n",
        "    def encode(self, input_sequences, sequence_lengths):\n",
        "        h_state = self.encoder(input_sequences, sequence_lengths)\n",
        "        h_state = self.init_h0(h_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]\n",
        "        h_state = h_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]\n",
        "        return h_state\n",
        "\n",
        "    def sort_batches(self, dest_sequences, dest_lengths, h_state):\n",
        "        sorted_dest_lengths, sorted_indices = torch.sort(dest_lengths, dim=0, descending=True)\n",
        "        sorted_dest_sequences = dest_sequences[:, sorted_indices]\n",
        "        h_state = h_state[:, sorted_indices, :]\n",
        "        # We won't decode at the <eos> position, since we've finished generating as soon as we generate <eos>.\n",
        "        # So, decoding lengths are actual lengths - 1\n",
        "        sorted_decode_lengths = (sorted_dest_lengths - 1).tolist() \n",
        "        return sorted_dest_sequences, sorted_decode_lengths, h_state\n",
        "\n",
        "    def decode(self, h_state, sorted_dest_sequences, sorted_decode_lengths, tf_ratio):\n",
        "        batch_size, last = sorted_dest_sequences.size(1), None\n",
        "        logits = torch.zeros(max(sorted_decode_lengths), batch_size, self.decoder.vocab_size).to(self.device)\n",
        "        for t in range(max(sorted_decode_lengths)):\n",
        "            batch_size_t = sum([l > t for l in sorted_decode_lengths])\n",
        "            if last is not None:\n",
        "                if np.random.rand() < tf_ratio:\n",
        "                    input_word_index = last[:batch_size_t] # in_ [batch_size,]\n",
        "                else:\n",
        "                    input_word_index = sorted_dest_sequences[t, :batch_size_t] # in_ [batch_size,]\n",
        "            else:\n",
        "                input_word_index = sorted_dest_sequences[t, :batch_size_t] # in_ [batch_size,]\n",
        "            logit, h_state = self.decoder(input_word_index, h_state[:, :batch_size_t, :].contiguous())\n",
        "            # logit: [batch_size, vocab_size] - h_state: [n_layers, batch_size, hidden_size]\n",
        "            logits[t, :batch_size_t, :] = logit\n",
        "            last = torch.argmax(F.softmax(logit, dim=1), dim=1) # [batch_size,]\n",
        "        return logits\n",
        "    \n",
        "    def forward(self, French_sequences, French_lengths, dest_sequences, dest_lengths, tf_ratio):\n",
        "        \"\"\"\n",
        "        :param Tensor[seq_len, batch_size] French_sequences\n",
        "        :param Tensor[batch_size,] French_lengths\n",
        "        :param Tensor[seq_len, batch_size] dest_sequences\n",
        "        :param Tensor[batch_size,] dest_lengths\n",
        "        :param float tf_ratio\n",
        "        :return Tensor[max(decode_lengths), batch_size, vocab_size] logits\n",
        "        :return Tensor[seq_len, batch_size] sorted_dest_sequences\n",
        "        :return list[max(decode_lengths) - 1] sorted_decode_lengths\n",
        "        \"\"\"\n",
        "        h_state = self.encode(French_sequences, French_lengths)\n",
        "        sorted_dest_sequences, sorted_decode_lengths, h_state = self.sort_batches(dest_sequences, dest_lengths, h_state)\n",
        "        logits = self.decode(h_state, sorted_dest_sequences, sorted_decode_lengths, tf_ratio)\n",
        "        return logits, sorted_dest_sequences, sorted_decode_lengths"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CzVIUeL2cAsz"
      },
      "source": [
        "#### Training routines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "5VPQdzdrddUu"
      },
      "outputs": [],
      "source": [
        "class AverageMeter:\n",
        "    \n",
        "    def __init__(self):\n",
        "        self.value = 0.\n",
        "        self.sum = 0.\n",
        "        self.count = 0\n",
        "        self.average = 0.\n",
        "        \n",
        "    def reset(self):\n",
        "        self.value = 0.\n",
        "        self.sum = 0.\n",
        "        self.count = 0\n",
        "        self.average = 0.\n",
        "        \n",
        "    def update(self, value, n=1):\n",
        "        self.value = value\n",
        "        self.sum += value * n\n",
        "        self.count += n\n",
        "        self.average = self.sum / self.count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "a4aTFulSfGTf"
      },
      "outputs": [],
      "source": [
        "def accuracy(outputs, target_sequences, k=5):\n",
        "    batch_size = outputs.size(1)\n",
        "    _, indices = outputs.topk(k, dim=1, largest=True, sorted=True)\n",
        "    correct = indices.eq(target_sequences.view(-1, 1).expand_as(indices))\n",
        "    correct_total = correct.view(-1).float().sum()  # 0D tensor\n",
        "    return correct_total.item() * (100.0 / batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "hCsp-2Fvb_li"
      },
      "outputs": [],
      "source": [
        "class Trainer:\n",
        "\n",
        "    def __init__(self, model, optimizer, criterion, train_iterator, valid_iterator):\n",
        "        self.model = model\n",
        "        self.optimizer = optimizer\n",
        "        self.criterion = criterion\n",
        "        self.train_iterator = train_iterator\n",
        "        self.valid_iterator = valid_iterator\n",
        "\n",
        "    def clip_gradients(self, grad_clip):\n",
        "        if grad_clip is not None:\n",
        "            for group in self.optimizer.param_groups:\n",
        "                for param in group['params']:\n",
        "                    if param.grad is not None:\n",
        "                        param.grad.data.clamp_(-grad_clip, grad_clip)\n",
        "\n",
        "    def adjust_lr(self, shrink_factor=0.9, verbose=True):\n",
        "        if verbose:\n",
        "            print(\"\\nDecaying learning rate.\")\n",
        "        for param_group in self.optimizer.param_groups:\n",
        "            param_group['lr'] = param_group['lr'] * shrink_factor\n",
        "        if verbose:\n",
        "            print(\"The new learning rate is %f\\n\" % (self.optimizer.param_groups[0]['lr'],))\n",
        "    \n",
        "    def adjust_tf(self, tf_ratio, shrink_factor=0.9, verbose=False):\n",
        "        tf_ratio = tf_ratio * shrink_factor\n",
        "        if verbose:\n",
        "            print(\"The teacher forcing rate is %f\\n\" % (tf_ratio,))\n",
        "        return tf_ratio\n",
        "    \n",
        "    def train_step(self, epoch, grad_clip, tf_ratio):\n",
        "        loss_tracker, acc_tracker = AverageMeter(), AverageMeter()\n",
        "        self.model.train()\n",
        "        progress_bar = tqdm.tqdm(enumerate(self.train_iterator), total=len(self.train_iterator))\n",
        "        for i, data in progress_bar:\n",
        "            # if isPrinted == False:\n",
        "            #     print(data.French)\n",
        "            #     print(data.English)\n",
        "            #     isPrinted = True\n",
        "            # print(len(data.French))\n",
        "            logits, sorted_dest_sequences, sorted_decode_lengths = self.model(*data.src, *data.trg, tf_ratio=tf_ratio)\n",
        "            sorted_dest_sequences = sorted_dest_sequences[1:, :] # Since we decoded starting with <sos>, the targets are all words after <sos>, up to <eos>\n",
        "            logits = nn.utils.rnn.pack_padded_sequence(logits, sorted_decode_lengths).data # Remove paddings\n",
        "            sorted_dest_sequences = nn.utils.rnn.pack_padded_sequence(sorted_dest_sequences, sorted_decode_lengths).data # Remove paddings\n",
        "            loss = criterion(logits, sorted_dest_sequences)\n",
        "            optimizer.zero_grad()\n",
        "            loss.backward()\n",
        "            self.clip_gradients(grad_clip)\n",
        "            optimizer.step()\n",
        "            loss_tracker.update(loss.item(), sum(sorted_decode_lengths))\n",
        "            acc_tracker.update(accuracy(logits, sorted_dest_sequences), sum(sorted_decode_lengths))\n",
        "            loss_, ppl_, acc_ = loss_tracker.average, np.exp(loss_tracker.average), acc_tracker.average\n",
        "            progress_bar.set_description(f'Epoch: {epoch+1:02d} -     loss: {loss_:.3f} -     ppl: {ppl_:.3f} -     acc: {acc_:.3f}%')\n",
        "        return loss_tracker.average, np.exp(loss_tracker.average), acc_tracker.average\n",
        "    \n",
        "    def validate(self, epoch):\n",
        "        loss_tracker, acc_tracker = AverageMeter(), AverageMeter()\n",
        "        self.model.eval()\n",
        "        with torch.no_grad():\n",
        "            progress_bar = tqdm.tqdm(enumerate(self.valid_iterator), total=len(self.valid_iterator))\n",
        "            for i, data in progress_bar:\n",
        "                logits, sorted_dest_sequences, sorted_decode_lengths = self.model(*data.src, *data.trg, tf_ratio=0.)\n",
        "                sorted_dest_sequences = sorted_dest_sequences[1:, :]\n",
        "                logits = nn.utils.rnn.pack_padded_sequence(logits, sorted_decode_lengths).data\n",
        "                sorted_dest_sequences = nn.utils.rnn.pack_padded_sequence(sorted_dest_sequences, sorted_decode_lengths).data\n",
        "                loss = criterion(logits, sorted_dest_sequences)\n",
        "                loss_tracker.update(loss.item(), sum(sorted_decode_lengths))\n",
        "                acc_tracker.update(accuracy(logits, sorted_dest_sequences), sum(sorted_decode_lengths))\n",
        "                loss_, ppl_, acc_ = loss_tracker.average, np.exp(loss_tracker.average), acc_tracker.average\n",
        "                progress_bar.set_description(f'Epoch: {epoch+1:02d} - val_loss: {loss_:.3f} - val_ppl: {ppl_:.3f} - val_acc: {acc_:.3f}%')\n",
        "        return loss_tracker.average, np.exp(loss_tracker.average), acc_tracker.average\n",
        "    \n",
        "    def train(self, n_epochs, grad_clip, tf_ratio):\n",
        "        history = {'acc': [], 'loss': [], 'ppl': [], 'val_ppl': [], 'val_acc': [], 'val_loss': []}\n",
        "        best_loss, last_improv = np.inf, 0\n",
        "        for epoch in range(n_epochs):\n",
        "            if last_improv == 4:\n",
        "                print('Training Finished - The model has stopped improving since last 4 epochs')\n",
        "                break\n",
        "            if last_improv > 0:\n",
        "                self.adjust_lr()\n",
        "            loss, ppl, acc = self.train_step(epoch, grad_clip, tf_ratio)\n",
        "            val_loss, val_ppl, val_acc = self.validate(epoch)\n",
        "            tf_ratio = self.adjust_tf(tf_ratio)\n",
        "            \n",
        "            if best_loss > val_loss:\n",
        "                best_loss, last_improv = val_loss, 0\n",
        "                torch.save(self.model.state_dict(), 'seq2seq.pth')\n",
        "            else:\n",
        "                last_improv += 1\n",
        "                print(f'Last improvement since epoch {epoch - last_improv + 1}')\n",
        "            \n",
        "            history['acc'].append(acc)\n",
        "            history['ppl'].append(ppl)\n",
        "            history['loss'].append(loss)\n",
        "            history['val_acc'].append(val_acc)\n",
        "            history['val_ppl'].append(val_ppl)\n",
        "            history['val_loss'].append(val_loss)\n",
        "        return history"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "adURzNzWkthV"
      },
      "source": [
        "#### Train the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "B9ghlNzJkrCH"
      },
      "outputs": [],
      "source": [
        "#hyper-params\n",
        "N_LAYERS = 2\n",
        "HIDDEN_SIZE = 256\n",
        "EMBED_SIZE = 300\n",
        "EMBED_DROPOUT = 0.25\n",
        "REC_DROPOUT = 0.25\n",
        "N_EPOCHS = 10\n",
        "BATCH_SIZE = 64\n",
        "LR = 1e-3\n",
        "GRAD_CLIP = 1.0\n",
        "TF_RATIO = 1.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3N7Sttcdk39H",
        "outputId": "93bea14b-d675-4af0-d955-f843103d646f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of parameters of the model: 11,031,203\n",
            "SeqToSeqNet(\n",
            "  (encoder): EncoderLayer(\n",
            "    (embedding): Embedding(10931, 300)\n",
            "    (gru): GRU(300, 256, num_layers=2, dropout=0.25, bidirectional=True)\n",
            "  )\n",
            "  (decoder): DecoderLayer(\n",
            "    (embedding): Embedding(8777, 300)\n",
            "    (gru): GRU(300, 256, num_layers=2, dropout=0.25)\n",
            "    (fc): Linear(in_features=256, out_features=8777, bias=True)\n",
            "  )\n",
            "  (init_h0): Linear(in_features=4, out_features=2, bias=True)\n",
            ")\n"
          ]
        }
      ],
      "source": [
        "encoder = EncoderLayer(vocab_size=len(FR_TEXT.vocab), embedding_size=EMBED_SIZE, hidden_size=HIDDEN_SIZE, n_layers=N_LAYERS,\n",
        "                       embedding_dropout=EMBED_DROPOUT, recurrent_dropout=REC_DROPOUT)\n",
        "decoder = DecoderLayer(vocab_size=len(EN_TEXT.vocab), embedding_size=EMBED_SIZE, hidden_size=HIDDEN_SIZE, n_layers=N_LAYERS,\n",
        "                       embedding_dropout=EMBED_DROPOUT, recurrent_dropout=REC_DROPOUT)\n",
        "seq2seq = SeqToSeqNet(encoder=encoder, decoder=decoder, device=DEVICE).to(DEVICE)\n",
        "optimizer = optim.RMSprop(params=seq2seq.parameters(), lr=LR)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "print(f'Number of parameters of the model: {sum(p.numel() for p in seq2seq.parameters() if p.requires_grad):,}')\n",
        "print(seq2seq)\n",
        "train_iterator, valid_iterator, test_iterator =  BucketIterator.splits((train, valid, test),\n",
        "                                                                       batch_size=BATCH_SIZE,\n",
        "                                                                       sort_key=lambda x: len(x.src),\n",
        "                                                                       sort_within_batch=True, device=DEVICE)\n",
        "trainer = Trainer(model=seq2seq, optimizer=optimizer, criterion=criterion, train_iterator=train_iterator, valid_iterator=valid_iterator)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "isE439WfmDJX",
        "outputId": "ed464d5f-0433-4324-d8e5-89630b091a9e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 01 -     loss: 5.774 -     ppl: 321.714 -     acc: 3.670%: 100%|██████████| 466/466 [00:49<00:00,  9.49it/s]\n",
            "Epoch: 01 - val_loss: 5.411 - val_ppl: 223.809 - val_acc: 4.093%: 100%|██████████| 16/16 [00:00<00:00, 18.29it/s]\n",
            "Epoch: 02 -     loss: 5.258 -     ppl: 192.043 -     acc: 4.271%: 100%|██████████| 466/466 [00:48<00:00,  9.56it/s]\n",
            "Epoch: 02 - val_loss: 5.136 - val_ppl: 169.981 - val_acc: 4.451%: 100%|██████████| 16/16 [00:00<00:00, 19.40it/s]\n",
            "Epoch: 03 -     loss: 4.924 -     ppl: 137.585 -     acc: 4.588%: 100%|██████████| 466/466 [00:49<00:00,  9.35it/s]\n",
            "Epoch: 03 - val_loss: 4.870 - val_ppl: 130.371 - val_acc: 4.795%: 100%|██████████| 16/16 [00:00<00:00, 20.15it/s]\n",
            "Epoch: 04 -     loss: 4.628 -     ppl: 102.348 -     acc: 4.886%: 100%|██████████| 466/466 [00:49<00:00,  9.33it/s]\n",
            "Epoch: 04 - val_loss: 4.752 - val_ppl: 115.759 - val_acc: 5.086%: 100%|██████████| 16/16 [00:00<00:00, 18.61it/s]\n",
            "Epoch: 05 -     loss: 4.352 -     ppl: 77.631 -     acc: 5.184%: 100%|██████████| 466/466 [00:50<00:00,  9.31it/s]\n",
            "Epoch: 05 - val_loss: 4.646 - val_ppl: 104.167 - val_acc: 5.236%: 100%|██████████| 16/16 [00:00<00:00, 18.99it/s]\n",
            "Epoch: 06 -     loss: 4.119 -     ppl: 61.515 -     acc: 5.467%: 100%|██████████| 466/466 [00:50<00:00,  9.30it/s]\n",
            "Epoch: 06 - val_loss: 4.591 - val_ppl: 98.555 - val_acc: 5.432%: 100%|██████████| 16/16 [00:00<00:00, 19.68it/s]\n",
            "Epoch: 07 -     loss: 3.901 -     ppl: 49.467 -     acc: 5.771%: 100%|██████████| 466/466 [00:50<00:00,  9.29it/s]\n",
            "Epoch: 07 - val_loss: 4.531 - val_ppl: 92.849 - val_acc: 5.585%: 100%|██████████| 16/16 [00:00<00:00, 17.78it/s]\n",
            "Epoch: 08 -     loss: 3.711 -     ppl: 40.910 -     acc: 6.087%: 100%|██████████| 466/466 [00:49<00:00,  9.34it/s]\n",
            "Epoch: 08 - val_loss: 4.507 - val_ppl: 90.659 - val_acc: 5.590%: 100%|██████████| 16/16 [00:00<00:00, 21.03it/s]\n",
            "Epoch: 09 -     loss: 3.551 -     ppl: 34.853 -     acc: 6.356%: 100%|██████████| 466/466 [00:50<00:00,  9.29it/s]\n",
            "Epoch: 09 - val_loss: 4.469 - val_ppl: 87.258 - val_acc: 5.742%: 100%|██████████| 16/16 [00:00<00:00, 19.43it/s]\n",
            "Epoch: 10 -     loss: 3.412 -     ppl: 30.315 -     acc: 6.618%: 100%|██████████| 466/466 [00:49<00:00,  9.41it/s]\n",
            "Epoch: 10 - val_loss: 4.467 - val_ppl: 87.119 - val_acc: 5.769%: 100%|██████████| 16/16 [00:00<00:00, 17.38it/s]\n"
          ]
        }
      ],
      "source": [
        "history = trainer.train(n_epochs=N_EPOCHS, grad_clip=GRAD_CLIP, tf_ratio=TF_RATIO)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "t-8rAeqiqjd7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 338
        },
        "outputId": "649cc581-1500-4aa6-acbc-6311a46aa813"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3gAAAFNCAYAAABSRs15AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3gVZfbA8e9JIaEEElqoSSjShZAgICpi76IriApWRFHXsvZ1XX/2dS27yroKiKIIiBS76NoIoFIkQOhSQ0dqIAECKef3x0zwEpOQQG7m5uZ8nuc+uZl6JpP73jkzbxFVxRhjjDHGGGNM5RfidQDGGGOMMcYYY8qHJXjGGGOMMcYYEyQswTPGGGOMMcaYIGEJnjHGGGOMMcYECUvwjDHGGGOMMSZIWIJnjDHGGGOMMUHCEjwTEETkXRF5toT5WSLSsiJjMsZUPiKSICIqImEnuJ3HRGRUOcV0k4j8WML8r0TkxvLYlzHGBDsR6SMim0qYP1xE/l6RMQUaS/DMUUQkXUTO9TqOwlS1lqquLWmZY33gjTHeccuWg+7Nmt/cmzq1vI6rOKr6vKreCuWXNJawr4tU9b1jLefG0NofMRhzotzPdsEr3+fzniUiA8tpHwWfRd99HfNCXkRSRGSPiESURxyBSETqisjnIrJXRLaIyMOlWEdFZL/7d9wpIh+ISLTP/BQRubWI9Yo6D1kiMqC49Up7jSYi1URk+Ylcz6nqUFV9phT7Cshr3vJgCZ4xLn9dvBljjrhMVWsBSUA34PGyrCwO+946Dla+GX9zb8TWcj/jG3A/7+5rXDnvLtpn2yVeyItIAnAGoMDl5RxHiSr4c/cQEAk0BjoCP5VyvS7uOWsJxABPlmGfvuehlqp+WJaAi/EQsKMctuNXgV6m2helKRURiRCRV927Qlvc9xHuvPoi8oWIZIjIbhGZWXARJiKPiMhmEckUkV9F5JwSdhMjIl+6y84RkVY++z9y51pELhaRZe5ym0XkQRGpCXwFNPG5k9TkGHH3EZFNbozbgNEiskRELvPZb7h7V6tr+f9VjamaVHUzzue1E4CI9BSRn90yJE1E+hQs694Jfk5EfgIOAC3daf8Qkbkisk9EPhWRukXtS0TqiMjbIrLVLS+eFZFQ9y7xQhG5210uVER+EpEn3N+fFJGx7mZmuD8z3LLlTLesO9lnPw1F5ICINCjuuEXkZfcpwjoRuajQMRY8LWwtItPdu/A7ReRDd3pBDGmF7pQPEZHVbjyfiUgTn+2qiNwlIquAVSLyXxF5pVBMn4nIX4qL2ZgTVcrv4cfc//d0KaenfT5uAGYD7wJHVYUWkeYi8pGI7BCRXSLyus+8IeI8Scp0rzmS3OlHPUkXnyYmxVxXxIhzjbTD/fx/ISLNfNavKyKj3b/NHhH5xJ1e1uuRHGC7qh5Q1T2qWtoEDwBV3Qd8BnQoy3rlSURaAIOAf5Ry+QdEZLtbvt/sM933nBR5jSoi7wNxwOdumfqwu/zlIrLUXT5FRNr7bDfdPbeLgP0i8pCITCkU0zARee2E/xgnyBI8U1p/A3oCiUAXoDu/331/ANgENABigccAFZG2wJ+BU1Q1CrgASC9hH9cAT+HcQVoNPFfMcm8Dt7vb7AT8oKr7gYuALT53krYcI26ARkBdIB64DRiDU7gUuBjYqqoLSojbGFMGItIc57O1QESaAl8Cz+J8Fh8EphRKlK7H+XxGAevdaTcAt+Dcrc4FhhWzu3fd+a2BrsD5wK2qehjns/60+wX+KBBK0eVOb/dnwd3q6cAEji4rrgW+V9Xi7jz3AH4F6gMvAm+LiBSx3DPANzjlYDPgPwCqWhBDl4I75SJyNs6F0NXu32G9G5evK9x9dwDeA66V32/A1QfOBcYXE7Mx5aE038P1gaY4CdhI9/qhJOvdRGq0+39ckhuAce7rAhGJBeemDvAFzucmwd3/BHdef5wnWTcAtXGe/O0qxbEWHI/vdUUIMNr9PQ44CLzus/z7QA2cp24NgX+708t6PfILzud7cCnjPIqIxOCUF7OPZ/1y8h+ca8iDpVi2EVAH57wNBv7rHkNhRV6jqur1HP2k+UURaQN8ANznLj8VJwGs5rO9a4FLgGhgLHChuNVaxXmqdw3OufOUJXimtAYCT6vqdvcC5imciy5w7ho1BuJVNUdVZ6qqAnlABNBBRMJVNV1V15Swj49Vda6q5uIUxInFLJfjbrO2e5dq/nHGDZAP/J+qHlLVgzgf1otFpLY7/3qcwtcYc+I+EZEM4EdgOvA8zgXMVFWdqqr5qvotMA/nYqbAu6q6VFVzVTXHnfa+qi5xb+78HbjavWA7wr2Quxi4T1X3q+p2nIunawBUdQlOYvkJTmJ5varmlfJYCpKlgiTtWGXFelV9y93+ezhlZmwRy+XgXAg2UdVsVS22cxac8u0dVZ2vqoeAvwKnilMlrcA/VHW3qh5U1bnAXqCgJsU1QIqq/lbikRpzYo71PQzwd/d7eDrODZ+ri9nWTuAUnM9IMs5Nn2Krf4rI6e6yE1U1FVgDXOfO7g40AR5yywffz9utwIuq+os6Vqvq+j/soGhHXVeo6i5VneI+WcvEuYl0phtfY5yb00Pd65kc928AZbgecZ8ojgT6AI+KyC3u9AgROSwidUqId75bLu/ESUBHlPI4AXa6T7oKXu2PvUrRRORKIFRVPy7lKjk4/1c5qjoVyAKKujFQ3DVqUQYAX6rqt+53zctAdaCXzzLDVHWje2634tTw6O/OuxDY6f6vecoSPFNaTfj9zjnu+4KqQC/hPHH7RkTWisijAKq6GucuyJPAdhGZID7Vh4qwzef9AaC4DhiuwrloWy9OVaZTjzNugB2qml3wi/vU7yfgKveOzEWU8OVhjCmTK1Q1WlXjVfVO96ZKPNDf9yIBOB3nC7nAxiK25TttPRCO8xTAV7w7favPtkfg3CUv8J673FRVXVXaA1HVOTjlVB8RaYfzhPCzElY5Ur6p6gH3bVFl3MOAAHPdakK3lLDNo8o3Vc3CecrQ1GeZwn+79/j9qcAg7AaW8b9jfQ/vcW/UHDVfROLEpwMPcP7HVXWee7PnN5xaQueLSFQx+74R+EZVd7q/j+f3aprNcW685BaxXnOcZPB4HHVdISI1RGSEiKwXkX04CUG0e0OqObBbVfcU3kgZr0cGA5+p6gycWgpPu2VHTyBNVfeWEG+SqkbjtN97E5gpIpGlPNb6bple8FruTs/FKXt9heMkWwW9FBec2+HiNLN5EbinlPsF2FXo3BV33VjkNWoxCpep+ThlaKUrUy3BM6W1BeciqECcOw1VzVTVB1S1JU41hvvFbWunquNVteAOmgL/PNFA3DtqfXEu0j4BJhbMKkvcJaxT8GHtD8xSp72QMcY/NuI8jfO9SKipqi/4LFPU57S5z/s4nAuHnYWW2Qgc4uiLkNqq2tFnmTdwqmld4N7tL0pxd3sLyorrgcm+F3XHS1W3qeoQVW0C3A68IcX3nHlU+eZeJNUDfMuswrGPBfqKSBegPU4Zaow/Het7OMb93z1qvqpu0KM7bilKwf/3H65nRaQ6zpPAM0Vkmzht4v4CdHH//zcCcVJ0ZxkbgVZFTAcnkajh83ujYmIq8ADOk6Ueqlqb36t8i7ufuuLTc2Uhpb0eCcNNqFR1Hc6TpH8CoyjldZf7xGoU0AK3ffQJ2IBT7dVXC9zkSZ1eigvO7VDgJHf5me55+gho7J63wtspk5KuUfnjuSpcpgrOd01JZeonQGcR6QRcSoA8FLAEzxQlXEQifV5hOHWSHxeRBm599ydwLhQQkUvF6RhAcKr/5AH5ItJWRM4WpzF1Nk6d6vwTCUycjhEGikgdtzDa57PN34B6haoiFBt3CT7B6eXvXgKgHrUxQW4scJmIXCBORyeR4nRU0OwY6w0SkQ4iUgN4GifBOqp6pVt95hvgFRGpLU7D+lYiUlA96nqcal434dw5fk+KHrphB045U3gszrHAlTgXYOVSVohIf59j34NzMeFbxvnG8AFws4gkuuXs88AcVU0vbvuqugmnrc77wBT3Kaox/lSa7+Gn3O/3M3AukicVtSER6eFeW4SISD2ctrcpxTyhugLneqQDTpOPRJybGjNx2tbNBbYCL4hITbfsOc1ddxTwoIgki6O1iBRc+C8ErnPLqwtxq1uWIArn+idDnM6g/q9ghltGfYVzIydGnI5UevusW9rrkY+AASJyhftkcB+QhpOkHihhvSPc9W52Y/Udliqs0DVh4SdzRfkQp2zq7v792uAk14XbCBdYgpNIFZynW3HKu0SKrsFRasVdo7qzC5epE4FLROQc9zgfwLlJ+HNx23dv7E3GeTo8V1U3nEi85cUSPFOUqTgf8ILXkzjtVOYBi4DFwHx3Gjh3Xr7Dqf88C3hDVafhtL97Aeeu+jacJ25/LYf4rgfS3aoOQ3Hq96OqK3C+SNaKUx2ryTHiLpJ7wTMF527TR+UQrzGmGKq6EeiL0/B9B86X+UMc+/vpfZwOVLbhVC0qrmrPDUA1YBlOwjQZ585wHPAqcINb7Ws8Tlnx78IbcKtUPgf85JYtPX1in4+ThM0s5SEfyynAHLdK2mfAvfr7GKBP4iShGSJytap+h9P+cArOhWor3PaFx/AecDIBUpXIBL1jfQ9vw/lsbsF5+jHU/T4vSkvgayATJyk4hNPpRVFuBEa7TwK3FbxwOjgZiPME7TKc6tUbcDriGACgqpNwPvPj3X19gtNxCjjJ1mVAhrudYz0FfxWnHddOnA5Mvi40/3qcGggrgO04TVtw4yjV9YiqzsJpW/h/OEnMDCAF6Ad8ICX3BJ7mljd7cP5mV6rqbp/5b3L0NeFon3kFPQsXvO534/kfTsdVo914puKUOyOLiT+30DnaDeS7v5e2XXRxirtGBaeTqsfdMvVBVf0V54bdf3DO12U4nbAcPsY+Aq5MleLbGRpTdYnTVXobVR10zIWNMRVKRFKAsao6KgBieQenOlmZxvTzkvuEYCxOpwN2EWA8I86QKGNV9VhP7Kssux4JfO4NwxVAI3WGm/BcQA/SZ4wX3CoUg/ljL1/GGHOE2zbkTzjDL1QKbrWje4FRltwZE9jseiTwiTPszP3AhEBJ7sCqaBpzFBEZglNF7Cu3NypjjPkDEXkGp4rYS26nBgFPnC7MM3B6KH3V43CMMSWw65HAJ07nQPuA8/BpWxkIrIqmMcYYY4wxxgQJe4JnjDHGGGOMMUHCEjxjjDHGGGOMCRKVrpOV+vXra0JCQqmW3b9/PzVr1jz2ggHOjiOwBMNxBNoxpKam7lTVBl7HcaKsfKqcguEYwI7DX4KhfLKyqfIKhuMIhmOAwDuOksqmSpfgJSQkMG/evFItm5KSQp8+ffwbUAWw4wgswXAcgXYMIrLe6xjKg5VPlVMwHAPYcfhLMJRPVjZVXsFwHMFwDBB4x1FS2WRVNI0xxhhjjDEmSFiCZ4wxxhhjjDFBwhI8Y4wxxhhjjAkSla4NnjHBJicnh02bNpGdnV1h+6xTpw7Lly+vsP0ViIyMpFmzZoSHh1f4vo0xZWflU/Aq7tx69fcvb77HUdXOrTGW4BnjsU2bNhEVFUVCQgIiUiH7zMzMJCoqqkL2VUBV2bVrF5s2baJFixYVum9jzPGx8il4FXduvfj7+0PBcVTFc2uMVdE0xmPZ2dnUq1evwi6evCIi1KtXr0KfBBhjToyVT8HLzq0xwcsSPGMCQLB/wRaoKsdpTDCpKp/bqnKcvqrKMVeV4zSmgCV4xlRxGRkZvPHGG2Ve7+KLLyYjI8MPERljjMPKp+Bl59YY/7EEz5gqrrgv2dzc3BLXmzp1KtHR0f4KyxhjrHwKYnZujfGfoE3wtmdm8/W6HFTV61CMCWiPPvooa9asITExkVNOOYUzzjiDyy+/nA4dOgBwxRVXkJycTMeOHRk5cuSR9RISEti5cyfp6em0b9+eIUOG0LFjR84//3wOHjzo1eEEvJy8fD5duJnVGXleh2JMwLPyKXjZuTXmaL+k72bmqh3lsq2gTfBmrNzJhF8PM31l+fyhjAlWL7zwAq1atWLhwoW89NJLzJ8/n9dee42VK1cC8M4775Camsq8efMYNmwYu3bt+sM2Vq1axV133cXSpUuJjo5mypQpFX0YlUaoCE98upSZm0q+S22MsfIpmNm5NeZ3Xy3eysBRc3jx61/Jzz/xh1NBO0zC5V2a8NxnixgxfS192jb0OhxjSuWpz5eybMu+ct1mhya1+b/LOpZ6+e7dux/VlfSwYcP4+OOPAdi4cSOrVq2iXr16R63TokULEhMTAUhOTiY9Pf3EAw9SISFCUlw0Kzfv9DoUY8rEyqfg5Xtu8/LyCA0NPeFt2rk1pnRG/7SOp79YRtfm0Yy68RRCQk68U6CgfYJXLSyE8xPCmbV2F2kbrTGuMaVVs2bNI+9TUlL47rvvmDVrFmlpaXTt2rXIrqYjIiKOvA8NDT1mG4qqLikuhs1Zyt6DOV6HYkylYuVT8LJza6qa/HzluS+X8dTnyzi/Qyzjh/Skbs1q5bLtoH2CB9CneRhT1+czcsZa/jswyetwjDmmstztLC9RUVFkZmYWOW/v3r3ExMRQo0YNVqxYwezZsys4uuCUHB8DwIINe6yGgak0rHwKXr7ntqIGOrdza6qyQ7l5PDhpEZ+nbeHGU+N54rKOhJbDk7sCQZ3gVQ8TBvWMZ8T0NaTv3E9C/ZrHXsmYKqZevXqcdtppdOrUierVqxMbG3tk3oUXXsjw4cNp3749bdu2pWfPnh5GGjy6NI9GgPkbMizBM6YEVj4FLzu3pqraezCH28bMY8663Tx6UTtu792y3MdqDOoED+DmXgm8PXMdo35cy7NXnOx1OMYEpPHjxxc5PSIigq+++qrIeQVtHerXr8+SJUuOTH/wwQfLPb5gUzMijLjaIcxfv8frUIwJeFY+BS87t6aq2ZJxkJtGz2Xdzv28dk0ifROb+mU/QdsGr0DD2pH8Kakpk+ZtYmfWIa/DMcYYAFpHh7Bgwx7yyqG3LGNM8BCRaBGZLCIrRGS5iJxaaH4fEdkrIgvd1xNexWqMKb3lW/dx5Rs/sTUjm/du6e635A6qQIIHcOsZLTmUm8+Yn9O9DsUYYwBoHR3K/sN5/Lqt6DYoxpgq6zXga1VtB3QBlhexzExVTXRfT1dseMaYsvpp9U6uHj4LQZh0x6n0alXfr/urEgle64a1OK9DLGNmr+fAYethyRjjvdbRTvGbusGqaRpjHCJSB+gNvA2gqodV1boCN6YS+2TBZm4aPZcm0dX5+K5etGtU2+/7rBIJHsDQM1uScSCHib9s9DoUY4yhfnWhYVSEtcMzxvhqAewARovIAhEZJSJF9RB3qoikichXIlLx3ZsaY45JVXkjZTX3fbiQ5PgYJg49lcZ1qlfIvoO+k5UCyfF16RYfw1sz1zGoZzxhoVUmtzXGBCARITk+hlRL8IwxvwsDkoC7VXWOiLwGPAr83WeZ+UC8qmaJyMXAJ8BJhTckIrcBtwHExsaSkpJy1Pw6deoUOUxBXl5escMXVCaFjyM7O/sPf4PKICsrq1LG7SsYjgHKdhz5qoxdfpgfNuTSo1Eog1tns2DOT/4N0EeVSfAAbj+zFUPGzOPLxVv92rDRGGNKIykuhq+WbGN7ZjYNoyK9DscY471NwCZVneP+PhknwTtCVff5vJ8qIm+ISH1V3VlouZHASIBu3bppnz59jtrR8uXLixzvrqLGwfO3wscRGRlJ165dPYzo+KSkpFD43FU2wXAMUPrjOHg4j3smLOCHDb9x+5kteeSCdoSU4xh3pVGlHmOd064hrRrUZMT0tahaz3XGHK9atWoBsGXLFvr161fkMn369GHevHkVGValk+QOeD5/vTWxMaY8VPaySVW3ARtFpK076Rxgme8yItJI3EGzRKQ7zrXcrgoN1AOV/dyaqmH3/sNcN2o23y3/jacu78hfL2pf4ckdVLEELyREuL13K5Zt3cePq3ceewVjTImaNGnC5MmTvQ6j0urUtDbVQkOYbx2tGFOuKnnZdDcwTkQWAYnA8yIyVESGuvP7AUtEJA0YBlyjVeiudSU/tyaIbdh1gKve/JllW/bx5sAkbuyV4FksVaqKJkDfrk14+ZtfGTF9LWec1MDrcIwJCI8++ijNmzfnrrvuAuDJJ58kLCyMadOmsWfPHnJycnj22Wfp27fvUeulp6dz6aWXsmTJEg4ePMjNN99MWloa7dq14+DBg14cSqUSERbKyc3qWEcrxhSjKpZNqroQ6FZo8nCf+a8Dr1doUH5QFc+tCV5pGzMY/N4v5OYr427tQbeEup7GU6We4IFzQXXL6S34cfVOlmze63U4xgSEAQMGMHHixCO/T5w4kRtvvJGPP/6Y+fPnM23aNB544IESqza/+eab1KhRg+XLl/PUU0+RmppaEaGXmohEishct+e5pSLylDu9hYjMEZHVIvKhiFRzp0e4v6925yf4I67k+BgWbd7Lodw8f2zemEqtKpRNVZWdWxMspq3YzjUjZxMZHsqUO3p5ntxBFXyCB3Bdjzhe/2E1I2as5T/XVr4GtyaIffUobFtcvttsdDJc9EKJi3Tt2pXt27ezZcsWduzYQUxMDI0aNeIvf/kLM2bMICQkhM2bN/Pbb7/RqFGjIrcxY8YM7rnnHgA6d+5M586dy/c4Ttwh4Gy357lw4EcR+Qq4H/i3qk4QkeHAYOBN9+ceVW0tItcA/wQGlHdQSXExjJyxlqVb9pEUF1Pemzem/HhQPlWRssl7Pue2el4uhJbD5aGdW1MFTJi7gb99soT2jaN456ZTAqbDtCqZ4NWODGdgjzjemrmWhy9oS/O6NbwOyRjP9e/fn8mTJ7Nt2zYGDBjAuHHj2LFjB6mpqYSHh5OQkEB2drbXYR43t41KlvtruPtS4GzgOnf6e8CTOAleX/c9OD3ZvS4iUt5tXZLiowGYv36PJXjGFCHYy6aqzM6tqaxUlX9/t4ph36/izDYN+O/AJGpFBE5aFTiRVLCbT2vBOz+tY9TMtTzVt5PX4RjjOMaTNn8aMGAAQ4YMYefOnUyfPp2JEyfSsGFDwsPDmTZtGuvXry9x/d69ezN+/HjOPvtslixZwqJFiyoo8tITkVAgFWgN/BdYA2Soaq67yCagYAyVpsBGAFXNFZG9QD2gXHtoahgVSVzdGqSu38OtZ5Tnlo0pZx6VT1WhbPKcz7k9WIHDJNi5NZVRTl4+j320mEmpm+if3Izn/3Qy4QE2vnaVTfAa1YnkisSmfDhvI/ee24a6Nat5HZIxnurYsSOZmZk0bdqUxo0bM3DgQC677DJOPvlkunXrRrt27Upc/4477uDmm2+mffv2tG/fnuTk5AqKvPRUNQ9IFJFo4GOg5IMqhWMNJlwc3wFTm0Ye4udVvzFt2jTc3s8rjWAYwDYYjgH8cxzFDYbtT4UHqI6Li2Pv3r00atSIWrVq0bdvX66++mo6duxI165dadOmDVlZWUfWyczMJCsri/z8fDIzMxk0aBB33HEHbdu2pW3btiQmJrJ///4ij6uyDoZdWVWF7x0TXLIO5XLnuPnMWLmDe885ifvOPSkgv7erbIIHcFvvlkxK3cSYWencd24br8MxxnOLF//evqZ+/frMmjWryOWyspyajgkJCSxZsgSA6tWrM2HCBP8HWQ5UNUNEpgGnAtEiEuY+xWsGbHYX2ww0BzaJSBhQhyLGmjrWYMLF8R0wdWPkemZ9soTWXXpUuirjwTCAbTAcA/jnOIobDNufihpoe+nSpUfeR0VFMXfu3CLXLSibOnXqxLJly44sP2XKlFLtu7IOhl2ZVZXvHVP5ZRzK55qRs1i+NZN/XnUyA06J8zqkYgXW88QKdlJsFOe2b8h7P6dz8LD1YGdMMBORBu6TO0SkOnAesByYhjOuFMCNwKfu+8/c33Hn/+CvsaaS4tx2eDYenjHGGBNw1uzI4tnZ2azZvp9RN3QL6OQOqniCB3D7ma3YcyCHSakbvQ7FGONfjYFp7uDBvwDfquoXwCPA/SKyGqeN3dvu8m8D9dzp9wOP+iuwtrFR1KwWSqqNh2eMMcYElAUb9tDvzZ85nKd8eHtPzmrX0OuQjqlKV9EE6BYfQ1JcNG/NXMt13eMIC7BGksaY8qGqi4A/1L1S1bVA9yKmZwP9KyA0wkJDSIyLtid4xhhjTACZtmI7d46bT8PaEdzVMYzOzaK9DqlUqnw2IyLcfmYrNu4+yNdLt3kdjqmi/FTzL+BUleM8HslxMSzfmsn+Q7nHXtiYClRVPrdV5Th9VZVjrirHacrX5NRN3DpmHq0a1mTy0F40rFF50ia/Rioi6SKyWEQWisi8Iub3EZG97vyFIvKEP+MpznntY2lZvyYjpq+1QsBUuMjISHbt2hX0/3uqyq5du4iMDIxBQANNUnwMeflK2qYMr0Mx5ggrn4KXnVtjiqaqvJmyhgcnpXFqy3pMuO1UGkRFeB1WmVREFc2zVLWkcaNmquqlFRBHsUJChCG9W/LXjxYza80uerWu72U4popp1qwZmzZtYseOHRW2z+zsbE++7CIjI2nWrFmF77cy6OoOcj5//R56tbIyyAQGK5+CV3Hn1qu/f3nzPY6qdm7N8cvPV575chmjf0rnsi5NeKV/F6qFVZ4ndwWqfBu8Ald2bcor36xk+Iy1luCZChUeHk6LFi0qdJ8pKSnWFXiAqVM9nDaxtayjFRNQrHwKXsWd22D5+wfLcZiKczg3nwcnpfFZ2hZuPi2Bv1/SgZCQwBvjrjT8nZIq8I2IpLqDARflVBFJE5GvRKSjn+MpVmR4KDeflsCMlTtYtmWfV2EYY6qw5PgY5m/IID8/uKtMGWOMMYEk61Aut7z7C5+lbeGRC9vxxKWVN7kD/z/BO11VN4tIQ+BbEVmhqjN85s8H4lU1S0QuBj4BTiq8ETc5vA0gNjaWlJSUUu08Kyur1MsCJOQqkaHwzKSfub1L4FRPKOtxBCo7jsARDMcQjJLiYvhg7kbW7syidcOKHVzaGGOMqYp2ZB7i5nfnsnxrJi/370K/5MpfndevCZ6qbnZ/bheRj3G6Ip/hM3+fz/upIvKGiNQv3GZPVUcCIwG6deumffr0KdX+U9WmVpwAACAASURBVFJSKO2yBRYcWsbon9N5qUt3msXUKNO6/nI8xxGI7DgCRzAcQzBKinfa4aWu32MJnjHGGONn63ft54Z35vLbvmzeuiGZs9vFeh1SufBbFU0RqSkiUQXvgfOBJYWWaSQi4r7v7sazy18xlcYtp7dAgLd/XOdlGMaYKqhl/ZpE1wi3dnjGGGOMny3ZvJer3vyZvQdzGD+kZ9Akd+DfNnixwI8ikgbMBb5U1a9FZKiIDHWX6QcscZcZBlyjHvfX2yS6OpcnNmHC3I3s2X/Yy1CMMVWMiJAc57TDM8YYY4x//LR6JwNGzCIiLJTJQ3uR5PZkHSz8VkVTVdcCXYqYPtzn/evA6/6K4Xjd1rslH83fzNjZ67n7nD80CTTGGL9Jio/h+xXbyThwmOga1bwOxxhjjAkqn6Vt4YGJC2lZvxbv3dKdRnUCp9+N8lL5BnaoAO0a1eastg149+d0snPyvA7HGFOFJLvt8BbYUzxjjDGmXI3+aR33fLCArs1jmHj7qUGZ3IEleMW6/cxW7Np/mMmpm7wOxRhThXRpFk1oiFg7PGOMMaacqCr//HoFT32+jPM7xDJmcHfq1Aj3Oiy/sQSvGD1a1KVL82jemrmWPBuTyhhTQapXC6Vjk9qW4BljjDHlICcvn4cmL+LNlDVc2z2ONwclExke6nVYfhW8Cd6B3cSnfwj5+ce1uogwtHdL1u86wP+Wbivn4IwxpnhJcTEs3JhBbt7xlV/GGGOMgYOH87j9/VQmp27i3nNO4vkrOxFaiQcwL63gTfBWfEGL9PHw3RPHvYnzOzYioV4NRkxfg8edexpjqpDk+BgO5uSxYlum16EYY4wxldKe/Ye5btRsUn7dzrNXdOIv57XBHZ0t6AVvgtf1ejY3uQh+/g/88vZxbSI0RBjSuyVpm/YyZ93ucg7QGGOKluwz4LkxxhhjymZzxkH6Df+ZpVv28cbAJAb1jPc6pAoVvAmeCKtbD4GTzoepD8Kqb49rM1clNaN+rWqMmL6mnAM0xpiiNYmuTqPakZbgGWOMMWW0Yts+/vTGT2zfd4gxt3Tnwk6NvQ6pwgVvggdoSCj0Gw2xnWDSTbBtcZm3ERkeyk29Epj26w5WbNtX/kEaY0wRkuNjmL/BEjxjjDGmtOau203/4bNQhYlDT6Vny3peh+SJoE7wAIioBddNhMg6MO5q2Lu5zJsY1DOeGtVCGTljrR8CNMaYP0qKj2HTnoP8ti/b61CMMcaYgPe/pdsY9PYcGtSKYModvWjfuLbXIXkm+BM8gNqNnSTvUCaMH+D8LIPoGtUYcEpzPlu4hS0ZB/0UpDHG/K6gHd58q6ZpjDHGlGj8nA3cMTaV9o1rM/mOXjSvW8PrkDxVNRI8gEad4Op3Yfsyp7pmXm6ZVh98egsUeOfHdf6IzhhjjtKhcW0iwkKsHZ4xxhhTgrdmrOWxjxfTu00DPhjSg7o1q3kdkueqToIH0PpcuOQVWP0dfPUQlGHog2YxNbisc2M+mLuBvQdy/BikMcZAtbAQujSLJtXa4RljjDFFGj9nA89NXc4lJzfmrRu6UaNamNchBYSqleABdLsZTrsX5r3jDKFQBrf1bsX+w3mMnbPeT8EZY8zvkuJjWLJ5L9k5eV6HYowxxgSUz9K28LdPFtOnbQP+PSCR8NCql9YUp2r+Jc55EjpcAd/+HZZ+UurVOjSpTe82DRj9U7pdcBlj/C45PoacPGXJ5r1eh2KMMcYEjB9W/Mb9Hy7klPi6vDkwmWphVTOlKU7V/GuEhMCVw6FZd/j4dtj4S6lXHdq7JTuzDvHxgrL3xmmMMWWRFBcN2IDnxlQlIhItIpNFZIWILBeRUwvNFxEZJiKrRWSRiCR5FasxXpi9dhd3jJ1Pu8ZRjLqpG9WrhXodUsCpmgkeQHh1uPYDiGoEH1wDu0vXecqprepxctM6vDVjLXn5pW/DZ4wxZVWvVgQJ9WpYgmdM1fIa8LWqtgO6AMsLzb8IOMl93Qa8WbHhGeOdtI0ZDH73F5rXrcGYW3pQOzLc65ACUtVN8ABq1oeBkyE/F8b1hwO7j7mKiHD7mS1Zu3M/3y77rQKCNMZUZUnugOdahk6hjDGVk4jUAXoDbwOo6mFVzSi0WF9gjDpmA9Ei0riCQzWmwq38LZMbR88lpmY1xg623jJLUrUTPID6J8E14yFjPXx4PeQeOuYqF3ZsRFzdGgyfvsYuuowxfpUcH8POrMNs3G1jcBpTBbQAdgCjRWSBiIwSkZqFlmkKbPT5fZM7zZigtWHXAQaNmkN4aAhjB/egUZ1Ir0MKaNaXKEDCadD3DfjoVvjsbrhyBIgUu3hYaAhDzmjB3z9dyi/pe+jeom4FBmuMqUoKBjxP3bCbuHpVe+BWY6qAMCAJuFtV54jIa8CjwN/LuiERuQ2nCiexsbGkpKSUar2srKxSLxvI7DgCx4kew57sfJ6fk82BXOWv3auTvuQX0sstutKrTOfCErwCnfvDnnSY9izEJMBZj5W4eL/k5vz7u1UMn77GEjxjjN+c1DCKqIgwUtfv4cquzbwOxxjjX5uATao6x/19Mk6C52sz0Nzn92butKOo6khgJEC3bt20T58+pQogJSWF0i4byOw4AseJHMOe/Ye5esQsDuSFMO62niQ2jy7f4MqgMp0Lq6Lpq/eDkDgQpv8TFo4vcdHq1UK55bQEflixne+sLZ4xxk9CQ4TEuGhS1xduhmOMCTaqug3YKCJt3UnnAMsKLfYZcIPbm2ZPYK+qbq3IOI2pCJnZOdw4ei7rdx/grRu7eZrcVTaW4PkSgUtfhRa94bN7YN2MEhcf0rslHRrX5uEpi9i+L7uCgjTGVDXJ8TH8um0fmdk5XodijPG/u4FxIrIISASeF5GhIjLUnT8VWAusBt4C7vQmTGP8Jzsnj1vfm8fSLft447okerWq73VIlYoleIWFVYOr34d6rWDCINjxa7GLRoSFMuzaRA4czuWBSWnk27AJxhg/SI6PIV8hbaMNeG5MsFPVharaTVU7q+oVqrpHVYer6nB3vqrqXaraSlVPVtV5XsdsTHnKycvnznHzmZu+m39d3YVzO8R6HVKlYwleUapHw3UTISwCxvWDrO3FLtq6YRSPX9KBmat2Mvrn9IqL0RhTaiLSXESmicgyEVkqIve6058Ukc0istB9Xeyzzl/dgYR/FZELvIseEptHI2IDnhtjjAluefnK/RPT+GHFdp7p24m+idZB7PGwBK84MfFw3QTI2gHjB8DhA8UuOrBHHOe2j+WfX61g+dZ9FRikMaaUcoEHVLUD0BO4S0Q6uPP+raqJ7msqgDvvGqAjcCHwhoiEehE4QFRkOG1jo0jdYAmeMcaY4KSqPP7JEj5P28IjF7ZjUM94r0OqtCzBK0nTZOj3NmxZAB8Ngfy8IhcTEf551cnUqRHOPR8sIDun6OWMMd5Q1a2qOt99nwksp+Rxo/oCE1T1kKquw2nr0t3/kRYvKT6GBRv2WFVwY4wxQUdVeeGrFXwwdwN39mnFHX1aeR1SpWYJ3rG0uwQueB5WfAHfPlHsYvVqRfBK/y6s2p7FP6Yur8AAjTFlISIJQFegoBvyP4vIIhF5R0Ri3GkBN5BwclwMmdm5rN6R5WUYxhhjTLl7I2UNI2asZVDPOB66oO2xVzAlsnHwSqPnHbBnHcx63Rkjr/uQIhfr3aYBg09vwds/ruPMtg04u501CjUmkIhILWAKcJ+q7hORN4FnAHV/vgLcUsZtVshgwrn78wEY/81s+jQPL0uIflWZBn4tTjAcA9hxGGMqpzGz0nnpf79yRWITnr68EyLidUiVniV4pSECF74AGRvgq4chOg7aFN3nwkMXtOWn1Tt5aNIivr6vNw2iIio4WGNMUUQkHCe5G6eqHwGo6m8+898CvnB/LdVAwu42KmQwYVXlxfnfkRXZkD59upR6PX+rTAO/FicYjgHsOIwxlc/HCzbxxKdLObd9LC/170JIiCV35cGqaJZWSChc9TbEdoJJN8OWhUUuFhkeyrBru5J1KJeHJqehau1ljPGaOLcD3waWq+q/fKY39lnsSmCJ+/4z4BoRiRCRFsBJwNyKircoIkJSfAzzrSdNY4wxQeB/S7fx4KRF9GpVj9ev60p4qKUl5cX+kmURUcsZPqF6jNOz5t5NRS7WJjaKv13SnpRfd/CeDZ1gTCA4DbgeOLvQkAgvishid0Dhs4C/AKjqUmAisAz4GrhLVT3vPSk5Poa1O/eze/9hr0MxxhhjjttPq3dy9/gFnNy0DiNv6EZkuGcdVQclS/DKqnZjGDgRDu+HcVdDdtHDIlzfM56z2zXk+a9WsGKbDZ1gjJdU9UdVFXfg4CNDIqjq9e5AwZ1V9XJV3eqzznPuQMJtVfUrL+MvkBzv9AFjT/GMMcZUVvM37GHImHm0qF+Td28+hVoR1mKsvFmCdzxiO8LV78GOFTDpRsjL+cMiIsKL/TpTOzKcez9YaEMnGGNO2MlN6xAeKjYenjHGmEpp+dZ93PTOXBpERfD+4O5E16jmdUhByRK849X6HLjsVVjzA3x2NxTR1q5+rQhe7t+ZX3/L5IWvVngQpDEmmESGh9KxSR1S7QmeMQFPRGJEpKOItBQRu94yVd66nfu5/u251KgWxtjBPWhYO9LrkIKWFTgnIukG6PNXSPsAvn+6yEX6tG3ITb0SePfndKb9ur2CAzTGBJukuBgWbcogJy/f61CMMYWISB0ReUxEFgOzgRE47XnXi8gkETnL2wiN8caWjIMMGjWHfFXG3tqD5nVreB1SULME70Sd+Qgk3Qg//gvmjCxykUcvakfb2CgemrSInVmHKjhAY0wwSY6PITsnn+VbrW2vMQFoMrAROMNtv3u6qnZT1ebAC0BfERnsbYjGVKx9h5RBb89h38EcxtzSndYNa3kdUtCzBO9EicAl/4K2Fztj5C395A+LFAydsC87h4cnL7KhE4wxxy0pPhrAqmkaE4BU9TxVfV9VM4qYl6qq96nq217EZowX9h7M4eV52WzJOMjbN51Cp6Z1vA4p8KjCwQzYsRK2l0+TLuu2pjyEhjlj5I3pCx8NgZr1IeH0oxZp2yiKxy5qx5OfL2Ps7PVcf2qCN7EaYyq1xnWq0zS6Oqnr93DzaS28DscYUwIRaQDcC1QHhqvqKo9DMqbCZGbncNPouWzOyuftm06he4u6XodUcVTh4B7I2g77t7s/d/j8vuPon3nu8EdxveCWE++42xK88lKtBlz3IbxzAXxwnXNyYjsetciNvRJIWbmDZ79cTo+W9WgTG+VRsMaYyiwpPobU9N1eh2GMObZXgLcABcYDp3gbjjEVY/+hXG559xcWb9rLnYkR9Gnb0OuQTpzmw/6dRSRpxSRv+bl/3EZIGNRs4LxqNYQG7aFWA6jZ0Pk9JqFcQrUErzzVqAuDpsCo82BsP7j1W6jT7MhsEeGlfl248NUZ3PPBAj7982lEhNnAjsaYskmOi+bztC1syThIk+jqXodjjHGJyP+A51R1hjupGpCOk+BFeBWXMRXp4OE8bn1vHqnr9/Cfa5OouftXr0M6MZtSIeV5zlz9A0wvooOzkHAnOavZAGrFQuzJRydttRr+/j4yGkL830LOErzyFh3nJHmjL4L3/wS3fO0kfq4GURG81L8zt7w7jxe//pW/X9rBw2CNMZVRcrxTpszfsMcSPGMCy9XA4yJyB/A48HfgHzhVNO/0MjBjKkJ2Th63vT+P2et28eqARC7p3JiUlEqa4G1Ng2nPw8qvoXpdNjW7nOadTi2UtDVwkjYRr6M9iiV4/tCoE1wzDsZeBR9cCzd8AuG/X4Sd3S6WG0+N5+0f13Fmmwb0btPAw2CNMZVNu8ZRVA8PJXX9Hi7t3MTrcIwxLlXdCzwkIi2B54AtwJ+L6nTFmGBzKDePO8amMnPVTl7q15m+iU29Dun4bF/uJHbLP4PIOnD249BjKGtmpdK8Zx+voysV60XTX1r0hitHwMY5MOVWyM87avZfL25Pm9haPDApjV02dIIxpgzCQ0Po0rwO860nTWMCioi0EpGXgVuBB4BPgA9F5B4RsTYZJmjl5OXz5/ELmPbrDp6/8mT6d2vudUhlt3MVTB4Mb5wKa6Y5Q6Hduwh6PwQRlavfDL8meCKSLiKLRWShiMwrYr6IyDARWS0ii0QkyZ/xVLhOf4IL/wErvoCpDzo96rgiw0N57Zqu7D2QwyNTFtvQCcaYMkmOj2Hpln1k5+Qde2FjTEX5APgImAa8r6ozVfUCIAP4xtPIjPGT3Lx87p2wgG+X/cZTl3fkuh5xXodUNrvXwsdD4b/d4depcPp9cN8iOOsxqB7tdXTHpSKqaJ6lqjuLmXcRcJL76gG86f4MHj3vgMyt8NNrENUEznzoyKz2jWvzyEXteOaLZYybs4FBPeM9DNQYU5kkxcWQm68s2rS3anU9bUxgiwDWAbWAGgUTVXWMiEzyLCpj/CQvX7l/YhpTF2/j8Uvac2OvBK9DKr2MjTDjRVg43undsuedcNp9Tru6Ss7rNnh9gTHqPL6aLSLRItJYVbd6HFf5OudJyNwG056FqEaQdP2RWTf3SmD6yh08++UyerasS+uGlesRsDHGG13jYgBnwHNL8IwJGHcCrwOHgaG+M1T1oCcRGeMn+fnKw5MX8VnaFh6+sC23ntHS65BKZ99WmPkKzH/P+b3bLXD6/VC7sbdxlSN/t8FT4BsRSRWR24qY3xTY6PP7JndacAkJgctfh1Znw+f3wsr/+cwSXu7XmRrVwrjng4UcyrXqVsaYY6tbsxotG9Qk1drhGRMwVPUnVb1KVa9V1TSv4zHGX/Lzlcc+XsyU+Zv4y7ltuLNPa69DOrasHfD1YzAsEVJHQ+J1cPd8uPiloEruwP9P8E5X1c0i0hD4VkRW+IwNU2pucngbQGxsLCkpKaVaLysrq9TLVoTQxkNI/G09NSYMYmHis2TWbntk3vVthdfm7+PeUd9zTbtqR60XaMdxvOw4AkcwHIOB5LgYvl+xHVVFAqyLZmOqIhH5HBgB/E9VcwrNawncBKSr6jsehGdMuVBV/u+zpUz4ZSN3ndWKe84J8OTuwG6nqdTckZCbDV2udTpOqdvC68j8xq8Jnqpudn9uF5GPge6Ab4K3GfDtZqeZO63wdkYCIwG6deumffr0KdX+U1JSKO2yFaZnN3j7PJKXvwCDv4X6zoeiD7Cr2mLGzt7AoHOSOP2k+kdWCcjjOA52HIEjGI6h0lKF5Z9TM2v3CW8qOT6GSambSN91gBb1a5ZDcMaYEzQEuB94TUR2AzuASCABWAO8rqqfeheeMSdGVXnmi+W8P3s9t/VuyYPntw3cG4wHM2DWf2H2m3A4C07uB2c+euTaO5j5rYqmiNQUkaiC98D5wJJCi30G3OD2ptkT2Bt07e8Kq9UQBn0EEgJjr3Ta5rn+dnEHWjesxf0TF7Jn/2EPgzTG+M2hTPj8HlqtGX3Cm0qO/70dnjHGe6q6TVUfVtVWQH/gGZyEr5OqnmfJnanMVJUXvl7BOz+t46ZeCfz1onaBmdwdyoTpL8FrnZ1OVFqfDXfOgqtGVYnkDvzbBi8W+FFE0oC5wJeq+rWIDBWRgobHU4G1wGrgLZzGycGvXisYOBH274Rx/SB7HwDVq4Xy2jWJ7DlwmEemLLKhE4wJRpG1ofdD1N2zENb8cEKbatWgFrUjwyzBMyYAqWq6qs5S1YWqesDreIw5Uf/+diUjpq9lYI84/u+yDoGX3B3eDz++Cq92djo2jOsFt8+Eq8dAw/ZeR1eh/FZFU1XXAl2KmD7c570Cd/krhoDWNNn5hxs/AD4cBAMnQ1g1Ojapw8MXtOO5qcuZ8MtGru1eycYSMcYc2ym3cnD6q1T/9glo0cfpiOk4hIQISfExNuC5McYYv/rP96sY9sNqru7WjGf6dvJPcqcK+XmQn+u8NO/o34+8Ck/Lg41z4cd/wf4d0Ppc6PMYNEsu/xgrCa+HSajaTjoP+r4On9zhvP70FoSEMPj0FkxfuYOnP19m3Z8bE4zCIljXYhAdlv8LFk+CLgOOe1PJcTH8a+VK9h7MoU718HIM0hhjjIER09fwyrcr+VPXpvzjT50JCSllcnd4P8wdSbdfRkNaSNGJme/veoI9ybfoDWeNhbieJ7adIGAJntcSr3MGQv/+aWeMvAueIyREeOXqLlz46gzunbCA+zpaVU1jgs32hmfQIeN7+OFZ6NAXwiOPaztJ8TGowsKNGZzZpvIPzmpMsBGRJFWd73UcxhyPd35cxz++WsGlnRvzYr/OhJYmucvJdoYhmPkK7N9Bbp1O0Ky9M5h4SKj70/dVeNqxfi9iWs0G0CTR/3+QSsISvEBw+v3OoIuzXoeoxtDrz8TWjuSFqzpz+/upTKkWzrlnex2kMaZcSQic9zSM6Qu/vAW97j6uzXRpHk2IwPz1eyzBM8ZjIpJUeBLwqYhcBogleqYyeX9WOk9/sYwLOzbi3wMSCQs9RnOC3MOwcKzTwUnmFveJ2jgWrj1oPXdXMEvwAoEIXPRPyPoNvvmb8yTv5H5c0LERA3vEMW7OBr5espULOwXXIIzGVHkt+0Crc2DGy9B1EFSPKfMmakWE0a5RbeZvsHZ4xgSAecBs4JDPtHrAvwAF7HatqRQmzN3A3z9dyrntGzLs2q6El5Tc5eXC4omQ8gJkrIdm3eHK4dDyTGf+2pQKidn8zp+9aJqyCAl12uDFnwYfDz3yYXjisg60rBPC/RPT+HVbprcxGmPK33lPQfZemPmv495EcnwMCzZkkJdv1bmN8Vh/IAd4UVXPUtWzgG3u+1IldyKSLiKLRWShiMwrYn4fEdnrzl8oIk+U8zGYKm5K6ib++vFizmzTgP8OTKJaWDHpQn4+LJkCb/R0+pKoHu10Gjj4m9+TO+MJS/ACSXgkXDMe6p8EEwbB1kVEhIVyd9cIakaEcdv789h7IMfrKI0x5anRydDlGpgzAjI2HtcmkuNjyDqUy8rf7CaQMV5S1SnAJcD5IjJJROJwntyV1Vmqmqiq3YqZP9Odn6iqTx93wMYU8unCzTw0OY1ereox4vpkIsJC/7iQKqz4EoafDpNvcdrADRgLt013OhAMtOETqiBL8AJNwd2PyNrOGHl70omJDGH4oCS2ZBzkngkL7C69McHmrL85P6c9d1yr24DnxgQOVc1S1b8AzwPvAVEeh2RMqXy1eCv3T0yjW0JdRt1wCpHhhZI7VVj9Hbx1Fky4DnKz4aq34Y6foP1lltgFEEvwAlGdpjBoivPBGXsV4Yf3kRxfl6cu78T0lTt4+ZtfvY7QGFOeoptDj9shbQJsW1zm1ZvFVKdBVISNh2dMAFHVBTht7lqVdVXgGxFJFZHbilnmVBFJE5GvRKTjCQVqDPDtst+4+4MFJDaP5p2bTqF6tULJXfqPMPoiGHsV7N8Fff8Ld82Fk/s5zYxMQLFOVgJVw/Zw7Yfw/hUkLvwrxIVxXfeLWLJlL2+mrKFjk9pc2rmJ11EaY8rLGffD/DHw3ZPODZ4yEBGS42JItY5WjAkoqqoikgIU7l2zJKer6mYRaQh8KyIrVHWGz/z5QLyqZonIxcAnwEmFN+Imh7cBxMbGkpKSUqqdZ2VllXrZQGbHUXppO3IZNv8QcbVDGNz6EPNm/XhkXu29v5KQPo66e9I4VK0u608aytbG56J7w2HmjyVs9Xd2LiqeJXiBLP5UuGY8IZPvhAnXQqPOPNX7EVZujeGhSYto1aAW7RvX9jpKY0x5qB4DvR+Ebx53Ollq2adMqyfHx/D10m3syDxEg6gIf0RojDk+Zaq3pqqb3Z/bReRjoDsww2f+Pp/3U0XkDRGpr6o7C21nJDASoFu3blrabupTUlKCokt7O47SmblqB//9bh7tGtdm/K09qVMj3JmxNQ2mPQ8rv4Ya9eGC54nodgttwqvTpoz7sHNR8ayKZqBrfQ5zu78Bfd+AQ5mETxzIBzzCJRELuG3ML+zZf9jrCI0x5eWUIVAnDr59wumdrAyS3HZ4NlyCMQHny9IuKCI1RSSq4D1wPrCk0DKNRJzGTiLSHedablf5hWuqipmrdjBkzDxa1q/J2ME9nORu+wqYeAOM6A0bZsE5T8C9aXDqXRBe3euQTSlZglcJaEgodB0If54Hfd8gPCeLl3NfYPiB+xn9zhvk5uZ5HaIxpjyER8LZjzt3TpeUrZpmp6a1qRYaYgmeMQFGVR8vw+KxwI8ikgbMBb5U1a9FZKiIDHWX6QcscZcZBlyjqtb7mimT75b9xuB355FQryZjb+1BTPZG+Og2Z8iD1d/DmY/AvYvgjAcgopbX4ZoysiqalUlomJPodR4Aiz4k7tt/cP+u/2Pbv8bR6PKnoO1F1oORMZXdyf1h1n/gh6ehw+UQVrrqlhFhoXRqWts6WjHGQyKSydHDIiiwE5gGPKKqJT5pU9W1QJcipg/3ef868Hq5BGyqpM/TtvCXDxfSsUltxlzVmDrfPwALx0NoNTjtHuh1L9Ss53WY5gTYE7zKyE30oh5YyEdxj5GdtddpozeiN6yY6nRja4ypnEJC4LynIWMD/DKqTKsmx8eQtmkvh3PLVr3TGFM+VDVKVWv7vOoA3YClwPBjrG6M302ct5F7JywguXkUEzv8TJ1RPWHRh3DKrXDvQuf7x5K7Ss8SvMosNIzLbnyIR5u8w6N5Qzl0wBI9Y4oiIs1FZJqILBORpSJyrzu9roh8KyKr3J8x7nQRkWEislpEFolIWXrAO3GtzoaWZ8GMl+BgRqlXS46P4XBuPku37PVjcMaYslDVPar6b8o+XIIx5WrMrHQenryIAXGZjJe/ETHjOWh7Mdw9Hy5+EaIaeR2iKSeW4FVy4aEhvD7oFKbXOJ/zDr1C1oXD4NA+S/RMUBORst5ezAUeUNUOQE/gLhHpADwKfK+qJwHfu78DXITT7fhJON2Mv1kugZfFeU85yd2P/y71KklxeFBy0wAAIABJREFUNuC5MYFIRMKxZjHGQ8Onr+HpT9N4tdE3PL/jz4Tu2wz934Or33PGYjVBxRK8IFC/VgQjrk9m2/5chixqS84dc91eNy3RM0FrtohMEpGLC3qTK4mqblXV+e77TGA50BToC7znLvYecIX7vi8wRh2zgWgRaVzuR1GSxl2g89UwZzjs3VSqVRrWjqR53erW0YoxHhGRPxXxGozTk+Zkr+MzVY+q8q9vV/Lp1/9jWp1nuCLjXaTD5c4g5R2v+H/27js8iqp74Pj3bHoPNZTQe4ckdEUQwYagdFSq0m342n/219eCFQvSexMQERtSNBSpCb33Kr0EQoC0+/tjFggQYAPZ7CY5n+eZZ3dn7wxnJME9O/eec+sTqGxJE7wconp4KB89Vo2lu07w4Z87rqq6qYmeyoHKY/V36gxsF5EPRcSh1jwiUhKoBSwHwowxh+xvHcaqYAdW8rc/zWEH7Puy1r1vgkm1ehE5KLJ4HmL3nkKL6inlEo9cs7UAKgKDjDHvuzIwlfsYY/j413VI9Mf86vMm4Z5x0GECtB2l6+xyOJ0ukIO0iQxn479nGPXPbqoWCaFNZLi96mZ7WDcVFg683DCdxq9r1U2VbdlLgs8F5opIE2AC0M9eNvw1Y8zS9I4TkUDgR+AFY8yZtDf/jDFGRDKcFYlIL6xpnISFhREdHe3QcfHx8Q6NLVP4IcLXTCLGsw7nAkvecnxQYhJHziTy4+y/ye/n/O/wHL0Od5YTrgH0OtyBMaa7q2NQCiA11TB48gxabX2Pyl57MdXaIw9+Av55XR2aygKa4OUwbzxUkc2HzvD6T+spFxZI9fBQ8PDSRE/lKPY1eE9i3cE7AjwLzAJqAtOAUukc44WV3E00xsyw7z4iIoWNMYfsUzCP2vcfBNIuSgi377uOMWYY1t1EoqKiTOPGjR26hujoaBwaW6c6fP03teN+gxbTbjm8wL9xjN+0GK/CFWhc0/k3HR2+DjeWE64B9DrcgYhMNca0tz//xBjzapr35hhjmrsuOpVbJCde4K9hL9P72EQu+oRiWk9EKrVwdVgqC+kUzRzG08PGt4/XokCgD73Hx3Ls7MUrb15K9NKburlpFqRqw3SVbSwFgoFHjTEPG2NmGGOSjTExpFOK3L5ObySw2RjzRZq3ZgFd7c+7Aj+n2d/FXk2zHhCXZipn1vLPazWa3T4Hdi+85fAKYUH4e3uwep/j1TeVUpmmXJrnza55r0BWBqJyp6R9sRz5rB7Nj49jR9gDBAyI0eQuF9IELwfKZy+6ciohkf4TV13fEyu9RG9qZ/gmApYPhYvxrglcKce9aYz5rzHmcvUREWkHYIz5JJ3xDbHu9t0rImvs20PAx0AzEdkO3Gd/DfA7sAvYAQwH+jnvUhxQpzeEFIO5b0PqzXvceXrYqFksVCtpKuUaN5vmrQtjlfMkXyR5znvYRt2Hx8XT/FljEJX6TUJ0SmaupAleDlW1aAiftKnOij0n+eC3TekPupzoxVqlcgMKwh+vwBeVYc5bDlfuU8oFXktn3+s3GmyMWWyMEWNMdWNMTfv2uzHmhDGmqTGmnDHmPmPMSft4Y4zpb4wpY4ypZr8z6DpevtDk/+Df1bBxxi2HR5bIw6ZDZ0hITM6C4JRSafiLSC0RiQT87M8jLr12dXAqhzoYS+qQu/Fc8gUzUu5iUbNfuP+xbq6OSrmQrsHLwVrVLMrGf88wbOEuqhQJpkPt4ukP9PC0SuVWeRT2r4Rl38HSb2Hpd9a++v2haGTWBq9UOkTkQeAhoKiIfJ3mrWCsXnc5V/X21u/k/Peh0iPg6XPDoREl8pCSali7P476ZbRSmlJZ6BBwaRr44TTPL72nVOZJugDRH2GWfM0pycNLSa/wSJuutI4Id3VkysU0wcvhXrm/ApsPneGtmRspFxZ0uRHyDRWrDcXGwOl91nTNVeNgw49QvD7U6wcVHwabR5bErlQ6/gVigJZAbJr9Z4EBLokoq9g8oNm7MKENxIyCen1vODSieB68PW38uOqAJnhKZSFjTJMbvScidbMyFpXD7V8JP/eD49uY49Oc1+I78GHHhjxYLWtbtir3pFM0czhPDxvfdKpFWIgPfSfEcvTMBccODC0O9/8PBmyE+z+CMwetdXpf14JlQ+DiWecGrlQ6jDFrjTFjgTLGmLFpthnGmJy/6KxMUyh1DywYCBfibjgsxM+Lbg1KMmPVAbYe1t9VpdzErcvgKnUrSedhzpswqjkpF8/xuv+7PHuuB190bqTJnbpME7xcINTfm2GdozhzPpk+E2K5mJyBapm+wVC/Hzy7GtqPg6BCMPtV+KKK9Q/M6f23PodSmUREptqfrhaRddduLg0uK4hAs/fh/ElY/NVNh/ZrXIYAH08+/XNLFgWnlLoF7Uek7sy+5TDkLljyDeeqPMEjKZ/yc3wlxnSrTZOKBV0dnXIjmuDlEpUKB/NZuxqs2nead2fdoOjKzXh4QuVW8NQceHo+lG0KSwfDoBowrTsciL31OZS6c8/bH1sAj6Sz5XxFakK1drBsMMSl25oPsL7Y6du4DPM2H2XF7pNZGKBS6ga0iqa6PYkJ8Of/waj7ITmRw62m0HxHa/YneDL+qTo0KJvf1REqN6MJXi7ycPXC9G1chskr9jFx+d7bP1F4FLQbDc+vte7u7ZgHI+6Fkc1h08/aT085TZpedAHGmL1pN9Jpbp5j3fsmmFSI/vCmw7o3KEVYsA8f/7EZY/SzpVLOJiK/iMisdLZfAF0QqzIs5PQm667d0m8hqgc72s6h5e9eJCQmM7lnPSJLaBsEdT2HEjwRCRARm/15eRFpKSJezg1NOcNLzStwT/kCvDtrIzF77vBb/dBi0PwDeHETPPAJnD0MU7tY6/SWDoYLZzInaKWuN1VEXrU3IvcTkW+Aj1wdVJbJUxJq94Q1k+Do5hsO8/P2YMB95Vm17zRzNh3JuviUyr0+Az5PZ/sMqwKwUo45dwJ+f4Waa96A1CToMosNtd6h/ZgNGOCH3vWpWjTE1VEqN+XoHbyFgK+IFAXmYDUMHuOsoJTzeNiErzvWomioH30mrOJwnINFV27GJwjq9YHnVkOHCRBcBP58Hb6sYk0pOL3vzv8Mpa5WFygGLAFWYlXXbOjSiLJao5fAOwjmvXvTYW0jwylTIICBs7eQnHLzJulKqTtjjFlws83V8alsIOGk1Q5nUHVYMYx/izwIfZeyyrM6nYYvw8/Lg2m961M+LMjVkSo35miCJ8aYBKA1MNgY0w6o4rywlDOF+HsxrEsU5xOT6T0hlgtJmTSl0uZh9efqMRt6/gXlmsOy72FQTZjWjXzHl1v/cCl155KA81iNg32B3caY3JW9+OeFuwfAttmwZ/ENh3l62HjlgYrsPHaO6bEHsjBApXIfESknIqNF5AsRCReRP0QkXkTWikhtV8en3Nj5U/DX/+Cr6rDoC+szVL9lbC/fm6UHLtJ5xHLyBXjzQ+96lMwf4OpolZtzOMETkfrAE8Bv9n3aDC0bKx8WxOfta7B2/2ne/nlD5q/PKRoJbUfCC+usRuk7/qLahg9hYCn4tg7Mes6aXnZyF+jaIJVxK7ESvNrA3UAnEcl9Jcjr9oHgojDnrZv+HjWvHEZE8VC+mred84m6RlYpJxoNLMWaVbAcGAXkB14CvnVhXMpdXYiD6I/hqxqwcCCUvRf6LrFqHRSsyLpjyXQbvYIioX5M7V2f8Dz+ro5YZQOOJngvAK8DPxljNopIaeBv54WlssIDVQvz3L1lmRpzgHFL76Doys2EhEPz/8JL21hd80No+jbkKQGbZsLMvtZ6vc/Kww+drXV7B2MhJck5saic5CljzNvGmCRjzCFjTCtglquDynJeftDk/+DfVbDxpxsOExFee7ASh89cYMySPVkXn1K5T6AxZpgx5jPgvDFmmjHmgjFmLuDj6uCUG7lwxupp+lU1iP4ISjeCPv9YLanCKgMwe8NhBq26SNmCgUzpVY+Cwb4uDlplF56ODLLPG18AYC+2ctwY85wzA1NZ44X7yrPp0Bne+2UjxfP506SCk/qoePkSF1oF7m5svU5NhWNbYN9S2L/cetxs/3zu5W9V6ixWD4rXg/DaVj8+pa6IFZEngdLGmPdFpDiw1dVBuUSNjrD0O2vNRsUW4Omd7rA6pfLStGJBBkfvoFOdYoT6pz9OKXVH0k4Vv7bSWO6aRq7Sd/EsLB9qVcU8fwoqPAyNX4PC1a8a9tu6Qzw3ZTWlQmxM6lmPED+tbagc51CCJyKTgD5ACtbUqGARGWSM+dSZwSnns9mEQR1r0X7oUp6ZuIppfRpQuUgWJFM2m/UNVVhlqP2Ute/Mv7BvmbXtXwaLPrNKwYsNwqpA8fpQrK71GFLU+TEqdzYY68PSvcD7wFngR6wpm7mLzQOavQcT20LsaKjb+4ZDX3mgIg8MWsjg6J288VClLAxSqVyjooisw2pqXsb+HPvr0q4LS7ncxXhYORz++RrOn4TyD1iJXZFa1w39fb2V3EUUD6VH2Yua3KkMcyjBAyobY86IyBPAH8BrQCygCV4OEODjyciutXn0u394auxKZvZvSJgrpgEEF4Gqra0NrG+5DsTYk76lsHoirBhmvRdSzLq7V7yedaevYCXrg67KLeoaYyJEZDWAMeaUiOTeW1Jl74OSd8OCT6BGpxve8a5QKIg2EeGMWbKHrg1KUjTUL4sDVSrH029O1NUSE2DlCPjnK0g4AWWbQePXITwy3eF/rD/Es5NXU6tYKKO71yFm6Y2LaCl1I44meF72vnePAt8aY5JERCtj5CCFQnwZ1a027YYsoceYlUztXZ8AH0d/PJzEJwjKNLE2gJRkOLIe9tmndO5eBOvtdTV8QqBYHSjRwPqgW6QmeOg3XjlYkoh4AAZARAqQm6c/iUCz92F4E/hnEDR964ZDBzQrz6y1//Ll3G181q5GFgapVM5njHHSgnaV7SSdh5hRsPhLOHcMytwLjd+AYjeeaDJ7g5Xc1SwWypgedQh09ecwlW05+pMzFNgDrAUWikgJrp9brrK5ykWC+fbxCJ4au5LnJq9mWJcoPGzi6rCu8PC0pjIUqWX13TMGTu+9Mq1z31KY/5411jvQurtX8i4r4Stc0zpe5RRfAz8BBUXkf0Bb4E3XhuRiRSOgahtrPV7tpyG4cPrDQv3o1qAkwxft4um7S1GxkK5vVUqpTJN0wZouv/hLiD8Cpe6BJm9Yn0luYvaGwzwzaTXVw0MY0722JnfqjjhaZOVrrA9Ul+wVkSbOCUm5UpOKBXmvZRXe+nkj//11E++2dON2hyKQp6S11eho7Ys/Bnv/sfqC7Vl0pQm0d6C1du9ywldDE75szBgzUURigaZYa1seNcZsdnFYrnfvW7BpFkR/CC2/ueGwfo3LMHnFPj6dvZWR3XLfskWllMp0yRdh1ThY9DmcPWR91mg3xppZdAtWcreK6uEhjO1RhyBfnYGk7oyjRVZCgHeARvZdC7AKG8Q5KS7lQp3rl2TviQRGLN5NiXz+dG9YytUhOS6wAFR51NoA4o9eSfh2L4J571j7vYOgxKWE7y4opAlfdiAiedO8PApMTvueMeZk1kflRvKWsu7erRgK9fpDwYrpDgv196Zv4zIMnL2VFbtPUqdU3nTHKaUyj4jkM8accHUcKpMlX4TV463m5GcOQvEG0HoYlGp062OBPzdayV01Te5UJnL0E+0oYAPQ3v66M1Yzz9bOCEq53usPVWLfyQT+++smiuXx577KYa4O6fYEFoQqj1kbWAnfnsVX7vBtn2Ptv5zw3W1P+KprwueeYrHW3aU3d9igVeqg0cuwZqJ19/rxKTcc1r1BKcYu2cPHf2zmx74NEHGj6dhK5RAishP4DZgAjAEquzQglXmSE61/axd9DnH7rSrfjw62pmQ6+O/pnI2H6T9xFVWLanKnMpejn2DLGGPapHn9noiscUZAyj142ISvOtak47BlPDt5NdP61Kdq0RBXh3XnAgteXanz7BHYu/jKHb5LCZ9PsDWls1SahE+rdLqcMSYb3U52kYB8cNcLVl+8XQug9D3pDvPz9mDAfeV5bcZ65mw6wv1VCmVxoErlfMaYMiIyAFgKdHd1PCqTxB2E8Y/B8a1Wr95HBllFVDLwRdncTUfoP8lK7sY9VYdgTe5UJnI0wTsvIncZYxYDiEhD4LzzwlLuwN/bkxFdo3jsO6uy5sz+DSmS08qqB4VZhSmq2r+/OHs4zR2+xbD9T2u/T7A1j75EQ/IfOwcHAiGoEASGabVOFxGR1sBdWHfuFhljZro4JPdRty+sngAz+0KfxeCf/hTMtpHhDF+0i4Gzt9C0YkE8PWxZHKhSOYuIzAF6XqqmKSL1sPoI9wZaAONcGJ7KDKf3wZgWVpPyTlOsfnYZnAExb9MR+k2MpXIRTe6Uczia4PUBxtnX4gGcAro6cqC9lHkMcNAY0+Ka97ph9dI7aN/1rTFmhIMxqSxQMMhqn9D2eyvJm9anfs6eQhBUCKq1tTaAM4fsa/gWWQnfttlUBdj48ZVj/PNDUGHr2Ks2+77AQtadQ00EM42IDAbKcmUNXh8RaWaM6e/CsNyHtz+0HQ0jm8HMftBpcrofQDw9bLzyQEV6j49leuwBOtYp7oJglcpRCqZJ7h7G+ozziDFmm4j0dm1o6o6d3A1jW8LFOOgyE4qm38vuZuZtOkLfibFULhzMuB6a3CnncLSK5lqghogE21+fEZEXgHUOHP48sBm4US3uH4wxzzgSh3KNCoWC+O6JCLqPWckzk1YzsmtU7vmmP7jw1QnfuePE/PUzUeWLWlWy4o9Yj2cPW9vh9XDuKJhrW7IJBBS4PgEMDEuTHBa2xujaP0fcC1QyxlzqgzcW2OjakNxMkZrQ/AP44xVY9j3U75fusOaVw4goHsqX87bRqmZR/Lx1KrJSd+CiiHQFigHPArWMMf/aPz8FuDY0dUdO7ISxj0BSAnT9xarGnUHzN6dJ7p6qS4ifJnfKOTL0SdIYk7b33YvAVzcbLyLhwMPA/+zjVTbVqHwBPni0Kq/PWM87szbywaNVc2dRhoD8xAeVgQqNbzwmJdlqahpvT/rOHrLW+l1OBA/BobVWwRerT/cVYoOAgpC/HOQvDwUqQoHykL+ClQTmxv/m6dsBFAcuNRUuZt+n0qrTC3YvhLlvWz2YikZcN0REeO3BSrQfupTRS3bTr3FZFwSqVI7xBPAakAgMBEaJyBKgFTDckROIyB7gLJACJBtjoq55X4BBwENAAtDNGLMqsy5ApePYNiu5S02Crr9CoaoZPsVfW47Qd8IqKmlyp7LAndwqcOST5lfAK0DQTca0EZFGwDZggDFm/x3EpJyoU53i7D2RwJAFOymVP4Cn79aCheny8LTu/N2g0fRlKcnW3b6zaRPBw1aZ5ePbYP10axrIJT4hVuKXNukrUB5CS+TGAjBBwGYRWYGVJdcBYkRkFoAxpqUrg3MbIlY/vKGNYHp36L0QfK8vllSnVF6aVizI99E7ebxOcUL9vV0QrFLZnzFmB/D0pdci8hdwH/CqMWZeBk7VxBhz/AbvPQiUs291ge/tj8oZjm62kjsEuv0GBStl+BR/bzlKn/GrqFAoiPE9NLlTzncnCZ652Zsi0gI4aoyJFZHGNxj2CzDZGHPRPjd9LNbUq2vP1QvoBRAWFkZ0dLRDAcbHxzs81p2503XU8TXEhHnwv982E/fvLiLDHP8RcqfruBPOuQ5/oAzYykAo1lbG4J14Cv+E/QScO4B/wn78zx3A/9hv+CROuHxkis2b835FORcQToJ/OAn+xeyPRTC29P8nkgP+Lt52dQDZhn9eaDMSRj8Ivzxvrc1L507wKw9U5IFBCxkcvZM3Hsr4Bxil1PWMMauB1Zl82lbAOPsU9WUiEioihY0xhzL5z1GH18O4VmDzsqZlFiif4VP8vfUovcfHUqFQEBOeqkuIvyZ3yvlu+ulcRM6SfiInwK3KKTYEWorIQ4AvECwiE4wxT14acE3DzxFY0xmuY4wZBgwDiIqKMo0bN77FH22Jjo7G0bHuzN2uo8FdKXQctozhG87QtEEUNYuFOnScu13H7XKL6zh/ypoycnwrHse2Enh8G4HHtsLRxVz+lRUPq/H1pTt9+StAgQqQvzzRS2Ncfw23yV646V1jTBNXx5JtFK8LTd+yeuOVugeirq/WXqFQEG0iwhmzZA9dG5SkaE6rmKtU9mGAOSJigKH2z0BpFQXSznY6YN+nCV5m+ncNjH8UvPyt5C5fmQyfItqe3JUvFKjJncpSN03wjDE3m1p5U8aY14HXAex38F5Km9zZ96f9xqklVjEW5eZ8vTwY0TWKR7/7h6fHruSnfg0pltff1WHlLn55rA/txa+ZlZOYACe2X07+OLbVmu65fY61dsCugVcIxPpba/7EBoh1V0dsaR4v7bel2S/p7E/vGAGfIKt6YyYzxqSISKqIhBhj4m59hAKgwfNWr8fZr0GxOhBW5bohA5qVZ9baf/ly7jY+a5fxAgJKqUxxlzHmoIgUBOaKyBZjzMKMnkRnP93+dQSd2U71de+Q4hHAmkrvcGH9fq7OqW9t/bFkBq2+SNFAG30qJLN6xT+3FUtO+PvICdcA2es6srxcn4i8D8QYY2YBz4lISyAZOAl0y+p41O3JH+jDmO61aT3Yap8wvW8DnVPuDrz9rcpe11b3Skmyyjvbk77jm1dQpHAYGGPfUu2VP9M8T7v/UlXQG75n0uw3kJoCKYnOvNJ4YL2IzAXOXdppjHnOmX9otmazwWNDYUhDmNYNekWD99VF/YqG+tGtQUmGL9rF03eXomKhGxU/VkrdjIg8AvxmzHUllW/JGHPQ/nhURH7CWmOcNsE7iFVY6pJwrrSbSnsenf10O9exfwVMeB+CCuDV9RfqhWa8fcyCbcf4Zl4MFQoFM/Hpune0rjkn/H3khGuA7HUdWZLgGWOigWj787fT7L98l09lP2ULBjGkcyRdRq6g/8RVjO5eG6/c0j4hu/HwsqZpFigPlR5hW2o0RbLJP1I3MMO+qYwILACth1trSn5/GR4dfN2Qfo3LMHnFPj6dvZWR3Wq7IEilcoQOwFci8iMwyhizxZGDRCQAsBljztqfNwfev2bYLOAZEZmCVVwlTtffZZK9S2BiO6uFUddfIKRohk+xcNsxeo6LoWyBwDtO7pS6XfppXN2RBmXy81HraizecZw3f9qAvS2ZUk5ljBkLTAWWGWPGXtpcHVe2UPoeuOcVWDMR1k657u1Qf2/6NS7L/C1HWb7rRDonUErdin1JSi1gJzBGRJaKSC8RudXSlzBgsYisBVZg3QWcLSJ9RKSPfczvwC6s1jDDgfSbXKqM2b0QJrSB4CJWtczbSO4WbdfkTrkHTfDUHWsXVYxn7y3LDzH7+X7BTleHo3IB+/SnNcBs++ual1okKAc0egVKNIRfX4Tj2697u1uDkoQF+/Dx7C36pY1St8neO3g6MAUoDDwGrBKRZ29yzC5jTA37VsUY8z/7/iHGmCH258YY098YU8YYU80YE5MFl5Oz7fzLunMXWsJK7m7V5igdi7cf5+mxMZS2J3d5AjS5U66jCZ7KFC82K0/LGkUYOHsrv67719XhqJzvXax1KacBjDFrAG3M6CgPT2gzAjx9rPV4SReuetvP24MB95Vn9b7TzNl0xDUxKpWNiUhL+/q5aMALqGOMeRCoAfzHlbGpa2yfC5M6Qr6y0O1XCCyY4VMs3n6cp8aupFT+AE3ulFvQBE9lChFhYNvqRJXIw4tT1xK795SrQ1I5W1I6FTRvWsxAREaJyFER2ZBm37siclBE1ti3h9K897qI7BCRrSJyfybH73rBRayiK0c2wJz/u+7ttpHhlCkQwMDZW0hOyXCdCKVyuzbAl/Y7bJ8aY44CGGMSgKdcG5q6bMvvMOVxKFjRWnMXkD/Dp/hnx5XkblLPeuTV5E65AU3wVKbx9fJgWJcoioT40nNcDPtOJLg6JJVzbRSRxwEPESknIt8AS25xzBjggXT2f2mMqWnffgcQkcpAR6CK/ZjB9v57OUv55tDgWVg5Ajb9fNVbnh42XnmgIjuPnWN67AEXBahUtvUu1ho6AETET0RKAhhj5rsmJHWVTbNgamcoVA26zAL/vBk+xRJN7pSb0gRPZaq8Ad6M7l6HVGPoNmYFcQlJtz5IqYx7Fiv5ughMAuKAF252gL2P1EkHz98KmGKMuWiM2Y1VzKDO7Yfrxu59G4pGws/Pwqk9V73VvHIYEcVD+XLeNs4nprgmPqWyp2lcPasgxb5PuYMNP1rT04tEQOefwC80w6dYsuM4PcaupGQ+a1qmJnfKnWiCpzJdqfwBDOscxYGT5+k9IYbEZJ3epTKHiPiKyAvAQGAfUN8YU9sY86Yx5sItDr+RZ0RknX0KZx77vqJc3dX2gH1fzuPpDW1HWc+n94DkK/0LRYTXHqzEkTMXGb1kt4sCVCpb8jTGXP5lsj/XDMAdrP0BfnwaitWFzjPANyTDp1i68wQ9xq6kRF4rucsX6OOEQJW6fVne6FzlDnVK5WVg2+q88MMaXp+xns/aVXd1SCpnGAskAYuAB4FK3OLO3S18D/wXMPbHz4EeGTmBiPQCegGEhYURHR3t0HHx8fEOj80K+cv2oerGT9g3tie7ynS/6r2aBTz4Zt5WiifuJ9BbrnrP3a7jduSEawC9DjdzTERaGmNmAYhIK+C4i2NSqyfCz/2h5F3w+A/gHZDhUyzbdYIeY1ZSPK8/E3tqcqfckyZ4ymkerVWUvScS+HLeNkrk86d6zlvBpLJeZWNMNQARGUmaNS63wxhzuUSkiAwHfrW/PAgUSzM03L4vvXMMA4YBREVFmcYONpCPjo7G0bFZozH4n6D4yhEUv/txKH+lrkzhimd5YNBC1iaF8X/NK191lPtdR8blhGsAvQ430weYKCLfAoI1I6CLa0PK5WLHwC8vQOkbBwl7AAAgAElEQVTG0HESePtn+BTLdp2g++iVhOfxY1LPeuTX5E65KZ2iqZzquaZlaR1RlC/mbmPxQV2Pp+7Y5R8iY0zynZ5MRNI2O3oMuFRhcxbQUUR8RKQUUI47TCazheb/g7Bq8FMfiLuSz1YoFESbiHDGLt3LwdPnXRigUtmDMWanMaYeUBmoZIxpYIzZ4eq4cq0Vw+GX56FcM+g05baSu+Wa3KlsRBM85VQiwsetq9OwbD5Grk/UanzqTtUQkTP27SxQ/dJzETlzswNFZDKwFKggIgdE5ClgoIisF5F1QBNgAIAxZiMwFdiE1Uy9vzEm51cZ8fKFdqMh+aK1RiXlSg49oFl5AL6cu81V0SmVrYjIw0A/4EUReVtE3nZ1TLnS0sHw+0tQ4SHoMMH6dy6DVuw+SfcxKylqT+4KBGlyp9ybJnjK6bw9bYzoUpvK+Wy8PH0tU1bsc3VIKpsyxngYY4LtW5AxxjPN8+BbHNvJGFPYGONljAk3xow0xnS296mqboxpaYw5lGb8/4wxZYwxFYwxfzj/6txE/nLQ4gvYtwQWfHJ5d9FQP7o1KMmPqw6w5fBNc2mlcj0RGQJ0wKr4K0A7oIRLg8qN/hkEf74OlVpCu7HgmfHEbMXuk3QbvYLCIb5M6llXkzuVLWiCp7KEn7cHz0f4ck/5Arw2Yz3jl+11dUhKqRup0RFqPA4LP4VdCy7v7te4DIE+nnw6e6sLg1MqW2hgjOkCnDLGvAfUB8q7OKZcpfjeaTD3bajS2qoU7JnxIqYr91xJ7ib3qkfBoIzf/VPKFTTBU1nG20MY2jmS+yoV5K2ZGxj9j5ZdV8ptPfSpdTdvRk+IPwpAqL83/RqXZf6WoyzfdcLFASrl1i61bUkQkSJY64cL32S8ykwLPqX07glQvQO0Hg4eXhk+Rcyek3QbtYJCIb5M7qnJncpeNMFTWcrH04PBT0Ryf5Uw3vtlE8MX7nJ1SEqp9PgEQtvRcP40/NQbUq1+lt0alCQs2IePZ2/BGOPiIJVyW7+ISCjwKbAK2ANMcmlEucXqCfD3BxwOawKPfg8eGS8YH7v3JF1HrSAs2JcpPetRMFiTO5W9aIKnspy3p41vH4/g4WqF+d/vmxkcrYXFlHJLharCgx/Dzr/gn68Aa7r1gPvKs3rfaf7ceOQWJ1Aq9xERGzDfGHPaGPMj1tq7isYYLbLibLsWWNUySzdha4VnwJbx/kyxe0/RddRKwoLt0zI1uVPZkCZ4yiW8PGwM6liTVjWLMHD2VgbN2+7qkJRS6YnsDpUfhb8+gH3LAWgbGU6ZAgF8+ucWUlL1Lp5SaRljUoHv0ry+aIyJc2FIucPRLfBDZ8hXDtqPxdhu587dKbqOWkGBIB8m96pHmCZ3KpvSBE+5jKeHjS/a16RNRDhfztvG53O26pQvpdyNCLT8GkLCYXoPSDiJp4eNVx6oyM5j5/hph/a3VCod80WkjYiIqwPJFeKPwqR2VguEJ6aCb0iGT7Fqn5Xc5Q/0ZnJPTe5U9qYJnnIpD5vwadvqdKxdjG/+2sEnszXJU8rt+IZY/fHij8DPz4AxNK8cRqc6xfh1VxIzVx+89TmUyl16A9OAi4726lS3KTEBJneE+GNWE/PQ4hk+xep9p+g6cgX5Ar2Z3KsehUI0uVPZmyZ4yuVsNuHDx6rxZL3iDFmwkw9+26xJnlLupmgkNHsPtv4Gy4ciIrzXsioV8th45cd1rN53ytURKuU27L05bcYYb0d7darbkJpqFYE6uArajICiERk+xZr9p+kycgV5A72Z0qsehUP8nBCoUlkr4xOUlXICm034b6uqeNpsjFy8m+SUVN5tWQWd3aKUG6nXD3YvhDlvQvG6eBepxbO1fBm4BnqOi2XWMw0pEqofjpQSkUbp7TfGLMzqWHK0ee/A5llw/4dQqUWGD1+7/zSdRy4nT4A1LVOTO5VT6B085TZEhHceqUzPu0sxdule/m/mBlK1gINS7kPEKjseWBCmdYcLZwj0FkZ2jeJiUgpPj40hITHZ1VEq5Q5eTrO9BfwCvOvKgHKcmFGw5Guo3dP68imD1u4/zZMjl5PH37pzp19OqZxEEzzlVkSENx6qRN/GZZi0fB+vzVinVfqUcif+ea2pUKf3wq8vgDGUCwvi68drseXwGQb8sEa/mFG5njHmkTRbM6AqoPOYM8v2efDbS1CuOTzwsfXlUwasO2Ald6H+XkzW5E7lQJrgKbcjIrxyfwWea1qOqTEHeHnaWk3ylHInJRpAkzdgw4/UXPMG7FpAk/IFeOOhSvy58QhfzN3m6giVcjcHgEquDiJHOLwBpnWDsMrQdlSGG5mvPxDHkyOWE+LnxeSe9SiqyZ3KgXQNnnJLIsKLzcrjaRO+mLuN5FTDF+1r4Omh30ko5Rbu+g/4huI370MY1xKKN+Cpe15le2Q43/69g3JhgbSqWdTVUSrlEiLyDXDpm0kbUBNY5bqIcogzh2BSe/AJgsenWo8ZsOFgHE+MWEawnxdTetUjPI+/kwJVyrU0wVNu7bmm5fDysPHJ7C0kp6YyqGMtvDTJU8r1bDao05PlZ0vQKGgvLPoCGd+Kj4rVI7BIS16eLhTP60+t4nlcHalSrhCT5nkyMNkY84+rgskRLsZbyd2FOOj+BwQXydDhVnK3nCBf686dJncqJ9MET7m9vo3L4OUhfPDbZpJTVvHt4xF4e2qSp5Q7SPXwhjo9oVZnWD0e26IveOvsG7TyrsjQse35v2f6UkQ/SKncZzpwwRiTAiAiHiLib4xJcHFc2VNqCvz4NBzZAJ1+gMLVM3T4peQu0MeTKb3qUSyv/pukcjb9lKyyhafvLs27j1RmzqYj9J0Qy8XkFFeHpJRKy8vXSvSeXwMPfUYlvzi+S3mf09/dy4Wtc0F7W6rcZT6QdnGXHzDPRbFkf3++Adv+gAcHQvnmGTp0479xPDlSkzuVu2iCp7KNbg1L8cGjVZm/5Si9xsVyIUmTPKXcjqcP1OmJ14C1bI18l9Cko/hObosZ2Rx2zNdET+UWvsaY+Esv7M81s7gdy4bA8iFQr7/1JVIGbPzXunPn7+XB5J6a3KncQxM8la08Wa8En7SpxsLtx3h6bAznEzXJU8otefpQ4ZEBzL73d/4vqQdnj+6FCa1hZDPYMU8TPZXTnRORiEsvRCQSOO/CeLKnrX/An69DxRbQ/L8ZOnTfmZTLyd2UXvUpnk+TO5V7aIKnsp0OtYvzWdsaLNl5nO5jVnDuojZWVspddW9UgaRa3Yg6M5C1Nd+1quBNaGMlets10VM51gvANBFZJCKLgR+AZ1wcU/by7xqY3gMK14DWw8Dm4fChmw+dYeDKC/h5eTC5Vz1N7lSuowmeypbaRIbzZYearNh9km6jVxCvSZ5SbklE+ODRatQsGUb7mIqsbf03tPgKzh6GiW1gxH2wXdfoqZzFGLMSqAj0BfoAlYwxsa6NKhuJOwCTOoB/PquoineAw4fuPBbPkyOW4+0hTOlVjxL5HD9WqZxCEzyVbbWqWZRvOkWwat9pOo9czpkLSa4OSSmVDm9PG98/GUGBIB96TlzH4XKd4NlV8MggiD8KE9vCiKawbY4meipHEJH+QIAxZoMxZgMQKCL9XB1XtnDhDExsD0kJVq+7oDCHDz14+jydRyxHBF6p7avJncq1NMFT2drD1Qvz3eMRbDgYx5MjlhOXoEmeUu4oX6API7vW5tzFZHqOi+F8qgdEdoNnY+2J3jGY1A6G36uJnsoJehpjTl96YYw5BWSsQkhulJIM07rB8a3QfiyEVXb40GNnL/LkiOWcvZjMuB51KRSgH3FV7qU//Srbe6BqIb5/IpIth87SYdhSjpy54OqQlFLpqFAoiK871WLDv3G8NG0tqakGPL3TJHpfQ8Jxe6LXBLb9qYmeyq48REQuvRARD8DbhfG4P2Pg95dg53x4+Asoc6/Dh8YlJNFl1AoOx11gTPfaVC4S7MRAlXJ/muCpHOG+ymGM6lab/ScTaPP9EnYfP+fqkJRS6WhaKYzXH6zIb+sPMWj+9itveHpDZFdr6mbLbyDhBExqD8PugdUTIFF/p1W2Mhv4QUSaikhTYLJ9n0PsjdFXi8iv6bzXTUSOicga+/Z0JsbtOku+gdjRcNcA698CByUkJtN9zAp2Ho1nWJdIIkvkdWKQSmUPmuCpHOOucvmZ3KseCYkptBuyhA0H41wdklIqHT3vLk27yHAGzd/Or+v+vfpNDy+I6GJP9L6FpPPwc3/4vCL8OsCqrKeU+3sV+AuryEpfrMbnL2fg+OeBzTd5/wdjTE37NuL2w3QTm36GuW9Blcfg3rcdPuxicgq9x8eyZv9pvu5Uk7vLFXBikEplH5rgqRylengo0/rUx8fTg47DlrF05wlXh6SUuoaI8MFjValdMg//mbqWdQdOXz/IwwsiOkP/FdB9NlR4CNZMsu7oDW0EMaOsYgxKuSFjTKoxZogxpq0xpi2wCfjGkWNFJBx4GMj+iZsjDsTAjF4QXgce/R5sjn00TU5J5fnJa1i0/TiftKnOA1ULOzlQpbIPTfBUjlOmQCDT+9ancIgvXUev4M+Nh10dklLqGj6eHnz/ZCT5A33oOS6Gw3E3WDsrAiXqQ+uh8J8t8OCnkJpi3c37vIJ1d2//Sl2rp9yOiNQSkYEisgd4H9ji4KFfAa8AqTcZ00ZE1onIdBEpdoehus6pPTC5IwQVgk6TwcvPocNSUw2vzVjP7I2HebtFZdpFZd//BEo5g6erA1DKGQqH+DG1d316jF1J3wmxfNS6Gh1qF3d1WEqpNPIH+jCyWxRtBi+h1/gYfuhVHz/vmzQz9ssDdXtBnZ5wcBWsGgPrf7TW6BWsbBVrqd7eGqeUC4hIeaCTfTuO1eBcjDFNHDy+BXDUGBMrIo1vMOwXYLIx5qKI9AbGAtdVJBGRXkAvgLCwMKKjox26hvj4eIfH3gnPpHhqrX4V78TzrK78LgkrNzh0nDGGSVsSmbs3mcfKelE6eS/R0XuvG5dV1+FsOeE6csI1QPa6Dk3wVI6VJ8CbiU/Xpc+EVbz643pOnkuizz2lSVPYTCnlYhULBTOoYy16jo/h5elr+aZTrVv/jopAeKS13f8hrJ8Oq8bCH6/A3LehciuI6AolGlhjlco6W4BFQAtjzA4AERmQgeMbAi1F5CHAFwgWkQnGmCcvDTDGpF17MAIYmN6JjDHDgGEAUVFRpnHjxg4FEB0djaNjb1tyIkxsAxeOQOefqFPqbocP/XLuNubu3c5Td5XizYcr3fDfiyy5jiyQE64jJ1wDZK/r0CmaKkfz9/ZkRJcoWtYowiezt/Dh75ut0uxKKbdxX+UwXn2gIr+uO8TX83dk7GCfIIjqDr2iofciqPUkbP0DxjwE39a2KvOdO+6MsJVKT2vgEPC3iAy3V9B0+FsGY8zrxphwY0xJoCPwV9rkDkBE0i42a8nNi7G4H2OsKda7F1oVczOQ3I1YtItB87fTPir8psmdUrmdJngqx/P2tPFVh5p0a1CS4Yt28/L0dSSl3Gxpg1Iqq/VuVJrWEUX5ct42flt36PZOUrg6PPy5tVav1WDwzwtz3rQqcE7rBjv/hlT93VfOY4yZaYzpCFQE/gZeAAqKyPci0vx2zysi74tIS/vL50Rko4isBZ4Dut1p3Flq0eewZgLc8xrU7OTwYVNX7ueD3zbzULVCfNS6uiZ3St2E06do2pt7xgAHjTEtrnnPBxgHRAIngA7GmD3OjknlPjab8M4jlckb4M0Xc7dxOiGRbx+PuPl6H6VUlhERPmpdjX0nEvjPtDUUz+tPtfCQ2zuZdwDUesLajm6GVeNg7WTY+BPkKWm1Yaj5hFXYQSknMMacAyYBk0QkD9AOq3XCnAycIxqItj9/O83+14HXMzHcrLP5F/jrv1CtHTR+zeHDfl9/iNdmrKNR+QJ82aEmHjZN7pS6may4g3ezXi5PAaeMMWWBL4FPsiAelUuJCM81LccHj1blr61H6TJqOXHnk1wdllLKzsfTgyGdI8kXYFXWPHrmBpU1M6JgJXjgI3hxC7QeASHFYP778EVlmPIEbJsDJuXO/xylbsAYc8oYM8wY09TVsbjUoXVWO4SikVaPSwfvwC3Ydoznp6wmongehjwZgY+nfjGr1K04NcFzoJdLK6zqTwDTgaai99yVkz1ZrwTfdopgzf7TdBi6NHM+RCqlMkX+QB9GdI3izIUkeo6L4UJSJiVfXr5QvR10+9Vqot7gGdi/HCa1o8GS7jCxHcz/r9Vw+eRubbugVGaKPwqTO1kVbjtOsn4fHbByz0l6j4+hXMEgRnarjb+31gZUyhHOvoN3q14uRYH9AMaYZCAOyOfkmJTi4eqFGd2tDvtOJtBmyBL2HD/n6pCUk4nIKBE5KiIb0uzLKyJzRWS7/TGPfb+IyNcissPeayrCdZHnPpUKW5U11x2M4+Xp6zK/MFK+MtDsfRiwCdqP42TeCIg7CIu/hKld4Oua8HEJGP0wzH4d1kyGIxshRe/4K5VhyRetu+UJJ6zkzsGp0RsOxtFj9EqKhPgx7qk6hPh5OTlQpXIOp30V4mAvF0fP5da9XJxNr8N5Xor04suY87T8egH/ifKhRPCtp36443VkVE64htswBvgWa93vJa8B840xH4vIa/bXrwIPAuXsW13ge/ujyiLN7JU1P/5jC4nJKXzZoWbmf3vv6Q2VW7HlaAiFGjeGpAtwdBMcXmdNJzu8DmJGQ/J5a7yHD4RVhkLVrYIuhWpAWBXw9s/cuJTKKYyBX56HAyug3RgoUtOhw3Yei6frqBUE+3kx4em65A/0cW6cSuUwzrzXfcteLsBBoBhwQEQ8gRCsYitXceteLllAr8N5GgON6sfTZeRyPotNZnjXmtQrffObyO54HRmVE64ho4wxC0Wk5DW7W2H9GIA1XTwaK8FrBYwzxhhgmYiEikhhY8xtlndUt6N3o9L4eNr476+baDdkKSO71qZQiGNTu26Lly8UjbC2S1JT4MQOe8K31nrcPMvquwcgNshXzp7wVYdC1aBwDauCp1K53ZKvrQJHjV+HKo85dMjB0+fpPGI5IjD+qToUCfVzcpBK5TxOS/DSVnmy38F76dpeLsAsoCuwFGiL1e9FFz6oLFW2YCA/9mtA55Er6DJqBd92qkXzKlpdL5cIS5O0HQbC7M8vTx+3O2Dfd12CpzMMnHsdpYAXInwYvOYMD3zxF89H+FAqJHOLLDh2DQXA+z4ocR8UN/hcPE5g/C6Czu4iMH4Xgdv+xnf9tMujL/jkJz6wNPGBpTkbZD1e9Mnv1Mbr+jOl3MrW2TD3HSuxu+dVhw45dvYiT45YztmLyfzQqz6lCwQ6OUilcqYsX60qIu8DMcaYWcBIYLyI7ABOYjX1VCrLFQ7xY1rv+vQYu5I+E2L5uHV12tcu5uqwVBYyxhgRyfAXTDrDwPnX0Ri4/+6z9Bizkk9iLvJl+5o8WK3wrQ5zWKZdw7kT1rTOw+vwPbQO38PryL/3B8D+Y+Wfzz69s8aVLU8psGXOcnj9mVJu48gm+PEp6852q8EOfbERl5BEl1ErOBx3gQlP16FykeAsCFSpnClLEryb9HK5gNUbRimXyxPgzcSn69Jnwipe+XEdJxMS6XNPGVeHpZzryKWplyJSGDhq339p+vgl4fZ9ykUqFAri52ca0mtcDH0nruLl+yvQr3EZ92p2HJAPyjSxtksSz1kFWg6tvbIt/Q5S7QVbfIKvrOm7lPTlKwceWi1QZVPnTsDkjuAdCB0nO7RGNSExme5jVrDj6FlGdq1NZAmd4qzUndD/gyiVhr+3JyO6RPHStLV8/McWTp5L5PUHK7rXh0iVmS5NE//Y/vhzmv3PiMgUrOIqcbr+zvXyB/owqWc9Xv1xHZ/+uZWdx+L5qHU19+6L5R0AxepY2yXJiXBs89VJX9piLp5+UKiqlexduuNXsBJ4aqEJ5eaSE2FqZzh7GLr/ASFFb3nIxeQUeo+PZc3+03z3eASNyhfIgkCVytk0wVPqGt6eNr7qUJM8/l4MW7iLE/GJfNKmGp4ezu4qopxJRCZjzfbLLyIHgHewErupIvIUsBdobx/+O/AQsANIALpnecAqXb5eHnzVoSZlCwTy+dxt7D+ZwNDOUeQN8HZ1aI7z9L5yt+6SlGR7MZc0Sd+6qbDS3kbW5mUleWmnd4ZV1Qqeyn0YA7//B/b+A61HQHjkLQ9JTknl+clrWLT9OAPbVs/UqddK5Waa4CmVDptNeLdlFfIF+vDF3G3EnU/k28cj8PVy4zsF6qaMMZ1u8FbTdMYaoL9zI1K3S0R4tmk5ShUI4D9T19Lqu8WM6lqbcmFBrg7t9nl4QsGK1lajg7UvNRVO7ba3bbAnfVt/h9XjrffFBvnLX0748h1PgP3+4JfXquLpG5pp6/uUuqXlQ2DVOLj7P1D91qtvUlMNr81Yz+yNh3m7RWXaR+m6d6UyiyZ4St2AiPBc03LkCfDm7Z830GXkCoZ3jXJ1WEopuxbVixCex5+nx8bQevASvn0ignty0vQum81qyp6vzJUS88bAmYP2hM+e+O1eBOt+oBrAhg/SnEDAL9Qq7nIp6bv8mOfq12nH6FRQlVE75sGfb0DFFtDkzVsON8bw/q+bmB57gBfuK0ePu0plQZBK5R6a4Cl1C53rlSCPvxcDflhDh6FL6V0x1dUhKaXsahYL5ednGvL02Bh6jFnJO49Upkv9kq4Oy3lEICTc2io+fGV//FFi/5pJZKVSkHASzp+8/vHMv3B4g/U8KeHGf4ZXQDpJYD7reWAYhBSz1laFhINviPOvWbm3Y9tgWg8oWBkeG+rQXeOv5m1nzJI99GhYiueblsuCIJXKXTTBU8oBLaoXIdTPm97jY3hnSSrhFU9Su6RW+VLKHRQN9WN6n/o8P2UNb/+8kZ1H43mrReXctW42sCBng8tDucaOjU+6kH4SmHACEk5dve/0fuvx/Gkut3y4xCcYgoteSTpDitoTwHBrf3BRa82hypkSTsLkDuDhBZ0mg8+t+9ZNWLaXQfO30y4ynDcfrqRFzJRyAk3wlHLQXeXyM7N/Q54cuohOw5bxTssqPFm3uP7PSSk3EODjydDOkXwyewvDFu5i94kEvn28FsG+Xq4OzT15+YJXEQgu4vgxqSkQfwTiDkLcfog7YE0XjTtgvf53lZUgXkXsd/0uJYHFrkkIwyGggFMbwCsnSUmCad2sv/+uv0Bo8VsecvZCEgNnb6Fh2Xx81LoaNpv+vSvlDJrgKZUB5cKCeLu+HzMOBvLWzA2sP3Ca91tV1eIrSrkBD5vwxkOVKJ0/gDdnbqDN4CWM6labYnm10mSmsHlYCWFwEShWO/0xiQnWVNC4/Vcnf3EHrebX2+ZcaQdxiYePlQDaE79yx05B/Cwr6RMbIPYEUNLsI80+29Xv33Qf1vOKD1tVSdXtm/0a7F5gNTIvXs+hQ8Yt3cuZC8m8+kDF3HWHXakspgmeUhkU4CUM7xLFV/O28fVfO9h6JJ4hT0ZQOMTP1aEppYCOdYpTPJ8/fSesotV3/zC0c6ROqc4q3v6Qv6y1pccYOH/qyh3Aa7fdiyhw/iyc9rTGYsCk2meGGmufSb3y/Gb7rp1Omlaekprg3YkVw60WHg2ehVpPOHRI/MVkhi/aRZMKBageHurkAJXK3TTBU+o22GzCi80rUKVoCC/+sIZHvlnM4CciqVNKP0Qq5Q4alLGmVPcYs5Inhi/n4zbVaB0R7uqwlIi9amfeq/sAprEkOprGjRtnzp9nbpAI2vTjz23btQD+eBXK3Q/3vefwYeOX7uV0QhLPaVEVpZxO748rdQfur1KIn59pSLCvF48PX8a4pXuwWqgppVytVP4AfurXgMgSeXhx6lo++3Mrqan6+5mriFhVHW0eVq9BT2+rDYRNp9XflhM7YWoXyF8O2oxw+L9jQqJ1965R+QLUKp7HyUEqpTTBU+oOlS0YxMxnGnJP+QK8/fNGXpm+jgtJKa4OSykFhPp7M+6pOnSqU4xv/97BM5NXcT5Rfz+VyrDzp2FSB2stY6cp4Bvs8KETl+3j5LlEnm96g6m7SqlMpQmeUpkg2NeL4V2ieK5pOabFHqDD0KUcijt/6wOVUk7n5WHjw8eq8ebDlfhjw2E6DFvK0TMXXB2WUtlHSjJM7wGndkP7cZDX8cbk5xNTGLpwJ3eVzU9kCV3GoFRW0ARPqUxiswkvNivP0M6R7Dx2jke+WczyXdeWDFdKuYKI8PTdpRneOYodR+Np9d0/bDgY5+qwlMoe5r4FO+fDw59DqbszdOikFfs4Hp+oa++UykKa4CmVye6vUoiZ/RsQ7OfFEyOWM3aJrstTyl3cVzmM6X0aIEC7IUv5c+NhV4eklHuLHQvLBkPdPhDZLUOHXkhKYciCndQrnVeLkCmVhTTBU8oJyhYMYmb/hjSuUIB3Zm3kZV2Xp5TbqFwkmJnPNKR8oSD6TIjl152JJKekujospdzPnn/gt/9AmXuh+f8yfPiUFfs4dvYizzct74TglFI3ogmeUk4S7OvFsM5RPN+0HNNjD9B+6FL+Pa3r8pRyBwWDfPmhVz1aVC/C9O1J3P/VQuZuOqJ325W65NQemNoZ8pSAtqOtKqQZcCEphe8X7KROybzUK61375TKSprgKeVENpswoFl5hneJYpd9Xd4yXZenlFvw9fLg6441ea6WDwboOS6GDsOWsWb/aVeHppRrXTgDkzpCajJ0+gH8Mt6YfFrsAY6cuchzTcshIk4IUil1I5rgKZUFmlUOY2b/hoT4W+vyRv+zW+8UKOUGRISIME/+fKERHzxalV3H4nn0u394ZtIq9p1IcHV4SmU9kwIzesLxbdBuDOTPeGuDi8kpfP/3DiJL5KFh2XyZH6NS6qY0wVMqi5QtGMjM/g1pUqEg7/2yif9MW6vr8pRyE14eNp6sV4Lol5vwXNNyzN98lKZfRPP+L5s4dS7R1bVil8kAABhfSURBVOEplWVK75oA22bDAx9ba+9uw4+xB/k37oLevVPKRTTBUyoLWevyIhlwX3lmrDpIuyFLOajr8pRyG4E+nrzYrDzRLzemTUQ4Y5bsptGnfzN0wU79QkblfGsmU3z/DIjqAXV63tYpklJS+e7vHdQoFkqjcvkzOUCllCM0wVMqi9lswvP3lWNElyj2HD9Hy28Ws3SnrstTyp2EBfvycZvqzH6hEVEl8vDRH1to+vmC/2/vzsOrKs/1j3+f7MwhCUMGyESAQGIYxYgMVsIgFbVQtVa0tdpJj1pLLUeLPbVFO2htfw6oZ6Baf1ZxQIqCU0WBCIqgyDwTkCFhSJgJKgJ5zx/ZciKFCoGw1l65P9fFtfdee2Xnfgn7Cc9+13oXLy2ooLZWh1dLQJmxs8XZMPR+aODM26T5FVTu/pSfafZOxDNq8EQ8Mrg4k5d/0o/miTF894m5/PVdnZcn4jedMpN58vu9ePZH59EiKYbbXljENx59l/fKt3sdTeT06z6Cxd1+A6GYBn35wcO1PDqjnK7ZqZQWpp/mcCJyotTgiXioQ/r/nZd3z6vLGTVB5+WJ+FHfgjSm3HI+D4/owe5PDvKdx+dy/ZMfsGrrPq+jiZxepzDrNnnhZjbt/FTn3ol4TA2eiMeS65+Xt6CSb/33bNZV13gdS0SOEhVlDO+RzbRR/fnlxUXM37CLoQ/P5I6Ji9i65zOv44l46tDhWh6dvobiNikMPivD6zgiTZoaPBEf+OK8vCeuK2HTzk8Z+vAsnnj3Y53rI+JD8TEhbrigAzPvGMAP+rXj5QWbKf3zDP7f1FXUHDjkdTwJADMLmdkCM3v1GM/FmdkLZlZuZnPNLP/MJ/xnryzezPodn2j2TsQH1OCJ+MigszKZetsF9CtI47evLmfEuDls2LHf61gicgzNE2P51aXFTBvVnwuLW/PI9HL63z+Dp99fz8HDtV7Hk8g2ElhxnOd+COxyzhUADwJ/PGOpjuNwreOR6eUUtU5mSHGm13FEmjw1eCI+k5kSzxPXlfCnb3VjxZa9XPTQLJ5+f71m80R8KrdlIo9cfTaTb+lHQUYz7pq8jK8/OJM3l23Vwkly0swsB7gEePw4uwwHngrfnwgMMo+nzF5bsoV11fu5dWBHoqI0eyfiNTV4Ij5kZlxZksubt11ASX4L7pq8jGv/OpeKXZ94HU1EjqN7bnOev6E3j3+vhKgo48anP+Lb//M+8zfu8jqaRJaHgDuA400DZwObAJxzh4A9QKszE+2f1dY6Hpm2ho4ZzRjapbVXMUSknmivA4jI8WU1T+BvP+jFcx9s4vevLeeih2bxq0vO4qpzc3WOg4gPmRmDizMpLUxnwrwKHnhrNZf/52wuLM7k+r759O3QSu9dOS4zuxSocs59ZGalp/haNwA3AGRmZlJWVnZCX1dTU3PC+wJ8sPUQa6oO8G/d45g5850GJG0cJzsOvwrCOIIwBoiscajBE/E5M+Oa8/L4Wsc07pi4mNGTlvDG0q3cd0VX2qQmeB1PRI4hOhTFNeflMbxHFn+ZtY6nZq/nreXbaJ+WxDXn5XHlObmkJjbsWmMSaP2AYWZ2MRAPpJjZM86579bbpxLIBSrMLBpIBXYc/ULOuXHAOICSkhJXWlp6QgHKyso40X1rax33jZ1Fh/Robr+qPyEfHZ55MuPwsyCMIwhjgMgahw7RFIkQuS0TGf+j87h7WGc++HgnQx6cyd8/qtA5PiI+lhQXzc8Gd+L9OwfxwLe70zwxht+9toLz7n2b219cxKJNu72OKD7inLvTOZfjnMsHRgDTj2ruAKYA14Xvfyu8jye/CKYu38rKrfu4dWBHXzV3Ik2dZvBEIkhUlHFd33z6d0rn9omLGPXiIt5YuoU/XN6VjOR4r+OJyHHEx4S4vGcOl/fMYfnmvTwzdwMvL6jkxY8q6Jqdynd75zGsezYJsSGvo4oPmdk9wDzn3BTgCeBpMysHdlLXCJ5xzjkenlZOu7QkLu3WxosIInIcmsETiUD5aUk8f0MffnXJWcxas50hD85k8sJKzeaJRIDirBT+cFlX5v5yEPcM78yBQ4f5xd+X0OsPbzNmyjLKq2q8jig+4Jwrc85dGr7/63Bzh3PuM+fclc65AudcL+fcOi/yvb2iihVb9nLLgAKiQ/rvpIif6B0pEqFCUcaPvtae10d+jfxWSYx8fiE3j5/P9poDXkcTkROQHB/D9/rk8+bPLmDCjX0YUJjB+LkbGPzAO1w9bg6vLd6i6+mJL9XN3q0mr2Ui3+yR5XUcETmKDtEUiXAd0psx8d/68JdZH/PgW6v54OOZ/O6bXRjaVYfMiEQCM6NXu5b0ateS7TXFTJi3iWfnbuSWZ+eTnhzHiHNzubpXHlnNtaiS+MOMVVUsrdzL/Vd00+ydiA/pXSkSANGhKG4q7cCrPz2frOYJ3DR+Pj99bgG79n/udbSIYGbrzWyJmS00s3nhbS3N7C0zWxO+beF1Tgm+tGZx3FxawDu3D+Cv15fQNTuVR2eUc/4fp/Pjv83jndXV1NbqUGzxzhfn3uW0SOCyntlexxGRY9AMnkiAdMpMZtLNffnvsrWMnb6G99ft4N7LujK4ONPraJFggHNue73Ho4Fpzrn7zGx0+PEvvIkmTU0oyhhYlMnAokw27fyE5z7YyAsfbuKt5dto2yqRa3rlcWVJLi2TYr2OKk3MO6urWbRpN/de3pUYzd6J+JLemSIBExOK4tZBHZl8y/mkNYvjR3+bx6gJi9jz6UGvo0Wa4cBT4ftPAd/0MIs0YbktE7njoiJm3zmQh0f0IDM5nnvfWEnve6fx8xcW8tGGXVpgSc6Iutm7NWSlxnNFzxyv44jIcWgGTySgirNSmHxLPx6dvobHytbyXvl27ruiK6WFGV5H8yMHTDUzB/xP+ALBmc65LeHntwKaBhVPxUWHGN4jm+E9slm1dR/j525g0vxKJi2o5Kw2KfRIPUj+9v3kpyV5HVUC6r3yHSzYuJvffrMLsdGaIxDxKzV4IgEWGx3Fz4cUMrg4k1ETFnH9kx8y4txc+qfq0/6jnO+cqzSzDOAtM1tZ/0nnnAs3f//EzG4AbgDIzMykrKzshL5hTU3NCe/rZ0EYR6SOYWAq9P1aLHO2RDF9Yw3PbanluZVltE40uqWH6J4eTWHLKKIj7ALUkfrzCLovVs5snRLPt0s0eyfiZ2rwRJqAbjnNeeXW83no7TWMm7mWqXEGmVu4qEtrzCLrP3+NwTlXGb6tMrOXgF7ANjNr45zbYmZtgKrjfO04YBxASUmJKy0tPaHvWVZWxonu62dBGEekj+EiYAww4fXpfJLajhmrqilbt4OpGz4jKTZEv4I0BhZlMKAog8yUeI/TfrVI/3kE1fvrdvDh+l3cPawzcdEhr+OIyL+gBk+kiYiPCTF6aBFDOmcy8uk53DR+Pn07tGLMsM50ykz2Op5nzCwJiHLO7QvfHwLcA0wBrgPuC99O9i6lyFfLSIyitF87ru/Xjk8+P8Ts8h1MX1XFjJVVTF2+DYDOWSkMKKxr9nrkNicUYbN74p2x09aQkRzHVefmeh1FRL5CozV4ZhYPzATiwt9nonPuN0ftcz3wJ6AyvOlR59zjjZVJRKBnXgvG9Ilnc0I7/jx1NUMfnsW1vdty24WdSE2I8TqeFzKBl8IzmdHAs865f5jZh8AEM/shsAH4tocZRU5KYmw0g4szGVyciXOOVdv2MX1lXbP3n2XlPDqjnJZJsfTvlE5pYTr9O6XTPFErcsqxzV23gznrdnLXpcXEx2j2TsTvGnMG7wAw0DlXY2YxwLtm9oZzbs5R+73gnPtJI+YQkaOEooxr++Rzabcs/jx1FU+9v54pizZzx9cLubIkt0l9qu+cWwd0P8b2HcCgM59I5PQyM4pap1DUOoWbSwvY88lB3llTzYyVVZStquKlBZVEGZzTtgWlhRkMLMqgqHWyDt+WI8ZOX0Nasziu6ZXndRQROQGN1uC5ujWba8IPY8J/tLKDiI+0SIrl95d15epeedz9yjJGT1rC+LkbGTOsM+e01XW9RYIoNTGGYd2zGNY9i8O1jkUVu5mxsorpK6v405ur+NObq8hKjae0KIMBhRn0K2hFYqzO6Giq5q3fyXvlO/iPi88iIVazdyKRoFErtpmFgI+AAuAx59zcY+x2hZldAKwGbnPObWrMTCLyz7pkpzLhxj5MWbSZP7y+giv+azaX98xm9EVFZETAogwi0jChKKNnXgt65rVg1JBCtu39jLJVdc3e5AWVPDt3I7HRUfRu34oBhen06dCKjhnJTWqWv6kbO72cVkmxfKe3Zu9EIkWjNnjOucNADzNrTt05Ll2cc0vr7fIK8Jxz7oCZ3UjdxYQHHv06WoZc4/CTIIzjeGNIBe7uFeLVdTFMWVDJ64sqGVYQw5C2MRG31LqInLzMlHiuOjePq87N48Chw8xbv6vu3L1VVdz9ynIAEmJCdMlOoVtOc7rlpNItpzn5rRJ1SGcALdi4i5mrqxk9tEizuCIR5Iy8W51zu81sBnWrOS+tt31Hvd0eB+4/ztdrGXKNwzeCMI6vGsNFwPrt+/nda8uZsKKKeTti+fU3inWRdJEmJC667hIL/QrSuOvSYjbs2M/8jbtYtGkPiyt288ycDRw4VAtASnz0lxq+7rmptE6JV9MX4cZOW0OLxBiu7d3W6ygichIacxXNdOBguLlLAC4E/njUPm2cc1vCD4cBKxorj4icnPy0JB6/7lxmrKrit68s5/onP2TwWRn86pJi8tOSvI4nImdY21ZJtG2VxGVn113k+uDhWlZv28eSij0sqqhr+sbNXMeh2rrT7dOT4+iek0rX7OZ0y02le05zWiZppc5IsbhiNzNWVXP71wtJitPsnUgkacx3bBvgqfB5eFHABOfcq2Z2DzDPOTcF+KmZDQMOATuB6xsxj4g0wIDCDPp1SOPJ9z5m7LQ1DHlwJj++oB03lxbol75IExYTiqJzViqds1IZ0atu22cHD7N8y14Wb9rN4oo9LKrYzbSVVbjwEms5LRLoHp7p65qTStfsVJLjm+TlWXxv7LQ1pCbE8L0+mr0TiTSNuYrmYuDsY2z/db37dwJ3NlYGETk9YqOjuLF/By47O5v73ljJYzPW8vePKrnz4iKGdc/SYVgiAkB8TOjIoi1f2PfZQZZW7mVxxf81fa8tqTt4xwzapyUdafq65Tbns0NacNtrSyv38PaKKn5+YSc14CIRSB+/i8gJy0iJ54GrevCd3nmMmbKckc8v5Jk5GxgzrDOds1K9jiciPpQcH0OfDq3o06HVkW07ag6wuHIPS8KHds4q386kBZVHnm/9wTQ6ZCTRPq0ZHdKT6JDRjPbpzWiTEk+UFnxqdI9MX0NyfDTX9c33OoqINIAaPBE5aee0bcnLt/TjxXmbuP/NVXzjkXe55rw8Rl1YSAudYyMiX6FVszgGFNZdZw/AOcfWvZ+xuGIPb81ZTG1yK9ZW7+flBZXsO3DoyNclxIRon55E+/S6xu/IbVozXaPtNNm0r5Y3l21j5KCOpCZo9k4kEqnBE5EGCUUZI3rlMbRLGx58ezVPz9nAK4u28O9DOnF1rzyiQ1FeRxSRCGFmtElNoE1qAnHVKykt7QHUNX7VNQdYW7Wfddtrjtwu3LSLVxdvPnJuH0B28wTapyfRIdz0dUivm/XLTInTYeQnYXL55zSLi+YH/dp5HUVEGkgNnoicktTEGMYM68zVvfIYM2UZd01exvi5GxkzrDO927f66hcQETkOMyMjOZ6M5PgvHeIJdQu6rN+xv67pq65hbXUNa6v38+K8Tez//PCR/ZJiQ3WHeKaFm7+MZpS0bUFGSvyZHo7vrdq6j3nbDvOTAQWkJmr2TiRSqcETkdOisHUyz/74PN5YupXfv7aCH/9tHrNHD9QJ+iLSKOJjQhS1TqGodcqXtjvn2Lb3wJeavrXVNXy4fhcvL9wMwMMjejC8R7YXsX3t/89eT3wIfni+Zu9EIpkaPBE5bcyMi7u2YUBhBsu37FVzJyJnnJnROjWe1qnx9C1I+9Jzn3x+iI+37ycrNcGjdP72m28U08GqdC61SIRTgycip11CbIhz2rb46h1FRM6gxNhorfj7L8THhChoocVqRCKdVkEQEREREREJCDV4IiIiIiIiAaEGT0REREREJCDU4ImIiIiIiASEGjwREREREZGAUIMnIiIiIiISEGrwREREREREAkINnoiIiIiISECowRMREREREQkINXgiIiIiIiIBYc45rzOcFDOrBjac4O5pwPZGjHOmaBz+EoRx+G0MbZ1z6V6HOFWqTxErCGMAjaOxRHx9Um2KaEEYRxDGAP4bx3FrU8Q1eCfDzOY550q8znGqNA5/CcI4gjCGSBeUn0EQxhGEMYDGIadHUP7+NQ7/CMIYILLGoUM0RUREREREAkINnoiIiIiISEAEvcEb53WA00Tj8JcgjCMIY4h0QfkZBGEcQRgDaBxyegTl71/j8I8gjAEiaByBPgdPRERERESkKQn6DJ6IiIiIiEiTEdgGz8wuMrNVZlZuZqO9ztMQZpZrZjPMbLmZLTOzkV5naigzC5nZAjN71essDWVmzc1sopmtNLMVZtbH60wNYWa3hf89LTWz58ws3utMTYlqk/+oPvmDapP3VJ/8RbXJPyKtPgWywTOzEPAYMBQoBq42s2JvUzXIIWCUc64Y6A3cEqHjABgJrPA6xCl6GPiHc64I6E4EjsfMsoGfAiXOuS5ACBjhbaqmQ7XJt1SfPKba5D3VJ19SbfKBSKxPgWzwgF5AuXNunXPuc+B5YLjHmU6ac26Lc25++P4+6t4U2d6mOnlmlgNcAjzudZaGMrNU4ALgCQDn3OfOud3epmqwaCDBzKKBRGCzx3maEtUmn1F98hXVJm+pPvmIapPvRFR9CmqDlw1sqve4ggh8c9dnZvnA2cBcb5M0yEPAHUCt10FOQTugGngyfLjE42aW5HWok+WcqwT+DGwEtgB7nHNTvU3VpKg2+Y/qkw+oNvmC6pO/qDb5RCTWp6A2eIFiZs2AvwM/c87t9TrPyTCzS4Eq59xHXmc5RdFAT+C/nHNnA/uBiDs/wcxaUPeJbDsgC0gys+96m0oiVSTXJlB98hPVJjndIrk+qTb5SyTWp6A2eJVAbr3HOeFtEcfMYqgrUOOdc5O8ztMA/YBhZraeusM9BprZM95GapAKoMI598WngBOpK1qRZjDwsXOu2jl3EJgE9PU4U1Oi2uQvqk/+odrkPdUn/1Bt8peIq09BbfA+BDqaWTszi6XuRMgpHmc6aWZm1B23vMI594DXeRrCOXency7HOZdP3c9hunPO1596HItzbiuwycwKw5sGAcs9jNRQG4HeZpYY/vc1iAg84TmCqTb5iOqTr6g2eU/1ySdUm3wn4upTtNcBGoNz7pCZ/QR4k7qVbv7qnFvmcayG6AdcCywxs4Xhbb90zr3uYaam7FZgfPgX3zrg+x7nOWnOublmNhGYT91KYwuAcd6majpUm6QRRXR9Um3ynuqTNJKIrk0QmfXJnHNeZxAREREREZHTIKiHaIqIiIiIiDQ5avBEREREREQCQg2eiIiIiIhIQKjBExERERERCQg1eCIiIiIiIgGhBk8anZkdNrOF9f6MPo2vnW9mS0/X64lI06L6JCJ+pNokpyKQ18ET3/nUOdfD6xAiIseg+iQifqTaJA2mGTzxjJmtN7P7zWyJmX1gZgXh7flmNt3MFpvZNDPLC2/PNLOXzGxR+E/f8EuFzOwvZrbMzKaaWYJngxKRQFB9EhE/Um2SE6EGT86EhKMOM7iq3nN7nHNdgUeBh8LbHgGecs51A8YDY8PbxwLvOOe6Az2BZeHtHYHHnHOdgd3AFY08HhEJDtUnEfEj1SZpMHPOeZ1BAs7MapxzzY6xfT0w0Dm3zsxigK3OuVZmth1o45w7GN6+xTmXZmbVQI5z7kC918gH3nLOdQw//gUQ45z7XeOPTEQineqTiPiRapOcCs3gidfcce6fjAP17h9G55aKyOmh+iQifqTaJP+SGjzx2lX1bt8P358NjAjf/w4wK3x/GnATgJmFzCz1TIUUkSZJ9UlE/Ei1Sf4ldetyJiSY2cJ6j//hnPtiud8WZraYuk+Srg5vuxV40sxuB6qB74e3jwTGmdkPqfu06SZgS6OnF5EgU30SET9SbZIG0zl44pnwceQlzrntXmcREalP9UlE/Ei1SU6EDtEUEREREREJCM3giYiIiIiIBIRm8ERERERERAJCDZ6IiIiIiEhAqMETEREREREJCDV4IiIiIiIiAaEGT0REREREJCDU4ImIiIiIiATE/wLHsHmQzN04pAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1080x360 with 3 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "_, axes = plt.subplots(1, 3, figsize=(15, 5))\n",
        "\n",
        "axes[0].plot(history['loss'], label='train')\n",
        "axes[0].plot(history['val_loss'], label='valid')\n",
        "axes[0].set_title('Loss history')\n",
        "axes[0].set_xlabel('Epoch')\n",
        "axes[0].set_ylabel('Loss')\n",
        "axes[0].grid(True)\n",
        "axes[0].legend()\n",
        "\n",
        "axes[1].plot(history['ppl'], label='train')\n",
        "axes[1].plot(history['val_ppl'], label='valid')\n",
        "axes[1].set_title('Perplexity history')\n",
        "axes[1].set_xlabel('Epoch')\n",
        "axes[1].set_ylabel('Perplexity')\n",
        "axes[1].grid(True)\n",
        "axes[1].legend()\n",
        "\n",
        "axes[2].plot(history['acc'], label='train')\n",
        "axes[2].plot(history['val_acc'], label='valid')\n",
        "axes[2].set_title('Top-5 Accuracy & BLEU-4 history')\n",
        "axes[2].set_xlabel('Epoch')\n",
        "axes[2].set_ylabel('Accuracy & BLEU-4 (%)')\n",
        "axes[2].grid(True)\n",
        "axes[2].legend()\n",
        "\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "YIIeF2cXCx-2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d6c22626-a54c-4b86-9613-83e88cc3c8da"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[321.7141894507325,\n",
              " 192.0432581547501,\n",
              " 137.58476550001484,\n",
              " 102.34762377822521,\n",
              " 77.63101671812713,\n",
              " 61.514573072040015,\n",
              " 49.4674457332398,\n",
              " 40.9100834941381,\n",
              " 34.85302525867917,\n",
              " 30.3148655286364]"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "history['ppl']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UIqN5273qP0L"
      },
      "source": [
        "# Evaluation "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Beam search & BLEU score"
      ],
      "metadata": {
        "id": "dSPH8vZYkb5O"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "PPgXr4ii_MD9"
      },
      "outputs": [],
      "source": [
        "#find the best path + defining Node\n",
        "def find_path(tree):\n",
        "    path = []\n",
        "    for nodes in reversed(tree):\n",
        "        if len(path) == 0:\n",
        "            path.append(nodes[0])\n",
        "        else:\n",
        "            parent_id = path[-1].parent_id\n",
        "            for node in nodes:\n",
        "                if node.id == parent_id:\n",
        "                    path.append(node)\n",
        "    return path\n",
        "\n",
        "def find_best_path(tree):\n",
        "    best = []\n",
        "    for nodes in reversed(tree):\n",
        "        if len(best) == 0:\n",
        "            best.append(nodes[0])\n",
        "        else:\n",
        "            nodes_eos = []\n",
        "            parent_id = best[-1].parent_id\n",
        "            for node in nodes:\n",
        "                if node.eos:\n",
        "                    nodes_eos.append(node)\n",
        "                if node.id == parent_id:\n",
        "                    best.append(node)\n",
        "            if len(nodes_eos) > 0:\n",
        "                candidates = sorted([best[-1], *nodes_eos],\n",
        "                                    key=lambda node: node.logps,\n",
        "                                    reverse=True)\n",
        "                candidate = candidates[0]\n",
        "                if candidate.eos:\n",
        "                    best = [candidate]\n",
        "    return best\n",
        "\n",
        "class Node:\n",
        "    id_ = 0\n",
        "    \n",
        "    def __init__(self, token, states, logp=0., parent=None, eos=False):\n",
        "        self.__id = self.__class__.id_\n",
        "        self.__token = token\n",
        "        self.__states = states\n",
        "        self.__logp = logp\n",
        "        self.__parent_id = None if parent is None else parent.id\n",
        "        self.__eos = eos\n",
        "        self.__level = 0 if parent is None else parent.level + 1\n",
        "        self.__logps = logp if parent is None else parent.logps + logp\n",
        "        self.__class__.id_ += 1\n",
        "        \n",
        "    def __str__(self):\n",
        "        return f'Node[id={self.__id}, ' + \\\n",
        "                    f'index={EN.vocab.itos[self.__token.cpu().item()]}, ' + \\\n",
        "                    f'logp={self.__logp}, ' + \\\n",
        "                    f'logps={self.__logps}, ' + \\\n",
        "                    f'parent_id={self.__parent_id}, ' + \\\n",
        "                    f'level={self.__level}]'\n",
        "    \n",
        "    @property\n",
        "    def token(self):\n",
        "        return self.__token\n",
        "    \n",
        "    @token.setter\n",
        "    def token(self, token):\n",
        "        self.__token = token\n",
        "    \n",
        "    @property\n",
        "    def parent_id(self):\n",
        "        return self.__parent_id\n",
        "    \n",
        "    @parent_id.setter\n",
        "    def parent_id(self, parent_id):\n",
        "        self.__parent_id = parent_id\n",
        "        \n",
        "    @property\n",
        "    def id(self):\n",
        "        return self.__id\n",
        "    \n",
        "    @id.setter\n",
        "    def id(self, id_):\n",
        "        self.__id = id_\n",
        "    \n",
        "    @property\n",
        "    def token(self):\n",
        "        return self.__token\n",
        "    \n",
        "    @token.setter\n",
        "    def token(self, token):\n",
        "        self.__token = token\n",
        "    \n",
        "    @property\n",
        "    def states(self):\n",
        "        return self.__states\n",
        "    \n",
        "    @states.setter\n",
        "    def states(self, states):\n",
        "        self.__states = states\n",
        "      \n",
        "    @property\n",
        "    def eos(self):\n",
        "        return self.__eos\n",
        "    \n",
        "    @eos.setter\n",
        "    def eos(self, eos):\n",
        "        self.__eos = eos\n",
        "    \n",
        "    @property\n",
        "    def logps(self):\n",
        "        return self.__logps\n",
        "    \n",
        "    @logps.setter\n",
        "    def logps(self, logps):\n",
        "        self.__logps = logps\n",
        "        \n",
        "    @property\n",
        "    def level(self):\n",
        "        return self.__level\n",
        "    \n",
        "    @level.setter\n",
        "    def level(self, level):\n",
        "        self.__level = level"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "axn3AumMmkvE"
      },
      "outputs": [],
      "source": [
        "def evaluate(model, data, beam_size, French_field, dest_field, max_len, device):\n",
        "    French_sentences = [*map(lambda example: example.French, data.examples)]\n",
        "    dest_sentences = [*map(lambda example: example.English, data.examples)]\n",
        "    data = [*zip([*map(lambda word_list: French_field.process([word_list]), French_sentences)],\n",
        "                 [*map(lambda word_list: dest_field.process([word_list]), dest_sentences)])]\n",
        "    references, hypotheses, sources = [], [], []\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for i, ((French_sequence, French_length), (dest_sequence, dest_length)) in tqdm.tqdm(enumerate(data), total=len(data)):\n",
        "            French_sequence, French_length = French_sequence.to(device), French_length.to(device)\n",
        "            dest_sequence, dest_length = dest_sequence.to(device), dest_length.to(device)\n",
        "            \n",
        "            # Encoding\n",
        "            h_state = model.encoder(input_sequences=French_sequence, sequence_lengths=French_length) # Encode\n",
        "            h_state = model.init_h0(h_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]\n",
        "            h_state = h_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]\n",
        "\n",
        "            # Decoding\n",
        "            tree = [[Node(token=torch.LongTensor([dest_field.vocab.stoi[dest_field.init_token]]).to(device), states=(h_state,))]]\n",
        "            for _ in range(max_len):\n",
        "                next_nodes = []\n",
        "                for node in tree[-1]:\n",
        "                    if node.eos: # Skip eos token\n",
        "                        continue\n",
        "                    logit, h_state = model.decoder(input_word_index=node.token, h_state_prev=node.states[0].contiguous()) # Decode\n",
        "                    # logit: [1, vocab_size]\n",
        "                    # h_state: [n_layers, 1, hidden_size]\n",
        "                    # c_state: [n_layers, 1, hidden_size]\n",
        "\n",
        "                    logp = F.log_softmax(logit, dim=1).squeeze(dim=0) # [vocab_size] Get scores                    \n",
        "                    topk_logps, topk_tokens = torch.topk(logp, beam_size) # Get top k tokens & logps\n",
        "                    for k in range(beam_size):\n",
        "                        next_nodes.append(Node(token=topk_tokens[k, None], states=(h_state,),\n",
        "                                               logp=topk_logps[k, None].cpu().item(), parent=node,\n",
        "                                               eos=topk_tokens[k].cpu().item() == dest_field.vocab[dest_field.eos_token]))\n",
        "                if len(next_nodes) == 0:\n",
        "                    break\n",
        "                next_nodes = sorted(next_nodes, key=lambda node: node.logps, reverse=True) # Sort next_nodes to get the best\n",
        "                tree.append(next_nodes[:beam_size]) # Update the tree\n",
        "                \n",
        "            best_path = find_best_path(tree) # Find the best path of the tree\n",
        "\n",
        "            # Get the translation\n",
        "            pred_translated = [*map(lambda node: dest_field.vocab.itos[node.token], best_path)]\n",
        "            pred_translated = [*filter(lambda word: word not in [dest_field.init_token, dest_field.eos_token], pred_translated[::-1])]\n",
        "\n",
        "            hypotheses.append(pred_translated) # Update hypotheses\n",
        "\n",
        "            # Update references\n",
        "            references.append([[dest_field.vocab.itos[indice] for indice in dest_sequence if indice not in (\n",
        "                dest_field.vocab.stoi[dest_field.init_token],\n",
        "                dest_field.vocab.stoi[dest_field.eos_token],\n",
        "                dest_field.vocab.stoi[dest_field.pad_token]\n",
        "            )]])\n",
        "\n",
        "            # Update sources\n",
        "            sources.append([French_field.vocab.itos[indice]  for indice in French_sequence  if indice not in (\n",
        "                French_field.vocab.stoi[French_field.init_token],\n",
        "                French_field.vocab.stoi[French_field.eos_token],\n",
        "                French_field.vocab.stoi[French_field.pad_token]\n",
        "            )])\n",
        "    \n",
        "        assert len(hypotheses) == len(references) == len(sources)\n",
        "        bleu4 = bleu_score(hypotheses, references, max_n=4, weights=[0.25, 0.25, 0.25, 0.25]) # Calculate BLEU-4 score\n",
        "    return hypotheses, references, sources, bleu4"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(model, data, beam_size, src_field, dest_field, max_len, device):\n",
        "    src_sentences = [*map(lambda example: example.src, data.examples)]\n",
        "    dest_sentences = [*map(lambda example: example.trg, data.examples)]\n",
        "    data = [*zip([*map(lambda word_list: src_field.process([word_list]), src_sentences)],\n",
        "                 [*map(lambda word_list: dest_field.process([word_list]), dest_sentences)])]\n",
        "    references, hypotheses, sources = [], [], []\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for i, ((src_sequence, src_length), (dest_sequence, dest_length)) in tqdm.tqdm(enumerate(data), total=len(data)):\n",
        "            src_sequence, src_length = src_sequence.to(device), src_length.to(device)\n",
        "            dest_sequence, dest_length = dest_sequence.to(device), dest_length.to(device)\n",
        "            \n",
        "            # Encoding\n",
        "            h_state = model.encoder(input_sequences=src_sequence, sequence_lengths=src_length) # Encode\n",
        "            h_state = model.init_h0(h_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]\n",
        "            h_state = h_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]\n",
        "\n",
        "            # Decoding\n",
        "            tree = [[Node(token=torch.LongTensor([dest_field.vocab.stoi[dest_field.init_token]]).to(device), states=(h_state,))]]\n",
        "            for _ in range(max_len):\n",
        "                next_nodes = []\n",
        "                for node in tree[-1]:\n",
        "                    if node.eos: # Skip eos token\n",
        "                        continue\n",
        "                    logit, h_state = model.decoder(input_word_index=node.token, h_state_prev=node.states[0].contiguous()) # Decode\n",
        "                    # logit: [1, vocab_size]\n",
        "                    # h_state: [n_layers, 1, hidden_size]\n",
        "                    # c_state: [n_layers, 1, hidden_size]\n",
        "\n",
        "                    logp = F.log_softmax(logit, dim=1).squeeze(dim=0) # [vocab_size] Get scores                    \n",
        "                    topk_logps, topk_tokens = torch.topk(logp, beam_size) # Get top k tokens & logps\n",
        "                    for k in range(beam_size):\n",
        "                        next_nodes.append(Node(token=topk_tokens[k, None], states=(h_state,),\n",
        "                                               logp=topk_logps[k, None].cpu().item(), parent=node,\n",
        "                                               eos=topk_tokens[k].cpu().item() == dest_field.vocab[dest_field.eos_token]))\n",
        "                if len(next_nodes) == 0:\n",
        "                    break\n",
        "                next_nodes = sorted(next_nodes, key=lambda node: node.logps, reverse=True) # Sort next_nodes to get the best\n",
        "                tree.append(next_nodes[:beam_size]) # Update the tree\n",
        "                \n",
        "            best_path = find_best_path(tree) # Find the best path of the tree\n",
        "\n",
        "            # Get the translation\n",
        "            pred_translated = [*map(lambda node: dest_field.vocab.itos[node.token], best_path)]\n",
        "            pred_translated = [*filter(lambda word: word not in [dest_field.init_token, dest_field.eos_token], pred_translated[::-1])]\n",
        "\n",
        "            hypotheses.append(pred_translated) # Update hypotheses\n",
        "\n",
        "            # Update references\n",
        "            references.append([[dest_field.vocab.itos[indice] for indice in dest_sequence if indice not in (\n",
        "                dest_field.vocab.stoi[dest_field.init_token],\n",
        "                dest_field.vocab.stoi[dest_field.eos_token],\n",
        "                dest_field.vocab.stoi[dest_field.pad_token]\n",
        "            )]])\n",
        "\n",
        "            # Update sources\n",
        "            sources.append([src_field.vocab.itos[indice]  for indice in src_sequence  if indice not in (\n",
        "                src_field.vocab.stoi[src_field.init_token],\n",
        "                src_field.vocab.stoi[src_field.eos_token],\n",
        "                src_field.vocab.stoi[src_field.pad_token]\n",
        "            )])\n",
        "    \n",
        "        assert len(hypotheses) == len(references) == len(sources)\n",
        "        bleu4 = bleu_score(hypotheses, references, max_n=4, weights=[0.25, 0.25, 0.25, 0.25]) # Calculate BLEU-4 score\n",
        "    return hypotheses, references, sources, bleu4"
      ],
      "metadata": {
        "id": "qEhLBJWG0Xmj"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "g-lc-nLat-gI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1d50ee24-30f0-4a04-e61d-6afc77ce0b28"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SeqToSeqNet(\n",
              "  (encoder): EncoderLayer(\n",
              "    (embedding): Embedding(10931, 300)\n",
              "    (gru): GRU(300, 256, num_layers=2, dropout=0.25, bidirectional=True)\n",
              "  )\n",
              "  (decoder): DecoderLayer(\n",
              "    (embedding): Embedding(8777, 300)\n",
              "    (gru): GRU(300, 256, num_layers=2, dropout=0.25)\n",
              "    (fc): Linear(in_features=256, out_features=8777, bias=True)\n",
              "  )\n",
              "  (init_h0): Linear(in_features=4, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "seq2seq.load_state_dict(torch.load('seq2seq.pth'))\n",
        "seq2seq.to(DEVICE)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "bleu_scores = []\n",
        "for beam_size in [1, 3, 5]:\n",
        "    for name, data in [('validation', valid), ('test', test)]:\n",
        "        _, _, _, bleu4 = evaluate(model=seq2seq, data=data, beam_size=beam_size, src_field=FR_TEXT, dest_field=EN_TEXT, max_len=50, device=DEVICE)\n",
        "        bleu_scores.append((beam_size, name, bleu4))\n",
        "        \n",
        "for bleu_score in bleu_scores:\n",
        "    print(f'BLEU-4: {bleu_score[2]*100:.3f}% with beam_size={bleu_score[0]} on {bleu_score[1]} data')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VAW5R1E6z3Pu",
        "outputId": "9f691149-a6c2-4105-a0dd-ad3f457e6389"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 996/996 [00:20<00:00, 49.02it/s]\n",
            "100%|██████████| 987/987 [00:19<00:00, 50.15it/s]\n",
            "100%|██████████| 996/996 [01:39<00:00, 10.00it/s]\n",
            "100%|██████████| 987/987 [01:39<00:00,  9.91it/s]\n",
            "100%|██████████| 996/996 [03:22<00:00,  4.91it/s]\n",
            "100%|██████████| 987/987 [03:20<00:00,  4.91it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU-4: 6.870% with beam_size=1 on validation data\n",
            "BLEU-4: 6.095% with beam_size=1 on test data\n",
            "BLEU-4: 7.298% with beam_size=3 on validation data\n",
            "BLEU-4: 6.389% with beam_size=3 on test data\n",
            "BLEU-4: 7.507% with beam_size=5 on validation data\n",
            "BLEU-4: 6.702% with beam_size=5 on test data\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "   \n",
        "for bleu_score in bleu_scores:\n",
        "    print(f'BLEU-4: {bleu_score[2]*100:.3f}% with beam_size={bleu_score[0]} on {bleu_score[1]} data')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4LTNBLI26CiI",
        "outputId": "536c9ace-3244-44c8-9b26-2ce2541bff43"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU-4: 6.870% with beam_size=1 on validation data\n",
            "BLEU-4: 6.095% with beam_size=1 on test data\n",
            "BLEU-4: 7.298% with beam_size=3 on validation data\n",
            "BLEU-4: 6.389% with beam_size=3 on test data\n",
            "BLEU-4: 7.507% with beam_size=5 on validation data\n",
            "BLEU-4: 6.702% with beam_size=5 on test data\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CQvQiVINwb_e"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "euSkrQikvFU4"
      },
      "outputs": [],
      "source": [
        "def translate(sentences, model, beam_size, src_field, dest_field, max_len, device):\n",
        "    if isinstance(sentences, list):\n",
        "        sentences = [*map(src_field.preprocess, sentences)]\n",
        "        targets = None\n",
        "    if isinstance(sentences, Dataset):\n",
        "        targets = [*map(lambda example: ' '.join(example.trg), sentences.examples)]\n",
        "        sentences = [*map(lambda example: example.src, sentences.examples)]\n",
        "    data = [*map(lambda word_list: src_field.process([word_list]), sentences)]\n",
        "    translated_sentences, pred_logps = [], []\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for i, (src_sequence, src_length) in tqdm.tqdm(enumerate(data), total=len(data)):\n",
        "            src_sequence, src_length = src_sequence.to(device), src_length.to(device)\n",
        "            h_state = model.encoder(input_sequences=src_sequence, sequence_lengths=src_length) # Encode\n",
        "            h_state = model.init_h0(h_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]\n",
        "            h_state = h_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]\n",
        "            tree = [[Node(token=torch.LongTensor([dest_field.vocab.stoi[dest_field.init_token]]).to(device), states=(h_state,))]]\n",
        "            for _ in range(max_len):\n",
        "                next_nodes = []\n",
        "                for node in tree[-1]:\n",
        "                    if node.eos: # Skip eos token\n",
        "                        continue\n",
        "                    logit, h_state = model.decoder(input_word_index=node.token, h_state_prev=node.states[0].contiguous())\n",
        "                    logp = F.log_softmax(logit, dim=1).squeeze(dim=0)                   \n",
        "                    topk_logps, topk_tokens = torch.topk(logp, beam_size)\n",
        "                    for k in range(beam_size):\n",
        "                        next_nodes.append(Node(token=topk_tokens[k, None], states=(h_state,),\n",
        "                                               logp=topk_logps[k, None].cpu().item(), parent=node,\n",
        "                                               eos=topk_tokens[k].cpu().item() == dest_field.vocab[dest_field.eos_token]))\n",
        "                if len(next_nodes) == 0:\n",
        "                    break\n",
        "                next_nodes = sorted(next_nodes, key=lambda node: node.logps, reverse=True)\n",
        "                tree.append(next_nodes[:beam_size])\n",
        "            best_path = find_best_path(tree)\n",
        "            # Get the translation\n",
        "            pred_translated = [*map(lambda node: dest_field.vocab.itos[node.token], best_path)]\n",
        "            pred_translated = [*filter(lambda word: word not in [\n",
        "                dest_field.init_token, dest_field.eos_token\n",
        "            ], pred_translated[::-1])]\n",
        "            translated_sentences.append(' '.join(pred_translated))\n",
        "            # Get probabilities\n",
        "            pred_logps.append(sum([*map(lambda node: node.logps, best_path)]))\n",
        "        sentences = [*map(lambda sentence: ' '.join(sentence), sentences)]\n",
        "    return sentences, translated_sentences, targets, pred_logps"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "uA8qEvYtzZn0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 953
        },
        "outputId": "9ff42fdf-f400-45b7-fd6c-5c613c863ecc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 987/987 [03:16<00:00,  5.01it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[526 285 435 839 329 740 328 627 219 378]\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> la commission vient de proposer la creation d un dispositif de reaction rapide</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> the commission recently tabled a proposal for the creation of a rapid reaction facility</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> the commission agrees to the commission s draft draft</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> la presidente invite l orateur a conclure j ai le droit de repondre et j en ai termine tout de suite</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> the president asked the speaker to conclude i have the right to reply and i shall finish soon</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> the president and and and and i have i and i speak to speak</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> mais elle ne peut pas etre a la charge des partis europeens car cela representerait une diminution de notre conscience europeenne</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> but it cannot work against the european parties because that would represent a reduction in our european selfawareness</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> however it cannot be allowed to support the european of the european of the european of the european european</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> le parlement doit restructurer son mode de fonctionnement de sorte a refleter l evolution des conditions de travail</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> this parliament must restructure its mode of operation to reflect the changed working conditions</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> parliament must keep this opportunity to work with the work of work</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> par votre intermediaire je voudrais donc reclamer l instauration de regles bien definies pour ce type de festivites</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> what i would like to ask through you is that these kinds of festivity be regulated properly</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> i i would like to see this type of this type of such as a product of</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> j ai deja evoque la reduction des budgets militaires dans les etats membres</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> i have previously discussed the shrinking defence budgets of the member states</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> i already already referred to the first of the in the member states</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> j estime monsieur le president que cette situation est indigne du parlement</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> i think mr president that that is unworthy of parliament</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> mr president mr president i believe that this is indeed the situation</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> je dois en outre affirmer que la nature des taches visees les taches de petersberg permet parfaitement de les traiter ici</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> i also have to say that the nature of the tasks which we have in mind the petersberg tasks is such that here is the ideal place where they can be dealt with</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> furthermore i have to see that the greek of the committees of the committees of the greek</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> pour cette raison la finlande a besoin d une possibilite independante d appliquer les restrictions a l importation qu elle juge raisonnables</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> that is why finland needs an independent mechanism regarding what it itself perceives as reasonable import restrictions</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> this is why this is the opportunity to serve to serve to serve to the the</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p><span style=\"color:black\"><b>Source:</b> en tant que rapporteur je souhaiterais egalement expressement remercier la commission pour sa disponibilite au dialogue</span><br /><span style=\"color:black\"><b>Ground truth translation:</b> i for my part as rapporteur should like expressly to thank the commission for its willingness to negotiate</span><br /><span style=\"color:pink\"><b>Predicted translation:</b> i would also like to thank the rapporteur for the commission for its opinion</span></p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "====================================================================================================\n"
          ]
        }
      ],
      "source": [
        "sentences, translated_sentences, dest_sentences, pred_logps = translate(sentences=test, model=seq2seq, beam_size=5, src_field=FR_TEXT,\n",
        "                                                                        dest_field=EN_TEXT, max_len=50, device=DEVICE)\n",
        "indexes = np.random.choice(len(test.examples), size=10, replace=False)\n",
        "print(indexes)\n",
        "print()\n",
        "for i in indexes:\n",
        "    html = f'<p><span style=\"color:black\"><b>Source:</b> {sentences[i]}</span><br />'\n",
        "    html += f'<span style=\"color:black\"><b>Ground truth translation:</b> {dest_sentences[i]}</span><br />'\n",
        "    html += f'<span style=\"color:pink\"><b>Predicted translation:</b> {translated_sentences[i]}</span></p>'\n",
        "    display(HTML(html))\n",
        "    print('='*100)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "0_noise_seq2seq_RNN.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}