NeuralMachineTranslation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Neural Machine Translation</b>\n",
    "\n",
    "This implementation follows Keon's <a href=\"https://github.com/keon/seq2seq/blob/master/model.py\">\"Minimal Seq2Seq model with Attention for Neural Machine Translation in PyTorch\"</a>.\n",
    "\n",
    "The dataset can be downloaded from <a href=\"http://www.manythings.org/anki/\">Tatoeba Project: Tab-delimited Bilingual Sentence Pairs</a>. The file is names \"deu-eng.zip\" (German-English) and contains deu.txt, which needs to be converted to a tsv and the english/german columns need to be swapped. The file should be called \"eng-ger-data.tsv\" and located in \".data\" or the reference below in the torch TabularDataset statement needs to be changed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tb 2018-09-27 \n",
      "\n",
      "CPython 3.6.4\n",
      "IPython 6.2.1\n",
      "\n",
      "sys 3.6.4 |Anaconda custom (64-bit)| (default, Jan 16 2018, 12:04:33) \n",
      "[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]\n",
      "numpy 1.14.2\n",
      "pandas 0.22.0\n",
      "sklearn 0.19.2\n",
      "torch 0.4.0a0+9692519\n",
      "IPython 6.2.1\n",
      "\n",
      "compiler   : GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)\n",
      "system     : Darwin\n",
      "release    : 17.5.0\n",
      "machine    : x86_64\n",
      "processor  : i386\n",
      "CPU cores  : 24\n",
      "interpreter: 64bit\n",
      "\n",
      "GPU Name: TITAN Xp\n",
      "GPU Memory: 12.0GB\n",
      "CUDA Version: (9, 1, 0)\n",
      "GPU Free/Total Memory: 84%\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import random\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import torchtext\n",
    "from torchtext.data import Field\n",
    "\n",
    "from utils import *\n",
    "\n",
    "# used to suppress \"UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad()\"\n",
    "# import warnings\n",
    "# warnings.filterwarnings(\"ignore\" ,category=FutureWarning)\n",
    "\n",
    "%load_ext watermark\n",
    "%watermark -a \"tb\" -d -v -m -p sys,numpy,pandas,sklearn,torch,IPython\n",
    "gpu_stat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.cuda.is_available = lambda : False\n",
    "#torch.backends.cudnn.enabled=False\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 0\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(seed)\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "H = HYPERPARAMETERS({\n",
    "    'seq_max_len' :         50,\n",
    "    'src_vocab_max_size' :  50000,\n",
    "    'tgt_vocab_max_size' :  50000,\n",
    "    'src_field_name' :      'src',\n",
    "    'tgt_field_name' :      'tgt',\n",
    "\n",
    "    'embedding_size' :      256,\n",
    "    'hidden_size' :         256,\n",
    "    'n_layers' :            2,\n",
    "    'n_epochs' :            10,\n",
    "    'batch_size' :          128,\n",
    "    'max_grad_norm' :       5,\n",
    "    'teacher_forcing_ratio':0.5,\n",
    "    'embedding_dropout' :   0.5,\n",
    "    'rnn_dropout' :         0.5,\n",
    "\n",
    "    'SYM_SOS' :             '<sos>',\n",
    "    'SYM_EOS' :             '<eos>',\n",
    "    'SYM_PAD' :             '<pad>',\n",
    "    'IDX_SOS' :             -1,\n",
    "    'IDX_EOS' :             -1,\n",
    "    'IDX_PAD' :             -1,\n",
    "    'device' :                None if torch.cuda.is_available() else -1\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "spacy_en = spacy.load('en')\n",
    "spacy_de = spacy.load('de')\n",
    "\n",
    "def tokenize_en(text):\n",
    "    return [tok.text for tok in spacy_en.tokenizer( text )]\n",
    "    return text.split()\n",
    "\n",
    "def tokenize_de(text):\n",
    "    return [tok.text for tok in spacy_de.tokenizer(text)]\n",
    "    return text.split()\n",
    "\n",
    "preproc = lambda seq: [H.SYM_SOS] + seq + [H.SYM_EOS]\n",
    "\n",
    "src = Field(sequential=True, tokenize=tokenize_en, lower=True, batch_first=True, include_lengths=True)\n",
    "tgt = Field(sequential=True, tokenize=tokenize_de, lower=True, batch_first=True, preprocessing=preproc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def len_filter(example):\n",
    "    return len(example.src) <= H.seq_max_len and len(example.tgt) <= H.seq_max_len\n",
    "\n",
    "train_data, valid_data, test_data= torchtext.data.TabularDataset(\n",
    "    path=\"./data/eng-ger-data.tsv\", format='tsv',\n",
    "    fields=[(H.src_field_name, src), (H.tgt_field_name, tgt)],\n",
    "    filter_pred=len_filter\n",
    "    ).split(split_ratio=[0.8, 0.1, 0.1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "src.build_vocab(train_data, max_size=H.src_vocab_max_size, min_freq=2)\n",
    "tgt.build_vocab(train_data, max_size=H.tgt_vocab_max_size, min_freq=2)\n",
    "\n",
    "input_vocab = src.vocab\n",
    "output_vocab = tgt.vocab\n",
    "\n",
    "H.IDX_SOS = output_vocab.stoi[H.SYM_SOS]\n",
    "H.IDX_EOS = output_vocab.stoi[H.SYM_EOS]\n",
    "H.IDX_PAD = output_vocab.stoi[H.SYM_PAD]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 6]) torch.Size([128]) torch.Size([128, 12])\n",
      "['where', \"'re\", 'tom', \"'s\", 'things', '?']\n",
      "len: 6\n",
      "['<sos>', 'wo', 'sind', 'toms', 'sachen', '?', '<eos>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1058, 133, 133)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_iter, valid_iter, test_iter = torchtext.data.BucketIterator.splits(\n",
    "                                (train_data, valid_data, test_data), \n",
    "                                batch_size=H.batch_size, device=H.device, repeat=False, \n",
    "                                sort=False, sort_within_batch=True, sort_key=lambda x: len(x.src))\n",
    "\n",
    "\n",
    "batch = next(train_iter.__iter__())\n",
    "input_variables = getattr(batch, 'src')\n",
    "target_variables = getattr(batch, 'tgt')\n",
    "\n",
    "print(input_variables[0].shape, input_variables[1].shape, target_variables.shape)\n",
    "print([input_vocab.itos[ni] for ni in input_variables[0][0].cpu()])\n",
    "print(\"len:\", input_variables[1][0].item())\n",
    "print([output_vocab.itos[ni] for ni in target_variables[0].cpu()])\n",
    "\n",
    "len(train_iter), len(valid_iter), len(test_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_size, hidden_size, n_layers=1, bidirectional = True, \n",
    "                 embedding_dropout=0, rnn_dropout=0):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embedding_size = embedding_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        self.bidirectional = bidirectional\n",
    "        self.embedding_dropout = embedding_dropout\n",
    "        self.rnn_dropout = rnn_dropout\n",
    "\n",
    "        self.embedding = nn.Sequential(\n",
    "            nn.Embedding(self.vocab_size, self.embedding_size),\n",
    "            nn.Dropout(self.embedding_dropout) \n",
    "        )\n",
    "        \n",
    "        self.rnn = nn.GRU(self.embedding_size, self.hidden_size, self.n_layers, \n",
    "                          batch_first=True, bidirectional=self.bidirectional, dropout=self.rnn_dropout)\n",
    "        \n",
    "        self.init_weights()\n",
    "\n",
    "    def forward(self, input_var, hidden=None):\n",
    "        \n",
    "        embedded = self.embedding(input_var) \n",
    "        \n",
    "        output_var, hidden = self.rnn(embedded, hidden)\n",
    "                \n",
    "        output_var = self._sum_outputs(output_var)\n",
    "        hidden = self._cat_hidden(hidden)\n",
    "        \n",
    "        return output_var, hidden\n",
    "\n",
    "    def _cat_hidden(self, h):\n",
    "        \"\"\" If the encoder is bidirectional, do the following transformation.\n",
    "            (#directions * #layers, #batch, hidden_size) -> (#layers, #batch, #directions * hidden_size)\n",
    "        \"\"\"\n",
    "        if self.bidirectional:\n",
    "            h = torch.cat([h[0:h.size(0):2], h[1:h.size(0):2]], 2)\n",
    "        return h\n",
    "    \n",
    "    def _sum_outputs(self, outputs):\n",
    "        # sum bidirectional outputs\n",
    "        outputs = (outputs[:, :, :self.hidden_size] +\n",
    "                   outputs[:, :, self.hidden_size:])\n",
    "        return outputs\n",
    "\n",
    "    def init_weights(self):\n",
    "        initrange = 0.1\n",
    "        self.embedding[0].weight.data.uniform_(-initrange, initrange)\n",
    "        for param in self.rnn.parameters():\n",
    "            param.data.uniform_(-0.08, 0.08)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary for model: EncoderRNN\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Shape                     Param #   \n",
      "=================================================================\n",
      "embedding.0 (Embedding)      ((9563, 256),)            2448128   \n",
      "_________________________________________________________________\n",
      "embedding.1 (Dropout)        ()                        0         \n",
      "_________________________________________________________________\n",
      "rnn (GRU)                    ((768, 256), (768, 256),  1972224   \n",
      "=================================================================\n",
      "Total params:         4,420,352\n",
      "Trainable params:     4,420,352\n",
      "_________________________________________________________________\n",
      "EncoderRNN(\n",
      "  (embedding): Sequential(\n",
      "    (0): Embedding(9563, 256)\n",
      "    (1): Dropout(p=0.5)\n",
      "  )\n",
      "  (rnn): GRU(256, 256, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([128, 6]),\n",
       " torch.Size([4, 128, 256]),\n",
       " torch.Size([128, 6, 256]),\n",
       " torch.Size([2, 128, 512]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder = EncoderRNN(len(input_vocab), H.embedding_size, H.hidden_size, n_layers=H.n_layers, \n",
    "                                        rnn_dropout=H.rnn_dropout, embedding_dropout=H.embedding_dropout)\n",
    "\n",
    "hidden = Variable(torch.zeros(2*H.n_layers, H.batch_size, H.hidden_size))\n",
    "\n",
    "encoder_output, encoder_hidden = encoder(input_variables[0].cpu(), hidden)\n",
    "\n",
    "print_model_summary(encoder)\n",
    "print(encoder)\n",
    "\n",
    "input_variables[0].shape, hidden.shape, encoder_output.shape, encoder_hidden.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_size, hidden_size, n_layers=1, embedding_dropout=0, rnn_dropout=0):\n",
    "                 \n",
    "        super(DecoderRNN, self).__init__()\n",
    "        self.vocab_size = vocab_size\n",
    "        self.embedding_size = embedding_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        self.embedding_dropout = embedding_dropout\n",
    "        self.rnn_dropout = rnn_dropout\n",
    "                \n",
    "        self.embedding = nn.Sequential(\n",
    "            nn.Embedding(self.vocab_size, self.embedding_size),\n",
    "            nn.Dropout(self.embedding_dropout) \n",
    "        )\n",
    "\n",
    "        self.rnn = nn.GRU(self.embedding_size, self.hidden_size, self.n_layers, \n",
    "                          batch_first=True, dropout=self.rnn_dropout)\n",
    "\n",
    "        self.fc = nn.Linear(self.hidden_size, self.vocab_size)\n",
    "        \n",
    "        self.init_weights()\n",
    "        \n",
    "    def forward(self, decoder_input, encoder_hidden):\n",
    "        batch_size, output_size = decoder_input.size(0), decoder_input.size(1)\n",
    "            \n",
    "        embedded = self.embedding(decoder_input)\n",
    "        \n",
    "        output, hidden = self.rnn(embedded, encoder_hidden)\n",
    "        \n",
    "        output = output.contiguous().view(-1, self.hidden_size)\n",
    "        output = self.fc(output)\n",
    "        \n",
    "        output = F.log_softmax(output, dim=1)\n",
    "        output = output.view(batch_size, output_size, -1)\n",
    "        return output, hidden\n",
    "    \n",
    "    def init_weights(self):\n",
    "        initrange = 0.1\n",
    "        self.embedding[0].weight.data.uniform_(-initrange, initrange)\n",
    "        for param in self.rnn.parameters():\n",
    "            param.data.uniform_(-0.08, 0.08)\n",
    "        self.fc.bias.data.zero_()\n",
    "        self.fc.weight.data.uniform_(-initrange, initrange)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary for model: DecoderRNN\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Shape                     Param #   \n",
      "=================================================================\n",
      "embedding.0 (Embedding)      ((15738, 512),)           8057856   \n",
      "_________________________________________________________________\n",
      "embedding.1 (Dropout)        ()                        0         \n",
      "_________________________________________________________________\n",
      "rnn (GRU)                    ((1536, 512), (1536, 512) 3151872   \n",
      "_________________________________________________________________\n",
      "fc (Linear)                  ((15738, 512), (15738,))  8073594   \n",
      "=================================================================\n",
      "Total params:         19,283,322\n",
      "Trainable params:     19,283,322\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([128, 6]), 2, torch.Size([128, 12, 15738]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder = DecoderRNN(len(output_vocab), 2 * H.hidden_size, 2 * H.hidden_size, n_layers=H.n_layers, \n",
    "                     rnn_dropout=H.rnn_dropout, embedding_dropout=H.embedding_dropout)\n",
    "\n",
    "decoder.train()\n",
    "decoder_output = decoder(target_variables.cpu(), encoder_hidden)\n",
    "\n",
    "print_model_summary(decoder)\n",
    "\n",
    "input_variables[0].shape, len(decoder_output), decoder_output[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NMT(nn.Module):\n",
    "    def __init__(self, encoder, decoder, sos_idx, teacher_forcing_ratio=0.5):\n",
    "        super(NMT, self).__init__()\n",
    "        self.encoder = encoder\n",
    "        self.decoder = decoder\n",
    "        self.sos_idx = sos_idx\n",
    "        self.teacher_forcing_ratio = teacher_forcing_ratio\n",
    "\n",
    "    def forward(self, input_var, target_var):\n",
    "\n",
    "        encoder_outputs, encoder_hidden = self.encoder(input_var)\n",
    "                \n",
    "        batch_size = input_var.size(0)\n",
    "        decoder_input = Variable(torch.LongTensor([self.sos_idx] * batch_size)).view(batch_size, 1)\n",
    "        if next(self.parameters()).is_cuda:\n",
    "            decoder_input = decoder_input.cuda()\n",
    "        \n",
    "        decoder_outputs = []\n",
    "        max_length = target_var.size(1) - 1 if not target_var is None else H.seq_max_len\n",
    "        for di in range(max_length):\n",
    "            \n",
    "            decoder_output, decoder_hidden = self.decoder(decoder_input, encoder_hidden)\n",
    "            step_output = decoder_output.squeeze(1)\n",
    "            \n",
    "            decoder_outputs.append(step_output)\n",
    "\n",
    "            use_teacher_forcing = True if self.training and random.random() < self.teacher_forcing_ratio else False       \n",
    "            if use_teacher_forcing:\n",
    "                decoder_input = Variable(target_var.data[:, di+1]).view(batch_size, 1)\n",
    "            else:\n",
    "                decoder_input = Variable(decoder_outputs[-1].topk(1)[1].data).view(batch_size, 1)\n",
    "\n",
    "            if next(self.parameters()).is_cuda:\n",
    "                decoder_input = decoder_input.cuda()\n",
    "                \n",
    "        return decoder_outputs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary for model: NMT\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Shape                     Param #   \n",
      "=================================================================\n",
      "encoder.embedding.0 (Embeddi ((9563, 256),)            2448128   \n",
      "_________________________________________________________________\n",
      "encoder.embedding.1 (Dropout ()                        0         \n",
      "_________________________________________________________________\n",
      "encoder.rnn (GRU)            ((768, 256), (768, 256),  1972224   \n",
      "_________________________________________________________________\n",
      "decoder.embedding.0 (Embeddi ((15738, 512),)           8057856   \n",
      "_________________________________________________________________\n",
      "decoder.embedding.1 (Dropout ()                        0         \n",
      "_________________________________________________________________\n",
      "decoder.rnn (GRU)            ((1536, 512), (1536, 512) 3151872   \n",
      "_________________________________________________________________\n",
      "decoder.fc (Linear)          ((15738, 512), (15738,))  8073594   \n",
      "=================================================================\n",
      "Total params:         23,703,674\n",
      "Trainable params:     23,703,674\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "NMT(\n",
       "  (encoder): EncoderRNN(\n",
       "    (embedding): Sequential(\n",
       "      (0): Embedding(9563, 256)\n",
       "      (1): Dropout(p=0.5)\n",
       "    )\n",
       "    (rnn): GRU(256, 256, num_layers=2, batch_first=True, dropout=0.5, bidirectional=True)\n",
       "  )\n",
       "  (decoder): DecoderRNN(\n",
       "    (embedding): Sequential(\n",
       "      (0): Embedding(15738, 512)\n",
       "      (1): Dropout(p=0.5)\n",
       "    )\n",
       "    (rnn): GRU(512, 512, num_layers=2, batch_first=True, dropout=0.5)\n",
       "    (fc): Linear(in_features=512, out_features=15738, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nmt = NMT(encoder, decoder, H.IDX_SOS, teacher_forcing_ratio=H.teacher_forcing_ratio).cuda()\n",
    "\n",
    "nmt.train()\n",
    "result = nmt(input_variables[0], target_variables)\n",
    "\n",
    "print_model_summary(nmt)\n",
    "nmt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_bleu import compute_bleu\n",
    "\n",
    "def _eos_parsing(sentence):\n",
    "    if H.IDX_EOS in sentence:\n",
    "        sentence = sentence[:sentence.index(H.IDX_EOS)+1]\n",
    "    if H.IDX_PAD in sentence:\n",
    "        sentence = sentence[:sentence.index(H.IDX_PAD)+1]\n",
    "    return sentence\n",
    "\n",
    "def blue_score(decoder_outputs, target_variables):\n",
    "    batch_size = target_variables.size(0)\n",
    "\n",
    "    output_seq = [[] for _ in range(batch_size)]\n",
    "    for decoder_output in decoder_outputs:\n",
    "        for batch_idx in range(batch_size):\n",
    "            output_seq[batch_idx].append(decoder_output[batch_idx].topk(1)[1].item())\n",
    "\n",
    "    target_seq = [[[]] for _ in range(batch_size)]\n",
    "    for batch_idx in range(batch_size):\n",
    "        target_seq[batch_idx][0] = [i.item() for i in target_variables[batch_idx]][1:-1]\n",
    "\n",
    "    for batch_idx in range(batch_size):\n",
    "        output_seq[batch_idx] = _eos_parsing(output_seq[batch_idx])\n",
    "        target_seq[batch_idx][0] = _eos_parsing(target_seq[batch_idx][0])\n",
    "             \n",
    "    return compute_bleu(target_seq, output_seq, smooth=False)[0]\n",
    "\n",
    "def get_perplexity(loss):\n",
    "    try:\n",
    "        return math.pow(2, loss)\n",
    "    except OverflowError:\n",
    "        return float('inf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = EncoderRNN(len(input_vocab), H.embedding_size, H.hidden_size, n_layers=H.n_layers, \n",
    "                     embedding_dropout=H.embedding_dropout, rnn_dropout=H.rnn_dropout)\n",
    "decoder = DecoderRNN(len(output_vocab), H.embedding_size*2, H.hidden_size*2, n_layers=H.n_layers, \n",
    "                     embedding_dropout=H.embedding_dropout, rnn_dropout=H.rnn_dropout)\n",
    "\n",
    "model = NMT(encoder, decoder, H.IDX_SOS, teacher_forcing_ratio=H.teacher_forcing_ratio)\n",
    "if torch.cuda.is_available():\n",
    "    model.cuda(H.device)\n",
    "\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(),\n",
    "                        amsgrad = False,\n",
    "                        betas = (0.9, 0.999),\n",
    "                        eps = 1e-08,\n",
    "                        lr = 0.001,\n",
    "                        weight_decay = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter, valid_iter, test_iter = torchtext.data.BucketIterator.splits(\n",
    "                                (train_data, valid_data, test_data), \n",
    "                                batch_size=H.batch_size, device=H.device, repeat=False, \n",
    "                                sort=False, sort_within_batch=True, sort_key=lambda x: len(x.src))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a642baf3ab2148f0bfd34769a5185163",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/torchtext/data/field.py:321: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  return Variable(arr, volatile=not train), lengths\n",
      "/anaconda3/lib/python3.6/site-packages/torchtext/data/field.py:322: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  return Variable(arr, volatile=not train)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "m = Metric([('train_loss', np.inf), ('valid_loss', np.inf), ('valid_ppl', np.inf), ('valid_score', np.inf)])\n",
    "\n",
    "epoch_itr = tqdm_notebook(range(1, H.n_epochs + 1))\n",
    "for epoch in epoch_itr:\n",
    "    epoch_itr.set_description('Epoch')\n",
    "\n",
    "    model.train(True)\n",
    "\n",
    "    avg_loss = 0.0\n",
    "    for idx_batch, batch in enumerate(train_iter.__iter__()):\n",
    "        \n",
    "        input_variables, input_lengths = getattr(batch, H.src_field_name)\n",
    "        target_variables = getattr(batch, H.tgt_field_name)\n",
    "\n",
    "        decoder_outputs = model(input_variables, target_variables)\n",
    "        \n",
    "        loss = 0\n",
    "        target_length = target_variables.size(1) - 1\n",
    "        for idx_seq in range(target_length):\n",
    "            seq_output = decoder_outputs[idx_seq]\n",
    "            out_batch_size = target_variables.size(0)\n",
    "            loss += criterion(seq_output.contiguous().view(out_batch_size, -1), target_variables[:, idx_seq + 1])\n",
    "\n",
    "        avg_loss += loss.item() / (idx_seq + 1)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), H.max_grad_norm)\n",
    "        optimizer.step()\n",
    "                \n",
    "        if not idx_batch % 500:\n",
    "            epoch_itr.set_postfix(epoch=epoch, batch=idx_batch,\n",
    "                                  train_loss=m.train_loss, valid_loss=m.valid_loss,\n",
    "                                  valid_ppl=m.valid_ppl, valid_score=m.valid_score)    \n",
    "            \n",
    "    m.train_loss = avg_loss / (idx_batch + 1)\n",
    "    \n",
    "    model.eval()   # Turn on evaluation mode which disables dropout and teacher_forcing.\n",
    "    with torch.no_grad():\n",
    "        \n",
    "        avg_loss = 0.0\n",
    "        avg_score = 0.0\n",
    "        for idx_batch, batch in enumerate(valid_iter.__iter__()):\n",
    "            input_variables, input_lengths = getattr(batch, H.src_field_name)\n",
    "            target_variables = getattr(batch, H.tgt_field_name)\n",
    "\n",
    "            decoder_outputs = model(input_variables, None)\n",
    "            \n",
    "            loss = 0\n",
    "            target_length = target_variables.size(1) - 1\n",
    "            for idx_seq in range(target_length):\n",
    "                seq_output = decoder_outputs[idx_seq]\n",
    "                loss += criterion(seq_output.contiguous(), target_variables[:, idx_seq + 1])\n",
    "\n",
    "            avg_loss += loss.item() / (idx_seq + 1)\n",
    "            avg_score += blue_score(decoder_outputs, target_variables)\n",
    "            \n",
    "        m.valid_loss = avg_loss / (idx_batch + 1)\n",
    "        m.valid_ppl = get_perplexity(m.valid_loss)\n",
    "        m.valid_score = avg_score / (idx_batch + 1)\n",
    "        \n",
    "    epoch_itr.set_postfix(epoch=epoch, batch=0,\n",
    "                          train_loss=m.train_loss, valid_loss=m.valid_loss,\n",
    "                          valid_ppl=m.valid_ppl, valid_score=m.valid_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA+gAAAGoCAYAAADVZM+hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XmcjXX/x/HXd5AZ+xh7dJOss5oZSxnLRCTZKiHuLC03KUWbRAu624RWQkUSLX7S3V0p2SvJOonsug3SILsxZub7++OaORmzGMzMdWbm/Xw8rsfMOedaPueY+pzP9d2MtRYRERERERERcZeP2wGIiIiIiIiIiAp0EREREREREa+gAl1ERERERETEC6hAFxEREREREfECKtBFREREREREvIAKdBEREREREREvoAJdRERERERExAuoQBcpwIwxu40xbd2OQ0RERC6NMSbKGPODMeaoMeawMeZ7Y0xjt+MSkdxR1O0AREREREQkPWNMGeALYBDwMXAF0AI4k4PXKGKtTcqp84nI5VELukghZIy5xxizPeVO/OfGmGopzxtjzARjzJ8pd+pjjDFBKa/dZIzZZIw5bozZa4x5xN13ISIiUuDVBbDWzrbWJllrT1trv7HWxoAnn29Oyc2bjDHhKc83MMYsMcYcMcb8aozpnHpCY8x0Y8wkY8yXxpiTQLQxprgxZpwx5n/GmAPGmMnGGD9X3rFIIacCXaSQMcZcDzwP3A5UBX4H5qS83A5oifOFoBzQAziU8to7wL+staWBIGBRHoYtIiJSGG0FkowxM4wxHYwx/qkvGGO6A88AdwJlgM7AIWNMMeA/wDdAJeABYJYxpt45570DeA4oDawAXsTJ/WHANcCVwFO5+9ZEJCMq0EUKn97Au9batdbaM8ATwLXGmJrAWZxkXR8w1trN1tr9KcedBRoaY8pYa/+y1q51IXYREZFCw1p7DIgCLDAViEvp+VYZuBt4yVr7s3Vst9b+DjQDSgEvWGsTrLWLcLrJ9zrn1POttd9ba5NxusvfAwy11h621h4H/g30zLM3KiIeKtBFCp9qOK3mAFhrT+C0kl+ZksTfAN4EDhhjpqSMfwO4FbgJ+N0Ys9QYc20exy0iIlLopNws72etrY7Tg60aMBGoAezI4JBqwJ6U4jvV7zit4qn2nPN7RaAEsCalS/wR4OuU50Ukj6lAFyl89gH/SH1gjCkJBAB7Aay1r1lrI4BAnO5uj6Y8/7O1tgtOd7nPcCarERERkTxirf0NmI5TqO8Bamew2z6ghjHm3O/5V5GS51NPdc7vB4HTQKC1tlzKVtZaWypHgxeRbFGBLlLwFTPG+KZuOIV1f2NMmDGmOE43tp+stbuNMY2NMU1Txq+dBOJxxr5dYYzpbYwpa609CxwDNOOriIhILjLG1DfGPGyMqZ7yuAZOV/WVwDTgEWNMRMokr9cYY/4B/ISTwx8zxhQzxrQGOvH3fDNppLS0TwUmGGMqpVznSmNM+9x+fyKSngp0kYLvS5w746lbC2AUMBfYj3P3PXWcWRmcJP0XTne4Q8C4lNf+Cew2xhwDBgJ98ih+ERGRwuo40BT4KWXG9ZXARuBha+0nOBO9fZiy32dAeWttAs6EcR1wWsffAu5MaX3PzOPAdmBlSp5fCNTLYn8RySXGWnvhvUREREREREQkV6kFXURERERERMQL5GqBbozZbYz5xRiz3hizOjevJSIiIpfPGPOuMeZPY8zGc54rb4z51hizLeWnf1bnEBERkUuTFy3o0dbaMGttZB5cS0RERC7PdODG854bDnxnra0DfJfyWERERHKYuriLiIiIh7V2GXD4vKe7ADNSfp8BdM3ToERERAqJorl8fgt8Y4yxwNvW2inn72CMuRe4F6BkyZIR9evXz+WQREREvNeaNWsOWmsruh3HeSpba/cDWGv3py7FdD7ldBERkb9dSk7P7QK9ubV2X0oi/9YY81vKnXmPlKJ9CkBkZKRdvVpD1UVEpPAyxvzudgyXSjldRETkb5eS03O1i7u1dl/Kzz+BeUCT3LyeiIiI5IoDxpiqACk//3Q5HhERkQIp1wp0Y0xJY0zp1N+BdsDGrI8SERERL/Q50Dfl977AfBdjERERKbBys4t7ZWCeMSb1Oh9aa7/OxeuJiIjIZTLGzAZaAxWMMbHA08ALwMfGmLuA/wHd3YtQRESk4Mq1At1auxMIza3zi4icPXuW2NhY4uPj3Q5F5KL5+vpSvXp1ihUr5nYoaVhre2XyUps8DURECgTlaikMcjKn5/YkcSIiuSY2NpbSpUtTs2ZNUnrriOQL1loOHTpEbGwstWrVcjscEZFco1wtBV1O53Stgy4i+VZ8fDwBAQFK+JLvGGMICAhQi5KIFHjK1VLQ5XROV4EuIvmaEr7kV/rbFZHCQv+/k4IuJ//GVaCLiIiIiIiIeAEV6CIil+jQoUOEhYURFhZGlSpVuPLKKz2PExISsnWO/v37s2XLliz3efPNN5k1a1ZOhJyhkSNHMnHixFw598mTJ2ndujXJycnZPuZi329ycjLt27enXLlydO3aNdP9Fi9eTKNGjShatCifffZZmtcefvhhAgMDadCgAUOHDsVaC8ANN9xAWFgYgYGB3HfffSQlJQHOv32bNm2oU6cO7du35+jRowB89tlnjBkzJtuxi4hI7iooudpNn376Kf/+97/z7HoLFy4kPDycoKAgBgwYQGJiYrp9du3aRXh4OGFhYQQFBTF16lTPa5nl7mHDhlGvXj1CQkK49dZbPbkbYP369TRr1ozAwECCg4M5e/YsAG3atEmzX56w1nrNFhERYUVEsmvTpk1uh+Dx9NNP25dffjnd88nJyTYpKcmFiLLvySeftBMmTMiVc0+cONG+8cYbuXLuVMnJyXbhwoV23rx5tkuXLpnut3PnThsTE2N79epl582b53l+6dKltkWLFjYpKcmePXvWNm7c2C5fvtxaa+3Ro0ettdYmJSXZLl262E8++cRaa+3QoUM9/95jxoyxI0aM8MQSGhpqT58+na3YM/obBlZbL8jJl7spp4uItcrV3urs2bPZ3rdJkyb28OHDuRjN3xITE+2VV15pt2/fbq219oknnrDTp09Pt198fLyNj4+31jq5ukaNGvbAgQOex9amz91ff/21530PGzbMk7sTEhJsUFCQjYmJsdZaGxcX5/l7mDZtmn3hhReyFXtO5XS1oIuI5LDt27cTFBTEwIEDCQ8PZ//+/dx7771ERkYSGBjI6NGjPftGRUWxfv16EhMTKVeuHMOHDyc0NJRrr72WP//8E0jbwh0VFcXw4cNp0qQJ9erV44cffgCclupbb72V0NBQevXqRWRkpOe8//znPwkODiYoKIjXXnsty9jXrl1L06ZN091dnjBhAg0bNiQ0NJQ+ffoAsGjRIkJDQwkLCyM8PJyTJ0+mO9+sWbPo0qUL4NwRj46O5rbbbqNOnTqMHDmS999/n8aNGxMSEsLu3buz/X7PZYyhTZs2lCpVKsv3VqtWLYKDg/HxSZv6jDHEx8eTkJDAmTNnSExMpFKlSgCUKVMGgKSkJM6cOeMZYzZ//nz69u0LQN++fT0t8sYYWrRowZdffpllLCIi4q78lqvnzJlDUFAQoaGhREdHA5CYmMjQoUMJCgoiJCSEt956C4Bvv/2WsLAwgoODueeeezw9BapXr86YMWNo3rw58+bNY9u2bbRv356IiAhatmzJ1q1b011306ZNlC5dGn9/fwD69OnD4MGDiY6Opnbt2ixbtoy+fftSv3597rrrLs9xGX2WR44coW7dumzfvh2A7t27895776W53p9//kmpUqWoXbs24LSGz507N11cxYsXp3jx4gCcOXOG5ORkT++3zHJ3+/btKVrUWcSsWbNmxMbGAvDVV18RERFBcHAwABUqVPB8V+jSpQsffvhhuuvnJi2zJiIFw0MPwfr1OXvOsDC4xK7fmzZt4r333mPy5MkAvPDCC5QvX57ExERPkdqwYcM0xxw9epRWrVrxwgsvMGzYMN59912GDx+e7tzWWlatWsXnn3/O6NGj+frrr3n99depUqUKc+fOZcOGDYSHhwOwZs0aDh48yC+//AI4yTErffr0YcqUKURFRTFixAjGjBnDuHHjeOmll/j999+54oorPOd4+eWXmTJlCk2bNuXEiRP4+vqmOVd8fDyxsbFUr17d89yGDRvYvHkzZcuWpWbNmtx33338/PPPvPLKK7zxxhuMGzcuW+83J7Vo0YLrrruOKlWqYK3loYceom7dup7X27Zty5o1a7j55pvp1q0b4HSZrFixIgBXXnkl+/fv9+wfGRnJ8uXLueWWW3I0ThGRfE+5+pJz9bPPPsuSJUuoXLmy5/VJkyaxb98+NmzYQJEiRTh8+DCnTp1iwIABLFmyhNq1a9O7d2+mTJnC/fffD0DJkiX5/vvvAYiOjmbatGnUrl2b77//nvvvv59vvvkmzXW///57IiIi0n0GixcvZu7cuXTq1Ikff/yR+vXrEx4ezsaNGwkKCsr0s3zttdfo168fgwYN4tSpU/Tv3z/NuStXrsypU6dYt24dYWFhzJ07lz179mT477d79246d+7M9u3bGT9+PJUrV/a8llHuPte7777rudG+detWrLW0a9eOgwcP0rt3bx5++GHAKdaPHz/OkSNHKFeuXIZx5DS1oIuI5ILatWvTuHFjz+PZs2cTHh5OeHg4mzdvZtOmTemO8fPzo0OHDgBERER4WpTPl1r4nbvPihUr6NmzJwChoaEEBgYCcM0117BlyxYefPBBFixYQNmyZTON+dChQ8THxxMVFQU4LcPLli0DIDAwkD59+jBr1iyKFSsGQPPmzXnooYd4/fXXOXbsGEWKFElzvj///JPy5cunea5p06ZUrlwZX19frr76atq3bw9AcHDwRb3fnLRlyxZ27NjB3r172bNnD1999VWalvqFCxeyb98+jh8/ztKlSzM8x7mzt1aqVIl9+/bleJwiIpKz8lOubt68OXfeeSfTpk3zzOuycOFCBg4c6Mm/5cuXZ/PmzdSpU8fTAn3nnXd6cjlAjx49AOcmwMqVK7n11lsJCwtj8ODBGeau/fv3e25Ip+rUqRPg5O5q1arRsGFDfHx8aNiwoee9ZvZZ3njjjdStW5cHH3wwzbjxVD4+Pnz44Yc88MADNG3alDJlynhavc9Xs2ZNYmJi2LZtG++88w4HDx70vJZV7n722WcpVaqU598iMTGR77//ntmzZ7N8+XI++uijNMdUrFgxzY343KYWdBEpGHJpkrNLVbJkSc/v27Zt49VXX2XVqlWUK1eOPn36ZLhW5hVXXOH5vUiRIhlOigJ4unSdu09qt67zBQQEEBMTw1dffcVrr73G3LlzmTJlSob7ZnYOgAULFrB06VLmz5/P2LFj2bhxIyNHjqRz587897//pXHjxixZsoQ6dep4jvHz80v3PlNjBycJpz728fG5qPebk/7v//6P6667zvNvduONN7Jy5Uquu+46zz5+fn506tSJ+fPnEx0dTUBAAHFxcVSsWJG9e/dSpUoVz77x8fH4+fnleJwiIvmecnWG+2YnV0+dOpWffvqJL774gtDQUGJiYrDWplveK6tcfu57ttZSoUIF1l+gR0NWufzcPJ76ODExMcvPMikpid9++w0/Pz/++usvqlWrlu6aUVFRrFixAoAvv/zygjfnr7zySurXr8+KFSvSTBZ7fu4GeOedd/jmm2/47rvvPPtVr16d1q1bExAQAECHDh1Yu3YtrVq1AvI+r6sFXUQklx07dozSpUtTpkwZ9u/fz4IFC3L8GlFRUXz88ccA/PLLL5471XFxcVhr6d69O88++yxr167N9BwVKlTAz8/P03o8c+ZMWrVqRVJSErGxsVx//fW8/PLLxMXFcerUKXbs2EFISAhPPPEEjRo1SjfDbcWKFT1ju73ZVVddxdKlS0lMTOTs2bMsXbqUBg0acPz4cf744w/Aubv+1VdfUb9+fQA6d+7MjBkzAJgxY4ZnnD04XeWCgoLy/o2IiMgl8/ZcvXPnTpo1a8aYMWPw9/dn7969tGvXjkmTJnlmKT98+DANGzZk27Zt7Ny5E4APPvjAU2iey9/fn6pVqzJv3jzAWRFlw4YN6fZr0KCBZ8x4dmX1WY4bN46wsDDef/99+vfvn+ENjtRx/fHx8bz00ksMHDgw3T6xsbGeov/QoUP8+OOP1K1bN8vc/d///pfx48fz+eefpxmW16FDB9atW8fp06dJTExk2bJlnqENycnJHDx4kBo1alzUZ3A51IIuIpLLwsPDadiwIUFBQVx99dU0b948x6/xwAMPcOeddxISEuJZmqRs2bLs2bOHu+66y3OX/cUXX8zyPDNnzmTQoEGcPn2aa665hvfee4/ExETuuOMOjh8/TnJyMo8//jilS5fmscceY/ny5fj4+BASEkK7du3Sna9Nmzb88MMPtG7dOsff87muvfZatm/fzokTJ6hevTozZsygTZs2PPnkkzRv3pybbrqJH3/8ke7du/PXX3/x9ddf89RTTxETE0PPnj1ZsmQJISEhAHTs2JEOHTqwb98+unTp4pl8pm3bttxzzz0AjBgxgttvv523336bWrVq8dFHH3liWbx4MePHj8/V9ysiIjnL23P10KFD2bVrl2esdFBQEPXq1WPbtm2EhIRQtGhRBg0axMCBA3nnnXe45ZZbSEpKomnTpp7cdb45c+YwaNAgnnnmGRISEujTpw+hoaFp9mndunWGY+yzktlnuXnzZmbMmMGqVasoVaoUzZo14/nnn2fUqFFpjn/++ef5+uuvSU5O5v7776dly5YA/PTTT545AzZu3Mijjz6Kj48P1lqeeOIJGjZsmGXuHjx4MMnJybRp0wZwhg28+eabBAQEMGTIECIiIvDx8aFTp06eIXirVq0iKioq3TC+3GQu1A0iL0VGRtrVq1e7HYaI5BObN2+mQYMGbofhFRITE0lMTMTX15dt27bRrl07tm3blum4rbzy888/89Zbb6WbpbWg2rdvH/369Us3yU5mMvobNsassdZG5kZ8eUk5XURAufpc3pqrL2Tw4MF0794912+2e6PBgwdz++23Z9gL4Xw5ldO9+69BRESy5cSJE7Rp04bExESstbz99ttekfAbN25MVFQUycnJ6ZY3K4j27NmT4Uz0IiIi3pqrL2TkyJGsWbPG7TBc0ahRo2wV5znJ+/8iRETkgsqVK+e1yfPcdVELuqZNm7odgoiIeClvztVZqVq1KjfffLPbYbji7rvvzvNrFvzmDBEREREREZF8QAW6iIiIiIiIiBdQgS4iIiIC8NtvbkcgIiKFnAp0ERERkalToWFDSFmXWERExA0q0EVERES6dYMSJeC559yORERECjEV6CIil6h169YsWLAgzXMTJ07kvvvuy/K4UqVKAc6a2bfddlum577QGtITJ07k1KlTnsc33XQTR44cyU7oWXrmmWdybamw06dP06pVK5KSkrJ9zOTJk3n//fezvf/69eu59tprCQwMJCQkhI8++ijL/T/99FOMMWk+75iYGM85goODiY+P59SpU3Ts2JH69esTGBjI8OHD05zn448/pmHDhgQGBnLHHXcAEBcXx4033pjt2MVFFSrA4MEwZw5s3ep2NCKSQwpqrnbTZ599xujRo/Psert27aJp06bUqVOHHj16kJCQkG6fQ4cOER0dTalSpbj//vvTvJaQkMC9995L3bp1qV+/PnPnzgWc7xfBwcGEhYURFRXFppQeVLt378bPz4+wsDDCwsIYOHCg51xt27blr7/+ysV3C1hrvWaLiIiwIiLZtWnTJlevP3nyZNuvX780zzVt2tQuW7Ysy+NKlix5wXO3atXK/vzzz1nu849//MPGxcVdONCL9PTTT9uXX345x89rrbVvvPGGnThxYq6cO9WWLVvs1q1brbXW7t2711apUsX+9ddfGe577Ngx26JFC9u0aVPP53327FkbHBxs169fb6219uDBgzYxMdGePHnSLlq0yFpr7ZkzZ2xUVJT98ssvrbXWbt261YaFhdnDhw9ba609cOCA5xr9+vWzK1asyPD6Gf0NA6utF+Tky93yZU4/cMBaPz9r+/Z1OxKRAkO5OndydU5LTEzM9r7XXnttnr6n7t2729mzZ1trrf3Xv/5l33rrrXT7nDhxwi5fvtxOmjTJDh48OM1rTz31lH3yySettdYmJSV5Yj969Khnn/nz59v27dtba63dtWuXDQwMzDCW6dOn27Fjx2b4Wk7ldLWgi0iB8NBD0Lp1zm4PPZT1NW+77Ta++OILzpw5Azh3XPft20dUVBQnTpygTZs2hIeHExwczPz589Mdv3v3boKCggCnZblnz56EhITQo0cPTp8+7dlv0KBBREZGEhgYyNNPPw3Aa6+9xr59+4iOjiY6OhqAmjVrcvDgQQDGjx9PUFAQQUFBTJw40XO9Bg0acM899xAYGEi7du3SXCcj69evp1mzZoSEhNCtWzfPXePXXnuNhg0bEhISQs+ePQFYunSp525zo0aNOH78eLrzzZo1iy5dugCwZMkSWrVqxe23307dunUZPnw4s2bNokmTJgQHB7Njxw4gbYt+69atefzxx2nSpAl169Zl+fLl6a5Rt25d6tSpA0C1atWoVKkScXFxGb6/UaNG8dhjj+Hr6+t57ptvviEkJITQ0FAAAgICKFKkCCVKlPB81ldccQXh4eHExsYCMHXqVAYPHoy/vz8AlSpV8pyva9euzJo1K8vPWbxEpUrwr3/BBx/Azp1uRyNS4ChX51yu/uSTTwgKCiI0NJSWLVsCkJSUxCOPPEJwcDAhISG8/vrrAHz33Xc0atSI4OBgBgwY4PksatasyejRo4mKiuKTTz5hx44d3HjjjURERNCiRQt+y2DizK1bt1K8eHEqVKgAQL9+/Rg0aBDR0dFcffXVLF26lAEDBtCgQQP69euX5edz9OhR6tWrx5YtWwDo1asXU6dOTXM9ay2LFi3y9GLo27cvn332Wbq4SpYsSVRUVJp8nurdd9/liSeeAMDHx8cTe5kyZTz7nDx5EmNMumPP17lzZ2bPnn3B/S6HCnQRkUsUEBBAkyZN+PrrrwGYM2cOPXr0wBiDr68v8+bNY+3atSxevJiHH34Y50ZqxiZNmkSJEiWIiYnhySefZM2aNZ7XnnvuOVavXk1MTAxLly4lJiaGIUOGUK1aNRYvXszixYvTnGvNmjW89957/PTTT6xcuZKpU6eybt06ALZt28bgwYP59ddfKVeunKebV2buvPNOXnzxRWJiYggODubZZ58F4IUXXmDdunXExMQwefJkAMaNG8ebb77J+vXrWb58OX5+fmnOlZCQwM6dO6lZs6bnuQ0bNvDqq6/yyy+/MHPmTLZu3cqqVau4++67PV8szpeYmMiqVauYOHGiJ57MrFq1ioSEBGrXrp3utXXr1rFnzx5uvvnmNM9v3boVYwzt27cnPDycl156Kd2xR44c4T//+Q9t2rTxHLN161aaN29Os2bNPH8TAJGRkRneSBAv9eijULQoPP+825GISA4oqLl69OjRLFiwgA0bNvD5558DMGXKFHbt2uXJz7179yY+Pp5+/frx0Ucf8csvv5CYmMikSZM85/H19WXFihX07NmTe++9l9dff501a9Ywbty4DIcBfP/994SHh6d57q+//mLRokVMmDCBTp06MXToUH799Vd++eUX1q9fn+nnU7ZsWd544w369evHnDlz+Ouvv7jnnnvSnPvQoUOUK1eOokWLAlC9enX27t2b6b/R+VKHE4waNYrw8HC6d+/OgQMHPK+/+eab1K5dm8cee4zXXnvN8/yuXbto1KgRrVq1SpPD/f39OXPmDIcOHcp2DBeraK6dWUQkD6XceM5zvXr1Ys6cOXTp0oU5c+bw7rvvAs4d3xEjRrBs2TJ8fHzYu3cvBw4coEqVKhmeZ9myZQwZMgSAkJAQQkJCPK99/PHHTJkyhcTERPbv38+mTZvSvH6+FStW0K1bN0qWLAnALbfcwvLly+ncuTO1atUiLCwMgIiICHbv3p3peY4ePcqRI0do1aoV4Ny17t69uyfG3r1707VrV7p27QpA8+bNGTZsGL179+aWW26hevXqac538OBBypUrl+a5xo0bU7VqVQBq165Nu3btAAgODk73ZSbVLbfckq349+/fzz//+U9mzJiBj0/a+9HJyckMHTqU6dOnpzsuMTGRFStW8PPPP1OiRAnatGlDRESEpxhPTEykV69eDBkyhKuvvtrz3LZt21iyZAmxsbG0aNGCjRs3Uq5cOSpVqsS+ffsyjVO8TLVqcPfdMGUKjBwJ//iH2xGJFBjK1X+73FzdvHlz+vXrx+233+7JiwsXLmTgwIGeYrZ8+fJs2LCBWrVqUbduXcDJ5W+++SYPpXQ96NGjBwAnTpzghx9+8OR5wNPSfq79+/dTsWLFNM916tQJYwzBwcFUrlyZ4OBgAAIDA9m9ezdhYWGZfj433HADn3zyCYMHD2bDhg3prpfRDZPstHSnSkxMJDY2lubNmzN+/HjGjx/PI488wsyZMwEYPHgwgwcP5sMPP2Ts2LHMmDGDqlWr8r///Y+AgADWrFlD165d+fXXXz0t7ql5PSAgINtxXAy1oIuIXIauXbvy3XffsXbtWk6fPu25qzxr1izi4uJYs2YN69evp3LlysTHx2d5rowSzq5duxg3bhzfffcdMTExdOzY8YLnyeruf/HixT2/FylShMTExCzPlZn//ve/DB48mDVr1hAREUFiYiLDhw9n2rRpnD59mmbNmqXrGufn55cu9nPj8fHx8Tz28fHJNLbUfbKK/9ixY3Ts2JGxY8fSrFmzdK8fP36cjRs30rp1a2rWrMnKlSvp3Lkzq1evpnr16rRq1YoKFSpQokQJbrrpJtauXes59t5776VOnTqeLzfg3NHv0qULxYoVo1atWtSrV49t27YBEB8fn643gXi5xx93fr74ortxiEiOKIi5evLkyYwdO5Y9e/YQFhbGoUOHsNamiy+r6wCeGwTJycmUK1eO9evXe7bNmzen2z+rXH5uHk99nJiYmOXnk5yczObNm/Hz8+Pw4cPprlehQgWOHDlFLTUaAAAgAElEQVTi+QxiY2OpVq1alu/pXAEBAZQoUYJu3boB0L179zQ5PVXPnj09XeeLFy/uKb4jIiKoXbs2W8+ZPDS387oKdBGRy1CqVClat27NgAED6NWrl+f5o0ePUqlSJYoVK8bixYv5/fffszxPy5YtPeOUN27cSExMDOAUmiVLlqRs2bIcOHCAr776ynNM6dKlMxzn3bJlSz777DNOnTrFyZMnmTdvHi1atLjo91a2bFn8/f09XbtmzpxJq1atSE5OZs+ePURHR/PSSy9x5MgRTpw4wY4dOwgODubxxx8nMjIyXYHu7+9PUlLSBb+0XK6EhAS6devGnXfemaYl4Pz3dvDgQXbv3s3u3btp1qwZn3/+OZGRkbRv356YmBhOnTpFYmIiS5cupWHDhgCMHDmSo0ePesYKpurataunxf/gwYNs3brV07q+detWz/hFySdq1ID+/eGdd+AiulKKiHcqiLl6x44dNG3alNGjR1OhQgX27NlDu3btmDx5sqeYPXz4MPXr12f37t1s374d+DuXn69MmTLUqlWLTz75BHAK+4xatBs0aOA5V3Zl9flMmDCBBg0aMHv2bAYMGMDZs2fTHGuMITo6mk8//RSAGTNmeOayyQ5jDJ06dWLJkiWAMx4/Naen3kgHp+Ehdf6auLg4z2ozO3fuZNu2bZ6cbq3ljz/+SDNcL6epQBcRuUy9evViw4YNnsnSAHr37s3q1auJjIxk1qxZ1K9fP8tzDBo0iBMnThASEsJLL71EkyZNAAgNDaVRo0YEBgYyYMAAmjdv7jnm3nvvpUOHDp6JZ1KFh4fTr18/mjRpQtOmTbn77rtp1KjRJb23GTNm8OijjxISEsL69et56qmnSEpKok+fPgQHB9OoUSOGDh1KuXLlmDhxomfCGj8/Pzp06JDufO3atWPFihWXFEt2ffzxxyxbtozp06d7Jq1LHQP31FNPecbqZcbf359hw4bRuHFjwsLCCA8Pp2PHjsTGxvLcc8+xadMmwsPDCQsLY9q0aQC0b9+egIAAGjZsSHR0NC+//LLn7vvixYvp2LFjrr5nyQXDh0NSEmQwB4GI5D8FLVc/+uijBAcHExQURMuWLQkNDeXuu+/mqquu8kx0+uGHH+Lr68t7771H9+7dCQ4OxsfHJ82yYeeaNWsW77zzDqGhoQQGBmY4aV7Lli1Zt27dBVvmz5XZ57N161amTZvGK6+8QosWLWjZsiVjx45Nd/yLL77I+PHjueaaazh06BB33XUXAJ9//jlPPfWUZ7+aNWsybNgwpk+fTvXq1T3Lpr344os888wzhISEMHPmTF555RUA3njjDQIDAwkLC2P8+PHMmDEDcIYypH6Gt912G5MnT6Z8+fKAM3dAs2bNPMMIcoO5mA83t0VGRtoLrSUoIpJq8+bNNGjQwO0w5CKsW7eO8ePHe8Z+FQYtW7Zk/vz5nhnez5XR37AxZo21NjKv4sstBSKnDxgAs2fDrl2QyZhUEcmacnXB8+CDD9KpUyfatm3rdih57sEHH6Rz586eeWnOlVM5XS3oIiKSZxo1akR0dLSn61hBFxcXx7BhwzIsziUfGDECEhIgpbVFRERgxIgRnDp1yu0wXBEUFJRhcZ6TVKCLSL7mTb2AJHsGDBhAkSJF3A4jT1SsWNEzy/359LebD1xzDfTqBW+9BXFxbkcjkm/p/3cFS+XKlencubPbYbji/GXgUuXk37gKdBHJt3x9fT2zlorkJ9ZaDh06hK+vr9uhyIU8+SScPg0TJrgdiUi+pFwtBV1O53Stgy4i+Vb16tWJjY0lTi1bkg/5+vqmWytevFCDBtC9O7z+OjzyCKRMFCQi2aNcLYVBTuZ0Fegikm+lrjktIpKrRo6Ejz+GV1+FZ591OxqRfEW5WuTiqIu7iIiISFaCg6FbN6dAP3rU7WhERKQAU4EuIiIiciEjRzrF+euvux2JiIgUYCrQRURERC4kPBxuvtmZLO74cbejERGRAkoFuoiIiEh2jBoFhw87y66JiIjkAhXoIiIiItnRpAm0bw+vvAInT7odjYiIFEAq0EVERESya9QoiIuDKVPcjkRERAogFegiIiIi2dW8OURHw0svwenTbkcjIiIFjAp0ERERkYvx1FPwxx/wzjtuRyIiIgWMCnQRERGRi9GqFURFwQsvwJkzbkcjIiIFiAp0ERERkYthjNOKvncvTJ/udjQiIlKAqEAXERERuVht20LTpvD883D2rNvRiIhIAaECXURERORiGePM6P777zBzptvRiIhIAaECXURERORS3HQThIfDv/8NiYluRyMiIgWACnQRERGRS5Hair5jB8ye7XY0IiJSAKhAFxEREblUnTtDSAg89xwkJbkdjYiI5HMq0EVEREQulY8PjBwJW7bAp5+6HY2IiORzKtBFRERELsett0KDBjBmDCQnux2NiIjkYyrQRURERC5Haiv6r7/CZ5+5HY2IiORjKtBFRERELlePHlCnjtOKbq3b0YiISD6lAl1ERETkchUpAk8+CevXwxdfuB2NiIjkUyrQRURERHLCHXdArVpqRRcRkUumAl1EREQkJxQrBk88AT//DAsWuB2NiIjkQyrQRURERHJK375QowaMHq1WdBERuWgq0EVERERyyhVXwPDh8OOPsGiR29GIiEg+owJdREREJCcNGADVqjlj0UVERC6CCnQRERGRnOTrC489BkuXwrJlbkcjIiL5iAp0ERERkZx2zz1QqZJa0UVE5KKoQBcRERHJaSVKwKOPwsKFznh0ERGRbFCBLiIiIpIbBg6EgAC1oouISLapQBcRERHJDaVKwcMPw1dfwerVbkcjIiL5QK4X6MaYIsaYdcaYL3L7WiIiIpJ7jDFDjTG/GmM2GmNmG2N83Y7J6w0eDP7+MHas25GIiEg+kBct6A8Cm/PgOiIiIpJLjDFXAkOASGttEFAE6OluVPlAmTLw4IMwfz5s2OB2NCIi4uVytUA3xlQHOgLTcvM6IiIikieKAn7GmKJACWCfy/HkD0OGOIW6WtFFROQCcrsFfSLwGJCc2Q7GmHuNMauNMavj4uJyORwRERG5FNbavcA44H/AfuCotfabc/dRTs+Evz888ADMnQu//up2NCIi4sVyrUA3xtwM/GmtXZPVftbaKdbaSGttZMWKFXMrHBEREbkMxhh/oAtQC6gGlDTG9Dl3H+X0LAwd6iy99txzbkciIiJeLDdb0JsDnY0xu4E5wPXGmA9y8XoiIiKSe9oCu6y1cdbas8D/Ade5HFP+ERDgTBj30UewZYvb0YiIiJfKtQLdWvuEtba6tbYmziQyi6y1fS5wmIiIiHin/wHNjDEljDEGaIMmgb04Dz8MxYvDv//tdiQiIuKltA66iIiIXJC19ifgU2At8AvOd4gprgaV31SqBAMHwqxZsHOn29GIiIgXypMC3Vq7xFp7c15cS0RERHKHtfZpa219a22Qtfaf1tozbseU7zz6KBQtCs8/73YkIiLihdSCLiIiIpJXqlaFu++G6dPh99/djkZERLyMCnQRERGRvPT442AMvPii25GIiIiXUYEuIiIikpdq1ID+/eGdd2DvXrejERERL6ICXURERCSvPfEEJCfDSy+5HYmIiHgRFegiIiIiea1mTfjnP2HKFPjjD7ejERERL6ECXURERMQNI0ZAQgKMG+d2JCIi4iVUoIuIiIi44Zpr4I47YNIkiItzOxoREfECKtBFRERE3PLkk3D6NIwf73YkIiLiBVSgi4iIiLilfn24/XZ44w04fNjtaERExGUq0EVERETc9OSTcOIEvPqq25GIiIjLVKCLiIiIuCk4GLp1cwr0o0fdjkZERFykAl1ERETEbaNGOcX566+7HYmIiLhIBbqIiIiI2xo1gptvhgkT4Phxt6MRERGXqEAXERER8QajRjkTxb31ltuRiIiIS1Sgi4iIiHiDJk2gfXt45RU4edLtaERExAUq0EVERES8xahREBcHb7/tdiQiIuICFegiIiIi3qJ5c7j+enj5ZTh92u1oREQkj6lAFxEREfEmo0bBH3/AtGluRyIiInlMBbqIiIiIN2nVClq0gBdfhDNn3I5GRETykAp0EREREW9ijNOKvncvTJ/udjQiIpKHVKCLiIiIeJu2baFpU3j+eTh71u1oREQkj6hAFxEREfE2xsBTT8Hvv8PMmW5HIyIieUQFuoiIiIg36tABIiLguecgMdHtaEREJA+oQBcRERHxRqlj0XfuhNmz3Y5GRETygAp0EREREW/VuTOEhDit6ElJbkcjIiK5TAW6iIiIiLcyBkaOhC1b4JNP3I5GRERymQp0EREREW92663QsCGMHQvJyW5HIyIiuUgFuoiIiIg38/GBJ5+EX3+FefPcjkZERHKRCnQRERERb9ejB9St67SiW+t2NCIikktUoIuIiIh4uyJFYMQIWL8evvjC7WhERCSXqEAXERERyQ/uuANq1YLRo9WKLiJSQKlAFxEREckPihVzWtFXr4YFC9yORkREcoEKdBEREZH84s474aqr1IouIlJAqUAXERERyS+uuAKGD4cff4RFi9yORkREcpgKdBEREZH8pH9/qFYNxoxxOxIREclhKtBFRERE8hNfX3jsMVi6FJYtczsaERHJQSrQRURERPKbe+6BypXVii4iUsCoQBcRERHJb0qUgEcegYULYdgwOHbM7YhERCQHqEAXERERyY/uvx/uvhsmToR69WDmTM3sLiKSz6lAFxEREcmPfH1h6lRYuRJq1HCWYGvRAtavdzsyERG5RCrQRURERPKzJk2cIn3aNNiyBSIinNb1w4fdjkxERC6SCnQRERGR/M7HB+66C7Zuhfvug0mTnG7v06ZBcrLb0YmISDapQBcREREpKPz94fXXYe1aaNDAme29WTNYtcrtyEREJBtUoIuIiIgUNKGhzjrps2ZBbCw0bepMKPfnn25HJiIiWVCBLiIiIlIQGQN33OGMS3/kEZgxA+rWdVrYExPdjk5ERDKgAl1ERESkICtdGl5+GWJioHFjGDIEwsNh2TK3IxMRkfOoQBcREREpDBo0gG++gU8/hSNHoFUr6NMH9u1zOzIREUmhAl1ERESksDAGbr0VfvsNRo50ivV69WDcOEhIcDs6EZFCTwW6iIiISGFTogSMGQO//grR0fDoo87Ect9+63ZkIiKFmgr0y7B6tZYWFRERkXysdm34/HP44gs4exbatYPbboPff3c7MhGRQkkF+iXauhWuuw46dtSKJSIiIpLPdewIGzfCc8/Bl18649XHjoX4eLcjExEpVFSgX6I6deDVV2HxYqdH2KJFbkckIiIichl8fWHECGd8eseOMGoUBAbCf/7jdmQiIoWGCvRLZAwMGgQ//QRly0Lbtk4e07KiIiIikq9ddRV88gksXAjFi0PnznDzzbB9u9uRiYgUeCrQL1NoKKxZA337Oj3BoqNhzx63oxIRERG5TG3awIYN8MorzprpgYHOzO8nT7odmYhIgaUCPQeULAnvvQczZ8L69RAW5sy3IiIiIpKvFSsGw4bBli3Qo4czRr1BA2d5Nmvdjk5EpMBRgZ6D+vSBtWvhH/+ALl3gwQfhzBm3oxIRERG5TFWrwvvvw/LlUL48dO8ON9wAmza5HZmISIGiAj2H1akDP/4IQ4bAa6/BtdfCtm1uRyUiIiKSA6KinHVm33zTGeMXGgqPPALHjrkdmYhIgaACPRcUL+7M8D5/PuzeDeHhMGuW21GJiIiI5ICiReG++5w1Z/v3h/HjoV49+OADdXsXEblMKtBzUefOztwqYWFO9/f+/TWvioiIiBQQFSvClCnOkjZXXQX//Ce0bOlMyCMiIpdEBXouq1HDWSt95EiYMQMiIyEmxu2oRERERHJI48bO+L533nHWUI+IgPvvh8OH3Y5MRCTfUYGeB4oWhTFjnOVEjxyBJk1g0iT1AhMREZECwscHBgxwur0PHux80alXD6ZNg+Rkt6MTEck3VKDnoeuvd7q8t27tDN3q3t0p2EVEREQKBH9/Z5bctWud5djuuQeaNYNVq9yOTEQkXyjqdgCFTaVK8OWX8MorMGKEMxHqnDlO7hIREREpEEJDYelSmD3bmeW9aVO46y54/nln7HpuSEqC06cz3+Ljs349qy0hwXlPHTo4y8v5++fOexCRQs9YL+pnHRkZaVevXu12GHnmp5+gZ0+IjYXnnnPyl4/6NIiIFGrGmDXW2ki347hchS2nSxaOH3fG+k2YAKVKwdNPO63rF1skX6jAPnv20mMsXhz8/JzN1/fv31M3Hx/ni9uRI87vzZo5xfqNNzrL9egLnIhk4FJyeq4V6MYYX2AZUBynpf5Ta+3TWR1TGJP5kSNO769PP4X27eH9951WdhERKZy8uUA3xpQDpgFBgAUGWGt/zGjfwpjT5QJ++w0eeMCZlCcrRYqkL5AvZsuowL7Q/tkpsBMTna76X30FX3/tdIME54tb+/ZOsd6uHVSocPmflYgUCN5WoBugpLX2hDGmGLACeNBauzKzYwprMrfWWaXkoYegXDlnGdE2bdyOSkRE3ODlBfoMYLm1dpox5gqghLU2w9lUCmtOlwuw1hmffuZM5gVz0XwyAvPPP2HBAqdYX7AADh0CY5zZgG+80Wlhj4x0bjiISKHkVQV6mosYUwKnQB9krf0ps/0KezL/5Rfo0cO5wTxiBDzzTP7JUSIikjO8tUA3xpQBNgBX22x8eSjsOV0KmaQkp0X966+dFvZVq5ybEQEBTqt6hw5OK7u6SYoUKpeS03N1wIwxpogxZj3wJ/BtRsW5MeZeY8xqY8zquLi43AzH6wUHw88/O6uUPPecM9v7//7ndlQiIiIAXA3EAe8ZY9YZY6YZY0qeu4NyuhRaRYo4E+E9/TSsXOm0rn/4Idx0k9Ol/847oXJlZ834UaPghx+cLvMiIufJqxb0csA84AFr7cbM9tPd9r/Nng333gvFisG770LXrm5HJCIiecGLW9AjgZVAc2vtT8aYV4Fj1tpRGe2vnC6SIjkZ1q1zWta/+sop4JOTnZngb7jh79b1qlXdjlREcpjXtaCnShmftgS4MS+uVxD06uX8v7xWLejWDYYMcYZriYiIuCQWiD2nN9ynQLiL8YjkDz4+EBEBI0fC99/DwYPw0UdO68vy5dC/P1SrBo0aOWMcly27vBnpRSRfy7UC3RhTMaXlHGOMH9AW+C23rlcQXXON0wNq6FB4/XW49lrYutXtqEREpDCy1v4B7DHG1Et5qg2wycWQRPInf3+4/Xani+TevU6LzPPPQ5ky8NJL0KqVMxP8bbfBtGnOerwiUmjkZgt6VWCxMSYG+BlnDPoXuXi9Aql4cRg/Hv7zH/j9d2epzQ8+cDsqEREppB4AZqXk9jDg3y7HI5K/GQNhYTB8OCxd6swEP3euU8CvXOmsxVujBoSEwGOPweLFkJDgdtQikovyZAx6dmm8WtZiY+GOO5zeUH37whtvQKlSbkclIiI5yVvHoF8s5XSRy2Qt/Prr32PXV6xwur6XKuWsx9uhg7NddZXbkYpIJrx2DLrkjOrVYdEiZ4LQ9993ltbcsMHtqEREREQkxxkDQUHw6KPOF8BDh+Czz6B3b6db/MCB8I9/QGAgPPywM1u8JiwSyfdUoOczRYs666MvWgTHjjkrerz1lnOTVUREREQKqNKloUsXmDwZdu+GTZvglVecCebeeMOZEb58eejUyflyuHOnviCK5ENF3Q5ALk3r1k7red++MHgwfPedM4+Iv7/bkYmIiIhIrjIGGjRwtmHD4ORJZ3x6anf4L86Z9snXF/z8nJ/nb5k9fynHZPS8j0ttgcnJznCAhASnV0FCQvrtYp6/0L5XXOFM7FehAgQEpP89IMBZO1kkG1Sg52MVKzr//50wwZlbJCwM5sxxZnsXERERkUKiZEm4+WZnsxa2bYNvv4UDByA+PuPt9Gnn57FjGT9/+rRT6F6OYsUuvqiHyy+sExMv/zM9nzHO7M1XXJF+O3PGWT7v5MnMjy9T5u/CPbNCXkW9oAI93/PxcYYdtWgBPXs6P8eOdSb6dOumpYiIiIi4xBioW9fZLldiYvrCPatiP7vPp7525Ej654xxit7MiuHSpTN+PrP9c+r5otkom+LjnbkCDh78+2dGvx844EwAeOgQnDiR+fnOLeozK+hV1Bc4KtAvx/79UKWK8z8SlzVp4swXcu+98MQTzhj1mTOhcmW3IxMRERGRfKloUWfWeC0blD2+vnDllc6WXalFfVYF/cGD8OefsHmz8/vlFvUBAU6RUL26c8NDct6JE84SXJdABfql2rrVqYrHjIEHHnA7GgDKlnW6uLdtC0OGQGios2Z627ZuRyYiIiIiIulcSlF/5syFC/qDByEu7sJFfZkyTqGe1VaunFc0SHqNY8ec4nvPHudn6nbu46NHL/n0KtAvVZ06EBXlLH3RqhWEhLgdEeD8t3PPPc449B49oF07Z3z66NHZ65kjIiIiIiJerHhxZ/b+atWyf8y5Rf2hQ05P4L170xaYGzc6z58/+3+JEumL9ho10j4OCMj/Rby1TmGdUcF97uPjx9MeZ8zfPRLq1IHrr//7c+nd+6LDMNaLll+IjIy0q1evdjuM7IuLcwpzf39Yvdr54/Uip07BQw/B1Klw3XUwezZcdZXbUYmISFaMMWustZFux3G58l1OFxERZ/b7P/5IW5ieX6Tu2wdJSWmPK178wi3xlSq5N0mWtfDXXxm/n3Mfn9/TwBioWjXrmxPVqjnzFGTgUnK62lQvR8WK8P77TjP1sGHOupRepEQJmDIF2rRxWtVDQ+G996BrV7cjExERERERr1OsmFOA1qiR+T5JSc5Ed5kV8T/84Pw8ezbtcUWLOl35syriq1S5+G6/1sLhw1l3OY+NdVovz+Xj4xTfNWpAUBDceGP6Arxq1TyfeE8t6Dnhscfg5Zdh7ly45Ra3o8nQjh3OLO+rV0Pz5nDDDc7Y9CZNNNmjiIg3UQu6iIjke8nJTnf6zIr41AI6Pj7tcalFc0bFe4kSmRfg55+nSBGnZTujFu/Ux5dyM+AiXUpOV4GeExISnKp3xw7YsCHrO04uSkiAV16BefOcQt1aZ+LG1q2dYr1tW2jQIP8PHxERyc9UoIuISKGQWbfz84vw87udn98Sn1EBXrmyU6S7TAW6m7Ztg/BwZ1u0yCv+ILJy+DAsWQLffgsLF8L27c7z1ar9Xay3aXNxc0+IiMjlU4EuIiJyjtSJ206d+nssu5fXWqk0Bt1NderAm29C377w73/DqFFuR5Sl8uWd3vipPfJ373YK9YUL4csvnaH1AIGBfxfsrVppqUQREREREclDZcs6WyGhFvScZC306QMffQRLlzrd3vOh5GSIifm7dX3ZMmdYR9Gi0KzZ3wW7xq+LiOQ8taCLiIgUDOri7g2OHYOwMGd2ww0boFw5tyO6bPHxzmSMqS3sGY1fv+EGqF9f49dFRC6XCnQREZGC4VJyuksL0RVgZcrAhx/C3r3wr385lWw+5+sL11/v9NxftcqZkPHTT6F3b9i8GR58EBo2dIaE9O0LM2fC/v1uRy0iIiIiIpK/qEDPDc2awZgx8PHHzsLjBUz58nDrrTBpkjM33q5dMHUqtGjhjF+/805ncrmgIHjoIfjiCzh+3O2oRUREREREvJu6uOeWpCSn3/dPP8HatVCvntsR5YnkZKdnf2p3+IzGr99wAzRurPHrIiIZURd3ERGRgkFd3L1JkSJOX28/P+jVC86ccTuiPOHjA40awaOPwoIFztKG333nPI6Ph2efdebOCwiAzp3htdecbvJedJ9IRKTAMMZEGWP6p/xe0RhTy+2YREREJHMq0HPTlVfCO+/AunXwxBNuR+OKc8ev//xz2vHrmzZp/LqISG4xxjwNPA6kJqBiwAfuRSQiIiIXonXQc1uXLnDffTBhArRrBzfe6HZErkodv37rrc7jXbucFvaFC+G//7249detdbrUnz0LiYl/bzn9+FLPUaYMXHMN1K7991aiRN591iJS6HUDGgFrAay1+4wxGfzfVERERLyFCvS8MG6cMxi7b19ngfHKld2OyGvUqgV33+1s545f//ZbePttePVVZ/y6v3/GxbBbfHycuFK3YsXSPi5aFA4fdrr4n6tq1bRF+7m/ly/vznsRkQIrwVprjTEWwBhT0u2AREREJGsq0POCnx/MmQORkU6R/uWXToUnaaSOX08dw566/vp33zmFbmbFcE4/zs4+2f3nO3wYduxwtu3b//59wYL0Xfn9/dMX7am/V62qNeZF5KJ9bIx5GyhnjLkHGABMdTkmERERyUK2ZnE3xjwIvAccB6bhdJkbbq39JieDKfAzvk6a5HR3f+UVGDbM7WjEZSdPws6dGRfwv//uLASQqkQJuPrqjAv4q65ybhqISMGQk7O4G2NuANoBBlhgrf02J86bHQU+p4uIiFzApeT07H6tH2CtfdUY0x6oCPTHKdhztEAv8AYOhG++geHDoXVrCA93OyJxUcmSEBzsbOc7e9Yp0s8t2rdvd9adX7DA6V2QqmhRqFkzfav7Ndc4Qwj8/PLsLYmIlzDGFMEpyNsCeVaUi4iIyOXJboGe2rn2JuA9a+0GY9Th9qIZA9OmQWgo9OzprI9eqpTbUYkXKlbMKbCvuSb9a8nJsG9f+pb37dth5Uo4ejTt/ldembZoP3fSunLl8ub9iEjestYmGWNOGWPKWmuPXvgIERER8QbZLdDXGGO+AWoBT6TMApuce2EVYAEB8MEHztpjQ4bAu++6HZHkMz4+zrJ01as7M9yfy1pn3Pv5Le87djiz5B84kHb/gIC0hXuVKhcej5/Zlp39Uvfx8clfY+qTk50hB1ltiYkX3id1P2OcYQvnb76+mp5CclQ88Isx5lvgZOqT1toh/9/enUfZUdb5H39/051OOhskJLIEQliUTQhgBDQjIoiCoOCCsv4Y1MSHLvQAACAASURBVGGOwyCOIgriijKcQVEZReGwyhL1JwiMMAqCAupP2ffFBRACEWIIdkiAJJ3n98fTfbqT3m53um5V336/znnOrVu37q3vLUiefO5T9VR5JUmSpP7UGtA/AuwMPJ5SWh4R08inuWso9toLTjkFvvY1eOc74UMfKrsiNYiIHLo32AB2373n6y+9lK97XzvA/+53eR7D1XX82W1dgn737SKGHphrDdU1TNUxbFpbc1jvfCyitbSMrB9INGTXdTRJkjRC1BrQ3wTcm1JaFhFHArsC3y6urFHgi1/M05Mfe2xOUrNnl12RRoFJk2CnnXJb24oVebb89vbe7wXfWytju5Urc3v55fx89Wpoaupqzc05gLa2rrl+7dbc3P/rg9lusJ+VUq5/+fLaWue2f/97z9e6z0dQqzFjBhfoW1vznAlTpsB66/X9OG7cuv8/2shSyj+StbXlS1E6H7svt7UN5/7SJRHRAryuY9VjKaWVw7cHSZI03GoN6N8D5kTEHOAk4ALgB8Bb+32X+jZ2LFxxBey8Mxx+eL5PulNxq0QtLbDhhmVXocFqb88hvdawP1BbsgSeeabn+pU1xLqWlv4DfG+Pa6+bPLmafxWuWNF3oK51XVvbwGepDOeZDRGxF3AJ8CR5LpnNIuLolNKtw7cXSZI0nGr9Z9CqlFKKiIOAb6eULoiIo4ssbFTYYgv4/vdzQP/yl+G008quSNII09SUR7cnTix2P50Bde0QOtDjk0+u+bz77QP7Usto/UCPEyfmsLt6dW2j1gOtq+VMhXHj8r6717H11j1r663ezuVJk/J/02HyDeAdKaXHACLidcB84A3DtgdJkjSsag3oSyPiZOAo4C0dt28ZW1xZo8hhh+Vbr33ta/D2t/ec9UuSKqClBaZPz22oOk/tH0zA73x85pmu50uXDryvztP4ly0beA6BiJ6BecaMHK77C9Pdl6dMqeQp/mM7wzlASumPEWHfLUlShdUa0D8EHE6+H/rfImIWcGZxZY0y//3f8NvfwpFHwn33wbRpZVckScOu++z1G2889M9pb8+j4gMF+2XL8oj0QCPXEyc27Oz5d0bEBcClHc+PAO4qsR5JkjSAmgJ6Ryi/HHhjRBwI3J5S+kGxpY0ikybB/PnwpjfBRz8KV17pFMuS1Iempq5wrX59DDgO+Dj5GvRbgXNKrUiSJPWrpjGDiPggcDtwCPBB4A8R8YEiCxt13vAGOP10+OlP4bzzyq5GkjTyNZPnjXlfSum9wNnA8F3hLkmShl2tJ/V9DnhjSunolNL/AXYDPl9cWaPUJz8J73gHfOIT8NBDZVcjSRrZbgJauz1vBX5ZUi2SJKkGtQb0MSml57s9XzyI96pWY8bAJZfkiyMPO2xoNziWJCkbn1J6qfNJx/KEEuuRJEkDqDVk/zwifhER/xwR/wxcB1xfXFmj2EYbwcUXwwMPwKc/XXY1kqSRa1lE7Nr5JCLmAi+XWI8kSRpArZPEfToi3g/MI080c15K6aeFVjaa7b9/Ps39W9/Kp7y/+91lVyRJGnlOAP5vRDwLJGAT8l1ZJElSRdV6mzVSSlcCVxZYi7o74wz49a/hmGPg/vthk03KrkiSNLJsAewCzALeC+xBDuqSJKmi+j3FPSKWRkRbL21pRLTVq8hRady4fOu1l1+Go47KN/6VJKl2n08ptQHrA/sC5wHfK7ckSZLUn34DekppckppSi9tckppSr2KHLW23RbOPhtuvhnOPLPsaiRJI0vnL7sHAN9PKV0DtJRYjyRJGoAzsVfdhz8MhxwCn/883H572dVIkkaOZyLiXOCDwPURMQ77fUmSKs2Ouuoi4Lzz8jXohx0GbV5ZIEmqyQeBXwD7pZReBKYB3h5EkqQKM6CPBOuvD1dcAU8+CccdV3Y1kqQRIKW0PKV0VUrpTx3PF6aUbii7LkmS1DcD+kgxbx588Ytw2WVw6aVlVyNJkiRJGmYG9JHkc5+Dt7wF/u3f4M9/LrsaSZIkSdIwMqCPJE1NeQS9uRkOPxxWrCi7IkmSJEnSMDGgjzSzZsH558Mdd8AXvlB2NZIkSZKkYWJAH4ne/3449lj4r/+Cm24quxpJkiRJ0jAwoI9U3/wmbLstHHUULFpUdjWSJEmSpHVkQB+pJkyA+fNh8WI45hhIqeyKJEmSJEnrwIA+ks2ZA2eeCdddB9/5TtnVSJIkSZLWgQF9pDv+eDjgAPj0p+H++8uuRpIkSZI0RAb0kS4CLroIpk6FQw+F5cvLrkiSJEmSNAQG9EYwYwZceik8+ih88pNlVyNJkiRJGgIDeqN4+9vzae7nngtXXVV2NZIkSZKkQTKgN5LTToO5c+GjH4Wnny67GkmSJEnSIBjQG0lLS7712sqVcOSR0N5edkWSJEmSpBoZ0BvN1lvDOefArbfC6aeXXY0kSZIkqUYG9EZ01FFwxBHw5S/D735XdjWSJEmSpBoY0BvVOefA5pvD4YfDiy+WXY0kSZIkaQCFBfSI2CwifhURj0TEQxFxQlH7Ui+mTIErroBnnoF//VdIqeyKJEkjXEQ0RcQ9EfGzsmuRJKkRFTmCvgr4VEppO2AP4LiI2L7A/Wltu+8OX/kK/PjHcNFFZVcjSRr5TgAeKbsISZIaVWEBPaW0MKV0d8fyUnKHPrOo/akPJ50Ee+8Nxx8Pjz1WdjWSpBEqIjYFDgDOL7sWSZIaVV2uQY+I2cAuwB96ee3YiLgzIu5ctGhRPcoZXZqa4NJLobUVDjsMXn217IokSSPTt4CTgNV9bWCfLknSuik8oEfEJOBK4BMppba1X08pnZdSmptSmjtjxoyiyxmdNtkkn+J+zz1wyillVyNJGmEi4kDg+ZTSXf1tZ58uSdK6KTSgR8RYcji/PKV0VZH70gDe/W447jg46yz4xjdg2bKyK5IkjRzzgPdExJPAD4G9I+KyckuSJKnxFDmLewAXAI+klM4qaj8ahDPPhH32gRNPhFmz4HOfg4ULy65KklRxKaWTU0qbppRmA4cCN6eUjiy5LEmSGk6RI+jzgKPIv7Lf29HeVeD+NJDWVrjxRvjNb+Ctb4X//E+YPRuOOQYeeKDs6iRJkiRpVCtyFvffpJQipbRTSmnnjnZ9UftTjSJg3jy46ir44x/hX/4l34Ztp53gne+EG27wnumSpD6llH6dUjqw7DokSWpEdZnFXRW19dbwne/A00/D6afnUfR3vjOH9YsucsZ3SZIkSaojA7pg2jQ4+WR44gm4+OI8yv7hD+fT37/2NVi8uOwKJUmSJKnhGdDVZdw4OPpouO++fKr7nDlw6qmw2WZ5Bvg//ansCiVJkiSpYRnQ1VME7Lsv/Pzn+bT3ww6D88+HbbaBgw+G227zOnVJkiRJGmYGdPXv9a+HCy6Av/4135bttttgzz1h993hRz+CVavKrlCSJEmSGoIBXbXZaCM47bQ8odw558CLL8Khh+aJ5r75TWhrK7tCSZIkSRrRDOganAkT4GMfg0cfhauvhlmz4JOfzNepn3hiDvCSJEmSpEEzoGtoxoyBgw6CW2+F22+Hd70LvvUt2GILOPxwuOuusiuUJEmSpBHFgK5198Y3wvz58Je/wAknwM9+BnPnwl57wbXXwurVZVcoSZIkSZVnQNfw2Xxz+MY38mnuX/86PP54HmXfbjv4/vdh+fKyK5QkSZKkyjKga/ittx586lN5RH3+fJgyJV+3PmsWfOEL8NxzZVcoSZIkSZVjQFdxxo7NM73ffjvccgvMmwdf/Woeaf/oR+Hhh8uuUJIkSZIqw4Cu4kXke6dfc02e/f2YY+CKK2CHHfLkcr/8JaRUdpWSJEmSVCoDuurrda+D730Pnnoq31f97rth331hl13gBz+AFSvKrlCSJEmSSmFAVzmmT4dTT4Unn4QLLoBVq+Doo/Nt2s44A5YsKbtCSZIkSaorA7rKNX48fPjD8MAD8L//C9tvDyefDJtuCscfnyeakyRJkqRRwICuaoiA/faDG2+Ee++FQw6Bc8+F174W3v9++J//gba2squUJEmSpMIY0FU9c+bAxRfn098/+1n41a/gPe+BadNgjz3glFPyxHLeV12SJElSAzGgq7o22QROPx2efRZuvjmf+t7UBGeemSeWmzoV9toLvvIV+O1vnWBOkiRJ0ohmQFf1jR8Pb3tbnvX9t7+FF16A66+Hj38cli6FL30J/umf8gj7/vvnAH/XXdDeXnblkiRJklSz5rILkAZt8uQcxPffPz9fvBhuuSWPst98M5x0Ul6//vp5hH3vvWGffWC77fK17pIkSZJUQQZ0jXwbbADve19uAAsX5uvWb74ZbroJrr46r99wwxzWO9sWWxjYJUmSJFWGAV2NZ+ON4fDDcwN44omu0fWbb4b58/P6zTfvGl1/29vyNe+SJEmSVBIDuhrfFlvARz6SW0rw6KNdYf3qq+Gii/J2227bNbq+1155ZF6SJEmS6sSArtElIl+Lvt12cNxxeSK5++7rCuyXXALnnJO3mzOnK7C/5S0wZUrZ1UuSJElqYAZ0jW5NTbDrrrmdeCKsXAl33NEV2L/zHTjrrLzdbrt1BfY3vQlaW8uuXpIkSVID8TZrUndjx8Kb3wynnpoD+osvwi9/CZ/5TD49/owz8jXrU6fmoP7Vr8LvfpeDvSRJkiStA0fQpf60tuZAvs8++XlbG9x2W9cM8Z//fG6TJsGee3aNsM+ZA2P8/UuSJElS7Qzo0mBMmQIHHJAbwN//Dr/+ddcp8ddfn9dPmwZ77AE77QQ77pgft9kmj9BLkiRJUi8M6NK6mD4dPvCB3ACeeabrHux33gk33ACrVuXXxo7Nk9N1D+077ZRvC+f92CVJkqRRz4AuDaeZM+HII3MDWLECHnsM7r8/twceyAH+ssu63jNtWs/QvsMOMHFiOd9BkiRJUikM6FKRWlpy8N5xRzjiiK71L7yQw/oDD3QF9wsvhGXL8usRsNVWa4b2HXeELbfMM8pLkiRJajgGdKkM06bBW9+aW6fVq+GJJ9YM7fffD1dfnWeQB5gwIY+udw/tO+6YT7WXJEmSNKIZ0KWqGDMmj5pvtRUcfHDX+uXL4eGH1wzt11wDF1zQtc3GG/c8TX7bbWHcuPp/D0mSJElDYkCXqm7CBJg7N7dOKcFzz/UcbT/77HzdO0Bzc545fu3T5DfbzEnpJEmSpAoyoEsjUQRstFFu++7btX7VKvjTn9YM7b//Pfzwh13brLdez9D++tfnW8hJkiRJKo0BXWokzc35Vm7bbQcf+lDX+rY2ePDBNWeTv+yyvL7T7Nn5fbNnw+abr/m44YaOukuSJEkFM6BLo8GUKfDmN+fWKSV4+uk1R9v/+Ee4/XZYvHjN948bl8P62sG9c90mmzi7vCRJkrSODOjSaBUBs2blduCBa7720kvw17/Ck0/2fLz22nz9e3fNzfna9t5G3zffHDbdFMaOrce3kiRJkkYsA7qkniZNyrdz22GH3l9/+WV46qneA/yNN8Kzz3bdGg7yDPUzZ6456t49xM+a5YzzkiRJGvUM6JIGr7U1zxC/zTa9v75iRT59vrcAf9ttMH8+tLev+Z6NN+599L3zccKEIr+RJEmSVDoDuqTh19LSdU/33qxaBc880/tp9HfcAVdeCStXrvme6dN7D++bbgqveQ3MmJH3K0mSJI1QBnRJ9dfc3HWq+5579nx99WpYuLD3AP/gg3DddfDKKz3fN2VKDuozZnSF9u5t7XXjxxf9TSVJkqSaGdAlVU/nNeszZ64583ynlOD553NgX7AAFi1asz3/fA70d9yRn69a1ft+Jk3qPcj3FfA9zV6SJEkFMqBLGnki8r3ZN9wQdtut/21Tgn/8I4f23oJ85/KCBXDPPXl5xYreP2vixNrD/Gtek7eXJEmSamRAl9TYImD99XN73esG3j4laGvrP8wvWgR/+1u+d/yiRfDqq71/Vmtr78F9+nTYYIPcpk1bc9nT7iVJkkYtA7okdRcB662X29ZbD7x9Svm+8f2F+c51Dz+cl19+ue/PmzChZ2jvLciv/djsX+eSJEkjnf+ik6R1EQGTJ+e25Za1vWfZMnjhBVi8uOfj2usefLBree1b03U3ZcrgQv0GG+QfIcaMGZ7jIEmSpHVmQJekeps4MbfNNqv9PZ2n3g8U6juX//KX/LhkSd+fOWYMTJ1aW6ifOjWfsj9+fFcbNy4/OnovSZI0LPxXlSSNBN1Pva91pB7yqPuSJQOP1C9eDM8+2zVi/9JLte+jqalnaF+7Fbl+3DjPBJAkSQ3BgC5JjaypKU9KN3364N736qs52HcG+CVL8r3nX3klv9a53L31t76tre9tV65c9+85dmzP4N7amn/QWH/9fAZA52SBncu9rZs8Of8YIkmSVAIDuiSpp3HjYKONcitae3tXYB9K+O9r3fLl+RZ7Tz2VZ9x/8cX8vD9jxgwc6Pt7bdy44o+XJElqWAZ0SVK5mpry7PUTJhS/r/b2PJq/ZEkO7C++2LXc17pnn+1afuWV/j9//PjBBfruy07aJ0nSqGdAlySNHk1NORBPnTq097/ySleI7y/Ud6577jl47LGudatX9/3ZEXk2fkmSNGoZ0CVJqtX48UM/9T+lPPneQCP2Z589/HVLkqQRwYAuSVI9RORJ6CZPhlmz+t7OgC5J0qjlxW6SJEmSJFWAAV2SJEmSpAowoEuSJEmSVAEGdEmSJEmSKsCALkmSJElSBRjQJUmSJEmqAAO6JEmSJEkVYECXJEmSJKkCCgvoEXFhRDwfEQ8WtQ9JklQfEbFZRPwqIh6JiIci4oSya5IkqdEUOYJ+MbBfgZ8vSZLqZxXwqZTSdsAewHERsX3JNUmS1FAKC+gppVuBF4r6fEmSVD8ppYUppbs7lpcCjwAzy61KkqTGUvo16BFxbETcGRF3Llq0qOxyJEnSACJiNrAL8Ie11tunS5K0DkoP6Cml81JKc1NKc2fMmFF2OZIkqR8RMQm4EvhESqmt+2v26ZIkrZvSA7okSRoZImIsOZxfnlK6qux6JElqNAZ0SZI0oIgI4ALgkZTSWWXXI0lSIyryNmvzgf8HbBMRCyLiI0XtS5IkFW4ecBSwd0Tc29HeVXZRkiQ1kuaiPjildFhRny1JkuorpfQbIMquQ5KkRuYp7pIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqwIAuSZIkSVIFGNAlSZIkSaoAA7okSZIkSRVgQJckSZIkqQIM6JIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqwIAuSZIkSVIFGNAlSZIkSaoAA7okSZIkSRVgQJckSZIkqQIM6JIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqwIAuSZIkSVIFGNAlSZIkSaoAA7okSZIkSRVgQJckSZIkqQIM6JIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqwIAuSZIkSVIFGNAlSZIkSaoAA7okSZIkSRVgQJckSZIkqQIM6JIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqwIAuSZIkSVIFGNAlSZIkSaoAA7okSZIkSRVgQJckSZIkqQIM6JIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqwIAuSZIkSVIFGNAlSZIkSaoAA7okSZIkSRVgQJckSZIkqQIM6JIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqwIAuSZIkSVIFGNAlSZIkSaoAA7okSZIkSRVgQJckSZIkqQIM6JIkSZIkVYABXZIkSZKkCjCgS5IkSZJUAQZ0SZIkSZIqoLnsAiRJkiRlq1fDypVrthUrel8e6LXVq7s+N6Xel+v9Wn/bAURAU9OabcyYnusGakN5T3/vi2DIUoL29vzfo719zeVa1xXxnlraYLZdl/fU8r6U8n+HMWNy6215oNcHs+1w7GsoDOiSJEkjRHs7vPQSLF0KbW2wbFnPgNOpt0DRV8gYzPrh2Dal/I/xwT4O5T3D9RmDDctDDdbt7b0fM5Wrtx8OOhv0H4z7+jM6khTxA8nYsTB+fO3vi8jHsrc/ywP9WR9o2/Z2WLVq8J810OtDYUCXJEkqUGeobmvrCtZLl6653Nfj2uuWLSv72ygiB4uxY6GlpffltZ+3tvb92mA+p7/XenveGR67197bcr1f62+7tUdRixqNHc6R384R07UDZn/rBnq9Xu+pJWgPdSRYQzvzwoAuSZK0lvb2roBcS5DuL3TXGqpbWmDyZJgypetxxgzYaque6zsfJ07s/R/PvY3Y9TWKN5j1w7Vtb6eJDvQ4lPcM12d0hvKWlp6hV5KGU6EBPSL2A74NNAHnp5TOKHJ/kiSpOI3cr197LZx4YlewXr68tve1tPQMzhtuCFtv3Xeo7u1x8mQYN67Y7yhJqr7CAnpENAHfBfYFFgB3RMS1KaWHi9qnJEkqRqP36xtsALvu2n+ANlRLkopW5Aj6bsCfU0qPA0TED4GDgIboyCVJGmUaul+fNy83SZLKVGRAnwk83e35AmD3tTeKiGOBYzuevhoRDxZYk7LpwN/LLmIU8DjXh8e5PjzO9bNN2QX0YcB+3T69FP7ZrB+PdX14nOvD41wfg+7Tiwzovc1Z12O6kJTSecB5ABFxZ0ppboE1CY9zvXic68PjXB8e5/qJiDvLrqEPA/br9un153GuH491fXic68PjXB9D6dOLnDR/AbBZt+ebAs8WuD9JklQc+3VJkgpWZEC/A3htRGwRES3AocC1Be5PkiQVx35dkqSCFXaKe0ppVUT8O/AL8u1YLkwpPTTA284rqh6tweNcHx7n+vA414fHuX4qeayH0K9X8ns0II9z/Xis68PjXB8e5/oY9HGOlHpcFi5JkiRJkuqsyFPcJUmSJElSjQzokiRJkiRVQCUCekTsFxGPRcSfI+KzZdfTiCJis4j4VUQ8EhEPRcQJZdfUyCKiKSLuiYiflV1LI4uI9SPiJxHxaMf/228qu6ZGFBH/0fH3xoMRMT8ixpddUyOIiAsj4vnu9wqPiGkRcWNE/KnjcWqZNQ6V/Xrx7Nfry369ePbp9WO/Xozh6tdLD+gR0QR8F9gf2B44LCK2L7eqhrQK+FRKaTtgD+A4j3OhTgAeKbuIUeDbwM9TStsCc/CYD7uImAl8HJibUno9eXKwQ8utqmFcDOy31rrPAjellF4L3NTxfESxX68b+/X6sl8vnn16HdivF+pihqFfLz2gA7sBf04pPZ5SWgH8EDio5JoaTkppYUrp7o7lpeS/9GaWW1VjiohNgQOA88uupZFFxBRgT+ACgJTSipTSi+VW1bCagdaIaAYm4L2vh0VK6VbghbVWHwRc0rF8CXBwXYsaHvbrdWC/Xj/268WzT687+/UCDFe/XoWAPhN4utvzBdjBFCoiZgO7AH8ot5KG9S3gJGB12YU0uC2BRcBFHacdnh8RE8suqtGklJ4Bvg48BSwE/pFSuqHcqhrahimlhZADGPCakusZCvv1OrNfL5z9evHs0+vEfr3uBt2vVyGgRy/rvPdbQSJiEnAl8ImUUlvZ9TSaiDgQeD6ldFfZtYwCzcCuwPdSSrsAyxiBpwNXXce1UgcBWwCbABMj4shyq1LF2a/Xkf16sezX68Y+vU7s16uvCgF9AbBZt+eb4mkWhYiIseRO/PKU0lVl19Og5gHviYgnyad17h0Rl5VbUsNaACxIKXWOGP2E3LlreL0deCKltCiltBK4CnhzyTU1suciYmOAjsfnS65nKOzX68R+vS7s1+vDPr1+7Nfra9D9ehUC+h3AayNii4hoIU9ScG3JNTWciAjydT2PpJTOKrueRpVSOjmltGlKaTb5/+WbU0r+KlmAlNLfgKcjYpuOVfsAD5dYUqN6CtgjIiZ0/D2yD07cU6RrgaM7lo8GrimxlqGyX68D+/X6sF+vD/v0urJfr69B9+vNhZZTg5TSqoj4d+AX5FkEL0wpPVRyWY1oHnAU8EBE3Nux7pSU0vUl1iStq+OByztCwOPAMSXX03BSSn+IiJ8Ad5Nnjb4HOK/cqhpDRMwH9gKmR8QC4IvAGcCPI+Ij5H9EHVJehUNjv1439utqNPbpdWC/Xpzh6tcjJS8LkyRJkiSpbFU4xV2SJEmSpFHPgC5JkiRJUgUY0CVJkiRJqgADuiRJkiRJFWBAlyRJkiSpAgzokgYUEXtFxM/KrkOSJK07+3WpugzokiRJkiRVgAFdaiARcWRE3B4R90bEuRHRFBEvRcQ3IuLuiLgpImZ0bLtzRPw+Iu6PiJ9GxNSO9VtHxC8j4r6O92zV8fGTIuInEfFoRFweEdGx/RkR8XDH53y9pK8uSVLDsV+XRh8DutQgImI74EPAvJTSzkA7cAQwEbg7pbQrcAvwxY63/AD4TEppJ+CBbusvB76bUpoDvBlY2LF+F+ATwPbAlsC8iJgGvBfYoeNzvlrst5QkaXSwX5dGJwO61Dj2Ad4A3BER93Y83xJYDfyoY5vLgH+KiPWA9VNKt3SsvwTYMyImAzNTSj8FSCm9klJa3rHN7SmlBSml1cC9wGygDXgFOD8i3gd0bitJktaN/bo0ChnQpcYRwCUppZ072jYppS/1sl0a4DP68mq35XagOaW0CtgNuBI4GPj5IGuWJEm9s1+XRiEDutQ4bgI+EBGvAYiIaRGxOfnP+Qc6tjkc+E1K6R/Akoh4S8f6o4BbUkptwIKIOLjjM8ZFxIS+dhgRk4D1UkrXk0+T27mILyZJ0ihkvy6NQs1lFyBpeKSUHo6IU4EbImIMsBI4DlgG7BARdwH/IF/PBnA08P2Ojvpx4JiO9UcB50bEVzo+45B+djsZuCYixpN/pf+PYf5akiSNSvbr0ugUKfV3VoykkS4iXkopTSq7DkmStO7s16XG5inukiRJkiRVgCPokiRJkiRVgCPokiRJkiRVgAFdkiRJc0gbEQAAACNJREFUkqQKMKBLkiRJklQBBnRJkiRJkirAgC5JkiRJUgX8f5O6ohIKls9HAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1ad5375b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_learning_curves(m, loss_ylim=(0, 5.0), score_ylim=(0.0, 10.0), figsize=(14,6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/torchtext/data/field.py:321: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  return Variable(arr, volatile=not train), lengths\n",
      "/anaconda3/lib/python3.6/site-packages/torchtext/data/field.py:322: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.\n",
      "  return Variable(arr, volatile=not train)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:  2.4905223673628756\n",
      "PPL:  5.619813942602337\n",
      "Bleu:  0.1562485560339747\n"
     ]
    }
   ],
   "source": [
    "model.eval()   # Turn on evaluation mode which disables dropout and teacher_forcing.\n",
    "with torch.no_grad():\n",
    "\n",
    "    avg_loss = 0.0\n",
    "    avg_score = 0.0\n",
    "    for idx_batch, batch in enumerate(test_iter.__iter__()):\n",
    "        input_variables, input_lengths = getattr(batch, H.src_field_name)\n",
    "        target_variables = getattr(batch, H.tgt_field_name)\n",
    "\n",
    "        decoder_outputs = model(input_variables, None)\n",
    "\n",
    "        loss = 0\n",
    "        target_length = target_variables.size(1) - 1\n",
    "        for idx_seq in range(target_length):\n",
    "            seq_output = decoder_outputs[idx_seq]\n",
    "            loss += criterion(seq_output.contiguous(), target_variables[:, idx_seq + 1])\n",
    "\n",
    "        avg_loss += loss.item() / (idx_seq + 1)\n",
    "                \n",
    "        avg_score += blue_score(decoder_outputs, target_variables)\n",
    "\n",
    "print(\"Loss: \", avg_loss / (idx_batch + 1))\n",
    "print(\"PPL: \", get_perplexity(avg_loss / (idx_batch + 1)))\n",
    "print(\"Bleu: \", avg_score / (idx_batch + 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type in a source sequence:What's your name?\n",
      ">>  What's your name?\n",
      "['what', \"'s\", 'your', 'name', '?']\n",
      "<<  wie ist dein name ?\n",
      "Type in a source sequence:Where do you live?\n",
      ">>  Where do you live?\n",
      "['where', 'do', 'you', 'live', '?']\n",
      "<<  wo wohnen sie ?\n",
      "Type in a source sequence:Hello\n",
      ">>  Hello\n",
      "['hello']\n",
      "<<  hallo !\n",
      "Type in a source sequence:We are traveling.\n",
      ">>  We are traveling.\n",
      "['we', 'are', 'traveling', '.']\n",
      "<<  wir reisen .\n",
      "Type in a source sequence:This has no effect.\n",
      ">>  This has no effect.\n",
      "['this', 'has', 'no', 'effect', '.']\n",
      "<<  das hat keine wirkung .\n"
     ]
    }
   ],
   "source": [
    "while True:\n",
    "    seq_str = input(\"Type in a source sequence:\")\n",
    "    print(\">> \", seq_str)\n",
    "    if not len(seq_str):\n",
    "        break\n",
    "    #seq = seq_str.strip().lower().split()\n",
    "    seq = tokenize_en(seq_str.strip().lower())\n",
    "    print(seq)\n",
    "\n",
    "    seq_id = [input_vocab.stoi[tok] for tok in seq]\n",
    "\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "\n",
    "        src_id_seq = Variable(torch.LongTensor(seq_id)).view(1, -1)\n",
    "\n",
    "        src_id_seq = src_id_seq.cuda() if torch.cuda.is_available() else src_id_seq\n",
    "\n",
    "        decoder_outputs = model(src_id_seq, None)\n",
    "        tokens = [step_output[-1].topk(1)[1].item() for step_output in decoder_outputs]\n",
    "        \n",
    "        tokens = np.array(tokens)\n",
    "        idx = np.equal(tokens, 2).nonzero()\n",
    "        if idx[0] :\n",
    "            idx = idx[0][0]\n",
    "            tokens = tokens[0:idx]\n",
    "        \n",
    "        tgt_seq = [output_vocab.itos[tok] for tok in tokens]\n",
    "\n",
    "        print(\"<< \", ' '.join(tgt_seq))\n",
    "\n",
    "print(\"Finished.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}