3-CRF-NER-CONLL.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lRC5Mk7DS7il"
   },
   "source": [
    "# BiLSTM et CRF\n",
    "\n",
    "Dans ce notebook nous nous utilisons les champs aléatoires conditionnels (conditional random fields ou CRFs) qui sont généralement utilisés pour le named entityy recognition dans le but de prédire des séquences valides.\n",
    "Par exemple, on ne devrait pas avoir de `B-PER` suivi directement par un autre `B-PER`.\n",
    "Pour l'implémentation CRF, on utilisera le package [pytorch-crf](https://github.com/kmkurn/pytorch-crf)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 256
    },
    "colab_type": "code",
    "id": "s7pqLoSXKbh6",
    "outputId": "0ca5ebdf-b819-4a70-9098-978868d85385"
   },
   "outputs": [],
   "source": [
    "#!pip install torchtext==0.6.0\n",
    "#!pip install pytorch-crf\n",
    "\n",
    "import time\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.optim import Adam\n",
    "from torchtext.data import Field, NestedField, BucketIterator\n",
    "from torchtext.datasets import SequenceTaggingDataset\n",
    "from torchtext.vocab import Vocab\n",
    "from collections import Counter\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchtext import data\n",
    "from torchtext import datasets\n",
    "\n",
    "import spacy\n",
    "from torchcrf import CRF\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import time\n",
    "import random\n",
    "import string\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "k4LF_qyLKvfK"
   },
   "source": [
    "# Préparation des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JAamZSXiKnJl"
   },
   "outputs": [],
   "source": [
    "# pour la reproductibilité\n",
    "SEED = 1234\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "TEXT = data.Field(lower = False) \n",
    "TAG = data.Field(unk_token = None) # les tags sont tous connus on a alors unk_token = None\n",
    "\n",
    "my_path = \"data_ner/\"\n",
    "train_data, valid_data, test_data = data.TabularDataset.splits(\n",
    "        path= my_path ,\n",
    "        train=\"train.csv\",\n",
    "        validation=\"valid.csv\",\n",
    "        test=\"test.csv\", format='csv', skip_header=True,\n",
    "        fields=((\"text\", TEXT), (\"tag\", TAG))\n",
    "    )\n",
    "MIN_FREQ = 0\n",
    "\n",
    "TEXT.build_vocab(train_data, \n",
    "                 min_freq = MIN_FREQ, # les mots qui apparaissent moins que MIN_FREQ fois seront ignorés du vocabulaire\n",
    "                 vectors = \"glove.6B.100d\",\n",
    "                 unk_init = torch.Tensor.normal_)\n",
    "\n",
    "\n",
    "TAG.build_vocab(train_data)\n",
    "BATCH_SIZE = 16\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "#device = 'cpu'\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "    batch_size = BATCH_SIZE,\n",
    "    device = device, sort=False)\n",
    "\n",
    "# padding index\n",
    "TEXT_PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]  \n",
    "TAG_PAD_IDX = TAG.vocab.stoi[TAG.pad_token]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qXY-SyrWKx4u"
   },
   "source": [
    "# Construire le modèle\n",
    "\n",
    "Pour la classe de modèle, il existe trois mises à jour principales:\n",
    "\n",
    "1. préparez la couche CRF lors de l'initialisation. Nous devons spécifier le nombre de tags possibles dans le text.\n",
    "2. inclure la logique de la couche CRF dans la séquence `forward ()`. Il y a un changement majeur dans le `forward ()` en raison de l'implémentation du package `pytorch-crf`. Jusqu'à présent, nous effectuons séparément la propagation forward et le calcul des pertes. Désormais, le calcul des pertes est intégré à la propagation forward.\n",
    "3. initialisez toutes les transitions impossibles avec un nombre vraiment bas (-100) dans la fonction `init_crf_transitions`. C'est là que nous imposons la logique de séquence BIO au modèle."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PW8shLquKyr-"
   },
   "outputs": [],
   "source": [
    "class BiLSTM(nn.Module):\n",
    "\n",
    "    def __init__(self,\n",
    "                 input_dim,\n",
    "                 embedding_dim,\n",
    "                 hidden_dim,\n",
    "                 output_dim,\n",
    "                 lstm_layers,\n",
    "                 emb_dropout,\n",
    "                 lstm_dropout,\n",
    "                 fc_dropout,\n",
    "                 word_pad_idx,\n",
    "                 tag_pad_idx):\n",
    "        super().__init__()\n",
    "        \n",
    "        # LAYER 1: Word Embedding\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.embedding = nn.Embedding(\n",
    "            num_embeddings=input_dim,\n",
    "            embedding_dim=embedding_dim,\n",
    "            padding_idx=word_pad_idx\n",
    "        )\n",
    "        self.emb_dropout = nn.Dropout(emb_dropout)\n",
    "        \n",
    "\n",
    "        # LAYER 2: BiLSTM\n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size=embedding_dim, \n",
    "            hidden_size=hidden_dim,\n",
    "            num_layers=lstm_layers,\n",
    "            bidirectional=True,\n",
    "            dropout=lstm_dropout if lstm_layers > 1 else 0\n",
    "        )\n",
    "        \n",
    "        # LAYER 3: Fully-connected  \n",
    "        self.fc_dropout = nn.Dropout(fc_dropout)\n",
    "        self.fc = nn.Linear(hidden_dim * 2, output_dim)  \n",
    "        # LAYER 4: CRF\n",
    "\n",
    "        self.tag_pad_idx = tag_pad_idx\n",
    "        self.crf = CRF(num_tags=output_dim)\n",
    "        \n",
    "        # init poids avec distribution normale \n",
    "        for name, param in self.named_parameters():\n",
    "            nn.init.normal_(param.data, mean=0, std=0.1)\n",
    "\n",
    "    def forward(self, words, tags=None):\n",
    "\n",
    "        # words = [sentence length, batch size]\n",
    "        # chars = [batch size, sentence length, word length)\n",
    "        # tags = [sentence length, batch size]\n",
    "        \n",
    "        # embedding_out = [sentence length, batch size, embedding dim]\n",
    "        embedding_out = self.emb_dropout(self.embedding(words))\n",
    "\n",
    "        # lstm_out = [sentence length, batch size, hidden dim * 2]\n",
    "        lstm_out, _ = self.lstm(embedding_out)\n",
    "        \n",
    "        # fc_out = [sentence length, batch size, output dim]\n",
    "        fc_out = self.fc(self.fc_dropout(lstm_out))\n",
    "      \n",
    "        if tags is not None:\n",
    "            mask = tags != self.tag_pad_idx\n",
    "            crf_out = self.crf.decode(fc_out, mask=mask)\n",
    "            crf_loss = -self.crf(fc_out, tags=tags, mask=mask) \n",
    "        else:\n",
    "            crf_out = self.crf.decode(fc_out)\n",
    "            crf_loss = None\n",
    "          \n",
    "        return crf_out , crf_loss\n",
    "\n",
    "   \n",
    "  \n",
    "    def init_crf_transitions(self, tag_names, imp_value=-100):\n",
    "        num_tags = len(tag_names)\n",
    "        for i in range(num_tags):\n",
    "            tag_name = tag_names[i]\n",
    "            # I and <pad> impossible au début\n",
    "            if tag_name[0] == \"I\" or tag_name == \"<pad>\":\n",
    "                torch.nn.init.constant_(self.crf.start_transitions[i], imp_value)\n",
    "        # transition impossible O - I\n",
    "        tag_is = {}\n",
    "        for tag_position in (\"B\", \"I\", \"O\"):\n",
    "            tag_is[tag_position] = [i for i, tag in enumerate(tag_names) if tag[0] == tag_position]\n",
    "        impossible_transitions_position = {\n",
    "            \"O\": \"I\"\n",
    "\n",
    "        }\n",
    "        for from_tag, to_tag_list in impossible_transitions_position.items():\n",
    "            to_tags = list(to_tag_list)\n",
    "            \n",
    "            for from_tag_i in tag_is[from_tag]:\n",
    "                for to_tag in to_tags:\n",
    "                    for to_tag_i in tag_is[to_tag]:\n",
    "                    \n",
    "                        torch.nn.init.constant_(\n",
    "                            self.crf.transitions[from_tag_i, to_tag_i], imp_value\n",
    "                        )\n",
    "        # transitions impossibles entre différents types\n",
    "        impossible_transitions_tags = {\n",
    "            \"B\": \"I\",\n",
    "            \"I\": \"I\"\n",
    "        }\n",
    "        for from_tag, to_tag_list in impossible_transitions_tags.items():\n",
    "            to_tags = list(to_tag_list)\n",
    "            for from_tag_i in tag_is[from_tag]:\n",
    "                for to_tag in to_tags:\n",
    "                    for to_tag_i in tag_is[to_tag]:\n",
    "                        if tag_names[from_tag_i].split(\"-\")[1] != tag_names[to_tag_i].split(\"-\")[1]:\n",
    "                            torch.nn.init.constant_(\n",
    "                                self.crf.transitions[from_tag_i, to_tag_i], imp_value\n",
    "                            )\n",
    "\n",
    "    def count_parameters(self):\n",
    "        return sum(p.numel() for p in self.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 237
    },
    "colab_type": "code",
    "id": "Zb03iYctNAUM",
    "outputId": "4be1d224-4f2d-4a36-b9ce-128b3ce019a6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Le modèle a 3,101,034 paramètres à entraîner.\n"
     ]
    }
   ],
   "source": [
    "embedding_dim=100\n",
    "tag_pad_idx=TAG_PAD_IDX\n",
    "model = BiLSTM(\n",
    "    input_dim=len(TEXT.vocab),\n",
    "    embedding_dim=100,\n",
    "    hidden_dim=256,\n",
    "    output_dim=len(TAG.vocab),\n",
    "    lstm_layers=1,\n",
    "    emb_dropout=0.1,\n",
    "    lstm_dropout=0.1,\n",
    "    fc_dropout=0.1,\n",
    "    word_pad_idx=TEXT_PAD_IDX,\n",
    "    tag_pad_idx=TAG_PAD_IDX\n",
    ")\n",
    "def init_weights(m):\n",
    "    for name, param in m.named_parameters():\n",
    "        nn.init.normal_(param.data, mean = 0, std = 0.1)\n",
    "        \n",
    "model.apply(init_weights)\n",
    "\n",
    "pretrained_embeddings = TEXT.vocab.vectors\n",
    "model.embedding.weight.data.copy_(pretrained_embeddings)\n",
    "model.embedding.weight.data[tag_pad_idx] = torch.zeros(embedding_dim)\n",
    "\n",
    "\n",
    "# CRF transitions initialisation\n",
    "model.init_crf_transitions(\n",
    "    tag_names=TAG.vocab.itos\n",
    ")\n",
    "print(f\"Le modèle a {model.count_parameters():,} paramètres à entraîner.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lGsjeyHlyWhx"
   },
   "source": [
    "Nous pouvons accéder à la matrice de transition et nous assurer que l'initialisation est effectuée comme prévu :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 697
    },
    "colab_type": "code",
    "id": "Wfe5T7dlrS2j",
    "outputId": "084157b9-5a3c-40d0-c506-4a89d50c913f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start and end tag transitions:\n",
      "TAG   \tSTART\tEND\n",
      "<pad> \t-100.0\t-0.01\n",
      "O     \t-0.04\t-0.03\n",
      "B-LOC \t0.06\t-0.18\n",
      "B-PER \t0.02\t-0.04\n",
      "B-ORG \t0.05\t0.09\n",
      "I-PER \t-100.0\t0.1\n",
      "I-ORG \t-100.0\t0.1\n",
      "B-MISC\t-0.02\t0.07\n",
      "I-LOC \t-100.0\t0.1\n",
      "I-MISC\t-100.0\t0.09\n",
      "\n",
      "Between tags transitions:\n",
      "   TO\tO    \tB-PER\tI-PER\n",
      "FROM\n",
      "O    \t-0.01\t0.04 \t-100.0\n",
      "B-PER\t-0.15\t-0.06\t0.02 \n",
      "I-PER\t-0.06\t0.04 \t-0.03\n"
     ]
    }
   ],
   "source": [
    "def print_crf_transitions(c, m):\n",
    "    tags = TAG.vocab.itos\n",
    "    max_len_tag = max([len(tag) for tag in tags])\n",
    "    print(\"Start and end tag transitions:\")\n",
    "    print(f\"{'TAG'.ljust(max_len_tag)}\\tSTART\\tEND\")\n",
    "    for tag, start_prob, end_prob in zip(tags, m.crf.start_transitions.tolist(), m.crf.end_transitions.tolist()):\n",
    "        print(f\"{tag.ljust(max_len_tag)}\\t{round(start_prob, 2)}\\t{round(end_prob, 2)}\")\n",
    "    print()\n",
    "    print(\"Between tags transitions:\")\n",
    "    persons_i = [i for i, tag in enumerate(TAG.vocab.itos) if \"PER\" in tag or tag == \"O\"]\n",
    "    max_len_tag = max([len(tag) for tag in TAG.vocab.itos if \"PER\" in tag ])\n",
    "    transitions = m.crf.transitions\n",
    "    to_tags = \"TO\".rjust(max_len_tag) + \"\\t\" + \"\\t\".join([tag.ljust(max_len_tag) for tag in tags if \"PER\" in tag or tag == \"O\"])\n",
    "    print(to_tags)\n",
    "    print(\"FROM\")\n",
    "    for from_tag_i, from_tag_probs in enumerate(transitions[persons_i]):\n",
    "        to_tag_str = f\"{tags[persons_i[from_tag_i]].ljust(max_len_tag)}\"\n",
    "        for to_tag_prob in from_tag_probs[persons_i]:\n",
    "            to_tag_str += f\"\\t{str(round(to_tag_prob.item(), 2)).ljust(max_len_tag)}\"\n",
    "        print(to_tag_str)\n",
    "\n",
    "print_crf_transitions(TEXT, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5lWbTw3yMiyr"
   },
   "source": [
    "# Entraînement\n",
    "\n",
    "Les sorties du modèle sont deux listes : les prédictions et les pertes.\n",
    "On prend en considération ces changements en modifiant la fonction qui calcule l'accuracy.\n",
    "\n",
    " - Optimiseur "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimiseur(model, lr=1e-5, eps=1e-6, weight_decay_rate=0.001, second_weight_decay_rate=0.0):\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "            'weight_decay_rate': weight_decay_rate},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "            'weight_decay_rate': second_weight_decay_rate}]\n",
    "    return optim.Adam(\n",
    "        optimizer_grouped_parameters,\n",
    "        lr=lr,\n",
    "        eps=eps\n",
    "    )\n",
    "\n",
    "optimizer = optimiseur(model, lr=1e-5, eps=1e-6, weight_decay_rate=0.001, second_weight_decay_rate=0.0)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " - Métriques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score, classification_report\n",
    "\n",
    "def f1_loss( preds, y, tag_pad_idx, full_report=False):\n",
    "    index_o = TAG.vocab.stoi[\"O\"]\n",
    "    positive_labels = [i for i in range(len(TAG.vocab.itos))\n",
    "                       if i not in (tag_pad_idx, index_o)]\n",
    "\n",
    "    flatten_preds = [pred for sent_pred in preds for pred in sent_pred]\n",
    "\n",
    "    positive_preds = [pred for pred in flatten_preds\n",
    "                      if pred not in (tag_pad_idx, index_o)]\n",
    "\n",
    "    flatten_y = [tag for sent_tag in y for tag in sent_tag]\n",
    "    if full_report:\n",
    "      \n",
    "        positive_names = [TAG.vocab.itos[i]\n",
    "                              for i in range(len(TAG.vocab.itos))\n",
    "                              if i not in (tag_pad_idx, index_o)]\n",
    "        print(classification_report(\n",
    "                y_true=flatten_y,\n",
    "                y_pred=flatten_preds,\n",
    "                labels=positive_labels,\n",
    "                target_names=positive_names\n",
    "            ))\n",
    "\n",
    "    return f1_score(\n",
    "            y_true=flatten_y,\n",
    "            y_pred=flatten_preds,\n",
    "            labels=positive_labels,\n",
    "            average=\"micro\"\n",
    "        ), flatten_preds, flatten_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, tag_pad_idx):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0    \n",
    "    model.train()\n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        text = batch.text\n",
    "        tags = batch.tag\n",
    "      \n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        pred_tags_list, batch_loss = model(text, tags)\n",
    "        \n",
    "        # pour calculer la loss et le score f1, on flatten true tags\n",
    "        true_tags_list = [\n",
    "                [tag for tag in sent_tag if tag != TAG_PAD_IDX]\n",
    "                for sent_tag in tags.permute(1, 0).tolist()\n",
    "            ]\n",
    "        f1,_,_ = f1_loss(pred_tags_list, true_tags_list, tag_pad_idx)\n",
    "\n",
    "        batch_loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        epoch_loss += batch_loss.item()\n",
    "        epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_f1 / len(iterator)\n",
    "\n",
    "def evaluate(model, iterator, tag_pad_idx,full_report):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1 = 0\n",
    "    \n",
    "    model.eval()\n",
    "    preds = []\n",
    "    labels = []\n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in iterator:\n",
    "\n",
    "            text = batch.text\n",
    "            tags = batch.tag\n",
    "        \n",
    "            pred_tags_list, batch_loss = model(text, tags)\n",
    "            true_tags_list = [\n",
    "                [tag for tag in sent_tag if tag != TAG_PAD_IDX]\n",
    "                for sent_tag in tags.permute(1, 0).tolist()\n",
    "                ]\n",
    "            \n",
    "            f1, pred, lab = f1_loss(pred_tags_list, true_tags_list, tag_pad_idx, full_report)\n",
    "            preds.append(pred)\n",
    "            labels.append(lab)\n",
    "            epoch_loss += batch_loss.item()\n",
    "            epoch_f1 += f1\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_f1 / len(iterator),preds, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 1m 28s\n",
      "\tTrain Loss: 186.306 | Train F1 score: 3.28%\n",
      "\t Val. Loss: 130.346 |  Val. F1 score: 1.10%\n",
      "Epoch: 02 | Epoch Time: 1m 27s\n",
      "\tTrain Loss: 104.158 | Train F1 score: 6.53%\n",
      "\t Val. Loss: 106.366 |  Val. F1 score: 5.68%\n",
      "Epoch: 03 | Epoch Time: 1m 27s\n",
      "\tTrain Loss: 90.119 | Train F1 score: 14.46%\n",
      "\t Val. Loss: 93.288 |  Val. F1 score: 13.02%\n",
      "Epoch: 04 | Epoch Time: 1m 29s\n",
      "\tTrain Loss: 81.585 | Train F1 score: 21.04%\n",
      "\t Val. Loss: 84.620 |  Val. F1 score: 20.80%\n",
      "Epoch: 05 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 75.177 | Train F1 score: 27.89%\n",
      "\t Val. Loss: 78.939 |  Val. F1 score: 26.26%\n",
      "Epoch: 06 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 70.408 | Train F1 score: 33.16%\n",
      "\t Val. Loss: 74.325 |  Val. F1 score: 31.21%\n",
      "Epoch: 07 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 66.334 | Train F1 score: 37.38%\n",
      "\t Val. Loss: 70.720 |  Val. F1 score: 34.70%\n",
      "Epoch: 08 | Epoch Time: 1m 29s\n",
      "\tTrain Loss: 63.040 | Train F1 score: 41.39%\n",
      "\t Val. Loss: 67.631 |  Val. F1 score: 38.76%\n",
      "Epoch: 09 | Epoch Time: 2m 28s\n",
      "\tTrain Loss: 60.106 | Train F1 score: 44.29%\n",
      "\t Val. Loss: 65.150 |  Val. F1 score: 42.16%\n",
      "Epoch: 10 | Epoch Time: 2m 21s\n",
      "\tTrain Loss: 57.687 | Train F1 score: 47.34%\n",
      "\t Val. Loss: 62.551 |  Val. F1 score: 46.12%\n",
      "Epoch: 11 | Epoch Time: 1m 26s\n",
      "\tTrain Loss: 55.743 | Train F1 score: 49.52%\n",
      "\t Val. Loss: 62.049 |  Val. F1 score: 46.89%\n",
      "Epoch: 12 | Epoch Time: 1m 26s\n",
      "\tTrain Loss: 54.714 | Train F1 score: 50.95%\n",
      "\t Val. Loss: 61.408 |  Val. F1 score: 47.83%\n",
      "Epoch: 13 | Epoch Time: 1m 53s\n",
      "\tTrain Loss: 53.743 | Train F1 score: 51.85%\n",
      "\t Val. Loss: 60.435 |  Val. F1 score: 49.10%\n",
      "Epoch: 14 | Epoch Time: 2m 30s\n",
      "\tTrain Loss: 52.778 | Train F1 score: 52.57%\n",
      "\t Val. Loss: 59.711 |  Val. F1 score: 49.92%\n",
      "Epoch: 15 | Epoch Time: 2m 25s\n",
      "\tTrain Loss: 51.769 | Train F1 score: 53.87%\n",
      "\t Val. Loss: 58.969 |  Val. F1 score: 50.66%\n",
      "Epoch: 16 | Epoch Time: 2m 20s\n",
      "\tTrain Loss: 51.018 | Train F1 score: 54.55%\n",
      "\t Val. Loss: 58.387 |  Val. F1 score: 51.79%\n",
      "Epoch: 17 | Epoch Time: 2m 16s\n",
      "\tTrain Loss: 50.101 | Train F1 score: 55.86%\n",
      "\t Val. Loss: 57.954 |  Val. F1 score: 52.58%\n",
      "Epoch: 18 | Epoch Time: 2m 20s\n",
      "\tTrain Loss: 49.446 | Train F1 score: 56.59%\n",
      "\t Val. Loss: 57.270 |  Val. F1 score: 53.38%\n",
      "Epoch: 19 | Epoch Time: 2m 33s\n",
      "\tTrain Loss: 48.758 | Train F1 score: 57.31%\n",
      "\t Val. Loss: 56.910 |  Val. F1 score: 54.00%\n",
      "Epoch: 20 | Epoch Time: 2m 30s\n",
      "\tTrain Loss: 48.023 | Train F1 score: 57.66%\n",
      "\t Val. Loss: 56.600 |  Val. F1 score: 54.25%\n",
      "Epoch: 21 | Epoch Time: 2m 30s\n",
      "\tTrain Loss: 47.415 | Train F1 score: 58.90%\n",
      "\t Val. Loss: 55.879 |  Val. F1 score: 54.95%\n",
      "Epoch: 22 | Epoch Time: 2m 17s\n",
      "\tTrain Loss: 47.177 | Train F1 score: 58.64%\n",
      "\t Val. Loss: 55.496 |  Val. F1 score: 55.33%\n",
      "Epoch: 23 | Epoch Time: 2m 21s\n",
      "\tTrain Loss: 46.963 | Train F1 score: 59.15%\n",
      "\t Val. Loss: 55.606 |  Val. F1 score: 55.30%\n",
      "Epoch: 24 | Epoch Time: 2m 32s\n",
      "\tTrain Loss: 46.417 | Train F1 score: 59.75%\n",
      "\t Val. Loss: 55.391 |  Val. F1 score: 55.72%\n",
      "Epoch: 25 | Epoch Time: 2m 34s\n",
      "\tTrain Loss: 46.087 | Train F1 score: 59.99%\n",
      "\t Val. Loss: 55.074 |  Val. F1 score: 56.02%\n",
      "Epoch: 26 | Epoch Time: 2m 25s\n",
      "\tTrain Loss: 45.939 | Train F1 score: 60.62%\n",
      "\t Val. Loss: 54.750 |  Val. F1 score: 56.62%\n",
      "Epoch: 27 | Epoch Time: 2m 30s\n",
      "\tTrain Loss: 45.581 | Train F1 score: 60.76%\n",
      "\t Val. Loss: 54.858 |  Val. F1 score: 56.54%\n",
      "Epoch: 28 | Epoch Time: 2m 13s\n",
      "\tTrain Loss: 45.219 | Train F1 score: 61.12%\n",
      "\t Val. Loss: 54.788 |  Val. F1 score: 56.42%\n",
      "Epoch: 29 | Epoch Time: 2m 15s\n",
      "\tTrain Loss: 44.879 | Train F1 score: 61.45%\n",
      "\t Val. Loss: 54.466 |  Val. F1 score: 56.99%\n",
      "Epoch: 30 | Epoch Time: 2m 39s\n",
      "\tTrain Loss: 44.531 | Train F1 score: 61.51%\n",
      "\t Val. Loss: 54.410 |  Val. F1 score: 57.29%\n",
      "Epoch: 31 | Epoch Time: 2m 39s\n",
      "\tTrain Loss: 44.467 | Train F1 score: 61.87%\n",
      "\t Val. Loss: 54.043 |  Val. F1 score: 57.68%\n",
      "Epoch: 32 | Epoch Time: 2m 35s\n",
      "\tTrain Loss: 44.297 | Train F1 score: 61.95%\n",
      "\t Val. Loss: 53.964 |  Val. F1 score: 57.74%\n",
      "Epoch: 33 | Epoch Time: 2m 24s\n",
      "\tTrain Loss: 44.175 | Train F1 score: 61.79%\n",
      "\t Val. Loss: 53.977 |  Val. F1 score: 57.68%\n",
      "Epoch: 34 | Epoch Time: 2m 13s\n",
      "\tTrain Loss: 43.968 | Train F1 score: 62.37%\n",
      "\t Val. Loss: 53.935 |  Val. F1 score: 57.85%\n",
      "Epoch: 35 | Epoch Time: 2m 18s\n",
      "\tTrain Loss: 43.771 | Train F1 score: 62.28%\n",
      "\t Val. Loss: 53.588 |  Val. F1 score: 58.37%\n",
      "Epoch: 36 | Epoch Time: 1m 38s\n",
      "\tTrain Loss: 43.810 | Train F1 score: 61.98%\n",
      "\t Val. Loss: 53.659 |  Val. F1 score: 58.16%\n",
      "Epoch: 37 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 43.681 | Train F1 score: 62.57%\n",
      "\t Val. Loss: 53.761 |  Val. F1 score: 57.96%\n",
      "Epoch: 38 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 43.500 | Train F1 score: 62.76%\n",
      "\t Val. Loss: 53.245 |  Val. F1 score: 58.71%\n",
      "Epoch: 39 | Epoch Time: 1m 24s\n",
      "\tTrain Loss: 43.410 | Train F1 score: 62.84%\n",
      "\t Val. Loss: 53.490 |  Val. F1 score: 58.27%\n",
      "Epoch: 40 | Epoch Time: 1m 29s\n",
      "\tTrain Loss: 43.289 | Train F1 score: 62.94%\n",
      "\t Val. Loss: 53.379 |  Val. F1 score: 58.50%\n",
      "Epoch: 41 | Epoch Time: 1m 22s\n",
      "\tTrain Loss: 43.066 | Train F1 score: 62.94%\n",
      "\t Val. Loss: 53.284 |  Val. F1 score: 58.66%\n",
      "Epoch: 42 | Epoch Time: 1m 23s\n",
      "\tTrain Loss: 42.988 | Train F1 score: 63.55%\n",
      "\t Val. Loss: 53.189 |  Val. F1 score: 58.73%\n",
      "Epoch: 43 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 42.904 | Train F1 score: 63.05%\n",
      "\t Val. Loss: 53.171 |  Val. F1 score: 58.69%\n",
      "Epoch: 44 | Epoch Time: 1m 25s\n",
      "\tTrain Loss: 43.042 | Train F1 score: 62.96%\n",
      "\t Val. Loss: 53.132 |  Val. F1 score: 58.83%\n",
      "Epoch: 45 | Epoch Time: 1m 27s\n",
      "\tTrain Loss: 42.814 | Train F1 score: 63.48%\n",
      "\t Val. Loss: 53.297 |  Val. F1 score: 58.63%\n",
      "Epoch: 46 | Epoch Time: 1m 29s\n",
      "\tTrain Loss: 42.741 | Train F1 score: 63.40%\n",
      "\t Val. Loss: 53.088 |  Val. F1 score: 58.84%\n",
      "Epoch: 47 | Epoch Time: 1m 29s\n",
      "\tTrain Loss: 42.645 | Train F1 score: 63.66%\n",
      "\t Val. Loss: 53.041 |  Val. F1 score: 58.89%\n",
      "Epoch: 48 | Epoch Time: 1m 31s\n",
      "\tTrain Loss: 42.631 | Train F1 score: 63.63%\n",
      "\t Val. Loss: 52.982 |  Val. F1 score: 58.93%\n",
      "Epoch: 49 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 42.627 | Train F1 score: 63.33%\n",
      "\t Val. Loss: 53.008 |  Val. F1 score: 58.89%\n",
      "Epoch: 50 | Epoch Time: 1m 30s\n",
      "\tTrain Loss: 42.576 | Train F1 score: 63.67%\n",
      "\t Val. Loss: 53.105 |  Val. F1 score: 58.89%\n"
     ]
    }
   ],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs\n",
    "\n",
    "N_EPOCHS = 50\n",
    "\n",
    "t_loss = []\n",
    "t_f1 = []\n",
    "v_loss = []\n",
    "v_f1 = []\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "   \n",
    "    \n",
    "    train_loss, train_f1 = train(model, train_iterator, optimizer, TAG_PAD_IDX)\n",
    "    t_loss.append(train_loss)\n",
    "    t_f1.append(train_f1) \n",
    "    \n",
    "    valid_loss, valid_f1,_,_ = evaluate(model, valid_iterator, TAG_PAD_IDX, full_report= False)\n",
    "    v_loss.append(valid_loss)\n",
    "    v_f1.append(valid_f1)\n",
    "    \n",
    "    scheduler.step()\n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'tut3-model_conll.pt')\n",
    "    \n",
    "\n",
    "    if epoch%1 == 0: \n",
    "        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "        print(f'\\tTrain Loss: {train_loss:.3f} | Train F1 score: {train_f1*100:.2f}%')\n",
    "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. F1 score: {valid_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f703dcb5110>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAELCAYAAAA4HCbKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3wU5b348c/M7DX32yYk4RJBoQFbRVCOl2obrFgJl7bWtDlaS620P7RV+/Mn6KGAgm0D9qhHoGgvWivH9qgtSFDBVu3xglYR8BIVyAUCCQm5keteZ35/bLIBDIQku2zY+b5fr2V3Z3dmnm8Svs8zzzPzjGIYhoEQQoiYpka7AEIIISJPkr0QQpiAJHshhDABSfZCCGECkuyFEMIEJNkLIYQJSLIXQggTkGQvTK+goIC33nor2sUQIqIk2QshhAlIsheiD16vl/vvv5/LLruMyy67jPvvvx+v1wtAU1MTP/rRj5g6dSoXXXQRxcXF6LoOwGOPPcaXv/xlJk+ezIwZM9i2bVs0wxAixBLtAggxHP3mN79h165dbNy4EUVRWLBgAWvXruX222/n8ccfJysrK5TId+3ahaIoVFRUsH79ep599lmysrI4cOBAqBIQItqkZS9EHzZt2sQtt9xCeno6aWlp3HLLLTz//PMAWCwWDh8+TE1NDVarlalTp6IoCpqm4fV6KS8vx+fzMXLkSEaPHh3lSIQIkmQvRB/q6+vJyckJvc/JyaG+vh6Am266iTFjxvCDH/yA6dOn89hjjwEwZswY7rnnHh555BEuueQS7rjjDurq6qJSfiGOJ8leiD5kZmZSU1MTel9bW0tmZiYACQkJLFq0iH/84x+sW7eOxx9/PNSlM2vWLJ5++mleffVVFEXhgQceiEr5hTieJHshAJ/Ph8fjCT1mzpzJb37zG5qammhqamLNmjXMmjULgFdffZV9+/ZhGAYJCQlomoaqqlRUVLBt2za8Xi82mw273Y6maVGOTIggGaAVApg/f/4x73/wgx9w7rnnMnv2bACuvvpqFixYAMC+fftYvnw5TU1NJCUl8d3vfpdp06bx6aef8utf/5ry8nKsViuTJ0/mvvvuO+2xCNEXRW5eIoQQsU+6cYQQwgQk2QshhAlIshdCCBOQZC+EECYgyV4IIUxAkr0QQpjAsD7Pvrm5A10f+Jmh6ekJNDa2R6BEw5fEbA5mi9ls8cLQYlZVhdTU+D4/G9bJXteNQSX7nnXNRmI2B7PFbLZ4ITIxSzeOEEKYgCR7IYQwgWHdjSOEOLMYhkFz82G8Xjcw9K6I+nrVdDeA6T9mBZvNQWqqC0VRTnm7kuyFEGHT3n4ERVHIyhqJogy948BiUfH7zZXs+4vZMHRaWhpobz9CYmLKKW9XunGEEGHT1dVOYmJKWBK96JuiqCQmptLVNbAzduQ3IoQIG10PoGnSYRBpmmZB1wMDWifmkv3vSst4eutn0S6GEKY1kH5kMTiD+RnHXBVc09CBx69z5eSc/r8shIhZN998Iz6fD7/fR3X1fs46axwA48dP4J57lp7SNjZseBaPx0NR0b8PaN/XXjuLlSsfZOzYswdc7kiJuWTvtFvodPujXQwhRJT99rd/BKC2toYf/vAGnnjivz/3Hb/fj8Vy4jQ4d+61ESvf6RZzyd5h02hu90a7GEKIYeraa2dRWDiH7dvfJScnl/nzF7Bs2X/Q0dGB1+vlkksuZcGC2wD4/e8fpauri1tvvZ0XXtjEyy+/RGJiEhUV5SQmJrBixUrS0zNOur8DB6pZteoXtLQ0o2ka8+ffwr/92yW43W5WrFhKVVUFmmZh9OgxLF/+K/btq+K++5bidrvR9QBf//osiotvGHLcMZfsnXYLBxs6ol0MIUzvzQ9reeOD2iFtQ1GgrxunXvalbC79Yvagt9vQ0MAjjzwKgMfjoaTkQeLi4vD7/fzsZ7fy9ttv8W//dsnn1vvkkzL++MenycoaQUnJCp599i/86Ee3nHRf9967mDlzvkFh4VwqKyu49dabeeqpZ/ngg520tbXx1FPPANDa2grAc889w8UXX8r3v//DY5YPVewle5t04wghTu7qq2eGXuu6ztq1D/Phhx8ABo2NjezZs7vPZP+lL51HVtYIACZNOpd3333npPvp7Oxg797dXHNN8Mb1Z501lrPPnsDHH3/I2Wefw/79Vfz61yVMnjyFSy65DIDzz7+ARx55EJ/PxwUXTOWCC6aGJeZ+k31JSQlbtmzh4MGDbNq0ifHjxwPw6quv8vDDD2MYBrqu85Of/ISrrroKgMrKShYtWkRLSwspKSmUlJSQl5cXlgL3x2HX6PT4MQxDzgoQIoou/eLQWt8QuYuq4uKcodd/+ct62tpaeeyxJ7Db7ZSU3I/X6+lzPZvNFnqtqhqBwMlPfzT6OiwheDZNbu5I1q9/hvfee5e3336Txx5bwx//+GcKCqYzceK5/Otfb/PUU0+wefPzLFmyfBBRHqvfUy+nT5/O+vXryc3NPSaAu+66i5UrV7Jx40ZWrVrFwoULQ5f4Ll26lOLiYrZs2UJxcTFLliwZckFPldNuQdcNvCa76k4IMThtbW2kp2dgt9s5fLieN974Z9i2HR+fwNlnj+fFF0sB2LevivLy3UyceC719XWoqsbll3+Fn/70/9LS0kxbWyvV1ftJS0vnmmtmMW/ezZSVfRyWsvTbsp86te9DCFVVaWtrA4I/rMzMTFRVpbGxkbKyMh5//HEACgsLWb58OU1NTaSlpYWl0CfjtGkAuD1+7FYt4vsTQpzZvv3t7/Dzny9k3rxiMjOzmDLlwrBuf+nSFaxa9Qv+53/+G03TWLz4PlJTU9m27U3WrVsNBC9Gu/7675OR4eKppx7npZdexGq1oCgKt932f8NSDsU40XHGcQoKCli3bl2oG2fbtm3cfvvtxMXF0dHRwaOPPsrkyZP56KOPWLhwIZs3bw6te80117Bq1SomTZoUlkKfzGvbq/n1f7/PukXTyXUlRHx/QoheH39cRk7OmGgXwxRqavYxadLEU/7+oAZo/X4/jz76KGvXrmXKlCls376dO+6445gEHw6Nje0DnsTf5wkOzh6sPYItDLPunSlcrkQOH26LdjFOK4l5+NF1Pax97DIR2onpuv65vwVVVUhP77uRO6jpEj755BPq6+uZMmUKAFOmTMHpdFJeXk52djZ1dXWhgYtAIEB9fT3Z2UMbqDlVTnuw66bLI2fkCCFEj0El+xEjRnDo0CEqKioAKC8vp6GhgdGjR5Oenk5+fj6lpcEBidLSUvLz809Lfz2AwxY8WOnyDGySICGEiGX9duOsWLGCrVu30tDQwLx580hJSWHz5s0sW7aM2267LXR64y9/+UtSUoJzKy9btoxFixaxdu1akpKSKCkpiWwUR+lp2bu90rIXQogepzxAGw2D6bNv7fRy+3+9QfGV53Dl1FERKtnwM9z7ciNBYh5+Dh3ax4gR4RuglT77E+vrZx32PvvhzNnTjeOVbhwhhOgRc8nealGxWlTcMkArhBAhMZfsAeIcFmnZC2FyP/vZT9iw4bljlhmGwbe/PZudO98/4Xr337+M5577CxCcz/4vf1nf5/deeGETixff1edn1147i4qKvYMseWTEZrK3W6VlL4TJzZw5mxde2HTMsh07tqNpGueff8EpbWPu3GsHfOOS4SrmZr0EcDoscp69EFHm2/0mvs/+d0jbUBSlz8nErBMuxzr+0pOue/nlX+E///NXVFZWcNZZYwHYvPl5rrlmFuXle/n1r3+F292F1+tl9uxvcN11xZ/bxtHz2ft8Ph58cCU7dmzH5cpk9Oi8U4pB5rOPIOnGEUJYrVa+9rWrefHFTSxYcBudnR28/vo/+fGPbyU+Pp6HHlqLzWajs7OT+fNv5KKLLiYv76wTbm/jxueora3hT3/6H/x+P7fccvMpXSwq89lHUJzdSmtb31OUCiFOD+v4S/ttffdnqKdezpw5hzvv/Anz59/CP/7xMl/60nm4XJk0NTWyevWv2Lt3N4qi0tBwmL17d5802b///na+/vVCLBYLFouFGTO+zgcf7Dzp/ofTfPax2WfvsNAlF1UJYXrnnDOe9PQM3nlnGy+88DwzZwaT7qOPriEtLZ0//GE9f/zj0+TnT8LrPfntTAdzSdKpzGd/4YXTeO+9d/j+97+Lx+OhoGA6v/nN78nNHclTTz3B8uXhmSI+JpN9sM9eunGEEMGB2j/84TGqq/dz2WVXANDe3kZmZhYWi4WKir3s2nXyFjrA1KkX8tJLL+D3+/F43Lz88kv9rnNGzWd/JoqzBwdo5W5VQoivfe3rrFnzX8yZ802sVisAN954E8uXL2Hr1hfJzc3l/PMn97ud2bO/yd69e7nhhuvIzMzi/POnUFt7sN/1zrj57KNhMNMlALy6q5Y/vfgJj955BVaLOW5gMtwvo48EiXn4kekShk6mSxiAOIfMfCmEEEeL7WQvg7RCCAHEbLIP9su5pWUvxGk3jHuGY8ZgfsYxmux7unGkZS/E6aSqGoGA/L+LtEDAj6oObDwyNpO9Pdiyl24cIU4vpzOBtrYWDMNcg6qnk2HotLU143T2PRB7IrF56mV3y166cYQ4vRISkmluPkxd3QFg6N05qqqi6+aqOPqPWcFmc5CQkDyg7cZksnfKAK0QUaEoCmlpmWHb3nA/1TQSIhVzv904JSUlFBQUMGHCBHbv3h1a7vF4WLp0KVdddRWzZs3i5z//eeizyspKioqKmDFjBkVFRVRVVYW94CfTM0ArffZCCBHUb8t++vTpfO973+Pf//3YOZ1XrVqF3W5ny5YtKIpCQ0ND6LOlS5dSXFzMnDlz2LhxI0uWLOHJJ58Mf+lPwGZR0VQFt8x8KYQQwCm07KdOnfq5aTw7OjrYsGEDt912W2g6goyMDAAaGxspKyujsLAQgMLCQsrKymhqagp32U9IURQcNk1a9kII0W1QZ+NUV1eTkpLC6tWr+eY3v8kNN9zAe++9B0BtbS1ZWVloWvC0IE3TyMzMpLa2NnylPgVOu0yGJoQQPQY1QOv3+6murmbixIksXLiQXbt28eMf/5iXX345rIU70RwPpyIhzoZOcLDDLMwUaw+JOfaZLV6ITMyDSvY5OTlYLJZQV815551HamoqlZWV5OTkUFdXRyAQQNM0AoEA9fX1p3RHl+MNdiI0lysRq6ZwpM1tmpF8OWvBHMwWs9nihaHFHPaJ0NLS0pg2bRpvvvkmEDz7prGxkTFjxpCenk5+fj6lpcH5m0tLS8nPzyctLW1QhR8s6cYRQohe/bbsV6xYwdatW2loaGDevHmkpKSwefNm7r33Xu655x5KSkqwWCysXLmSpKQkAJYtW8aiRYtYu3YtSUlJlJSURDyQ4zlsGofkPHshhABidD57lyuRB/70Ljt2H+ahn345AiUbfuRw1xzMFrPZ4oVh1o1zJnDaLXTJefZCCAHEcrK3afj8Ov6AuebVEEKIvsRssnfYuydDk9a9EELEbrJ32mROeyGE6BG7yd4evIJXkr0QQsRwspduHCGE6BWzyV66cYQQolfsJvuebhy5sEoIIWI32TtCLXvpxhFCiJhN9j0te7d04wghROwme7tVQ0G6cYQQAmI42SuKgkNmvhRCCCCGkz0Eu3KkG0cIIWI+2ctkaEIIAbGe7G0WOc9eCCGI8WTvsGu4ZYBWCCFiO9kHW/bSjSOEELGd7O2anHophBDEeLJ32Cy4pWUvhBD9J/uSkhIKCgqYMGECu3fv/tznq1ev/txnlZWVFBUVMWPGDIqKiqiqqgproU+V027B4wsM6j62QggRS/pN9tOnT2f9+vXk5uZ+7rOPP/6YnTt3kpOTc8zypUuXUlxczJYtWyguLmbJkiXhK/EAOG3dUyZIV44QwuT6TfZTp04lOzv7c8u9Xi/33XcfS5cuRVGU0PLGxkbKysooLCwEoLCwkLKyMpqamsJY7FPTM6d9p5x+KYQwOctgV3z44YeZPXs2o0aNOmZ5bW0tWVlZaFqwVa1pGpmZmdTW1pKWljagfaSnJwy2eLhciWS5gus74x24XImD3taZwgwxHk9ijn1mixciE/Ogkv2OHTv48MMPufPOO8NdnmM0NrYPqr/d5Urk8OE2fG4fADWHjhBvUfpZ68zWE7OZSMyxz2zxwtBiVlXlhI3kQZ2N8+6771JRUcH06dMpKCjg0KFD3HTTTbzxxhtkZ2dTV1dHIBA8CyYQCFBfX99nV1Ck9XTjyLn2QgizG1TLfv78+cyfPz/0vqCggHXr1jF+/HgA8vPzKS0tZc6cOZSWlpKfnz/gLpxwkAFaIYQI6jfZr1ixgq1bt9LQ0MC8efNISUlh8+bNJ11n2bJlLFq0iLVr15KUlERJSUnYCjwQTrvch1YIIeAUkv3ixYtZvHjxSb/zyiuvHPN+3LhxPPPMM0MrWRg4pRtHCCGAGL+C1i7dOEIIAcR4slcVBYdNk5a9EML0YjrZQ88NTKRlL4Qwt5hP9g6b3JpQCCFiPtnLrQmFEMIMyV5a9kIIEZvJ3jB6p1hwSMteCCFiL9l3/eM3NL36VOi93HRcCCFiMNkbPjede94LvXfYNUn2QgjTi7lkr6WPxtdYg+H3AsGWvdsbQDfkblVCCPOKuWSvZowBQ0dvOgD0TpngkX57IYSJxVyy19JHAxBo3A8Eu3FAJkMTQphbzCV7JTEDxR6H3p3snbbuydCkZS+EMLHYS/aKij1zTKhl39ONI+faCyHMLOaSPYAt6yz0xmoMXcfZ040j8+MIIUwsJpO9fcRZ4PdgtNaHunHcMvOlEMLEYjLZ27LygOAgrQzQCiFErCb7jFGgaOiN+3vvViUDtEIIE+s32ZeUlFBQUMCECRPYvXs3AM3Nzdx8883MmDGDWbNmceutt9LU1BRap7KykqKiImbMmEFRURFVVVURC6AvisWKmppDoHEfjp67VUnLXghhYv0m++nTp7N+/Xpyc3NDyxRF4Yc//CFbtmxh06ZNjBo1igceeCD0+dKlSykuLmbLli0UFxezZMmSyJT+JNSM0egN+9FUFZtVlQFaIYSp9Zvsp06dSnZ29jHLUlJSmDZtWuj9+eefT01NDQCNjY2UlZVRWFgIQGFhIWVlZce0/E8HLX00RtcR9M4WmQxNCGF6Q+6z13Wdp59+moKCAgBqa2vJyspC04LdJ5qmkZmZSW1t7VB3NSBq95W0emN1cJpjORtHCGFilqFuYPny5cTFxXH99deHozzHSE9PGPS6meMnsg9wuutIjE8jALhciWEr23AU6/H1RWKOfWaLFyIT85CSfUlJCfv27WPdunWoavAgITs7m7q6OgKBAJqmEQgEqK+v/1xX0KlobGxH1wc+W6XLlUhTu4GSmEHrvt1Y1YtpbfNw+HDbgLd1pnC5EmM6vr5IzLHPbPHC0GJWVeWEjeRBd+M8+OCDfPTRR6xZswabzRZanp6eTn5+PqWlpQCUlpaSn59PWlraYHc1aFr6aPTG/ThsmgzQCiFMrd+W/YoVK9i6dSsNDQ3MmzePlJQUHnroIdatW0deXh7f+c53ABg5ciRr1qwBYNmyZSxatIi1a9eSlJRESUlJZKM4ATV9NP6qHSSM0NkvA7RCCBPrN9kvXryYxYsXf275Z599dsJ1xo0bxzPPPDO0koVBcJDWIEtpZrtHi3ZxhBAiamLyCtoeWsYYADL0w3R5/cfciFwIIcwkppO9Ep8G9nhSfHUYBnh9erSLJIQQURHbyV5R0NJHk+ipA2SaYyGEecV0sodgv31c1yFUdLmKVghhWjGf7LX00ai6H5failtmvhRCmFTMJ3s1IzhtQq6lWVr2QgjTiv1kn5KNoVoYqTVJshdCmFbMJ3tFtWAkZ5NraZLJ0IQQphXzyR5ATRtDrtZMl8cX7aIIIURUmCLZW1xjSFTd6B0t0S6KEEJEhSmSvTUzeCWtrb0myiURQojoMEWy19JGAeDskGQvhDAnUyR7xeakyUgi0X0o2kURQoioMEWyB2hQXaT5amUyNCGEKZkm2R+05ZGgt6E3VEW7KEIIcdqZJtnXOCcQQMVX/k60iyKEEKedaZK95kxgjz4Sf/k7GIZMdSyEMBfTJPtxuUm80zkGo6OZwKE90S6OEEKcVqZJ9heMd/GxdyQBxYJfunKEECbTb7IvKSmhoKCACRMmsHv37tDyyspKioqKmDFjBkVFRVRVVZ3SZ9GSkmBnVG4Ge8jDX/Euhi7z5AghzKPfZD99+nTWr19Pbm7uMcuXLl1KcXExW7Zsobi4mCVLlpzSZ9E0ZYKLN1pHYrjbCNR8Eu3iCCHEadNvsp86dSrZ2dnHLGtsbKSsrIzCwkIACgsLKSsro6mp6aSfRdsF41184svFr9rx7X072sURQojTxjKYlWpra8nKykLTNAA0TSMzM5Pa2uBFSyf6LC0tbUD7SU9PGEzxAHC5EvtcNjo3jT2+sUzc9z4ZqQ4Ui3XQ+xhu+oo51knMsc9s8UJkYh5Usj9dGhvb0fWBX/HqciVy+HBbn5+dNy6d17blkJ/4CbU73sKad8FQizksnCzmWCUxxz6zxQtDi1lVlRM2kgd1Nk52djZ1dXUEAsFBzkAgQH19PdnZ2Sf9bDiYMt7Fbl82PkscfunKEUKYxKCSfXp6Ovn5+ZSWlgJQWlpKfn4+aWlpJ/1sOMjJiCcrPYHPGId//04MnyfaRRJCiIjrN9mvWLGCyy+/nEOHDjFv3jxmzpwJwLJly3jqqaeYMWMGTz31FPfee29onZN9NhxMmeDileYc8Hvx79sR7eIIIUTEKcYwngYyEn32APsOtXHfE/9iZdZG4nLG4Zxx21CKOSxI36Y5mC1ms8ULw6zP/kw3OiuB9GQnn6ln46/+AMPTEe0iCSFERJky2SuKwgXjXfy9MRv0AP7K7dEukhBCRJQpkz3A1AmZVHnT8DjSZdpjIUTMM22yH5ubRHK8nc/UswnUlKG3HY52kYQQImJMm+zV7q6c5+tHg6rh2f58tIskhBARY9pkD8FTMA97nTRnX4x/zxvoLbXRLpIQQkSEqZP9+FEpxDssvOb9Img2PO/9LdpFEkKIiDB1srdoKpPPcfFOpRtt0pX4K/5FoGFftIslhBBhZ+pkD3DJuSPo8vjZpp8Htjg87/012kUSQoiwM32y/8KYVCbmpbLxX3Wo515NYP8uAnV7o10sIYQIK9Mne4BvXTGO9i4fr3ROQHEm4fnXswzjWSSEEGLAJNkDZ2UnMWWCixffO4Q+6esEaj8lcLAs2sUSQoiwkWTf7RtfHovHF+ClpjyU+DQ87z4nrXshRMyQZN8tJyOeS7+Yzd931OGbOBP9cAWBfTujXSwhhAgLSfZHmXPpWYDBhpoRKMlZeN57DsPQo10sIYQYMkn2R0lPdlBwwUje+KiOrgnXoDcdwPfhy9EulhBCDJkk++Ncc/EYbFaNZypSseRdgOedv+Cv+STaxRJCiCGRZH+cpDgbV180mvd2N1D3hSLU5Czcf1+L3t4Y7aIJIcSgDTnZv/rqq8ydO5c5c+Ywa9Ystm7dCkBlZSVFRUXMmDGDoqIiqqqqhrqr0+aqC0eR4LTy3Fs1OK/6KUbAR9fLqzH83mgXTQghBmVIyd4wDO666y5WrlzJxo0bWbVqFQsXLkTXdZYuXUpxcTFbtmyhuLiYJUuWhKvMEee0Wyi8JI+Pq5r5sMGK46vz0Q9X4n7jT3I6phDijDTklr2qqrS1BW+O29bWRmZmJs3NzZSVlVFYWAhAYWEhZWVlNDU1DXV3p81XJ+cy0hXP4y9+SlfmudgumI1/9+v4Pnk12kUTQogBswxlZUVReOihh1iwYAFxcXF0dHTw6KOPUltbS1ZWFpqmAaBpGpmZmdTW1pKWlhaWgkea1aIyf9Yk7vvjuzzxwqfc+s05BA5X4XlzPWraKCwjzol2EYUQ4pQNKdn7/X4effRR1q5dy5QpU9i+fTt33HEHK1euDEvh0tMTBr2uy5U45P27XIncOHMSv3/+I3ZWtnDldXdy8A934f3HGlw3rcKSOLwqrnDEfKaRmGOf2eKFyMQ8pGT/ySefUF9fz5QpUwCYMmUKTqcTu91OXV0dgUAATdMIBALU19eTnZ09oO03Nraj6wPvI3e5Ejl8uG3A6/Xl4nwX2z5I5bENH5KTehGu6bfSuWE5B/60DGfhQlRnUlj2M1ThjPlMITHHPrPFC0OLWVWVEzaSh9RnP2LECA4dOkRFRQUA5eXlNDQ0MGbMGPLz8yktLQWgtLSU/Pz8M6YL52iqonDTzIlYNZXfbvoYIzkH54zb0Fvr6dq8CsPdHu0iCiFEvxRjiKeXPP/88/z2t79FURQAfvrTn3LllVdSXl7OokWLaG1tJSkpiZKSEsaOHTugbQ+Hln2P9z6tZ+2Gjyi8JI9vXj4W/4GP6NryEGpqLnEz70Kxx4d1fwMlLSBzMFvMZosXIteyH3Kyj6ThlOwBfr+5jLc+OsTC4gsYPyoF//5ddG19BDV9FHEz/x+KLS7s+zxV8p/CHMwWs9nihWHajWM2xVeOJyPZwe9Ky+jy+LGMPg/n125Bb9hP54v/ieHtinYRhRCiT5LsB8Bpt3Bz4SQaW908+vzH+AM6ljGTcVz5f9DrK+h66UEMnyfaxRRCiM+RZD9AZ49M5oarJvBBeSO/3VSGrhtYz5qKo+BHBOr20PnCKvTOlmgXUwghjiHJfhC+MjmX6756Nu9+Ws8TL36KbhhYx03DMX0BeuN+Ov+6TG5aLoQYViTZD9LV00Yz+9I83viwlj//fQ+GYWAdeyFxc38OmpXOTb/E+8lr0S6mEEIAkuyHZM5lZ/G1qaP4+/YDbHi9EgAtbRTx31iKlpOP5/UncP/vExgBX5RLKoQwuyFdQWt2iqLwneln4/H52fRWFQ67xtenjUFxJOC8+md43/sr3p2lBJqqcX7tVtT41GgXWQhhUtKyHyJFUfjejC9wUX4mz7xazj+2HwguV1XsF12L48pb0JsO0PnXpfirdkS5tEIIs5KWfRioqsIPCyfi8+usf3k37V0+Zl+ah6IoWMdeiJqSg/vVdXRtfRjL+MtwXFIc1QuwhBDmIy37MLFoKv9n7rlc+sURbHyjkj9t+Sx09a+Wlm8/ECkAABXgSURBVEvc3KXYJs/Cv+dNOp79Of6DZVEusRDCTCTZh5FFU/nBNfnMvHgMr+2sYc3fPsTrCwCgaBbsF36LuDmLQbPStXkl7jefwvDLRVhCiMiTZB9miqLwrSvG8d0rz2Hnngb+8y876XT3no2jZY4j/lv3Yp10Jb6P/07Hc0vw79sptzsUQkSUJPsI+drUUfxoziTKa1r55fr3aW7rbcErFjuOS6/HOfMuALq2PERXaQmBw5XRKq4QIsZJso+gi/KzuOO682g84mbFk+/x2f7mYz635E4k/tv3Y7/0evTmg3T+7V66/rEOve1wlEoshIhVkuwjbGJeGov+/QJsFpWVT+9gw+sVBHQ99LmiWrBNupL476zEdn4h/qrtdPzlbtxv/xnD0xHFkgshYokk+9NgdFYiS75/IRdPGsHzb1ax6r930NTqPuY7is2J/aJriS8qwXL2xfg+2ELHnxfiLXsFQw9EqeRCiFghyf40cdot/LBwIjcXTmRffTtL//Av3t/9+e4aNSEN51duIu5b96KmjcTzxpPBC7LkVE0hxBBIsj/NLj53BMvmXUhGipPVf/2QP239DI/38y13LX00zsKFOK68BcPnpmvzSrq2/hd6a30USi2EONPJbQmjxB/Qee6f5Wz5VzUZyQ6+//UvMDGv7xuyG34v3g+34t2xCfQA2ohzUOLTUBPSUOJTURPSSBs5iiN+p6muzD0Tfs/hZraYzRYvDON70Ho8Hn7xi1+wbds27HY7559/PsuXL6eyspJFixbR0tJCSkoKJSUl5OXlDWjbsZzse3y2v5knXvyUuuYuLj8vm+u+ejZxDmuf39U7mvHu2ESgcT9GexNGZzMc9+tTHIkoyVmoSZmoSVmoyVmoKdmoabkoamzNjnEm/Z7DxWwxmy1eGMbJfsWKFaiqyt13342iKDQ0NJCRkcH3vvc9vvWtbzFnzhw2btzIc889x5NPPjmgbZsh2QN4fQE2vlnJlneqSYy38r2rJjB5vKvf9Qw9gNF5BKOjiQStiyMHq9GP1KG31qMfqcPoaOr9smZFzRiDljmu+zEWJSEdRVEiGFlknWm/53AwW8xmixeGabLv6Ojgiiuu4J///Cfx8fGh5Y2NjcyYMYN33nkHTdMIBAJMmzaNrVu3kpbWd1dFX8yS7HtUHWrl8Rc+pbq+nQu/kMl3pp9DaqL9lNbtK2bD70VvPYzefIBAfQWB+nL0hioI+AFQnEmoqbnBln9KDmpqDmpKNkpcyhlRCZypv+ehMFvMZosXIpfsh3RcX11dTUpKCqtXr+add94hPj6e2267DYfDQVZWFpqmAaBpGpmZmdTW1g4o2Z+o0KfC5Uoc9LrR4nIlMnliNs+9uoc/b93Nrr0NzPryWK4tOIeEONsprf852enAF0JvjYAPb90+3DV78NSW42s4gLf8bQxPZ+g7qj0Oa8ZIbBkjsbpGYcsYhS1jJFpSxrCrBM7E3/NQmS1ms8ULkYl5SMne7/dTXV3NxIkTWbhwIbt27eLHP/4xDz/8cFgKZ7aWfY+C83L44phUNrxewV9f3cuLb1VxzcVjuHLKSGxWrc91BhSzNQvGZKGMuQwbYDUMjM4W9JZa9OYa9JYa/M01eHe/h7HrlaPWcwTHAhJdKIkZqIku1KQMlEQXakI6itUx9OAH4Ez/PQ+G2WI2W7wwTFv2OTk5WCwWCgsLATjvvPNITU3F4XBQV1dHIBAIdePU19eTnZ09lN2ZiivFyc2zJnH1tDE8989ynn2tnL+/V82cy87isi9lo6nhO2tWUZTgWT3xqZA78ZjPdHdbsAJorkFvPhgcD2ipRa/+AI6/3aLVgeJMRo1LDnYFxSWjxKWipmShJmejJrlQtL4Hn4UQkTWkZJ+Wlsa0adN48803ueyyy6isrKSxsZG8vDzy8/MpLS1lzpw5lJaWkp+fP6AuHBE0KjOB2799Hp/tb+bZ18r540ufUfpWFQUXjOTL5+WQ4Ixs8lQdiajZEyB7wjHLDcPA6DqC0daA3nYYvb0pOFjc2YLRdSR4xlD1EfB19a6kKMGjgOQRqMlZwcrAkYjiSEBxJKE6ElAciWCLQwljZSaECMPZONXV1dxzzz20tLRgsVi4/fbbueKKKygvL2fRokW0traSlJRESUkJY8eOHdC2zdqNcyKGYbCrvJGX363mk33N2Cwq/zZpBFdOHcnkidnDMmbD24necgj9SPej5RD6kVr0I/Vwwrn8FbDHodjighWBLQ7FHo+SkIaako3WPZicOXLEsIw5kmL1b/tEzBYvDNOzcSJNkv2JHahv5+/bD7Dt40P4/DpfOjuDi77g4pyRKWQkO4bdQGpfDL8Hw92O4W7D6GoLPrvbMDydGJ52DE/Hsa/bG0NnEgFo8SmQNAI1KRPFmdT7cCQGn+1xoAcwdD8EAhDwBecZ0v0oFjtY7ShWR/drB4o2/K9DMMPf9tHMFi9Ish8QM/2BtHf5eH1XDa/tquFwc7DLJDnBxtm5ycHHyGTGZCVi0c78bhFD1zHaG0KDyNauBjoP7cNoO4zhbgdD738jJ6NqwSMIezx0Pyv2hOCz46j3joTe71ns0F2BGAE/6P5ghWTooKigqqBooKooihbch8056K4qM/1tg/niBUn2A2LGP5C09AR2ltVSfvAIew4eYe+BIzQcCc6sadFUxmQlcFZOEmNzkhibk4zrDGn9n8zRv2fD0MHTie5uDR4ldLVieDtRVA1UC2iW4BXEmgVUDfxeDJ8Hw9cFPg+Gzw0+N4a3s/uIogPD3RE6qsDn7qc0A9XdVdVTqTgSgoPXinpUJRF8KBY7ijMRxZFIcqaLNp81ePRijw9WHqrWvU6wMkHpqUSMo56M3v0qyhnzuzfj/+VheTaOGD40VWF0ViKjsxL56gUjAWhp97D3wBHKa45QUdPK/+6s4e/vHQAgMc7KWdndyT87ibNykog/wTQNZwJFUcGRgOZIgJTwb9/Q/cEuJXd38ve0B1/7vcdVJBYUTQu25g0ddB3DCICugxGAgB/D29VdobT3VijudoyADwwdQ9eD6/as7/dA970NwjcNntJdQSiAimK1HzVYnhA6gsFqD5ZdD3SXLdBdru6jGT0Q7CLrObLBQHUmdc/dlI6SkBYca4lPA8PA8LkxvF3g68LwuoOVraEHf249P8Oen6eq4SUJvcUDWnel3VO5qZbgMkULa8Vl6H7wB38PWOxnRNfeqYqdSMTnpCTYmfqFTKZ+IRMITr528HAHFbWtVHRXAB+WN4bafFlpcYztrgDG5SYx0pUQE90/4aCoFhRnEjiTorJ/Q/djuDtIdeo01h4KHr14O7uTbgCMQPd4RHciBlAUoDsRKt3/GEZvRWIYgBGsXHzu7jGTdvTWegx3BYa77aht9dEl1XOUFErOwXTib9iH0XmE3qOJwevs/yu9RzRHHdUo3Ucw9FQEitr781CU3veGHjzK83u7k3zg89vuHtdRrA6w2EJHXUrPNnqOxrpf9x5pqb1HWd0/6+DPv+eISw9WLnp3I6CnAs3OQ7l43pB/dseTZG8iFk1lzIhExoxI5KuTcwHodPupOtRKRU0rlbWtlFU1se3jQwDYLCp52cHEf3ZOMuNyk0mK7/9KXhF+impBiUvG5krEoqSeln0ahhFMfoNoPRsBP0ZHM3pHE0Z7I3pHczA52pzBQXGbM/QaRese6+geQA/4QfdhBAIkJVhpbWnvHmgPdH/vqNfdlV0oaRr6sd1WoeQafA72Wuu9yxUFxWIDzRZ8tthQLMHuNMPn6e7uc3e/7n7uqShDR18BDMPXffSmB7sUQ6+DlUeo8jn6WVWOOaJRLDZQNSyJ6UTidkWS7E0uzmFhYl5aaHplwzBoavVQXnOEvQePUH6wla3/quZFfT8AqYl2cjLiyc2IJ6fnkR5PnEP+lGKNoiigDO73qmgWlCQXalL/E/qdTIIrkS6T9dmnRWicQv6HimMoikJ6soP0ZAcX5WcBwVk599W1sffgEQ7Ut1PT0MlrOw7i9fee/ZKeZCdvRBJ52YnkjUhizIjEiF/wJYQ4dZLsRb9sVo1zRqZwzsjekU9dN2hodVNzuIODDe1U17dTdaiN7UfdatGV4mBMViJZaXFkpcaRleYkKzWOxDjrGXM2iBCxQpK9GBRVVchMcZKZ4uT8czJCyzvcPvYdaqPqUBuVta3sr2/n/d0N6Eed4eu0a2SlxjEirfuRHnzOSovDfoKJ3oQQQyPJXoRVvMN6zBgABM8Cajzipq65k7qmLg41d1Lf1MmeAy28XVZ3zPppSXay04PjANkZceSkB8cFpEtIiKGRZC8izqKpwa6ctDgYd+xnHl+AuqZODh31qG3o5J/Vx44JJMXbGJHqxJXixNX9PP4sH1bDkG4hIU6BJHsRVXarFroY7Gi6YdB0xE1NYwc1DZ3UNHZQ39RJ2b5mmj86dMx3LZqCw2bBYdNw2Cw47dpR7zXsPc/W3uWpiXbSkx2kJTqwWuRaAhH7JNmLYUlVFDJSnGSkOPnScUcDXl+AhiNuvAbs2ddES7sHtzeA2+PH7Q3Q5fHT2umlviWAx+vH4wvg9gaOvzd7SHK8LXgGUpKDpHgbCU4rCU4r8U4LCQ4r8U4rTrsFm0XFYlGxaio2qxrWewoIEWmS7MUZx2bVyMmIx+VKJM8V3/8KBK8f8Pl1PL5gZdDU6qGx1U3jETcNrW6aWt3sr2ujtdNHl8ff/wYJVkh2m0ai00pCXLCCSHRaSYyz9b6PC75P7H7ttFuky0lEhSR7YQqKomCzatisGolxNjJT40743YCu0+H209Hlo7374fEG8Pp1fKFHAF9Ax+0J0N7lo63LR0u7hwOH22nr9OHz9z0Dp6YqxHcfOfQ+LMQ7rSQ6e48qeiqLBKcVh01DVRVUqSTEEEiyF+I4mqqSFGcj6RRu8n4iHm+Ati4vbZ0+2jp9tB/z2heqSOqaOymvCb73B04+l4wCaFow6SuqgkXtrsAsKlaLht2qYrNqWLu7miwWFYumBF93v7dbg2MXdpuGo/v56GV2a/A7tu5lqhqsYAzDwCB4fUVPd5hFO3NmzxSS7IWICLtNw25zkpHsPKXvG4aB2xsIHUm0d/lo764Y3F4/ugEB3cDhsNLe4UHXDfwBPXS04fUFgg+/Tqfbjz+g4wvo+AM6fr+OLxD8/omOOE5EU5Vggj/B5xZNwWrprkw0FatFxXZUxWOzqNhtGjaLhkVT0FQVTVPQVAVNU7B0v++tnIIVlEULbmfEEQ9dnZ7Q4HvPQHtPJSROnSR7IYYBRVFw2i047RZcKSeuIIY6v7uuG3h8gd6HNzh47T1umccXHN/wB/TgkYRC77OqYBj0Vih+o/s5+L6n4vH4dTrcPrzd2woEdAK6gV83CAQMAgF90PNiho5y1O4KRA1WIKraU5H0LuupsHyBo7vhghUhCsecuXV0haJpCprSu83QvrTe7Vq0nvdqcNLLnvJ1H/EcvSy04KgYVLX3Z9vTVZc/LkCSPfwXF4Yt2a9evZpHHnmETZs2MX78eCorK1m0aBEtLS2kpKRQUlJCXl5euHYnhBgEVe2tVIaDniOUYMVh4PfroUrE59dxxNmoq2/H7Q2eadVTMfl1A7374dd1dN0g0PMIGAR0/ajXRjAxd3dvWS1qqKtL7z6iOnr7Le0evL7ebeqGEaqojt6HHqH7PqUlOXhgwSVh325YfuMff/wxO3fuJCcnJ7Rs6dKlFBcXM2fOHDZu3MiSJUt48sknw7E7IUSMUFUFmxocI+iLy5XI4WTHaS7VqQlWAr0VS2ia+u7xDYzjZvQ/rnLQjeB39e4KRTeCld/YMWm4OzxhL++Qk73X6+W+++7jgQce4MYbbwSgsbGRsrIyHn/8cQAKCwtZvnw5TU1NpKWlnWxzQghxRlAVBdWiYCW811skxtkikuyHXMqHH36Y2bNnM2rUqNCy2tpasrKy0LRgba1pGpmZmdTW1g51d0IIIQZhSC37HTt28OGHH3LnnXeGqzzHONGNc0+Fy5XY/5dijMRsDmaL2WzxQmRiHlKyf/fdd6moqGD69OkAHDp0iJtuuom7776buro6AoEAmqYRCASor68nOzt7QNtvbGxH1wc+CCJ3pDcHiTn2mS1eGFrMqqqcsJE8pG6c+fPn88Ybb/DKK6/wyiuvMGLECH7/+99zzTXXkJ+fT2lpKQClpaXk5+dLf70QQkRJxM6/WrZsGYsWLWLt2rUkJSVRUlISqV0JIYToR1iT/SuvvBJ6PW7cOJ555plwbl4IIcQgDY8rK05gKJdEm/FyaonZHMwWs9nihcHHfLL1FMOI0GVgQgghhg25+4IQQpiAJHshhDABSfZCCGECkuyFEMIEJNkLIYQJSLIXQggTkGQvhBAmIMleCCFMQJK9EEKYQEwl+8rKSoqKipgxYwZFRUVUVVVFu0hhV1JSQkFBARMmTGD37t2h5bEae3NzMzfffDMzZsxg1qxZ3HrrrTQ1NQGxGzPAggULmD17NnPnzqW4uJhPPvkEiO2Ye6xevfqYv+9YjrmgoICrr76aOXPmMGfOHF5//XUgQjEbMeSGG24wNmzYYBiGYWzYsMG44YYbolyi8Hv33XeNmpoa46tf/arx2WefhZbHauzNzc3G22+/HXr/q1/9yrj77rsNw4jdmA3DMFpbW0OvX375ZWPu3LmGYcR2zIZhGB999JFx0003GV/5yldCf9+xHPPx/497RCLmmEn2DQ0NxpQpUwy/328YhmH4/X5jypQpRmNjY5RLFhlH/5GYKfaXXnrJuPHGG00V89/+9jfjG9/4RszH7PF4jOuuu87Yv39/6O871mPuK9lHKuZhPevlQJzsvrexftMUs8Su6zpPP/00BQUFpoj5P/7jP3jzzTcxDIPf/e53MR/zQO9nHQsxA9x5550YhsGUKVP42c9+FrGYY6rPXsS25cuXExcXx/XXXx/topwW999/P6+99hp33HEHK1eujHZxIqrnftbFxcXRLspptX79ep5//nmee+45DMPgvvvui9i+YibZZ2dnh+57Cwz6vrdnIjPEXlJSwr59+3jooYdQVdUUMfeYO3cu77zzDiNGjIjZmI++n3VBQUHoftb79++P2ZiBUBw2m43i4mLef//9iP1tx0yyT09PN+19b2M99gcffJCPPvqINWvWYLPZgNiOuaOjg9ra2tD7V155heTk5JiO2Yz3s+7s7KStLXhjccMweOGFF8jPz4/Y7zmmbl5SXl7OokWLaG1tDd33duzYsdEuVlitWLGCrVu30tDQQGpqKikpKWzevDlmY9+zZw+FhYXk5eXhcDgAGDlyJGvWrInZmBsaGliwYAFdXV2oqkpycjILFy5k0qRJMRvz8QoKCli3bh3jx4+P2Zirq6v5yU9+QiAQQNd1xo0bx+LFi8nMzIxIzDGV7IUQQvQtZrpxhBBCnJgkeyGEMAFJ9kIIYQKS7IUQwgQk2QshhAlIshdCCBOQZC+EECYgyV4IIUzg/wP2LSe30z8yvgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt \n",
    "import numpy as np \n",
    "sns.set()\n",
    "\n",
    "x = np.linspace(0, N_EPOCHS,N_EPOCHS)\n",
    "\n",
    "plt.plot(x,t_loss)\n",
    "plt.plot(x,v_loss)\n",
    "plt.title(\"Loss\")\n",
    "plt.legend([\"Train loss\", \"Valid loss\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f703dbe3bd0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAELCAYAAADTK53JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXyU5b3//9d93zOTfZ1sww5hC8iOKLJUWRrFaGxdsKk9tVU8PW6nWs9Pjj0CHu0Sz692sdqeelpbi7VW24pGirhUBRRQRAkEAllIWCaTZLJOllnu+/7+MRhFliwkTDLzeT4eecww3PfM58ryzpVrrvu6FNM0TYQQQoQdNdQFCCGEGBgS8EIIEaYk4IUQIkxJwAshRJiSgBdCiDAlAS+EEGFKAl4IIcKUJdQFCHGulixZQn19PZqmdT22adMmMjMzefDBB9m5cydVVVX88Ic/5Ktf/WoIKxXi/JKAF2Hh17/+NZdccskpj0+ePJkVK1bwP//zPyGo6mSBQACLRX7kxPkjQzQirH39619n/vz5REVFdXvsO++8w4oVK5g1axaLFi3it7/9bdf/vfHGG+Tn5zN79myWLVvGu+++C4DL5eI73/kO8+bNY/ny5fzlL3/pOufxxx/n7rvv5r777mP27Nn8/e9/p7W1lQceeICFCxeyaNEifvrTn6Lrev83XAikBy9El+9///v87Gc/Y+7cuTQ3N3P06FEA9uzZw/33388vfvEL5s+fT11dHR6PB4Dvfe97jB8/ni1btlBRUcG3vvUtRo4cyfz58wF48803+fnPf86jjz6Kz+fj3nvvJS0tjc2bN9PR0cG//uu/4nA4uPHGG0PWbhG+pAcvwsIdd9zB3LlzmTt3LrfffnufnsNisVBWVobH4yEpKYmpU6cC8OKLL3LttdeyYMECVFUlMzOT7OxsnE4nu3bt4r777iMqKoqcnByuv/56NmzY0PWcM2fOZNmyZaiqisfj4d133+WBBx4gNjYWu93OzTffzKuvvtovnwMhvkh68CIsPPHEE6cdg++NX/ziF/zqV7/iJz/5CZMmTeJ73/ses2bNwul08qUvfemU42tra0lKSiI+Pr7rsWHDhrF3796uf2dlZXXdP378OIFAgIULF3Y9ZhgGDofjnOoW4kwk4IU4Yfr06fzqV7/C7/fz7LPP8t3vfpd33nkHh8NBdXX1KcdnZGTQ3NyMx+PpCnmn00lmZmbXMYqidN3PysrCZrOxfft2ebNVnBcyRCPCms/nw+v1YpomgUAAr9eLYRinPe7ll1+mtbUVq9VKXFxc17TL6667jr/97W+8//77GIaBy+WivLwch8PBrFmzeOyxx/B6vRw4cIAXX3yRq6666rS1ZGRksGDBAn784x/j8XgwDIPq6mp27tw5oJ8DEbkk4EVYu+WWW5g+fTq7d+/mwQcfZPr06XzwwQenPXbDhg0sWbKE2bNn8+c//5lHH30UCPbsf/SjH/HDH/6QOXPmcNNNN3H8+HEAHnvsMY4dO8aiRYu48847ueuuu1iwYMEZ63n00Ufx+/2sWLGCCy+8kLvvvpu6urr+b7gQgCIbfgghRHiSHrwQQoQpCXghhAhTEvBCCBGmJOCFECJMScALIUSYkoAXQogwNagup2tsbMMwej9r026Px+32DEBFg5e0OTJImyNDX9usqgopKXFn/P9BFfCGYfYp4D89N9JImyODtDkyDESbZYhGCCHClAS8EEKEqUE1RHM6pmnS2FiHz9cJnP5PmNpa9bQLSIWz3rRZ0yzExycTE3PmsTohRPgZ9AHv8TSjKAqZmSNQlNP/wWGxqAQCkRXwPW2zaZr4/T6amoILWknICxE5Bv0QTUeHh4SE5DOGuzg7RVGw2aJITk7H42kKdTlCiPNo0KemYeho2qD/Q2PQs1pt6Hog1GUIIc6jIZGcn98VR/SNfA5FuDBNk8M1rew+VA9AzqhksocnYbNqfXqu2qYOyo42AxBl1bBZVWwWDduJ+63tfuqaOqhv7qCuqTN4v6mDDp+OqiqoioKmKqhq8DbKqpGZEoPDHkeWPZas1OBHfKyVhuZOjta1cazeE7yt89DQ4mXtqotJj7f16+cJhkjADxarVn0Tv99PIODnyJFqxo7NBmDixEk88MDaHj3HSy+9iNfrZeXKr/fqte+88zZcLhdxccEx9NGjR/PQQz/i0KFSHnuskIMHS5k/fwGPPPJo7xolxBAQ0A1KjzSx+2Aduw/V09jqRT3RaSl67zAWTSF7WBKTR6eQMzqFMVkJWC3qaTs2re0+9lc1UnK4gZLDjdQ3d/aoBlVRSE2MIj05hpkT0oiNtnZdu6ObJqZhohsm7Z0Bahrb2Xe4kYD+2ftkmqqgf26uuz0xiuHp8cwYn8borETaPT2rozck4Hvhqaf+AIDTeZxbb/0Gv//9n045JhAInHW/zWuuua7Pr//d797HggWLgM/eZE1JSeXOO+/l0KFSPvxwR5+fW4j+YpombZ0B2jr8eP36Zx8+A59fp9Ov4/XpdPoCdH7u1uvTMYGoKAt+nw6AooBumJQdbabdG8BmUblgnJ2vLk5jxvg0NFXh0NEm9lc1cqCqiZe3VrJhayUQDNRom0aUTSPKqhFt0/AHTI7VeTCBmCiNyaNSyJ03ismjkrFaNXwnavX5g7X6AgZx0RbSk2NITYxCU3s+qm0YJu6WTmoa2qlxt9Po8ZKREsOI9HiG2eOIjf4sJ+JirBLwg9V1111FXl4+u3Z9wLBhw7nttttZt+77tLW14fP5uOSSBdx++78D8Nvf/i8dHR3ceed32bjxFV5/fRMJCYlUVJSTkBDPI488it2e1uPXTktLJy0tnaqqyoFqnhCnVdPQzp6yepwN7TR7fDS3+Whp89Lc5iOg9+yqzCibRvSJ8I2yaSgoaJ0BAv5g2H+639ysiWnMnpjOlDGpRH1hKGZ6dhrTs4M/M22dfkqrm3C620788vjcLxO/DibMnTyWqWNSGeNI6FVg95aqKqQnx5CeHMO0cfYBe52zGVIBv63YydY9zlMeV5TPvhH6auF0BwumOfp8fn19PY8//r8AeL1eCgt/SmxsLIFAgHvvvZPt29/j4osvOeW8/ftL+MMfniMzM4vCwkd48cXn+dd/veO0r/Gzn/3/PPXUrwC48cavcfnlp9/cWYjTafJ42VPuZk+5m6O1HqwWFZtVI8r66a1GTJRGenIMWalxOOyxZKTEYNGCIagbBmVHm/mkzM3HZfXUNLQDkBBrJSkuiuR4G8PssSTG20iKiyI+xkKU1UKUTSXqxPN/+jqfBrp6miGU9PQE6upa+9TGuGgrsyemA+l9/jyFkyEV8IPZ5Zdf2XXfMAyefPLnFBfvAUzcbjeHDh08bcBPnz6DzMwsAKZOvYAPPjjzMMvphmhE+DPN4J/6x+ra6PAG8AU+Gz749NZmUUmItZEQayUhxkr8ifsNLV72lNfzSbmbqppgaKYmRjF+eBK6bnYNnzR7fHj9Ou3eAC1tvq7XVhWFtORo0pKiqapppa0zgKYqTB6dwtI5I5gx3k5aUkyoPjWiG0Mq4BdMO30vezCEXWzsZ9/kzz//LK2tLfzmN78nKiqKwsIf4PN5T3uezfbZO+eqqqHr+oDXKgYH0wy+KecPGPh1g8CJ2/bOAEfrPBxxeahp6qDixPjzmVg09aQ3875IUSB7eBLXfmkcM7LTGJ4ed9ZZVR3eQNe4sbOhnRp3G7VNHUzPTmPmhDQuGJtKTNSQio6IJV+lAdDa2ordnkZUVBR1dbVs3foO11xzbajLEiHS6QtwvL6dY/UenPXtHKtv43i9h4ZW71mHFm1WlbHDkpiXk8HIjHhGZMQTH2Mlyqp1Da9YLSqqoqAbBm0dAVrbfbS2+2nt8NPa7iM2ysLUsakkxPZ8Cl5MlIWxjkTGOhL7ofUilCTgB8D119/Igw/ez7e+VUBGRiZz5lw4YK/ldB7n9ttvpbOzE5/Py1e+soJbbrmNvLxrBuw1xemZpom7uZPqWg/VrlaO1Ho4Uus5aRqeRVNx2GOZMCKZtORobBYNi6ZitZz40FSibBrD0uLISI4hMzOxR+PRmqqSGGcjMa7/51KLoUsxze7fnqysrGT16tU0NTWRnJxMYWEhY8aMOeW4jRs38qtf/QrTNFEUhaeffpq0tJ7PCHG7PaesiVxTU0VW1uiznjcYhmjOt760uSefy8HsXN586w+eDj8lhxto9vho6/QHpwJ2+mnvDNDa7qemoZ2OE0MpCpBlj2VkRjzD0+MZnhbH8LQ40pNjUNWeX3QW6jaHgrS551RVwW6PP+P/96gHv3btWgoKCsjPz2fDhg2sWbOGZ5555qRjiouL+eUvf8kf/vAH0tPTaW1tPWl8WYihqMnj5aODdewqraO0ugnjc/2h2CgLcTEWYqOtxEdbuHhKJiMz4xmVkcDw9LhTpvMJcb51G/But5uSkhKefvppAPLy8nj44YdpaGggNTW167jf//73fPvb3yY9PTg9KSEhYYBKFmJg+AMGDS2d1Ld0csTl4aNDdZQfbcYEslJjueLiUcyakE5GSgyxUZZe9cSFCIVuA97pdJKZmYmmBXsjmqaRkZGB0+k8KeDLy8sZMWIEX//612lvb2f58uX827/9W6/WQDndnxq1tSoWS/cXI/TkmHDT2zarqkp6+tD+xdtf9R+v97Bzn4uD1Y3UNrRT29hOY+vJM53GDUui4PLJXDLNwcjMhJCt5zPUv2Z9IW3uH/32Jquu65SWlvL000/j8/m49dZbGTZsGNdc0/M3+043Bm8YRrdjzTIG3zOGYQzpsc1zGZs1DJOK4y3sLqvj40P1ON3Bi3TSkqJJT45h6thU0hKjsSdFY0+MJiMlhtTE6K7z6+tDswm0jEdHhpCNwTscDlwuF7quo2nBedq1tbU4HCfPRx82bBiXX345NpsNm83G0qVL2bNnT68CXoj+1NDSyf6qRvZXNVJc4aa13Y+mKkwcmcyls4Yzc3wa6clykY7oH6ZpgmkEL6tXtUGxgmu3AW+328nJyaGoqIj8/HyKiorIyck5aXgGgmPz77zzDvn5+QQCAbZv305ubu6AFS4ij2GY/POjo7y28wgx0RbST/S+05JjSE+KJiUxmuP1bcFQP9yAq7EDgPgYK1PGpDBrQjrTxqUSG20NcUvCn9HkxGhrREsfi2Lr2y9R0zQwvW2Yna3g92HqPgj4Qfd/dh9A1bo+lE/vGwam14PZ6fnsttODGfCiRCeixiWjxKWgxCajxiajxCQGX6utEaOtEbO9MXi/vQl8nZhG4MTrBm/RA8HHDB0MI3hrfuEixa66LCiaJXif04S+ZiXxhv8P1P5fr6ZHQzTr1q1j9erVPPnkkyQmJlJYWAjAqlWruPvuu5k2bRpXXnkle/fuZcWKFaiqysKFC7nuur6vnCjE5x2t9VD43G5KqxoZPzyJaJvGkbo2Pi6rP2Vhq2ibxqSRyVw2aziTR6cwIiP+tGueiJOZAS9GwzGU6HiU2GQUS+9mwZn+TgLlO/GVvovhKjvxqIKaOgItawJa5ni0zPEoCengbcPwuDE89ZieBozWesy2BsyOFo742wh4mjG9nnNfZOpEDUpUHETHo1hsGPVVBDqaz/7cioISkxT8JWCNRrHFnAhpC2jWz+5/8ReLemLmlKGDoX/2S0APgHGGq5E1K1psIvT/YpI9mwd/vgz2efD33nsXixdfetJVqaZpcsMN+Xz/++uYOXP2ac/7wQ/WMXlyDtdeu/Ks68Fv3PgK77235bRruvfHevBDcR6816/z8rZKNu88QlyMlRsuy2b+1KyuP38N06Sp1UtdUwcNLcHlWEdnJXQtkDXUDeR4tGnoGHWVBI6VoB8rQXeVnRxCUXGocSkocSnB29gT92OTUeKSg78EohPRa8sJlL6Lv3wnBLyoyQ6skxajpo5Ary1Hd5Whu8rBH/yLCtVyathpNpT4VNSYRKKSUvCrMSjRCSc+4lGsMWCxnghXK1hswVtF+VxP+tNQ1VFQus7FFovyhVUjTcPA7GzBbGvCbG/E6GhB+bS9sSkosUnB0D5PQjoPXgRdeeXVPP/8sycF/O7du9A07Yzh/kWyHnzP7a1w88xrpdQ3d7JwuoN/u24m3vaTZ7oEN2GIPukN0XBl+r2Y/o4TQwX+zw1X+EFRTwTfiQD89MPQMTtaTvowOlowm2sIHD/QFbqqfRTWC5ahZWSDv/PEMEXTiSGLBgL1VZgdrcAX+4NK8DFrNNbsi7BOXoyakd31C9gyclqwdsPAaDqGXlOG0eJCjU1BSbCjxqehxKcGw/jEOefjTVZFVVFikyE2GRgzoK8VSkMq4P0Ht+EvffeUxxVF4Vz/ELFOWox14oKzHrN48aU89tiPqaysYOzYcQC8+urLrFhxFeXlZfzkJz+ms7MDn8/H1Vd/hRtuKDjlOT6/Hrzf7+enP32U3bt3kZ6ewahRY3pddzisBx/QDWoa2jlaG9zG7Ghd8BL/xlYvWamx3F8wi0mjUkiMs1HXfvpF28KN0dGCUV9F48EaOqoPotdXYbbU9s+TaxaU+DSs2RehDZ+CNmwyakz3686YRgCzvQWzvQmjvfFE77cJNTEDy7gLUaxn/iWrqCpa6ki01JH90wbRI0Mq4EPNarWyfPnl/OMfr3D77f9Oe3sbW7a8w3e+cydxcXH87GdPYrPZaG9v57bbvsm8efMZM2bsGZ9vw4a/4nQe549//AuBQIA77lh1yuykzwun9eA7vAF2ldbx/r4aDh5p6trKTFMVHPY4Jo9KZtywJBbPGIY1TK9xME0zGJZNzi98HMdsawSgA1AS0tHSRqNOXBDs6X6uh/5prx3TBN2H+fleve4HRUOJTUSNTkSJCX5gje7TDA9FtaDEp0J8KnKN7tAwpALeOnHBaXvZ53Me/JVX5nPffXdx22138OabrzN9+gzS0zNoaHDzy1/+mLKygyiKSn19HWVlB88a8B99tIsrrsjDYrFgsVjIzb2CPXs+PuPxQ309eN0wKDncyPt7a/joYB2+gEFGcgzL545kVGZwtcSs1NiwGT8/HaOllkD1JwSO7EGvKftsXBrAEoWa7EBzTEZLG4WaNoaMiVNo8Ayat8nEEDOkAn4wmDBhInZ7Gjt2vM/GjS93DcP87/8+QWqqnd/97lksFgv33HMHPp/vrM81iN7fHlBHaz1s2+tk+z4XzW3BJWwvmebgkguyyB6WOCjmC3fHaK0jUPUx+vH9wd6yJSo4y8RiO3EbhWKLRrHFQlQsii0WJSoOxRaD0VwbDPTqTzCaawBQkjKxjr8YNWU4arIDNdkRnLHxhc+FFhMPnsi66Ef0Hwn4Prjyyqv53e9+g8vlZOHCLwHg8bSSnT0Bi8VCRUUZn3zyMcuXX37W55k790I2bdrIkiXL0fUAr7++qWt3p6Gutd3HjhIX24prqHK1oqkK07PtXHJBFtOz0wb9sItpmhj1hwlU7SZweDdGwxEAlMQMFEsUZsAHAe+JW9+Zp8B9SrOgOSYTNXUplpHTUJPC4+ssBjcJ+D5YvvwKnnjiF+TnfxWrNXjRzDe/eQsPP7yGzZv/wfDhw5k5c1a3z3P11V+lrKyMb3zjBjIyMpk5cw5O57Fe1TKY1oM3TZPiCjfvfuLkk7J6dMNkVGY8X1s2gYumZJLYi00nzhfTNIMzS5prguPfn97WV2G2N4GioGVOIOrilVhGzUJNPn0wm4YO/k5Mbzum78SHtx28bSgxiWjDclCsUee5dSLSyTz4IWqwrQff5PHyx9dK2X2onsRYKxdPzWLBNAcjM848R7e3znX6nGmamG0N6LUV6LXlGHWV6O5q8H1uHFyzoiZloqYMxzJyOtqo6ajRoVv4StZliQwyD14MSqZpsr3ExZ9eP4gvYHDDZeNZNnfEeXuj1PR1oDccwaivwnAfCc4eUVUURQNVhRO3Zms9em05ZkdL8ETVgpo2Cuv4+cEx8KQs1OQslHg7ijK4h4+E6CkJeNFnTR4vz2wq5eOyerKHJ/LtFTk47HED+ppGSy2Bw7vQ66ow6g9jNLv49OIbJToBbDGfXdVoGsGhE0NHiU1GG3EBWvo4tIxxqPaRwSmGQoSxIRHwn24BKPquP0fiDNNk+74annvjUFev/csXjhywDTCMjhYCFTs5dvgDvMdKAVDi7Whpo7GMnx+cI542OnjpvHyfCNFl0Ae8qmroegCLRXpb58Lv96Fpff9ym6bJ4ZpWdpS4+OBALY2tXsYPT+JbKyb3e6/d1ANdY+X+svfRj+wFU8eaPgrbvOuD0wvj+3/lPSHCzaAP+JiYeFpbm0hOlrHRvjBNE7/fR1NTHQkJKb0+/2idh537XewsqaW2qQNNVZg2zs4Nl43nwskZ59RrN3V/cJGr2orgaoKtdcHb9saulf6UuFRs03OxjJ9P1uQpEffmmxDnYtAHfHx8Eo2NdbhcRzl1oaMgVVUxjMiaRdObNmuahYSEFGJiet7TbmwNzor5uKweRYEpo1O4cv5oZk9KJ+4c1lM3fR0EjuwhULmLwJE94O8ElOAqhQlpaMNyUBPSgh/JDtSMcfKLXYg+GvQBrygKqakZZz1GplX1H9M02VZcw5/fPIRfN7j2S+NYOH0YSXF9n8NuBrwEKj7AX74T/VgJGAGUmESs2RdhGTMHbdjkXq89LoTo3qAPeHH+NLR08odNpRRXuJkwIolvr8ghMzW2z8+nNx7Dv/9t/Ae3gq8DJSEd69SlWMbOQcsYf8oa3UKI/iUBLzBNky17nDz/1iF0w+RryyawdM6IPu2CZOp+ApW78O//J7qzFFQLlnFzseZchpY1UWa5CHEeScBHsNZ2Hzv317J1j5MqVyuTRyVz8xWTyUjpfa/daK7Bt/9tAge3YXa2oiRmEHXRDVgmLuzRWuNCiP4nAR9h/AGdT8rcvLe3huIKN7phMjIjnpuvmMzC6Y5e9dpNPUDg8C78+98OrrKoaFhGz8Q65TK04VPkzVEhQkwCPkK4GtrZ/OERduxz0e4NkBRvY/nckcy/IKvX68UYHS3492zCX7ol2FtPSMN24bVYJy1CjU0eoBYIIXpLAj7MlR9rZtOOaj46WIemKcydlMEl07KYMjq113PYTT2Af9/reHe9DAEvltGzsOZcijZiqvTWhRiEJODDkGGafFJWz6Yd1Rw62kxslIUV80ezbM4IkuJ7v2StaZro1Z/Quf05zGYX2sjpRM2/ES152ABUL4ToLz0K+MrKSlavXk1TUxPJyckUFhYyZsyYk455/PHH+dOf/kRGRnDO+uzZs1m7dm2/FyzOrtLZwu9e3c+x+jbsidF8bekEFs1wEG3r2+9yvfE43vf/hH50L2pSFtGX34tl1PR+rloIMRB69FO/du1aCgoKyM/PZ8OGDaxZs4ZnnnnmlOOuueYa7r///n4vUnTPME1e21nN396pIDnexm1XT+HCyRlofZhrbhoG+rG9+Eu3Eqj8EKxRRM3/GtapS1FU+aNPiKGi259Wt9tNSUkJTz/9NAB5eXk8/PDDNDQ0kJqaOuAFiu41e7z836v72VfZwJxJ6dx8xeQ+LSegNxzDf3ArgbL3g7sZRcVhnboM26w8meooxBDUbcA7nU4yMzPRNA0ATdPIyMjA6XSeEvCvvvoqW7duJT09nbvuuotZs7rftk6cm70Vbv6vqIQOn86/5E7iSzOH9epiItPXgb/sffylWzDqKkFR0UZOxzppIZZRM2TNdCGGsH77e/vGG2/kO9/5DlarlW3btnH77bezceNGUlJ6voLh2bae6k56eui2VQuFgG7wyvZq/v52GaOyEvjhN+YyOqvnvWxf3RFaPnoNz563MX0d2DJGk7D8W8RPXYQWlzSAlZ+bSPs6g7Q5UgxEm7sNeIfDgcvlQtd1NE1D13Vqa2txOBxfKC696/6CBQtwOBwcOnSIefPm9biY0+3J2hORttiYYZqsf+MQb+86yqWzhnPjkvHYNKXbz4FpBAgc3o2/5K3ghUmqBcu4C7FNXYqakY1PUWhoB9oH5+cy0r7OIG2OFCHbk9Vut5OTk0NRURH5+fkUFRWRk5NzyvCMy+UiMzMTgP3793Ps2DHGjh3b64JF9158u5y3dx3lK4vHcdUlY3p0TuDoXjq3/AGztQ4l3o5t3nVYJy2WsXUhwliPhmjWrVvH6tWrefLJJ0lMTKSwsBCAVatWcffddzNt2jQee+wx9u3bh6qqWK1WHn300ZN69aJ/bN5ZzaYd1Vy5YCx580d3e7zpbaPz/T8TOLgFJSmLmC//O9qoGbKSoxARQDH7c7POcyRDNGe3vaSG37xcwtxJ6fzXrfNpcHvOery/chferc9gdrZim7EC2+yrh/S665Hydf48aXNkCNkQjRgc9lU28Nui/Uwamcyqq6agnWWZAaO9Ge976wlUfIBqH0XMFfegpY05f8UKIQYFCfghoKqmlV/+vRiHPY67rp2O1aKd9jiz04Nv3xv4ijeD7sN24XXYZlwuFycJEaHkJ3+Qq23q4Kd/+Zj4aCv33DCD2OhTv2RGWyO+4tfwl/yzaxEw20XXy1oxQkQ4CfhBzDBMfvPyPnTD5P6VM0hJOHmhMKOlFt8nG/GXbgVTx5J9MbaZV6KljghRxUKIwUQCfhB7Y9dRKo63cNvVU3DY4wAwDR39yB5q/vke7WUfgqJhnbQQ24wVqIln35xcCBFZJOAHqbqmDv72bjnTs+1clJOJ0VqH/8C7+A9uxWxrRItLxjZjBdapy1Djen61sBAickjAD0KmafKHTQdQFYV/uTiBjn/8BP3oPgC0kdOwXnITjjkLqW/oCHGlQojBTAJ+ENpWXEPJ4UZuWj6BqA9/h95Si2321VgnL0aNtwOgaPKlE0KcnaTEINPc5uP5tw4xYUQSC9PceHdVErX4W9gmfynUpQkhhhi5Xn2Qefb1g3j9BjdfPgn/Ry+jxNuxTlgQ6rKEEEOQBPwg8tHBOj48UMvVC8aQ3lGBUVeBbdZVMhwjhOgTCfhBor3Tzx83lzIyI57ceSPx7toQ7L1PXBjq0oQQQ5QE/CDxl3+W0dLm4+YrJqPU7MeoLcc2M09670KIPpOAHwSKK9y8+4mT3HmjGJOVgHfXSyhxqVgnLQp1aUKIIUwCPsTaO8VVOqYAABicSURBVP38/h8HGJYWx1cWjUU/VoLhKsM2S3rvQohzIwEfYn964xDNHh+35uVg0VR8H22Q3rsQol9IwIfQ7oN1vLe3hivnj2ZMViL68f3oNQexzVyBollDXZ4QYoiTgA+R1nYff9h0gFEZ8Vy1YAymaeLb9RJKbDLWSYtDXZ4QIgxIwIfIHzcfpK0zwK15U7BoKrrzwIne+5VDels9IcTgIQEfAjv3u/jwQC35C8cyIiM+2Hv/4G/B3rssSSCE6CcS8OdZs8fLH18rZawjkSsuHgVAoGInuusQtjnXSO9dCNFvJODPs2deK8UXMLg1LwdNVTEDXrzbn0e1j5axdyFEv5KAP48OHmli96F68heO7dqhyffxRsy2BqIuKUBR5cshhOg/PUqUyspKVq5cSW5uLitXruTw4cNnPLaiooIZM2ZQWFjYXzWGjVe2VZIYa2XpnOCeqYbHje+TjVjGzcPimBTi6oQQ4aZHAb927VoKCgp47bXXKCgoYM2aNac9Ttd11q5dy7Jly/q1yHBQfqyZfYcbyb1oFFFWDQDv9ucBhaiLV4a2OCFEWOo24N1uNyUlJeTl5QGQl5dHSUkJDQ0Npxz7m9/8hksvvZQxY8b0e6FD3SvvHSY+xspls4YDEHCWEqjYiW3miq5dmoQQoj91G/BOp5PMzEw0Ldjr1DSNjIwMnE7nSccdOHCArVu3cvPNNw9IoUPZ4ZoW9pS7+fKFI4m2WTANA+97z6LEpWKbcUWoyxNChKl+Wc3K7/fz4IMP8qMf/ajrF0Ff2O3xfT43PT2hz+cOtP99pYS4GCsrcycTG22l5aPNeNzVZHzlXuIdaX1+3sHc5oEibY4M0ub+0W3AOxwOXC4Xuq6jaRq6rlNbW4vD4eg6pq6ujurqam677TYAWlpaME0Tj8fDww8/3ONi3G4PhmH2uhHp6QnU1bX2+rzzodrVyo59NeQvHEtbayeeejdt//wTWtZE2tOm0dHHugdzmweKtDkySJt7TlWVs3aMuw14u91OTk4ORUVF5OfnU1RURE5ODqmpqV3HDBs2jB07dnT9+/HHH6e9vZ3777+/1wWHm6L3DhMTpbFsbnDmjPejlzE7PURd8nUURQlxdUKIcNajWTTr1q1j/fr15Obmsn79eh566CEAVq1aRXFx8YAWOJQdq/Owq7SOpXNGEhdtxfC48e99A+vkRWhpo0NdnhAizPVoDD47O5sXXnjhlMefeuqp0x5/1113nVtVYaLo/SpsNo0vXzgSCF7UhAK22fkhrkwIEQnk0skB4nS3sbPExZLZw4mPsWK0N+EvfQfrxAUyLVIIcV5IwA+QV9+vwmpRyb0wuKCYb88mMHRsM/NCXJkQIlJIwA+A2qYOtu9zcems4STG2TA6W/GXvIUl+2LUxIxQlyeEiBAS8APgtR3VqCrkzgv23v3FmyHgwzZLeu9CiPNHAr6fNbf52Frs5JILskhJiML0tuHb+waWsXPRUoaHujwhRASRgO9nb3x4hEDA4PKLgtMgffveBH8HtllXhbgyIUSkkYDvRx3eAP/86BizJ6WTlRqL6e/EX7wZbdQMmfcuhDjvJOD70TsfH6fdG2DFxcEw9+//J6bXQ5T03oUQISAB30/8AYPNH1STMzqFsY5EzIAP3yeb0IZPQcscH+ryhBARSAK+n2zfV0OTx9e1kba/dAtmR7OMvQshQkYCvh8Ypsk/dlQzKjOeqWNSMfUAvk82omVOQHNMDnV5QogIJQHfD3YfrKemoZ0VF49GUZTg2LvHjW32VbJipBAiZCTgz5FpmmzcXkV6cjRzJqVjetvw7noJbfhUtBHTQl2eECKCScCfo9LqJiqdLVx+0Wg0VcX70cvgayfq4hul9y6ECCkJ+HO0cUcVibFWFlyQhdFcg3/fG1gnLUazjwx1aUKICCcBfw6qXa3srWhg2dyR2Kwa3h1/Ac2Kbe5XQl2aEEJIwJ+Ldz4+js2ictns4QSO7ydw+CNsM69EjU0OdWlCCCEB31f+gM6OEhezJ6YTG6Xhff/PKPF2bNNyQ12aEEIAEvB99nGZm3ZvgAXTHAQOvYfhriJq3vUoFluoSxNCCEACvs+2FTtJSYhi8rBYvDtfRM0YhyX7olCXJYQQXSTg+6DZ42VvRQOXXJBFoPgfmO1NRF/8NZkWKYQYVCTg++D9fS4M02RBdjS+T/6BZdw8tKwJoS5LCCFOYunJQZWVlaxevZqmpiaSk5MpLCxkzJgxJx3z17/+ld///veoqophGFx//fX8y7/8y0DUHFKmabJtr5PsYYkkH92C39CJmnd9qMsSQohT9KgHv3btWgoKCnjttdcoKChgzZo1pxyTm5vLyy+/zIYNG3juued4+umnOXDgQL8XHGrVLg/H6tpYMDWdwMFtWEbPRE1MD3VZQghxim4D3u12U1JSQl5ecMPovLw8SkpKaGhoOOm4+Pj4rjHozs5O/H5/WI5Jby12YtFU5sYdw+xsxZrzpVCXJIQQp9VtwDudTjIzM9E0DQBN08jIyMDpdJ5y7JtvvsmVV17JZZddxq233sqkSZP6v+IQCugGO0pczJqQhlq+DSXejjb8glCXJYQQp9WjMfieWrp0KUuXLuX48ePccccdLF68mHHjxvX4fLs9vs+vnZ6e0Odze+r94uN4OvzkzUxA37yPlMU3kpKZNOCveybno82DjbQ5Mkib+0e3Ae9wOHC5XOi6jqZp6LpObW0tDofjjOcMGzaMadOm8fbbb/cq4N1uD4Zh9vj4T6WnJ1BX19rr83rrH9sqSYqzkebaSUBR8I286Ly87umcrzYPJtLmyCBt7jlVVc7aMe52iMZut5OTk0NRUREARUVF5OTkkJqaetJx5eXlXfcbGhrYsWMHEydO7HXBg1VLu4895W4umZKOfnALllEzUeNSQl2WEEKcUY+GaNatW8fq1at58sknSUxMpLCwEIBVq1Zx9913M23aNJ5//nm2bduGxWLBNE1uuukmFi5cOKDFn0879rnQDZNF9jrM8hask+XNVSHE4NajgM/OzuaFF1445fGnnnqq6/4DDzzQf1UNQtuKnYzOSiDR+S5GXCraSNmtSQgxuMmVrD1wpNZDda2HJROj0Y/sxTppEYqqhbosIYQ4Kwn4Hnh/bw2aqjBTKQXAOnlxiCsSQojuScD3wMdl9UwZlYRSvhVt5DTUeHuoSxJCiG5JwHejtrGdmoZ2FqXVY7Y3yZWrQoghQwK+G8UVwSUZxnfsQYlNxjJqRogrEkKInpGA70ZxhZvsFB2tZh/WiQtR1H69+FcIIQaMBPxZ+Pw6+6sa+XJKNZimzH0XQgwpEvBnUXqkCX9AJ9tbgjYsR5YFFkIMKRLwZ7Gn3M2EKDfWDjfWCZeEuhwhhOgVCfizKC53syT1GGg2LGPnhrocIYToFQn4M3A1tONu8jAhcAjLmFkotphQlySEEL0iAX8Ge8rdTLYex6q3Y50wP9TlCCFEr0nAn0FxhZuFCdUo0QloI2TXJiHE0CMBfxpen87h6lomKYexZF8kc9+FEEOSBPxp7K9uZKpWiWbqMntGCDFkScCfRnGFm3nRlSiJmajpY0NdjhBC9IkE/BeYpklVeRXZlhqsEy5BUZRQlySEEH0iAf8FNQ3tjPUeQAGZPSOEGNIk4L9gT1k9F9oqMOzjUBMzQl2OEEL0mQT8FxwrK8VhaSImJ3w2DBdCRCYJ+M/p9AVIc3+CgYp13LxQlyOEEOdEAv5z9le6mWWrwJs+BSU6PtTlCCHEOZGA/5ya/btJVjtImiabagshhr4eXaJZWVnJ6tWraWpqIjk5mcLCQsaMGXPSMU888QQbN25E0zQsFgv33HMPixYtGoiaB4RpmsTX7MKn2YgfMzPU5QghxDnrUcCvXbuWgoIC8vPz2bBhA2vWrOGZZ5456Zjp06fz7W9/m5iYGA4cOMBNN93E1q1biY6OHpDC+9sRVwsTqKLVPhW7xRbqcoQQ4px1O0TjdrspKSkhLy8PgLy8PEpKSmhoaDjpuEWLFhETE1xSd9KkSZimSVNT0wCUPDDK9u0nTvWRMl421RZChIduA97pdJKZmYmmaQBomkZGRgZOp/OM57z00kuMGjWKrKys/qt0gLVX7QUgYdy0EFcihBD9o9+XSdy5cyc///nP+d3vftfrc+32vs9cSU9P6PO5ja2dpLZX0ZmQyrixY/r8POfbubR5qJI2RwZpc//oNuAdDgculwtd19E0DV3Xqa2txeFwnHLs7t27+Y//+A+efPJJxo0b1+ti3G4PhmH2+rz09ATq6lp7fd6ntnxylIkWF2TOOafnOZ/Otc1DkbQ5Mkibe05VlbN2jLsdorHb7eTk5FBUVARAUVEROTk5pKamnnTcnj17uOeee/jFL37B1KlTe11oKB0tPUCs6iMpW4ZnhBDho0fz4NetW8f69evJzc1l/fr1PPTQQwCsWrWK4uJiAB566CE6OztZs2YN+fn55OfnU1paOnCV9xN/wMB0Beu0DMsJcTVCCNF/ejQGn52dzQsvvHDK40899VTX/b/+9a/9V9V5VHqkkbGKE19MGglxKaEuRwgh+k3EX8m651Ad2VYX0SOH1rCSEEJ0J6ID3jRNXOWlxCh+bCNkeEYIEV4iOuCP17eR7q0CQBs2OcTVCCFE/4rogP+k3M0EiwszMQs1NjnU5QghRL+K7IA/VMt4Wy22EVNCXYoQQvS7iA341nYfflcFNvxoDhmeEUKEn4gN+L0VDYy31AAy/i6ECE8RG/Afl9UzOboOJWU4akxiqMsRQoh+F5EBH9ANSirrGKu5sEjvXQgRpiIy4A8dbSZdd2ExZfxdCBG+IjLgPymrZ5LNBcj4uxAifEVswE+Pd6OmjkSNjrx1p4UQkSHiAr6+qYP6Rg8O3Sm9dyFEWIu4gD9Q3cRoSz2a6ZeAF0KEtYgL+NIjjUyNqQMULFmTQl2OEEIMmMgL+OompsbWodpHokT3fQ9YIYQY7CIq4OubO2hqbiMz4EST3ZuEEGEuogK+tLqJcZZaVDOAZbgEvBAivEVWwB9pYmqMCxRNLnASQoS9yAr46kYuiHahZWajWKNDXY4QQgyoiAn4hpZO2pqbsOu1aCNk/1UhRPiLmIAvrW5ioqUGBRPLcAl4IUT4i5yAP9LI1OgasMagpo8NdTlCCDHgehTwlZWVrFy5ktzcXFauXMnhw4dPOWbr1q189atf5YILLqCwsLC/6zxnB6qbyIkKLg+sqFqoyxFCiAHXo4Bfu3YtBQUFvPbaaxQUFLBmzZpTjhk5ciSPPPIIt9xyS78Xea4aW73ozbUkGM0y/i6EiBjdBrzb7aakpIS8vDwA8vLyKCkpoaGh4aTjRo8ezZQpU7BYLANT6TkorW5ksvU4AJbhF4S4GiGEOD+6TWOn00lmZiaaFhzW0DSNjIwMnE4nqamp/VqM3d73pQPS08+87G9VXTk5US60xDQyxo9HUZQ+v85gcrY2hytpc2SQNvePQdXddrs9GIbZ6/PS0xOoq2s94/9/ctDFUqsT1TGP+nrPuZQ4aHTX5nAkbY4M0uaeU1XlrB3jbodoHA4HLpcLXdcB0HWd2tpaHA5Hr4sJhSaPF1vzUaJML5pMjxRCRJBuA95ut5OTk0NRUREARUVF5OTk9PvwzEA5eKSJSVYnANrwKSGuRgghzp8ezaJZt24d69evJzc3l/Xr1/PQQw8BsGrVKoqLiwH48MMPWbx4MU8//TR//vOfWbx4MVu2bBm4ynvoQHUTOTYnin0UakxiqMsRQojzpkdj8NnZ2bzwwgunPP7UU0913Z87dy7vvvtu/1XWTyqqarnaUotl+JdDXYoQQpxXYX0la3Obj9jWw2gYWEbI9EghRGQJ64A/eKSJyRYnpqqhZU0IdTlCCHFehXXAH6huZJLNiZY5EcUSFepyhBDivArrgD9adZxhWiMWWZ5ACBGBwjbgW9p9JLaWA0jACyEiUtgG/JsfHmWS1YlhjUW1jw51OUIIcd6FZcC7Gtr5x47DTItxYRsxBUUNy2YKIcRZhV3ymabJs68fZLi1lVjDgybTI4UQESrsAn5XaR17Kxv42hgXKBqWUTNCXZIQQoREWAV8py/Ac28eIjs9Ckfjbizj5qLGpYS6LCGECImwCvhXth2msdXLzZMbwd+B7YLloS5JCCFCJmwC/lh9G5s/OMLCaVkkH9uGmj4WNSM71GUJIUTIhEXAm6bJs5tLibZpXDfZh9HkxDZ1Wdjs3CSEEH0RFgH/7u5jHKhu4tovZWM59E+UmEQs2fNCXZYQQoTUkA/4Dm+A372yl7GOBBaO0dCr92DNuQxFs4a6NCGECKkhH/B7yt00tXq56cuTCOx/CxQVa86loS5LCCFCblBtut0XcyenM2fqMjRfO55NW7CMu1CmRgohBGHQg9dUFUdaHP6D205MjVwW6pKEEGJQGPIBD2CaBv59b8jUSCGE+JywCPiOyj3BqZEXLJepkUIIcUJYBHzLBxuDUyPHXRjqUoQQYtAY8gFvNLtoL/tIpkYKIcQX9CjgKysrWblyJbm5uaxcuZLDhw+fcoyu6zz00EMsW7aM5cuX88ILL/R3raelu6tRomKwTrnsvLyeEEIMFT0K+LVr11JQUMBrr71GQUEBa9asOeWYV155herqajZv3szzzz/P448/ztGjR/u94C+yjruQ0Xf9BjU2ecBfSwghhpJuA97tdlNSUkJeXh4AeXl5lJSU0NDQcNJxGzdu5Prrr0dVVVJTU1m2bBmbNm0amKq/QI2KOS+vI4QQQ0m3Ae90OsnMzETTNAA0TSMjIwOn03nKccOGDev6t8PhoKampp/LFUII0VOD6kpWuz2+z+empyf0YyVDg7Q5MkibI8NAtLnbgHc4HLhcLnRdR9M0dF2ntrYWh8NxynHHjx9n+vTpwKk9+p5wuz0YhtmrcyD4iamra+31eUOZtDkySJsjQ1/brKrKWTvG3Q7R2O12cnJyKCoqAqCoqIicnBxSU1NPOu7yyy/nhRdewDAMGhoaeOONN8jNze11wUIIIfpHj2bRrFu3jvXr15Obm8v69et56KGHAFi1ahXFxcUA5OfnM2LECL785S9zww03cMcddzBy5MiBq1wIIcRZKaZp9n5MZIDIEE3PSZsjg7Q5MgzUEM2gepNVVfu+jsy5nDtUSZsjg7Q5MvSlzd2dM6h68EIIIfrPkF+LRgghxOlJwAshRJiSgBdCiDAlAS+EEGFKAl4IIcKUBLwQQoQpCXghhAhTEvBCCBGmJOCFECJMDfmA78l+sUNZYWEhS5YsYdKkSRw8eLDr8XBud2NjI6tWrSI3N5errrqKO++8s2sHsXBu9+23387VV1/NNddcQ0FBAfv37wfCu80Av/zlL0/6/g7n9i5ZsoTLL7+c/Px88vPz2bJlCzCAbTaHuG984xvmSy+9ZJqmab700kvmN77xjRBX1L8++OAD8/jx4+Zll11mlpaWdj0ezu1ubGw0t2/f3vXvH//4x+Z//ud/mqYZ3u1uaWnpuv/666+b11xzjWma4d3mvXv3mrfccot56aWXdn1/h3N7v/hz/KmBavOQDvj6+npzzpw5ZiAQME3TNAOBgDlnzhzT7XaHuLL+9/lvjEhqt2ma5qZNm8xvfvObEdXuv//97+ZXvvKVsG6z1+s1b7jhBrO6urrr+zuc22uapw/4gWzzoFpNsrfOtl/sFzckCSeR1G7DMHjuuedYsmRJRLT7+9//Ptu2bcM0Tf7v//4vrNv885//nKuvvvqkfSPCub2fuu+++zBNkzlz5nDvvfcOaJuH/Bi8CG8PP/wwsbGx3HTTTaEu5bz4wQ9+wNtvv80999zDo48+GupyBszu3bspLi6moKAg1KWcV88++ywvv/wyf/3rXzFNk//+7/8e0Ncb0gH/+f1igTPuFxtuIqXdhYWFVFVV8bOf/QxVVSOm3QDXXHMNO3bsICsrKyzb/MEHH1BRUcHSpUtZsmQJNTU13HLLLVRXV4dlez/1aTtsNhsFBQV89NFHA/p9PaQDvqf7xYabSGj3T3/6U/bu3csTTzyBzWYDwrvdbW1tOJ3Orn+/9dZbJCUlhW2bb7vtNrZu3cpbb73FW2+9RVZWFr/97W9ZsWJFWLYXoL29ndbW4K5NpmmyceNGcnJyBvRrPOQ3/CgvL2f16tW0tLSQmJhIYWEh48aNC3VZ/eaRRx5h8+bN1NfXk5KSQnJyMq+++mpYt/vQoUPk5eUxZswYoqOjARgxYgRPPPFE2La7vr6e22+/nY6ODlRVJSkpifvvv5+pU6eGbZs/b8mSJfz6179m4sSJYdveI0eOcNddd6HrOoZhkJ2dzX/913+RkZExYG0e8gEvhBDi9Ib0EI0QQogzk4AXQogwJQEvhBBhSgJeCCHClAS8EEKEKQl4IYQIUxLwQggRpiTghRAiTP0/sV0kBS38KQgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.linspace(0, N_EPOCHS,N_EPOCHS)\n",
    "\n",
    "plt.plot(x,t_f1)\n",
    "plt.plot(x,v_f1)\n",
    "plt.title(\"F1 score\")\n",
    "plt.legend([\"Train F1\", \"Valid F1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss: 51.947 |  Test F1 score: 52.48%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, precision_recall_fscore_support\n",
    "model.load_state_dict(torch.load('tut3-model_conll.pt'))\n",
    "\n",
    "test_loss, test_f1, preds, labels = evaluate(model, test_iterator, TAG_PAD_IDX, full_report=False)\n",
    "print(f'Test Loss: {test_loss:.3f} |  Test F1 score: {test_f1*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict =  [item for sublist in preds for item in sublist]\n",
    "true =  [item for sublist in labels for item in sublist]\n",
    "confusion = confusion_matrix(true, predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Matrice de confusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>O</th>\n",
       "      <th>LOC</th>\n",
       "      <th>PER</th>\n",
       "      <th>ORG</th>\n",
       "      <th>MISC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>38178</td>\n",
       "      <td>49</td>\n",
       "      <td>144</td>\n",
       "      <td>101</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LOC</th>\n",
       "      <td>794</td>\n",
       "      <td>937</td>\n",
       "      <td>74</td>\n",
       "      <td>99</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PER</th>\n",
       "      <td>1282</td>\n",
       "      <td>25</td>\n",
       "      <td>1346</td>\n",
       "      <td>106</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORG</th>\n",
       "      <td>969</td>\n",
       "      <td>149</td>\n",
       "      <td>209</td>\n",
       "      <td>1115</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MISC</th>\n",
       "      <td>524</td>\n",
       "      <td>35</td>\n",
       "      <td>28</td>\n",
       "      <td>46</td>\n",
       "      <td>282</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          O  LOC   PER   ORG  MISC\n",
       "O     38178   49   144   101    48\n",
       "LOC     794  937    74    99    20\n",
       "PER    1282   25  1346   106    12\n",
       "ORG     969  149   209  1115    51\n",
       "MISC    524   35    28    46   282"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_df =pd.DataFrame(confusion)\n",
    "\n",
    "confusion_df.columns=[i for i in TAG.vocab.itos]\n",
    "s = pd.Series([i for i in TAG.vocab.itos])\n",
    "confusion_df = confusion_df.set_index([s])\n",
    "\n",
    "confusion_df['LOC'] = confusion_df['B-LOC'] + confusion_df['I-LOC']\n",
    "confusion_df['PER'] = confusion_df['B-PER'] + confusion_df['I-PER']\n",
    "confusion_df['ORG'] = confusion_df['B-ORG'] + confusion_df['I-ORG']\n",
    "confusion_df['MISC'] = confusion_df['B-MISC'] + confusion_df['I-MISC']\n",
    "\n",
    "\n",
    "confusion_df = confusion_df.drop(columns=[ i for i in TAG.vocab.itos if i != 'O'])\n",
    "\n",
    "confusion_df.loc['LOC'] = confusion_df.loc['B-LOC'] + confusion_df.loc['I-LOC']\n",
    "confusion_df.loc['PER'] = confusion_df.loc['B-PER'] + confusion_df.loc['I-PER']\n",
    "confusion_df.loc['ORG'] = confusion_df.loc['B-ORG'] + confusion_df.loc['I-ORG']\n",
    "confusion_df.loc['MISC'] = confusion_df.loc['B-MISC'] + confusion_df.loc['I-MISC']\n",
    "\n",
    "confusion_df = confusion_df.drop([i for i in TAG.vocab.itos if i != 'O'  ])\n",
    "\n",
    "confusion_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Precision-Recall-F1Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1-score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>O</th>\n",
       "      <td>0.914509</td>\n",
       "      <td>0.991121</td>\n",
       "      <td>0.951275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LOC</th>\n",
       "      <td>0.784100</td>\n",
       "      <td>0.487006</td>\n",
       "      <td>0.600834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PER</th>\n",
       "      <td>0.747363</td>\n",
       "      <td>0.485745</td>\n",
       "      <td>0.588801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORG</th>\n",
       "      <td>0.760055</td>\n",
       "      <td>0.447252</td>\n",
       "      <td>0.563131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MISC</th>\n",
       "      <td>0.682809</td>\n",
       "      <td>0.308197</td>\n",
       "      <td>0.424699</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Precision    Recall  F1-score\n",
       "O      0.914509  0.991121  0.951275\n",
       "LOC    0.784100  0.487006  0.600834\n",
       "PER    0.747363  0.485745  0.588801\n",
       "ORG    0.760055  0.447252  0.563131\n",
       "MISC   0.682809  0.308197  0.424699"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm = confusion_df.to_numpy()\n",
    "\n",
    "TP = np.diag(cm)\n",
    "FP = np.sum(cm, axis=0) - TP\n",
    "FN = np.sum(cm, axis=1) - TP\n",
    "\n",
    "num_classes = 4\n",
    "TN = []\n",
    "for i in range(num_classes):\n",
    "    temp = np.delete(cm, i, 0)    \n",
    "    temp = np.delete(temp, i, 1) \n",
    "    TN.append(sum(sum(temp)))\n",
    "    \n",
    "precision = TP/(TP+FP)\n",
    "recall = TP/(TP+FN)\n",
    "f1 = (2*precision*recall)/(precision+recall)\n",
    "\n",
    "prf_df =pd.DataFrame()\n",
    "prf_df['Precision'] = precision\n",
    "prf_df['Recall'] = recall\n",
    "prf_df['F1-score'] = f1\n",
    "\n",
    "s = pd.Series([i for i in confusion_df.index])\n",
    "prf_df = prf_df.set_index([s])\n",
    "\n",
    "prf_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inférence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tag_sentence(model, device, sentence, text_field, tag_field):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    if isinstance(sentence, str):\n",
    "        nlp = spacy.load('en')\n",
    "        tokens = [token.text for token in nlp(sentence)]\n",
    "    else:\n",
    "        tokens = [token for token in sentence]\n",
    "\n",
    "    if text_field.lower:\n",
    "        tokens = [t.lower() for t in tokens]\n",
    "        \n",
    "    max_word_len = max([len(token) for token in tokens])\n",
    "\n",
    "    numericalized_tokens = [text_field.vocab.stoi[t] for t in tokens]\n",
    "    unk_idx = text_field.vocab.stoi[text_field.unk_token]  \n",
    "    unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n",
    "    \n",
    "    token_tensor = torch.as_tensor(numericalized_tokens)    \n",
    "    token_tensor = token_tensor.unsqueeze(-1).to(device)\n",
    "\n",
    "    predictions, _ = model(token_tensor)\n",
    "    predicted_tags = [tag_field.vocab.itos[t] for t in predictions[0]]\n",
    "    \n",
    "    return tokens, predicted_tags, unks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Trailing', 'by', '213', ',', 'Somerset', 'got', 'a', 'solid', 'start', 'to', 'their', 'second', 'innings', 'before', 'Simmons', 'stepped', 'in', 'to', 'bundle', 'them', 'out', 'for', '174', '.']\n",
      "['O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
     ]
    }
   ],
   "source": [
    "example_index = 6\n",
    "\n",
    "sentence = vars(valid_data.examples[example_index])['text']\n",
    "actual_tags = vars(valid_data.examples[example_index])['tag']\n",
    "\n",
    "print(sentence)\n",
    "print(actual_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "['O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
     ]
    }
   ],
   "source": [
    "tokens, pred_tags, unks = tag_sentence(model, \n",
    "                                       device, \n",
    "                                       sentence, \n",
    "                                       TEXT, \n",
    "                                       TAG\n",
    "                                      )\n",
    "print(pred_tags)\n",
    "print(actual_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pred. Tag\t\t\t\tActual Tag\t\t\t\tCorrect?\t\t\t\tToken\n",
      "\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t Trailing\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t by\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t 213\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t ,\n",
      "O \t\t\t\t\t B-ORG \t\t\t\t ✘ \t\t\t\t\t Somerset\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t got\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t a\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t solid\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t start\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t to\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t their\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t second\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t innings\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t before\n",
      "B-PER \t\t\t\t B-PER \t\t\t\t ✔ \t\t\t\t\t Simmons\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t stepped\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t in\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t to\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t bundle\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t them\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t out\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t for\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t 174\n",
      "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t .\n"
     ]
    }
   ],
   "source": [
    "print(\"Pred. Tag\\t\\t\\t\\tActual Tag\\t\\t\\t\\tCorrect?\\t\\t\\t\\tToken\\n\")\n",
    "for token, pred_tag, actual_tag in zip(tokens, pred_tags, actual_tags):\n",
    "    correct = '✔' if pred_tag == actual_tag else '✘'\n",
    "    space = 5 if pred_tag == 'O'else 4\n",
    "    space1 = 5 if actual_tag == 'O'else 4   \n",
    "    print(pred_tag,\"\\t\"*space, actual_tag, \"\\t\"*space1, correct,\"\\t\"*5, token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "Pred. Tag\tToken\n",
      "\n",
      "O \t\t The\n",
      "O \t\t will\n",
      "O \t\t deliver\n",
      "O \t\t a\n",
      "O \t\t speech\n",
      "O \t\t about\n",
      "O \t\t the\n",
      "O \t\t conflict\n",
      "O \t\t in\n",
      "B-LOC \t North\n",
      "I-LOC \t Korea\n",
      "O \t\t tomorrow\n",
      "O \t\t in\n",
      "B-LOC \t New\n",
      "I-LOC \t York\n",
      "O \t\t with\n",
      "O \t\t my\n",
      "O \t\t friend\n",
      "B-LOC \t Mary\n",
      "I-LOC \t Kate\n",
      "O \t\t .\n"
     ]
    }
   ],
   "source": [
    "sentence = 'The will deliver a speech about the conflict in North Korea tomorrow in New York with my friend Mary Kate.'\n",
    "tokens, tags, unks = tag_sentence(model, \n",
    "                                  device, \n",
    "                                  sentence, \n",
    "                                  TEXT, \n",
    "                                  TAG\n",
    "                                )\n",
    "\n",
    "print(unks)\n",
    "print(\"Pred. Tag\\tToken\\n\")\n",
    "\n",
    "\n",
    "for token, tag in zip(tokens, tags):\n",
    "    space = 2 if tag == 'O'else 1\n",
    "    print(tag, \"\\t\"*space, token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "k4LF_qyLKvfK"
   ],
   "name": "4 - Sequence Labeling with Conditional Random Field (CRF) as Output Layer",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}