Neural_Machine_Translation_tf_1L_8H_BigData.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Neural_Machine_Translation_tf_1L_8H-BigData.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#### Mounting Google Drive"
      ],
      "metadata": {
        "id": "0dqNGaFLKKAJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "bVenoImvJZs9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e317fbfc-a654-4489-f383-ac619107ea1b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd \"/content/drive/MyDrive/IASNLP\""
      ],
      "metadata": {
        "id": "iNpRDc1-KMj8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "644d5f78-fb57-4144-ba72-824bd4bc30fb"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/IASNLP\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Importing Necessary Libraries"
      ],
      "metadata": {
        "id": "OlMuBUdIApMX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentencepiece\n",
        "!pip install sacrebleu\n",
        "!pip install tensorflow-gpu # ---> For GPU"
      ],
      "metadata": {
        "id": "Iwi6TDKsB0St",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f49470e2-b4ae-483e-9348-3d067e4892b9"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 5.8 MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.96\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-2.1.0-py3-none-any.whl (92 kB)\n",
            "\u001b[K     |████████████████████████████████| 92 kB 3.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (1.21.6)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (2022.6.2)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.5-py2.py3-none-any.whl (16 kB)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.4.0-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (0.8.9)\n",
            "Installing collected packages: portalocker, colorama, sacrebleu\n",
            "Successfully installed colorama-0.4.5 portalocker-2.4.0 sacrebleu-2.1.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting tensorflow-gpu\n",
            "  Downloading tensorflow_gpu-2.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (511.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 511.7 MB 6.2 kB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (57.4.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (3.3.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.6.3)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.14.1)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (3.1.0)\n",
            "Collecting flatbuffers<2,>=1.12\n",
            "  Downloading flatbuffers-1.12-py2.py3-none-any.whl (15 kB)\n",
            "Collecting gast<=0.4.0,>=0.2.1\n",
            "  Downloading gast-0.4.0-py3-none-any.whl (9.8 kB)\n",
            "Collecting keras<2.10.0,>=2.9.0rc0\n",
            "  Downloading keras-2.9.0-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 52.0 MB/s \n",
            "\u001b[?25hCollecting tensorflow-estimator<2.10.0,>=2.9.0rc0\n",
            "  Downloading tensorflow_estimator-2.9.0-py2.py3-none-any.whl (438 kB)\n",
            "\u001b[K     |████████████████████████████████| 438 kB 61.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (21.3)\n",
            "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (3.17.3)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.1.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.1.2)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (14.0.1)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (4.1.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (0.26.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (0.2.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.1.0)\n",
            "Collecting tensorboard<2.10,>=2.9\n",
            "  Downloading tensorboard-2.9.1-py3-none-any.whl (5.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.8 MB 58.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.21.6)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.46.3)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow-gpu) (0.37.1)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow-gpu) (1.5.2)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (1.35.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (0.6.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (2.23.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (3.3.7)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (1.0.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (1.8.1)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (4.8)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow-gpu) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow-gpu) (4.11.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow-gpu) (3.8.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (2022.6.15)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow-gpu) (3.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->tensorflow-gpu) (3.0.9)\n",
            "Installing collected packages: tensorflow-estimator, tensorboard, keras, gast, flatbuffers, tensorflow-gpu\n",
            "  Attempting uninstall: tensorflow-estimator\n",
            "    Found existing installation: tensorflow-estimator 2.8.0\n",
            "    Uninstalling tensorflow-estimator-2.8.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.8.0\n",
            "  Attempting uninstall: tensorboard\n",
            "    Found existing installation: tensorboard 2.8.0\n",
            "    Uninstalling tensorboard-2.8.0:\n",
            "      Successfully uninstalled tensorboard-2.8.0\n",
            "  Attempting uninstall: keras\n",
            "    Found existing installation: keras 2.8.0\n",
            "    Uninstalling keras-2.8.0:\n",
            "      Successfully uninstalled keras-2.8.0\n",
            "  Attempting uninstall: gast\n",
            "    Found existing installation: gast 0.5.3\n",
            "    Uninstalling gast-0.5.3:\n",
            "      Successfully uninstalled gast-0.5.3\n",
            "  Attempting uninstall: flatbuffers\n",
            "    Found existing installation: flatbuffers 2.0\n",
            "    Uninstalling flatbuffers-2.0:\n",
            "      Successfully uninstalled flatbuffers-2.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.8.2+zzzcolab20220527125636 requires keras<2.9,>=2.8.0rc0, but you have keras 2.9.0 which is incompatible.\n",
            "tensorflow 2.8.2+zzzcolab20220527125636 requires tensorboard<2.9,>=2.8, but you have tensorboard 2.9.1 which is incompatible.\n",
            "tensorflow 2.8.2+zzzcolab20220527125636 requires tensorflow-estimator<2.9,>=2.8, but you have tensorflow-estimator 2.9.0 which is incompatible.\u001b[0m\n",
            "Successfully installed flatbuffers-1.12 gast-0.4.0 keras-2.9.0 tensorboard-2.9.1 tensorflow-estimator-2.9.0 tensorflow-gpu-2.9.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras import Sequential\n",
        "from tensorflow.keras.layers import Dense, Lambda, Layer, Embedding, LayerNormalization\n",
        "import sentencepiece as spm\n",
        "from sacrebleu.metrics import BLEU, CHRF, TER\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from collections import Counter"
      ],
      "metadata": {
        "id": "DXSYYy2b-jMM"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading Data"
      ],
      "metadata": {
        "id": "ToLMSNAHBnvJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We load the `train_data`, `train_dev_data`, `test_val_data` and `test_data` as well as the byte-pair encoder tokenizer for English and Bengali i.e. `sp_en_bpe` and `s_ben_bpe`."
      ],
      "metadata": {
        "id": "zXIf-FL7IG7f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_data, train_dev_data = pd.read_csv(\"train_data_big.csv\")[['src', 'tgt']], pd.read_csv(\"train_dev.csv\")[['src', 'tgt']] \n",
        "test_val_data, test_data = pd.read_csv(\"test_val.csv\")[['src', 'tgt']], pd.read_csv(\"test_data.csv\")[['src', 'tgt']]\n",
        "sp_en_bpe, sp_ben_bpe = spm.SentencePieceProcessor(), spm.SentencePieceProcessor()\n",
        "sp_en_bpe.load('eng_big_bpe.model'); sp_ben_bpe.load('ben_big_bpe.model');"
      ],
      "metadata": {
        "id": "IXsCg4mBBeoo"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Preprocessing"
      ],
      "metadata": {
        "id": "auf4TlPFhNtn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We decide on some formats of the data and batches. We take the maximum tokens we can have per tokenized English and Bengali sentence to be `MAX_TOKENS = 60`. We use the vocabulary size of `32000`(i.e. `ENCODER_VOCAB = DECODER_VOCAB = 32000`). A batch size of `256` was choosen for our Mini-Batch Gradient Descent which is used to train our Model."
      ],
      "metadata": {
        "id": "3z7BUnn_G3WL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "MAX_TOKENS = 60\n",
        "VOCAB_SIZE = 32000\n",
        "BATCH_SIZE = 256\n",
        "BUFFER_SIZE = BATCH_SIZE*4"
      ],
      "metadata": {
        "id": "7jTuQSqOAybO"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following two functions help in tokenizing an detokenizing the data based on our sentencepiece byte-pair encoding model."
      ],
      "metadata": {
        "id": "NlEW7GndbCfw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize(sentence, sp_model, add_bos = True, add_eos = True):\n",
        "    bos = [sp_model.bos_id()] if add_bos else []\n",
        "    eos = [sp_model.eos_id()] if add_eos else []\n",
        "    inputs = bos + sp_model.encode_as_ids(sentence) + eos\n",
        "    return np.reshape(np.array(inputs), [1, -1])\n",
        "def detokenize(tokenized, sp_model, is_bos = True, is_eos = True):\n",
        "    integers = np.squeeze(tokenized).tolist()\n",
        "    if is_eos:\n",
        "        return sp_model.DecodeIdsWithCheck(integers[int(is_bos):integers.index(sp_model.eos_id())])\n",
        "    else:\n",
        "        if sp_model.pad_id() in tokenized:\n",
        "            return sp_model.DecodeIdsWithCheck(integers[int(is_bos):integers.index(sp_model.pad_id())])\n",
        "        else:\n",
        "            return sp_model.DecodeIdsWithCheck(integers[int(is_bos):])"
      ],
      "metadata": {
        "id": "VC-6hav2BrJJ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have tokenized the data."
      ],
      "metadata": {
        "id": "5-76mwBnbuzp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Tokenization\n",
        "src_train_data_enc = [np.squeeze(tokenize(train_data['src'].iloc[i], sp_en_bpe)) for i in range(train_data.shape[0])]\n",
        "tgt_train_data_enc = [np.squeeze(tokenize(train_data['tgt'].iloc[i], sp_ben_bpe)) for i in range(train_data.shape[0])]\n",
        "src_train_dev_data_enc = [np.squeeze(tokenize(train_dev_data['src'].iloc[i], sp_en_bpe)) for i in range(train_dev_data.shape[0])]\n",
        "tgt_train_dev_data_enc = [np.squeeze(tokenize(train_dev_data['tgt'].iloc[i], sp_ben_bpe)) for i in range(train_dev_data.shape[0])]\n",
        "src_test_val_data_enc = [np.squeeze(tokenize(test_val_data['src'].iloc[i], sp_en_bpe)) for i in range(test_val_data.shape[0])]\n",
        "tgt_test_val_data_enc = [np.squeeze(tokenize(test_val_data['tgt'].iloc[i], sp_ben_bpe)) for i in range(test_val_data.shape[0])]"
      ],
      "metadata": {
        "id": "nT0QpepuDoFF"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have padded the data with post-truncation and post padding upto MAX_TOKENS number of tokens."
      ],
      "metadata": {
        "id": "wdyEjbm5hWXg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Padding\n",
        "# The reason why we add MAX_TOKENS+1 after-pad length for Bengali is that when we form the tf.Dataset later, we will shift the target sentence once and will truncate the last token once\n",
        "# Hence, it will get adjusted\n",
        "train_src = tf.keras.preprocessing.sequence.pad_sequences(src_train_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "train_tgt = tf.keras.preprocessing.sequence.pad_sequences(tgt_train_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "train_dev_src = tf.keras.preprocessing.sequence.pad_sequences(src_train_dev_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "train_dev_tgt = tf.keras.preprocessing.sequence.pad_sequences(tgt_train_dev_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "test_val_src = tf.keras.preprocessing.sequence.pad_sequences(src_test_val_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "test_val_tgt = tf.keras.preprocessing.sequence.pad_sequences(tgt_test_val_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')"
      ],
      "metadata": {
        "id": "NT63vC8PcK-2"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Below, we discard all those pairs where the tokenized source or target sentences are larger than MAX_TOKENS number of tokens."
      ],
      "metadata": {
        "id": "FvxtnfKniMRA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Removing all pairs whose either the target sentence or source setence exceeds MAX_TOKENS number of tokens\n",
        "same_idx = list(set([i for i in range(train_src.shape[0]) if train_src[i][-1] == 1 or train_src[i][-1] == 0]).intersection(set([i for i in range(train_tgt.shape[0]) if train_tgt[i][-1] == 1 or train_tgt[i][-1] == 0])))\n",
        "train_src = train_src[same_idx]\n",
        "train_tgt = train_tgt[same_idx]\n",
        "same_idx = list(set([i for i in range(train_dev_src.shape[0]) if train_dev_src[i][-1] == 1 or train_dev_src[i][-1] == 0]).intersection(set([i for i in range(train_dev_tgt.shape[0]) if train_dev_tgt[i][-1] == 1 or train_dev_tgt[i][-1] == 0])))\n",
        "train_dev_src = train_dev_src[same_idx]\n",
        "train_dev_tgt = train_dev_tgt[same_idx]\n",
        "same_idx = list(set([i for i in range(test_val_src.shape[0]) if test_val_src[i][-1] == 1 or test_val_src[i][-1] == 0]).intersection(set([i for i in range(test_val_tgt.shape[0]) if test_val_tgt[i][-1] == 1 or test_val_tgt[i][-1] == 0])))\n",
        "test_val_src = test_val_src[same_idx]\n",
        "test_val_tgt = test_val_tgt[same_idx]"
      ],
      "metadata": {
        "id": "L6-cV6zpcO5e"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_labels = np.zeros(train_tgt.shape)\n",
        "train_labels[:,0:train_tgt.shape[1] -1] = train_tgt[:,1:]\n",
        "train_dev_labels = np.zeros(train_dev_tgt.shape)\n",
        "train_dev_labels[:,0:train_dev_tgt.shape[1] -1] = train_dev_tgt[:,1:]\n",
        "test_val_labels = np.zeros(test_val_tgt.shape)\n",
        "test_val_labels[:,0:test_val_tgt.shape[1] -1] = test_val_tgt[:,1:]"
      ],
      "metadata": {
        "id": "JGMvPvIxOx-0"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"*** Preprocessed Data ***\")\n",
        "print(\"Train Data Shape(Source): \", train_src.shape)\n",
        "print(\"Train Data Shape(Target): \", train_tgt.shape)\n",
        "print(\"Train Labels Shape(Target): \", train_labels.shape)\n",
        "print(\"Train Dev Data Shape(Source): \", train_dev_src.shape)\n",
        "print(\"Train Dev Data Shape(Target): \", train_dev_tgt.shape)\n",
        "print(\"Train Dev Labels Shape(Target): \", train_dev_labels.shape)\n",
        "print(\"Test Val Data Shape(Source): \", test_val_src.shape)\n",
        "print(\"Test Val Data Shape(Target): \", test_val_tgt.shape)\n",
        "print(\"Test Labels Shape(Target): \", test_val_labels.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rO5HD33XOGxf",
        "outputId": "07df3b4b-f5b4-41d7-faa2-d5d11e201490"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "*** Preprocessed Data ***\n",
            "Train Data Shape(Source):  (728047, 60)\n",
            "Train Data Shape(Target):  (728047, 60)\n",
            "Train Labels Shape(Target):  (728047, 60)\n",
            "Train Dev Data Shape(Source):  (3649, 60)\n",
            "Train Dev Data Shape(Target):  (3649, 60)\n",
            "Train Dev Labels Shape(Target):  (3649, 60)\n",
            "Test Val Data Shape(Source):  (1194, 60)\n",
            "Test Val Data Shape(Target):  (1194, 60)\n",
            "Test Labels Shape(Target):  (1194, 60)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Below we prepare the data in the form ready to feed to the Model."
      ],
      "metadata": {
        "id": "xtsMxbAYzePz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_data_prep = tf.data.Dataset.from_tensor_slices((train_src, train_tgt, train_labels))\n",
        "train_dev_data_prep = tf.data.Dataset.from_tensor_slices((train_dev_src, train_dev_tgt, train_dev_labels))\n",
        "test_val_data_prep = tf.data.Dataset.from_tensor_slices((test_val_src, test_val_tgt, test_val_labels))"
      ],
      "metadata": {
        "id": "_OkR5tbtyBQj"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Below, we can see how our data looks"
      ],
      "metadata": {
        "id": "jSGBKDhc2sc-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for encoder_inputs, decoder_inputs, labels in train_data_prep.take(1):\n",
        "    print(f'encoder_inputs.shape: {encoder_inputs.shape}')\n",
        "    print(f'decoder_inputs.shape: {decoder_inputs.shape}')\n",
        "    print(f\"labels.shape: {labels.shape}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ao1TX6GHzWn7",
        "outputId": "6171a5a3-8d94-4990-e45b-935f2924b659"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "encoder_inputs.shape: (60,)\n",
            "decoder_inputs.shape: (60,)\n",
            "labels.shape: (60,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model"
      ],
      "metadata": {
        "id": "zjXk6pt-28kL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will use the Transformer Architechture with the following hyperparameters."
      ],
      "metadata": {
        "id": "a9UFHuFO3dHe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Transformer parameters\n",
        "d_model = 256 # 512 in the original paper\n",
        "d_k = 32 # 64 in the original paper\n",
        "d_v = 32 # 64 in the original paper\n",
        "n_heads = 8 # 8 in the original paper\n",
        "n_encoder_layers = 1 # 6 in the original paper\n",
        "n_decoder_layers = 1 # 6 in the original paper\n",
        "\n",
        "max_token_length = MAX_TOKENS # 512 in the original paper"
      ],
      "metadata": {
        "id": "v9ULcpgbJ9WP"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class SingleHeadAttention(Layer):\n",
        "  def __init__(self, input_shape=(3, -1, d_model), dropout=.0, masked=None):\n",
        "    super(SingleHeadAttention, self).__init__()\n",
        "    self.q = Dense(d_k, input_shape=(-1, d_model), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    self.normalize_q = Lambda(lambda x: x / np.sqrt(d_k))\n",
        "    self.k = Dense(d_k, input_shape=(-1, d_model), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    self.v = Dense(d_v, input_shape=(-1, d_model), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    self.dropout = dropout\n",
        "    self.masked = masked\n",
        "  \n",
        "  # Inputs: [query, key, value]\n",
        "  def call(self, inputs, training=None):\n",
        "    assert len(inputs) == 3\n",
        "    # We use a lambda layer to divide vector q by sqrt(d_k) according to the equation\n",
        "    q = self.normalize_q(self.q(inputs[0]))\n",
        "    k = self.k(inputs[1])\n",
        "    # The dimensionality of q is (batch_size, query_length, d_k) and that of k is (batch_size, key_length, d_k)\n",
        "    # So we will do a matrix multication by batch after transposing last 2 dimensions of k\n",
        "    # tf.shape(attn_weights) = (batch_size, query_length, key_length)\n",
        "    attn_weights = tf.matmul(q, tf.transpose(k, perm=[0,2,1]))\n",
        "    if self.masked: # Prevent future attentions in decoding self-attention\n",
        "      # Create a matrix where the strict upper triangle (not including main diagonal) is filled with -inf and 0 elsewhere\n",
        "      length = tf.shape(attn_weights)[-1]\n",
        "      #attn_mask = np.triu(tf.fill((length, length), -np.inf), k=1) # We need to use tensorflow functions instead of numpy\n",
        "      attn_mask = tf.fill((length, length), -np.inf)\n",
        "      attn_mask = tf.linalg.band_part(attn_mask, 0, -1) # Get upper triangle\n",
        "      attn_mask = tf.linalg.set_diag(attn_mask, tf.zeros((length))) # Set diagonal to zeros to avoid operations with infinity\n",
        "      # This matrix is added to the attention weights so all future attention will have -inf logits (0 after softmax)\n",
        "      attn_weights += attn_mask\n",
        "    # Softmax along the last dimension\n",
        "    attn_weights = tf.nn.softmax(attn_weights, axis=-1)\n",
        "    if training: # Attention dropout included in the original paper. This is possibly to encourage multihead diversity.\n",
        "      attn_weights = tf.nn.dropout(attn_weights, rate=self.dropout)\n",
        "    v = self.v(inputs[2])\n",
        "    return tf.matmul(attn_weights, v)\n",
        "\n",
        "class MultiHeadAttention_my(Layer):\n",
        "  def __init__(self, dropout=.0, masked=None):\n",
        "    super(MultiHeadAttention_my, self).__init__()\n",
        "    self.attn_heads = list()\n",
        "    for i in range(n_heads): \n",
        "      self.attn_heads.append(SingleHeadAttention(dropout=dropout, masked=masked))\n",
        "    self.linear = Dense(d_model, input_shape=(-1, n_heads * d_v), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    \n",
        "  def call(self, x, training=None):\n",
        "    attentions = [self.attn_heads[i](x, training=training) for i in range(n_heads)]\n",
        "    concatenated_attentions = tf.concat(attentions, axis=-1)\n",
        "    return self.linear(concatenated_attentions)\n",
        "\n",
        "class TransformerEncoder(Layer):\n",
        "  def __init__(self, dropout=.1, attention_dropout=.0, **kwargs):\n",
        "    super(TransformerEncoder, self).__init__(**kwargs)\n",
        "    self.dropout_rate = dropout\n",
        "    self.attention_dropout_rate = attention_dropout\n",
        "  def build(self, input_shape):\n",
        "    self.multihead_attention = MultiHeadAttention_my(dropout=self.attention_dropout_rate)\n",
        "    self.dropout1 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization1 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    \n",
        "    self.linear1 = Dense(input_shape[-1] * 4, input_shape=input_shape, activation='relu',\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.linear2 = Dense(input_shape[-1], input_shape=self.linear1.compute_output_shape(input_shape),\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.dropout2 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization2 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    super(TransformerEncoder, self).build(input_shape)\n",
        "  def call(self, x, training=None):\n",
        "    sublayer1 = self.multihead_attention((x, x, x), training=training)\n",
        "    sublayer1 = self.dropout1(sublayer1, training=training)\n",
        "    layernorm1 = self.layer_normalization1(x + sublayer1)\n",
        "    \n",
        "    sublayer2 = self.linear2(self.linear1(layernorm1))\n",
        "    sublayer1 = self.dropout2(sublayer2, training=training)\n",
        "    layernorm2 = self.layer_normalization2(layernorm1 + sublayer2)\n",
        "    return layernorm2\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    return input_shape\n",
        "\n",
        "class TransformerDecoder(Layer):\n",
        "  def __init__(self, dropout=.0, attention_dropout=.0, **kwargs):\n",
        "    super(TransformerDecoder, self).__init__(**kwargs)\n",
        "    self.dropout_rate = dropout\n",
        "    self.attention_dropout_rate = attention_dropout\n",
        "  def build(self, input_shape):\n",
        "    self.multihead_self_attention = MultiHeadAttention_my(dropout=self.attention_dropout_rate, masked=True)\n",
        "    self.dropout1 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization1 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    \n",
        "    self.multihead_encoder_attention = MultiHeadAttention_my(dropout=self.attention_dropout_rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization2 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    \n",
        "    self.linear1 = Dense(input_shape[-1] * 4, input_shape=input_shape, activation='relu',\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.linear2 = Dense(input_shape[-1], input_shape=self.linear1.compute_output_shape(input_shape),\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.dropout3 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization3 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    super(TransformerDecoder, self).build(input_shape)\n",
        "  def call(self, x, hidden, training=None):\n",
        "    sublayer1 = self.multihead_self_attention((x, x, x))\n",
        "    sublayer1 = self.dropout1(sublayer1, training=training)\n",
        "    layernorm1 = self.layer_normalization1(x + sublayer1)\n",
        "    \n",
        "    sublayer2 = self.multihead_encoder_attention((x, hidden, hidden))\n",
        "    sublayer2 = self.dropout2(sublayer2, training=training)\n",
        "    layernorm2 = self.layer_normalization2(layernorm1 + sublayer2)\n",
        "    \n",
        "    sublayer3 = self.linear2(self.linear1(layernorm1))\n",
        "    sublayer3 = self.dropout3(sublayer3, training=training)\n",
        "    layernorm3 = self.layer_normalization2(layernorm2 + sublayer3)\n",
        "    return layernorm3\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    return input_shape\n",
        "\n",
        "class SinusoidalPositionalEncoding(Layer): # This is a TensorFlow implementation of https://github.com/graykode/nlp-tutorial/blob/master/5-1.Transformer/Transformer_Torch.ipynb\n",
        "  def __init__(self):\n",
        "    super(SinusoidalPositionalEncoding, self).__init__()\n",
        "    self.sinusoidal_encoding = np.array([self.get_positional_angle(pos) for pos in range(max_token_length)], dtype=np.float32)\n",
        "    self.sinusoidal_encoding[:, 0::2] = np.sin(self.sinusoidal_encoding[:, 0::2])\n",
        "    self.sinusoidal_encoding[:, 1::2] = np.cos(self.sinusoidal_encoding[:, 1::2])\n",
        "    self.sinusoidal_encoding = tf.cast(self.sinusoidal_encoding, dtype=tf.float32) # Casting the array to Tensor for slicing\n",
        "  def call(self, x):\n",
        "    return x + self.sinusoidal_encoding[:tf.shape(x)[1]]\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    return input_shape\n",
        "  def get_angle(self, pos, dim):\n",
        "    return pos / np.power(10000, 2 * (dim // 2) / d_model)\n",
        "  def get_positional_angle(self, pos):\n",
        "    return [self.get_angle(pos, dim) for dim in range(d_model)]\n",
        "\n",
        "class Transformer(Model):\n",
        "  def __init__(self, dropout=.1, attention_dropout=.0, **kwargs):\n",
        "    super(Transformer, self).__init__(**kwargs)\n",
        "    self.encoding_embedding = Embedding(VOCAB_SIZE, d_model)\n",
        "    self.decoding_embedding = Embedding(VOCAB_SIZE, d_model)\n",
        "    self.pos_encoding = SinusoidalPositionalEncoding()\n",
        "    self.encoder = [TransformerEncoder(dropout=dropout, attention_dropout=attention_dropout) for i in range(n_encoder_layers)]\n",
        "    self.decoder = [TransformerDecoder(dropout=dropout, attention_dropout=attention_dropout) for i in range(n_decoder_layers)]\n",
        "    self.decoder_final = Dense(VOCAB_SIZE, input_shape=(None, d_model))\n",
        "  def call(self, inputs, training=None): # Source_sentence and decoder_input\n",
        "    source_sentence, decoder_input = inputs\n",
        "    embedded_source = self.encoding_embedding(source_sentence)\n",
        "    encoder_output = self.pos_encoding(embedded_source)\n",
        "    for encoder_unit in self.encoder:\n",
        "      encoder_output = encoder_unit(encoder_output, training=training)\n",
        "    \n",
        "    embedded_target = self.decoding_embedding(decoder_input)\n",
        "    decoder_output = self.pos_encoding(embedded_target)\n",
        "    for decoder_unit in self.decoder:\n",
        "      decoder_output = decoder_unit(decoder_output, encoder_output, training=training)\n",
        "    if training:\n",
        "      decoder_output = self.decoder_final(decoder_output)\n",
        "      decoder_output = tf.nn.softmax(decoder_output, axis=-1)\n",
        "    else:\n",
        "      decoder_output = self.decoder_final(decoder_output[:, -1:, :])\n",
        "      decoder_output = tf.nn.softmax(decoder_output, axis=-1)\n",
        "    return decoder_output"
      ],
      "metadata": {
        "id": "KvsIOzPnzt4F"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transformer = Transformer(dropout=0.3) # Instantiating a new transformer model\n",
        "train = [tf.cast(train_src, dtype=tf.float32), tf.cast(train_tgt, dtype=tf.float32)] # Cast the tuples to tensors\n",
        "validation = [tf.cast(train_dev_src, dtype=tf.float32), tf.cast(train_dev_tgt, dtype=tf.float32)]"
      ],
      "metadata": {
        "id": "eRsGRYTR6XQW"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.build([train_src.shape, train_tgt.shape])\n",
        "transformer.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IHXUpv-0eh-b",
        "outputId": "c86b27d2-85ab-4535-9e32-3fd3443a4cbf"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"transformer\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       multiple                  8192000   \n",
            "                                                                 \n",
            " embedding_1 (Embedding)     multiple                  8192000   \n",
            "                                                                 \n",
            " sinusoidal_positional_encod  multiple                 0         \n",
            " ing (SinusoidalPositionalEn                                     \n",
            " coding)                                                         \n",
            "                                                                 \n",
            " transformer_encoder (Transf  multiple                 789760    \n",
            " ormerEncoder)                                                   \n",
            "                                                                 \n",
            " transformer_decoder (Transf  multiple                 1052928   \n",
            " ormerDecoder)                                                   \n",
            "                                                                 \n",
            " dense (Dense)               multiple                  8224000   \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 26,450,688\n",
            "Trainable params: 26,450,688\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training"
      ],
      "metadata": {
        "id": "0JkChx5Ndp1s"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])"
      ],
      "metadata": {
        "id": "InXkWf-LawJi"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "EPOCHS = 10\n",
        "checkpoint_filepath = './Models/'\n",
        "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_filepath,\n",
        "    save_weights_only=True,\n",
        "    monitor = \"accuracy\",\n",
        "    mode='max',\n",
        "    save_best_only=True)"
      ],
      "metadata": {
        "id": "LMttctIwaNxG"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with tf.device('/device:GPU:0'):\n",
        "    history = transformer.fit(train, tf.cast(train_labels, dtype=tf.float32), batch_size=256, epochs=EPOCHS, callbacks=[model_checkpoint_callback])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bzdsT8kmBEMo",
        "outputId": "83b78b75-10ce-4cd7-8e35-5b0d88a7adf4"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "2844/2844 [==============================] - 1522s 531ms/step - loss: 1.4319 - accuracy: 0.8152\n",
            "Epoch 2/10\n",
            "2844/2844 [==============================] - 1516s 533ms/step - loss: 1.0605 - accuracy: 0.8368\n",
            "Epoch 3/10\n",
            "2844/2844 [==============================] - 1519s 534ms/step - loss: 0.9442 - accuracy: 0.8466\n",
            "Epoch 4/10\n",
            "2844/2844 [==============================] - 1520s 535ms/step - loss: 0.8805 - accuracy: 0.8522\n",
            "Epoch 5/10\n",
            "2844/2844 [==============================] - 1519s 534ms/step - loss: 0.8385 - accuracy: 0.8561\n",
            "Epoch 6/10\n",
            "2844/2844 [==============================] - 1519s 534ms/step - loss: 0.8063 - accuracy: 0.8592\n",
            "Epoch 7/10\n",
            "2844/2844 [==============================] - 1519s 534ms/step - loss: 0.7802 - accuracy: 0.8617\n",
            "Epoch 8/10\n",
            "2844/2844 [==============================] - 1519s 534ms/step - loss: 0.7583 - accuracy: 0.8639\n",
            "Epoch 9/10\n",
            "2844/2844 [==============================] - 1519s 534ms/step - loss: 0.7394 - accuracy: 0.8659\n",
            "Epoch 10/10\n",
            "2844/2844 [==============================] - 1519s 534ms/step - loss: 0.7231 - accuracy: 0.8676\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.save_weights(\"model_weights\")\n",
        "# transformer.save(\"./Models/\")"
      ],
      "metadata": {
        "id": "6tmeafiFdZ_U"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 10))\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train'], loc='upper left')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "JbSCoj0cft6O",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 621
        },
        "outputId": "a0d897e4-913c-4d94-8c7a-97671c6f1355"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAJcCAYAAAC8DwN/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXxU5d3///cnO4EkQBLWsJOwy2JYVMAFF8Rd77ZqcWlduty21p9dtFq13l1sv3e1tXWpW11atbaKYosLWhAQRJE1bElYEwgwSUgIgaxz/f6Y0TsiQoRMziyv5+PB4zEz55yZd0CZN+c657rMOScAAACEhzivAwAAAOD/UM4AAADCCOUMAAAgjFDOAAAAwgjlDAAAIIxQzgAAAMII5QxAxDOzp83sF63cd6uZnRnqTABwrChnAAAAYYRyBgBhwswSvM4AwHuUMwDtIjic+CMzW21mtWb2pJl1N7M3zKzGzN4xsy4t9r/QzNaaWZWZzTezYS22jTWz5cHj/i4p5ZDPOt/MVgaPXWxmJ7Qy43lmtsLM9plZiZndc8j2ycH3qwpuvzb4egcz+52ZbTOzajNbFHztNDMrPczvw5nBx/eY2T/N7K9mtk/StWY2wcyWBD+jzMz+ZGZJLY4fYWZzzazSzHab2U/NrIeZHTCzzBb7jTMzn5kltuZnBxA+KGcA2tNlks6SlCfpAklvSPqppGwF/j76viSZWZ6kFyT9ILhtjqTXzSwpWFRelfScpK6S/hF8XwWPHSvpKUnfkpQp6c+SZptZcivy1Uq6WlJnSedJ+o6ZXRx8337BvH8MZhojaWXwuP+VdKKkk4OZfizJ38rfk4sk/TP4mX+T1CzpFklZkk6SNE3Sd4MZ0iS9I+lNSb0kDZb0rnNul6T5kr7a4n2vkvSic66xlTkAhAnKGYD29Efn3G7n3A5JCyUtdc6tcM7VSZolaWxwv69J+rdzbm6wXPyvpA4KlJ9JkhIl/d451+ic+6ekj1p8xo2S/uycW+qca3bOPSOpPnjcETnn5jvn1jjn/M651QoUxFODm6+U9I5z7oXg51Y451aaWZykb0q62Tm3I/iZi51z9a38PVninHs1+JkHnXMfO+c+cM41Oee2KlAuP8lwvqRdzrnfOefqnHM1zrmlwW3PSJopSWYWL+kKBQosgAhDOQPQnna3eHzwMM87BR/3krTtkw3OOb+kEkm9g9t2OOdci2O3tXjcT9KtwWHBKjOrktQneNwRmdlEM5sXHA6slvRtBc5gKfgemw5zWJYCw6qH29YaJYdkyDOzf5nZruBQ569akUGSXpM03MwGKHB2sto59+ExZgLgIcoZgHC0U4GSJUkyM1OgmOyQVCapd/C1T/Rt8bhE0i+dc51b/Ep1zr3Qis99XtJsSX2ccxmSHpX0yeeUSBp0mGPKJdV9wbZaSaktfo54BYZEW3KHPH9E0gZJuc65dAWGfVtmGHi44MGzjy8pcPbsKnHWDIhYlDMA4eglSeeZ2bTgBe23KjA0uVjSEklNkr5vZolmdqmkCS2OfVzSt4NnwczMOgYv9E9rxeemSap0ztWZ2QQFhjI/8TdJZ5rZV80swcwyzWxM8KzeU5LuN7NeZhZvZicFr3ErlJQS/PxESXdKOtq1b2mS9knab2ZDJX2nxbZ/SeppZj8ws2QzSzOziS22PyvpWkkXinIGRCzKGYCw45zbqMAZoD8qcGbqAkkXOOcanHMNki5VoIRUKnB92istjl0m6QZJf5K0V1JxcN/W+K6ke82sRtJdCpTET953u6QZChTFSgVuBhgd3PxDSWsUuPatUtJvJMU556qD7/mEAmf9aiV95u7Nw/ihAqWwRoGi+fcWGWoUGLK8QNIuSUWSTm+x/X0FbkRY7pxrOdQLIILYZy/bAABEMjP7j6TnnXNPeJ0FwLGhnAFAlDCz8ZLmKnDNXI3XeQAcG4Y1ASAKmNkzCsyB9gOKGRDZOHMGAAAQRkJ65szMppvZRjMrNrPbDrO9b3BOoRUWWNJlRvD1rweXXvnkl9/MxoQyKwAAQDgI2Zmz4Hw+hQrcWVSqwF1MVzjn1rXY5zFJK5xzj5jZcElznHP9D3mfUZJedc4dbg6hT2VlZbn+/fsfaRcAAICw8PHHH5c75w6d91CSlBDCz50gqdg5t1mSzOxFBdaQW9diHycpPfg4Q4GJJw91haQXj/Zh/fv317Jly44rMAAAQHswsy+c7iaU5ay3PrssSamkiYfsc4+kt83se5I6SjrzMO/zNQVK3eeY2Y0KrKOnvn37Hm4XAACAiOL13ZpXSHraOZejwOSOzwUXEZYUWOdO0gHnXMHhDnbOPeacy3fO5WdnH/bMIAAAQEQJZTnbocBaeJ/ICb7W0nUKzsDtnFuiwOLBWS22Xy6pNevhAQAARIVQDmt+JCnXzAYoUMou12fXqZOk7ZKmSXrazIYpUM58khQ8g/ZVSVOONUBjY6NKS0tVV1d3rG8RMVJSUpSTk6PExESvowAAgOMQsnLmnGsys5skvSUpXtJTzrm1ZnavpGXOudkKrFH3uJndosDNAde6/7t9dKqkkk9uKDgWpaWlSktLU//+/WVmx/cDhTHnnCoqKlRaWqoBAwZ4HQcAAByHUJ45k3NujqQ5h7x2V4vH6ySd8gXHzpc06Xg+v66uLuqLmSSZmTIzM+Xz+byOAgAAjpPXNwSEXLQXs0/Eys8JAEC0i/pyBgAAEEkoZyFWVVWlhx9++EsfN2PGDFVVVYUgEQAACGeUsxD7onLW1NR0xOPmzJmjzp07hyoWAAAIUyG9IQDSbbfdpk2bNmnMmDFKTExUSkqKunTpog0bNqiwsFAXX3yxSkpKVFdXp5tvvlk33nijpP9bjmr//v0699xzNXnyZC1evFi9e/fWa6+9pg4dOnj8kwEAgFCImXL289fXat3OfW36nsN7pevuC0YccZ/77rtPBQUFWrlypebPn6/zzjtPBQUFn0558dRTT6lr1646ePCgxo8fr8suu0yZmZmfeY+ioiK98MILevzxx/XVr35VL7/8smbOnNmmPwsAAAgPMVPOwsWECRM+MxfZgw8+qFmzZkmSSkpKVFRU9LlyNmDAAI0ZM0aSdOKJJ2rr1q3tlhcAALSvmClnRzvD1V46duz46eP58+frnXfe0ZIlS5SamqrTTjvtsKsZJCcnf/o4Pj5eBw8ebJesAACg/XFDQIilpaWppqbmsNuqq6vVpUsXpaamasOGDfrggw/aOR0AAAg3MXPmzCuZmZk65ZRTNHLkSHXo0EHdu3f/dNv06dP16KOPatiwYRoyZIgmTTquBREAAEAUsP9byjKy5efnu2XLln3mtfXr12vYsGEeJWp/sfbzAgAQqczsY+dc/uG2MawJAAAQRihnAAAAYSTqy1m0DNseTaz8nAAARLuoLmcpKSmqqKiI+uLinFNFRYVSUlK8jgIAAI5TVN+tmZOTo9LSUvl8Pq+jhFxKSopycnK8jgEAQMRzzsnMPPv8qC5niYmJn5mNHwAA4FDOOW3cXaP3Nvo0f6NP/bM66teXjvIsT1SXMwAAgMPZV9eo94vK9V6hT+8V+lRWHVihZ2iPNE3tmu1pNsoZAACIes45rSvbp/kbA2Vs+ba9avI7pSUnaHJuln5wZrZOzeumHhneX79NOQMAAFGp+kCjFhb7Pi1kvpp6SdKIXum6cepAnTakm8b27azE+PC6P5JyBgAAooLf77R25z7N37hH8wt9WrF9r/xOyuiQqCm5WTptSDdNzc1St3Tvz44dCeUMAABErMraBi0s8um9jT4tKPKpfH+DJOmEnAzddPpgnTqkm0bnZCghzM6OHQnlDAAARIxmv9Pq0iq9VxgYrlxVWiXnpC6piZqal63ThmRrSm62sjolex31mFHOAABAWCvfX68FwbsqFxT6tPdAo8ykMX066+ZpuTptSDeN6p2h+Djv5iZrS5QzAAAQVpqa/VpVWvXphfyrS6slSVmdknT60G46NS9bU3Oz1aVjksdJQ4NyBgAAPLdnX11gqLLQp0VF5ao+2Kg4k8b17aIfnp2nU/O6aUSvdMVFydmxI6GcAQCAdtfY7NeK7VWBOys3+rSubJ8kqVtass4e3l2nDemmyYOzlJGa6HHS9kc5AwAA7WJXdZ3eKwyUsUXF5aqpa1J8nOnEfl304+lDdGpetob3TPd0XctwQDkDAAAh0dDk17JtlYElkjb6tGFXjSSpR3qKzhvVU6cNydbJg7OUnhJ7Z8eOhHIGAADazI6qg5q/cY/e2+jT+8Xlqm1oVmK8aXz/rrr93KE6bUg35XXvFPNnx46EcgYAAI5ZfVOzPtqyN1DICn0q2rNfktS7cwddPLa3Ts0LnB3rlEzlaC1+pwAAwJdSUnng0wv5F2+q0MHGZiXFx2niwK762vg+Om1ItgZlc3bsWFHOAADAETnnVLBjn94oKNOba3dps69WktS3a6q+kp+j04Zka9LATKUmUSvaAr+LAADgc/x+p5WlVXqzYJfmrClT6d6Dio8znTQwU1dN6qfThnRT/8xUzo6FAOUMAABIChSyj7fv1Zw1ZXqzYJfKquuUGG+aPDhL35+Wq7OGdY/aWfnDCeUMAIAY1tTs14dbK/XGml16c+0u+WrqlZQQp1PzsvWjc4Zo2rDuyujAVBftiXIGAECMaWz2a8mmCr1RUKa31u5WZW2DUhLjdMbQbpo+sqfOGNqNuys9xO88AAAxoL6pWe8Xl2vOml2au263qg82qmNSvKYN665zR/bQqUOyuaA/TPCnAABAlKprbNZ7hT69saZM767fo5r6JqWlJOis4d117siempKbpZTEeK9j4hCUMwAAokhtfZPmb/RpTkGZ5m3YowMNzeqcmqhzR/XQuaN66pRBWUpKiPM6Jo6AcgYAQISrqWvUfzbs0Zw1ZZq/0af6Jr+yOiXp4rG9NWNkT00c2FWJ8RSySEE5AwAgAlUfaNTc9bv1xpoyLSwqV0OzX93Tk3X5+D46d1RPje/fVfFxzEEWiShnAABEiMraBr29dpfmFOzS4uJyNfmdemWk6KqT+mnGqB4a26eL4ihkEY9yBgBAGNtTU6e31gbOkC3dUqlmv1Pfrqm6bsoAzRjZUyfkZDBLf5ShnAEAEGbKqg/qzYJdemPNLn20rVLOSQOzO+o7pw7SuaN6aHjPdApZFKOcAQAQBkoqDwQKWUGZlm+vkiQN6Z6mm6flasaonsrt1olCFiMoZwAAeGRLea3eKAisY7m6tFqSNKJXun50zhBNH9lDg7I7eZwQXqCcAQDQjor31GjOml2as6ZMG3bVSJJG9+ms288dqnNH9lTfzFSPE8JrlDMAAELIOacNu2r0xpoyvVGwS0V79kuS8vt10c/OH67pI3uod+cOHqdEOKGcAQDQxpxzKtixT3OCQ5ZbymsVZ9KEAV111UkjdM6IHuqenuJ1TIQpyhkAAG3gk0L2+uqdmrOmTKV7Dyo+znTyoEzdMGWgzh7RXVmdkr2OiQhAOQMA4DjsrDqoV1fu0CvLd6h4z34lxpsmD87S96fl6qxh3dWlY5LXERFhKGcAAHxJ++ub9GbBLr2yvFRLNlfIucA1ZL+6ZJTOG9VTGamJXkdEBKOcAQDQCs1+p0XF5Zq1vFRvrt2luka/+nZN1c3TcnXJ2N7ql9nR64iIEpQzAACOYH3ZPr2yvFSvrdypPTX1Sk9J0KXjcnTZuN4a17cLE8OizVHOAAA4xJ59dXpt5U69vLxUG3bVKCHOdPrQbrp0bG+dMaybkhPivY6IKEY5AwBA0sGGZr29bpdeXr5Di4p88rvA5LD3XjRC55/QS125sB/thHIGAIhZfr/TB5sr9MqKHXpjTZlqG5rVu3MHffe0wbpkXG+WT4InKGcAgJhTvKdGLy/foddW7NDO6jp1Sk7QeSf01KXjcjShf1fFxXEdGbxDOQMAxITy/fV6fdVOzVqxQ6tLqxUfZ5qam6XbZgzTWcO6q0MS15EhPFDOAABRq66xWe+u36NXlpfqvUKfmvxOI3ql62fnD9eFo3spO40Z+xF+KGcAgKjinNNHW/dq1opS/Wt1mWrqmtQ9PVnXTRmgS8fmaEiPNK8jAkdEOQMARIWt5bV6ZcUOzVpRqpLKg0pNitf0ET106bgcnTQoU/FcR4YIQTkDAESsqgMNen11mWYtL9Xy7VUykyYPztItZ+bpnBE91DGZrzlEHv6rBQBElIYmv+ZtDFxH9p8Ne9TY7JTXvZNuP3eoLhrTWz0yUryOCBwXyhkAIOw557SipEqzlu/Q66t3qupAo7I6Jenqk/rrkrG9NaJXOssoIWpQzgAAYauk8oBeXbFDr6zYoS3ltUpOiNPZI3ro0nG9NWVwlhLi47yOCLQ5yhkAIKzsq2vUnNVlemXFDn24pVKSNGlgV33n1EGaPqqH0lMSPU4IhBblDADgucZmvxYW+fTy8h16Z91u1Tf5NTC7o354dp4uHttbOV1SvY4ItBvKGQDAE845rd25Ty8vL9Xrq3aqfH+DuqQm6vLxfXTJuByNzsngOjLEJMoZAKBdlVUf1KsrduqV5aUq2rNfSfFxmjasmy4dl6NT87KVlMB1ZIhtlDMAQMg1+53mbdijZ5Zs1aLicjknndivi355yUidP6qXMlK5jgz4BOUMABAy1Qcb9Y9lJXp2yTZtrzygHukp+t4Zubp0bG/1z+rodTwgLFHOAABtrmh3jZ5ZslWvLN+hAw3NGt+/i34yfajOHtFdiUx/ARwR5QwA0CY+Gbp8enFg6DIpIU4Xje6la07ur5G9M7yOB0QMyhkA4LgcbujyR+cM0eXj+yizU7LX8YCIQzkDAByTT4YuX/54hw42MnQJtBXKGQCg1Ri6BEKPcgYAOKpPhi6fWbJVJZUHGboEQohyBgD4QkW7a/T04sBdlwcbmzWhf1fdNn0YQ5dACFHOAACfwdAl4C3KGQBA0ueHLntmMHQJeIFyBgAxjqFLILxQzgAgBjX7nf6zYY+eYegSCDuUMwCIIV80dHnFhL7q2jHJ63gAFOJyZmbTJf1BUrykJ5xz9x2yva+kZyR1Du5zm3NuTnDbCZL+LCldkl/SeOdcXSjzAkC0OtzQ5e3nDtPZw7srgaFLIKyErJyZWbykhySdJalU0kdmNts5t67FbndKesk594iZDZc0R1J/M0uQ9FdJVznnVplZpqTGUGUFgGj0ydDl04u36P3iCiUlxOniMYGhyxG9GLoEwlUoz5xNkFTsnNssSWb2oqSLJLUsZ06BM2OSlCFpZ/Dx2ZJWO+dWSZJzriKEOQEgqlQfaNRLy0r07AcMXQKRKJTlrLekkhbPSyVNPGSfeyS9bWbfk9RR0pnB1/MkOTN7S1K2pBedc7899APM7EZJN0pS37592zQ8AEQahi6B6OD1DQFXSHraOfc7MztJ0nNmNjKYa7Kk8ZIOSHrXzD52zr3b8mDn3GOSHpOk/Px8177RAcB7DF0C0SeU5WyHpD4tnucEX2vpOknTJck5t8TMUiRlKXCWbYFzrlySzGyOpHGS3hUAgKFLIIqFspx9JCnXzAYoUMoul3TlIftslzRN0tNmNkxSiiSfpLck/djMUiU1SDpV0gMhzAoAEYGhSyD6haycOeeazOwmBYpWvKSnnHNrzexeScucc7Ml3SrpcTO7RYGbA651zjlJe83sfgUKnpM0xzn371BlBYBwxtAlEFss0IUiX35+vlu2bJnXMQCgzRxu6HLmpH4MXQJRIHgtff7htnl9QwAA4BCFwaHLWQxdAjGJcgYAYWLJpgr9aV4RQ5dAjKOcAYDHPtpaqfvfLtSSzRXqlpbMXZdAjKOcAYBHlm/fqwfmFmphUbmyOiXrrvOH68qJfZWSGO91NAAeopwBQDtbXVqlB+YWat5Gn7p2TNIdM4Zp5qR+6pBEKQNAOQOAdrN2Z7UemFukd9bvVufURP1k+lBdfVI/dUzmr2IA/4e/EQAgxDbs2qffzy3Sm2t3KT0lQbeeladrT+mvtJREr6MBCEOUMwAIkeI9NXrgnSL9e3WZ0pITdPO0XH1z8gBldKCUAfhilDMAaGObffv14LtFem3VTqUmxuum0wfr+ikD1DmVuy8BHB3lDADayLaKWj34brFmrShVckK8bpw6UN+aOogpMQB8KZQzADhOJZUH9NC8Yv3j41IlxJm+ecoAfevUQcpOS/Y6GoAIRDkDgGO0s+qgHppXrJeWlchkumpSP333tEHqlp7idTQAEYxyBgBf0u59dXp4XrFe+LBETk5fG99H/336YPXM6OB1NABRgHIGAK3kq6nXI/M36W9Lt6nZ7/SV/Bz99+mDldMl1etoAKII5QwAjqJif70eW7BZzyzZqsZmp0vH9tb3zshV30xKGYC2RzkDgC+wt7ZBjy/crKcXb1VdY7MuGtNb35+WqwFZHb2OBiCKUc4A4BDVBxv15MLNeur9raptaNL5J/TSzdMGa3C3NK+jAYgBlDMACKqpa9Rf3t+qxxduVk1dk2aM6qGbp+VpSA9KGYD2QzkDEPNq65v09OJAKas60KizhnfXLWfmaXivdK+jAYhBlDMAMetgQ7OeXbJVf16wWZW1DTpjaDfdcmaeRuVkeB0NQAyjnAGIOXWNzfrb0u16ZP4mle+v19S8bN1yZq7G9u3idTQAoJwBiB31Tc168cMSPTSvWHtq6nXK4Ew9euY45ffv6nU0APgU5QxA1Gto8uulZYFSVlZdpwn9u+rBK8Zq0sBMr6MBwOdQzgBErcZmv15ZXqoH3y3WjqqDGte3s/73K6N18qBMmZnX8QDgsChnAKJOU7Nfr67cqQffLdL2ygManZOhX14yUqfmZVPKAIQ9yhmAqNHsd3p91U794d0ibSmv1Yhe6XrymnydMbQbpQxAxKCcAYh4fr/TnIIy/f6dIhXv2a+hPdL056tO1NnDu1PKAEQcyhmAiOX3O729bpcemFukjbtrlNutkx66cpzOHdlDcXGUMgCRiXIGIOI45/TO+j16YG6h1pXt08CsjvrD5WN0/gm9FE8pAxDhKGcAIoZzTvMLfXpgbqFWl1arX2aq7v/qaF04upcS4uO8jgcAbYJyBiAibK84oNtnrdb7xRXK6dJBv73sBF0yrrcSKWUAogzlDEBYa/Y7/eX9LfrftzcqIS5OP79whK6Y0FdJCZQyANGJcgYgbBXurtGP/7laK0uqdMbQbvrlJSPVM6OD17EAIKQoZwDCTkOTX4/M36Q/zStSWkqi/nD5GF04uhfTYgCICZQzAGFlVUmVfvLyam3YVaMLRvfSPRcMV2anZK9jAUC7oZwBCAsHG5r1wDuFemLhZmWnJevxq/N11vDuXscCgHZHOQPguSWbKnT7K6u1teKArpjQR7fPGKb0lESvYwGAJyhnADyzr65R972xQc8v3a6+XVP1/PUTdfLgLK9jAYCnKGcAPPGfDbv101cKtKemTtdPHqBbzx6iDknxXscCAM9RzgC0q4r99br3X+v02sqdyuveSY/MPFlj+3bxOhYAhA3KGYB24ZzT66vLdM/staqpa9TN03L136cPZjJZADgE5QxAyO2qrtOdr67RO+v3aHROhn7zXxM1tEe617EAICxRzgCEjHNOL35Uol/9e70a/X7dMWOYvjl5gOLjmEwWAL4I5QxASGyrqNVtL6/Rks0VmjSwq+679AT1z+rodSwACHuUMwBtquVC5YlxcfrVJaN0+fg+iuNsGQC0CuUMQJvZuKtGP355tVaVVGna0G76BQuVA8CXRjkDcNwamvx6eH6xHppXzELlAHCcKGcAjsvKkir95J+rtXF3jS4a00t3nc9C5QBwPChnAI7JwYZm3T93o55ctEXd0lL0xNX5OpOFygHguFHOAHxpizeV67aX12h75QFdObGvbjt3KAuVA0AboZwBaLV9dY369ZwNeuHD7eqXmarnb5iokwexUDkAtCXKGYBWeWfdbt3x6hr5aup149SBuuXMPBYqB4AQoJwBOKKK/fX6+evrNHvVTg3pnqY/X5WvMX06ex0LAKIW5QzAYTnnNHvVTt0ze6321zfpljPz9J3TBrFQOQCEGOUMwOeUVR/UnbMK9O6GPRrdp7N+e9kJGtIjzetYABATKGcAPuX3O73w0Xb9es4GNfn9uvO8YfrGKSxUDgDtiXIGQJK0tbxWt72yWh9srtRJAzN132Wj1C+ThcoBoL1RzoAY1+x3emrRFv1ubmCh8vsuHaWvje/D0ksA4BHKGRDDNu6q0Y//uUqrSqt15rBu+sXFo9QjI8XrWAAQ0yhnQAxqaPLroXnFenh+sdJTEvXHK8bq/BN6crYMAMIA5QyIMStLqvTjf65S4e79unhML911wQh17ZjkdSwAQBDlDIgRBxua9bu3N+qp97eoe3qKnro2X2cMZaFyAAg3lDMgBrRcqPzrwYXK01ioHADCEuUMiGKBhcrX64UPS9Q/M1Uv3jhJkwZmeh0LAHAElDMgSrVcqPxbUwfqByxUDgARgXIGRJmK/fW65/V1en3VTg3tkabHrsrXaBYqB4CIQTkDosjrq3bqrtcKWKgcACIY5QyIEo++t0n3vbFBY/p01m//6wTldWehcgCIRJQzIMI55/S7twv1p3nFumB0L93/1dFKjOdsGQBEKsoZEMH8fqd7/7VOTy/eqsvH99EvLxml+Dhm+QeASEY5AyJUs9/p9ldW66Vlpbpu8gDded4wll8CgChAOQMiUEOTX7f8faX+vaZMN0/L1Q/OzKWYAUCUoJwBEaausVnf+evHmrfRpztmDNMNUwd6HQkA0IYoZ0AE2V/fpOuf+UhLt1TqV5eM0pUT+3odCQDQxihnQISoOtCga/7ykQp2VOv3Xxuji8b09joSACAEKGdABPDV1OuqJ5dqs69Wj3x9nM4e0cPrSACAEKGcAWFuR9VBzXxiqXZV1+mpa8drcm6W15EAACFEOQPC2JbyWs18Yqn21TXqr9dP0In9unodCQAQYpQzIExt2LVPM5/4UH7n9MINkzSyd4bXkQAA7YByBoShVSVVuuYvHyo5IU4vXj9Jg7uxTiYAxArKGRBmPthcoeue/khdOyXp+esnqU/XVK8jAQDaEeUMCCPzNu7Rt5/7WH26puqv101Uj4wUryMBANoZ5exFv8YAACAASURBVAwIE3PWlOnmF1cor3uanv3mBGV2SvY6EgDAA5QzIAz8Y1mJfvLyao3r20VPfWO80lMSvY4EAPAI5Qzw2DOLt+ru2Ws1JTdLf77qRKUm8b8lAMSyuFC+uZlNN7ONZlZsZrcdZntfM5tnZivMbLWZzQi+3t/MDprZyuCvR0OZE/DKQ/OKdffstTp7eHc9cU0+xQwAELozZ2YWL+khSWdJKpX0kZnNds6ta7HbnZJecs49YmbDJc2R1D+4bZNzbkyo8gFecs7pt29t1CPzN+niMb30/74yWonxIf23EgAgQoTy22CCpGLn3GbnXIOkFyVddMg+TlJ68HGGpJ0hzAOEBb/f6a7X1uqR+Zv09Yl9df9Xx1DMAACfCuU3Qm9JJS2elwZfa+keSTPNrFSBs2bfa7FtQHC48z0zm3K4DzCzG81smZkt8/l8bRgdCI2mZr9++M9Veu6Dbbpx6kD94uKRioszr2MBAMKI1/9cv0LS0865HEkzJD1nZnGSyiT1dc6NlfT/SXrezNIPPdg595hzLt85l5+dnd2uwYEvq76pWTc9v0KvLN+hW8/K0+3nDpUZxQwA8FmhLGc7JPVp8Twn+FpL10l6SZKcc0skpUjKcs7VO+cqgq9/LGmTpLwQZgVC6mBDs2549mO9uXaXfnb+cH1vWi7FDABwWKEsZx9JyjWzAWaWJOlySbMP2We7pGmSZGbDFChnPjPLDt5QIDMbKClX0uYQZgVCpqauUdc89aEWFvn0m8tG6brJA7yOBAAIYyG7W9M512RmN0l6S1K8pKecc2vN7F5Jy5xzsyXdKulxM7tFgZsDrnXOOTObKuleM2uU5Jf0bedcZaiyAqGyt7ZB1/zlQ63buU8PXj5WF4zu5XUkAECYM+ec1xnaRH5+vlu2bJnXMYBP7dlXp5lPLtXWigN6dOY4nTG0u9eRAABhwsw+ds7lH24bM14CIVBSeUAzn1wqX029nv7GeJ08KMvrSACACEE5A9rYJt9+zXxiqWrrm/TX6ydqXN8uXkcCAEQQyhnQhtbt3KernlwqM+nv3zpJw3p+bgYYAACOiHIGtJHl2/fq2qc+VMfkBP31+okalN3J60gAgAhEOQPawOLicl3/7DJlpyXrb9dPVE6XVK8jAQAiFOUMOE7vrt+t7/xtufpnpuqv101Ut/QUryMBACIY5Qw4Dq+v2qlb/r5Sw3ul65lvTFCXjkleRwIARDjKGXCM/v7Rdt32yhqN79dVT16br7SURK8jAQCiAOUMOAZPLtqi//nXOp2al61HZ56oDknxXkcCAEQJyhnwJTjn9Mf/FOv+uYU6d2QP/f7yMUpOoJgBANoO5QxoJeecfv3GBj22YLMuG5ej31w2SgnxcV7HAgBEGcoZ0ArNfqefvVag55du19Un9dM9F4xQXJx5HQsAEIUoZ8BRNDb79cN/rNJrK3fqO6cN0o/PGSIzihkAIDQoZ8AR1DU263svrNDcdbv1o3OG6L9PH+x1JABAlKOcAV/gQEOTbnz2Yy0qLtfPLxyha07u73UkAEAMoJwBh1F9sFHffPojrdi+V//7ldH6rxNzvI4EAIgRlDPgEBX763X1Ux+qcHeN/nTlOM0Y1dPrSACAGEI5A1rYVV2nmU8uVUnlAT12db5OH9LN60gAgBhDOQOCtlcc0Nef/EB7axv17DcnaOLATK8jAQBiEOUMkFS8p0Zff2Kp6pv8+tv1EzW6T2evIwEAYhTlDDGvYEe1rn7qQ8WZ6e83nqQhPdK8jgQAiGGsPYOYtmxrpa547AN1SIzXP75NMQMAeI8zZ4hZC4t8uvHZj9UjI0V/vX6ienfu4HUkAAAoZ4hNb6/dpZueX6GB2R313HUTlZ2W7HUkAAAkUc4Qg15dsUO3/mOVRvXO0NPfGK/OqUleRwIA4FOUM8SUvy3dpjtfLdDEAV31xDXj1SmZ/wUAAOGFbybEjDcLynTHrAKdMbSbHv76OKUkxnsdCQCAz6GcISbU1DXq7tlrNaJXuh6deaKSErhRGQAQnihniAkPzC3Snpp6/fmqfIoZACCs8S2FqLd2Z7WeXrxFV07oqzHM/A8ACHOUM0Q1v9/pzlcL1LVjkn58zlCv4wAAcFSUM0S1Fz8q0YrtVfrpjGHKSE30Og4AAEdFOUPUKt9fr9+8uUGTBnbVJWN7ex0HAIBWoZwhav16zgYdaGjSLy4eKTPzOg4AAK1COUNUWrq5Qi8vL9UNUwZqcDcWMwcARA7KGaJOQ5Nfd75aoJwuHfS9M3K9jgMAwJfCPGeIOk8u2qKiPfv15DX56pDEKgAAgMjCmTNEldK9B/Tgu0U6e3h3TRvW3es4AAB8aZQzRJV7Zq+TJN194QiPkwAAcGwoZ4gac9ft1jvrd+sHZ+aqd+cOXscBAOCYUM4QFQ40NOme2Ws1pHuavjl5gNdxAAA4ZtwQgKjw4LvF2lF1UP/49klKjOffHACAyMW3GCJe4e4aPbFws75yYo7G9+/qdRwAAI4L5QwRzbnAwuadUhJ0+4xhXscBAOC4Uc4Q0V5evkMfbqnUbdOHqmvHJK/jAABw3ChniFhVBxr0qznrNa5vZ301v4/XcQAAaBOUM0Ss37y5UdUHG/XLS0YpLo6FzQEA0YFyhoi0fPtevfDhdn3j5P4a1jPd6zgAALQZyhkiTlOzX3fMKlCP9BT94Kw8r+MAANCmKGeIOM8s2ab1Zft09wXD1SmZqfoAANGFcoaIsqu6Tve/vVGnDcnW9JE9vI4DAECbo5whovzPv9apye9074UjZcZNAACA6EM5Q8R4r9Cnf68p002nD1bfzFSv4wAAEBKUM0SEusZm3fVagQZmd9SNpw70Og4AACHD1dSICA/P36RtFQf0/PUTlZwQ73UcAABChjNnCHubffv16PxNumhML508OMvrOAAAhBTlDGHNOae7Xlur5MQ43XEeC5sDAKJfq8qZmb1iZueZGWUO7er11WVaVFyuH50zRN3SUryOAwBAyLW2bD0s6UpJRWZ2n5kNCWEmQJK0r65R//OvdTohJ0Nfn9jP6zgAALSLVpUz59w7zrmvSxonaaukd8xssZl9w8wSQxkQsev+twtVvr9ev7h4pOJZ2BwAECNaPUxpZpmSrpV0vaQVkv6gQFmbG5JkiGkFO6r17JKtumpSP52Q09nrOAAAtJtWTaVhZrMkDZH0nKQLnHNlwU1/N7NloQqH2NTsd7pj1hp17ZisW89mBB0AEFtaO8/Zg865eYfb4JzLb8M8gJ7/cLtWlVbr918bo4wOjJoDAGJLa4c1h5vZp2NLZtbFzL4bokyIYb6aev32zQ06eVCmLhrTy+s4AAC0u9aWsxucc1WfPHHO7ZV0Q2giIZb9as561TU2696LWNgcABCbWlvO4q3FN6WZxUtKCk0kxKrFm8o1a8UOfWvqIA3u1snrOAAAeKK115y9qcDF/38OPv9W8DWgTTQ0+fWzVwvUp2sH3XTGYK/jAADgmdaWs58oUMi+E3w+V9ITIUmEmPT4ws3a5KvVX64dr5REFjYHAMSuVpUz55xf0iPBX0CbKqk8oAffLdL0ET10+tBuXscBAMBTrZ3nLFfSryUNl/TpAofOuYEhyoUY4ZzT3bPXKj7OdNcFw72OAwCA51p7Q8BfFDhr1iTpdEnPSvprqEIhdry1drf+s2GPbjkzT706d/A6DgAAnmttOevgnHtXkjnntjnn7pF0XuhiIRbU1jfp56+v1dAeabr2lP5exwEAICy09oaAejOLk1RkZjdJ2iGJuQ5wXP7wbpHKquv0xyvGKjG+1cu8AgAQ1Vr7jXizpFRJ35d0oqSZkq4JVShEvw279unJRVv0tfw+yu/f1es4AACEjaOeOQtOOPs159wPJe2X9I2Qp0JU8/ud7pxVoPSUBN127lCv4wAAEFaOeubMOdcsaXI7ZEGM+OfHpVq2ba9uP3eYunRkoQkAAFpq7TVnK8xstqR/SKr95EXn3CshSYWotbe2Qb9+Y73y+3XRf52Y43UcAADCTmvLWYqkCklntHjNSaKc4Uv5zZsbtK+uSb+4ZKTi4ljYHACAQ7V2hQCuM8Nx+3hbpV78qEQ3Th2ooT3SvY4DAEBYau0KAX9R4EzZZzjnvtnmiRCVmpr9umNWgXplpOjmablexwEAIGy1dljzXy0ep0i6RNLOto+DaPX04q3asKtGj848UR2TW/ufHQAAsae1w5ovt3xuZi9IWhSSRIg6ZdUH9cDcQp0xtJvOGdHd6zgAAIS1Y52WPVdSt7YMguh17+vr1Oycfn7hCJlxEwAAAEfS2mvOavTZa852SfpJSBIhqszbuEdvFOzSj84Zoj5dU72OAwBA2GvtsGZaqIMg+tQ1Nuvu19ZqUHZH3TBloNdxAACICK0a1jSzS8wso8XzzmZ2cehiIRo8NK9Y2ysP6H8uHqmkBBY2BwCgNVr7jXm3c676kyfOuSpJdx/tIDObbmYbzazYzG47zPa+ZjbPzFaY2Wozm3GY7fvN7IetzIkwscm3X4++t0mXjO2tkwdleR0HAICI0dpydrj9jjgkGlww/SFJ50oaLukKMxt+yG53SnrJOTdW0uWSHj5k+/2S3mhlRoQJ55x+9mqBOiTG66czhnkdBwCAiNLacrbMzO43s0HBX/dL+vgox0yQVOyc2+yca5D0oqSLDtnHSfpkqvgMtZg7LThsukXS2lZmRJiYvWqnFm+q0I+mD1V2WrLXcQAAiCitLWffk9Qg6e8KlKw6Sf99lGN6Sypp8bw0+FpL90iaaWalkuYEP0dm1kmBu0F/fqQPMLMbzWyZmS3z+Xyt+0kQUtUHG/U//1qv0TkZunJCX6/jAAAQcVp7t2atpM9dM9YGrpD0tHPud2Z2kqTnzGykAqXtAefc/iPNi+Wce0zSY5KUn5//ueWl0P5+9/ZGVdbW6+lvjFc8C5sDAPCltfZuzblm1rnF8y5m9tZRDtshqU+L5znB11q6TtJLkuScW6LA0lBZkiZK+q2ZbZX0A0k/NbObWpMV3lldWqXnPtimq0/qr5G9M45+AAAA+JzWLnKYFbxDU5LknNtrZkdbIeAjSblmNkCBUna5pCsP2We7pGmSnjazYQqUM59zbsonO5jZPZL2O+f+1Mqs8ECz3+mOWQXK6pSs/+/sPK/jAAAQsVp7zZnfzD69gMjM+uuzKwZ8jnOuSdJNkt6StF6BuzLXmtm9ZnZhcLdbJd1gZqskvSDpWuccw5MR6G9Lt2nNjmr97PzhSk9J9DoOAAARq7Vnzu6QtMjM3pNkkqZIuvFoBznn5ihwoX/L1+5q8XidpFOO8h73tDIjPLKnpk7/782Nmjw4Sxec0NPrOAAARLTW3hDwppnlK1DIVkh6VdLBUAZD5Pjlv9ervsmvey9iYXMAAI5Xaxc+v17SzQpc1L9S0iRJSySdEbpoiATvF5frtZU79f1puRqY3cnrOAAARLzWXnN2s6TxkrY5506XNFZS1ZEPQbSrb2rWz14tUL/MVH33tEFexwEAICq0tpzVOefqJMnMkp1zGyQNCV0sRILH3tuszeW1uveikUpJjPc6DgAAUaG1NwSUBuc5e1XSXDPbK2lb6GIh3G2vOKA/zSvWeaN66tS8bK/jAAAQNVp7Q8AlwYf3mNk8BdbBfDNkqRDWnHO6a3aBEuJMPzv/0LXsAQDA8WjtmbNPOefeC0UQRI43C3Zp/kaffnb+cPXISPE6DgAAUaW115wBkqT99U36+evrNLxnuq45qZ/XcQAAiDpf+swZYtvv5xZqd02dHp45TgnxdHsAANoa365otfVl+/SXxVt1+fi+Gte3i9dxAACISpQztIrf73THrDXq3CFRP5nOLCoAAIQK5Qyt8tKyEi3fXqXbZwxT59Qkr+MAABC1KGc4qsraBt335gZNGNBVl43r7XUcAACiGuUMR/XrOeu1v65Jv7h4JAubAwAQYpQzHNFHWyv1j49Ldf2UgcrrnuZ1HAAAoh7lDF+osdmvO2cVqHfnDvr+tMFexwEAICYwzxm+0FOLtmjj7ho9fnW+UpP4TwUAgPbAmTMc1o6qg/r9O0U6c1h3nTW8u9dxAACIGZQzHNbPZ6+VJN1zIQubAwDQnihn+Jx31+/W2+t26/vTcpXTJdXrOAAAxBTKGT7jYEOz7p69VrndOum6yQO8jgMAQMzhKm98xh//U6TSvQf19xsnKSmB7g4AQHvj2xefKt5To8cXbtZl43I0cWCm13EAAIhJlDN86ndvFyo1KUE/nTHU6ygAAMQsyhkkSQ1Nfi0o9On8E3oqs1Oy13EAAIhZlDNIkpZv36vahmZNzcv2OgoAADGNcgZJ0sIin+LjTCcN4lozAAC8RDmDJGlBYbnG9e2s9JREr6MAABDTKGdQxf56Feys1pRchjQBAPAa5QxaVFwu58T1ZgAAhAHKGbSwqFydUxM1qneG11EAAIh5lLMY55zTwiKfThmcpfg48zoOAAAxj3IW4zburtHuffU6levNAAAIC5SzGLewsFySNCUvy+MkAABAopzFvAVFPuV266SeGR28jgIAAEQ5i2l1jc1auqWSKTQAAAgjlLMYtnRLpRqa/JrKkCYAAGGDchbDFhb6lJQQp4kDWLIJAIBwQTmLYQuKfJrQv6s6JMV7HQUAAARRzmJUWfVBFe7ez5AmAABhhnIWoxYWBafQ4GYAAADCCuUsRi0o9KlbWrKG9kjzOgoAAGiBchaDmv1Oi4rLNSU3W2Ys2QQAQDihnMWggh3VqjrQyPVmAACEIcpZDFpY5JMkTR5MOQMAINxQzmLQgsJyjeydrsxOyV5HAQAAh6CcxZiaukYt375XU7lLEwCAsEQ5izFLNlWoye+YQgMAgDBFOYsxC4p86pgUrxP7dfE6CgAAOAzKWYxZWFSukwZlKimBP3oAAMIR39AxZFtFrbZVHGBIEwCAMEY5iyELgks2Tc2jnAEAEK4oZzFkQaFPOV06qH9mqtdRAADAF6CcxYjGZr+WbKrQ1DyWbAIAIJxRzmLEiu1V2l/fpKm5rAoAAEA4o5zFiAWFPsXHmU5mySYAAMIa5SxGLCzyaWyfzkpPSfQ6CgAAOALKWQyorG3Q6h3VTKEBAEAEoJzFgEXF5XJOmprHkCYAAOGOchYDFhb6lNEhUSfkdPY6CgAAOArKWZRzzmlBkU+TB2cpPo4pNAAACHeUsyhXtGe/du+r1xSm0AAAICJQzqLcgkKfJJZsAgAgUlDOotyConIN7tZJvTp38DoKAABoBcpZFKtrbNbSzRUMaQIAEEEoZ1Hswy2Vqm/yM6QJAEAEoZxFsYVFPiXFx2nigK5eRwEAAK1EOYtiCwrLNX5AF6UmJXgdBQAAtBLlLErt3lenjbtrWLIJAIAIQzmLUp9OoUE5AwAgolDOotTConJldUrWsJ5pXkcBAABfAuUsCvn9TouKyzU1N0tmLNkEAEAkoZxFoYKd1aqsbWAKDQAAIhDlLAotLCqXJE1m8lkAACIO5SwKvVfo04he6crqlOx1FAAA8CVRzqLM/vomLd+2lyk0AACIUJSzKLNkU4Wa/E5T8xjSBAAgElHOoszCIp9Sk+J1Yr8uXkcBAADHgHIWZRYU+jRpYKaSE+K9jgIAAI4B5SyKbK84oK0VBzSVuzQBAIhYlLMosqAosGTTFOY3AwAgYlHOosiCQp96d+6ggVkdvY4CAACOEeUsSjQ2+7VkU4Wm5rFkEwAAkYxyFiVWllSppr5JU5nfDACAiEY5ixILC32KM+nkwdwMAABAJAtpOTOz6Wa20cyKzey2w2zva2bzzGyFma02sxnB1yeY2crgr1Vmdkkoc0aD94rKNaZPZ2V0SPQ6CgAAOA4hK2dmFi/pIUnnShou6QozG37IbndKesk5N1bS5ZIeDr5eICnfOTdG0nRJfzazhFBljXR7axu0urRKU7lLEwCAiBfKM2cTJBU75zY75xokvSjpokP2cZLSg48zJO2UJOfcAedcU/D1lOB++ALvbyqXc2I9TQAAokAoy1lvSSUtnpcGX2vpHkkzzaxU0hxJ3/tkg5lNNLO1ktZI+naLsqYW+9xoZsvMbJnP52vr/BFjQaFP6SkJGp2T4XUUAABwnLy+IeAKSU8753IkzZD0nJnFSZJzbqlzboSk8ZJuN7OUQw92zj3mnMt3zuVnZ8fmWSPnnBYWleuUwVlKiPf6jxMAAByvUH6b75DUp8XznOBrLV0n6SVJcs4tUWAI8zO3Gzrn1kvaL2lkyJJGsOI9+1VWXcf1ZgAARIlQlrOPJOWa2QAzS1Lggv/Zh+yzXdI0STKzYQqUM1/wmITg6/0kDZW0NYRZI9aConJJ0hTW0wQAICqE7A5I51yTmd0k6S1J8ZKecs6tNbN7JS1zzs2WdKukx83sFgUu+r/WOefMbLKk28ysUZJf0nedc+WhyhrJFhT6NDC7o3K6pHodBQAAtIGQTk/hnJujwIX+LV+7q8XjdZJOOcxxz0l6LpTZokFdY7OWbqnQ5eP7eh0FAAC0Ea4gj2DLtu5VXaNfU/MY0gQAIFpQziLYgiKfEuNNkwZmeh0FAAC0EcpZBFtQ6FN+v65KTWLxBAAAogXlLELt2VenDbtqmEIDAIAoQzmLUAuDU2hwvRkAANGFchahFhT5lNUpScN6pB99ZwAAEDEoZxHI7w8s2TQlN1txceZ1HAAA0IYoZxFoXdk+VdY2sCoAAABRiHIWgd4r9EmSJlPOAACIOpSzCLSwyKdhPdPVLS3F6ygAAKCNUc4iTG19kz7etpe7NAEAiFKUswjzweYKNTY7nZrL/GYAAEQjylmEWVDoU4fEeJ3Yv4vXUQAAQAhQziLMgqJyTRrYVckJ8V5HAQAAIUA5iyAllQe0pbxWUxjSBAAgalHOIsiCosAUGqynCQBA9KKcRZCFheXqlZGiQdkdvY4CAABChHIWIZqa/Xp/U7mm5mXLjCWbAACIVpSzCLGqtEo1dU0MaQIAEOUoZxHivcJyxZl0yiAmnwUAIJpRziLEgkKfRvfprIzURK+jAACAEKKcRYCqAw1aXVrFFBoAAMQAylkEeL+4Qn4nncp6mgAARD3KWQRYWORTWkqCRud09joKAAAIMcpZmHPOaUGhT6cMylJCPH9cAABEO77tw9wmX612VtcxhQYAADGCchbmFhQGlmyaksv1ZgAAxALKWZhbUOTTwKyO6tM11esoAACgHVDOwlh9U7M+2FzBWTMAAGII5SyMLdu6V3WNfq43AwAghlDOwtiCIp8S402TBmZ6HQUAALQTylkYW1BYrhP7dVHH5ASvowAAgHZCOQtTe2rqtL5sH0OaAADEGMpZmFpUVC5Jmsp6mgAAxBTKWZhaUOhTZsckDe+Z7nUUAADQjihnYcjvd1pUXK7JuVmKizOv4wAAgHZEOQtD68r2qXx/A0OaAADEIMpZGFoYvN6MyWcBAIg9lLMwtKDQp6E90tQtPcXrKAAAoJ1RzsLMgYYmLdtWqVOZQgMAgJhEOQszH2yuUGOz0xSuNwP+//buPbbOu77j+OfjWxzHiZM4N8idxFlJaUtKljakdie6ScA2mDSm0a0dQ9O2P4DRCWkrE2MT4p9J0y5/dBsItjGoYKxrJcQqQGMox+my9JKktGnLOU5CEye9nJOLc3Pt2P7uj3OKTJZ2PvY5/j3Hfr+kKOc85+KP8yjHHz+/3/P8AGBeopxlTC5fUntrk3ZuWpY6CgAASIByljG5QlG3be5We2tz6igAACABylmGDJ67omPFyyzZBADAPEY5y5DXL6Fx5zYuoQEAwHxFOcuQXL6ot3S1a8vKztRRAABAIpSzjBgbn9BjAyX19ayUzZJNAADMV5SzjHh6cEgXXhtTL0OaAADMa5SzjMjli7KlO7ZSzgAAmM8oZxnRXyjq5nVLtbSjLXUUAACQEOUsA4auXNXhk+d1JwudAwAw71HOMuC/j5Y0EeL6ZgAAgHKWBblCUYsXtOiW9UtTRwEAAIlRzhKLCOXyJb17a7dam9kdAADMd7SBxI6VLuvU+WH19jCkCQAAKGfJ5fJFSdKdzDcDAACinCXXXyhpU3eH1i/vSB0FAABkAOUsoZGxce0/eoazNAEAwE9QzhJ66sVzGr46rj7mmwEAgArKWUK5fEktTdbtW7pTRwEAABlBOUuov1DUuzYuU+eCltRRAABARlDOEileHNGR0xeYbwYAAH4K5SyRfQPlS2gw3wwAAExGOUukP1/S8kVtuvGtS1JHAQAAGUI5S2BiIpQrlHTH1hVqanLqOAAAIEMoZwm88PJFlS6NMN8MAAD8H5SzBHKF8nyz3p4ViZMAAICsoZwl0F8o6oY1i7V6SXvqKAAAIGMoZ7PsyuiYnjh+jqNmAADguihns+zAsbMaHZ9gvhkAALguytksyxWKWtDSpJ/dtDx1FAAAkEGUs1mWyxd129u61d7anDoKAADIIMrZLDp1flhHi5fVx3wzAADwBihns6g/X1myiflmAADgDVDOZlF/oaQ1S9rVs6ozdRQAAJBRlLNZMj4R2jdQUm/PCtks2QQAAK6PcjZLnh48r6HhqwxpAgCAN0U5myX9+ZJs6Y6tnAwAAADeGOVsluQKRd28tkvLFrWljgIAADKMcjYLhoav6vDJ8wxpAgCA/xflbBbsP1rS+ESot4dyBgAA3hzlbBbkCiV1LmjRjg1LU0cBAAAZRzmrs4hQLl/U7i3dam3mnxsAALw52kKdHS9d1uC5YeabAQCAKaGc1Vl/oSRJrKcJAACmpK7lzPZ7bf/I9oDt+6/z+AbbP7B9yPYPbb+/sv0XbD9l+5nK3++pZ856yuWL2tjdoY3di1JHAQAADaBu5cx2s6QHJL1P0nZJd9vefs3TPiPpmxGxQ9KHJf1dZXtJ0i9HxE2SPiLpq/XKWU+jYxPaf+yM+jhLEwAATFE9j5ztkjQQEcciYlTSNyR98JrnhKQlldtdkk5LUkQciojTle1HJC20OGDx7gAACdVJREFUvaCOWeviqRfP6crouHoZ0gQAAFNUz3K2VtLJSfcHK9sm+3NJ99gelPSopE9c531+VdLBiBi59gHbv2f7SdtPFovF2qSuof5CUS1N1u4t3amjAACABpH6hIC7Jf1zRKyT9H5JX7X9k0y2b5T0F5J+/3ovjogvRsTOiNi5cmX2hg5zhaJu3bBMi9tbU0cBAAANop7l7JSk9ZPur6tsm+x3JH1TkiJiv6R2SSskyfY6SY9I+q2IOFrHnHVRujSiZ09dUN82hjQBAMDU1bOcPSGpx/Zm220qT/j/1jXPOSHpLkmy/XaVy1nR9lJJ/yHp/oh4rI4Z6+axgfIlNFiyCQAAVKNu5SwixiR9XNJ3JT2v8lmZR2x/zvYHKk/7lKTftf20pK9L+u2IiMrrtkr6rO3DlT+r6pW1Hvbmi1rW0ap3rO1KHQUAADSQlnq+eUQ8qvJE/8nbPjvp9nOS9lzndZ+X9Pl6ZquniFB/oaQ7elaqucmp4wAAgAaS+oSAOemFly+qeHGES2gAAICqUc7qoL9QvqwHF58FAADVopzVQS5f0rbVnVrT1Z46CgAAaDCUsxobHh3X4z8+y1EzAAAwLZSzGjtw/IxGxybUu41yBgAAqkc5q7FcvqQFLU26bfPy1FEAAEADopzVWH+hqF2bl6u9tTl1FAAA0IAoZzV0+vywCq9eYr4ZAACYNspZDe0rlJds6mO+GQAAmCbKWQ3tLRS1eskCbVvdmToKAABoUJSzGhmfCO0rlNTbs1I2SzYBAIDpoZzVyDOnhjQ0fJUlmwAAwIxQzmokly/Klno5GQAAAMwA5axG+gtF3bS2S8sXtaWOAgAAGhjlrAYuvHZVB0+cZ0gTAADMGOWsBvYfPaPxieD6ZgAAYMYoZzWQyxe1qK1ZOzYsSx0FAAA0OMrZDEWEcoWidm9ZobYW/jkBAMDM0CZm6MUzV3Ty7LD6tjHfDAAAzBzlbIZyhaIkMd8MAADUBOVshnL5kjYs79CmFYtSRwEAAHMA5WwGRscmtP9oiUtoAACAmqGczcChE+d0eXRcfdsY0gQAALVBOZuBXKGo5iZr95bu1FEAAMAcQTmbgVy+pFs3LNWS9tbUUQAAwBxBOZumM5dG9OzpIc7SBAAANUU5m6Z9AyVFSL3MNwMAADVEOZum/kJJSztaddPartRRAADAHEI5m4aIUH+hqD1bV6i5yanjAACAOYRyNg35Vy7plQsjupP5ZgAAoMYoZ9OQy5eXbOplPU0AAFBjlLNpyBWK6lnVqbd0LUwdBQAAzDGUsyq9dnVcB46fZVUAAABQF5SzKh04flajYxOspwkAAOqCclal/nxRbS1Num0zSzYBAIDao5xVKVcoatem5VrY1pw6CgAAmIMoZ1V4aWhY+VcuqY+zNAEAQJ1QzqrQXyhJknq5vhkAAKgTylkVcvmiVi1eoBvWLE4dBQAAzFGUsykanwjtGyipt2elbJZsAgAA9UE5m6LSpRGtWdLOfDMAAFBXLakDNIrVS9r1nfv6FBGpowAAgDmMI2dVYkgTAADUE+UMAAAgQyhnAAAAGUI5AwAAyBDKGQAAQIZQzgAAADKEcgYAAJAhlDMAAIAMoZwBAABkCOUMAAAgQyhnAAAAGUI5AwAAyBDKGQAAQIZQzgAAADKEcgYAAJAhlDMAAIAMoZwBAABkCOUMAAAgQyhnAAAAGUI5AwAAyBDKGQAAQIZQzgAAADKEcgYAAJAhjojUGWrCdlHSi7PwpVZIKs3C10F9sP8aH/uw8bEPGx/7cOY2RsTK6z0wZ8rZbLH9ZETsTJ0D08P+a3zsw8bHPmx87MP6YlgTAAAgQyhnAAAAGUI5q94XUwfAjLD/Gh/7sPGxDxsf+7COmHMGAACQIRw5AwAAyBDKGQAAQIZQzqbI9ntt/8j2gO37U+dBdWyvt/0D28/ZPmL7k6kzoXq2m20fsv3t1FlQPdtLbT9k+wXbz9venToTqmP7Dyufoc/a/rrt9tSZ5iLK2RTYbpb0gKT3Sdou6W7b29OmQpXGJH0qIrZLul3Sx9iHDemTkp5PHQLT9reSvhMRN0i6RezLhmJ7raQ/kLQzIt4hqVnSh9OmmpsoZ1OzS9JARByLiFFJ35D0wcSZUIWIeCkiDlZuX1T5h8LatKlQDdvrJP2ipC+lzoLq2e6S1Cfpy5IUEaMRcT5tKkxDi6SFtlskdUg6nTjPnEQ5m5q1kk5Ouj8ofrA3LNubJO2QdCBtElTpbyT9kaSJ1EEwLZslFSX9U2Vo+ku2F6UOhamLiFOS/lLSCUkvSRqKiO+lTTU3Uc4wr9julPTvku6LiAup82BqbP+SpFcj4qnUWTBtLZJulfT3EbFD0mVJzN9tILaXqTxqtFnSWyUtsn1P2lRzE+Vsak5JWj/p/rrKNjQQ260qF7MHI+Lh1HlQlT2SPmD7xypPK3iP7a+ljYQqDUoajIjXj1g/pHJZQ+P4eUnHI6IYEVclPSzp3YkzzUmUs6l5QlKP7c2221SeAPmtxJlQBdtWea7L8xHxV6nzoDoR8emIWBcRm1T+//dfEcFv7A0kIl6WdNL2z1Q23SXpuYSRUL0Tkm633VH5TL1LnNRRFy2pAzSCiBiz/XFJ31X57JR/jIgjiWOhOnsk3SvpGduHK9v+JCIeTZgJmG8+IenByi+5xyR9NHEeVCEiDth+SNJBlc+APySWcaoLlm8CAADIEIY1AQAAMoRyBgAAkCGUMwAAgAyhnAEAAGQI5QwAACBDKGcAMEO2f872t1PnADA3UM4AAAAyhHIGYN6wfY/tx20ftv0F2822L9n+a9tHbH/f9srKc99p+39s/9D2I5V1BWV7q+3/tP207YO2t1TevtP2Q7ZfsP1g5QrqAFA1yhmAecH22yX9uqQ9EfFOSeOSflPSIklPRsSNkvZK+rPKS/5F0h9HxM2Snpm0/UFJD0TELSqvK/hSZfsOSfdJ2i7pbSqvSgEAVWP5JgDzxV2S3iXpicpBrYWSXpU0IelfK8/5mqSHbXdJWhoReyvbvyLp32wvlrQ2Ih6RpIh4TZIq7/d4RAxW7h+WtEnSvvp/WwDmGsoZgPnCkr4SEZ/+qY32n17zvOmuaTcy6fa4+HwFME0MawKYL74v6UO2V0mS7eW2N6r8OfihynN+Q9K+iBiSdM52b2X7vZL2RsRFSYO2f6XyHgtsd8zqdwFgzuM3OwDzQkQ8Z/szkr5nu0nSVUkfk3RZ0q7KY6+qPC9Nkj4i6R8q5euYpI9Wtt8r6Qu2P1d5j1+bxW8DwDzgiOkewQeAxmf7UkR0ps4BAK9jWBMAACBDOHIGAACQIRw5AwAAyBDKGQAAQIZQzgAAADKEcgYAAJAhlDMAAIAM+V88/9k2exINTgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 10))\n",
        "plt.plot(history.history['loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train'], loc='upper left')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "9dIpZk6AK8Zl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 621
        },
        "outputId": "38d555ed-00c2-4dd4-ba8b-7b795271c43c"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmEAAAJcCAYAAACxEXM4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXjdZZ3+8ftzsjZpkpM2adIm53TftxxIy1YWQbGALIIUUJxxmQEcQcVlfjqO46jjyAw4MgIuqAwzzlhgAEGQTRBkp02bdN9pm6Rpm7TN2jTreX5/JO0U7JI05+R7lvfrunqZ5ntyctc/vG6f5zmfx5xzAgAAwPDyeR0AAAAgGVHCAAAAPEAJAwAA8AAlDAAAwAOUMAAAAA9QwgAAADxACQOQFMzsQTP7pwG+doeZfXCo7wMAJ0IJAwAA8AAlDAAAwAOUMAAxo38b8GtmttrMDprZr8ysyMyeNbNWM3vRzPKPev0VZrbOzJrM7BUzm3nUs5CZrez/uYclZb7vd33EzKr6f/ZNM5t3ipn/2sy2mtkBM/udmY3r/76Z2Y/MrN7MWsxsjZnN6X92qZmt78+2y8y+ekr/hQGIa5QwALHmGkkfkjRN0uWSnpX0d5IK1fe/WV+QJDObJmmppC/1P3tG0lNmlm5m6ZKekPRrSaMk/W//+6r/Z0OSHpB0s6TRkn4u6XdmljGYoGZ2oaQfSFoiaayknZIe6n98saTz+v8def2v2d//7FeSbnbO5UiaI+mPg/m9ABIDJQxArLnHObfXObdL0muS3nHOVTrnOiT9VlKo/3XXSfq9c+4PzrluSXdJGiHpbElnSkqTdLdzrts596ik5Uf9jpsk/dw5945zrtc595+SOvt/bjA+IekB59xK51ynpG9IOsvMJkjqlpQjaYYkc85tcM7t7v+5bkmzzCzXOdfonFs5yN8LIAFQwgDEmr1HfX3oGH8f2f/1OPWtPEmSnHNhSTWSSvqf7XLOuaN+dudRX4+X9JX+rcgmM2uSFOj/ucF4f4Y29a12lTjn/ijpXkn3Sao3s/vNLLf/pddIulTSTjP7k5mdNcjfCyABUMIAxKs69ZUpSX1nsNRXpHZJ2i2ppP97hwWP+rpG0vedc/6j/mQ555YOMUO2+rY3d0mSc+7HzrnTJc1S37bk1/q/v9w5d6WkMerbNn1kkL8XQAKghAGIV49IuszMLjKzNElfUd+W4puS3pLUI+kLZpZmZldLWnjUz/5C0i1mdkb/AfpsM7vMzHIGmWGppE+bWVn/ebJ/Vt/26Q4zW9D//mmSDkrqkBTuP7P2CTPL699GbZEUHsJ/DwDiFCUMQFxyzm2SdKOkeyTtU98h/sudc13OuS5JV0v6lKQD6js/9vhRP1sh6a/Vt13YKGlr/2sHm+FFSd+S9Jj6Vt8mS7q+/3Gu+speo/q2LPdLurP/2Scl7TCzFkm3qO9sGYAkY+89MgEAAIDhwEoYAACAByhhAAAAHqCEAQAAeIASBgAA4IFUrwMMVkFBgZswYYLXMQAAAE5qxYoV+5xzhcd6FnclbMKECaqoqPA6BgAAwEmZ2c7jPWM7EgAAwAOUMAAAAA9QwgAAADwQd2fCjqW7u1u1tbXq6OjwOkrUZWZmqrS0VGlpaV5HAQAAQ5AQJay2tlY5OTmaMGGCzMzrOFHjnNP+/ftVW1uriRMneh0HAAAMQUJsR3Z0dGj06NEJXcAkycw0evTopFjxAwAg0SVECZOU8AXssGT5dwIAkOgSpoQBAADEE0pYBDQ1NeknP/nJoH/u0ksvVVNTUxQSAQCAWEcJi4DjlbCenp4T/twzzzwjv98frVgAACCGJcSnI7329a9/Xdu2bVNZWZnS0tKUmZmp/Px8bdy4UZs3b9ZVV12lmpoadXR06Itf/KJuuukmSf93BVNbW5suueQSLVq0SG+++aZKSkr05JNPasSIER7/ywAAQLQkXAn7zlPrtL6uJaLvOWtcrr59+ezjPr/jjju0du1aVVVV6ZVXXtFll12mtWvXHhkj8cADD2jUqFE6dOiQFixYoGuuuUajR49+z3ts2bJFS5cu1S9+8QstWbJEjz32mG688caI/jsAAEDsSLgSFgsWLlz4njleP/7xj/Xb3/5WklRTU6MtW7b8WQmbOHGiysrKJEmnn366duzYMWx5AQDA8Eu4EnaiFavhkp2dfeTrV155RS+++KLeeustZWVl6YILLjjmnK+MjIwjX6ekpOjQoUPDkhUAAHiDg/kRkJOTo9bW1mM+a25uVn5+vrKysrRx40a9/fbbw5wOAADEooRbCfPC6NGjdc4552jOnDkaMWKEioqKjjxbvHixfvazn2nmzJmaPn26zjzzTA+TAgCAWGHOOa8zDEp5ebmrqKh4z/c2bNigmTNnepRo+CXbvxcAgHhlZiucc+XHesZ2JAAAgAcoYQAAAB5ImBIWb9uqpypZ/p0AACS6hChhmZmZ2r9/f8IXFOec9u/fr8zMTK+jAACAIUqIT0eWlpaqtrZWDQ0NXkeJuszMTJWWlnodAwAADFFClLC0tLT3TKgfivV1Lbr1Nyt1xzXztHDiqIi8JwAAwPslxHZkJBXlZujdfQdVWd3odRQAAJDAKGHvM3pkhsaPzlJldZPXUQAAQAKjhB1DKODXyurGhD/oDwAAvEMJO4aygF/1rZ3a3fznF20DAABEAiXsGELBfElSVQ1bkgAAIDooYccwc2yu0lN9HM4HAABRQwk7hvRUn+aW5HE4HwAARA0l7DjKAn6t2dWs7t6w11EAAEACooQdRyjoV2dPWBt3t3odBQAAJCBK2HEcPpxfWcO5MAAAEHmUsOMYl5epwpwMzoUBAICooIQdh5kpFPAzpgIAAEQFJewEQsF8bd93UI0Hu7yOAgAAEgwl7ARCQb8khrYCAIDIo4SdwNySPPlMqqSEAQCACKOEnUB2RqqmF+cyOR8AAEQcJewkQsG+w/nhsPM6CgAASCCUsJMoC/jV2tGjd/e1eR0FAAAkEErYSZzWfzifeWEAACCSolbCzOwBM6s3s7Uned0CM+sxs49FK8tQTCoYqZzMVA7nAwCAiIrmStiDkhaf6AVmliLpXyS9EMUcQ+LzmcoCflbCAABAREWthDnnXpV04CQvu03SY5Lqo5UjEkIBvzbtaVF7V4/XUQAAQILw7EyYmZVI+qiknw7gtTeZWYWZVTQ0NEQ/3PuEgvkKO2l1bfOw/24AAJCYvDyYf7ek/+ecC5/shc65+51z5c658sLCwmGI9l5lAQ7nAwCAyEr18HeXS3rIzCSpQNKlZtbjnHvCw0zHlJ+drgmjsxjaCgAAIsazEuacm3j4azN7UNLTsVjADgsF8/X61n1yzqm/OAIAAJyyaI6oWCrpLUnTzazWzD5rZreY2S3R+p3RFAr61dDaqbrmDq+jAACABBC1lTDn3A2DeO2nopUjUkKBfElSZXWjSvwjPE4DAADiHRPzB2jG2BxlpPpUxeF8AAAQAZSwAUpL8WluSR6T8wEAQERQwgYhFPRrza5mdfWcdKoGAADACVHCBqEskK+unrA27G7xOgoAAIhzlLBBCAX7hrZWsSUJAACGiBI2CGPzMlWUm8HQVgAAMGSUsEEwM4UC+RzOBwAAQ0YJG6SyoF8797frwMEur6MAAIA4RgkbpFDg8LkwtiQBAMCpo4QN0tzSPKX4TJUMbQUAAENACRukrPRUzSjOoYQBAIAhoYSdgrKAX6tqmhQOO6+jAACAOEUJOwWhYL5aO3u0raHN6ygAACBOUcJOweGhrWxJAgCAU0UJOwUTR2crNzNVlXxCEgAAnCJK2Cnw+UxlwXxWwgAAwCmjhJ2iUMCvzXtb1dbZ43UUAAAQhyhhpygU9CvspNW1rIYBAIDBo4SdorIjk/MpYQAAYPAoYafIn5WuSQXZnAsDAACnhBI2BGVBvyqrm+QcQ1sBAMDgUMKGIBTwa19bp2obD3kdBQAAxBlK2BCEgvmSOBcGAAAGjxI2BNOLc5SZ5uNcGAAAGDRK2BCkpfg0r8TP5HwAADBolLAhKgv6ta6uRZ09vV5HAQAAcYQSNkShgF9dPWFt2N3qdRQAABBHKGFDdPhwfmU1W5IAAGDgKGFDVJyXqeLcTA7nAwCAQaGERUAo6GdMBQAAGBRKWASEgn5VH2jXvrZOr6MAAIA4QQmLgCNDW9mSBAAAA0QJi4A54/KU4jO2JAEAwIBRwiJgRHqKZo7NYWgrAAAYMEpYhIQC+VpV06zesPM6CgAAiAOUsAgpC/jV1tmjrfVtXkcBAABxgBIWIaGgX5JUxZYkAAAYAEpYhEwsyFbeiDSGtgIAgAGhhEWImSkU9FPCAADAgFDCIqgs4Nfm+la1dfZ4HQUAAMQ4SlgEhYL5ck5azbwwAABwEpSwCCor7TucX0kJAwAAJ0EJi6C8rDRNLsxWZTWfkAQAACdGCYuwskC+qmqa5BxDWwEAwPFRwiIsFPRrX1uXahsPeR0FAADEMEpYhB0e2rqSLUkAAHAClLAIm16UoxFpKcwLAwAAJ0QJi7DUFJ/mluapik9IAgCAE6CERUEo6Nf6uhZ19vR6HQUAAMQoSlgUhAL56uoNa11di9dRAABAjKKERcHhw/lVnAsDAADHQQmLgqLcTI3Ly2RyPgAAOC5KWJSEgvlMzgcAAMdFCYuSsoBftY2H1NDa6XUUAAAQgyhhUXLkXBhbkgAA4BgoYVEypyRPqT5jSxIAABwTJSxKMtNSNGtcLpPzAQDAMVHCoqgs4Nfq2ib1hp3XUQAAQIyhhEVRKOjXwa5ebalv9ToKAACIMZSwKAoF8iWJLUkAAPBnKGFRNH50lvKz0jicDwAA/kzUSpiZPWBm9Wa29jjPrzSz1WZWZWYVZrYoWlm8YmYqC/gZUwEAAP5MNFfCHpS0+ATPX5I03zlXJukzkn4ZxSyeCQXztaW+TS0d3V5HAQAAMSRqJcw596qkAyd43uacO/yxwWxJCfkRwlDQL+ek1TXNXkcBAAAxxNMzYWb2UTPbKOn36lsNO97rburfsqxoaGgYvoARMK/08OR8zoUBAID/42kJc8791jk3Q9JVkr53gtfd75wrd86VFxYWDl/ACMgbkaYpY0byCUkAAPAeMfHpyP6ty0lmVuB1lmgIBfyqrGnS/+2+AgCAZOdZCTOzKWZm/V+fJilD0n6v8kRTWdCvAwe7VH2g3esoAAAgRqRG643NbKmkCyQVmFmtpG9LSpMk59zPJF0j6S/MrFvSIUnXuQRdKjo8tLWqpknjR2d7nAYAAMSCqJUw59wNJ3n+L5L+JVq/P5ZMKxqprPQUVVY36cqyEq/jAACAGBATZ8ISXWqKT/NK85icDwAAjqCEDZOyQL7W725RR3ev11EAAEAMoIQNk1DQr+5ep3V1LV5HAQAAMYASNkxCgb6hrWxJAgAAiRI2bMbkZqrEP0KVXOYNAABECRtWZUG/qpicDwAARAkbVqGAX7uaDqm+pcPrKAAAwGOUsGEUCvYNbWVLEgAAUMKG0exxuUpLMS7zBgAAlLDhlJmWolljc1VVwyckAQBIdpSwYRYK5mt1bbN6esNeRwEAAB6ihA2zUNCv9q5ebd7b5nUUAADgIUrYMCvrH9paxeF8AACSGiVsmAVHZWlUdjqT8wEASHKUsGFmZgoF/IypAAAgyVHCPFAW8GtrfZuaD3V7HQUAAHiEEuaBw0NbV9eyGgYAQLKihHlgXiBPZmJoKwAASYwS5oHczDRNHTOSw/kAACQxSphHygJ+VdU0yTnndRQAAOABSphHQsF8NbZ3a+f+dq+jAAAAD1DCPBIK9g1treQeSQAAkhIlzCNTx+QoOz2Fw/kAACQpSphHUnymeaV+ri8CACBJUcI8FAr6tb6uRR3dvV5HAQAAw4wS5qFQMF89Yae1u5q9jgIAAIYZJcxDZYG+w/lsSQIAkHwoYR4qzMlQaf4IDucDAJCEKGEeCwXzmZwPAEASooR5rCzgV11zh/a2dHgdBQAADCNKmMeODG1lSxIAgKRCCfPY7HG5Sk/xMTkfAIAkQwnzWEZqimaNy2UlDACAJEMJiwFlAb/W1DarpzfsdRQAADBMKGExIBT061B3rzbtbfU6CgAAGCaUsBhwWjBfEofzAQBIJpSwGFCaP0Kjs9MpYQAAJBFKWAwwM4WCflXxCUkAAJIGJSxGhIL52tZwUM3t3V5HAQAAw4ASFiNChy/zrmVLEgCAZEAJixFzS/NkJu6RBAAgSVDCYkROZpqmjclRVQ0rYQAAJANKWAwJBf2qrG6Sc87rKAAAIMooYTGkLOBX86Fubd930OsoAAAgyihhMSTUP7SVLUkAABIfJSyGTBkzUiMzUhnaCgBAEqCExZAUn2l+IE+VDG0FACDhUcJiTFnArw27W3Woq9frKAAAIIooYTEmFMhXb9hpbV2z11EAAEAUUcJiTFmwb3I+Q1sBAEhslLAYUzAyQ8FRWRzOBwAgwVHCYlBZwM+YCgAAEhwlLAaFgn7tbu7Q7uZDXkcBAABRQgmLQUeGtrIlCQBAwqKExaCZY3OUnuJTJVuSAAAkLEpYDMpITdHsklxWwgAASGCUsBgVCuRr9a4mdfeGvY4CAACigBIWo0JBvzq6w9q0p9XrKAAAIAooYTGqLNA/tJVzYQAAJCRKWIwqzR+hgpEZTM4HACBBUcJilJkpFPRzOB8AgAQVtRJmZg+YWb2ZrT3O80+Y2WozW2Nmb5rZ/GhliVdlAb/e3XdQTe1dXkcBAAARFs2VsAclLT7B8+2SznfOzZX0PUn3RzFLXAr1X+bNFUYAACSeqJUw59yrkg6c4PmbzrnDB57ellQarSzxal6pXz4Tl3kDAJCAYuVM2GclPXu8h2Z2k5lVmFlFQ0PDMMby1siMVE0ryuETkgAAJCDPS5iZfUB9Jez/He81zrn7nXPlzrnywsLC4QsXA0JBv1bVNCkcdl5HAQAAEeRpCTOzeZJ+KelK59x+L7PEqlAgX82HurV9/0GvowAAgAjyrISZWVDS45I+6Zzb7FWOWHf4cD7nwgAASCzRHFGxVNJbkqabWa2ZfdbMbjGzW/pf8g+SRkv6iZlVmVlFtLLEs8mFI5WTkcrQVgAAEkxqtN7YOXfDSZ7/laS/itbvTxQ+n2l+wM+YCgAAEoznB/NxcqGgXxv3tKq9q8frKAAAIEIoYXEgFPSrN+y0prbZ6ygAACBCKGFxYH4pk/MBAEg0lLA4MHpkhsaPzuITkgAAJBBKWJwIBfxaWd0o5xjaCgBAIqCExYmygF/1rZ3a3dzhdRQAABABlLA4EQrmS+JcGAAAiYISFidmjs1VeqqPoa0AACQISlicSE/1aW5JHofzAQBIEJSwOFIW8GvNrmZ194a9jgIAAIaIEhZHQkG/OnvC2ri71esoAABgiChhceTw4fzKGs6FAQAQ7yhhcWRcXqbG5GRwLgwAgARACYsjZqaygJ8xFQAAJABKWJwJBfO1fd9BNR7s8joKAAAYAkpYnAkFucwbAIBEQAmLM3NL8uQzMbQVAIA4RwmLM9kZqZpenKtKVsIAAIhrlLA4FAr2Hc4Ph53XUQAAwCmihMWhUMCv1o4evbuvzesoAADgFFHC4tDhw/nMCwMAIH5RwuLQpIKRyslM5VwYAABxjBIWh3y+vqGtrIQBABC/KGFxKhTwa9OeFh3s7PE6CgAAOAWUsDgVCuYr7KQ1u5q9jgIAAE4BJSxOlQU4nA8AQDyjhMWp/Ox0TSzIZnI+AABxihIWx8oCflXWNMk5hrYCABBvKGFxLBT0q6G1U3XNHV5HAQAAg0QJi2OhQL4kLvMGACAeUcLi2IyxOcpI9XE4HwCAOEQJi2NpKT7NLclTFZPzAQCIO5SwOBcK+rVmV7O6esJeRwEAAINACYtzoWC+unrC2rC7xesoAABgEChhce7w0Fa2JAEAiC+UsDg3Ni9TRbkZfEISAIA4QwmLc2amUCBflayEAQAQVyhhCaAs6NfO/e3a39bpdRQAADBAlLAEEOo/F7aqltUwAADiBSUsAcwtzVOKzxjaCgBAHKGEJYCs9FTNKM6hhAEAEEcoYQmiLODXqpomhcPO6ygAAGAAKGEJIhTMV2tnj7Y1tHkdBQAADAAlLEGEgn2H89mSBAAgPlDCEsTE0dnKzUxVZQ1DWwEAiAeUsATh85nKgvmshAEAECcoYQkkFPBr895WtXX2eB0FAACcBCUsgYSCfoWdtJqhrQAAxDxKWAIpC3A4HwCAeEEJSyD+rHRNKshWFZd5AwAQ8yhhCaYs6FdldZOcY2grAACxjBKWYELBfO1r61Rt4yGvowAAgBOghCWYUP+5MLYkAQCIbZSwBDO9OEeZaT4O5wMAEOMoYQkmLcWneSV+JucDABDjKGEJqCzo17pdLers6fU6CgAAOA5KWAIKBfzq6g1rw+5Wr6MAAIDjoIQloFAwX5JUWc2WJAAAsYoSloCK8zI1Ni+Tw/kAAMQwSliCKgv4GVMBAEAMi1oJM7MHzKzezNYe5/kMM3vLzDrN7KvRypGsQkG/qg+0a19bp9dRAADAMURzJexBSYtP8PyApC9IuiuKGZLW4XNhVWxJAgAQk6JWwpxzr6qvaB3veb1zbrmk7mhlSGZzxuUpxWfMCwMAIEbFxZkwM7vJzCrMrKKhocHrOHFhRHqKZo7N4VwYAAAxKi5KmHPufudcuXOuvLCw0Os4cSMUyNeqmmb1hp3XUQAAwPvERQnDqQkF/Wrr7NHW+javowAAgPehhCWwsoBfklTFuTAAAGJONEdULJX0lqTpZlZrZp81s1vM7Jb+58VmVivpy5L+vv81udHKk4wmFmQrb0QaQ1sBAIhBqdF6Y+fcDSd5vkdSabR+PyQzUyjop4QBABCD2I5McGUBvzbXt6q1g0kgAADEEkpYggsF8+WctKa22esoAADgKJSwBFdW2nc4v5J5YQAAxBRKWILLy0rT5MJsVVbzCUkAAGLJgEqYmX3RzHKtz6/MbKWZXRztcIiMskC+Kqub5BxDWwEAiBUDXQn7jHOuRdLFkvIlfVLSHVFLhYgKBf3af7BLtY2HvI4CAAD6DbSEWf9/Xirp1865dUd9DzEuFOw7F7aSLUkAAGLGQEvYCjN7QX0l7Hkzy5EUjl4sRNL0ohyNSEthXhgAADFkoMNaPyupTNK7zrl2Mxsl6dPRi4VISk3xaW5pnqr4hCQAADFjoCthZ0na5JxrMrMbJf29JAZPxZFQ0K/1dS3q7On1OgoAANDAS9hPJbWb2XxJX5G0TdJ/RS0VIi4UyFdXb1jr6lq8jgIAADTwEtbj+uYbXCnpXufcfZJyohcLkXb4cD7nwgAAiA0DPRPWambfUN9oinPNzCcpLXqxEGlFuZkal5fJuTAAAGLEQFfCrpPUqb55YXsklUq6M2qpEBWhYD6T8wEAiBEDKmH9xet/JOWZ2UckdTjnOBMWZ0JBv2obD6mhtdPrKAAAJL2BXlu0RNIySddKWiLpHTP7WDSDIfLKAn3nwtiSBADAewM9E/ZNSQucc/WSZGaFkl6U9Gi0giHy5pTkKdVnqqxu1IdmFXkdBwCApDbQM2G+wwWs3/5B/CxiRGZaimaNy+UTkgAAxICBroQ9Z2bPS1ra//frJD0TnUiIprKAX4+tqFVv2CnFx/WfAAB4ZaAH878m6X5J8/r/3O+c+3/RDIboCAX9OtjVqy31rV5HAQAgqQ10JUzOucckPRbFLBgGoUC+pL6hrTOKcz1OAwBA8jrhSpiZtZpZyzH+tJoZ99/EofGjs5SflabXt+7zOgoAAEnthCXMOZfjnMs9xp8c5xzLKHHIzHRteUC/X71br2yqP/kPAACAqOATjknoyx+apqljRupvH12txoNdXscBACApUcKSUGZain50XZka27v0zSfWqO9udgAAMJwoYUlqTkmebv/QND2zZo+eqNrldRwAAJIOJSyJ3XzeZJWPz9c/PLFOu5oOeR0HAICkQglLYik+078tKVPYOX3lkSqFw2xLAgAwXChhSS44Okvfvny23n73gB54Y7vXcQAASBqUMOja8lJ9aFaR/vW5Tdq0h0n6AAAMB0oYZGb6wdVzlTsiVV96uEqdPb1eRwIAIOFRwiBJKhiZoTuunqcNu1v0oz9s8ToOAAAJjxKGIz44q0g3LAzo569u07LtB7yOAwBAQqOE4T3+/rJZCuRn6cuPVKm1o9vrOAAAJCxKGN4jOyNVP7puvuqaDum7T633Og4AAAmLEoY/c/r4UfqbC6bof1fU6rm1e7yOAwBAQqKE4Zi+cNFUzSnJ1d/9do3qWzu8jgMAQMKhhOGY0lN9+tGSMh3s7NE3HuOSbwAAIo0ShuOaWpSjr18yQy9trNdDy2u8jgMAQEKhhOGE/vKsCVo0pUDfe3q9duw76HUcAAASBiUMJ+Tzme68dp5SfaYvP1Klnt6w15EAAEgIlDCc1Ni8Efqnj87Vyuom/exP27yOAwBAQqCEYUCumD9OV8wfp7tf3KI1tc1exwEAIO5RwjBg37tyjgpGZuhLD1eqo5tLvgEAGApKGAYsLytNd147T9saDuqOZzd6HQcAgLhGCcOgnDu1UJ86e4IefHOHXtvS4HUcAADiFiUMg/b1S2ZocmG2vvq/q9TU3uV1HAAA4hIlDIOWmZaiu68LaX9bl7715Dqv4wAAEJcoYTglc0vz9KUPTtVTq+r0ZNUur+MAABB3KGE4ZbecP1mnBf361hNrVdd0yOs4AADEFUoYTllqik//tqRMPWGnrz26SuEwl3wDADBQlDAMyYSCbH3rI7P0xtb9evDNHV7HAQAgblDCMGTXLwjoohljdMdzG7Vlb6vXcQAAiAuUMAyZmemOa+ZpZEaqvvRwlbp6uOQbAICToYQhIgpzMvSDq+dqXV2L/v2lzV7HAQAg5lHCEDEfnl2sJeWl+ukr21Sx44DXcQAAiGmUMETUP1w+WyX5I/TlR1aprbPH6zgAAMQsShgiamRGqv5tSZlqGtv1T0+v9zoOAAAxixKGiFswYZRuOX+yHlpeoz+s3+t1HAAAYhIlDFFx+wenaebYXH39sdXa19bpdRwAAGIOJQxRkZ7q093Xlam1s0dff2yNnGOaPgAAR4taCTOzB8ys3szWHue5mTy5EiYAACAASURBVNmPzWyrma02s9OilQXemF6co7/98HS9uGGvHqmo8ToOAAAxJZorYQ9KWnyC55dImtr/5yZJP41iFnjkM+dM1FmTRus7T63Xzv0HvY4DAEDMiFoJc869KulEw6KulPRfrs/bkvxmNjZaeeANn89015L5SvGZvvzIKvVyyTcAAJK8PRNWIunoPara/u/9GTO7ycwqzKyioaFhWMIhckr8I/S9K+doxc5G/exP27yOAwBATIiLg/nOufudc+XOufLCwkKv4+AUXFk2TpfNG6sf/WGz1u5q9joOAACe87KE7ZIUOOrvpf3fQwIyM33/qjkalZ2u2x+uUkd3r9eRAADwlJcl7HeS/qL/U5JnSmp2zu32MA+izJ+Vrjuvna8t9W268/lNXscBAMBTqdF6YzNbKukCSQVmVivp25LSJMk59zNJz0i6VNJWSe2SPh2tLIgd508r1F+eNV6/en27LpwxRudMKfA6EgAAnrB4G6JZXl7uKioqvI6BITjU1avL7nlNh7p69dyXzlPeiDSvIwEAEBVmtsI5V36sZ3FxMB+JZUR6iu6+rkwNrZ369pPHnOULAEDCo4TBE/NK/brtwql6oqpOT62q8zoOAADDjhIGz3z+A5M1P+DX3z+xVnuaO7yOAwDAsKKEwTOpKT79aMl8dfWE9bVHVynMNH0AQBKhhMFTkwpH6puXzdRrW/bp12/v9DoOAADDhhIGz33ijKAumF6of35mg7bWt3kdBwCAYUEJg+fMTP96zTxlpafo9oer1N0b9joSAABRRwlDTBiTm6kfXD1Xa3Y1656XtngdBwCAqKOEIWYsnjNW15xWqntf3qqV1Y1exwEAIKooYYgp375ilsbmjdCXH67Swc4er+MAABA1lDDElNzMNP1wyXztPNCu7z+zwes4AABEDSUMMefMSaN107mT9Jt3qvXHjXu9jgMAQFRQwhCTvnzxNM0oztHfPrpG+9s6vY4DAEDEUcIQkzJSU/Sj68rUcqhb33h8jZxjmj4AILFQwhCzZo7N1Vc/PE0vrN+rR1fUeh0HAICIooQhpn120SSdMXGUvvPUetUcaPc6DgAAEUMJQ0xL8Zl+uGS+JOkrj6xSL5d8AwASBCUMMa80P0vfuWK2lu04oF+89q7XcQAAiAhKGOLC1aeV6JI5xfrhC5u0vq7F6zgAAAwZJQxxwcz0/Y/OlT8rXbc/XKWO7l6vIwEAMCSUMMSNUdnp+tePzdOmva364QubvI4DAMCQUMIQVz4wfYxuPDOoX76+XW9u2+d1HAAAThklDHHn7y6dqQmjs/XVR1appaPb6zgAAJwSShjiTlZ6qv5tyXztbe3UPz65zus4AACcEkoY4lIomK9bPzBFj1fu0u9X7/Y6DgAAg0YJQ9y69cIpml+ap28+sUZ7Wzq8jgMAwKBQwhC30lJ8+rfrytTR3auvPbqaS74BAHGFEoa4NrlwpL556Uy9urlB//1OtddxAAAYMEoY4t6NZ47XedMK9f3fr9e7DW1exwEAYEAoYYh7ZqY7PzZPmWkpuv3hKnX3hr2OBADASVHCkBCKcjP1/avmalVts+57eavXcQAAOClKGBLGZfPG6qOhEt3zx62qqmnyOg4AACdECUNC+ccrZqsoJ0NffKhSNQfavY4DAMBxUcKQUPJGpOnHN4R04GCXLr/3db26ucHrSAAAHBMlDAmnfMIoPXXrIhXnZuov/2OZ7nt5q8JhZogBAGILJQwJaUJBth7/m7N1+bxxuvP5Tbrlv1eolcu+AQAxhBKGhJWVnqp/v75M//CRWXppY72uvPcNbdnb6nUsAAAkUcKQ4MxMn1k0Ub/5qzPU0tGjK+97Q8+s4cJvAID3KGFICmdMGq2nb1uk6cU5+pv/WakfPLNBPQx1BQB4iBKGpFGcl6mHbzpLnzxzvH7+6rv65K+WaX9bp9exAABJihKGpJKe6tP3rpqjOz82TyuqG3X5Pa9rFYNdAQAeoIQhKV1bHtDjnztbZqZrf/aWHlpW7XUkAECSoYQhac0pydPTty3SGZNG6euPr9E3Hl+tzp5er2MBAJIEJQxJLT87XQ9+eqE+/4HJWrqsRkt+9pbqmg55HQsAkAQoYUh6KT7T1z48Qz//5Ona1nBQl9/zut7cus/rWACABEcJA/p9eHaxnrz1HOVnp+vGX72j+1/dJue47ggAEB2UMOAokwtH6onPn6PFc4r1z89s1K2/qVRbZ4/XsQAACYgSBrzPyIxU3ffx0/SNS2bo2bW79dH73tC2hjavYwEAEgwlDDgGM9PN50/Wrz97hvYf7NKV976h59ft8ToWACCBUMKAEzhnSoGeum2RJhdm6+Zfr9Bdz29Sb5hzYgCAoaOEASdR4h+hh28+S9cvCOjel7fq0w8uV+PBLq9jAQDiHCUMGIDMtBTdcc08/eDquXp7235dfu/rWrur2etYAIA4RgkDBuGGhUE9cstZ6g07XfPTN/XoilqvIwEA4hQlDBiksoBfT922SKcF8/XV/12lbz2xVl09Ya9jAQDiDCUMOAUFIzP0688u1E3nTdKv396p6+9/S3tbOryOBQCII5Qw4BSlpvj0d5fO1L0fD2njnlZd9uPXtWz7Aa9jAQDiBCUMGKKPzBunJz5/jnIzU/XxX7yt/3hjO9cdAQBOihIGRMC0ohw9ces5+sCMMfrOU+v1pYer1N7FdUcAgOOjhAERkpuZpp/feLq+evE0/W5Vna7+yZvauf+g17EAADGKEgZEkM9nuvXCqXrw0wu1u7lDl9/zul7eWO91LABADKKEAVFw/rRCPX3bIpXmZ+kz/7lcd7+4WWGuOwIAHCWqJczMFpvZJjPbamZfP8bz8Wb2kpmtNrNXzKw0mnmA4RQYlaXHPne2Phoq0d0vbtFf/1eFmg91ex0LABAjolbCzCxF0n2SLpE0S9INZjbrfS+7S9J/OefmSfqupB9EKw/ghRHpKfrhtfP1vStn60+bG3TFva9r454Wr2MBAGJANFfCFkra6px71znXJekhSVe+7zWzJP2x/+uXj/EciHtmpk+eNUEP33ymDnX16qP3vaknq3Z5HQsA4LFolrASSTVH/b22/3tHWyXp6v6vPyopx8xGv/+NzOwmM6sws4qGhoaohAWi7fTxo/T0FxZpTkmuvvhQlb771Hp193LdEQAkK68P5n9V0vlmVinpfEm7JPW+/0XOufudc+XOufLCwsLhzghEzJicTP3mr8/Up86eoAfe2K5P/PId1bdy3REAJKNolrBdkgJH/b20/3tHOOfqnHNXO+dCkr7Z/72mKGYCPJeW4tM/XjFbd19XptW1Tbr8nte1Ymej17EAAMMsmiVsuaSpZjbRzNIlXS/pd0e/wMwKzOxwhm9IeiCKeYCYclWoRI9/7hxlpKbo+vvf0q/f3sl1RwCQRKJWwpxzPZJulfS8pA2SHnHOrTOz75rZFf0vu0DSJjPbLKlI0vejlQeIRbPG5eqpWxdp0ZQCfeuJtfrao6vV0f1nO/IAgARk8fb/vMvLy11FRYXXMYCICoed/v2lLfr3l7ZoTkmufvqJ0xUYleV1LADAEJnZCudc+bGeeX0wH4D6rju6/UPT9Ku/LNfO/e26/N7X9epmPgkMAImMEgbEkItmFumpWxepKCdTf/kfy3Tfy1s5JwYACYoSBsSYCQXZ+u3nz9ZH5o3Tnc9v0s2/XqHWDq47AoBEQwkDYlBWeqp+fH2ZvvWRWXppY72uvO8Nbdnb6nUsAEAEUcKAGGVm+uyiifqfvzpDLYe6ddV9b+jZNbu9jgUAiBBKGBDjzpw0Wk/fdq6mFefoc/+zUj94doN6uO4IAOIeJQyIA8V5mXropjN145lB/fxP7+ovHlimuqZDXscCAAwBJQyIExmpKfqnq+bqzo/N04qdjbrgrlf03afWa19bp9fRAACngBIGxJlrywN66Svn66qycfrPt3bovH99WXc+v1HN7XyCEgDiCRPzgTj2bkObfvTiFj21qk65mam66bxJ+vQ5E5Wdkep1NACATjwxnxIGJIANu1v0wxc268UNezU6O12fu2CybjxzvDLTUryOBgBJjRIGJInK6kb98IXNen3rPhXnZuq2i6ZoSXlAaSmcPAAAL1DCgCTz5rZ9uuv5TVpZ3aTgqCzd/qGpumJ+iVJ85nU0AEgqXOANJJmzJxfosc+drQc+Va6RGam6/eFVWnz3q3pu7W7uogSAGEEJAxKUmenCGUV6+rZFuu/jpynsnG7575W64t439MqmesoYAHiMEgYkOJ/PdNm8sXr+S+fprmvnq7G9S5/6j+Va8vO39M67+72OBwBJizNhQJLp6gnr4Yoa3fPSFtW3durcqQX66sXTNT/g9zoaACQcDuYD+DMd3b369Vs79ZNXtqqxvVsXzyrSVy6erunFOV5HA4CEQQkDcFytHd164PUd+uVr76qtq0dXzh+nL31wmiYUZHsdDQDiHiUMwEk1tXfpZ396Vw++uV3dvU5Lykt124VTNc4/wutoABC3KGEABqy+tUM/eXmbfvNOtSTpE2cG9TcXTFFhTobHyQAg/lDCAAxabWO77nlpqx5dWav0FJ8+fc4E3XzeZOVlpXkdDQDiBiUMwCl7t6FNd7+4Rb9bVaeczFTddO4kfXrRRI3kknAAOClKGIAhO/qS8FHZ6fobLgkHgJOihAGIGC4JB4CBo4QBiLi3tu3XXS9s0oqdjQqOytKXPjhVV5ZxSTgAHI0LvAFE3FmTR+vRW87Sf3xqgXIyU/XlR/ouCX92DZeEA8BAUMIAnDIz0wdmjNFTty7STz7Rd0n45/5npS6/93W9zCXhAHBClDAAQ+bzmS6dO1Yv3H6+7rp2vprau/Xp/kvC3+aScAA4Js6EAYi4w5eE3/vHLdrbwiXhAJIXB/MBeIJLwgEkO0oYAE+1dfbogde36xev9l0SfsX8cbqdS8IBJAFKGICY0NTepZ+/+q4efGOHunrDuvb0Un3hIi4JB5C4KGEAYsr7Lwn/+BlBff4DXBIOIPFQwgDEpF1Nh/TjF7ccuST8U+dM0M3nTZI/K93raAAQEZQwADFt+76D+tEfNuup1XUamZGqvz53kj7DJeEAEgAlDEBc2Lin75LwP6zfq5EZqboqNE7XLwhqTkme19EA4JRQwgDElVU1TfrPt3bo96t3q7MnrHmlebphYVCXzx/H6hiAuEIJAxCXmtu79dvKWi1dVqNNe1uVnZ6iK8pK9PGFQc0tZXUMQOyjhAGIa845VdY0aek71XpqdZ06usOaPS5XNywM6sqyccrJTPM6IgAcEyUMQMJo6ejWk1V1+s071dqwu0Uj0lJ0xfxxun5hQGUBv8zM64gAcAQlDEDCcc5pdW2zli6r1u9W1am9q1czinP08TOCurKsRHkjWB0D4D1KGICE1trRrd+tqtNDy2q0ZlezMtN8+si8cbphYVCnBVkdA+AdShiApLGmtllLl1frycpdOtjVq2lFI3XDwqCuDpUqL4vVMQDDixIGIOkc7OzR06vr9JtlNVpV06SMVJ8umztW1y8MasGEfFbHAAwLShiApLaurlkPLavRE5W71NrZo8mF2bphYVDXnFaq/GyuSAIQPZQwAJDU3tWjp1fv1kPLqrWyuknpKT5dMrdYNywM6oyJo1gdAxBxlDAAeJ+Ne1r00LIaPb6yVi0dPZpUkK3rFwZ0zWmlGj0yw+t4ABIEJQwAjqOju1fPrNmtpcuqtXxHo9JSTB+e3bc6dtak0fL5WB0DcOooYQAwAFv2tmrpsho9XlmrpvZujR+dpesXBPWx00tVmMPqGIDBo4QBwCB0dPfq+XV79Jt3qvXO9gNK9Zkunl2kGxYGdc7kAlbHAAwYJQwATtHW+jY9vLxaj66oVWN7twKjRuj6BUFde3qpxuRmeh0PQIyjhAHAEHX29OqFdXu1dFm13ty2Xyk+0wdnjtH1C4M6b2qhUlgdA3AMJyphqcMdBgDiUUZqii6fP06Xzx+n7fsO6qHl1Xq0olbPr9urEv8IXbcgoCXlARXnsToGYGBYCQOAU9TVE9aLG/pWx17bsk8+ky6cUaSPnxHQ+dPGsDoGgJUwAIiG9FSfLp07VpfOHaud+w/q4eU1eqSiVi9u2KuxeZlaUh7QdQsCGucf4XVUADGIlTAAiKDu3rBe2rBXS5fV6NUtDTJJF0wfo+sXBHThjDFKTfF5HRHAMGIlDACGSVqKT4vnjNXiOWNVc6Bdj1TU6OHlNbppY72KcjO0pLzv7FhgVJbXUQF4jJUwAIiynt6wXt7UoKXLqvXKpno5SYumFOjyeeP0wVlFGsUl4kDCYkQFAMSIXU2H9MjyGj22sla1jYeU4jOdMXGUFs8p1odnF6uI2WNAQvGshJnZYkn/LilF0i+dc3e873lQ0n9K8ve/5uvOuWdO9J6UMACJwDmndXUtem7tHj27dre2NRyUJJ0+Pl+LZxdr8ZxitiyBBOBJCTOzFEmbJX1IUq2k5ZJucM6tP+o190uqdM791MxmSXrGOTfhRO9LCQOQiLbWt+rZNXv03Lo9WlfXIkmaPS5Xl8wp1uI5YzVlzEiPEwI4FV4dzF8oaatz7t3+EA9JulLS+qNe4yTl9n+dJ6kuinkAIGZNGZOj2y7K0W0XTVX1/nY9v65vheyuFzbrrhc2a8qYkbqkf8ty9rhcmTGDDIh30VwJ+5ikxc65v+r/+yclneGcu/Wo14yV9IKkfEnZkj7onFtxjPe6SdJNkhQMBk/fuXNnVDIDQKzZ09yhF9bv0bNr9uid7fsVdlJwVNaRM2ShgJ8LxYEY5tV25EBK2Jf7M/zQzM6S9CtJc5xz4eO9L9uRAJLV/rZOvbhhr55du0dvbN2n7l6notwMfbj/DNnCCaOYQwbEGK+2I3dJChz199L+7x3ts5IWS5Jz7i0zy5RUIKk+irkAIC6NHpmh6xYEdd2CoFo6uvXHDfV6du1uPVJRo/96a6dGZafrQzOLtHhusc6ePFoZqSleRwZwAtEsYcslTTWzieorX9dL+vj7XlMt6SJJD5rZTEmZkhqimAkAEkJuZpquCpXoqlCJ2rt69KdNDXpu3R79fs1uPVxRo5yMVF00c4wWzynW+dPGaEQ6hQyINdEeUXGppLvVN37iAefc983su5IqnHO/6/9E5C8kjVTfIf2/dc69cKL3ZDsSAI6vs6dXb27dr2fX7tYf1u9VY3u3RqSl6ILphVo8p1gXzhijnMw0r2MCSYNhrQCQhHp6w1q2/YCeXbtHz6/bo/rWTqWn+HTOlNG6ZM5YfWhWkfKZ1g9EFSUMAJJcOOxUWdN4ZBbZ0dP6L5lTrIuZ1g9EBSUMAHDE4Wn9z67drWfX7tG7DQdlJp0WzD8yi4xp/UBkUMIAAMe1ZW9r//VJe7R+d9+0/jkluf3XJzGtHxgKShgAYEB27j/YP61/jyqrmyTpyLT+xXOKNWss0/qBwaCEAQAGbU9zx5Hrk5ZtP8C0fuAUUMIAAEOyv61Tf1i/V8+t+79p/cW5mfrw7CJ9mGn9wHFRwgAAEdN8qFt/3LhXz63doz9tblBHd1ijstN18ay+QnbO5AKlp1LIAIkSBgCIksPT+p9du0d/3Fivts4e5WSm6qIZY3ThzCItmlKgUcwiQxKjhAEAoq6ju1dvbtunZ9fs0R827FVTe7fMpNnjcnXu1EKdO7VAp4/P505LJBVKGABgWPWGndbsatZrmxv02pZ9WlndqJ6w04i0FJ0xadSRUjZ1zEg+bYmERgkDAHiqrbNHb2/br9e2NOi1rfv0bsNBSVJRbsaRQnbOlAIVjMzwOCkQWZQwAEBMqW1s1+tb9um1rfv0xtZ9amrvltS3dbloaoHOm1qo08fnKzONrUvEN0oYACBm9Yad1tU167Ut+/Tq5gatrG5Ud69TZppPCyeO1nlTC3Tu1EJNK2LrEvGHEgYAiBsHO3v0zvb9enXzPr2+dZ+21rdJksbkZGjR1AKdO7VAi6YUqjCHrUvEvhOVsNThDgMAwIlkZ6TqwhlFunBGkSSprunQka3LlzfW6/GVuyRJM8fm6tz+UrZgwii2LhF3WAkDAMSNcNhpXV2LXtvaoNc279OKnY3q6g0rI9WnhRNH9ZeyQs0ozmHrEjGB7UgAQEJq7+rRO9sP6LXN+/T61gZt3tu3dVmYk6FFUw5vXRZoTG6mx0mRrNiOBAAkpKz0VH1g+hh9YPoYSX2Xjr+2pUGvb+075P/byr6tyxnFOX2FbGqhFk4YpRHpbF3Ce6yEAQASUjjstH53i17b0rdKtnx739ZleqpPCyeMOnLIf2Zxrnw+ti4RHWxHAgCS3qGuXi3bceDIFP9Ne1slSQUj03XOlIIjQ2OL2LpEBLEdCQBIeiPSU3T+tEKdP61QkrS3paPvU5f925dPVtVJkqYVjTxSyM6YOJqtS0QNK2EAgKQXDjtt3NN6pJC9s/2AunrCSk/xqXxC/pFSNmssW5cYHLYjAQAYhI7uXi3bfuDIAf+Ne/q2Lkdlp2vRlAIt6r/rssQ/wuOkiHWUMAAAhqC+tUNvbN2n1zb3DY1taO2UJJX4R+j08flaMCFf5RNGaVpRjlJYKcNRKGEAAESIc31bl++8u1/LdzaqYscB7W3pK2U5Gak67ahSNr/Uz5myJEcJAwAgSpxzqm08pIqdB7R8R6NW7Gg88snLVJ9pTkmeFkzI1+njR6l8Qr4KRnLnZTKhhAEAMIya2ru0srpRFTv6/lTVNqmrJyxJmlSQ3b+F2VfKJhZkc8VSAqOEAQDgoc6eXq3d1ayKHY19q2U7D6ixvVuSNDo7/T2lbPa4PKWn+jxOjEihhAEAEEOcc9rWcFAVO/q2MCt2HtDO/e2SpIxUn8oC/iOl7LTx+crNTPM4MU4VJQwAgBhX39qhFUetlK2ta1Fv2MlMml6Uc6SUlU8YxWiMOEIJAwAgzrR39aiquunIStnKnY062NUrSRqXl6nyw6Vs/ChNL2Y0Rqzi2iIAAOJMVnqqzp5SoLOnFEiSenrD2rinVSt2Nmr5jgN6Z/t+/W5V31VLORmpCo3P14LxfStlZQFGY8QDVsIAAIhDh0djHC5lFTsatbm+Vc71jcaYXZJ3pJQxGsM7bEcCAJAEmtu7tbL6/0rZ0aMxJhZkq3x8/pFzZZMYjTEsKGEAACShvtEYLarYcUAV/dP9D4/GGJWd/p5SNofRGFHBmTAAAJJQRmqKTh+fr9PH5+tmvXc0xuFS9sL6vf2v9Wl+wH/kyqXTgvnKG8FojGhiJQwAgCRW39qhlTv7RmNU7DigdXUt6gn3dYNJBdmaW5qnuSV9f2aX5GlkBus3g8F2JAAAGJD2rh5V1TRpxY5GrdnVrDW7mrW7uUOSZCZNLhx5pJTNK83TrHG5ykqnmB0P25EAAGBAstJTdfbkAp09ueDI9xpaO7V2V7NW1zZrza4mvbltn35buUuS5DNpypiRmlvi19ySXM0t9WvW2FxGZAwAK2EAAGDQ9rZ0aE1t85HVstW1zdrX1ilJSvGZpo4ZeWS1bE5JnmaOzVVmWvIVM1bCAABARBXlZqpoVqY+OKtIUt+h/70tnVpd29S3ararWX/cWK//XVErqW922bSinL6tzNK+cja9OEcZqclXzA5jJQwAAESFc051zYdXzJq0urZZa3c1HxmTkZZiml6co7klfs3r/wDAtKKchBqVwcF8AAD+f3v3FiN3WYdx/Pt0l3LoCZQisEUolADlWE5BicaIJhgNqEEBhRhuvAEEYqJgPIVrI3pBFIIaFCJqhYQQIkQwJCQCRaiFclAsAltQikhLORTa/XkxU9jWYpjK7LszfD9Jk513/p15Nm86feb/vjN/TQubv+l/8zLmA+NrWTH+Aute3QjAzJEZHLLXHA7rLmUePrYrB75vNjuMDGYxs4RJkqRpq6p46vlXWLH6hS32mb24uZiNzmDxXnPf2F92xIJ5LJo/m9EBKGaWMEmSNFAmJoonnn+5e7ass5S58ul1rN/QKWY77bC5mO36xj6zA+bPZmTG9LoUkyVMkiQNvImJ4vF/vfTm2bLxtTz49Fpefm0TALvMHOHQvedusZS5/+6zmNGwmFnCJEnSUNo0UTz+3HpWjK99Y+P/yqfX8crrnWI2a+YIh47N44ju2bLDx+ax33unrpj5FRWSJGkojcwIi/aYw6I95vDZoxcAsHHTBH9b89KbS5mr1/KLu55gw8YJAObsOMqhY3P5zJIxTj/u/c2yW8IkSdJQGR2ZwUF7zuGgPedw2jFvFrO/Prv+jaXMFavX8s91G9rmbPrskiRJU2B0ZAaH7DWXQ/aay+eP26d1HACm/2c7JUmShpAlTJIkqQFLmCRJUgOWMEmSpAYsYZIkSQ1YwiRJkhqwhEmSJDVgCZMkSWrAEiZJktSAJUySJKkBS5gkSVIDfS1hSU5O8miSx5JcvI37L0uyvPvnL0le6GceSZKk6aJvF/BOMgJcDnwcGAeWJbmxqh7afExVXTTp+POBJf3KI0mSNJ3080zY8cBjVbWqql4DrgNO/R/Hnwn8so95JEmSpo1+lrAx4KlJt8e7Y/8lyb7AQuD2t7j/y0nuTXLvmjVr3vGgkiRJU226bMw/A1haVZu2dWdVXVlVx1bVsfPnz5/iaJIkSe+8fpaw1cA+k24v6I5tyxm4FClJkt5F+lnClgEHJlmYZCadonXj1gclORjYDfhjH7NIkiRNK30rYVW1ETgPuAV4GPh1Va1McmmSUyYdegZwXVVVv7JIkiRNN337igqAqroZuHmrsW9vdfu7/cwgSZI0HU2XjfmSJEnvKpYwSZKkBixhkiRJDWTQ9sMnWQM8MQVPtTvw3BQ8j/rD+Rt8zuHgcw4Hm/P3zti3qrb5JacDV8KmSpJ7q+rY1jm0fZy/weccDj7ncLA5f/3ncqQkSVIDljBJkqQGLGFv7crWAfR/cf4Gn3M4+JzDweb89Zl7wiRJkhrwTJgkSVIDmz8MZAAABCNJREFUljBJkqQGLGFbSXJykkeTPJbk4tZ51Jsk+yT5Q5KHkqxMckHrTOpdkpEk9ye5qXUW9S7JrkmWJnkkycNJPtA6k3qT5KLua+iDSX6ZZKfWmYaRJWySJCPA5cAngMXAmUkWt02lHm0EvlpVi4ETgHOdw4F0AfBw6xDabj8EfldVBwNH4lwOlCRjwFeAY6vqMGAEOKNtquFkCdvS8cBjVbWqql4DrgNObZxJPaiqZ6rqvu7PL9J58R9rm0q9SLIA+CRwVess6l2SecCHgZ8AVNVrVfVC21TaDqPAzklGgV2ApxvnGUqWsC2NAU9Nuj2O/4EPrCT7AUuAu9smUY9+AHwNmGgdRNtlIbAG+Fl3SfmqJLNah9LbV1Wrge8BTwLPAGur6ta2qYaTJUxDKcls4LfAhVW1rnUevT1JPgU8W1V/ap1F220UOBr4UVUtAV4C3F87QJLsRmcVaCGwNzAryVltUw0nS9iWVgP7TLq9oDumAZJkBzoF7Nqqur51HvXkROCUJH+nsx3go0muaRtJPRoHxqtq8xnopXRKmQbHx4DHq2pNVb0OXA98sHGmoWQJ29Iy4MAkC5PMpLMR8cbGmdSDJKGzF+Xhqvp+6zzqTVVdUlULqmo/Ov/+bq8q34EPkKr6B/BUkoO6QycBDzWMpN49CZyQZJfua+pJ+OGKvhhtHWA6qaqNSc4DbqHzaZCfVtXKxrHUmxOBs4EHkizvjn2jqm5umEl6tzkfuLb7ZnYVcE7jPOpBVd2dZClwH51PnN+PlzDqCy9bJEmS1IDLkZIkSQ1YwiRJkhqwhEmSJDVgCZMkSWrAEiZJktSAJUyS3qYkH0lyU+sckoaDJUySJKkBS5ikoZPkrCT3JFme5IokI0nWJ7ksycoktyWZ3z32qCR3JVmR5IbudfNIsijJ75P8Ocl9SQ7oPvzsJEuTPJLk2u43iktSzyxhkoZKkkOA04ETq+ooYBPwRWAWcG9VHQrcAXyn+1d+Dny9qo4AHpg0fi1weVUdSee6ec90x5cAFwKLgf3pXKVBknrmZYskDZuTgGOAZd2TVDsDzwITwK+6x1wDXJ9kHrBrVd3RHb8a+E2SOcBYVd0AUFWvAnQf756qGu/eXg7sB9zZ/19L0rCxhEkaNgGurqpLthhMvrXVcdt7zbYNk37ehK+jkraTy5GShs1twGlJ9gBI8p4k+9J5vTute8wXgDurai3w7yQf6o6fDdxRVS8C40k+3X2MHZPsMqW/haSh5zs4SUOlqh5K8k3g1iQzgNeBc4GXgOO79z1LZ98YwJeAH3dL1irgnO742cAVSS7tPsbnpvDXkPQukKrtPSMvSYMjyfqqmt06hyRt5nKkJElSA54JkyRJasAzYZIkSQ1YwiRJkhqwhEmSJDVgCZMkSWrAEiZJktTAfwDJCay7E11DLQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# dot_img_file = \"d_mod-\"+str(d_model)+\"-d_k-\"+str(d_k)+\"-d_v-\"+str(d_v)+\"-n_heads-\"+str(n_heads)+\"-enc_lays-\"+str(n_encoder_layers)+\"-dec_lays-\"+str(n_decoder_layers)+'-transformer.png'\n",
        "# tf.keras.utils.plot_model(transformer, to_file=dot_img_file, show_shapes=True)"
      ],
      "metadata": {
        "id": "RSK1eufbeQPL"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#transformer = keras.models.load_model('./Models/')"
      ],
      "metadata": {
        "id": "Oly3kYeU3FnV"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Testing"
      ],
      "metadata": {
        "id": "KJ_Gx7RVeNla"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def greedy_translate(model, source_sentence, target_sentence_start=sp_ben_bpe.bos_id()):\n",
        "  # Tokenizing and padding\n",
        "  source_seq = tokenize(source_sentence, sp_en_bpe)\n",
        "  source_seq = tf.keras.preprocessing.sequence.pad_sequences(source_seq, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "  predict_seq = [[target_sentence_start]]\n",
        "  \n",
        "  predict_sentence = [target_sentence_start] # Deep copy here to prevent updates on target_sentence_start\n",
        "  while predict_sentence[-1] != sp_ben_bpe.eos_id() and len(predict_sentence) < MAX_TOKENS:\n",
        "    predict_output = model([np.array(source_seq), np.array(predict_seq)], training=None)\n",
        "    predict_label = tf.argmax(predict_output, axis=-1) # Pick the label with highest softmax score\n",
        "    predict_seq = tf.concat([predict_seq, predict_label], axis=-1) # Updating the prediction sequence\n",
        "    predict_sentence.append(predict_label[0][0].numpy())\n",
        "  \n",
        "  if len(predict_sentence) == MAX_TOKENS:\n",
        "      return detokenize(predict_sentence, sp_ben_bpe, True, False)\n",
        "  return detokenize(predict_sentence, sp_ben_bpe)"
      ],
      "metadata": {
        "id": "zfg1AaKDnPQM"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow_probability as tfp"
      ],
      "metadata": {
        "id": "pLrtBO__iPPl"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I love you.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", greedy_translate(transformer, sentence))"
      ],
      "metadata": {
        "id": "KtWeJl1JoapJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c0d686ff-dc2a-4b46-aa62-e6cc9b91bfa8"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I love you.\n",
            "Bengali Sentence:  তোমাকে ভালবাসি।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I am tired.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", greedy_translate(transformer, sentence))"
      ],
      "metadata": {
        "id": "K2axdVLjq6z1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9b4c3e4a-6494-4635-956c-e65e655b2a85"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I am tired.\n",
            "Bengali Sentence:  ক্লান্ত হয়ে পড়েছি।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences = list(test_val_data['src'][:250])\n",
        "refs = [list(test_val_data['tgt'][:250])]\n",
        "sys = [greedy_translate(transformer, sentence) for sentence in sentences]"
      ],
      "metadata": {
        "id": "9XNckV-wBgiN"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bleu = BLEU()\n",
        "bleu.corpus_score(sys, refs)"
      ],
      "metadata": {
        "id": "otRhPJC1VKyj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e2028b6d-a39e-418a-936f-0896d3733cac"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BLEU = 3.03 21.8/6.0/1.9/0.6 (BP = 0.884 ratio = 0.890 hyp_len = 3190 ref_len = 3583)"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "chrf = CHRF()\n",
        "chrf.corpus_score(sys, refs)"
      ],
      "metadata": {
        "id": "mlhToSrvYNqd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "608cdd5f-9794-4e22-d674-48c6602c8341"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "chrF2 = 29.33"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ter = TER()\n",
        "ter.corpus_score(sys, refs)"
      ],
      "metadata": {
        "id": "DtIadLMJYe1w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6641a8ed-649a-44a3-b303-20a1b1f40311"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TER = 96.24"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def sampling_translate(model, source_sentence, temperature=0.6, target_sentence_start=sp_ben_bpe.bos_id()):\n",
        "  # Tokenizing and padding\n",
        "  source_seq = tokenize(source_sentence, sp_en_bpe)\n",
        "  source_seq = tf.keras.preprocessing.sequence.pad_sequences(source_seq, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "  predict_seq = [[target_sentence_start]]\n",
        "  logprob = 0\n",
        "  \n",
        "  predict_sentence = [target_sentence_start] # Deep copy here to prevent updates on target_sentence_start\n",
        "  while predict_sentence[-1] != sp_ben_bpe.eos_id() and len(predict_sentence) < MAX_TOKENS:\n",
        "    predict_output = model([np.array(source_seq), np.array(predict_seq)], training=None)\n",
        "    probs = predict_output[-1].numpy()[0]\n",
        "    log_probs = np.log(probs)\n",
        "    u = np.random.uniform(low=1e-6, high=1.0 - 1e-6, size=log_probs.shape)\n",
        "    g = -np.log(-np.log(u))\n",
        "    sample_idx = np.argmax(log_probs + g * temperature, axis=-1)\n",
        "    sample_prob = predict_output[-1].numpy()[0][sample_idx]\n",
        "    logprob += np.log(sample_prob)\n",
        "    predict_label = np.arange(predict_output.shape[-1])[sample_idx] # Sample the label softmax score\n",
        "    predict_seq = tf.concat([predict_seq, tf.cast([[predict_label]], dtype=tf.int32)], axis=-1) # Updating the prediction sequence\n",
        "    predict_sentence.append(predict_label)\n",
        "  \n",
        "  if len(predict_sentence) == MAX_TOKENS:\n",
        "      return detokenize(predict_sentence, sp_ben_bpe, True, False), logprob\n",
        "  return detokenize(predict_sentence, sp_ben_bpe), logprob"
      ],
      "metadata": {
        "id": "2dmVo_eQhE8v"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I love you.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", sampling_translate(transformer, sentence))"
      ],
      "metadata": {
        "id": "CB5iHHlxh1cC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "39016900-d90c-4098-85b0-e31be866a8c5"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I love you.\n",
            "Bengali Sentence:  ('আমি তোমাকে পছন্দ করি।', -4.3833165644609835)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I am hungry.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", sampling_translate(transformer, sentence))"
      ],
      "metadata": {
        "id": "IiZmzCY3q1S9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c94f4ad9-1a27-4a65-ce4f-eb39a5dedc5d"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I am hungry.\n",
            "Bengali Sentence:  ('আমি খিদে পেয়ে থাকি।', -6.214136440859875)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_samples(sentence, n_samples, model=None, temperature=0.6):\n",
        "    samples, log_probs = [], []\n",
        "    for _ in range(n_samples):\n",
        "        sample, logp = sampling_translate(model, sentence, temperature)\n",
        "        samples.append(sample)\n",
        "        log_probs.append(logp)\n",
        "    return samples, log_probs"
      ],
      "metadata": {
        "id": "xIxJ9HqTtxHY"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def rouge1_similarity(system, reference):\n",
        "    sys_counter = Counter(system)\n",
        "    ref_counter = Counter(reference)\n",
        "    overlap = 0\n",
        "    for token in sys_counter:\n",
        "        token_count_sys = sys_counter[token]\n",
        "        token_count_ref = ref_counter[token]\n",
        "        overlap += min(token_count_ref, token_count_sys)\n",
        "    precision = overlap / sum(sys_counter.values())\n",
        "    recall = overlap / sum(ref_counter.values())\n",
        "    if precision + recall != 0:\n",
        "        rouge1_score = 2 * ((precision * recall)/(precision + recall))\n",
        "    else:\n",
        "        rouge1_score = 0     \n",
        "    return rouge1_score"
      ],
      "metadata": {
        "id": "uhIlVoVwt7YC"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def weighted_avg_overlap(samples, log_probs):\n",
        "    scores = {}\n",
        "    for index_candidate, candidate in enumerate(samples):    \n",
        "        overlap, weight_sum = 0.0, 0.0\n",
        "        for index_sample, (sample, logp) in enumerate(zip(samples, log_probs)):           \n",
        "            if index_candidate == index_sample:\n",
        "                continue\n",
        "            sample_p = float(np.exp(logp))\n",
        "            weight_sum += sample_p\n",
        "            sample_overlap = rouge1_similarity(candidate, sample)\n",
        "            overlap += sample_p * sample_overlap\n",
        "        score = overlap / weight_sum\n",
        "        scores[index_candidate] = score\n",
        "    return scores"
      ],
      "metadata": {
        "id": "7KxiaRENt-C_"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def mbr_decode(sentence, n_samples, model=None, temperature=0.6):\n",
        "    samples, log_probs = generate_samples(sentence, n_samples, model, temperature)\n",
        "    scores = weighted_avg_overlap(samples, log_probs)\n",
        "    max_score_key = max(scores, key=scores.get)\n",
        "    return Counter({sample:score for sample, score in zip(samples, scores.values())})"
      ],
      "metadata": {
        "id": "ny5EYLFWt_5u"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I love you.\"\n",
        "translated_sentence = mbr_decode(sentence, 2, transformer, 1)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "id": "lzrxKRMAaOgr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "25b6aad4-ea2d-4669-b60f-8b2f579df157"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  I love you.\n",
            "Bengali:  Counter({'ভালোবাসি।': 0.368421052631579, 'আমার সঙ্গে ভালবাসার সম্পর্ক ।': 0.368421052631579})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"How are you.\"\n",
        "translated_sentence = mbr_decode(sentence, 2, transformer, 1)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "id": "3ACvLvfouZz0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1e2a1301-3c84-4ba2-9962-1b925a60fa8a"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  How are you.\n",
            "Bengali:  Counter({'তরফুকাই।': 0.4, 'তুমি কী ভাবে আছি।': 0.4})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"How are you.\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "id": "hFnTZVktqKjJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4d001235-84e8-4476-a31b-1ead013674a1"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  How are you.\n",
            "Bengali:  Counter({'কেমন আছো তুমি।': 0.9961131669375238, 'তুমি কেমন আছো।': 0.9954941899091438, 'তুমি কেমন আছো': 0.9599541081364434, 'আপনি কেমন আছেন?': 0.620897257125933})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I am hungry.\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "id": "QlRYyoz_qXcQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dd40eb41-b67e-40f0-b979-d8cf22a7f09d"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  I am hungry.\n",
            "Bengali:  Counter({'আমার ক্ষুধা লেগেছে।': 0.8768672503938452, 'আমার ক্ষুধা লাগে।': 0.8125457238152827, 'আমি ক্ষুধার্ত।': 0.642096754224992, 'আমার খিদে পেয়েছে।': 0.5856401856147169, 'আমার খাচ্ছি।': 0.5839517886939615, 'আমার খিদে পেয়ে যাব।': 0.5673338738975026, 'আমি খেয়েছি।': 0.4631965084103791})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"Hello!\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "id": "EqW3eaYJqpyB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03ef56fd-44d2-4282-f62e-f695a70313f4"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  Hello!\n",
            "Bengali:  Counter({'হ্যালো!': 0.9961633542261612, 'হ্যালো !': 0.9333333333333337})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"Hyderabad is a beautiful city.\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GGS4rCJM9yHH",
        "outputId": "0c6c4fa6-b6aa-4c11-8104-7510773371e0"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  Hyderabad is a beautiful city.\n",
            "Bengali:  Counter({'হায়দরাবাদের সুন্দর শহর।': 0.8576307628382545, 'হায়দ্রাবাদের একটি শহর।': 0.857378034121985, 'হায়দরাবাদের একটি শহর।': 0.8401757824612711, 'হায়দরাবাদ শহরের সুন্দর শহর।': 0.8240833433763276, 'হায়দ্রাবাদ শহরের একটি শহর।': 0.8166577006185474, 'হায়দরাবাদ শহর।': 0.7938977092476417})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"Africa\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r6K_H866940B",
        "outputId": "ea60df4d-1a59-4759-f62a-8475c2841e85"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  Africa\n",
            "Bengali:  Counter({'আফ্রিকা': 1.0})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"My name Rishi.\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fFPhbopy-ClI",
        "outputId": "5b6a968f-c909-461d-b62c-49d7f9d1115d"
      },
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  My name Rishi.\n",
            "Bengali:  Counter({'আমার নাম।': 0.8710596405658085, 'আমার নাম ঋষি।': 0.8368881837786692, 'আমার নাম বলে।': 0.7783439609099393, 'আমার নাম আমার নাম।': 0.6906105902278183})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "H4-KXsKr-H6G"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}